Skip to content

Commit d7a59fb

Browse files
artgromovthevar1able
artgromov
authored and
thevar1able
committed
progress
1 parent 53afddf commit d7a59fb

File tree

4 files changed

+110
-96
lines changed

4 files changed

+110
-96
lines changed

‎internal/controllers/database/controller.go

+6-3
Original file line numberDiff line numberDiff line change
@@ -50,12 +50,15 @@ func (r *DatabaseReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c
5050
r.Log.Info("database resources not found")
5151
return controllers.Ok()
5252
} else {
53-
r.Log.Error(err, "unexpected error")
53+
r.Log.Error(err, "unexpected Get error")
5454
}
5555
return controllers.NoRequeue(err)
5656
}
57-
58-
return r.Sync(ctx, database)
57+
result, err := r.Sync(ctx, database)
58+
if err != nil {
59+
r.Log.Error(err, "unexpected Sync error")
60+
}
61+
return result, err
5962
}
6063

6164
func ignoreDeletionPredicate() predicate.Predicate {

‎internal/controllers/storage/controller.go

+6-3
Original file line numberDiff line numberDiff line change
@@ -57,12 +57,15 @@ func (r *StorageReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
5757
r.Log.Info("storage resources not found")
5858
return controllers.Ok()
5959
} else {
60-
r.Log.Error(err, "unexpected error")
60+
r.Log.Error(err, "unexpected Get error")
6161
}
6262
return controllers.NoRequeue(err)
6363
}
64-
65-
return r.Sync(ctx, storage)
64+
result, err := r.Sync(ctx, storage)
65+
if err != nil {
66+
r.Log.Error(err, "unexpected Sync error")
67+
}
68+
return result, err
6669
}
6770

6871
func ignoreDeletionPredicate() predicate.Predicate {

‎internal/controllers/storage/sync.go

+93-77
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ import (
2424

2525
const (
2626
DefaultRequeueDelay = 10 * time.Second
27-
HealthcheckRequeueDelay = 30 * time.Second
27+
SelfCheckRequeueDelay = 30 * time.Second
2828
StorageInitializationRequeueDelay = 30 * time.Second
2929

3030
StorageInitializedCondition = "StorageInitialized"
@@ -62,22 +62,26 @@ func (r *StorageReconciler) Sync(ctx context.Context, cr *ydbv1alpha1.Storage) (
6262
return result, err
6363
}
6464

65-
result, err = r.waitForHealthCheck(ctx, &storage)
66-
if err != nil || !result.IsZero() {
67-
return result, err
68-
}
69-
7065
if !meta.IsStatusConditionTrue(storage.Status.Conditions, StorageInitializedCondition) {
7166
result, err = r.setInitialStatus(ctx, &storage)
7267
if err != nil || !result.IsZero() {
7368
return result, err
7469
}
70+
result, err = r.runSelfCheck(ctx, &storage, true)
71+
if err != nil || !result.IsZero() {
72+
return result, err
73+
}
7574
result, err = r.runInitScripts(ctx, &storage)
7675
if err != nil || !result.IsZero() {
7776
return result, err
7877
}
7978
}
8079

80+
result, err = r.runSelfCheck(ctx, &storage, false)
81+
if err != nil || !result.IsZero() {
82+
return result, err
83+
}
84+
8185
return controllers.Ok()
8286
}
8387

@@ -225,103 +229,115 @@ func (r *StorageReconciler) waitForStatefulSetToScale(ctx context.Context, stora
225229
}
226230

227231
func (r *StorageReconciler) setInitialStatus(ctx context.Context, storage *resources.StorageClusterBuilder) (ctrl.Result, error) {
228-
meta.SetStatusCondition(&storage.Status.Conditions, metav1.Condition{
229-
Type: StorageInitializedCondition,
230-
Status: "False",
231-
Reason: StorageInitializedReasonInProgress,
232-
Message: "Storage is not initialized",
233-
})
234-
if _, err := r.setState(ctx, storage); err != nil {
235-
return controllers.NoRequeue(err)
232+
if meta.FindStatusCondition(storage.Status.Conditions, StorageInitializedCondition) == nil {
233+
meta.SetStatusCondition(&storage.Status.Conditions, metav1.Condition{
234+
Type: StorageInitializedCondition,
235+
Status: "False",
236+
Reason: StorageInitializedReasonInProgress,
237+
Message: "Storage is not initialized",
238+
})
239+
if _, err := r.setState(ctx, storage); err != nil {
240+
return controllers.NoRequeue(err)
241+
}
236242
}
237243

238-
configMapName := storage.Name
239-
if storage.Spec.ClusterConfig != "" {
240-
configMapName = storage.Spec.ClusterConfig
244+
if meta.FindStatusCondition(storage.Status.Conditions, InitStorageStepCondition) == nil {
245+
meta.SetStatusCondition(&storage.Status.Conditions, metav1.Condition{
246+
Type: InitStorageStepCondition,
247+
Status: "False",
248+
Reason: InitStorageStepReasonInProgress,
249+
Message: "InitStorageStep is required",
250+
})
251+
if _, err := r.setState(ctx, storage); err != nil {
252+
return controllers.NoRequeue(err)
253+
}
241254
}
242255

243-
configMap := &corev1.ConfigMap{}
244-
err := r.Get(ctx, types.NamespacedName{
245-
Name: configMapName,
246-
Namespace: storage.Namespace,
247-
}, configMap)
256+
if meta.FindStatusCondition(storage.Status.Conditions, InitRootStorageStepCondition) == nil {
257+
configMapName := storage.Name
258+
if storage.Spec.ClusterConfig != "" {
259+
configMapName = storage.Spec.ClusterConfig
260+
}
248261

249-
if err != nil && errors.IsNotFound(err) {
250-
return controllers.Ok()
251-
} else if err != nil {
252-
r.Recorder.Event(
253-
storage,
254-
corev1.EventTypeNormal,
255-
"Syncing",
256-
fmt.Sprintf("Failed to get ConfigMap: %s", err),
257-
)
258-
return controllers.NoRequeue(err)
259-
}
262+
configMap := &corev1.ConfigMap{}
263+
err := r.Get(ctx, types.NamespacedName{
264+
Name: configMapName,
265+
Namespace: storage.Namespace,
266+
}, configMap)
260267

261-
meta.SetStatusCondition(&storage.Status.Conditions, metav1.Condition{
262-
Type: InitStorageStepCondition,
263-
Status: "False",
264-
Reason: InitStorageStepReasonInProgress,
265-
Message: "InitStorageStep is required",
266-
})
267-
if _, err := r.setState(ctx, storage); err != nil {
268-
return controllers.NoRequeue(err)
268+
if err != nil && errors.IsNotFound(err) {
269+
return controllers.Ok()
270+
} else if err != nil {
271+
r.Recorder.Event(
272+
storage,
273+
corev1.EventTypeNormal,
274+
"Syncing",
275+
fmt.Sprintf("Failed to get ConfigMap: %s", err),
276+
)
277+
return controllers.NoRequeue(err)
278+
}
279+
280+
if _, ok := configMap.Data["init_root_storage.bash"]; ok {
281+
meta.SetStatusCondition(&storage.Status.Conditions, metav1.Condition{
282+
Type: InitRootStorageStepCondition,
283+
Status: "False",
284+
Reason: InitRootStorageStepReasonInProgress,
285+
Message: fmt.Sprintf("InitRootStorageStep is required, init_root_storage.bash script is specified in ConfigMap: %s", configMapName),
286+
})
287+
if _, err := r.setState(ctx, storage); err != nil {
288+
return controllers.NoRequeue(err)
289+
}
290+
} else {
291+
meta.SetStatusCondition(&storage.Status.Conditions, metav1.Condition{
292+
Type: InitRootStorageStepCondition,
293+
Status: "True",
294+
Reason: InitRootStorageStepReasonNotRequired,
295+
Message: "InitRootStorageStep is not required",
296+
})
297+
if _, err := r.setState(ctx, storage); err != nil {
298+
return controllers.NoRequeue(err)
299+
}
300+
}
269301
}
270302

271-
if _, ok := configMap.Data["init_root_storage.bash"]; ok {
303+
if meta.FindStatusCondition(storage.Status.Conditions, InitCMSStepCondition) == nil {
272304
meta.SetStatusCondition(&storage.Status.Conditions, metav1.Condition{
273-
Type: InitRootStorageStepCondition,
305+
Type: InitCMSStepCondition,
274306
Status: "False",
275-
Reason: InitRootStorageStepReasonInProgress,
276-
Message: fmt.Sprintf("InitRootStorageStep (init_root_storage.bash script) is specified in ConfigMap: %s", configMapName),
277-
})
278-
if _, err := r.setState(ctx, storage); err != nil {
279-
return controllers.NoRequeue(err)
280-
}
281-
} else {
282-
meta.SetStatusCondition(&storage.Status.Conditions, metav1.Condition{
283-
Type: InitRootStorageStepCondition,
284-
Status: "True",
285-
Reason: InitRootStorageStepReasonNotRequired,
286-
Message: "InitRootStorageStep is not required",
307+
Reason: InitCMSStepReasonInProgress,
308+
Message: "InitCMSStep is required",
287309
})
288310
if _, err := r.setState(ctx, storage); err != nil {
289311
return controllers.NoRequeue(err)
290312
}
291313
}
292314

293-
meta.SetStatusCondition(&storage.Status.Conditions, metav1.Condition{
294-
Type: InitCMSStepCondition,
295-
Status: "False",
296-
Reason: InitCMSStepReasonInProgress,
297-
Message: "InitCMSStep is required",
298-
})
299-
if _, err := r.setState(ctx, storage); err != nil {
300-
return controllers.NoRequeue(err)
301-
}
302-
303315
return controllers.Ok()
304316
}
305317

306-
func (r *StorageReconciler) waitForHealthCheck(ctx context.Context, storage *resources.StorageClusterBuilder) (ctrl.Result, error) {
307-
err := healthcheck.CheckBootstrapHealth(ctx, storage)
318+
func (r *StorageReconciler) runSelfCheck(ctx context.Context, storage *resources.StorageClusterBuilder, waitForGoodResultWithoutIssues bool) (ctrl.Result, error) {
319+
result, err := healthcheck.GetSelfCheckResult(ctx, storage)
308320

309321
if err != nil {
310-
r.Recorder.Event(
311-
storage,
312-
corev1.EventTypeNormal,
313-
"HealthcheckInProgress",
314-
fmt.Sprintf("Waiting for healthcheck, current status: %s", err),
315-
)
316-
return controllers.RequeueAfter(HealthcheckRequeueDelay, err)
322+
r.Log.Error(err, "GetSelfCheckResult error")
323+
return controllers.RequeueAfter(SelfCheckRequeueDelay, err)
324+
}
325+
326+
eventType := corev1.EventTypeNormal
327+
if result.SelfCheckResult.String() != "GOOD" {
328+
eventType = corev1.EventTypeWarning
317329
}
318330

319331
r.Recorder.Event(
320332
storage,
321-
corev1.EventTypeNormal,
322-
"HealthcheckOK",
323-
"Bootstrap healthcheck is green",
333+
eventType,
334+
"SelfCheck",
335+
fmt.Sprintf("SelfCheck result: %s, issues found: %d", result.SelfCheckResult.String(), len(result.IssueLog)),
324336
)
337+
338+
if waitForGoodResultWithoutIssues && (result.SelfCheckResult.String() != "GOOD" || len(result.IssueLog) > 0) {
339+
return controllers.RequeueAfter(SelfCheckRequeueDelay, err)
340+
}
325341
return controllers.Ok()
326342
}
327343

‎internal/healthcheck/healthcheck.go

+5-13
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,6 @@ package healthcheck
22

33
import (
44
"context"
5-
"errors"
6-
"fmt"
75

86
"github.com/golang/protobuf/proto"
97
"github.com/ydb-platform/ydb-go-genproto/protos/Ydb_Monitoring"
@@ -15,7 +13,7 @@ const (
1513
selfCheckEndpoint = "/Ydb.Monitoring.V1.MonitoringService/SelfCheck"
1614
)
1715

18-
func CheckBootstrapHealth(ctx context.Context, cluster *resources.StorageClusterBuilder) error {
16+
func GetSelfCheckResult(ctx context.Context, cluster *resources.StorageClusterBuilder) (*Ydb_Monitoring.SelfCheckResult, error) {
1917
client := grpc.InsecureGrpcClient{
2018
Context: ctx,
2119
Target: cluster.GetEndpoint(),
@@ -28,20 +26,14 @@ func CheckBootstrapHealth(ctx context.Context, cluster *resources.StorageCluster
2826
&response,
2927
)
3028

29+
result := &Ydb_Monitoring.SelfCheckResult{}
3130
if err != nil {
32-
return err
31+
return result, err
3332
}
3433

35-
result := &Ydb_Monitoring.SelfCheckResult{}
3634
if err := proto.Unmarshal(response.Operation.Result.GetValue(), result); err != nil {
37-
return err
38-
}
39-
40-
if len(result.IssueLog) > 0 {
41-
return errors.New(
42-
fmt.Sprintf("healthcheck API status code %s", result.SelfCheckResult.String()),
43-
)
35+
return result, err
4436
}
4537

46-
return nil
38+
return result, err
4739
}

0 commit comments

Comments
 (0)