Fix Database: Ready when pods are still pending (#88)

* refactor: move storage ready status assignment

* fix: database was ready when pods were pending

* fix: dead variable

* fix: code review nested blocks

* chore: bump chart version

* fix: review suggestions

Author: Jorres

deploy/ydb-operator/Chart.yaml

@@ -15,10 +15,10 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.4.15
+version: 0.4.16
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "0.4.15"
+appVersion: "0.4.16"

internal/controllers/database/sync.go

@@ -19,6 +19,7 @@ import (
 
 	ydbv1alpha1 "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1"
 	"github.com/ydb-platform/ydb-kubernetes-operator/internal/cms"
+	"github.com/ydb-platform/ydb-kubernetes-operator/internal/labels"
 	"github.com/ydb-platform/ydb-kubernetes-operator/internal/resources"
 )
 
@@ -134,43 +135,72 @@ func (r *Reconciler) waitForClusterResources(ctx context.Context, database *reso
 	return Continue, ctrl.Result{Requeue: false}, nil
 }
 
-func (r *Reconciler) waitForStatefulSetToScale(ctx context.Context, database *resources.DatabaseBuilder) (bool, ctrl.Result, error) {
-	r.Log.Info("running step waitForStatefulSetToScale")
+func (r *Reconciler) waitForStatefulSetToScale(
+	ctx context.Context,
+	database *resources.DatabaseBuilder,
+) (bool, ctrl.Result, error) {
+	r.Log.Info("running step waitForStatefulSetToScale for Database")
 
-	if database.Spec.ServerlessResources == nil {
-		found := &appsv1.StatefulSet{}
-		err := r.Get(ctx, types.NamespacedName{
-			Name:      database.Name,
-			Namespace: database.Namespace,
-		}, found)
-		if err != nil {
-			if apierrors.IsNotFound(err) {
-				return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, nil
-			}
-			r.Recorder.Event(
-				database,
-				corev1.EventTypeNormal,
-				"Syncing",
-				fmt.Sprintf("Failed to get StatefulSets: %s", err),
-			)
-			return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err
+	if database.Spec.ServerlessResources != nil {
+		return Continue, ctrl.Result{Requeue: false}, nil
+	}
+
+	found := &appsv1.StatefulSet{}
+	err := r.Get(ctx, types.NamespacedName{
+		Name:      database.Name,
+		Namespace: database.Namespace,
+	}, found)
+	if err != nil {
+		if apierrors.IsNotFound(err) {
+			return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, nil
 		}
+		r.Recorder.Event(
+			database,
+			corev1.EventTypeNormal,
+			"Syncing",
+			fmt.Sprintf("Failed to get StatefulSets: %s", err),
+		)
+		return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err
+	}
 
-		if found.Status.Replicas != database.Spec.Nodes {
-			msg := fmt.Sprintf("Waiting for number of running pods to match expected: %d != %d",
-				found.Status.Replicas,
-				database.Spec.Nodes,
-			)
-			r.Recorder.Event(database, corev1.EventTypeNormal, "Provisioning", msg)
-			database.Status.State = string(Provisioning)
-			return r.setState(ctx, database)
+	podLabels := labels.Common(database.Name, make(map[string]string))
+	podLabels.Merge(map[string]string{
+		labels.ComponentKey: labels.DynamicComponent,
+	})
+
+	matchingLabels := client.MatchingLabels{}
+	for k, v := range podLabels {
+		matchingLabels[k] = v
+	}
+
+	podList := &corev1.PodList{}
+	opts := []client.ListOption{
+		client.InNamespace(database.Namespace),
+		matchingLabels,
+	}
+
+	err = r.List(ctx, podList, opts...)
+	if err != nil {
+		r.Recorder.Event(
+			database,
+			corev1.EventTypeNormal,
+			"Syncing",
+			fmt.Sprintf("Failed to list cluster pods: %s", err),
+		)
+		database.Status.State = string(Provisioning)
+		return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err
+	}
+
+	runningPods := 0
+	for _, e := range podList.Items {
+		if e.Status.Phase == "Running" {
+			runningPods++
 		}
 	}
 
-	if database.Status.State != string(Ready) &&
-		meta.IsStatusConditionTrue(database.Status.Conditions, TenantInitializedCondition) {
-		r.Recorder.Event(database, corev1.EventTypeNormal, "ResourcesReady", "Resource are ready and DB is initialized")
-		database.Status.State = string(Ready)
+	if runningPods != int(database.Spec.Nodes) {
+		msg := fmt.Sprintf("Waiting for number of running dynamic pods to match expected: %d != %d", runningPods, database.Spec.Nodes)
+		r.Recorder.Event(database, corev1.EventTypeNormal, string(Provisioning), msg)
 		return r.setState(ctx, database)
 	}
 
@@ -372,6 +402,15 @@ func (r *Reconciler) handleTenantCreation(
 		Reason:  TenantInitializedReasonCompleted,
 		Message: "Tenant creation is complete",
 	})
+
+	r.Recorder.Event(
+		database,
+		corev1.EventTypeNormal,
+		"DatabaseReady",
+		"Database is initialized",
+	)
+	database.Status.State = string(Ready)
+
 	return r.setState(ctx, database)
 }
 

internal/controllers/storage/init.go

@@ -47,7 +47,7 @@ func (r *Reconciler) processSkipInitPipeline(storage *resources.StorageClusterBu
 		storage,
 		corev1.EventTypeNormal,
 		"ResourcesReady",
-		"Everything should be in sync",
+		"All resources are ready",
 	)
 
 	storage.Status.State = string(Ready)
@@ -120,7 +120,7 @@ func (r *Reconciler) runInitScripts(
 			storage,
 			corev1.EventTypeNormal,
 			"ResourcesReady",
-			"Everything should be in sync",
+			"All resources are ready",
 		)
 		storage.Status.State = string(Ready)
 

internal/controllers/storage/sync.go

@@ -101,7 +101,7 @@ func (r *Reconciler) waitForStatefulSetToScale(
 	ctx context.Context,
 	storage *resources.StorageClusterBuilder,
 ) (bool, ctrl.Result, error) {
-	r.Log.Info("running step waitForStatefulSetToScale")
+	r.Log.Info("running step waitForStatefulSetToScale for Storage")
 	found := &appsv1.StatefulSet{}
 	err := r.Get(ctx, types.NamespacedName{
 		Name:      storage.Name,
@@ -155,7 +155,7 @@ func (r *Reconciler) waitForStatefulSetToScale(
 	}
 
 	if runningPods != int(storage.Spec.Nodes) {
-		msg := fmt.Sprintf("Waiting for number of running pods to match expected: %d != %d", runningPods, storage.Spec.Nodes)
+		msg := fmt.Sprintf("Waiting for number of running storage pods to match expected: %d != %d", runningPods, storage.Spec.Nodes)
 		r.Recorder.Event(storage, corev1.EventTypeNormal, string(Provisioning), msg)
 		storage.Status.State = string(Provisioning)
 		return r.setState(ctx, storage)