Processing remoteNodeSet objects in storage/database controllers (#175)

Author: kobzonega

api/v1alpha1/databasenodeset_types.go

@@ -19,9 +19,8 @@ type DatabaseNodeSetSpec struct {
 
 // DatabaseNodeSetStatus defines the observed state
 type DatabaseNodeSetStatus struct {
-	State                      constants.ClusterState `json:"state"`
-	Conditions                 []metav1.Condition     `json:"conditions,omitempty"`
-	ObservedDatabaseGeneration int64                  `json:"observedDatabaseGeneration,omitempty"`
+	State      constants.ClusterState `json:"state"`
+	Conditions []metav1.Condition     `json:"conditions,omitempty"`
 }
 
 // DatabaseNodeSetSpecInline describes an group nodes object inside parent object

api/v1alpha1/storagenodeset_types.go

@@ -19,9 +19,8 @@ type StorageNodeSetSpec struct {
 
 // StorageNodeSetStatus defines the observed state
 type StorageNodeSetStatus struct {
-	State                     constants.ClusterState `json:"state"`
-	Conditions                []metav1.Condition     `json:"conditions,omitempty"`
-	ObservedStorageGeneration int64                  `json:"observedStorageGeneration,omitempty"`
+	State      constants.ClusterState `json:"state"`
+	Conditions []metav1.Condition     `json:"conditions,omitempty"`
 }
 
 // StorageNodeSetSpecInline describes an group nodes object inside parent object

deploy/ydb-operator/crds/databasenodeset.yaml

@@ -4635,9 +4635,6 @@ spec:
                   - type
                   type: object
                 type: array
-              observedDatabaseGeneration:
-                format: int64
-                type: integer
               state:
                 type: string
             required:

deploy/ydb-operator/crds/remotedatabasenodeset.yaml

@@ -2792,6 +2792,10 @@ spec:
               storageEndpoint:
                 description: YDB Storage Node broker address
                 type: string
+              terminationGracePeriodSeconds:
+                description: (Optional) If specified, the pod's terminationGracePeriodSeconds.
+                format: int64
+                type: integer
               tolerations:
                 description: (Optional) If specified, the pod's tolerations.
                 items:
@@ -4632,9 +4636,6 @@ spec:
                   - type
                   type: object
                 type: array
-              observedDatabaseGeneration:
-                format: int64
-                type: integer
               state:
                 type: string
             required:

deploy/ydb-operator/crds/remotestoragenodeset.yaml

@@ -2785,6 +2785,10 @@ spec:
                 required:
                 - name
                 type: object
+              terminationGracePeriodSeconds:
+                description: (Optional) If specified, the pod's terminationGracePeriodSeconds.
+                format: int64
+                type: integer
               tolerations:
                 description: (Optional) If specified, the pod's tolerations.
                 items:
@@ -4625,9 +4629,6 @@ spec:
                   - type
                   type: object
                 type: array
-              observedStorageGeneration:
-                format: int64
-                type: integer
               state:
                 type: string
             required:

deploy/ydb-operator/crds/storagenodeset.yaml

@@ -4628,9 +4628,6 @@ spec:
                   - type
                   type: object
                 type: array
-              observedStorageGeneration:
-                format: int64
-                type: integer
               state:
                 type: string
             required:

deploy/ydb-operator/templates/rbac-operator.yaml

@@ -179,7 +179,9 @@ rules:
   - ydb.tech
   resources:
   - databasenodesets
+  - remotedatabasenodesets
   - storagenodesets
+  - remotestoragenodesets
   verbs:
   - create
   - delete
@@ -192,14 +194,18 @@ rules:
   - ydb.tech
   resources:
   - databasenodesets/finalizers
+  - remotedatabasenodesets/finalizers
   - storagenodesets/finalizers
+  - remotestoragenodesets/finalizers
   verbs:
   - update
 - apiGroups:
   - ydb.tech
   resources:
   - databasenodesets/status
+  - remotedatabasenodesets/status
   - storagenodesets/status
+  - remotestoragenodesets/status
   verbs:
   - get
   - patch

e2e/tests/smoke_test.go

@@ -456,7 +456,6 @@ var _ = Describe("Operator smoke test", func() {
 		checkPodsRunningAndReady(ctx, "ydb-cluster", "kind-database", databaseSample.Spec.Nodes)
 
 		database := v1alpha1.Database{}
-		databaseNodeSetList := v1alpha1.DatabaseNodeSetList{}
 		databasePods := corev1.PodList{}
 		By("delete nodeSetSpec inline to check inheritance...")
 		Eventually(func(g Gomega) error {
@@ -476,23 +475,6 @@ var _ = Describe("Operator smoke test", func() {
 			return k8sClient.Update(ctx, &database)
 		}, Timeout, Interval).Should(BeNil())
 
-		By("check that ObservedDatabaseGeneration changed...")
-		Eventually(func(g Gomega) bool {
-			g.Expect(k8sClient.Get(ctx, types.NamespacedName{
-				Name:      databaseSample.Name,
-				Namespace: testobjects.YdbNamespace,
-			}, &database)).Should(Succeed())
-			g.Expect(k8sClient.List(ctx, &databaseNodeSetList,
-				client.InNamespace(testobjects.YdbNamespace),
-			)).Should(Succeed())
-			for _, databaseNodeSet := range databaseNodeSetList.Items {
-				if database.GetGeneration() != databaseNodeSet.Status.ObservedDatabaseGeneration {
-					return false
-				}
-			}
-			return true
-		}, Timeout, Interval).Should(BeTrue())
-
 		By("expecting databaseNodeSet pods deletion...")
 		Eventually(func(g Gomega) bool {
 			g.Expect(k8sClient.Get(ctx, types.NamespacedName{

internal/controllers/database/controller.go

@@ -13,6 +13,7 @@ import (
 	"k8s.io/client-go/tools/record"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
 	"sigs.k8s.io/controller-runtime/pkg/event"
 	"sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
@@ -53,20 +54,21 @@ type Reconciler struct {
 func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
 	r.Log = log.FromContext(ctx)
 
-	database := &ydbv1alpha1.Database{}
-	err := r.Get(ctx, req.NamespacedName, database)
+	resource := &ydbv1alpha1.Database{}
+	err := r.Get(ctx, req.NamespacedName, resource)
 	if err != nil {
 		if errors.IsNotFound(err) {
-			r.Log.Info("database resources not found")
+			r.Log.Info("Database resource not found")
 			return ctrl.Result{Requeue: false}, nil
 		}
 		r.Log.Error(err, "unexpected Get error")
 		return ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err
 	}
-	result, err := r.Sync(ctx, database)
+	result, err := r.Sync(ctx, resource)
 	if err != nil {
 		r.Log.Error(err, "unexpected Sync error")
 	}
+
 	return result, err
 }
 
@@ -88,10 +90,36 @@ func ignoreDeletionPredicate() predicate.Predicate {
 
 // SetupWithManager sets up the controller with the Manager.
 func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error {
-	controller := ctrl.NewControllerManagedBy(mgr).For(&ydbv1alpha1.Database{})
+	resource := &ydbv1alpha1.Database{}
+	resourceGVK, err := apiutil.GVKForObject(resource, r.Scheme)
+	if err != nil {
+		r.Log.Error(err, "does not recognize GVK for resource")
+		return err
+	}
 
-	r.Recorder = mgr.GetEventRecorderFor("Database")
+	r.Recorder = mgr.GetEventRecorderFor(resourceGVK.Kind)
+	controller := ctrl.NewControllerManagedBy(mgr).For(resource)
+	if err := mgr.GetFieldIndexer().IndexField(
+		context.Background(),
+		&ydbv1alpha1.RemoteDatabaseNodeSet{},
+		OwnerControllerKey,
+		func(obj client.Object) []string {
+			// grab the RemoteDatabaseNodeSet object, extract the owner...
+			remoteDatabaseNodeSet := obj.(*ydbv1alpha1.RemoteDatabaseNodeSet)
+			owner := metav1.GetControllerOf(remoteDatabaseNodeSet)
+			if owner == nil {
+				return nil
+			}
+			// ...make sure it's a Database...
+			if owner.APIVersion != ydbv1alpha1.GroupVersion.String() || owner.Kind != resourceGVK.Kind {
+				return nil
+			}
 
+			// ...and if so, return it
+			return []string{owner.Name}
+		}); err != nil {
+		return err
+	}
 	if err := mgr.GetFieldIndexer().IndexField(
 		context.Background(),
 		&ydbv1alpha1.DatabaseNodeSet{},
@@ -104,7 +132,7 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error {
 				return nil
 			}
 			// ...make sure it's a Database...
-			if owner.APIVersion != ydbv1alpha1.GroupVersion.String() || owner.Kind != "Database" {
+			if owner.APIVersion != ydbv1alpha1.GroupVersion.String() || owner.Kind != resourceGVK.Kind {
 				return nil
 			}
 
@@ -115,6 +143,7 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error {
 	}
 
 	return controller.
+		Owns(&ydbv1alpha1.RemoteDatabaseNodeSet{}).
 		Owns(&ydbv1alpha1.DatabaseNodeSet{}).
 		Owns(&corev1.Service{}).
 		Owns(&appsv1.StatefulSet{}).

internal/controllers/database/sync.go

@@ -461,11 +461,11 @@ func (r *Reconciler) syncNodeSetSpecInline(
 	database *resources.DatabaseBuilder,
 ) (bool, ctrl.Result, error) {
 	r.Log.Info("running step syncNodeSetSpecInline")
-
-	databaseNodeSets := &v1alpha1.DatabaseNodeSetList{}
 	matchingFields := client.MatchingFields{
 		OwnerControllerKey: database.Name,
 	}
+
+	databaseNodeSets := &v1alpha1.DatabaseNodeSetList{}
 	if err := r.List(ctx, databaseNodeSets,
 		client.InNamespace(database.Namespace),
 		matchingFields,
@@ -483,10 +483,12 @@ func (r *Reconciler) syncNodeSetSpecInline(
 		databaseNodeSet := databaseNodeSet.DeepCopy()
 		isFoundDatabaseNodeSetSpecInline := false
 		for _, nodeSetSpecInline := range database.Spec.NodeSets {
-			databaseNodeSetName := database.Name + "-" + nodeSetSpecInline.Name
-			if databaseNodeSet.Name == databaseNodeSetName {
-				isFoundDatabaseNodeSetSpecInline = true
-				break
+			if nodeSetSpecInline.Remote == nil {
+				nodeSetName := database.Name + "-" + nodeSetSpecInline.Name
+				if databaseNodeSet.Name == nodeSetName {
+					isFoundDatabaseNodeSetSpecInline = true
+					break
+				}
 			}
 		}
 		if !isFoundDatabaseNodeSetSpecInline {
@@ -509,24 +511,53 @@ func (r *Reconciler) syncNodeSetSpecInline(
 					databaseNodeSet.Name),
 			)
 		}
+	}
 
-		oldGeneration := databaseNodeSet.Status.ObservedDatabaseGeneration
-		if oldGeneration != database.Generation {
-			databaseNodeSet.Status.ObservedDatabaseGeneration = database.Generation
-			if err := r.Status().Update(ctx, databaseNodeSet); err != nil {
+	remoteDatabaseNodeSets := &v1alpha1.RemoteDatabaseNodeSetList{}
+	if err := r.List(ctx, remoteDatabaseNodeSets,
+		client.InNamespace(database.Namespace),
+		matchingFields,
+	); err != nil {
+		r.Recorder.Event(
+			database,
+			corev1.EventTypeWarning,
+			"ProvisioningFailed",
+			fmt.Sprintf("Failed to list RemoteDatabaseNodeSets: %s", err),
+		)
+		return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err
+	}
+
+	for _, remoteDatabaseNodeSet := range remoteDatabaseNodeSets.Items {
+		remoteDatabaseNodeSet := remoteDatabaseNodeSet.DeepCopy()
+		isFoundRemoteDatabaseNodeSetSpecInline := false
+		for _, nodeSetSpecInline := range database.Spec.NodeSets {
+			if nodeSetSpecInline.Remote != nil {
+				nodeSetName := database.Name + "-" + nodeSetSpecInline.Name
+				if remoteDatabaseNodeSet.Name == nodeSetName {
+					isFoundRemoteDatabaseNodeSetSpecInline = true
+					break
+				}
+			}
+		}
+
+		if !isFoundRemoteDatabaseNodeSetSpecInline {
+			if err := r.Delete(ctx, remoteDatabaseNodeSet); err != nil {
 				r.Recorder.Event(
-					databaseNodeSet,
+					database,
 					corev1.EventTypeWarning,
-					"ControllerError",
-					fmt.Sprintf("Failed setting status: %s", err),
+					"ProvisioningFailed",
+					fmt.Sprintf("Failed to delete RemoteDatabaseNodeSet: %s", err),
 				)
 				return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err
 			}
 			r.Recorder.Event(
-				databaseNodeSet,
+				database,
 				corev1.EventTypeNormal,
-				"StatusChanged",
-				fmt.Sprintf("DatabaseNodeSet updated observedStorageGeneration from %d to %d", oldGeneration, database.Generation),
+				"Syncing",
+				fmt.Sprintf("Resource: %s, Namespace: %s, Name: %s, deleted",
+					reflect.TypeOf(remoteDatabaseNodeSet),
+					remoteDatabaseNodeSet.Namespace,
+					remoteDatabaseNodeSet.Name),
 			)
 		}
 	}