Use new configuration flow

Author: Konstantin Bogdanov

api/v1alpha1/const.go

@@ -21,9 +21,12 @@ const (
 	DiskNumberMaxDigits = 2
 	DiskFilePath        = "/data"
 
-	ConfigDir = "/opt/kikimr/cfg"
+	ConfigDir      = "/opt/ydb/cfg"
+	ConfigFileName = "config.yaml"
+
+	BinariesDir      = "/opt/ydb/bin"
+	DaemonBinaryName = "ybdd"
 
-	DefaultDomain    = "root"
 	TenantNameFormat = "/%s/%s"
 )
 

api/v1alpha1/database_types.go

@@ -13,8 +13,13 @@ type DatabaseSpec struct {
 
 	// ConfigMap name with custom YDB configuration, where key is config file name and value is config file content.
 	// +optional
+	// +deprecated
 	ClusterConfig string `json:"config,omitempty"`
 
+	// YDB configuration in YAML format. Will be applied on top of generated one in internal/configuration
+	// +optional
+	Configuration string `json:"configuration"`
+
 	// (Optional) Storage services parameter overrides
 	// Default: (not specified)
 	// +optional

api/v1alpha1/storage_types.go

@@ -17,6 +17,7 @@ type StorageSpec struct {
 	ClusterConfig string `json:"config,omitempty"`
 
 	// YDB configuration in YAML format. Will be applied on top of generated one in internal/configuration
+	// +optional
 	Configuration string `json:"configuration"`
 
 	// Data storage mode.

deploy/ydb-operator/crds/database.yaml

@@ -879,6 +879,10 @@ spec:
                 description: ConfigMap name with custom YDB configuration, where key
                   is config file name and value is config file content.
                 type: string
+              configuration:
+                description: YDB configuration in YAML format. Will be applied on
+                  top of generated one in internal/configuration
+                type: string
               domain:
                 default: root
                 description: '(Optional) Name of the root storage domain Default:
@@ -3440,6 +3444,10 @@ spec:
                             type: array
                         type: object
                     type: object
+                  config:
+                    description: ConfigMap name with custom YDB configuration, where
+                      key is config file name and value is config file content.
+                    type: string
                   configuration:
                     description: YDB configuration in YAML format. Will be applied
                       on top of generated one in internal/configuration
@@ -5195,7 +5203,6 @@ spec:
                       of the YDB image Default: ""'
                     type: string
                 required:
-                - configuration
                 - dataStore
                 - erasure
                 - nodes

deploy/ydb-operator/crds/storage.yaml

@@ -875,6 +875,10 @@ spec:
                         type: array
                     type: object
                 type: object
+              config:
+                description: ConfigMap name with custom YDB configuration, where key
+                  is config file name and value is config file content.
+                type: string
               configuration:
                 description: YDB configuration in YAML format. Will be applied on
                   top of generated one in internal/configuration
@@ -2585,7 +2589,6 @@ spec:
                   YDB image Default: ""'
                 type: string
             required:
-            - configuration
             - dataStore
             - erasure
             - nodes

internal/controllers/storage/init.go

@@ -3,25 +3,15 @@ package storage
 import (
 	"context"
 	"fmt"
-	"path"
 
 	"github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1"
 	"github.com/ydb-platform/ydb-kubernetes-operator/internal/exec"
 	"github.com/ydb-platform/ydb-kubernetes-operator/internal/resources"
-	corev1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/types"
 	ctrl "sigs.k8s.io/controller-runtime"
 )
 
-const (
-	InitStorageScript     = "init_storage.bash"
-	InitRootStorageScript = "init_root_storage.bash"
-	InitCMSScript         = "init_cms.bash"
-)
-
 func (r *StorageReconciler) setInitialStatus(ctx context.Context, storage *resources.StorageClusterBuilder) (bool, ctrl.Result, error) {
 	r.Log.Info("running step setInitialStatus")
 	var changed bool = false
@@ -43,56 +33,6 @@ func (r *StorageReconciler) setInitialStatus(ctx context.Context, storage *resou
 		})
 		changed = true
 	}
-	if meta.FindStatusCondition(storage.Status.Conditions, InitCMSStepCondition) == nil {
-		meta.SetStatusCondition(&storage.Status.Conditions, metav1.Condition{
-			Type:    InitCMSStepCondition,
-			Status:  "False",
-			Reason:  InitCMSStepReasonInProgress,
-			Message: "InitCMSStep is required",
-		})
-		changed = true
-	}
-	if meta.FindStatusCondition(storage.Status.Conditions, InitRootStorageStepCondition) == nil {
-		configMapName := storage.Name
-		//if storage.Spec.ClusterConfig != "" {
-		//	configMapName = storage.Spec.ClusterConfig
-		//}
-		configMap := &corev1.ConfigMap{}
-		err := r.Get(ctx, types.NamespacedName{
-			Name:      configMapName,
-			Namespace: storage.Namespace,
-		}, configMap)
-
-		if err != nil {
-			if !errors.IsNotFound(err) {
-				r.Recorder.Event(
-					storage,
-					corev1.EventTypeNormal,
-					"Syncing",
-					fmt.Sprintf("Failed to get ConfigMap: %s", err),
-				)
-			}
-			return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err
-		}
-
-		if _, ok := configMap.Data[InitRootStorageScript]; ok {
-			meta.SetStatusCondition(&storage.Status.Conditions, metav1.Condition{
-				Type:    InitRootStorageStepCondition,
-				Status:  "False",
-				Reason:  InitRootStorageStepReasonInProgress,
-				Message: fmt.Sprintf("InitRootStorageStep is required, %s script is specified in ConfigMap: %s", InitRootStorageScript, configMapName),
-			})
-			changed = true
-		} else {
-			meta.SetStatusCondition(&storage.Status.Conditions, metav1.Condition{
-				Type:    InitRootStorageStepCondition,
-				Status:  "True",
-				Reason:  InitRootStorageStepReasonNotRequired,
-				Message: "InitRootStorageStep is not required",
-			})
-			changed = true
-		}
-	}
 	if storage.Status.State != string(Initializing) {
 		storage.Status.State = string(Initializing)
 		changed = true
@@ -108,7 +48,12 @@ func (r *StorageReconciler) runInitScripts(ctx context.Context, storage *resourc
 	podName := fmt.Sprintf("%s-0", storage.Name)
 
 	if !meta.IsStatusConditionTrue(storage.Status.Conditions, InitStorageStepCondition) {
-		cmd := []string{"/bin/bash", path.Join(v1alpha1.ConfigDir, InitStorageScript)}
+		cmd := []string{
+			fmt.Sprintf("%s/%s", v1alpha1.BinariesDir, v1alpha1.DaemonBinaryName),
+			"admin", "blobstorage", "config", "init",
+			"--yaml-file",
+			fmt.Sprintf("%s/%s", v1alpha1.ConfigDir, v1alpha1.ConfigFileName),
+		}
 		_, _, err := exec.ExecInPod(r.Scheme, r.Config, storage.Namespace, podName, "ydb-storage", cmd)
 		if err != nil {
 			return Stop, ctrl.Result{RequeueAfter: StorageInitializationRequeueDelay}, err
@@ -122,36 +67,6 @@ func (r *StorageReconciler) runInitScripts(ctx context.Context, storage *resourc
 		return r.setState(ctx, storage)
 	}
 
-	if !meta.IsStatusConditionTrue(storage.Status.Conditions, InitRootStorageStepCondition) {
-		cmd := []string{"/bin/bash", path.Join(v1alpha1.ConfigDir, InitRootStorageScript)}
-		_, _, err := exec.ExecInPod(r.Scheme, r.Config, storage.Namespace, podName, "ydb-storage", cmd)
-		if err != nil {
-			return Stop, ctrl.Result{RequeueAfter: StorageInitializationRequeueDelay}, err
-		}
-		meta.SetStatusCondition(&storage.Status.Conditions, metav1.Condition{
-			Type:    InitRootStorageStepCondition,
-			Status:  "True",
-			Reason:  InitRootStorageStepReasonCompleted,
-			Message: "InitRootStorageStep completed successfully",
-		})
-		return r.setState(ctx, storage)
-	}
-
-	if !meta.IsStatusConditionTrue(storage.Status.Conditions, InitCMSStepCondition) {
-		cmd := []string{"/bin/bash", path.Join(v1alpha1.ConfigDir, InitCMSScript)}
-		_, _, err := exec.ExecInPod(r.Scheme, r.Config, storage.Namespace, podName, "ydb-storage", cmd)
-		if err != nil {
-			return Stop, ctrl.Result{RequeueAfter: StorageInitializationRequeueDelay}, err
-		}
-		meta.SetStatusCondition(&storage.Status.Conditions, metav1.Condition{
-			Type:    InitCMSStepCondition,
-			Status:  "True",
-			Reason:  InitCMSStepReasonCompleted,
-			Message: "InitCMSStep completed successfully",
-		})
-		return r.setState(ctx, storage)
-	}
-
 	meta.SetStatusCondition(&storage.Status.Conditions, metav1.Condition{
 		Type:    StorageInitializedCondition,
 		Status:  "True",

internal/controllers/storage/sync.go

@@ -63,7 +63,7 @@ func (r *StorageReconciler) Sync(ctx context.Context, cr *ydbv1alpha1.Storage) (
 	var result ctrl.Result
 	var err error
 
-	storage := resources.NewCluster(cr, r.Log)
+	storage := resources.NewCluster(cr)
 	storage.SetStatusOnFirstReconcile()
 
 	stop, result, err = r.handleResourcesSync(ctx, &storage)
@@ -79,7 +79,7 @@ func (r *StorageReconciler) Sync(ctx context.Context, cr *ydbv1alpha1.Storage) (
 		if stop {
 			return result, err
 		}
-		stop, result, err = r.runSelfCheck(ctx, &storage, true)
+		stop, result, err = r.runSelfCheck(ctx, &storage, false)
 		if stop {
 			return result, err
 		}
@@ -166,12 +166,12 @@ func (r *StorageReconciler) handleResourcesSync(ctx context.Context, storage *re
 	r.Log.Info("running step handleResourcesSync")
 
 	for _, builder := range storage.GetResourceBuilders() {
-		new_resource := builder.Placeholder(storage)
+		newResource := builder.Placeholder(storage)
 
-		result, err := resources.CreateOrUpdateIgnoreStatus(ctx, r.Client, new_resource, func() error {
+		result, err := resources.CreateOrUpdateIgnoreStatus(ctx, r.Client, newResource, func() error {
 			var err error
 
-			err = builder.Build(new_resource)
+			err = builder.Build(newResource)
 			if err != nil {
 				r.Recorder.Event(
 					storage,
@@ -181,7 +181,7 @@ func (r *StorageReconciler) handleResourcesSync(ctx context.Context, storage *re
 				)
 				return err
 			}
-			err = ctrl.SetControllerReference(storage.Unwrap(), new_resource, r.Scheme)
+			err = ctrl.SetControllerReference(storage.Unwrap(), newResource, r.Scheme)
 			if err != nil {
 				r.Recorder.Event(
 					storage,
@@ -195,11 +195,11 @@ func (r *StorageReconciler) handleResourcesSync(ctx context.Context, storage *re
 			return nil
 		})
 
-		var eventMessage string = fmt.Sprintf(
+		var eventMessage = fmt.Sprintf(
 			"Resource: %s, Namespace: %s, Name: %s",
-			reflect.TypeOf(new_resource),
-			new_resource.GetNamespace(),
-			new_resource.GetName(),
+			reflect.TypeOf(newResource),
+			newResource.GetNamespace(),
+			newResource.GetName(),
 		)
 		if err != nil {
 			r.Recorder.Event(

internal/resources/database.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 
 	api "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1"
+	"github.com/ydb-platform/ydb-kubernetes-operator/internal/configuration"
 	"github.com/ydb-platform/ydb-kubernetes-operator/internal/labels"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -59,10 +60,24 @@ func (b *DatabaseBuilder) GetPath() string {
 func (b *DatabaseBuilder) GetResourceBuilders() []ResourceBuilder {
 	ll := labels.DatabaseLabels(b.Unwrap())
 
+	var optionalBuilders []ResourceBuilder
+
+	cfg, _ := configuration.Build(b.StorageRef)
+
+	optionalBuilders = append(
+		optionalBuilders,
+		&ConfigMapBuilder{
+			Object: b,
+			Data:   cfg,
+			Labels: ll,
+		},
+	)
+
 	if b.Spec.ServerlessResources != nil {
 		return []ResourceBuilder{}
 	}
-	return []ResourceBuilder{
+	return append(
+		optionalBuilders,
 		&ServiceBuilder{
 			Object:         b,
 			NameFormat:     grpcServiceNameFormat,
@@ -104,5 +119,5 @@ func (b *DatabaseBuilder) GetResourceBuilders() []ResourceBuilder {
 			IPFamilyPolicy: b.Spec.Service.Status.IPFamilyPolicy,
 		},
 		&DatabaseStatefulSetBuilder{Database: b.Unwrap(), Labels: ll},
-	}
+	)
 }

internal/resources/database_statefulset.go

@@ -94,17 +94,17 @@ func (b *DatabaseStatefulSetBuilder) buildPodTemplateSpec() corev1.PodTemplateSp
 }
 
 func (b *DatabaseStatefulSetBuilder) buildVolumes() []corev1.Volume {
-	var volumes []corev1.Volume
+	configMapName := b.Name
 
-	if b.Spec.ClusterConfig != "" {
-		volumes = append(volumes, corev1.Volume{
+	volumes := []corev1.Volume{
+		{
 			Name: configVolumeName,
 			VolumeSource: corev1.VolumeSource{
 				ConfigMap: &corev1.ConfigMapVolumeSource{
-					LocalObjectReference: corev1.LocalObjectReference{Name: b.Spec.ClusterConfig},
+					LocalObjectReference: corev1.LocalObjectReference{Name: configMapName},
 				},
 			},
-		})
+		},
 	}
 
 	if b.Spec.Service.GRPC.TLSConfiguration != nil && b.Spec.Service.GRPC.TLSConfiguration.Enabled {
@@ -182,13 +182,11 @@ func (b *DatabaseStatefulSetBuilder) buildContainer() corev1.Container {
 func (b *DatabaseStatefulSetBuilder) buildVolumeMounts() []corev1.VolumeMount {
 	var volumeMounts []corev1.VolumeMount
 
-	if b.Spec.ClusterConfig != "" {
-		volumeMounts = append(volumeMounts, corev1.VolumeMount{
-			Name:      configVolumeName,
-			ReadOnly:  true,
-			MountPath: "/opt/kikimr/cfg",
-		})
-	}
+	volumeMounts = append(volumeMounts, corev1.VolumeMount{
+		Name:      configVolumeName,
+		ReadOnly:  true,
+		MountPath: v1alpha1.ConfigDir,
+	})
 
 	if b.Spec.Service.GRPC.TLSConfiguration.Enabled {
 		volumeMounts = append(volumeMounts, corev1.VolumeMount{
@@ -210,7 +208,7 @@ func (b *DatabaseStatefulSetBuilder) buildVolumeMounts() []corev1.VolumeMount {
 }
 
 func (b *DatabaseStatefulSetBuilder) buildContainerArgs() ([]string, []string) {
-	command := []string{"/opt/kikimr/bin/kikimr"}
+	command := []string{fmt.Sprintf("%s/%s", v1alpha1.BinariesDir, v1alpha1.DaemonBinaryName)}
 
 	db := NewDatabase(b.DeepCopy())
 
@@ -225,6 +223,9 @@ func (b *DatabaseStatefulSetBuilder) buildContainerArgs() ([]string, []string) {
 		"--ic-port",
 		fmt.Sprintf("%d", v1alpha1.InterconnectPort),
 
+		"--yaml-config",
+		fmt.Sprintf("%s/%s", v1alpha1.ConfigDir, v1alpha1.ConfigFileName),
+
 		"--tenant",
 		tenantName,
 
@@ -247,13 +248,6 @@ func (b *DatabaseStatefulSetBuilder) buildContainerArgs() ([]string, []string) {
 		)
 	}
 
-	if b.Spec.ClusterConfig != "" {
-		command = []string{"/bin/bash"}
-		args = []string{
-			"-c",
-			fmt.Sprintf("export kikimr_tenant='%s' && source /opt/kikimr/cfg/dynamic_server.cfg && exec /opt/kikimr/bin/kikimr ${kikimr_arg}", tenantName),
-		}
-	}
 	return command, args
 }