YDBOPS-9608 support dynconfig

Makefile

@@ -79,7 +79,7 @@ kind-load:
 
 .PHONY: unit-test
 unit-test: manifests generate fmt vet envtest ## Run unit tests
-	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use --arch=amd64 $(ENVTEST_K8S_VERSION) -p path)" go test -v -timeout 1800s -p 1 ./internal/controllers/... -ginkgo.v -coverprofile cover.out
+	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use --arch=amd64 $(ENVTEST_K8S_VERSION) -p path)" go test -v -timeout 1800s -p 1 ./internal/... -ginkgo.v -coverprofile cover.out
 
 .PHONY: e2e-test
 e2e-test: manifests generate fmt vet docker-build kind-init kind-load ## Run e2e tests

api/v1alpha1/configuration.go

@@ -1,6 +1,7 @@
 package v1alpha1
 
 import (
+	"bytes"
 	"crypto/sha256"
 	"fmt"
 	"path"
@@ -24,7 +25,7 @@ func hash(text string) string {
 	return fmt.Sprintf("%x", h.Sum(nil))
 }
 
-func generateSomeDefaults(cr *Storage, crDB *Database) schema.Configuration {
+func generateHosts(cr *Storage) []schema.Host {
 	var hosts []schema.Host
 
 	for i := 0; i < int(cr.Spec.Nodes); i++ {
@@ -57,6 +58,10 @@ func generateSomeDefaults(cr *Storage, crDB *Database) schema.Configuration {
 		}
 	}
 
+	return hosts
+}
+
+func generateKeyConfig(cr *Storage, crDB *Database) *schema.KeyConfig {
 	var keyConfig *schema.KeyConfig
 	if crDB != nil && crDB.Spec.Encryption != nil && crDB.Spec.Encryption.Enabled {
 		keyConfig = &schema.KeyConfig{
@@ -71,25 +76,10 @@ func generateSomeDefaults(cr *Storage, crDB *Database) schema.Configuration {
 		}
 	}
 
-	return schema.Configuration{
-		Hosts:     hosts,
-		KeyConfig: keyConfig,
-	}
+	return keyConfig
 }
 
-func tryFillMissingSections(
-	resultConfig map[string]interface{},
-	generatedConfig schema.Configuration,
-) {
-	if resultConfig["hosts"] == nil {
-		resultConfig["hosts"] = generatedConfig.Hosts
-	}
-	if generatedConfig.KeyConfig != nil {
-		resultConfig["key_config"] = generatedConfig.KeyConfig
-	}
-}
-
-func BuildConfiguration(cr *Storage, crDB *Database) (string, error) {
+func BuildConfiguration(cr *Storage, crDB *Database) ([]byte, error) {
 	config := make(map[string]interface{})
 
 	// If any kind of configuration exists on Database object, then
@@ -103,18 +93,47 @@ func BuildConfiguration(cr *Storage, crDB *Database) (string, error) {
 		rawYamlConfiguration = cr.Spec.Configuration
 	}
 
-	err := yaml.Unmarshal([]byte(rawYamlConfiguration), &config)
+	dynconfig, err := ParseDynconfig(rawYamlConfiguration)
+	if err == nil {
+		if dynconfig.Config["hosts"] == nil {
+			hosts := generateHosts(cr)
+			dynconfig.Config["hosts"] = hosts
+		}
+
+		return yaml.Marshal(dynconfig)
+	}
+
+	err = yaml.Unmarshal([]byte(rawYamlConfiguration), &config)
 	if err != nil {
-		return "", err
+		return nil, err
 	}
 
-	generatedConfig := generateSomeDefaults(cr, crDB)
-	tryFillMissingSections(config, generatedConfig)
+	if config["hosts"] == nil {
+		hosts := generateHosts(cr)
+		config["hosts"] = hosts
+	}
 
-	data, err := yaml.Marshal(config)
-	if err != nil {
-		return "", err
+	// Will be removed by YDBOPS-9692
+	keyConfig := generateKeyConfig(cr, crDB)
+	if keyConfig != nil {
+		config["key_config"] = keyConfig
 	}
 
-	return string(data), nil
+	return yaml.Marshal(config)
+}
+
+func ParseConfig(rawYamlConfiguration string) (schema.Configuration, error) {
+	config := schema.Configuration{}
+	dec := yaml.NewDecoder(bytes.NewReader([]byte(rawYamlConfiguration)))
+	dec.KnownFields(false)
+	err := dec.Decode(&config)
+	return config, err
+}
+
+func ParseDynconfig(rawYamlConfiguration string) (schema.Dynconfig, error) {
+	dynconfig := schema.Dynconfig{}
+	dec := yaml.NewDecoder(bytes.NewReader([]byte(rawYamlConfiguration)))
+	dec.KnownFields(true)
+	err := dec.Decode(&dynconfig)
+	return dynconfig, err
 }

api/v1alpha1/database_webhook.go

@@ -58,6 +58,10 @@ func (r *DatabaseDefaulter) Default(ctx context.Context, obj runtime.Object) err
 	database := obj.(*Database)
 	databaselog.Info("default", "name", database.Name)
 
+	if !database.Spec.OperatorSync {
+		return nil
+	}
+
 	if database.Spec.StorageClusterRef.Namespace == "" {
 		database.Spec.StorageClusterRef.Namespace = database.Namespace
 	}
@@ -146,7 +150,7 @@ func (r *DatabaseDefaulter) Default(ctx context.Context, obj runtime.Object) err
 		if err != nil {
 			return err
 		}
-		database.Spec.Configuration = configuration
+		database.Spec.Configuration = string(configuration)
 	}
 
 	return nil

api/v1alpha1/storage_webhook.go

@@ -61,18 +61,15 @@ func (r *Storage) GetGRPCServiceEndpoint() string {
 	return fmt.Sprintf("%s:%d", host, GRPCPort)
 }
 
-// +k8s:deepcopy-gen=false
-type PartialHostsConfig struct {
-	Hosts []schema.Host `yaml:"hosts,flow"`
-}
-
 func (r *Storage) GetHostFromConfigEndpoint() string {
+	configuration := make(map[string]interface{})
+
 	// skip handle error because we already checked in webhook
-	hostsConfig := PartialHostsConfig{}
-	_ = yaml.Unmarshal([]byte(r.Spec.Configuration), &hostsConfig)
+	_ = yaml.Unmarshal([]byte(r.Spec.Configuration), &configuration)
+	hostsConfig := configuration["hosts"].([]schema.Host)
 
 	randNum := rand.Int31n(r.Spec.Nodes) // #nosec G404
-	host := hostsConfig.Hosts[randNum].Host
+	host := hostsConfig[randNum].Host
 	return fmt.Sprintf("%s:%d", host, GRPCPort)
 }
 
@@ -97,15 +94,6 @@ func (r *Storage) IsRemoteNodeSetsOnly() bool {
 	return true
 }
 
-// +k8s:deepcopy-gen=false
-type PartialDomainsConfig struct {
-	DomainsConfig struct {
-		SecurityConfig struct {
-			EnforceUserTokenRequirement bool `yaml:"enforce_user_token_requirement"`
-		} `yaml:"security_config"`
-	} `yaml:"domains_config"`
-}
-
 // StorageDefaulter mutates Storages
 // +k8s:deepcopy-gen=false
 type StorageDefaulter struct {
@@ -119,6 +107,10 @@ func (r *StorageDefaulter) Default(ctx context.Context, obj runtime.Object) erro
 	storage := obj.(*Storage)
 	storagelog.Info("default", "name", storage.Name)
 
+	if !storage.Spec.OperatorSync {
+		return nil
+	}
+
 	if storage.Spec.Image == nil {
 		storage.Spec.Image = &PodImage{}
 	}
@@ -170,7 +162,7 @@ func (r *StorageDefaulter) Default(ctx context.Context, obj runtime.Object) erro
 	if err != nil {
 		return err
 	}
-	storage.Spec.Configuration = configuration
+	storage.Spec.Configuration = string(configuration)
 
 	return nil
 }
@@ -183,23 +175,28 @@ var _ webhook.Validator = &Storage{}
 func (r *Storage) ValidateCreate() error {
 	storagelog.Info("validate create", "name", r.Name)
 
-	configuration := make(map[string]interface{})
-	err := yaml.Unmarshal([]byte(r.Spec.Configuration), &configuration)
-	if err != nil {
-		return fmt.Errorf("failed to parse .spec.configuration, error: %w", err)
+	var configuration schema.Configuration
+
+	rawYamlConfiguration := r.Spec.Configuration
+	dynconfig, err := ParseDynconfig(r.Spec.Configuration)
+	if err == nil {
+		config, err := yaml.Marshal(dynconfig.Config)
+		if err != nil {
+			return fmt.Errorf("failed to parse .config from dynconfig, error: %w", err)
+		}
+		rawYamlConfiguration = string(config)
 	}
 
-	hostsConfig := PartialHostsConfig{}
-	err = yaml.Unmarshal([]byte(r.Spec.Configuration), &hostsConfig)
+	configuration, err = ParseConfig(rawYamlConfiguration)
 	if err != nil {
-		return fmt.Errorf("failed to parse YAML to determine `hosts`, error: %w", err)
+		return fmt.Errorf("failed to parse .spec.configuration, error: %w", err)
 	}
 
 	var nodesNumber int32
-	if len(hostsConfig.Hosts) == 0 {
+	if len(configuration.Hosts) == 0 {
 		nodesNumber = r.Spec.Nodes
 	} else {
-		nodesNumber = int32(len(hostsConfig.Hosts))
+		nodesNumber = int32(len(configuration.Hosts))
 	}
 
 	minNodesPerErasure := map[ErasureType]int32{
@@ -211,15 +208,11 @@ func (r *Storage) ValidateCreate() error {
 		return fmt.Errorf("erasure type %v requires at least %v storage nodes", r.Spec.Erasure, minNodesPerErasure[r.Spec.Erasure])
 	}
 
-	yamlConfig := PartialDomainsConfig{}
-	err = yaml.Unmarshal([]byte(r.Spec.Configuration), &yamlConfig)
-	if err != nil {
-		return fmt.Errorf("failed to parse YAML to determine `enforce_user_token_requirement`, error: %w", err)
-	}
-
 	var authEnabled bool
-	if yamlConfig.DomainsConfig.SecurityConfig.EnforceUserTokenRequirement {
-		authEnabled = true
+	if configuration.DomainsConfig.SecurityConfig != nil {
+		if configuration.DomainsConfig.SecurityConfig.EnforceUserTokenRequirement != nil {
+			authEnabled = *configuration.DomainsConfig.SecurityConfig.EnforceUserTokenRequirement
+		}
 	}
 
 	if (authEnabled && r.Spec.OperatorConnection == nil) || (!authEnabled && r.Spec.OperatorConnection != nil) {
@@ -281,23 +274,28 @@ func hasUpdatesBesidesFrozen(oldStorage, newStorage *Storage) (bool, string) {
 func (r *Storage) ValidateUpdate(old runtime.Object) error {
 	storagelog.Info("validate update", "name", r.Name)
 
-	configuration := make(map[string]interface{})
-	err := yaml.Unmarshal([]byte(r.Spec.Configuration), &configuration)
-	if err != nil {
-		return fmt.Errorf("failed to parse .spec.configuration, error: %w", err)
+	var configuration schema.Configuration
+
+	rawYamlConfiguration := r.Spec.Configuration
+	dynconfig, err := ParseDynconfig(r.Spec.Configuration)
+	if err == nil {
+		config, err := yaml.Marshal(dynconfig.Config)
+		if err != nil {
+			return fmt.Errorf("failed to parse .config from dynconfig, error: %w", err)
+		}
+		rawYamlConfiguration = string(config)
 	}
 
-	hostsConfig := PartialHostsConfig{}
-	err = yaml.Unmarshal([]byte(r.Spec.Configuration), &hostsConfig)
+	configuration, err = ParseConfig(rawYamlConfiguration)
 	if err != nil {
-		return fmt.Errorf("failed to parse YAML to determine `hosts`, error: %w", err)
+		return fmt.Errorf("failed to parse .spec.configuration, error: %w", err)
 	}
 
 	var nodesNumber int32
-	if len(hostsConfig.Hosts) == 0 {
+	if len(configuration.Hosts) == 0 {
 		nodesNumber = r.Spec.Nodes
 	} else {
-		nodesNumber = int32(len(hostsConfig.Hosts))
+		nodesNumber = int32(len(configuration.Hosts))
 	}
 
 	minNodesPerErasure := map[ErasureType]int32{
@@ -309,6 +307,17 @@ func (r *Storage) ValidateUpdate(old runtime.Object) error {
 		return fmt.Errorf("erasure type %v requires at least %v storage nodes", r.Spec.Erasure, minNodesPerErasure[r.Spec.Erasure])
 	}
 
+	var authEnabled bool
+	if configuration.DomainsConfig.SecurityConfig != nil {
+		if configuration.DomainsConfig.SecurityConfig.EnforceUserTokenRequirement != nil {
+			authEnabled = *configuration.DomainsConfig.SecurityConfig.EnforceUserTokenRequirement
+		}
+	}
+
+	if (authEnabled && r.Spec.OperatorConnection == nil) || (!authEnabled && r.Spec.OperatorConnection != nil) {
+		return fmt.Errorf("field 'spec.operatorConnection' does not align with config option `enforce_user_token_requirement: %t`", authEnabled)
+	}
+
 	if !r.Spec.OperatorSync {
 		oldStorage := old.(*Storage)
 
@@ -327,21 +336,6 @@ func (r *Storage) ValidateUpdate(old runtime.Object) error {
 		}
 	}
 
-	yamlConfig := PartialDomainsConfig{}
-	err = yaml.Unmarshal([]byte(r.Spec.Configuration), &yamlConfig)
-	if err != nil {
-		return fmt.Errorf("failed to parse YAML to determine `enforce_user_token_requirement`, error: %w", err)
-	}
-
-	var authEnabled bool
-	if yamlConfig.DomainsConfig.SecurityConfig.EnforceUserTokenRequirement {
-		authEnabled = true
-	}
-
-	if (authEnabled && r.Spec.OperatorConnection == nil) || (!authEnabled && r.Spec.OperatorConnection != nil) {
-		return fmt.Errorf("field 'spec.operatorConnection' does not align with config option `enforce_user_token_requirement: %t`", authEnabled)
-	}
-
 	if r.Spec.NodeSets != nil {
 		var nodesInSetsCount int32
 		for _, nodeSetInline := range r.Spec.NodeSets {

e2e/tests/data/storage-block-4-2-config-tls.yaml

@@ -102,7 +102,7 @@ channel_profile_config:
 grpc_config:
   start_grpc_proxy: true
   ssl_port: 2135
-  ca: /tls/grpc/ca.crt
+  ca: /etc/ssl/certs/ca-certificates.crt
   cert: /tls/grpc/tls.crt
   key: /tls/grpc/tls.key
   grpc_memory_quota_bytes: '1073741824'

internal/configuration/schema/configuration.go

@@ -1,6 +1,26 @@
 package schema
 
+type Dynconfig struct {
+	Metadata       *Metadata              `yaml:"metadata"`
+	Config         map[string]interface{} `yaml:"config"`
+	AllowedLabels  map[string]interface{} `yaml:"allowed_labels"`
+	SelectorConfig []SelectorConfig       `yaml:"selector_config"`
+}
 type Configuration struct {
-	Hosts     []Host     `yaml:"hosts"`
-	KeyConfig *KeyConfig `yaml:"key_config,omitempty"`
+	DomainsConfig *DomainsConfig `yaml:"domains_config"`
+	Hosts         []Host         `yaml:"hosts,omitempty"`
+	KeyConfig     *KeyConfig     `yaml:"key_config,omitempty"`
+}
+
+type Metadata struct {
+	Kind    string `yaml:"kind"`
+	Cluster string `yaml:"cluster"`
+	Version uint64 `yaml:"version"`
+	ID      uint64 `yaml:"id,omitempty"`
+}
+
+type SelectorConfig struct {
+	Description string                 `yaml:"description"`
+	Selector    map[string]interface{} `yaml:"selector"`
+	Config      map[string]interface{} `yaml:"config"`
 }

internal/configuration/schema/domains.go

@@ -0,0 +1,9 @@
+package schema
+
+type DomainsConfig struct {
+	SecurityConfig *SecurityConfig `yaml:"security_config,omitempty"`
+}
+
+type SecurityConfig struct {
+	EnforceUserTokenRequirement *bool `yaml:"enforce_user_token_requirement,omitempty"`
+}