From 6f07d13fdb15d823b1975d129acd07c0cbaa4631 Mon Sep 17 00:00:00 2001 From: Aleksei Kobzev Date: Tue, 28 Jan 2025 01:09:57 +0800 Subject: [PATCH 01/21] fix bug: missing error handler for arg --auth-token-file (#283) * fix bug: missing error handler for arg --auth-token-file * fix golangci-lint --- .../unreleased/Added-20250124-141810.yaml | 3 + .../unreleased/Fixed-20250124-141631.yaml | 3 + api/v1alpha1/const.go | 14 +++- .../controllers/storage/controller_test.go | 78 +++++++++++++++++++ internal/resources/database_statefulset.go | 33 +++----- internal/resources/secret.go | 22 ------ internal/resources/storage_statefulset.go | 33 +++----- 7 files changed, 119 insertions(+), 67 deletions(-) create mode 100644 .changes/unreleased/Added-20250124-141810.yaml create mode 100644 .changes/unreleased/Fixed-20250124-141631.yaml diff --git a/.changes/unreleased/Added-20250124-141810.yaml b/.changes/unreleased/Added-20250124-141810.yaml new file mode 100644 index 00000000..58e21c15 --- /dev/null +++ b/.changes/unreleased/Added-20250124-141810.yaml @@ -0,0 +1,3 @@ +kind: Added +body: 'annotations overrides default secret name and key for arg --auth-token-file' +time: 2025-01-24T14:18:10.344319+08:00 diff --git a/.changes/unreleased/Fixed-20250124-141631.yaml b/.changes/unreleased/Fixed-20250124-141631.yaml new file mode 100644 index 00000000..31c864f9 --- /dev/null +++ b/.changes/unreleased/Fixed-20250124-141631.yaml @@ -0,0 +1,3 @@ +kind: Fixed +body: 'bug: missing error handler for arg --auth-token-file' +time: 2025-01-24T14:16:31.463111+08:00 diff --git a/api/v1alpha1/const.go b/api/v1alpha1/const.go index 6f6e9f6a..a9fe280d 100644 --- a/api/v1alpha1/const.go +++ b/api/v1alpha1/const.go @@ -28,18 +28,24 @@ const ( DiskPathPrefix = "/dev/kikimr_ssd" DiskNumberMaxDigits = 2 DiskFilePath = "/data" - YdbAuthToken = "ydb-auth-token-file" - ConfigDir = "/opt/ydb/cfg" - ConfigFileName = "config.yaml" + AuthTokenSecretName = "ydb-auth-token-file" + AuthTokenSecretKey = "ydb-auth-token-file" + AuthTokenFileArg = "--auth-token-file" DatabaseEncryptionKeySecretDir = "database_encryption" DatabaseEncryptionKeySecretFile = "key" DatabaseEncryptionKeyConfigFile = "key.txt" + ConfigDir = "/opt/ydb/cfg" + ConfigFileName = "config.yaml" + BinariesDir = "/opt/ydb/bin" DaemonBinaryName = "ydbd" + AdditionalSecretsDir = "/opt/ydb/secrets" + AdditionalVolumesDir = "/opt/ydb/volumes" + DefaultRootUsername = "root" DefaultRootPassword = "" DefaultDatabaseDomain = "Root" @@ -60,6 +66,8 @@ const ( AnnotationGRPCPublicHost = "ydb.tech/grpc-public-host" AnnotationNodeHost = "ydb.tech/node-host" AnnotationNodeDomain = "ydb.tech/node-domain" + AnnotationAuthTokenSecretName = "ydb.tech/auth-token-secret-name" + AnnotationAuthTokenSecretKey = "ydb.tech/auth-token-secret-key" AnnotationValueTrue = "true" diff --git a/internal/controllers/storage/controller_test.go b/internal/controllers/storage/controller_test.go index e332cc27..b23e2498 100644 --- a/internal/controllers/storage/controller_test.go +++ b/internal/controllers/storage/controller_test.go @@ -62,6 +62,23 @@ var _ = Describe("Storage controller medium tests", func() { }) It("Checking field propagation to objects", func() { + getStatefulSet := func(objName string) (appsv1.StatefulSet, error) { + foundStatefulSets := appsv1.StatefulSetList{} + err := k8sClient.List(ctx, &foundStatefulSets, client.InNamespace( + testobjects.YdbNamespace, + )) + if err != nil { + return appsv1.StatefulSet{}, err + } + for _, statefulSet := range foundStatefulSets.Items { + if statefulSet.Name == objName { + return statefulSet, nil + } + } + + return appsv1.StatefulSet{}, fmt.Errorf("Statefulset with name %s was not found", objName) + } + storageSample := testobjects.DefaultStorage(filepath.Join("..", "..", "..", "tests", "data", "storage-mirror-3-dc-config.yaml")) tmpFilesDir := "/tmp/mounted_volume" @@ -227,5 +244,66 @@ var _ = Describe("Storage controller medium tests", func() { return len(foundStatefulSets.Items) }, test.Timeout, test.Interval).Should(Equal(1)) }) + + By("check --auth-token-file arg in StatefulSet...", func() { + By("create auth-token Secret with default name...") + defaultAuthTokenSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: v1alpha1.AuthTokenSecretName, + Namespace: testobjects.YdbNamespace, + }, + StringData: map[string]string{ + v1alpha1.AuthTokenSecretKey: "StaffApiUserToken: 'default-token'", + }, + } + Expect(k8sClient.Create(ctx, defaultAuthTokenSecret)) + + By("append auth-token Secret inside Storage manifest...") + Eventually(func() error { + foundStorage := v1alpha1.Storage{} + Expect(k8sClient.Get(ctx, types.NamespacedName{ + Name: testobjects.StorageName, + Namespace: testobjects.YdbNamespace, + }, &foundStorage)) + foundStorage.Spec.Secrets = []*corev1.LocalObjectReference{ + { + Name: v1alpha1.AuthTokenSecretName, + }, + } + return k8sClient.Update(ctx, &foundStorage) + }, test.Timeout, test.Interval).ShouldNot(HaveOccurred()) + + checkAuthTokenArgs := func() error { + statefulSet, err := getStatefulSet(testobjects.StorageName) + if err != nil { + return err + } + podContainerArgs := statefulSet.Spec.Template.Spec.Containers[0].Args + var argExist bool + var currentArgValue string + authTokenFileArgValue := fmt.Sprintf("%s/%s/%s", + v1alpha1.AdditionalSecretsDir, + v1alpha1.AuthTokenSecretName, + v1alpha1.AuthTokenSecretKey, + ) + for idx, arg := range podContainerArgs { + if arg == v1alpha1.AuthTokenFileArg { + argExist = true + currentArgValue = podContainerArgs[idx+1] + break + } + } + if !argExist { + return fmt.Errorf("arg `%s` did not found in StatefulSet podTemplate args: %v", v1alpha1.AuthTokenFileArg, podContainerArgs) + } + if authTokenFileArgValue != currentArgValue { + return fmt.Errorf("current arg `%s` value `%s` did not match with expected: %s", v1alpha1.AuthTokenFileArg, currentArgValue, authTokenFileArgValue) + } + return nil + } + + By("check that --auth-token-file arg was added to Statefulset template...") + Eventually(checkAuthTokenArgs, test.Timeout, test.Interval).ShouldNot(HaveOccurred()) + }) }) }) diff --git a/internal/resources/database_statefulset.go b/internal/resources/database_statefulset.go index 8293a864..06b5cbc5 100644 --- a/internal/resources/database_statefulset.go +++ b/internal/resources/database_statefulset.go @@ -1,10 +1,8 @@ package resources import ( - "context" "errors" "fmt" - "log" "regexp" "strconv" @@ -619,30 +617,23 @@ func (b *DatabaseStatefulSetBuilder) buildContainerArgs() ([]string, []string) { ) } + authTokenSecretName := api.AuthTokenSecretName + authTokenSecretKey := api.AuthTokenSecretKey + if value, ok := b.ObjectMeta.Annotations[api.AnnotationAuthTokenSecretName]; ok { + authTokenSecretName = value + } + if value, ok := b.ObjectMeta.Annotations[api.AnnotationAuthTokenSecretKey]; ok { + authTokenSecretKey = value + } for _, secret := range b.Spec.Secrets { - exist, err := CheckSecretKey( - context.Background(), - b.GetNamespace(), - b.RestConfig, - &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: secret.Name, - }, - Key: api.YdbAuthToken, - }, - ) - if err != nil { - log.Default().Printf("Failed to inspect a secret %s: %s\n", secret.Name, err.Error()) - continue - } - if exist { + if secret.Name == authTokenSecretName { args = append(args, - "--auth-token-file", + api.AuthTokenFileArg, fmt.Sprintf( "%s/%s/%s", wellKnownDirForAdditionalSecrets, - secret.Name, - api.YdbAuthToken, + authTokenSecretName, + authTokenSecretKey, ), ) } diff --git a/internal/resources/secret.go b/internal/resources/secret.go index 675f74b8..d9c81a4a 100644 --- a/internal/resources/secret.go +++ b/internal/resources/secret.go @@ -13,28 +13,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -func CheckSecretKey( - ctx context.Context, - namespace string, - config *rest.Config, - secretKeyRef *corev1.SecretKeySelector, -) (bool, error) { - clientset, err := kubernetes.NewForConfig(config) - if err != nil { - return false, fmt.Errorf("failed to create kubernetes clientset, error: %w", err) - } - - getCtx, cancel := context.WithTimeout(ctx, time.Second) - defer cancel() - secret, err := clientset.CoreV1().Secrets(namespace).Get(getCtx, secretKeyRef.Name, metav1.GetOptions{}) - if err != nil { - return false, fmt.Errorf("failed to get secret %s, error: %w", secretKeyRef.Name, err) - } - - _, exist := secret.Data[secretKeyRef.Key] - return exist, nil -} - func GetSecretKey( ctx context.Context, namespace string, diff --git a/internal/resources/storage_statefulset.go b/internal/resources/storage_statefulset.go index 1201d8f7..9c74dcb5 100644 --- a/internal/resources/storage_statefulset.go +++ b/internal/resources/storage_statefulset.go @@ -1,10 +1,8 @@ package resources import ( - "context" "errors" "fmt" - "log" "strconv" appsv1 "k8s.io/api/apps/v1" @@ -521,30 +519,23 @@ func (b *StorageStatefulSetBuilder) buildContainerArgs() ([]string, []string) { ) } + authTokenSecretName := api.AuthTokenSecretName + authTokenSecretKey := api.AuthTokenSecretKey + if value, ok := b.ObjectMeta.Annotations[api.AnnotationAuthTokenSecretName]; ok { + authTokenSecretName = value + } + if value, ok := b.ObjectMeta.Annotations[api.AnnotationAuthTokenSecretKey]; ok { + authTokenSecretKey = value + } for _, secret := range b.Spec.Secrets { - exist, err := CheckSecretKey( - context.Background(), - b.GetNamespace(), - b.RestConfig, - &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: secret.Name, - }, - Key: api.YdbAuthToken, - }, - ) - if err != nil { - log.Default().Printf("Failed to inspect a secret %s: %s\n", secret.Name, err.Error()) - continue - } - if exist { + if secret.Name == authTokenSecretName { args = append(args, - "--auth-token-file", + api.AuthTokenFileArg, fmt.Sprintf( "%s/%s/%s", wellKnownDirForAdditionalSecrets, - secret.Name, - api.YdbAuthToken, + authTokenSecretName, + authTokenSecretKey, ), ) } From 1ed91e3c40b86d310237b562f5c76f4939d1d732 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 28 Jan 2025 01:10:14 +0800 Subject: [PATCH 02/21] Bump golang.org/x/net from 0.23.0 to 0.33.0 (#281) * Bump golang.org/x/net from 0.23.0 to 0.33.0 Bumps [golang.org/x/net](https://github.com/golang/net) from 0.23.0 to 0.33.0. - [Commits](https://github.com/golang/net/compare/v0.23.0...v0.33.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-type: indirect ... Signed-off-by: dependabot[bot] * changie info --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Aleksei Kobzev --- .../unreleased/Security-20250127-170538.yaml | 3 ++ go.mod | 14 ++++----- go.sum | 29 +++++++++---------- 3 files changed, 24 insertions(+), 22 deletions(-) create mode 100644 .changes/unreleased/Security-20250127-170538.yaml diff --git a/.changes/unreleased/Security-20250127-170538.yaml b/.changes/unreleased/Security-20250127-170538.yaml new file mode 100644 index 00000000..219ea97a --- /dev/null +++ b/.changes/unreleased/Security-20250127-170538.yaml @@ -0,0 +1,3 @@ +kind: Security +body: bump golang.org/x/net from 0.23.0 to 0.33.0 (by dependabot) +time: 2025-01-27T17:05:38.777767+08:00 diff --git a/go.mod b/go.mod index d862ad64..f0297899 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/banzaicloud/k8s-objectmatcher v1.7.0 github.com/go-logr/logr v1.2.4 github.com/golang-jwt/jwt/v4 v4.5.1 - github.com/google/go-cmp v0.5.9 + github.com/google/go-cmp v0.6.0 github.com/onsi/ginkgo/v2 v2.9.4 github.com/onsi/gomega v1.27.6 github.com/pkg/errors v0.9.1 @@ -63,14 +63,14 @@ require ( go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.24.0 // indirect - golang.org/x/net v0.23.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/oauth2 v0.7.0 // indirect - golang.org/x/sync v0.3.0 // indirect - golang.org/x/sys v0.18.0 // indirect - golang.org/x/term v0.18.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/sync v0.10.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/term v0.27.0 // indirect + golang.org/x/text v0.21.0 // indirect golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.12.0 // indirect + golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect diff --git a/go.sum b/go.sum index fb940a91..a863f035 100644 --- a/go.sum +++ b/go.sum @@ -211,8 +211,8 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= @@ -438,7 +438,6 @@ golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -464,8 +463,8 @@ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -479,8 +478,8 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -506,11 +505,11 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= -golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -518,8 +517,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -542,8 +541,8 @@ golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss= -golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From 06b6f6385c6c6453b5fa87a8265a0634596c2a85 Mon Sep 17 00:00:00 2001 From: Aleksei Kobzev Date: Tue, 28 Jan 2025 01:10:30 +0800 Subject: [PATCH 03/21] fix remoteObj revision field (#232) * fix remoteObj revision is status * changie info --- .changes/unreleased/Fixed-20250127-125607.yaml | 3 +++ .../controllers/remotedatabasenodeset/remote_objects.go | 7 +++---- .../controllers/remotestoragenodeset/remote_objects.go | 7 +++---- 3 files changed, 9 insertions(+), 8 deletions(-) create mode 100644 .changes/unreleased/Fixed-20250127-125607.yaml diff --git a/.changes/unreleased/Fixed-20250127-125607.yaml b/.changes/unreleased/Fixed-20250127-125607.yaml new file mode 100644 index 00000000..59654796 --- /dev/null +++ b/.changes/unreleased/Fixed-20250127-125607.yaml @@ -0,0 +1,3 @@ +kind: Fixed +body: fix field resourceVersion inside .status.remoteResources.conditions +time: 2025-01-27T12:56:07.577721+08:00 diff --git a/internal/controllers/remotedatabasenodeset/remote_objects.go b/internal/controllers/remotedatabasenodeset/remote_objects.go index d600e93d..8b1305d5 100644 --- a/internal/controllers/remotedatabasenodeset/remote_objects.go +++ b/internal/controllers/remotedatabasenodeset/remote_objects.go @@ -60,7 +60,6 @@ func (r *Reconciler) syncRemoteObjects( for _, remoteObj := range remoteObjects { remoteObjName := remoteObj.GetName() remoteObjKind := remoteObj.GetObjectKind().GroupVersionKind().Kind - remoteObjRV := remoteObj.GetResourceVersion() var remoteResource *v1alpha1.RemoteResource for idx := range remoteDatabaseNodeSet.Status.RemoteResources { if resources.EqualRemoteResourceWithObject(&remoteDatabaseNodeSet.Status.RemoteResources[idx], remoteObj) { @@ -103,11 +102,11 @@ func (r *Reconciler) syncRemoteObjects( fmt.Sprintf("Failed to get resource %s with name %s: %s", remoteObjKind, remoteObjName, remoteGetErr), ) } - remoteDatabaseNodeSet.UpdateRemoteResourceStatus(remoteResource, metav1.ConditionFalse, remoteObjRV) - return r.updateStatusRemoteObjects(ctx, remoteDatabaseNodeSet, DefaultRequeueDelay) + return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, remoteGetErr } // Check object existence in local cluster + remoteObjRV := remoteObj.GetResourceVersion() localObj := resources.CreateResource(remoteObj) getErr := r.Client.Get(ctx, types.NamespacedName{ Name: localObj.GetName(), @@ -145,7 +144,7 @@ func (r *Reconciler) syncRemoteObjects( "Provisioning", fmt.Sprintf("RemoteSync CREATE resource %s with name %s", remoteObjKind, remoteObjName), ) - remoteDatabaseNodeSet.UpdateRemoteResourceStatus(remoteResource, metav1.ConditionFalse, remoteObjRV) + remoteDatabaseNodeSet.UpdateRemoteResourceStatus(remoteResource, metav1.ConditionTrue, remoteObjRV) return r.updateStatusRemoteObjects(ctx, remoteDatabaseNodeSet, StatusUpdateRequeueDelay) } diff --git a/internal/controllers/remotestoragenodeset/remote_objects.go b/internal/controllers/remotestoragenodeset/remote_objects.go index a54f6a39..4f8c670f 100644 --- a/internal/controllers/remotestoragenodeset/remote_objects.go +++ b/internal/controllers/remotestoragenodeset/remote_objects.go @@ -60,7 +60,6 @@ func (r *Reconciler) syncRemoteObjects( for _, remoteObj := range remoteObjects { remoteObjName := remoteObj.GetName() remoteObjKind := remoteObj.GetObjectKind().GroupVersionKind().Kind - remoteObjRV := remoteObj.GetResourceVersion() var remoteResource *v1alpha1.RemoteResource for idx := range remoteStorageNodeSet.Status.RemoteResources { if resources.EqualRemoteResourceWithObject(&remoteStorageNodeSet.Status.RemoteResources[idx], remoteObj) { @@ -103,11 +102,11 @@ func (r *Reconciler) syncRemoteObjects( fmt.Sprintf("Failed to get resource %s with name %s: %s", remoteObjKind, remoteObjName, remoteGetErr), ) } - remoteStorageNodeSet.UpdateRemoteResourceStatus(remoteResource, metav1.ConditionFalse, remoteObjRV) - return r.updateStatusRemoteObjects(ctx, remoteStorageNodeSet, DefaultRequeueDelay) + return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, remoteGetErr } // Check object existence in local cluster + remoteObjRV := remoteObj.GetResourceVersion() localObj := resources.CreateResource(remoteObj) getErr := r.Client.Get(ctx, types.NamespacedName{ Name: localObj.GetName(), @@ -145,7 +144,7 @@ func (r *Reconciler) syncRemoteObjects( "Provisioning", fmt.Sprintf("RemoteSync CREATE resource %s with name %s", remoteObjKind, remoteObjName), ) - remoteStorageNodeSet.UpdateRemoteResourceStatus(remoteResource, metav1.ConditionFalse, remoteObjRV) + remoteStorageNodeSet.UpdateRemoteResourceStatus(remoteResource, metav1.ConditionTrue, remoteObjRV) return r.updateStatusRemoteObjects(ctx, remoteStorageNodeSet, StatusUpdateRequeueDelay) } From 1a3a3a7dcb1d7dcb80c92d7397831e9442cd4808 Mon Sep 17 00:00:00 2001 From: Aleksei Kobzev Date: Tue, 28 Jan 2025 13:54:05 +0800 Subject: [PATCH 04/21] use oldObj as statefulSet inside IgnoreChangesFunction (#286) * use oldObj as statefulSet inside IgnoreChangesFunction * fix typo --- .changes/unreleased/Fixed-20250127-141003.yaml | 3 +++ internal/controllers/database/sync.go | 4 ++-- internal/controllers/storage/sync.go | 4 ++-- 3 files changed, 7 insertions(+), 4 deletions(-) create mode 100644 .changes/unreleased/Fixed-20250127-141003.yaml diff --git a/.changes/unreleased/Fixed-20250127-141003.yaml b/.changes/unreleased/Fixed-20250127-141003.yaml new file mode 100644 index 00000000..aa6c20f7 --- /dev/null +++ b/.changes/unreleased/Fixed-20250127-141003.yaml @@ -0,0 +1,3 @@ +kind: Fixed +body: panic when create object with .spec.pause is true +time: 2025-01-27T14:10:03.497565+08:00 diff --git a/internal/controllers/database/sync.go b/internal/controllers/database/sync.go index 6255d521..0ad5e9ad 100644 --- a/internal/controllers/database/sync.go +++ b/internal/controllers/database/sync.go @@ -359,8 +359,8 @@ func (r *Reconciler) waitForStatefulSetToScale( func shouldIgnoreDatabaseChange(database *resources.DatabaseBuilder) resources.IgnoreChangesFunction { return func(oldObj, newObj runtime.Object) bool { - if _, ok := newObj.(*appsv1.StatefulSet); ok { - if database.Spec.Pause && *oldObj.(*appsv1.StatefulSet).Spec.Replicas == 0 { + if statefulSet, ok := oldObj.(*appsv1.StatefulSet); ok { + if database.Spec.Pause && *statefulSet.Spec.Replicas == 0 { return true } } diff --git a/internal/controllers/storage/sync.go b/internal/controllers/storage/sync.go index ebf833ec..55cde3ce 100644 --- a/internal/controllers/storage/sync.go +++ b/internal/controllers/storage/sync.go @@ -292,8 +292,8 @@ func (r *Reconciler) waitForNodeSetsToProvisioned( func shouldIgnoreStorageChange(storage *resources.StorageClusterBuilder) resources.IgnoreChangesFunction { return func(oldObj, newObj runtime.Object) bool { - if _, ok := newObj.(*appsv1.StatefulSet); ok { - if storage.Spec.Pause && *oldObj.(*appsv1.StatefulSet).Spec.Replicas == 0 { + if statefulSet, ok := oldObj.(*appsv1.StatefulSet); ok { + if storage.Spec.Pause && *statefulSet.Spec.Replicas == 0 { return true } } From c3664d0a8c5feafecd795cbf3dc21dda88b72576 Mon Sep 17 00:00:00 2001 From: Aleksei Kobzev Date: Tue, 28 Jan 2025 18:43:51 +0800 Subject: [PATCH 05/21] ObservedGeneration in conditions (#275) --- .../unreleased/Added-20250127-130609.yaml | 3 + internal/controllers/database/init.go | 63 +++--- internal/controllers/database/sync.go | 211 ++++++++++-------- internal/controllers/databasenodeset/sync.go | 132 ++++++----- internal/controllers/storage/init.go | 25 ++- internal/controllers/storage/sync.go | 201 +++++++++-------- internal/controllers/storagenodeset/sync.go | 136 ++++++----- internal/resources/remotedatabasenodeset.go | 2 +- internal/resources/remotestoragenodeset.go | 2 +- 9 files changed, 428 insertions(+), 347 deletions(-) create mode 100644 .changes/unreleased/Added-20250127-130609.yaml diff --git a/.changes/unreleased/Added-20250127-130609.yaml b/.changes/unreleased/Added-20250127-130609.yaml new file mode 100644 index 00000000..16003da3 --- /dev/null +++ b/.changes/unreleased/Added-20250127-130609.yaml @@ -0,0 +1,3 @@ +kind: Added +body: field ObservedGeneration inside .status.conditions +time: 2025-01-27T13:06:09.845302+08:00 diff --git a/internal/controllers/database/init.go b/internal/controllers/database/init.go index 910f6d6c..1a2770be 100644 --- a/internal/controllers/database/init.go +++ b/internal/controllers/database/init.go @@ -24,10 +24,11 @@ func (r *Reconciler) setInitPipelineStatus( ) (bool, ctrl.Result, error) { if database.Status.State == DatabasePreparing { meta.SetStatusCondition(&database.Status.Conditions, metav1.Condition{ - Type: DatabaseInitializedCondition, - Status: metav1.ConditionFalse, - Reason: ReasonInProgress, - Message: "Database has not been initialized yet", + Type: DatabaseInitializedCondition, + Status: metav1.ConditionFalse, + Reason: ReasonInProgress, + ObservedGeneration: database.Generation, + Message: "Database has not been initialized yet", }) database.Status.State = DatabaseInitializing return r.updateStatus(ctx, database, StatusUpdateRequeueDelay) @@ -58,16 +59,18 @@ func (r *Reconciler) setInitDatabaseCompleted( message string, ) (bool, ctrl.Result, error) { meta.SetStatusCondition(&database.Status.Conditions, metav1.Condition{ - Type: DatabaseInitializedCondition, - Status: metav1.ConditionTrue, - Reason: ReasonCompleted, - Message: message, + Type: DatabaseInitializedCondition, + Status: metav1.ConditionTrue, + Reason: ReasonCompleted, + ObservedGeneration: database.Generation, + Message: message, }) meta.SetStatusCondition(&database.Status.Conditions, metav1.Condition{ - Type: CreateDatabaseOperationCondition, - Status: metav1.ConditionTrue, - Reason: ReasonCompleted, - Message: "Tenant creation operation is completed", + Type: CreateDatabaseOperationCondition, + Status: metav1.ConditionTrue, + Reason: ReasonCompleted, + ObservedGeneration: database.Generation, + Message: "Tenant creation operation is completed", }) return r.updateStatus(ctx, database, StatusUpdateRequeueDelay) } @@ -90,10 +93,11 @@ func (r *Reconciler) checkCreateDatabaseOperation( errMessage, ) meta.SetStatusCondition(&database.Status.Conditions, metav1.Condition{ - Type: CreateDatabaseOperationCondition, - Status: metav1.ConditionFalse, - Reason: ReasonFailed, - Message: errMessage, + Type: CreateDatabaseOperationCondition, + Status: metav1.ConditionFalse, + Reason: ReasonFailed, + ObservedGeneration: database.Generation, + Message: errMessage, }) return r.updateStatus(ctx, database, DatabaseInitializationRequeueDelay) } @@ -124,10 +128,11 @@ func (r *Reconciler) checkCreateDatabaseOperation( errMessage, ) meta.SetStatusCondition(&database.Status.Conditions, metav1.Condition{ - Type: CreateDatabaseOperationCondition, - Status: metav1.ConditionFalse, - Reason: ReasonCompleted, - Message: errMessage, + Type: CreateDatabaseOperationCondition, + Status: metav1.ConditionFalse, + Reason: ReasonCompleted, + ObservedGeneration: database.Generation, + Message: errMessage, }) return r.updateStatus(ctx, database, DatabaseInitializationRequeueDelay) } @@ -140,10 +145,11 @@ func (r *Reconciler) checkCreateDatabaseOperation( fmt.Sprintf("Tenant creation operation is not completed, operationID: %s", operationID), ) meta.SetStatusCondition(&database.Status.Conditions, metav1.Condition{ - Type: CreateDatabaseOperationCondition, - Status: metav1.ConditionUnknown, - Reason: ReasonInProgress, - Message: operationID, + Type: CreateDatabaseOperationCondition, + Status: metav1.ConditionUnknown, + Reason: ReasonInProgress, + ObservedGeneration: database.Generation, + Message: operationID, }) return r.updateStatus(ctx, database, DatabaseInitializationRequeueDelay) } @@ -296,10 +302,11 @@ func (r *Reconciler) initializeTenant( fmt.Sprintf("Tenant creation operation in progress, operationID: %s", operationID), ) meta.SetStatusCondition(&database.Status.Conditions, metav1.Condition{ - Type: CreateDatabaseOperationCondition, - Status: metav1.ConditionUnknown, - Reason: ReasonInProgress, - Message: operationID, + Type: CreateDatabaseOperationCondition, + Status: metav1.ConditionUnknown, + Reason: ReasonInProgress, + ObservedGeneration: database.Generation, + Message: operationID, }) return r.updateStatus(ctx, database, DatabaseInitializationRequeueDelay) } diff --git a/internal/controllers/database/sync.go b/internal/controllers/database/sync.go index 0ad5e9ad..095f2b51 100644 --- a/internal/controllers/database/sync.go +++ b/internal/controllers/database/sync.go @@ -48,11 +48,6 @@ func (r *Reconciler) Sync(ctx context.Context, ydbCr *v1alpha1.Database) (ctrl.R return result, err } - stop, result, err = r.syncNodeSetSpecInline(ctx, &database) - if stop { - return result, err - } - if !meta.IsStatusConditionTrue(database.Status.Conditions, DatabaseInitializedCondition) { return r.handleTenantCreation(ctx, &database) } @@ -87,17 +82,19 @@ func (r *Reconciler) setInitialStatus( if database.Spec.Pause { meta.SetStatusCondition(&database.Status.Conditions, metav1.Condition{ - Type: DatabasePausedCondition, - Status: metav1.ConditionUnknown, - Reason: ReasonInProgress, - Message: "Transitioning to state Paused", + Type: DatabasePausedCondition, + Status: metav1.ConditionUnknown, + Reason: ReasonInProgress, + ObservedGeneration: database.Generation, + Message: "Transitioning to state Paused", }) } else { meta.SetStatusCondition(&database.Status.Conditions, metav1.Condition{ - Type: DatabaseReadyCondition, - Status: metav1.ConditionUnknown, - Reason: ReasonInProgress, - Message: "Transitioning to state Ready", + Type: DatabaseReadyCondition, + Status: metav1.ConditionUnknown, + Reason: ReasonInProgress, + ObservedGeneration: database.Generation, + Message: "Transitioning to state Ready", }) } @@ -113,10 +110,11 @@ func (r *Reconciler) waitForClusterResources(ctx context.Context, database *reso if database.Status.State == DatabasePending { meta.SetStatusCondition(&database.Status.Conditions, metav1.Condition{ - Type: DatabasePreparedCondition, - Status: metav1.ConditionUnknown, - Reason: ReasonInProgress, - Message: fmt.Sprintf("Waiting for sync resources for generation %d", database.Generation), + Type: DatabasePreparedCondition, + Status: metav1.ConditionUnknown, + Reason: ReasonInProgress, + ObservedGeneration: database.Generation, + Message: "Waiting for sync resources", }) database.Status.State = DatabasePreparing return r.updateStatus(ctx, database, StatusUpdateRequeueDelay) @@ -167,9 +165,10 @@ func (r *Reconciler) waitForClusterResources(ctx context.Context, database *reso ), ) meta.SetStatusCondition(&database.Status.Conditions, metav1.Condition{ - Type: DatabasePreparedCondition, - Status: metav1.ConditionFalse, - Reason: ReasonInProgress, + Type: DatabasePreparedCondition, + Status: metav1.ConditionFalse, + Reason: ReasonInProgress, + ObservedGeneration: database.Generation, Message: fmt.Sprintf( "Referenced storage cluster (%s, %s) is not initialized", database.Spec.StorageClusterRef.Name, @@ -193,10 +192,11 @@ func (r *Reconciler) waitForNodeSetsToProvisioned( if database.Status.State == DatabaseInitializing { meta.SetStatusCondition(&database.Status.Conditions, metav1.Condition{ - Type: DatabaseProvisionedCondition, - Status: metav1.ConditionUnknown, - Reason: ReasonInProgress, - Message: "Waiting for NodeSets conditions to be Provisioned", + Type: DatabaseProvisionedCondition, + Status: metav1.ConditionUnknown, + Reason: ReasonInProgress, + ObservedGeneration: database.Generation, + Message: "Waiting for NodeSet resources to be Provisioned", }) database.Status.State = DatabaseProvisioning return r.updateStatus(ctx, database, StatusUpdateRequeueDelay) @@ -242,8 +242,8 @@ func (r *Reconciler) waitForNodeSetsToProvisioned( nodeSetConditions = nodeSetObject.(*v1alpha1.DatabaseNodeSet).Status.Conditions } - // TODO: also check observedGeneration to guarantee that compare with updated object - if !meta.IsStatusConditionTrue(nodeSetConditions, NodeSetProvisionedCondition) { + condition := meta.FindStatusCondition(nodeSetConditions, NodeSetProvisionedCondition) + if condition == nil || condition.ObservedGeneration != nodeSetObject.GetGeneration() || condition.Status != metav1.ConditionTrue { r.Recorder.Event( database, corev1.EventTypeNormal, @@ -255,9 +255,10 @@ func (r *Reconciler) waitForNodeSetsToProvisioned( ), ) meta.SetStatusCondition(&database.Status.Conditions, metav1.Condition{ - Type: DatabaseProvisionedCondition, - Status: metav1.ConditionFalse, - Reason: ReasonInProgress, + Type: DatabaseProvisionedCondition, + Status: metav1.ConditionFalse, + Reason: ReasonInProgress, + ObservedGeneration: database.Generation, Message: fmt.Sprintf( "Waiting %s with name %s for condition NodeSetProvisioned to be True", nodeSetKind, @@ -270,10 +271,11 @@ func (r *Reconciler) waitForNodeSetsToProvisioned( if !meta.IsStatusConditionTrue(database.Status.Conditions, DatabaseProvisionedCondition) { meta.SetStatusCondition(&database.Status.Conditions, metav1.Condition{ - Type: DatabaseProvisionedCondition, - Status: metav1.ConditionTrue, - Reason: ReasonCompleted, - Message: "Successfully scaled to desired number of nodes", + Type: DatabaseProvisionedCondition, + Status: metav1.ConditionTrue, + Reason: ReasonCompleted, + ObservedGeneration: database.Generation, + Message: fmt.Sprintf("Successfully scaled to desired number of nodes: %d", database.Spec.Nodes), }) return r.updateStatus(ctx, database, StatusUpdateRequeueDelay) } @@ -290,10 +292,11 @@ func (r *Reconciler) waitForStatefulSetToScale( if database.Status.State == DatabaseInitializing { meta.SetStatusCondition(&database.Status.Conditions, metav1.Condition{ - Type: DatabaseProvisionedCondition, - Status: metav1.ConditionUnknown, - Reason: ReasonInProgress, - Message: fmt.Sprintf("Waiting for scale to desired nodes: %d", database.Spec.Nodes), + Type: DatabaseProvisionedCondition, + Status: metav1.ConditionUnknown, + Reason: ReasonInProgress, + ObservedGeneration: database.Generation, + Message: fmt.Sprintf("Waiting for scale to desired number of nodes: %d", database.Spec.Nodes), }) database.Status.State = DatabaseProvisioning return r.updateStatus(ctx, database, StatusUpdateRequeueDelay) @@ -335,20 +338,22 @@ func (r *Reconciler) waitForStatefulSetToScale( fmt.Sprintf("Waiting for number of running pods to match expected: %d != %d", foundStatefulSet.Status.ReadyReplicas, database.Spec.Nodes), ) meta.SetStatusCondition(&database.Status.Conditions, metav1.Condition{ - Type: DatabaseProvisionedCondition, - Status: metav1.ConditionFalse, - Reason: ReasonInProgress, - Message: fmt.Sprintf("Number of running pods does not match expected: %d != %d", foundStatefulSet.Status.ReadyReplicas, database.Spec.Nodes), + Type: DatabaseProvisionedCondition, + Status: metav1.ConditionFalse, + Reason: ReasonInProgress, + ObservedGeneration: database.Generation, + Message: fmt.Sprintf("Number of running nodes does not match expected: %d != %d", foundStatefulSet.Status.ReadyReplicas, database.Spec.Nodes), }) return r.updateStatus(ctx, database, DefaultRequeueDelay) } if !meta.IsStatusConditionTrue(database.Status.Conditions, DatabaseProvisionedCondition) { meta.SetStatusCondition(&database.Status.Conditions, metav1.Condition{ - Type: DatabaseProvisionedCondition, - Status: metav1.ConditionTrue, - Reason: ReasonCompleted, - Message: fmt.Sprintf("Successfully scaled to desired number of nodes: %d", database.Spec.Nodes), + Type: DatabaseProvisionedCondition, + Status: metav1.ConditionTrue, + Reason: ReasonCompleted, + ObservedGeneration: database.Generation, + Message: fmt.Sprintf("Successfully scaled to desired number of nodes: %d", database.Spec.Nodes), }) return r.updateStatus(ctx, database, StatusUpdateRequeueDelay) } @@ -437,10 +442,11 @@ func (r *Reconciler) handleResourcesSync( eventMessage+fmt.Sprintf(", failed to sync, error: %s", err), ) meta.SetStatusCondition(&database.Status.Conditions, metav1.Condition{ - Type: DatabasePreparedCondition, - Status: metav1.ConditionFalse, - Reason: ReasonInProgress, - Message: fmt.Sprintf("Failed to sync resources for generation %d", database.Generation), + Type: DatabasePreparedCondition, + Status: metav1.ConditionFalse, + Reason: ReasonInProgress, + ObservedGeneration: database.Generation, + Message: "Failed to sync resources", }) return r.updateStatus(ctx, database, DefaultRequeueDelay) } else if result == controllerutil.OperationResultCreated || result == controllerutil.OperationResultUpdated { @@ -453,6 +459,28 @@ func (r *Reconciler) handleResourcesSync( } } + if err := r.syncNodeSetSpecInline(ctx, database); err != nil { + meta.SetStatusCondition(&database.Status.Conditions, metav1.Condition{ + Type: DatabasePreparedCondition, + Status: metav1.ConditionFalse, + Reason: ReasonInProgress, + ObservedGeneration: database.Generation, + Message: "Failed to sync NodeSet resources", + }) + return r.updateStatus(ctx, database, DefaultRequeueDelay) + } + + if !meta.IsStatusConditionTrue(database.Status.Conditions, DatabasePreparedCondition) { + meta.SetStatusCondition(&database.Status.Conditions, metav1.Condition{ + Type: DatabasePreparedCondition, + Status: metav1.ConditionTrue, + Reason: ReasonCompleted, + ObservedGeneration: database.Generation, + Message: "Successfully synced resources", + }) + return r.updateStatus(ctx, database, StatusUpdateRequeueDelay) + } + r.Log.Info("complete step handleResourcesSync") return Continue, ctrl.Result{Requeue: false}, nil } @@ -469,15 +497,17 @@ func (r *Reconciler) updateStatus( meta.IsStatusConditionFalse(database.Status.Conditions, DatabaseProvisionedCondition) { if database.Spec.Pause { meta.SetStatusCondition(&database.Status.Conditions, metav1.Condition{ - Type: DatabasePausedCondition, - Status: metav1.ConditionFalse, - Reason: ReasonInProgress, + Type: DatabasePausedCondition, + Status: metav1.ConditionFalse, + Reason: ReasonInProgress, + ObservedGeneration: database.Generation, }) } else { meta.SetStatusCondition(&database.Status.Conditions, metav1.Condition{ - Type: DatabaseReadyCondition, - Status: metav1.ConditionFalse, - Reason: ReasonInProgress, + Type: DatabaseReadyCondition, + Status: metav1.ConditionFalse, + Reason: ReasonInProgress, + ObservedGeneration: database.Generation, }) } } @@ -526,9 +556,7 @@ func (r *Reconciler) updateStatus( func (r *Reconciler) syncNodeSetSpecInline( ctx context.Context, database *resources.DatabaseBuilder, -) (bool, ctrl.Result, error) { - r.Log.Info("running step syncNodeSetSpecInline") - +) error { databaseNodeSets := &v1alpha1.DatabaseNodeSetList{} if err := r.List(ctx, databaseNodeSets, client.InNamespace(database.Namespace), @@ -542,7 +570,7 @@ func (r *Reconciler) syncNodeSetSpecInline( "ProvisioningFailed", fmt.Sprintf("Failed to list DatabaseNodeSets: %s", err), ) - return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err + return err } for _, databaseNodeSet := range databaseNodeSets.Items { @@ -565,7 +593,7 @@ func (r *Reconciler) syncNodeSetSpecInline( "ProvisioningFailed", fmt.Sprintf("Failed to delete DatabaseNodeSet: %s", err), ) - return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err + return err } r.Recorder.Event( database, @@ -592,7 +620,7 @@ func (r *Reconciler) syncNodeSetSpecInline( "ProvisioningFailed", fmt.Sprintf("Failed to list RemoteDatabaseNodeSets: %s", err), ) - return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err + return err } for _, remoteDatabaseNodeSet := range remoteDatabaseNodeSets.Items { @@ -616,7 +644,7 @@ func (r *Reconciler) syncNodeSetSpecInline( "ProvisioningFailed", fmt.Sprintf("Failed to delete RemoteDatabaseNodeSet: %s", err), ) - return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err + return err } r.Recorder.Event( database, @@ -630,18 +658,7 @@ func (r *Reconciler) syncNodeSetSpecInline( } } - if !meta.IsStatusConditionTrue(database.Status.Conditions, DatabasePreparedCondition) { - meta.SetStatusCondition(&database.Status.Conditions, metav1.Condition{ - Type: DatabasePreparedCondition, - Status: metav1.ConditionTrue, - Reason: ReasonCompleted, - Message: "Successfully synced resources", - }) - return r.updateStatus(ctx, database, StatusUpdateRequeueDelay) - } - - r.Log.Info("complete step syncNodeSetSpecInline") - return Continue, ctrl.Result{Requeue: false}, nil + return nil } func (r *Reconciler) handlePauseResume( @@ -653,16 +670,18 @@ func (r *Reconciler) handlePauseResume( if database.Status.State == DatabaseProvisioning { if database.Spec.Pause { meta.SetStatusCondition(&database.Status.Conditions, metav1.Condition{ - Type: DatabasePausedCondition, - Status: metav1.ConditionTrue, - Reason: ReasonCompleted, + Type: DatabasePausedCondition, + Status: metav1.ConditionTrue, + Reason: ReasonCompleted, + ObservedGeneration: database.Generation, }) database.Status.State = DatabasePaused } else { meta.SetStatusCondition(&database.Status.Conditions, metav1.Condition{ - Type: DatabaseReadyCondition, - Status: metav1.ConditionTrue, - Reason: ReasonCompleted, + Type: DatabaseReadyCondition, + Status: metav1.ConditionTrue, + Reason: ReasonCompleted, + ObservedGeneration: database.Generation, }) database.Status.State = DatabaseReady } @@ -672,10 +691,11 @@ func (r *Reconciler) handlePauseResume( if database.Status.State == DatabaseReady && database.Spec.Pause { r.Log.Info("`pause: true` was noticed, moving Database to state `Paused`") meta.SetStatusCondition(&database.Status.Conditions, metav1.Condition{ - Type: DatabaseReadyCondition, - Status: metav1.ConditionFalse, - Reason: ReasonNotRequired, - Message: "Transitioning to state Paused", + Type: DatabaseReadyCondition, + Status: metav1.ConditionFalse, + Reason: ReasonNotRequired, + ObservedGeneration: database.Generation, + Message: "Transitioning to state Paused", }) database.Status.State = DatabasePaused return r.updateStatus(ctx, database, StatusUpdateRequeueDelay) @@ -684,10 +704,11 @@ func (r *Reconciler) handlePauseResume( if database.Status.State == DatabasePaused && !database.Spec.Pause { r.Log.Info("`pause: false` was noticed, moving Database to state `Ready`") meta.SetStatusCondition(&database.Status.Conditions, metav1.Condition{ - Type: DatabasePausedCondition, - Status: metav1.ConditionFalse, - Reason: ReasonNotRequired, - Message: "Transitioning to state Ready", + Type: DatabasePausedCondition, + Status: metav1.ConditionFalse, + Reason: ReasonNotRequired, + ObservedGeneration: database.Generation, + Message: "Transitioning to state Ready", }) database.Status.State = DatabaseReady return r.updateStatus(ctx, database, StatusUpdateRequeueDelay) @@ -696,18 +717,20 @@ func (r *Reconciler) handlePauseResume( if database.Spec.Pause { if !meta.IsStatusConditionTrue(database.Status.Conditions, DatabasePausedCondition) { meta.SetStatusCondition(&database.Status.Conditions, metav1.Condition{ - Type: DatabasePausedCondition, - Status: metav1.ConditionTrue, - Reason: ReasonCompleted, + Type: DatabasePausedCondition, + Status: metav1.ConditionTrue, + Reason: ReasonCompleted, + ObservedGeneration: database.Generation, }) return r.updateStatus(ctx, database, StatusUpdateRequeueDelay) } } else { if !meta.IsStatusConditionTrue(database.Status.Conditions, DatabaseReadyCondition) { meta.SetStatusCondition(&database.Status.Conditions, metav1.Condition{ - Type: DatabaseReadyCondition, - Status: metav1.ConditionTrue, - Reason: ReasonCompleted, + Type: DatabaseReadyCondition, + Status: metav1.ConditionTrue, + Reason: ReasonCompleted, + ObservedGeneration: database.Generation, }) return r.updateStatus(ctx, database, StatusUpdateRequeueDelay) } diff --git a/internal/controllers/databasenodeset/sync.go b/internal/controllers/databasenodeset/sync.go index 0e0b2cde..34f2147a 100644 --- a/internal/controllers/databasenodeset/sync.go +++ b/internal/controllers/databasenodeset/sync.go @@ -62,17 +62,19 @@ func (r *Reconciler) setInitialStatus( if databaseNodeSet.Spec.Pause { meta.SetStatusCondition(&databaseNodeSet.Status.Conditions, metav1.Condition{ - Type: NodeSetPausedCondition, - Status: metav1.ConditionUnknown, - Reason: ReasonInProgress, - Message: "Transitioning to state Paused", + Type: NodeSetPausedCondition, + Status: metav1.ConditionUnknown, + Reason: ReasonInProgress, + ObservedGeneration: databaseNodeSet.Generation, + Message: "Transitioning to state Paused", }) } else { meta.SetStatusCondition(&databaseNodeSet.Status.Conditions, metav1.Condition{ - Type: NodeSetReadyCondition, - Status: metav1.ConditionUnknown, - Reason: ReasonInProgress, - Message: "Transitioning to state Ready", + Type: NodeSetReadyCondition, + Status: metav1.ConditionUnknown, + Reason: ReasonInProgress, + ObservedGeneration: databaseNodeSet.Generation, + Message: "Transitioning to state Ready", }) } @@ -102,10 +104,11 @@ func (r *Reconciler) handleResourcesSync( if databaseNodeSet.Status.State == DatabaseNodeSetPending { meta.SetStatusCondition(&databaseNodeSet.Status.Conditions, metav1.Condition{ - Type: NodeSetPreparedCondition, - Status: metav1.ConditionUnknown, - Reason: ReasonInProgress, - Message: fmt.Sprintf("Waiting for sync resources for generation %d", databaseNodeSet.Generation), + Type: NodeSetPreparedCondition, + Status: metav1.ConditionUnknown, + Reason: ReasonInProgress, + ObservedGeneration: databaseNodeSet.Generation, + Message: "Waiting for sync resources", }) databaseNodeSet.Status.State = DatabaseNodeSetPreparing return r.updateStatus(ctx, databaseNodeSet, StatusUpdateRequeueDelay) @@ -155,10 +158,11 @@ func (r *Reconciler) handleResourcesSync( eventMessage+fmt.Sprintf(", failed to sync, error: %s", err), ) meta.SetStatusCondition(&databaseNodeSet.Status.Conditions, metav1.Condition{ - Type: NodeSetPreparedCondition, - Status: metav1.ConditionFalse, - Reason: ReasonInProgress, - Message: fmt.Sprintf("Failed to sync resources for generation %d", databaseNodeSet.Generation), + Type: NodeSetPreparedCondition, + Status: metav1.ConditionFalse, + Reason: ReasonInProgress, + ObservedGeneration: databaseNodeSet.Generation, + Message: "Failed to sync resources", }) return r.updateStatus(ctx, databaseNodeSet, DefaultRequeueDelay) } else if result == controllerutil.OperationResultCreated || result == controllerutil.OperationResultUpdated { @@ -173,10 +177,11 @@ func (r *Reconciler) handleResourcesSync( if !meta.IsStatusConditionTrue(databaseNodeSet.Status.Conditions, NodeSetPreparedCondition) { meta.SetStatusCondition(&databaseNodeSet.Status.Conditions, metav1.Condition{ - Type: NodeSetPreparedCondition, - Status: metav1.ConditionTrue, - Reason: ReasonCompleted, - Message: "Successfully synced resources", + Type: NodeSetPreparedCondition, + Status: metav1.ConditionTrue, + Reason: ReasonCompleted, + ObservedGeneration: databaseNodeSet.Generation, + Message: "Successfully synced resources", }) return r.updateStatus(ctx, databaseNodeSet, StatusUpdateRequeueDelay) } @@ -193,10 +198,11 @@ func (r *Reconciler) waitForStatefulSetToScale( if databaseNodeSet.Status.State == DatabaseNodeSetPreparing { meta.SetStatusCondition(&databaseNodeSet.Status.Conditions, metav1.Condition{ - Type: NodeSetProvisionedCondition, - Status: metav1.ConditionUnknown, - Reason: ReasonInProgress, - Message: fmt.Sprintf("Waiting for scale to desired nodes: %d", databaseNodeSet.Spec.Nodes), + Type: NodeSetProvisionedCondition, + Status: metav1.ConditionUnknown, + Reason: ReasonInProgress, + ObservedGeneration: databaseNodeSet.Generation, + Message: fmt.Sprintf("Waiting for scale to desired nodes: %d", databaseNodeSet.Spec.Nodes), }) databaseNodeSet.Status.State = DatabaseNodeSetProvisioning return r.updateStatus(ctx, databaseNodeSet, StatusUpdateRequeueDelay) @@ -234,20 +240,22 @@ func (r *Reconciler) waitForStatefulSetToScale( fmt.Sprintf("Waiting for number of running nodes to match expected: %d != %d", foundStatefulSet.Status.ReadyReplicas, databaseNodeSet.Spec.Nodes), ) meta.SetStatusCondition(&databaseNodeSet.Status.Conditions, metav1.Condition{ - Type: NodeSetProvisionedCondition, - Status: metav1.ConditionFalse, - Reason: ReasonInProgress, - Message: fmt.Sprintf("Number of running nodes does not match expected: %d != %d", foundStatefulSet.Status.ReadyReplicas, databaseNodeSet.Spec.Nodes), + Type: NodeSetProvisionedCondition, + Status: metav1.ConditionFalse, + Reason: ReasonInProgress, + ObservedGeneration: databaseNodeSet.Generation, + Message: fmt.Sprintf("Number of running nodes does not match expected: %d != %d", foundStatefulSet.Status.ReadyReplicas, databaseNodeSet.Spec.Nodes), }) return r.updateStatus(ctx, databaseNodeSet, DefaultRequeueDelay) } if !meta.IsStatusConditionTrue(databaseNodeSet.Status.Conditions, NodeSetProvisionedCondition) { meta.SetStatusCondition(&databaseNodeSet.Status.Conditions, metav1.Condition{ - Type: NodeSetProvisionedCondition, - Status: metav1.ConditionTrue, - Reason: ReasonCompleted, - Message: fmt.Sprintf("Successfully scaled to desired number of nodes: %d", databaseNodeSet.Spec.Nodes), + Type: NodeSetProvisionedCondition, + Status: metav1.ConditionTrue, + Reason: ReasonCompleted, + ObservedGeneration: databaseNodeSet.Generation, + Message: fmt.Sprintf("Successfully scaled to desired number of nodes: %d", databaseNodeSet.Spec.Nodes), }) return r.updateStatus(ctx, databaseNodeSet, StatusUpdateRequeueDelay) } @@ -267,15 +275,17 @@ func (r *Reconciler) updateStatus( meta.IsStatusConditionFalse(databaseNodeSet.Status.Conditions, NodeSetProvisionedCondition) { if databaseNodeSet.Spec.Pause { meta.SetStatusCondition(&databaseNodeSet.Status.Conditions, metav1.Condition{ - Type: NodeSetPausedCondition, - Status: metav1.ConditionFalse, - Reason: ReasonInProgress, + Type: NodeSetPausedCondition, + Status: metav1.ConditionFalse, + ObservedGeneration: databaseNodeSet.Generation, + Reason: ReasonInProgress, }) } else { meta.SetStatusCondition(&databaseNodeSet.Status.Conditions, metav1.Condition{ - Type: NodeSetReadyCondition, - Status: metav1.ConditionFalse, - Reason: ReasonInProgress, + Type: NodeSetReadyCondition, + Status: metav1.ConditionFalse, + ObservedGeneration: databaseNodeSet.Generation, + Reason: ReasonInProgress, }) } } @@ -340,16 +350,18 @@ func (r *Reconciler) handlePauseResume( if databaseNodeSet.Status.State == DatabaseNodeSetProvisioning { if databaseNodeSet.Spec.Pause { meta.SetStatusCondition(&databaseNodeSet.Status.Conditions, metav1.Condition{ - Type: NodeSetPausedCondition, - Status: metav1.ConditionTrue, - Reason: ReasonCompleted, + Type: NodeSetPausedCondition, + Status: metav1.ConditionTrue, + Reason: ReasonCompleted, + ObservedGeneration: databaseNodeSet.Generation, }) databaseNodeSet.Status.State = DatabaseNodeSetPaused } else { meta.SetStatusCondition(&databaseNodeSet.Status.Conditions, metav1.Condition{ - Type: NodeSetReadyCondition, - Status: metav1.ConditionTrue, - Reason: ReasonCompleted, + Type: NodeSetReadyCondition, + Status: metav1.ConditionTrue, + Reason: ReasonCompleted, + ObservedGeneration: databaseNodeSet.Generation, }) databaseNodeSet.Status.State = DatabaseNodeSetReady } @@ -359,10 +371,11 @@ func (r *Reconciler) handlePauseResume( if databaseNodeSet.Status.State == DatabaseNodeSetReady && databaseNodeSet.Spec.Pause { r.Log.Info("`pause: true` was noticed, moving DatabaseNodeSet to state `Paused`") meta.SetStatusCondition(&databaseNodeSet.Status.Conditions, metav1.Condition{ - Type: NodeSetReadyCondition, - Status: metav1.ConditionFalse, - Reason: ReasonNotRequired, - Message: "Transitioning to state Paused", + Type: NodeSetReadyCondition, + Status: metav1.ConditionFalse, + Reason: ReasonNotRequired, + ObservedGeneration: databaseNodeSet.Generation, + Message: "Transitioning to state Paused", }) databaseNodeSet.Status.State = DatabaseNodeSetPaused return r.updateStatus(ctx, databaseNodeSet, StatusUpdateRequeueDelay) @@ -371,10 +384,11 @@ func (r *Reconciler) handlePauseResume( if databaseNodeSet.Status.State == DatabaseNodeSetPaused && !databaseNodeSet.Spec.Pause { r.Log.Info("`pause: false` was noticed, moving DatabaseNodeSet to state `Ready`") meta.SetStatusCondition(&databaseNodeSet.Status.Conditions, metav1.Condition{ - Type: NodeSetPausedCondition, - Status: metav1.ConditionFalse, - Reason: ReasonNotRequired, - Message: "Transitioning to state Ready", + Type: NodeSetPausedCondition, + Status: metav1.ConditionFalse, + Reason: ReasonNotRequired, + ObservedGeneration: databaseNodeSet.Generation, + Message: "Transitioning to state Ready", }) databaseNodeSet.Status.State = DatabaseNodeSetReady return r.updateStatus(ctx, databaseNodeSet, StatusUpdateRequeueDelay) @@ -383,18 +397,20 @@ func (r *Reconciler) handlePauseResume( if databaseNodeSet.Spec.Pause { if !meta.IsStatusConditionTrue(databaseNodeSet.Status.Conditions, NodeSetPausedCondition) { meta.SetStatusCondition(&databaseNodeSet.Status.Conditions, metav1.Condition{ - Type: NodeSetPausedCondition, - Status: metav1.ConditionTrue, - Reason: ReasonCompleted, + Type: NodeSetPausedCondition, + Status: metav1.ConditionTrue, + Reason: ReasonCompleted, + ObservedGeneration: databaseNodeSet.Generation, }) return r.updateStatus(ctx, databaseNodeSet, StatusUpdateRequeueDelay) } } else { if !meta.IsStatusConditionTrue(databaseNodeSet.Status.Conditions, NodeSetReadyCondition) { meta.SetStatusCondition(&databaseNodeSet.Status.Conditions, metav1.Condition{ - Type: NodeSetReadyCondition, - Status: metav1.ConditionTrue, - Reason: ReasonCompleted, + Type: NodeSetReadyCondition, + Status: metav1.ConditionTrue, + Reason: ReasonCompleted, + ObservedGeneration: databaseNodeSet.Generation, }) return r.updateStatus(ctx, databaseNodeSet, StatusUpdateRequeueDelay) } diff --git a/internal/controllers/storage/init.go b/internal/controllers/storage/init.go index a13dbab6..ec2f2207 100644 --- a/internal/controllers/storage/init.go +++ b/internal/controllers/storage/init.go @@ -34,10 +34,11 @@ func (r *Reconciler) setInitPipelineStatus( ) (bool, ctrl.Result, error) { if storage.Status.State == StoragePreparing { meta.SetStatusCondition(&storage.Status.Conditions, metav1.Condition{ - Type: StorageInitializedCondition, - Status: metav1.ConditionUnknown, - Reason: ReasonInProgress, - Message: "Storage has not been initialized yet", + Type: StorageInitializedCondition, + Status: metav1.ConditionUnknown, + Reason: ReasonInProgress, + ObservedGeneration: storage.Generation, + Message: "Storage has not been initialized yet", }) storage.Status.State = StorageInitializing return r.updateStatus(ctx, storage, StatusUpdateRequeueDelay) @@ -67,10 +68,11 @@ func (r *Reconciler) setInitStorageCompleted( message string, ) (bool, ctrl.Result, error) { meta.SetStatusCondition(&storage.Status.Conditions, metav1.Condition{ - Type: StorageInitializedCondition, - Status: metav1.ConditionTrue, - Reason: ReasonCompleted, - Message: message, + Type: StorageInitializedCondition, + Status: metav1.ConditionTrue, + Reason: ReasonCompleted, + ObservedGeneration: storage.Generation, + Message: message, }) return r.updateStatus(ctx, storage, StatusUpdateRequeueDelay) } @@ -196,9 +198,10 @@ func (r *Reconciler) initializeBlobstorage( "Failed initBlobstorage Job, check Pod logs for additional info", ) meta.SetStatusCondition(&storage.Status.Conditions, metav1.Condition{ - Type: StorageInitializedCondition, - Status: metav1.ConditionFalse, - Reason: ReasonInProgress, + Type: StorageInitializedCondition, + Status: metav1.ConditionFalse, + Reason: ReasonInProgress, + ObservedGeneration: storage.Generation, }) if err := r.Delete(ctx, initJob, client.PropagationPolicy(metav1.DeletePropagationForeground)); err != nil { r.Recorder.Event( diff --git a/internal/controllers/storage/sync.go b/internal/controllers/storage/sync.go index 55cde3ce..60852d93 100644 --- a/internal/controllers/storage/sync.go +++ b/internal/controllers/storage/sync.go @@ -41,11 +41,6 @@ func (r *Reconciler) Sync(ctx context.Context, cr *v1alpha1.Storage) (ctrl.Resul return result, err } - stop, result, err = r.syncNodeSetSpecInline(ctx, &storage) - if stop { - return result, err - } - if !meta.IsStatusConditionTrue(storage.Status.Conditions, StorageInitializedCondition) { return r.handleBlobstorageInit(ctx, &storage) } @@ -100,17 +95,19 @@ func (r *Reconciler) setInitialStatus( if storage.Spec.Pause { meta.SetStatusCondition(&storage.Status.Conditions, metav1.Condition{ - Type: StoragePausedCondition, - Status: metav1.ConditionUnknown, - Reason: ReasonInProgress, - Message: "Transitioning to state Paused", + Type: StoragePausedCondition, + Status: metav1.ConditionUnknown, + Reason: ReasonInProgress, + ObservedGeneration: storage.Generation, + Message: "Transitioning to state Paused", }) } else { meta.SetStatusCondition(&storage.Status.Conditions, metav1.Condition{ - Type: StorageReadyCondition, - Status: metav1.ConditionUnknown, - Reason: ReasonInProgress, - Message: "Transitioning to state Ready", + Type: StorageReadyCondition, + Status: metav1.ConditionUnknown, + Reason: ReasonInProgress, + ObservedGeneration: storage.Generation, + Message: "Transitioning to state Ready", }) } @@ -129,10 +126,11 @@ func (r *Reconciler) waitForStatefulSetToScale( if storage.Status.State == StorageInitializing { meta.SetStatusCondition(&storage.Status.Conditions, metav1.Condition{ - Type: StorageProvisionedCondition, - Status: metav1.ConditionUnknown, - Reason: ReasonInProgress, - Message: fmt.Sprintf("Waiting for scale to desired nodes: %d", storage.Spec.Nodes), + Type: StorageProvisionedCondition, + Status: metav1.ConditionUnknown, + Reason: ReasonInProgress, + ObservedGeneration: storage.Generation, + Message: fmt.Sprintf("Waiting for scale to desired number of nodes: %d", storage.Spec.Nodes), }) storage.Status.State = StorageProvisioning return r.updateStatus(ctx, storage, StatusUpdateRequeueDelay) @@ -170,20 +168,22 @@ func (r *Reconciler) waitForStatefulSetToScale( fmt.Sprintf("Waiting for number of running nodes to match expected: %d != %d", foundStatefulSet.Status.ReadyReplicas, storage.Spec.Nodes), ) meta.SetStatusCondition(&storage.Status.Conditions, metav1.Condition{ - Type: StorageProvisionedCondition, - Status: metav1.ConditionFalse, - Reason: ReasonInProgress, - Message: fmt.Sprintf("Number of running nodes does not match expected: %d != %d", foundStatefulSet.Status.ReadyReplicas, storage.Spec.Nodes), + Type: StorageProvisionedCondition, + Status: metav1.ConditionFalse, + Reason: ReasonInProgress, + ObservedGeneration: storage.Generation, + Message: fmt.Sprintf("Number of running nodes does not match expected: %d != %d", foundStatefulSet.Status.ReadyReplicas, storage.Spec.Nodes), }) return r.updateStatus(ctx, storage, DefaultRequeueDelay) } if !meta.IsStatusConditionTrue(storage.Status.Conditions, StorageProvisionedCondition) { meta.SetStatusCondition(&storage.Status.Conditions, metav1.Condition{ - Type: StorageProvisionedCondition, - Status: metav1.ConditionTrue, - Reason: ReasonCompleted, - Message: fmt.Sprintf("Successfully scaled to desired number of nodes: %d", storage.Spec.Nodes), + Type: StorageProvisionedCondition, + Status: metav1.ConditionTrue, + Reason: ReasonCompleted, + ObservedGeneration: storage.Generation, + Message: fmt.Sprintf("Successfully scaled to desired number of nodes: %d", storage.Spec.Nodes), }) return r.updateStatus(ctx, storage, StatusUpdateRequeueDelay) } @@ -200,10 +200,11 @@ func (r *Reconciler) waitForNodeSetsToProvisioned( if storage.Status.State == StorageInitializing { meta.SetStatusCondition(&storage.Status.Conditions, metav1.Condition{ - Type: StorageProvisionedCondition, - Status: metav1.ConditionUnknown, - Reason: ReasonInProgress, - Message: "Waiting for NodeSets conditions to be Provisioned", + Type: StorageProvisionedCondition, + Status: metav1.ConditionUnknown, + Reason: ReasonInProgress, + ObservedGeneration: storage.Generation, + Message: "Waiting for NodeSet resources to be Provisioned", }) storage.Status.State = StorageProvisioning return r.updateStatus(ctx, storage, StatusUpdateRequeueDelay) @@ -250,8 +251,8 @@ func (r *Reconciler) waitForNodeSetsToProvisioned( nodeSetConditions = nodeSetObject.(*v1alpha1.StorageNodeSet).Status.Conditions } - // TODO: also check observedGeneration to guarantee that compare with updated object - if !meta.IsStatusConditionTrue(nodeSetConditions, NodeSetProvisionedCondition) { + condition := meta.FindStatusCondition(nodeSetConditions, NodeSetProvisionedCondition) + if condition == nil || condition.ObservedGeneration != nodeSetObject.GetGeneration() || condition.Status != metav1.ConditionTrue { r.Recorder.Event( storage, corev1.EventTypeNormal, @@ -263,9 +264,10 @@ func (r *Reconciler) waitForNodeSetsToProvisioned( ), ) meta.SetStatusCondition(&storage.Status.Conditions, metav1.Condition{ - Type: StorageProvisionedCondition, - Status: metav1.ConditionFalse, - Reason: ReasonInProgress, + Type: StorageProvisionedCondition, + Status: metav1.ConditionFalse, + Reason: ReasonInProgress, + ObservedGeneration: storage.Generation, Message: fmt.Sprintf( "Waiting %s with name %s for condition NodeSetProvisioned to be True", nodeSetKind, @@ -278,10 +280,11 @@ func (r *Reconciler) waitForNodeSetsToProvisioned( if !meta.IsStatusConditionTrue(storage.Status.Conditions, StorageProvisionedCondition) { meta.SetStatusCondition(&storage.Status.Conditions, metav1.Condition{ - Type: StorageProvisionedCondition, - Status: metav1.ConditionTrue, - Reason: ReasonCompleted, - Message: "Successfully scaled to desired number of nodes", + Type: StorageProvisionedCondition, + Status: metav1.ConditionTrue, + Reason: ReasonCompleted, + ObservedGeneration: storage.Generation, + Message: fmt.Sprintf("Successfully scaled to desired number of nodes: %d", storage.Spec.Nodes), }) return r.updateStatus(ctx, storage, StatusUpdateRequeueDelay) } @@ -309,10 +312,11 @@ func (r *Reconciler) handleResourcesSync( if storage.Status.State == StoragePending { meta.SetStatusCondition(&storage.Status.Conditions, metav1.Condition{ - Type: StoragePreparedCondition, - Status: metav1.ConditionUnknown, - Reason: ReasonInProgress, - Message: fmt.Sprintf("Waiting for sync resources for generation %d", storage.Generation), + Type: StoragePreparedCondition, + Status: metav1.ConditionUnknown, + Reason: ReasonInProgress, + ObservedGeneration: storage.Generation, + Message: "Waiting for sync resources", }) storage.Status.State = StoragePreparing return r.updateStatus(ctx, storage, StatusUpdateRequeueDelay) @@ -374,10 +378,11 @@ func (r *Reconciler) handleResourcesSync( eventMessage+fmt.Sprintf(", failed to sync, error: %s", err), ) meta.SetStatusCondition(&storage.Status.Conditions, metav1.Condition{ - Type: StoragePreparedCondition, - Status: metav1.ConditionFalse, - Reason: ReasonInProgress, - Message: fmt.Sprintf("Failed to sync resources for generation %d", storage.Generation), + Type: StoragePreparedCondition, + Status: metav1.ConditionFalse, + Reason: ReasonInProgress, + ObservedGeneration: storage.Generation, + Message: "Failed to sync resources", }) return r.updateStatus(ctx, storage, DefaultRequeueDelay) } else if result == controllerutil.OperationResultCreated || result == controllerutil.OperationResultUpdated { @@ -390,12 +395,24 @@ func (r *Reconciler) handleResourcesSync( } } + if err := r.syncNodeSetSpecInline(ctx, storage); err != nil { + meta.SetStatusCondition(&storage.Status.Conditions, metav1.Condition{ + Type: StoragePreparedCondition, + Status: metav1.ConditionFalse, + Reason: ReasonInProgress, + ObservedGeneration: storage.Generation, + Message: "Failed to sync NodeSet resources", + }) + return r.updateStatus(ctx, storage, DefaultRequeueDelay) + } + if !meta.IsStatusConditionTrue(storage.Status.Conditions, StoragePreparedCondition) { meta.SetStatusCondition(&storage.Status.Conditions, metav1.Condition{ - Type: StoragePreparedCondition, - Status: metav1.ConditionTrue, - Reason: ReasonCompleted, - Message: "Successfully synced resources", + Type: StoragePreparedCondition, + Status: metav1.ConditionTrue, + Reason: ReasonCompleted, + ObservedGeneration: storage.Generation, + Message: "Successfully synced resources", }) return r.updateStatus(ctx, storage, StatusUpdateRequeueDelay) } @@ -407,8 +424,7 @@ func (r *Reconciler) handleResourcesSync( func (r *Reconciler) syncNodeSetSpecInline( ctx context.Context, storage *resources.StorageClusterBuilder, -) (bool, ctrl.Result, error) { - r.Log.Info("running step syncNodeSetSpecInline") +) error { matchingFields := client.MatchingFields{ OwnerControllerField: storage.Name, } @@ -424,7 +440,7 @@ func (r *Reconciler) syncNodeSetSpecInline( "ProvisioningFailed", fmt.Sprintf("Failed to list StorageNodeSets: %s", err), ) - return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err + return err } for _, storageNodeSet := range storageNodeSets.Items { @@ -448,7 +464,7 @@ func (r *Reconciler) syncNodeSetSpecInline( "ProvisioningFailed", fmt.Sprintf("Failed to delete StorageNodeSet: %s", err), ) - return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err + return err } r.Recorder.Event( storage, @@ -473,7 +489,7 @@ func (r *Reconciler) syncNodeSetSpecInline( "ProvisioningFailed", fmt.Sprintf("Failed to list RemoteStorageNodeSets: %s", err), ) - return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err + return err } for _, remoteStorageNodeSet := range remoteStorageNodeSets.Items { @@ -497,7 +513,7 @@ func (r *Reconciler) syncNodeSetSpecInline( "ProvisioningFailed", fmt.Sprintf("Failed to delete RemoteStorageNodeSet: %s", err), ) - return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err + return err } r.Recorder.Event( storage, @@ -511,18 +527,7 @@ func (r *Reconciler) syncNodeSetSpecInline( } } - if !meta.IsStatusConditionTrue(storage.Status.Conditions, StoragePreparedCondition) { - meta.SetStatusCondition(&storage.Status.Conditions, metav1.Condition{ - Type: StoragePreparedCondition, - Status: metav1.ConditionTrue, - Reason: ReasonCompleted, - Message: "Successfully synced resources", - }) - return r.updateStatus(ctx, storage, StatusUpdateRequeueDelay) - } - - r.Log.Info("complete step syncNodeSetSpecInline") - return Continue, ctrl.Result{Requeue: false}, nil + return nil } func (r *Reconciler) runSelfCheck( @@ -595,15 +600,17 @@ func (r *Reconciler) updateStatus( meta.IsStatusConditionFalse(storage.Status.Conditions, StorageProvisionedCondition) { if storage.Spec.Pause { meta.SetStatusCondition(&storage.Status.Conditions, metav1.Condition{ - Type: StoragePausedCondition, - Status: metav1.ConditionFalse, - Reason: ReasonInProgress, + Type: StoragePausedCondition, + Status: metav1.ConditionFalse, + Reason: ReasonInProgress, + ObservedGeneration: storage.Generation, }) } else { meta.SetStatusCondition(&storage.Status.Conditions, metav1.Condition{ - Type: StorageReadyCondition, - Status: metav1.ConditionFalse, - Reason: ReasonInProgress, + Type: StorageReadyCondition, + Status: metav1.ConditionFalse, + Reason: ReasonInProgress, + ObservedGeneration: storage.Generation, }) } } @@ -657,16 +664,18 @@ func (r *Reconciler) handlePauseResume( if storage.Status.State == StorageProvisioning { if storage.Spec.Pause { meta.SetStatusCondition(&storage.Status.Conditions, metav1.Condition{ - Type: StoragePausedCondition, - Status: metav1.ConditionTrue, - Reason: ReasonCompleted, + Type: StoragePausedCondition, + Status: metav1.ConditionTrue, + Reason: ReasonCompleted, + ObservedGeneration: storage.Generation, }) storage.Status.State = StoragePaused } else { meta.SetStatusCondition(&storage.Status.Conditions, metav1.Condition{ - Type: StorageReadyCondition, - Status: metav1.ConditionTrue, - Reason: ReasonCompleted, + Type: StorageReadyCondition, + Status: metav1.ConditionTrue, + Reason: ReasonCompleted, + ObservedGeneration: storage.Generation, }) storage.Status.State = StorageReady } @@ -676,10 +685,11 @@ func (r *Reconciler) handlePauseResume( if storage.Status.State == StorageReady && storage.Spec.Pause { r.Log.Info("`pause: true` was noticed, moving Storage to state `Paused`") meta.SetStatusCondition(&storage.Status.Conditions, metav1.Condition{ - Type: StorageReadyCondition, - Status: metav1.ConditionFalse, - Reason: ReasonNotRequired, - Message: "Transitioning to state Paused", + Type: StorageReadyCondition, + Status: metav1.ConditionFalse, + Reason: ReasonNotRequired, + ObservedGeneration: storage.Generation, + Message: "Transitioning to state Paused", }) storage.Status.State = StoragePaused return r.updateStatus(ctx, storage, StatusUpdateRequeueDelay) @@ -688,10 +698,11 @@ func (r *Reconciler) handlePauseResume( if storage.Status.State == StoragePaused && !storage.Spec.Pause { r.Log.Info("`pause: false` was noticed, moving Storage to state `Ready`") meta.SetStatusCondition(&storage.Status.Conditions, metav1.Condition{ - Type: StoragePausedCondition, - Status: metav1.ConditionFalse, - Reason: ReasonNotRequired, - Message: "Transitioning to state Ready", + Type: StoragePausedCondition, + Status: metav1.ConditionFalse, + Reason: ReasonNotRequired, + ObservedGeneration: storage.Generation, + Message: "Transitioning to state Ready", }) storage.Status.State = StorageReady return r.updateStatus(ctx, storage, StatusUpdateRequeueDelay) @@ -700,18 +711,20 @@ func (r *Reconciler) handlePauseResume( if storage.Spec.Pause { if !meta.IsStatusConditionTrue(storage.Status.Conditions, StoragePausedCondition) { meta.SetStatusCondition(&storage.Status.Conditions, metav1.Condition{ - Type: StoragePausedCondition, - Status: metav1.ConditionTrue, - Reason: ReasonCompleted, + Type: StoragePausedCondition, + Status: metav1.ConditionTrue, + Reason: ReasonCompleted, + ObservedGeneration: storage.Generation, }) return r.updateStatus(ctx, storage, StatusUpdateRequeueDelay) } } else { if !meta.IsStatusConditionTrue(storage.Status.Conditions, StorageReadyCondition) { meta.SetStatusCondition(&storage.Status.Conditions, metav1.Condition{ - Type: StorageReadyCondition, - Status: metav1.ConditionTrue, - Reason: ReasonCompleted, + Type: StorageReadyCondition, + Status: metav1.ConditionTrue, + Reason: ReasonCompleted, + ObservedGeneration: storage.Generation, }) return r.updateStatus(ctx, storage, StatusUpdateRequeueDelay) } diff --git a/internal/controllers/storagenodeset/sync.go b/internal/controllers/storagenodeset/sync.go index fc0b9ed4..12c7781a 100644 --- a/internal/controllers/storagenodeset/sync.go +++ b/internal/controllers/storagenodeset/sync.go @@ -62,17 +62,19 @@ func (r *Reconciler) setInitialStatus( if storageNodeSet.Spec.Pause { meta.SetStatusCondition(&storageNodeSet.Status.Conditions, metav1.Condition{ - Type: NodeSetPausedCondition, - Status: metav1.ConditionUnknown, - Reason: ReasonInProgress, - Message: "Transitioning to state Paused", + Type: NodeSetPausedCondition, + Status: metav1.ConditionUnknown, + Reason: ReasonInProgress, + ObservedGeneration: storageNodeSet.Generation, + Message: "Transitioning to state Paused", }) } else { meta.SetStatusCondition(&storageNodeSet.Status.Conditions, metav1.Condition{ - Type: NodeSetReadyCondition, - Status: metav1.ConditionUnknown, - Reason: ReasonInProgress, - Message: "Transitioning to state Ready", + Type: NodeSetReadyCondition, + Status: metav1.ConditionUnknown, + Reason: ReasonInProgress, + ObservedGeneration: storageNodeSet.Generation, + Message: "Transitioning to state Ready", }) } @@ -102,10 +104,11 @@ func (r *Reconciler) handleResourcesSync( if storageNodeSet.Status.State == StorageNodeSetPending { meta.SetStatusCondition(&storageNodeSet.Status.Conditions, metav1.Condition{ - Type: NodeSetPreparedCondition, - Status: metav1.ConditionUnknown, - Reason: ReasonInProgress, - Message: fmt.Sprintf("Waiting for sync resources for generation %d", storageNodeSet.Generation), + Type: NodeSetPreparedCondition, + Status: metav1.ConditionUnknown, + Reason: ReasonInProgress, + ObservedGeneration: storageNodeSet.Generation, + Message: "Waiting for sync resources", }) storageNodeSet.Status.State = StorageNodeSetPreparing return r.updateStatus(ctx, storageNodeSet, StatusUpdateRequeueDelay) @@ -155,10 +158,11 @@ func (r *Reconciler) handleResourcesSync( eventMessage+fmt.Sprintf(", failed to sync, error: %s", err), ) meta.SetStatusCondition(&storageNodeSet.Status.Conditions, metav1.Condition{ - Type: NodeSetPreparedCondition, - Status: metav1.ConditionFalse, - Reason: ReasonInProgress, - Message: fmt.Sprintf("Failed to sync resources for generation %d", storageNodeSet.Generation), + Type: NodeSetPreparedCondition, + Status: metav1.ConditionFalse, + Reason: ReasonInProgress, + ObservedGeneration: storageNodeSet.Generation, + Message: "Failed to sync resources", }) return r.updateStatus(ctx, storageNodeSet, DefaultRequeueDelay) } else if result == controllerutil.OperationResultCreated || result == controllerutil.OperationResultUpdated { @@ -173,10 +177,11 @@ func (r *Reconciler) handleResourcesSync( if !meta.IsStatusConditionTrue(storageNodeSet.Status.Conditions, NodeSetPreparedCondition) { meta.SetStatusCondition(&storageNodeSet.Status.Conditions, metav1.Condition{ - Type: NodeSetPreparedCondition, - Status: metav1.ConditionTrue, - Reason: ReasonCompleted, - Message: "Successfully synced resources", + Type: NodeSetPreparedCondition, + Status: metav1.ConditionTrue, + Reason: ReasonCompleted, + ObservedGeneration: storageNodeSet.Generation, + Message: "Successfully synced resources", }) return r.updateStatus(ctx, storageNodeSet, StatusUpdateRequeueDelay) } @@ -193,10 +198,11 @@ func (r *Reconciler) waitForStatefulSetToScale( if storageNodeSet.Status.State == StorageNodeSetPreparing { meta.SetStatusCondition(&storageNodeSet.Status.Conditions, metav1.Condition{ - Type: NodeSetProvisionedCondition, - Status: metav1.ConditionUnknown, - Reason: ReasonInProgress, - Message: fmt.Sprintf("Waiting for scale to desired nodes: %d", storageNodeSet.Spec.Nodes), + Type: NodeSetProvisionedCondition, + Status: metav1.ConditionUnknown, + Reason: ReasonInProgress, + ObservedGeneration: storageNodeSet.Generation, + Message: fmt.Sprintf("Waiting for scale to desired nodes: %d", storageNodeSet.Spec.Nodes), }) storageNodeSet.Status.State = StorageNodeSetProvisioning return r.updateStatus(ctx, storageNodeSet, StatusUpdateRequeueDelay) @@ -234,20 +240,22 @@ func (r *Reconciler) waitForStatefulSetToScale( fmt.Sprintf("Waiting for number of running nodes to match expected: %d != %d", foundStatefulSet.Status.ReadyReplicas, storageNodeSet.Spec.Nodes), ) meta.SetStatusCondition(&storageNodeSet.Status.Conditions, metav1.Condition{ - Type: NodeSetProvisionedCondition, - Status: metav1.ConditionFalse, - Reason: ReasonInProgress, - Message: fmt.Sprintf("Number of running nodes does not match expected: %d != %d", foundStatefulSet.Status.ReadyReplicas, storageNodeSet.Spec.Nodes), + Type: NodeSetProvisionedCondition, + Status: metav1.ConditionFalse, + Reason: ReasonInProgress, + ObservedGeneration: storageNodeSet.Generation, + Message: fmt.Sprintf("Number of running nodes does not match expected: %d != %d", foundStatefulSet.Status.ReadyReplicas, storageNodeSet.Spec.Nodes), }) return r.updateStatus(ctx, storageNodeSet, DefaultRequeueDelay) } if !meta.IsStatusConditionTrue(storageNodeSet.Status.Conditions, NodeSetProvisionedCondition) { meta.SetStatusCondition(&storageNodeSet.Status.Conditions, metav1.Condition{ - Type: NodeSetProvisionedCondition, - Status: metav1.ConditionTrue, - Reason: ReasonCompleted, - Message: fmt.Sprintf("Successfully scaled to desired number of nodes: %d", storageNodeSet.Spec.Nodes), + Type: NodeSetProvisionedCondition, + Status: metav1.ConditionTrue, + Reason: ReasonCompleted, + ObservedGeneration: storageNodeSet.Generation, + Message: fmt.Sprintf("Successfully scaled to desired number of nodes: %d", storageNodeSet.Spec.Nodes), }) return r.updateStatus(ctx, storageNodeSet, StatusUpdateRequeueDelay) } @@ -267,15 +275,17 @@ func (r *Reconciler) updateStatus( meta.IsStatusConditionFalse(storageNodeSet.Status.Conditions, NodeSetProvisionedCondition) { if storageNodeSet.Spec.Pause { meta.SetStatusCondition(&storageNodeSet.Status.Conditions, metav1.Condition{ - Type: NodeSetPausedCondition, - Status: metav1.ConditionFalse, - Reason: ReasonInProgress, + Type: NodeSetPausedCondition, + Status: metav1.ConditionFalse, + Reason: ReasonInProgress, + ObservedGeneration: storageNodeSet.Generation, }) } else { meta.SetStatusCondition(&storageNodeSet.Status.Conditions, metav1.Condition{ - Type: NodeSetReadyCondition, - Status: metav1.ConditionFalse, - Reason: ReasonInProgress, + Type: NodeSetReadyCondition, + Status: metav1.ConditionFalse, + Reason: ReasonInProgress, + ObservedGeneration: storageNodeSet.Generation, }) } } @@ -322,8 +332,8 @@ func (r *Reconciler) updateStatus( func shouldIgnoreStorageNodeSetChange(storageNodeSet *resources.StorageNodeSetResource) resources.IgnoreChangesFunction { return func(oldObj, newObj runtime.Object) bool { - if _, ok := newObj.(*appsv1.StatefulSet); ok { - if storageNodeSet.Spec.Pause && *oldObj.(*appsv1.StatefulSet).Spec.Replicas == 0 { + if statefulSet, ok := oldObj.(*appsv1.StatefulSet); ok { + if storageNodeSet.Spec.Pause && *statefulSet.Spec.Replicas == 0 { return true } } @@ -340,16 +350,18 @@ func (r *Reconciler) handlePauseResume( if storageNodeSet.Status.State == StorageNodeSetProvisioning { if storageNodeSet.Spec.Pause { meta.SetStatusCondition(&storageNodeSet.Status.Conditions, metav1.Condition{ - Type: NodeSetPausedCondition, - Status: metav1.ConditionTrue, - Reason: ReasonCompleted, + Type: NodeSetPausedCondition, + Status: metav1.ConditionTrue, + Reason: ReasonCompleted, + ObservedGeneration: storageNodeSet.Generation, }) storageNodeSet.Status.State = StorageNodeSetPaused } else { meta.SetStatusCondition(&storageNodeSet.Status.Conditions, metav1.Condition{ - Type: NodeSetReadyCondition, - Status: metav1.ConditionTrue, - Reason: ReasonCompleted, + Type: NodeSetReadyCondition, + Status: metav1.ConditionTrue, + Reason: ReasonCompleted, + ObservedGeneration: storageNodeSet.Generation, }) storageNodeSet.Status.State = StorageNodeSetReady } @@ -359,10 +371,11 @@ func (r *Reconciler) handlePauseResume( if storageNodeSet.Status.State == StorageNodeSetReady && storageNodeSet.Spec.Pause { r.Log.Info("`pause: true` was noticed, moving StorageNodeSet to state `Paused`") meta.SetStatusCondition(&storageNodeSet.Status.Conditions, metav1.Condition{ - Type: NodeSetReadyCondition, - Status: metav1.ConditionFalse, - Reason: ReasonNotRequired, - Message: "Transitioning to state Paused", + Type: NodeSetReadyCondition, + Status: metav1.ConditionFalse, + Reason: ReasonNotRequired, + ObservedGeneration: storageNodeSet.Generation, + Message: "Transitioning to state Paused", }) storageNodeSet.Status.State = StorageNodeSetPaused return r.updateStatus(ctx, storageNodeSet, StatusUpdateRequeueDelay) @@ -371,10 +384,11 @@ func (r *Reconciler) handlePauseResume( if storageNodeSet.Status.State == StorageNodeSetPaused && !storageNodeSet.Spec.Pause { r.Log.Info("`pause: false` was noticed, moving StorageNodeSet to state `Ready`") meta.SetStatusCondition(&storageNodeSet.Status.Conditions, metav1.Condition{ - Type: NodeSetPausedCondition, - Status: metav1.ConditionFalse, - Reason: ReasonNotRequired, - Message: "Transitioning to state Ready", + Type: NodeSetPausedCondition, + Status: metav1.ConditionFalse, + Reason: ReasonNotRequired, + ObservedGeneration: storageNodeSet.Generation, + Message: "Transitioning to state Ready", }) storageNodeSet.Status.State = StorageNodeSetReady return r.updateStatus(ctx, storageNodeSet, StatusUpdateRequeueDelay) @@ -383,18 +397,20 @@ func (r *Reconciler) handlePauseResume( if storageNodeSet.Spec.Pause { if !meta.IsStatusConditionTrue(storageNodeSet.Status.Conditions, NodeSetPausedCondition) { meta.SetStatusCondition(&storageNodeSet.Status.Conditions, metav1.Condition{ - Type: NodeSetPausedCondition, - Status: metav1.ConditionTrue, - Reason: ReasonCompleted, + Type: NodeSetPausedCondition, + Status: metav1.ConditionTrue, + Reason: ReasonCompleted, + ObservedGeneration: storageNodeSet.Generation, }) return r.updateStatus(ctx, storageNodeSet, StatusUpdateRequeueDelay) } } else { if !meta.IsStatusConditionTrue(storageNodeSet.Status.Conditions, NodeSetReadyCondition) { meta.SetStatusCondition(&storageNodeSet.Status.Conditions, metav1.Condition{ - Type: NodeSetReadyCondition, - Status: metav1.ConditionTrue, - Reason: ReasonCompleted, + Type: NodeSetReadyCondition, + Status: metav1.ConditionTrue, + Reason: ReasonCompleted, + ObservedGeneration: storageNodeSet.Generation, }) return r.updateStatus(ctx, storageNodeSet, StatusUpdateRequeueDelay) } diff --git a/internal/resources/remotedatabasenodeset.go b/internal/resources/remotedatabasenodeset.go index 93fcf5c1..e4d776a8 100644 --- a/internal/resources/remotedatabasenodeset.go +++ b/internal/resources/remotedatabasenodeset.go @@ -204,7 +204,7 @@ func (b *RemoteDatabaseNodeSetResource) CreateRemoteResourceStatus( &b.Status.RemoteResources[len(b.Status.RemoteResources)-1].Conditions, metav1.Condition{ Type: RemoteResourceSyncedCondition, - Status: "Unknown", + Status: metav1.ConditionUnknown, Reason: ReasonInProgress, }, ) diff --git a/internal/resources/remotestoragenodeset.go b/internal/resources/remotestoragenodeset.go index c8dbf8d9..ad34c985 100644 --- a/internal/resources/remotestoragenodeset.go +++ b/internal/resources/remotestoragenodeset.go @@ -183,7 +183,7 @@ func (b *RemoteStorageNodeSetResource) CreateRemoteResourceStatus(remoteObj clie &b.Status.RemoteResources[len(b.Status.RemoteResources)-1].Conditions, metav1.Condition{ Type: RemoteResourceSyncedCondition, - Status: "Unknown", + Status: metav1.ConditionUnknown, Reason: ReasonInProgress, }, ) From 25b0d7ae4eb4475294ee0cb625059322527ad71d Mon Sep 17 00:00:00 2001 From: Aleksei Kobzev Date: Tue, 28 Jan 2025 21:59:38 +0800 Subject: [PATCH 06/21] New field `ExternalPort` for GRPC Service to override `--grpc-public-port` arg (#278) --- .../unreleased/Added-20250104-143452.yaml | 3 + .../unreleased/Fixed-20250104-143747.yaml | 3 + .gitignore | 2 + Makefile | 22 +++-- api/v1alpha1/const.go | 1 + api/v1alpha1/service_types.go | 3 +- deploy/ydb-operator/crds/database.yaml | 3 + deploy/ydb-operator/crds/databasenodeset.yaml | 3 + .../crds/remotedatabasenodeset.yaml | 3 + .../crds/remotestoragenodeset.yaml | 3 + deploy/ydb-operator/crds/storage.yaml | 3 + deploy/ydb-operator/crds/storagenodeset.yaml | 3 + .../controllers/database/controller_test.go | 82 ++++++++++++++++++ internal/resources/database.go | 5 +- internal/resources/database_statefulset.go | 29 +++++-- internal/resources/service.go | 6 ++ tests/cfg/kind-cluster-config.yaml | 5 ++ tests/cfg/operator-local-values.yaml | 5 ++ tests/e2e/smoke_test.go | 35 ++++++++ tests/test-utils/test-utils.go | 85 +++++++++++++++++-- 20 files changed, 280 insertions(+), 24 deletions(-) create mode 100644 .changes/unreleased/Added-20250104-143452.yaml create mode 100644 .changes/unreleased/Fixed-20250104-143747.yaml diff --git a/.changes/unreleased/Added-20250104-143452.yaml b/.changes/unreleased/Added-20250104-143452.yaml new file mode 100644 index 00000000..d70253c6 --- /dev/null +++ b/.changes/unreleased/Added-20250104-143452.yaml @@ -0,0 +1,3 @@ +kind: Added +body: field externalPort for grpc service to override --grpc-public-port arg +time: 2025-01-04T14:34:52.706824+08:00 diff --git a/.changes/unreleased/Fixed-20250104-143747.yaml b/.changes/unreleased/Fixed-20250104-143747.yaml new file mode 100644 index 00000000..4a4439c7 --- /dev/null +++ b/.changes/unreleased/Fixed-20250104-143747.yaml @@ -0,0 +1,3 @@ +kind: Fixed +body: update the Makefile with the changes in GitHub CI +time: 2025-01-04T14:37:47.689565+08:00 diff --git a/.gitignore b/.gitignore index e3f63895..8d25f855 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,8 @@ com_crashlytics_export_strings.xml crashlytics.properties crashlytics-build.properties fabric.properties +log.json +log.txt bin/ config/ diff --git a/Makefile b/Makefile index 967c8c19..86c88e5a 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,8 @@ VERSION ?= 0.1.0 IMG ?= cr.yandex/yc/ydb-operator:latest # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. ENVTEST_K8S_VERSION = 1.26 +# Image URL which uses in tests +YDB_IMAGE ?= $(shell grep "anchor_for_fetching_image_from_workflow" ./tests/**/*.go | grep -o -E '"cr\.yandex.*"' | tr -d '"') # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) ifeq (,$(shell go env GOBIN)) @@ -65,25 +67,29 @@ vet: ## Run go vet against code. kind-init: if kind get clusters | grep "kind-ydb-operator"; then exit 0; fi; \ - kind create cluster --config e2e/kind-cluster-config.yaml --name kind-ydb-operator; \ - docker pull k8s.gcr.io/ingress-nginx/kube-webhook-certgen:v1.0; \ - kind load docker-image k8s.gcr.io/ingress-nginx/kube-webhook-certgen:v1.0 --name kind-ydb-operator; \ - docker pull cr.yandex/crptqonuodf51kdj7a7d/ydb:24.2.7; \ - kind load docker-image cr.yandex/crptqonuodf51kdj7a7d/ydb:24.2.7 --name kind-ydb-operator + kind create cluster \ + --config tests/cfg/kind-cluster-config.yaml \ + --image=kindest/node:v1.31.2@sha256:18fbefc20a7113353c7b75b5c869d7145a6abd6269154825872dc59c1329912e \ + --name kind-ydb-operator + docker pull k8s.gcr.io/ingress-nginx/kube-webhook-certgen:v1.0 + kind load docker-image k8s.gcr.io/ingress-nginx/kube-webhook-certgen:v1.0 --name kind-ydb-operator + docker pull ${YDB_IMAGE} + kind load docker-image ${YDB_IMAGE} --name kind-ydb-operator kind-load: - docker tag cr.yandex/yc/ydb-operator:latest kind/ydb-operator:current + docker tag ${IMG} kind/ydb-operator:current kind load docker-image kind/ydb-operator:current --name kind-ydb-operator opts ?= '' .PHONY: unit-test unit-test: manifests generate fmt vet envtest ## Run unit tests - KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use --arch=amd64 $(ENVTEST_K8S_VERSION) -p path)" go test -v -timeout 900s -p 1 ./internal/... -ginkgo.v -coverprofile cover.out $(opts) + KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use --arch=amd64 $(ENVTEST_K8S_VERSION) -p path)" \ + go test -v -timeout 900s -p 1 ./internal/... -ginkgo.vv -coverprofile cover.out $(opts) .PHONY: e2e-test e2e-test: manifests generate fmt vet docker-build kind-init kind-load ## Run e2e tests - go test -v -timeout 3600s -p 1 ./e2e/... -ginkgo.v $(opts) + go test -v -timeout 3600s -p 1 ./tests/e2e/... -ginkgo.vv $(opts) .PHONY: test test: unit-test e2e-test ## Run all tests diff --git a/api/v1alpha1/const.go b/api/v1alpha1/const.go index a9fe280d..0fe9e0ec 100644 --- a/api/v1alpha1/const.go +++ b/api/v1alpha1/const.go @@ -64,6 +64,7 @@ const ( AnnotationDisableLivenessProbe = "ydb.tech/disable-liveness-probe" AnnotationDataCenter = "ydb.tech/data-center" AnnotationGRPCPublicHost = "ydb.tech/grpc-public-host" + AnnotationGRPCPublicPort = "ydb.tech/grpc-public-port" AnnotationNodeHost = "ydb.tech/node-host" AnnotationNodeDomain = "ydb.tech/node-domain" AnnotationAuthTokenSecretName = "ydb.tech/auth-token-secret-name" diff --git a/api/v1alpha1/service_types.go b/api/v1alpha1/service_types.go index 981cce60..8c540c74 100644 --- a/api/v1alpha1/service_types.go +++ b/api/v1alpha1/service_types.go @@ -21,7 +21,8 @@ type GRPCService struct { Service `json:""` TLSConfiguration *TLSConfiguration `json:"tls,omitempty"` - ExternalHost string `json:"externalHost,omitempty"` // TODO implementation + ExternalHost string `json:"externalHost,omitempty"` + ExternalPort int32 `json:"externalPort,omitempty"` IPDiscovery *IPDiscovery `json:"ipDiscovery,omitempty"` } diff --git a/deploy/ydb-operator/crds/database.yaml b/deploy/ydb-operator/crds/database.yaml index eae6f463..7493ceec 100644 --- a/deploy/ydb-operator/crds/database.yaml +++ b/deploy/ydb-operator/crds/database.yaml @@ -4075,6 +4075,9 @@ spec: type: object externalHost: type: string + externalPort: + format: int32 + type: integer ipDiscovery: properties: enabled: diff --git a/deploy/ydb-operator/crds/databasenodeset.yaml b/deploy/ydb-operator/crds/databasenodeset.yaml index 2417044a..d5164b28 100644 --- a/deploy/ydb-operator/crds/databasenodeset.yaml +++ b/deploy/ydb-operator/crds/databasenodeset.yaml @@ -2687,6 +2687,9 @@ spec: type: object externalHost: type: string + externalPort: + format: int32 + type: integer ipDiscovery: properties: enabled: diff --git a/deploy/ydb-operator/crds/remotedatabasenodeset.yaml b/deploy/ydb-operator/crds/remotedatabasenodeset.yaml index f24ffa2e..ac65fe56 100644 --- a/deploy/ydb-operator/crds/remotedatabasenodeset.yaml +++ b/deploy/ydb-operator/crds/remotedatabasenodeset.yaml @@ -2688,6 +2688,9 @@ spec: type: object externalHost: type: string + externalPort: + format: int32 + type: integer ipDiscovery: properties: enabled: diff --git a/deploy/ydb-operator/crds/remotestoragenodeset.yaml b/deploy/ydb-operator/crds/remotestoragenodeset.yaml index 651c504d..ce30bb27 100644 --- a/deploy/ydb-operator/crds/remotestoragenodeset.yaml +++ b/deploy/ydb-operator/crds/remotestoragenodeset.yaml @@ -2715,6 +2715,9 @@ spec: type: object externalHost: type: string + externalPort: + format: int32 + type: integer ipDiscovery: properties: enabled: diff --git a/deploy/ydb-operator/crds/storage.yaml b/deploy/ydb-operator/crds/storage.yaml index d989e383..1fed259e 100644 --- a/deploy/ydb-operator/crds/storage.yaml +++ b/deploy/ydb-operator/crds/storage.yaml @@ -5216,6 +5216,9 @@ spec: type: object externalHost: type: string + externalPort: + format: int32 + type: integer ipDiscovery: properties: enabled: diff --git a/deploy/ydb-operator/crds/storagenodeset.yaml b/deploy/ydb-operator/crds/storagenodeset.yaml index 9ce14e79..cf3ea5ee 100644 --- a/deploy/ydb-operator/crds/storagenodeset.yaml +++ b/deploy/ydb-operator/crds/storagenodeset.yaml @@ -2714,6 +2714,9 @@ spec: type: object externalHost: type: string + externalPort: + format: int32 + type: integer ipDiscovery: properties: enabled: diff --git a/internal/controllers/database/controller_test.go b/internal/controllers/database/controller_test.go index ee099129..467726a5 100644 --- a/internal/controllers/database/controller_test.go +++ b/internal/controllers/database/controller_test.go @@ -24,6 +24,7 @@ import ( . "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/constants" "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/database" "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/storage" + "github.com/ydb-platform/ydb-kubernetes-operator/internal/resources" "github.com/ydb-platform/ydb-kubernetes-operator/internal/test" testobjects "github.com/ydb-platform/ydb-kubernetes-operator/tests/test-k8s-objects" ) @@ -306,4 +307,85 @@ var _ = Describe("Database controller medium tests", func() { Expect(args).To(ContainElements([]string{"--grpc-public-address-v4", "--grpc-public-target-name-override"})) }) + + It("Check externalPort GRPC Service field propagation", func() { + By("Create test database") + databaseSample = *testobjects.DefaultDatabase() + Expect(k8sClient.Create(ctx, &databaseSample)).Should(Succeed()) + + checkPublicPortArg := func(expectedGRPCPort string) error { + foundStatefulSet := appsv1.StatefulSet{} + Eventually(func() error { + return k8sClient.Get(ctx, + types.NamespacedName{ + Name: testobjects.DatabaseName, + Namespace: testobjects.YdbNamespace, + }, + &foundStatefulSet, + ) + }, test.Timeout, test.Interval).Should(Succeed()) + podContainerArgs := foundStatefulSet.Spec.Template.Spec.Containers[0].Args + for idx, argKey := range podContainerArgs { + if argKey == "--grpc-public-port" { + if podContainerArgs[idx+1] != expectedGRPCPort { + return fmt.Errorf( + "Found arg `--grpc-public-port` value %s does not match with expected: %s", + podContainerArgs[idx+1], + expectedGRPCPort, + ) + } + } + } + return nil + } + + By("Check that args `--grpc-public-host` and `--grpc-public-port` propagated to StatefulSet pods...") + Eventually( + checkPublicPortArg(fmt.Sprintf("%d", v1alpha1.GRPCPort)), + test.Timeout, + test.Interval).ShouldNot(HaveOccurred()) + + externalPort := int32(30001) + By("Update externalHost and externalPort for Database GRPC Service...", func() { + database := v1alpha1.Database{} + Expect(k8sClient.Get(ctx, types.NamespacedName{ + Name: testobjects.DatabaseName, + Namespace: testobjects.YdbNamespace, + }, &database)) + database.Spec.Service.GRPC.ExternalPort = externalPort + Expect(k8sClient.Update(ctx, &database)).Should(Succeed()) + }) + + By("Check that type was updated for Database GRPC Service to NodePort...") + Eventually(func() error { + databaseGRPCService := corev1.Service{} + Expect(k8sClient.Get(ctx, types.NamespacedName{ + Name: fmt.Sprintf(resources.GRPCServiceNameFormat, databaseSample.Name), + Namespace: testobjects.YdbNamespace, + }, &databaseGRPCService)) + if databaseGRPCService.Spec.Type != corev1.ServiceTypeNodePort { + return fmt.Errorf( + "Found GRPC Service .spec.type %s does not match with expected: %s", + databaseGRPCService.Spec.Type, + corev1.ServiceTypeNodePort, + ) + } + for _, port := range databaseGRPCService.Spec.Ports { + if port.NodePort != externalPort { + return fmt.Errorf( + "Found GRPC Service NodePort value %d does not match with expected: %s", + port.NodePort, + fmt.Sprintf("%d", externalPort), + ) + } + } + return nil + }, test.Timeout, test.Interval).ShouldNot(HaveOccurred()) + + By("Check that args `--grpc-public-port` was updated in StatefulSet...") + Eventually( + checkPublicPortArg(fmt.Sprintf("%d", externalPort)), + test.Timeout, + test.Interval).ShouldNot(HaveOccurred()) + }) }) diff --git a/internal/resources/database.go b/internal/resources/database.go index c2543680..e2935036 100644 --- a/internal/resources/database.go +++ b/internal/resources/database.go @@ -175,8 +175,9 @@ func (b *DatabaseBuilder) GetResourceBuilders(restConfig *rest.Config) []Resourc SelectorLabels: databaseSelectorLabels, Annotations: b.Spec.Service.GRPC.AdditionalAnnotations, Ports: []corev1.ServicePort{{ - Name: api.GRPCServicePortName, - Port: api.GRPCPort, + Name: api.GRPCServicePortName, + Port: api.GRPCPort, + NodePort: b.Spec.Service.GRPC.ExternalPort, }}, IPFamilies: b.Spec.Service.GRPC.IPFamilies, IPFamilyPolicy: b.Spec.Service.GRPC.IPFamilyPolicy, diff --git a/internal/resources/database_statefulset.go b/internal/resources/database_statefulset.go index 06b5cbc5..df84df23 100644 --- a/internal/resources/database_statefulset.go +++ b/internal/resources/database_statefulset.go @@ -4,7 +4,6 @@ import ( "errors" "fmt" "regexp" - "strconv" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -74,6 +73,15 @@ func (b *DatabaseStatefulSetBuilder) buildEnv() []corev1.EnvVar { var envVars []corev1.EnvVar envVars = append(envVars, + corev1.EnvVar{ + Name: "POD_NAME", // for `--grpc-public-host` flag + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "metadata.name", + }, + }, + }, corev1.EnvVar{ Name: "NODE_NAME", // for `--grpc-public-host` flag ValueFrom: &corev1.EnvVarSource{ @@ -640,10 +648,11 @@ func (b *DatabaseStatefulSetBuilder) buildContainerArgs() ([]string, []string) { } publicHostOption := "--grpc-public-host" - publicHost := fmt.Sprintf(api.InterconnectServiceFQDNFormat, b.Database.Name, b.GetNamespace(), domain) // FIXME .svc.cluster.local + publicHostDomain := fmt.Sprintf(api.InterconnectServiceFQDNFormat, b.Database.Name, b.GetNamespace(), domain) + publicHost := fmt.Sprintf("%s.%s", "$(POD_NAME)", publicHostDomain) if b.Spec.Service.GRPC.ExternalHost != "" { - publicHost = b.Spec.Service.GRPC.ExternalHost + publicHost = fmt.Sprintf("%s.%s", "$(POD_NAME)", b.Spec.Service.GRPC.ExternalHost) } if value, ok := b.ObjectMeta.Annotations[api.AnnotationGRPCPublicHost]; ok { publicHost = value @@ -667,22 +676,28 @@ func (b *DatabaseStatefulSetBuilder) buildContainerArgs() ([]string, []string) { args = append( args, "--grpc-public-target-name-override", - fmt.Sprintf("%s.%s", "$(NODE_NAME)", targetNameOverride), + fmt.Sprintf("%s.%s", "$(POD_NAME)", targetNameOverride), ) } } publicPortOption := "--grpc-public-port" - publicPort := api.GRPCPort + publicPort := fmt.Sprintf("%d", api.GRPCPort) + if b.Spec.Service.GRPC.ExternalPort > 0 { + publicPort = fmt.Sprintf("%d", b.Spec.Service.GRPC.ExternalPort) + } + if value, ok := b.ObjectMeta.Annotations[api.AnnotationGRPCPublicPort]; ok { + publicPort = value + } args = append( args, publicHostOption, - fmt.Sprintf("%s.%s", "$(NODE_NAME)", publicHost), // fixme $(NODE_NAME) + publicHost, publicPortOption, - strconv.Itoa(publicPort), + publicPort, ) if value, ok := b.ObjectMeta.Annotations[api.AnnotationDataCenter]; ok { diff --git a/internal/resources/service.go b/internal/resources/service.go index 07a2e289..e32d1aaf 100644 --- a/internal/resources/service.go +++ b/internal/resources/service.go @@ -62,6 +62,12 @@ func (b *ServiceBuilder) Build(obj client.Object) error { service.Spec.ClusterIP = "None" } + for _, port := range service.Spec.Ports { + if port.NodePort > 0 { + service.Spec.Type = corev1.ServiceTypeNodePort + } + } + return nil } diff --git a/tests/cfg/kind-cluster-config.yaml b/tests/cfg/kind-cluster-config.yaml index 18829857..82348230 100644 --- a/tests/cfg/kind-cluster-config.yaml +++ b/tests/cfg/kind-cluster-config.yaml @@ -4,6 +4,11 @@ nodes: - role: control-plane - role: worker + extraPortMappings: + - containerPort: 30001 + hostPort: 30001 + listenAddress: "127.0.0.1" + protocol: tcp labels: topology.kubernetes.io/zone: az-1 worker: true diff --git a/tests/cfg/operator-local-values.yaml b/tests/cfg/operator-local-values.yaml index edc33299..f81cad01 100644 --- a/tests/cfg/operator-local-values.yaml +++ b/tests/cfg/operator-local-values.yaml @@ -1,3 +1,8 @@ +image: + pullPolicy: IfNotPresent + repository: kind/ydb-operator + tag: current + webhook: enabled: true diff --git a/tests/e2e/smoke_test.go b/tests/e2e/smoke_test.go index e473efd8..3562e913 100644 --- a/tests/e2e/smoke_test.go +++ b/tests/e2e/smoke_test.go @@ -733,6 +733,41 @@ var _ = Describe("Operator smoke test", func() { ExecuteSimpleTableE2ETest(podName, testobjects.YdbNamespace, storageEndpoint, databasePath) }) + It("Check externalPort for Database", func() { + By("create storage...") + Expect(k8sClient.Create(ctx, storageSample)).Should(Succeed()) + defer DeleteStorageSafely(ctx, k8sClient, storageSample) + By("create database...") + databaseSample.Spec.Nodes = 1 + databaseSample.Spec.NodeSelector = map[string]string{ + "topology.kubernetes.io/zone": "az-1", + } + databaseSample.Annotations = map[string]string{ + v1alpha1.AnnotationGRPCPublicHost: "localhost", + } + databaseSample.Spec.Service.GRPC.ExternalPort = 30001 + Expect(k8sClient.Create(ctx, databaseSample)).Should(Succeed()) + defer func() { + Expect(k8sClient.Delete(ctx, databaseSample)).Should(Succeed()) + }() + + By("waiting until Storage is ready...") + WaitUntilStorageReady(ctx, k8sClient, storageSample.Name, testobjects.YdbNamespace) + + By("checking that all the storage pods are running and ready...") + CheckPodsRunningAndReady(ctx, k8sClient, "ydb-cluster", "kind-storage", storageSample.Spec.Nodes) + + By("waiting until database is ready...") + WaitUntilDatabaseReady(ctx, k8sClient, databaseSample.Name, testobjects.YdbNamespace) + + By("checking that all the database pods are running and ready...") + CheckPodsRunningAndReady(ctx, k8sClient, "ydb-cluster", "kind-database", databaseSample.Spec.Nodes) + + By("execute simple query with ydb-go-sdk...") + databasePath := DatabasePathWithDefaultDomain(databaseSample) + ExecuteSimpleTableE2ETestWithSDK(databaseSample.Name, testobjects.YdbNamespace, databasePath) + }) + AfterEach(func() { UninstallOperatorWithHelm(testobjects.YdbNamespace) Expect(k8sClient.Delete(ctx, &namespace)).Should(Succeed()) diff --git a/tests/test-utils/test-utils.go b/tests/test-utils/test-utils.go index 43cb86ed..66761a10 100644 --- a/tests/test-utils/test-utils.go +++ b/tests/test-utils/test-utils.go @@ -3,6 +3,7 @@ package testutils import ( "bufio" "context" + "database/sql" "fmt" "io" "os" @@ -23,6 +24,9 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + ydb "github.com/ydb-platform/ydb-go-sdk/v3" + "github.com/ydb-platform/ydb-go-sdk/v3/retry" + v1alpha1 "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" . "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/constants" testobjects "github.com/ydb-platform/ydb-kubernetes-operator/tests/test-k8s-objects" @@ -34,11 +38,17 @@ const ( Interval = time.Second * 2 YdbOperatorRemoteChart = "ydb/ydb-operator" YdbOperatorReleaseName = "ydb-operator" + TestTablePath = "testfolder/testtable" ) var ( pathToHelmValuesInLocalInstall = filepath.Join("..", "cfg", "operator-local-values.yaml") pathToHelmValuesInRemoteInstall = filepath.Join("..", "cfg", "operator-values.yaml") + + createTableQuery = fmt.Sprintf("CREATE TABLE `%s` (testColumnA Utf8, testColumnB Utf8, PRIMARY KEY (testColumnA));", TestTablePath) + insertQuery = fmt.Sprintf("INSERT INTO `%s` (testColumnA, testColumnB) VALUES ('valueA', 'valueB');", TestTablePath) + selectQuery = fmt.Sprintf("SELECT testColumnA, testColumnB FROM `%s`;", TestTablePath) + dropTableQuery = fmt.Sprintf("DROP TABLE `%s`;", TestTablePath) ) func InstallLocalOperatorWithHelm(namespace string) { @@ -203,8 +213,6 @@ func BringYdbCliToPod(podName, podNamespace string) { } func ExecuteSimpleTableE2ETest(podName, podNamespace, storageEndpoint string, databasePath string) { - tablePath := "testfolder/testtable" - tableCreatingInterval := time.Second * 10 Eventually(func(g Gomega) { @@ -217,7 +225,7 @@ func ExecuteSimpleTableE2ETest(podName, podNamespace, storageEndpoint string, da "-e", storageEndpoint, "yql", "-s", - fmt.Sprintf("CREATE TABLE `%s` (testColumnA Utf8, testColumnB Utf8, PRIMARY KEY (testColumnA));", tablePath), + createTableQuery, } output, _ := exec.Command("kubectl", args...).CombinedOutput() fmt.Println(string(output)) @@ -232,7 +240,7 @@ func ExecuteSimpleTableE2ETest(podName, podNamespace, storageEndpoint string, da "-e", storageEndpoint, "yql", "-s", - fmt.Sprintf("INSERT INTO `%s` (testColumnA, testColumnB) VALUES ('valueA', 'valueB');", tablePath), + insertQuery, } output, err := exec.Command("kubectl", argsInsert...).CombinedOutput() Expect(err).ShouldNot(HaveOccurred(), string(output)) @@ -247,7 +255,7 @@ func ExecuteSimpleTableE2ETest(podName, podNamespace, storageEndpoint string, da "yql", "--format", "csv", "-s", - fmt.Sprintf("SELECT * FROM `%s`;", tablePath), + selectQuery, } output, err = exec.Command("kubectl", argsSelect...).CombinedOutput() Expect(err).ShouldNot(HaveOccurred(), string(output)) @@ -262,12 +270,77 @@ func ExecuteSimpleTableE2ETest(podName, podNamespace, storageEndpoint string, da "-e", storageEndpoint, "yql", "-s", - fmt.Sprintf("DROP TABLE `%s`;", tablePath), + dropTableQuery, } output, err = exec.Command("kubectl", argsDrop...).CombinedOutput() Expect(err).ShouldNot(HaveOccurred(), string(output)) } +func ExecuteSimpleTableE2ETestWithSDK(databaseName, databaseNamespace, databasePath string) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + cc, err := ydb.Open( + ctx, + fmt.Sprintf("grpc://localhost:30001/%s", databasePath), + ) + Expect(err).ShouldNot(HaveOccurred()) + defer func() { _ = cc.Close(ctx) }() + + c, err := ydb.Connector(cc, + ydb.WithAutoDeclare(), + ydb.WithTablePathPrefix(fmt.Sprintf("%s/%s", databasePath, TestTablePath)), + ) + Expect(err).ShouldNot(HaveOccurred()) + defer func() { _ = c.Close() }() + + db := sql.OpenDB(c) + defer func() { _ = db.Close() }() + + err = retry.Do(ctx, db, func(ctx context.Context, cc *sql.Conn) error { + _, err = cc.ExecContext(ydb.WithQueryMode(ctx, ydb.SchemeQueryMode), createTableQuery) + if err != nil { + return err + } + return nil + }, retry.WithIdempotent(true)) + Expect(err).ShouldNot(HaveOccurred()) + + err = retry.DoTx(ctx, db, func(ctx context.Context, tx *sql.Tx) error { + if _, err = tx.ExecContext(ctx, insertQuery); err != nil { + return err + } + return nil + }, retry.WithIdempotent(true)) + Expect(err).ShouldNot(HaveOccurred()) + + var ( + testColumnA string + testColumnB string + ) + err = retry.Do(ctx, db, func(ctx context.Context, cc *sql.Conn) (err error) { + row := cc.QueryRowContext(ctx, selectQuery) + if err = row.Scan(&testColumnA, &testColumnB); err != nil { + return err + } + + return nil + }, retry.WithIdempotent(true)) + Expect(err).ShouldNot(HaveOccurred()) + Expect(testColumnA).To(BeEquivalentTo("valueA")) + Expect(testColumnB).To(BeEquivalentTo("valueB")) + + err = retry.Do(ctx, db, func(ctx context.Context, cc *sql.Conn) error { + _, err = cc.ExecContext(ydb.WithQueryMode(ctx, ydb.SchemeQueryMode), dropTableQuery) + if err != nil { + return err + } + + return nil + }, retry.WithIdempotent(true)) + Expect(err).ShouldNot(HaveOccurred()) +} + func PortForward( ctx context.Context, svcName, svcNamespace, serverName string, From 3a1af221e48ffb439ca3725fd0fabacfc15c4cb2 Mon Sep 17 00:00:00 2001 From: Egor Tarasov Date: Wed, 29 Jan 2025 13:34:03 +0100 Subject: [PATCH 07/21] Use correct operator version in e2e tests (#287) * fix: use correct operator version * fix: init job launching on the same nodes * fix: operator local values --- Makefile | 1 + tests/test-k8s-objects/objects.go | 3 +++ tests/test-utils/test-utils.go | 2 +- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 86c88e5a..a3266722 100644 --- a/Makefile +++ b/Makefile @@ -73,6 +73,7 @@ kind-init: --name kind-ydb-operator docker pull k8s.gcr.io/ingress-nginx/kube-webhook-certgen:v1.0 kind load docker-image k8s.gcr.io/ingress-nginx/kube-webhook-certgen:v1.0 --name kind-ydb-operator + kubectl cluster-info --context kind-kind-ydb-operator # yes, kind prefixes all context with one more 'kind-' docker pull ${YDB_IMAGE} kind load docker-image ${YDB_IMAGE} --name kind-ydb-operator diff --git a/tests/test-k8s-objects/objects.go b/tests/test-k8s-objects/objects.go index 447f6c49..b4c1394c 100644 --- a/tests/test-k8s-objects/objects.go +++ b/tests/test-k8s-objects/objects.go @@ -109,6 +109,9 @@ func DefaultStorage(storageYamlConfigPath string) *v1alpha1.Storage { AdditionalLabels: map[string]string{"ydb-cluster": "kind-storage"}, Affinity: storageAntiAffinity, }, + InitJob: &v1alpha1.StorageInitJobSpec{ + AdditionalLabels: map[string]string{"ydb-cluster": "kind-storage-init"}, + }, }, } } diff --git a/tests/test-utils/test-utils.go b/tests/test-utils/test-utils.go index 66761a10..d06d013b 100644 --- a/tests/test-utils/test-utils.go +++ b/tests/test-utils/test-utils.go @@ -108,7 +108,7 @@ func UpgradeOperatorWithHelm(namespace, version string) { "ydb-operator", YdbOperatorRemoteChart, "--version", version, - "-f", pathToHelmValuesInLocalInstall, + "-f", pathToHelmValuesInRemoteInstall, } cmd := exec.Command("helm", args...) From c3f7de5a7578c2603135e629b9d5df5ae54c5295 Mon Sep 17 00:00:00 2001 From: Nikita Kozlovskii Date: Wed, 29 Jan 2025 16:18:16 +0100 Subject: [PATCH 08/21] append secrets and volumes from storage spec to init-job volumes (#282) * apend secrets and volumes from storage spec to init-job volumes * wip * rename grpcTLSVolumeName to GRPCTLSVolumeName * remove one test * add changie * fix test --- .../unreleased/Fixed-20250129-134226.yaml | 3 ++ internal/resources/database_statefulset.go | 6 +-- internal/resources/resource.go | 2 +- internal/resources/storage_init_job.go | 25 ++++++++-- internal/resources/storage_statefulset.go | 6 +-- tests/e2e/smoke_test.go | 48 +++++++++++++++++++ 6 files changed, 78 insertions(+), 12 deletions(-) create mode 100644 .changes/unreleased/Fixed-20250129-134226.yaml diff --git a/.changes/unreleased/Fixed-20250129-134226.yaml b/.changes/unreleased/Fixed-20250129-134226.yaml new file mode 100644 index 00000000..2405bd52 --- /dev/null +++ b/.changes/unreleased/Fixed-20250129-134226.yaml @@ -0,0 +1,3 @@ +kind: Fixed +body: Passing additional secret volumes to blobstorage-init. The init container can now use them without issues. +time: 2025-01-29T13:42:26.145577+01:00 diff --git a/internal/resources/database_statefulset.go b/internal/resources/database_statefulset.go index df84df23..a27f977b 100644 --- a/internal/resources/database_statefulset.go +++ b/internal/resources/database_statefulset.go @@ -192,7 +192,7 @@ func (b *DatabaseStatefulSetBuilder) buildVolumes() []corev1.Volume { } if b.Spec.Service.GRPC.TLSConfiguration.Enabled { - volumes = append(volumes, buildTLSVolume(grpcTLSVolumeName, b.Spec.Service.GRPC.TLSConfiguration)) + volumes = append(volumes, buildTLSVolume(GRPCTLSVolumeName, b.Spec.Service.GRPC.TLSConfiguration)) } if b.Spec.Service.Interconnect.TLSConfiguration.Enabled { @@ -314,7 +314,7 @@ func (b *DatabaseStatefulSetBuilder) buildCaStorePatchingInitContainerVolumeMoun if b.Spec.Service.GRPC.TLSConfiguration.Enabled { volumeMounts = append(volumeMounts, corev1.VolumeMount{ - Name: grpcTLSVolumeName, + Name: GRPCTLSVolumeName, ReadOnly: true, MountPath: grpcTLSVolumeMountPath, }) @@ -482,7 +482,7 @@ func (b *DatabaseStatefulSetBuilder) buildVolumeMounts() []corev1.VolumeMount { if b.Spec.Service.GRPC.TLSConfiguration.Enabled { volumeMounts = append(volumeMounts, corev1.VolumeMount{ - Name: grpcTLSVolumeName, + Name: GRPCTLSVolumeName, ReadOnly: true, MountPath: grpcTLSVolumeMountPath, }) diff --git a/internal/resources/resource.go b/internal/resources/resource.go index c48965cf..81acea22 100644 --- a/internal/resources/resource.go +++ b/internal/resources/resource.go @@ -35,7 +35,7 @@ const ( StatusServiceNameFormat = "%s-status" DatastreamsServiceNameFormat = "%s-datastreams" - grpcTLSVolumeName = "grpc-tls-volume" + GRPCTLSVolumeName = "grpc-tls-volume" interconnectTLSVolumeName = "interconnect-tls-volume" datastreamsTLSVolumeName = "datastreams-tls-volume" statusTLSVolumeName = "status-tls-volume" diff --git a/internal/resources/storage_init_job.go b/internal/resources/storage_init_job.go index 871ab535..4308e29b 100644 --- a/internal/resources/storage_init_job.go +++ b/internal/resources/storage_init_job.go @@ -75,6 +75,7 @@ func (b *StorageInitJobBuilder) buildInitJobPodTemplateSpec() corev1.PodTemplate DNSConfig: &corev1.PodDNSConfig{ Searches: dnsConfigSearches, }, + InitContainers: b.Spec.InitContainers, }, } @@ -92,8 +93,7 @@ func (b *StorageInitJobBuilder) buildInitJobPodTemplateSpec() corev1.PodTemplate } } - // InitContainer only needed for CaBundle manipulation for now, - // may be probably used for other stuff later + // append an init container for updating the ca.crt if we have any certificates if b.AnyCertificatesAdded() { podTemplate.Spec.InitContainers = append( []corev1.Container{b.buildCaStorePatchingInitContainer()}, @@ -137,7 +137,7 @@ func (b *StorageInitJobBuilder) buildInitJobVolumes() []corev1.Volume { } if b.Spec.Service.GRPC.TLSConfiguration.Enabled { - volumes = append(volumes, buildTLSVolume(grpcTLSVolumeName, b.Spec.Service.GRPC.TLSConfiguration)) + volumes = append(volumes, buildTLSVolume(GRPCTLSVolumeName, b.Spec.Service.GRPC.TLSConfiguration)) } if b.Spec.OperatorConnection != nil { @@ -153,6 +153,21 @@ func (b *StorageInitJobBuilder) buildInitJobVolumes() []corev1.Volume { }) } + for _, secret := range b.Spec.Secrets { + volumes = append(volumes, corev1.Volume{ + Name: secret.Name, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: secret.Name, + }, + }, + }) + } + + for _, volume := range b.Spec.Volumes { + volumes = append(volumes, *volume) + } + if b.AnyCertificatesAdded() { volumes = append(volumes, corev1.Volume{ Name: systemCertsVolumeName, @@ -219,7 +234,7 @@ func (b *StorageInitJobBuilder) buildJobVolumeMounts() []corev1.VolumeMount { if b.Spec.Service.GRPC.TLSConfiguration.Enabled { volumeMounts = append(volumeMounts, corev1.VolumeMount{ - Name: grpcTLSVolumeName, + Name: GRPCTLSVolumeName, ReadOnly: true, MountPath: grpcTLSVolumeMountPath, }) @@ -302,7 +317,7 @@ func (b *StorageInitJobBuilder) buildCaStorePatchingInitContainerVolumeMounts() if b.Spec.Service.GRPC.TLSConfiguration.Enabled { volumeMounts = append(volumeMounts, corev1.VolumeMount{ - Name: grpcTLSVolumeName, + Name: GRPCTLSVolumeName, ReadOnly: true, MountPath: grpcTLSVolumeMountPath, }) diff --git a/internal/resources/storage_statefulset.go b/internal/resources/storage_statefulset.go index 9c74dcb5..5fb87671 100644 --- a/internal/resources/storage_statefulset.go +++ b/internal/resources/storage_statefulset.go @@ -216,7 +216,7 @@ func (b *StorageStatefulSetBuilder) buildVolumes() []corev1.Volume { } if b.Spec.Service.GRPC.TLSConfiguration.Enabled { - volumes = append(volumes, buildTLSVolume(grpcTLSVolumeName, b.Spec.Service.GRPC.TLSConfiguration)) + volumes = append(volumes, buildTLSVolume(GRPCTLSVolumeName, b.Spec.Service.GRPC.TLSConfiguration)) } if b.Spec.Service.Interconnect.TLSConfiguration.Enabled { @@ -326,7 +326,7 @@ func (b *StorageStatefulSetBuilder) buildCaStorePatchingInitContainerVolumeMount if b.Spec.Service.GRPC.TLSConfiguration.Enabled { volumeMounts = append(volumeMounts, corev1.VolumeMount{ - Name: grpcTLSVolumeName, + Name: GRPCTLSVolumeName, ReadOnly: true, MountPath: grpcTLSVolumeMountPath, }) @@ -438,7 +438,7 @@ func (b *StorageStatefulSetBuilder) buildVolumeMounts() []corev1.VolumeMount { if b.Spec.Service.GRPC.TLSConfiguration.Enabled { volumeMounts = append(volumeMounts, corev1.VolumeMount{ - Name: grpcTLSVolumeName, + Name: GRPCTLSVolumeName, ReadOnly: true, MountPath: grpcTLSVolumeMountPath, }) diff --git a/tests/e2e/smoke_test.go b/tests/e2e/smoke_test.go index 3562e913..59e51c10 100644 --- a/tests/e2e/smoke_test.go +++ b/tests/e2e/smoke_test.go @@ -768,6 +768,54 @@ var _ = Describe("Operator smoke test", func() { ExecuteSimpleTableE2ETestWithSDK(databaseSample.Name, testobjects.YdbNamespace, databasePath) }) + It("Check init job with additional volumes and GRPCS enabled", func() { + By("create stls secrets...") + storageCert := testobjects.StorageCertificate() + + secret := storageCert.DeepCopy() + secret.Name = "another-secret" + + Expect(k8sClient.Create(ctx, storageCert)).Should(Succeed()) + Expect(k8sClient.Create(ctx, secret)).Should(Succeed()) + + By("create storage...") + storage := testobjects.DefaultStorage(filepath.Join("..", "data", "storage-mirror-3-dc-config-tls.yaml")) + + storage.Spec.Service.GRPC.TLSConfiguration = testobjects.TLSConfiguration( + testobjects.StorageCertificateSecretName, + ) + + storage.Spec.Secrets = []*corev1.LocalObjectReference{ + { + Name: secret.Name, + }, + } + + mountPath := fmt.Sprintf("%s/%s", v1alpha1.AdditionalSecretsDir, secret.Name) + + storage.Spec.InitContainers = []corev1.Container{ + { + Name: "init-container", + Image: storage.Spec.Image.Name, + Command: []string{"bash", "-xc"}, + Args: []string{fmt.Sprintf("ls -la %s", mountPath)}, + VolumeMounts: []corev1.VolumeMount{ + { + Name: secret.Name, + MountPath: mountPath, + ReadOnly: true, + }, + }, + }, + } + + Expect(k8sClient.Create(ctx, storage)).Should(Succeed()) + defer DeleteStorageSafely(ctx, k8sClient, storage) + + By("waiting until Storage is ready ...") + WaitUntilStorageReady(ctx, k8sClient, storage.Name, testobjects.YdbNamespace) + }) + AfterEach(func() { UninstallOperatorWithHelm(testobjects.YdbNamespace) Expect(k8sClient.Delete(ctx, &namespace)).Should(Succeed()) From 3ae10efa44f84a7653b9d4909f4c3de570e61c61 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 29 Jan 2025 17:21:44 +0100 Subject: [PATCH 09/21] Release v0.6.0 (#288) Co-authored-by: Jorres --- .../unreleased/Added-20241113-181628.yaml | 3 -- .../unreleased/Added-20241118-222810.yaml | 4 --- .../unreleased/Added-20241204-122609.yaml | 3 -- .../unreleased/Added-20241204-122633.yaml | 3 -- .../unreleased/Added-20241204-153550.yaml | 3 -- .../unreleased/Added-20241209-182500.yaml | 3 -- .../unreleased/Added-20250104-143452.yaml | 3 -- .../unreleased/Added-20250124-141810.yaml | 3 -- .../unreleased/Added-20250127-130609.yaml | 3 -- .../unreleased/Changed-20241118-222537.yaml | 3 -- .../unreleased/Changed-20241204-122659.yaml | 3 -- .../unreleased/Changed-20241204-122713.yaml | 3 -- .../unreleased/Fixed-20241113-214512.yaml | 3 -- .../unreleased/Fixed-20241129-213809.yaml | 3 -- .../unreleased/Fixed-20241204-122736.yaml | 3 -- .../unreleased/Fixed-20250104-143747.yaml | 3 -- .../unreleased/Fixed-20250124-141631.yaml | 3 -- .../unreleased/Fixed-20250127-125607.yaml | 3 -- .../unreleased/Fixed-20250127-141003.yaml | 3 -- .../unreleased/Fixed-20250129-134226.yaml | 3 -- .../unreleased/Security-20241118-222433.yaml | 3 -- .../unreleased/Security-20250127-170538.yaml | 3 -- .changes/v0.6.0.md | 27 ++++++++++++++++++ CHANGELOG.md | 28 +++++++++++++++++++ deploy/ydb-operator/Chart.yaml | 4 +-- 25 files changed, 57 insertions(+), 69 deletions(-) delete mode 100644 .changes/unreleased/Added-20241113-181628.yaml delete mode 100644 .changes/unreleased/Added-20241118-222810.yaml delete mode 100644 .changes/unreleased/Added-20241204-122609.yaml delete mode 100644 .changes/unreleased/Added-20241204-122633.yaml delete mode 100644 .changes/unreleased/Added-20241204-153550.yaml delete mode 100644 .changes/unreleased/Added-20241209-182500.yaml delete mode 100644 .changes/unreleased/Added-20250104-143452.yaml delete mode 100644 .changes/unreleased/Added-20250124-141810.yaml delete mode 100644 .changes/unreleased/Added-20250127-130609.yaml delete mode 100644 .changes/unreleased/Changed-20241118-222537.yaml delete mode 100644 .changes/unreleased/Changed-20241204-122659.yaml delete mode 100644 .changes/unreleased/Changed-20241204-122713.yaml delete mode 100644 .changes/unreleased/Fixed-20241113-214512.yaml delete mode 100644 .changes/unreleased/Fixed-20241129-213809.yaml delete mode 100644 .changes/unreleased/Fixed-20241204-122736.yaml delete mode 100644 .changes/unreleased/Fixed-20250104-143747.yaml delete mode 100644 .changes/unreleased/Fixed-20250124-141631.yaml delete mode 100644 .changes/unreleased/Fixed-20250127-125607.yaml delete mode 100644 .changes/unreleased/Fixed-20250127-141003.yaml delete mode 100644 .changes/unreleased/Fixed-20250129-134226.yaml delete mode 100644 .changes/unreleased/Security-20241118-222433.yaml delete mode 100644 .changes/unreleased/Security-20250127-170538.yaml create mode 100644 .changes/v0.6.0.md diff --git a/.changes/unreleased/Added-20241113-181628.yaml b/.changes/unreleased/Added-20241113-181628.yaml deleted file mode 100644 index ac1658a2..00000000 --- a/.changes/unreleased/Added-20241113-181628.yaml +++ /dev/null @@ -1,3 +0,0 @@ -kind: Added -body: starting with this release, deploying to dockerhub (ydbplatform/ydb-kubernetes-operator) -time: 2024-11-13T18:16:28.275365313+01:00 diff --git a/.changes/unreleased/Added-20241118-222810.yaml b/.changes/unreleased/Added-20241118-222810.yaml deleted file mode 100644 index 7c6c51ef..00000000 --- a/.changes/unreleased/Added-20241118-222810.yaml +++ /dev/null @@ -1,4 +0,0 @@ -kind: Added -body: 'added the ability to create metadata announce for customize dns domain (default: - cluster.local)' -time: 2024-11-18T22:28:10.452679+03:00 diff --git a/.changes/unreleased/Added-20241204-122609.yaml b/.changes/unreleased/Added-20241204-122609.yaml deleted file mode 100644 index b7524b8b..00000000 --- a/.changes/unreleased/Added-20241204-122609.yaml +++ /dev/null @@ -1,3 +0,0 @@ -kind: Added -body: new field additionalPodLabels for Storage and Database CRD -time: 2024-12-04T12:26:09.597907+07:00 diff --git a/.changes/unreleased/Added-20241204-122633.yaml b/.changes/unreleased/Added-20241204-122633.yaml deleted file mode 100644 index 389dfcf0..00000000 --- a/.changes/unreleased/Added-20241204-122633.yaml +++ /dev/null @@ -1,3 +0,0 @@ -kind: Added -body: new method buildPodTemplateLabels to append additionalPodLabels for statefulset builders -time: 2024-12-04T12:26:33.654222+07:00 diff --git a/.changes/unreleased/Added-20241204-153550.yaml b/.changes/unreleased/Added-20241204-153550.yaml deleted file mode 100644 index a324b32b..00000000 --- a/.changes/unreleased/Added-20241204-153550.yaml +++ /dev/null @@ -1,3 +0,0 @@ -kind: Added -body: compatibility tests running automatically on each new tag -time: 2024-12-04T15:35:50.352507104+01:00 diff --git a/.changes/unreleased/Added-20241209-182500.yaml b/.changes/unreleased/Added-20241209-182500.yaml deleted file mode 100644 index 74088f43..00000000 --- a/.changes/unreleased/Added-20241209-182500.yaml +++ /dev/null @@ -1,3 +0,0 @@ -kind: Added -body: customize Database and Storage container securityContext -time: 2024-12-09T18:25:00.648464+01:00 diff --git a/.changes/unreleased/Added-20250104-143452.yaml b/.changes/unreleased/Added-20250104-143452.yaml deleted file mode 100644 index d70253c6..00000000 --- a/.changes/unreleased/Added-20250104-143452.yaml +++ /dev/null @@ -1,3 +0,0 @@ -kind: Added -body: field externalPort for grpc service to override --grpc-public-port arg -time: 2025-01-04T14:34:52.706824+08:00 diff --git a/.changes/unreleased/Added-20250124-141810.yaml b/.changes/unreleased/Added-20250124-141810.yaml deleted file mode 100644 index 58e21c15..00000000 --- a/.changes/unreleased/Added-20250124-141810.yaml +++ /dev/null @@ -1,3 +0,0 @@ -kind: Added -body: 'annotations overrides default secret name and key for arg --auth-token-file' -time: 2025-01-24T14:18:10.344319+08:00 diff --git a/.changes/unreleased/Added-20250127-130609.yaml b/.changes/unreleased/Added-20250127-130609.yaml deleted file mode 100644 index 16003da3..00000000 --- a/.changes/unreleased/Added-20250127-130609.yaml +++ /dev/null @@ -1,3 +0,0 @@ -kind: Added -body: field ObservedGeneration inside .status.conditions -time: 2025-01-27T13:06:09.845302+08:00 diff --git a/.changes/unreleased/Changed-20241118-222537.yaml b/.changes/unreleased/Changed-20241118-222537.yaml deleted file mode 100644 index 53d107cc..00000000 --- a/.changes/unreleased/Changed-20241118-222537.yaml +++ /dev/null @@ -1,3 +0,0 @@ -kind: Changed -body: up CONTROLLER_GEN_VERSION to 0.16.5 and ENVTEST_VERSION to release-0.17 -time: 2024-11-18T22:25:37.274092+03:00 diff --git a/.changes/unreleased/Changed-20241204-122659.yaml b/.changes/unreleased/Changed-20241204-122659.yaml deleted file mode 100644 index 0968be5c..00000000 --- a/.changes/unreleased/Changed-20241204-122659.yaml +++ /dev/null @@ -1,3 +0,0 @@ -kind: Changed -body: refactor package labels to separate methods buildLabels, buildSelectorLabels and buildeNodeSetLabels for each resource -time: 2024-12-04T12:26:59.096105+07:00 diff --git a/.changes/unreleased/Changed-20241204-122713.yaml b/.changes/unreleased/Changed-20241204-122713.yaml deleted file mode 100644 index 9bf0b3f9..00000000 --- a/.changes/unreleased/Changed-20241204-122713.yaml +++ /dev/null @@ -1,3 +0,0 @@ -kind: Changed -body: propagate labels ydb.tech/database-nodeset, ydb.tech/storage-nodeset and ydb.tech/remote-cluster with method makeCommonLabels between resource recasting -time: 2024-12-04T12:27:13.265234+07:00 diff --git a/.changes/unreleased/Fixed-20241113-214512.yaml b/.changes/unreleased/Fixed-20241113-214512.yaml deleted file mode 100644 index b5688449..00000000 --- a/.changes/unreleased/Fixed-20241113-214512.yaml +++ /dev/null @@ -1,3 +0,0 @@ -kind: Fixed -body: e2e tests and unit tests flapped because of the race between storage finalizers and uninstalling operator helm chart -time: 2024-11-13T21:45:12.19273022+01:00 diff --git a/.changes/unreleased/Fixed-20241129-213809.yaml b/.changes/unreleased/Fixed-20241129-213809.yaml deleted file mode 100644 index bb3a9eb1..00000000 --- a/.changes/unreleased/Fixed-20241129-213809.yaml +++ /dev/null @@ -1,3 +0,0 @@ -kind: Fixed -body: regenerate CRDs in upload-artifacts workflow (as opposed to manually) -time: 2024-11-29T21:38:09.848071991+01:00 diff --git a/.changes/unreleased/Fixed-20241204-122736.yaml b/.changes/unreleased/Fixed-20241204-122736.yaml deleted file mode 100644 index 4f4c6808..00000000 --- a/.changes/unreleased/Fixed-20241204-122736.yaml +++ /dev/null @@ -1,3 +0,0 @@ -kind: Fixed -body: additional kind worker to maintain affinity rules for blobstorage init job -time: 2024-12-04T12:27:36.97703+07:00 diff --git a/.changes/unreleased/Fixed-20250104-143747.yaml b/.changes/unreleased/Fixed-20250104-143747.yaml deleted file mode 100644 index 4a4439c7..00000000 --- a/.changes/unreleased/Fixed-20250104-143747.yaml +++ /dev/null @@ -1,3 +0,0 @@ -kind: Fixed -body: update the Makefile with the changes in GitHub CI -time: 2025-01-04T14:37:47.689565+08:00 diff --git a/.changes/unreleased/Fixed-20250124-141631.yaml b/.changes/unreleased/Fixed-20250124-141631.yaml deleted file mode 100644 index 31c864f9..00000000 --- a/.changes/unreleased/Fixed-20250124-141631.yaml +++ /dev/null @@ -1,3 +0,0 @@ -kind: Fixed -body: 'bug: missing error handler for arg --auth-token-file' -time: 2025-01-24T14:16:31.463111+08:00 diff --git a/.changes/unreleased/Fixed-20250127-125607.yaml b/.changes/unreleased/Fixed-20250127-125607.yaml deleted file mode 100644 index 59654796..00000000 --- a/.changes/unreleased/Fixed-20250127-125607.yaml +++ /dev/null @@ -1,3 +0,0 @@ -kind: Fixed -body: fix field resourceVersion inside .status.remoteResources.conditions -time: 2025-01-27T12:56:07.577721+08:00 diff --git a/.changes/unreleased/Fixed-20250127-141003.yaml b/.changes/unreleased/Fixed-20250127-141003.yaml deleted file mode 100644 index aa6c20f7..00000000 --- a/.changes/unreleased/Fixed-20250127-141003.yaml +++ /dev/null @@ -1,3 +0,0 @@ -kind: Fixed -body: panic when create object with .spec.pause is true -time: 2025-01-27T14:10:03.497565+08:00 diff --git a/.changes/unreleased/Fixed-20250129-134226.yaml b/.changes/unreleased/Fixed-20250129-134226.yaml deleted file mode 100644 index 2405bd52..00000000 --- a/.changes/unreleased/Fixed-20250129-134226.yaml +++ /dev/null @@ -1,3 +0,0 @@ -kind: Fixed -body: Passing additional secret volumes to blobstorage-init. The init container can now use them without issues. -time: 2025-01-29T13:42:26.145577+01:00 diff --git a/.changes/unreleased/Security-20241118-222433.yaml b/.changes/unreleased/Security-20241118-222433.yaml deleted file mode 100644 index af068092..00000000 --- a/.changes/unreleased/Security-20241118-222433.yaml +++ /dev/null @@ -1,3 +0,0 @@ -kind: Security -body: bump golang-jwt to v4.5.1 (by dependabot) -time: 2024-11-18T22:24:33.337464+03:00 diff --git a/.changes/unreleased/Security-20250127-170538.yaml b/.changes/unreleased/Security-20250127-170538.yaml deleted file mode 100644 index 219ea97a..00000000 --- a/.changes/unreleased/Security-20250127-170538.yaml +++ /dev/null @@ -1,3 +0,0 @@ -kind: Security -body: bump golang.org/x/net from 0.23.0 to 0.33.0 (by dependabot) -time: 2025-01-27T17:05:38.777767+08:00 diff --git a/.changes/v0.6.0.md b/.changes/v0.6.0.md new file mode 100644 index 00000000..aeffbf29 --- /dev/null +++ b/.changes/v0.6.0.md @@ -0,0 +1,27 @@ +## v0.6.0 - 2025-01-29 +### Added +* starting with this release, deploying to dockerhub (ydbplatform/ydb-kubernetes-operator) +* added the ability to create metadata announce for customize dns domain (default: cluster.local) +* new field additionalPodLabels for Storage and Database CRD +* new method buildPodTemplateLabels to append additionalPodLabels for statefulset builders +* compatibility tests running automatically on each new tag +* customize Database and Storage container securityContext +* field externalPort for grpc service to override --grpc-public-port arg +* annotations overrides default secret name and key for arg --auth-token-file +* field ObservedGeneration inside .status.conditions +### Changed +* up CONTROLLER_GEN_VERSION to 0.16.5 and ENVTEST_VERSION to release-0.17 +* refactor package labels to separate methods buildLabels, buildSelectorLabels and buildeNodeSetLabels for each resource +* propagate labels ydb.tech/database-nodeset, ydb.tech/storage-nodeset and ydb.tech/remote-cluster with method makeCommonLabels between resource recasting +### Fixed +* e2e tests and unit tests flapped because of the race between storage finalizers and uninstalling operator helm chart +* regenerate CRDs in upload-artifacts workflow (as opposed to manually) +* additional kind worker to maintain affinity rules for blobstorage init job +* update the Makefile with the changes in GitHub CI +* bug: missing error handler for arg --auth-token-file +* fix field resourceVersion inside .status.remoteResources.conditions +* panic when create object with .spec.pause is true +* Passing additional secret volumes to blobstorage-init. The init container can now use them without issues. +### Security +* bump golang-jwt to v4.5.1 (by dependabot) +* bump golang.org/x/net from 0.23.0 to 0.33.0 (by dependabot) diff --git a/CHANGELOG.md b/CHANGELOG.md index 94e0e4cb..b7593b9d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,34 @@ # Changelog +## v0.6.0 - 2025-01-29 +### Added +* starting with this release, deploying to dockerhub (ydbplatform/ydb-kubernetes-operator) +* added the ability to create metadata announce for customize dns domain (default: cluster.local) +* new field additionalPodLabels for Storage and Database CRD +* new method buildPodTemplateLabels to append additionalPodLabels for statefulset builders +* compatibility tests running automatically on each new tag +* customize Database and Storage container securityContext +* field externalPort for grpc service to override --grpc-public-port arg +* annotations overrides default secret name and key for arg --auth-token-file +* field ObservedGeneration inside .status.conditions +### Changed +* up CONTROLLER_GEN_VERSION to 0.16.5 and ENVTEST_VERSION to release-0.17 +* refactor package labels to separate methods buildLabels, buildSelectorLabels and buildeNodeSetLabels for each resource +* propagate labels ydb.tech/database-nodeset, ydb.tech/storage-nodeset and ydb.tech/remote-cluster with method makeCommonLabels between resource recasting +### Fixed +* e2e tests and unit tests flapped because of the race between storage finalizers and uninstalling operator helm chart +* regenerate CRDs in upload-artifacts workflow (as opposed to manually) +* additional kind worker to maintain affinity rules for blobstorage init job +* update the Makefile with the changes in GitHub CI +* bug: missing error handler for arg --auth-token-file +* fix field resourceVersion inside .status.remoteResources.conditions +* panic when create object with .spec.pause is true +* Passing additional secret volumes to blobstorage-init. The init container can now use them without issues. +### Security +* bump golang-jwt to v4.5.1 (by dependabot) +* bump golang.org/x/net from 0.23.0 to 0.33.0 (by dependabot) + ## v0.5.32 - 2024-11-05 ### Fixed * Chart.yaml version is bumped up automatically when a new release PR is created diff --git a/deploy/ydb-operator/Chart.yaml b/deploy/ydb-operator/Chart.yaml index 297e7771..e80d4fe9 100644 --- a/deploy/ydb-operator/Chart.yaml +++ b/deploy/ydb-operator/Chart.yaml @@ -15,10 +15,10 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: "0.5.32" +version: "0.6.0" # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "0.5.32" +appVersion: "0.6.0" From 84db0f332a49ebb61c899da9229506b658adc1da Mon Sep 17 00:00:00 2001 From: Egor Tarasov Date: Thu, 30 Jan 2025 10:57:34 +0100 Subject: [PATCH 10/21] Fix AWS CLI <-> S3 compatibility (#290) --- .github/workflows/upload-artifacts.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/upload-artifacts.yml b/.github/workflows/upload-artifacts.yml index 030106fe..1f5a8191 100644 --- a/.github/workflows/upload-artifacts.yml +++ b/.github/workflows/upload-artifacts.yml @@ -36,7 +36,7 @@ jobs: - name: install-aws-cli uses: unfor19/install-aws-cli-action@v1 with: - version: 2 + version: "2.22.35" - name: initialize-aws-cli run: | aws configure set aws_access_key_id ${{ secrets.CI_PUBLIC_HELM_S3_KEY_IDENTIFIER }} From 5522cd11c5ecf4f9cdf338ab189d737848c177a4 Mon Sep 17 00:00:00 2001 From: Egor Tarasov Date: Thu, 30 Jan 2025 11:18:36 +0100 Subject: [PATCH 11/21] Allow tag creation from job (#291) --- .github/workflows/upload-artifacts.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/upload-artifacts.yml b/.github/workflows/upload-artifacts.yml index 1f5a8191..5e5376f7 100644 --- a/.github/workflows/upload-artifacts.yml +++ b/.github/workflows/upload-artifacts.yml @@ -8,6 +8,9 @@ on: - 'CHANGELOG.md' workflow_dispatch: +permissions: + contents: write + jobs: tag-and-release: runs-on: ubuntu-latest From 1d1153513cda55822203e4a8e80f520e826ca72e Mon Sep 17 00:00:00 2001 From: Egor Tarasov Date: Thu, 30 Jan 2025 11:39:10 +0100 Subject: [PATCH 12/21] Allow dispatching `compatibility-tests` workflow from `upload-artifacts` (#292) --- .github/workflows/compatibility-tests.yaml | 8 +++++--- .github/workflows/tmp-workflow.yaml | 18 ++++++++++++++++++ .github/workflows/upload-artifacts.yml | 5 +++++ 3 files changed, 28 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/tmp-workflow.yaml diff --git a/.github/workflows/compatibility-tests.yaml b/.github/workflows/compatibility-tests.yaml index db7af57c..3114bcbc 100644 --- a/.github/workflows/compatibility-tests.yaml +++ b/.github/workflows/compatibility-tests.yaml @@ -1,10 +1,9 @@ name: compatibility-tests on: - push: - tags: - - '*' workflow_dispatch: + repository_dispatch: + types: [compatibility-tests] jobs: test-compatibility: @@ -56,6 +55,9 @@ jobs: PREVIOUS_VERSION="0.5.30" fi + echo "Current version is $NEW_VERSION" + echo "Will be tested for compatibility from $PREVIOUS_VERSION" + echo "NEW_VERSION=$NEW_VERSION" >> $GITHUB_ENV echo "PREVIOUS_VERSION=$PREVIOUS_VERSION" >> $GITHUB_ENV diff --git a/.github/workflows/tmp-workflow.yaml b/.github/workflows/tmp-workflow.yaml new file mode 100644 index 00000000..38f5cbab --- /dev/null +++ b/.github/workflows/tmp-workflow.yaml @@ -0,0 +1,18 @@ +name: tmp + +on: + workflow_dispatch: + +permissions: + contents: write + +jobs: + tag-and-release: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: dispatch-compatibility-tests + uses: peter-evans/repository-dispatch@v3 + with: + event-type: compatibility-tests diff --git a/.github/workflows/upload-artifacts.yml b/.github/workflows/upload-artifacts.yml index 5e5376f7..40da712e 100644 --- a/.github/workflows/upload-artifacts.yml +++ b/.github/workflows/upload-artifacts.yml @@ -131,3 +131,8 @@ jobs: tag_name: ${{ env.VERSION }} env: GITHUB_TOKEN: ${{ github.token }} + + - name: dispatch-compatibility-tests + uses: peter-evans/repository-dispatch@v3 + with: + event-type: compatibility-tests From cf0af29aa99220f4e97ff7a916618e25cf73fd8a Mon Sep 17 00:00:00 2001 From: Egor Tarasov Date: Thu, 30 Jan 2025 11:48:14 +0100 Subject: [PATCH 13/21] Cleanup compatibility test debug artifacts (#293) --- .github/workflows/compatibility-tests.yaml | 7 ------- .github/workflows/tmp-workflow.yaml | 18 ------------------ 2 files changed, 25 deletions(-) delete mode 100644 .github/workflows/tmp-workflow.yaml diff --git a/.github/workflows/compatibility-tests.yaml b/.github/workflows/compatibility-tests.yaml index 3114bcbc..931c0bcd 100644 --- a/.github/workflows/compatibility-tests.yaml +++ b/.github/workflows/compatibility-tests.yaml @@ -48,13 +48,6 @@ jobs: exit 1 fi - # remove after creating 0.6.0 tag. - # Basically, we are incompatible with 0.4, and while there is no 0.6 (and prev minor being 0.5), - # we will run compat tests from previous patch version - if [ "$PREVIOUS_VERSION" = "0.4.42" ]; then - PREVIOUS_VERSION="0.5.30" - fi - echo "Current version is $NEW_VERSION" echo "Will be tested for compatibility from $PREVIOUS_VERSION" diff --git a/.github/workflows/tmp-workflow.yaml b/.github/workflows/tmp-workflow.yaml deleted file mode 100644 index 38f5cbab..00000000 --- a/.github/workflows/tmp-workflow.yaml +++ /dev/null @@ -1,18 +0,0 @@ -name: tmp - -on: - workflow_dispatch: - -permissions: - contents: write - -jobs: - tag-and-release: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: dispatch-compatibility-tests - uses: peter-evans/repository-dispatch@v3 - with: - event-type: compatibility-tests From ed6c344f51d5926e1b815dd27c10287883fa07d9 Mon Sep 17 00:00:00 2001 From: Nikita Kozlovskii Date: Wed, 12 Feb 2025 10:14:26 +0100 Subject: [PATCH 14/21] fix blobstorage-init-job, fix passing interconnect volume (#294) * fix blobstorage-init-job, fix passing interconnect volume * add changie --- .../unreleased/Fixed-20250210-112112.yaml | 3 + internal/resources/storage_init_job.go | 69 +++++++++---------- 2 files changed, 35 insertions(+), 37 deletions(-) create mode 100644 .changes/unreleased/Fixed-20250210-112112.yaml diff --git a/.changes/unreleased/Fixed-20250210-112112.yaml b/.changes/unreleased/Fixed-20250210-112112.yaml new file mode 100644 index 00000000..e1f24365 --- /dev/null +++ b/.changes/unreleased/Fixed-20250210-112112.yaml @@ -0,0 +1,3 @@ +kind: Fixed +body: fix passing interconnet TLS volume in blobstorage-init job +time: 2025-02-10T11:21:12.37515+01:00 diff --git a/internal/resources/storage_init_job.go b/internal/resources/storage_init_job.go index 4308e29b..51da7006 100644 --- a/internal/resources/storage_init_job.go +++ b/internal/resources/storage_init_job.go @@ -136,6 +136,10 @@ func (b *StorageInitJobBuilder) buildInitJobVolumes() []corev1.Volume { }, } + if b.Spec.Service.Interconnect.TLSConfiguration.Enabled { + volumes = append(volumes, buildTLSVolume(interconnectTLSVolumeName, b.Spec.Service.Interconnect.TLSConfiguration)) + } + if b.Spec.Service.GRPC.TLSConfiguration.Enabled { volumes = append(volumes, buildTLSVolume(GRPCTLSVolumeName, b.Spec.Service.GRPC.TLSConfiguration)) } @@ -222,16 +226,7 @@ func (b *StorageInitJobBuilder) buildInitJobContainer() corev1.Container { // to return container } -func (b *StorageInitJobBuilder) buildJobVolumeMounts() []corev1.VolumeMount { - volumeMounts := []corev1.VolumeMount{ - { - Name: configVolumeName, - ReadOnly: true, - MountPath: fmt.Sprintf("%s/%s", api.ConfigDir, api.ConfigFileName), - SubPath: api.ConfigFileName, - }, - } - +func (b *StorageInitJobBuilder) appendTLSVolumeMounts(volumeMounts []corev1.VolumeMount) []corev1.VolumeMount { if b.Spec.Service.GRPC.TLSConfiguration.Enabled { volumeMounts = append(volumeMounts, corev1.VolumeMount{ Name: GRPCTLSVolumeName, @@ -240,12 +235,11 @@ func (b *StorageInitJobBuilder) buildJobVolumeMounts() []corev1.VolumeMount { }) } - if b.Spec.OperatorConnection != nil { - secretName := fmt.Sprintf(OperatorTokenSecretNameFormat, b.Storage.Name) + if b.Spec.Service.Interconnect.TLSConfiguration.Enabled { volumeMounts = append(volumeMounts, corev1.VolumeMount{ - Name: operatorTokenVolumeName, + Name: interconnectTLSVolumeName, ReadOnly: true, - MountPath: fmt.Sprintf("%s/%s", wellKnownDirForAdditionalSecrets, secretName), + MountPath: interconnectTLSVolumeMountPath, }) } @@ -260,6 +254,29 @@ func (b *StorageInitJobBuilder) buildJobVolumeMounts() []corev1.VolumeMount { MountPath: systemCertsDir, }) } + return volumeMounts +} + +func (b *StorageInitJobBuilder) buildJobVolumeMounts() []corev1.VolumeMount { + volumeMounts := []corev1.VolumeMount{ + { + Name: configVolumeName, + ReadOnly: true, + MountPath: fmt.Sprintf("%s/%s", api.ConfigDir, api.ConfigFileName), + SubPath: api.ConfigFileName, + }, + } + + if b.Spec.OperatorConnection != nil { + secretName := fmt.Sprintf(OperatorTokenSecretNameFormat, b.Storage.Name) + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: operatorTokenVolumeName, + ReadOnly: true, + MountPath: fmt.Sprintf("%s/%s", wellKnownDirForAdditionalSecrets, secretName), + }) + } + + volumeMounts = b.appendTLSVolumeMounts(volumeMounts) return volumeMounts } @@ -301,29 +318,7 @@ func (b *StorageInitJobBuilder) buildCaStorePatchingInitContainer() corev1.Conta } func (b *StorageInitJobBuilder) buildCaStorePatchingInitContainerVolumeMounts() []corev1.VolumeMount { - volumeMounts := []corev1.VolumeMount{} - - if b.AnyCertificatesAdded() { - volumeMounts = append(volumeMounts, corev1.VolumeMount{ - Name: localCertsVolumeName, - MountPath: localCertsDir, - }) - - volumeMounts = append(volumeMounts, corev1.VolumeMount{ - Name: systemCertsVolumeName, - MountPath: systemCertsDir, - }) - } - - if b.Spec.Service.GRPC.TLSConfiguration.Enabled { - volumeMounts = append(volumeMounts, corev1.VolumeMount{ - Name: GRPCTLSVolumeName, - ReadOnly: true, - MountPath: grpcTLSVolumeMountPath, - }) - } - - return volumeMounts + return b.appendTLSVolumeMounts([]corev1.VolumeMount{}) } func (b *StorageInitJobBuilder) buildBlobStorageInitCommandArgs() ([]string, []string) { From 1ca6c032552880e6a893c45f39d38bacfa4b65af Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 13 Feb 2025 11:38:23 +0100 Subject: [PATCH 15/21] Release v0.6.1 (#295) Co-authored-by: nikitka <95808+nikitka@users.noreply.github.com> --- .changes/unreleased/Fixed-20250210-112112.yaml | 3 --- .changes/v0.6.1.md | 3 +++ CHANGELOG.md | 4 ++++ deploy/ydb-operator/Chart.yaml | 4 ++-- 4 files changed, 9 insertions(+), 5 deletions(-) delete mode 100644 .changes/unreleased/Fixed-20250210-112112.yaml create mode 100644 .changes/v0.6.1.md diff --git a/.changes/unreleased/Fixed-20250210-112112.yaml b/.changes/unreleased/Fixed-20250210-112112.yaml deleted file mode 100644 index e1f24365..00000000 --- a/.changes/unreleased/Fixed-20250210-112112.yaml +++ /dev/null @@ -1,3 +0,0 @@ -kind: Fixed -body: fix passing interconnet TLS volume in blobstorage-init job -time: 2025-02-10T11:21:12.37515+01:00 diff --git a/.changes/v0.6.1.md b/.changes/v0.6.1.md new file mode 100644 index 00000000..2f7a24f4 --- /dev/null +++ b/.changes/v0.6.1.md @@ -0,0 +1,3 @@ +## v0.6.1 - 2025-02-12 +### Fixed +* fix passing interconnet TLS volume in blobstorage-init job diff --git a/CHANGELOG.md b/CHANGELOG.md index b7593b9d..99daea0c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ # Changelog +## v0.6.1 - 2025-02-12 +### Fixed +* fix passing interconnet TLS volume in blobstorage-init job + ## v0.6.0 - 2025-01-29 ### Added * starting with this release, deploying to dockerhub (ydbplatform/ydb-kubernetes-operator) diff --git a/deploy/ydb-operator/Chart.yaml b/deploy/ydb-operator/Chart.yaml index e80d4fe9..504b463a 100644 --- a/deploy/ydb-operator/Chart.yaml +++ b/deploy/ydb-operator/Chart.yaml @@ -15,10 +15,10 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: "0.6.0" +version: "0.6.1" # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "0.6.0" +appVersion: "0.6.1" From b2d3704bfd52aaef46773bd9a58ffbed05ebe835 Mon Sep 17 00:00:00 2001 From: Aleksei Kobzev Date: Mon, 24 Feb 2025 15:12:20 +0800 Subject: [PATCH 16/21] Fix prefix with POD_NAME in `--grpc-public-host` arg (#296) --- .../unreleased/Fixed-20250223-222850.yaml | 3 + .golangci.yml | 26 +----- .../controllers/database/controller_test.go | 85 +++++++++++++------ internal/resources/database_statefulset.go | 4 + 4 files changed, 66 insertions(+), 52 deletions(-) create mode 100644 .changes/unreleased/Fixed-20250223-222850.yaml diff --git a/.changes/unreleased/Fixed-20250223-222850.yaml b/.changes/unreleased/Fixed-20250223-222850.yaml new file mode 100644 index 00000000..d951ee77 --- /dev/null +++ b/.changes/unreleased/Fixed-20250223-222850.yaml @@ -0,0 +1,3 @@ +kind: Fixed +body: 'bug: regression with pod name in grpc-public-host arg' +time: 2025-02-23T22:28:50.688471+08:00 diff --git a/.golangci.yml b/.golangci.yml index 1dc7fd9d..38c43fd7 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -35,7 +35,8 @@ run: # output configuration options output: # colored-line-number|line-number|json|tab|checkstyle, default is "colored-line-number" - format: colored-line-number + formats: + - format: colored-line-number print-issued-lines: true @@ -61,13 +62,6 @@ linters-settings: # report about assignment of errors to blank identifier: `num, _ := strconv.Atoi(numStr)`; # default is false: such cases aren't reported by default. check-blank: false - govet: - # report about shadowed variables - shadow: true - fieldalignment: true - golint: - # minimal confidence for issues, default is 0.8 - min-confidence: 0.8 gofmt: # simplify code: gofmt with `-s` option, true by default simplify: true @@ -85,9 +79,6 @@ linters-settings: - G101 - G115 - G601 # no longer actual since 1.22 - fieldalignment: - # print struct with more effective memory layout or not, false by default - suggest-new: true misspell: # Correct spellings using locale preferences for US or UK. # Default is to use a neutral variety of English. @@ -118,17 +109,7 @@ linters-settings: - name: empty-block - name: superfluous-else - name: unreachable-code - unused: - # treat code as a program (not a library) and report unused exported identifiers; default is false. - # XXX: if you enable this setting, unused will report a lot of false-positives in text editors: - # if it's called for subdir of a project it can't find funcs usages. All text editor integrations - # with golangci-lint call it on a directory with the changed file. - check-exported: false unparam: - # call graph construction algorithm (cha, rta). In general, use cha for libraries, - # and rta for programs with main packages. Default is cha. - algo: cha - # Inspect exported functions, default is false. Set to true if no external program/library imports your code. # XXX: if you enable this setting, unparam will report a lot of false-positives in text editors: # if it's called for subdir of a project it can't find external interfaces. All text editor integrations @@ -192,9 +173,6 @@ issues: # Default value for this option is true. exclude-use-default: true - # Maximum issues count per one linter. Set to 0 to disable. Default is 50. - max-per-linter: 0 - # Maximum count of issues with the same text. Set to 0 to disable. Default is 3. max-same-issues: 0 diff --git a/internal/controllers/database/controller_test.go b/internal/controllers/database/controller_test.go index 467726a5..a36e0694 100644 --- a/internal/controllers/database/controller_test.go +++ b/internal/controllers/database/controller_test.go @@ -308,40 +308,47 @@ var _ = Describe("Database controller medium tests", func() { Expect(args).To(ContainElements([]string{"--grpc-public-address-v4", "--grpc-public-target-name-override"})) }) + checkContainerArg := func(expectedArgKey, expectedArgValue string) error { + foundStatefulSet := appsv1.StatefulSet{} + Eventually(func() error { + return k8sClient.Get(ctx, + types.NamespacedName{ + Name: testobjects.DatabaseName, + Namespace: testobjects.YdbNamespace, + }, + &foundStatefulSet, + ) + }, test.Timeout, test.Interval).Should(Succeed()) + podContainerArgs := foundStatefulSet.Spec.Template.Spec.Containers[0].Args + for idx, argKey := range podContainerArgs { + if argKey == expectedArgKey { + if podContainerArgs[idx+1] != expectedArgValue { + return fmt.Errorf( + "Found arg `%s` value %s does not match with expected: %s", + expectedArgKey, + podContainerArgs[idx+1], + expectedArgValue, + ) + } + } + } + return nil + } + It("Check externalPort GRPC Service field propagation", func() { By("Create test database") databaseSample = *testobjects.DefaultDatabase() + databaseSample.Spec.Service.GRPC.ExternalHost = fmt.Sprintf("%s.%s", testobjects.YdbNamespace, "k8s.external.net") Expect(k8sClient.Create(ctx, &databaseSample)).Should(Succeed()) - checkPublicPortArg := func(expectedGRPCPort string) error { - foundStatefulSet := appsv1.StatefulSet{} - Eventually(func() error { - return k8sClient.Get(ctx, - types.NamespacedName{ - Name: testobjects.DatabaseName, - Namespace: testobjects.YdbNamespace, - }, - &foundStatefulSet, - ) - }, test.Timeout, test.Interval).Should(Succeed()) - podContainerArgs := foundStatefulSet.Spec.Template.Spec.Containers[0].Args - for idx, argKey := range podContainerArgs { - if argKey == "--grpc-public-port" { - if podContainerArgs[idx+1] != expectedGRPCPort { - return fmt.Errorf( - "Found arg `--grpc-public-port` value %s does not match with expected: %s", - podContainerArgs[idx+1], - expectedGRPCPort, - ) - } - } - } - return nil - } - By("Check that args `--grpc-public-host` and `--grpc-public-port` propagated to StatefulSet pods...") Eventually( - checkPublicPortArg(fmt.Sprintf("%d", v1alpha1.GRPCPort)), + checkContainerArg("--grpc-public-host", fmt.Sprintf("%s.%s", "$(POD_NAME)", databaseSample.Spec.Service.GRPC.ExternalHost)), + test.Timeout, + test.Interval).ShouldNot(HaveOccurred()) + + Eventually( + checkContainerArg("--grpc-public-port", fmt.Sprintf("%d", v1alpha1.GRPCPort)), test.Timeout, test.Interval).ShouldNot(HaveOccurred()) @@ -384,7 +391,29 @@ var _ = Describe("Database controller medium tests", func() { By("Check that args `--grpc-public-port` was updated in StatefulSet...") Eventually( - checkPublicPortArg(fmt.Sprintf("%d", externalPort)), + checkContainerArg("--grpc-public-port", fmt.Sprintf("%d", externalPort)), + test.Timeout, + test.Interval).ShouldNot(HaveOccurred()) + }) + + It("Checking args propagation from annotation to StatefulSet", func() { + By("Check that Database with annotations was created...") + databaseSample = *testobjects.DefaultDatabase() + databaseSample.Annotations = map[string]string{ + v1alpha1.AnnotationGRPCPublicHost: fmt.Sprintf("%s.%s", testobjects.YdbNamespace, "k8s.external.net"), + v1alpha1.AnnotationGRPCPublicPort: fmt.Sprintf("%d", 30001), + } + Expect(k8sClient.Create(ctx, &databaseSample)).Should(Succeed()) + + By("Check that args `--grpc-public-host` propagated to StatefulSet pods...") + Eventually( + checkContainerArg("--grpc-public-host", fmt.Sprintf("%s.%s.%s", "$(POD_NAME)", testobjects.YdbNamespace, "k8s.external.net")), + test.Timeout, + test.Interval).ShouldNot(HaveOccurred()) + + By("Check that args `--grpc-public-port` propagated to StatefulSet pods...") + Eventually( + checkContainerArg("--grpc-public-port", fmt.Sprintf("%d", 30001)), test.Timeout, test.Interval).ShouldNot(HaveOccurred()) }) diff --git a/internal/resources/database_statefulset.go b/internal/resources/database_statefulset.go index a27f977b..fa04465b 100644 --- a/internal/resources/database_statefulset.go +++ b/internal/resources/database_statefulset.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "regexp" + "strings" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -657,6 +658,9 @@ func (b *DatabaseStatefulSetBuilder) buildContainerArgs() ([]string, []string) { if value, ok := b.ObjectMeta.Annotations[api.AnnotationGRPCPublicHost]; ok { publicHost = value } + if !(strings.HasPrefix(publicHost, "$(POD_NAME)") || strings.HasPrefix(publicHost, "$(NODE_NAME)")) { + publicHost = fmt.Sprintf("%s.%s", "$(POD_NAME)", publicHost) + } if b.Spec.Service.GRPC.IPDiscovery != nil && b.Spec.Service.GRPC.IPDiscovery.Enabled { targetNameOverride := b.Spec.Service.GRPC.IPDiscovery.TargetNameOverride From 3245f6f0ec475d5512be2a7dd34eef8a99ed5381 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 24 Feb 2025 15:34:36 +0800 Subject: [PATCH 17/21] Release v0.6.2 (#298) Co-authored-by: kobzonega <122476665+kobzonega@users.noreply.github.com> --- .changes/unreleased/Fixed-20250223-222850.yaml | 3 --- .changes/v0.6.2.md | 3 +++ CHANGELOG.md | 4 ++++ deploy/ydb-operator/Chart.yaml | 4 ++-- 4 files changed, 9 insertions(+), 5 deletions(-) delete mode 100644 .changes/unreleased/Fixed-20250223-222850.yaml create mode 100644 .changes/v0.6.2.md diff --git a/.changes/unreleased/Fixed-20250223-222850.yaml b/.changes/unreleased/Fixed-20250223-222850.yaml deleted file mode 100644 index d951ee77..00000000 --- a/.changes/unreleased/Fixed-20250223-222850.yaml +++ /dev/null @@ -1,3 +0,0 @@ -kind: Fixed -body: 'bug: regression with pod name in grpc-public-host arg' -time: 2025-02-23T22:28:50.688471+08:00 diff --git a/.changes/v0.6.2.md b/.changes/v0.6.2.md new file mode 100644 index 00000000..f7d29295 --- /dev/null +++ b/.changes/v0.6.2.md @@ -0,0 +1,3 @@ +## v0.6.2 - 2025-02-24 +### Fixed +* bug: regression with pod name in grpc-public-host arg diff --git a/CHANGELOG.md b/CHANGELOG.md index 99daea0c..b27a5f44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ # Changelog +## v0.6.2 - 2025-02-24 +### Fixed +* bug: regression with pod name in grpc-public-host arg + ## v0.6.1 - 2025-02-12 ### Fixed * fix passing interconnet TLS volume in blobstorage-init job diff --git a/deploy/ydb-operator/Chart.yaml b/deploy/ydb-operator/Chart.yaml index 504b463a..6bf8e4ed 100644 --- a/deploy/ydb-operator/Chart.yaml +++ b/deploy/ydb-operator/Chart.yaml @@ -15,10 +15,10 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: "0.6.1" +version: "0.6.2" # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "0.6.1" +appVersion: "0.6.2" From 53e8441c96b1193451cecb102d838172baf2530c Mon Sep 17 00:00:00 2001 From: Egor Tarasov Date: Thu, 8 May 2025 00:10:47 +0200 Subject: [PATCH 18/21] Fix `operatorSync: false` breaking on implicit webhook mutations (#299) --- api/v1alpha1/storage_webhook.go | 42 ++++++++++++++++++++++++++++----- 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/api/v1alpha1/storage_webhook.go b/api/v1alpha1/storage_webhook.go index d6be5085..816f4ba4 100644 --- a/api/v1alpha1/storage_webhook.go +++ b/api/v1alpha1/storage_webhook.go @@ -116,10 +116,6 @@ func (r *StorageDefaulter) Default(ctx context.Context, obj runtime.Object) erro storage := obj.(*Storage) storagelog.Info("default", "name", storage.Name) - if !storage.Spec.OperatorSync { - return nil - } - if storage.Spec.OperatorConnection != nil { if storage.Spec.OperatorConnection.Oauth2TokenExchange != nil { if storage.Spec.OperatorConnection.Oauth2TokenExchange.SignAlg == "" { @@ -311,9 +307,43 @@ func hasUpdatesBesidesFrozen(oldStorage, newStorage *Storage) (bool, string) { oldStorageCopy.Spec.OperatorSync = false newStorageCopy.Spec.OperatorSync = false - ignoreNonSpecFields := cmpopts.IgnoreFields(Storage{}, "Status", "ObjectMeta", "TypeMeta") + // We will allow configuration diffs if they are limited to + // formatting, order of keys in the map etc. If two maps are + // meaningfully different (not deep-equal), we still disallow + // the diff of course. + configurationCompareOpt := cmp.FilterPath( + func(path cmp.Path) bool { + if sf, ok := path.Last().(cmp.StructField); ok { + return sf.Name() == "Configuration" + } + return false + }, + cmp.Comparer(func(a, b string) bool { + var o1, o2 any + + if err := yaml.Unmarshal([]byte(a), &o1); err != nil { + return false + } + if err := yaml.Unmarshal([]byte(b), &o2); err != nil { + return false + } + + diff := cmp.Diff(o1, o2) + if diff != "" { + storagelog.Info(fmt.Sprintf("Configurations are different:\n%v\n%v", o1, o2)) + } + + return diff == "" + }), + ) - diff := cmp.Diff(oldStorageCopy, newStorageCopy, ignoreNonSpecFields) + ignoreNonSpecFields := cmpopts.IgnoreFields(Storage{}, "Status", "ObjectMeta", "TypeMeta") + diff := cmp.Diff( + oldStorageCopy, + newStorageCopy, + ignoreNonSpecFields, + configurationCompareOpt, + ) return diff != "", diff } From 95ebf5b9a09b9aa1eb0ece49d2001dc623a35b51 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 8 May 2025 00:16:22 +0200 Subject: [PATCH 19/21] Release v0.6.3 (#300) Co-authored-by: Jorres --- .changes/v0.6.3.md | 1 + CHANGELOG.md | 2 ++ deploy/ydb-operator/Chart.yaml | 4 ++-- 3 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 .changes/v0.6.3.md diff --git a/.changes/v0.6.3.md b/.changes/v0.6.3.md new file mode 100644 index 00000000..23cebc35 --- /dev/null +++ b/.changes/v0.6.3.md @@ -0,0 +1 @@ +## v0.6.3 - 2025-05-07 diff --git a/CHANGELOG.md b/CHANGELOG.md index b27a5f44..3f93cff9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ # Changelog +## v0.6.3 - 2025-05-07 + ## v0.6.2 - 2025-02-24 ### Fixed * bug: regression with pod name in grpc-public-host arg diff --git a/deploy/ydb-operator/Chart.yaml b/deploy/ydb-operator/Chart.yaml index 6bf8e4ed..f44e072f 100644 --- a/deploy/ydb-operator/Chart.yaml +++ b/deploy/ydb-operator/Chart.yaml @@ -15,10 +15,10 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: "0.6.2" +version: "0.6.3" # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "0.6.2" +appVersion: "0.6.3" From b8ad65c56694ae65408c687efe1b77e195976317 Mon Sep 17 00:00:00 2001 From: Egor Tarasov Date: Tue, 20 May 2025 12:13:30 +0200 Subject: [PATCH 20/21] Add GRPC Service port configuration (#301) --- .../unreleased/Added-20250512-140119.yaml | 3 + .../unreleased/Added-20250512-143551.yaml | 3 + .../unreleased/Fixed-20250513-113445.yaml | 3 + .github/workflows/run-tests.yml | 8 +- api/v1alpha1/const.go | 11 +- api/v1alpha1/service_types.go | 3 + api/v1alpha1/storage_webhook.go | 119 ++++++++++++-- deploy/ydb-operator/crds/database.yaml | 6 + deploy/ydb-operator/crds/databasenodeset.yaml | 6 + .../crds/remotedatabasenodeset.yaml | 6 + .../crds/remotestoragenodeset.yaml | 6 + deploy/ydb-operator/crds/storage.yaml | 6 + deploy/ydb-operator/crds/storagenodeset.yaml | 6 + .../configuration/schema/configuration.go | 1 + internal/configuration/schema/grpc.go | 6 + .../databasenodeset/controller_test.go | 11 +- .../controllers/monitoring/monitoring_test.go | 42 ++++- .../controllers/storage/controller_test.go | 148 ++++++++++++++++++ internal/resources/storage.go | 27 +++- internal/resources/storage_statefulset.go | 45 +++++- internal/test/k8s_helpers.go | 15 ++ 21 files changed, 447 insertions(+), 34 deletions(-) create mode 100644 .changes/unreleased/Added-20250512-140119.yaml create mode 100644 .changes/unreleased/Added-20250512-143551.yaml create mode 100644 .changes/unreleased/Fixed-20250513-113445.yaml create mode 100644 internal/configuration/schema/grpc.go diff --git a/.changes/unreleased/Added-20250512-140119.yaml b/.changes/unreleased/Added-20250512-140119.yaml new file mode 100644 index 00000000..d9ea0343 --- /dev/null +++ b/.changes/unreleased/Added-20250512-140119.yaml @@ -0,0 +1,3 @@ +kind: Added +body: '`insecurePort` can be specified in GRPC Service spec to create a Service with a second port (for non-tls grpc port in storage)' +time: 2025-05-12T14:01:19.020259248+02:00 diff --git a/.changes/unreleased/Added-20250512-143551.yaml b/.changes/unreleased/Added-20250512-143551.yaml new file mode 100644 index 00000000..9fa427b4 --- /dev/null +++ b/.changes/unreleased/Added-20250512-143551.yaml @@ -0,0 +1,3 @@ +kind: Added +body: Default 2135 port on the GRPC service can now be overridden +time: 2025-05-12T14:35:51.255373275+02:00 diff --git a/.changes/unreleased/Fixed-20250513-113445.yaml b/.changes/unreleased/Fixed-20250513-113445.yaml new file mode 100644 index 00000000..0daedb3f --- /dev/null +++ b/.changes/unreleased/Fixed-20250513-113445.yaml @@ -0,0 +1,3 @@ +kind: Fixed +body: '[development] mutating\validating webhooks now run during medium tests (previously e2e only)' +time: 2025-05-13T11:34:45.560660001+02:00 diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 36fbacfa..9d7bb5d4 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -28,7 +28,7 @@ jobs: - name: setup-go uses: actions/setup-go@v3 with: - go-version: '1.20' + go-version: '1.23' - name: setup-medium-test-class-binaries run: | # This installs kube-apiserver and etcd binaries for `medium` @@ -36,6 +36,10 @@ jobs: make envtest KUBEBUILDER_ASSETS=$(./bin/setup-envtest use 1.26 -p path) echo "KUBEBUILDER_ASSETS=$KUBEBUILDER_ASSETS" >> $GITHUB_ENV + - name: render-webhook-manifests + run: | + # This renders webhook manifests to ./config/webhook + make manifests - name: setup-gotestsum run: | go install gotest.tools/gotestsum@v1.12.0 @@ -76,7 +80,7 @@ jobs: - name: setup-go uses: actions/setup-go@v3 with: - go-version: '1.20' + go-version: '1.23' - name: install-dependencies run: | sudo apt-get update diff --git a/api/v1alpha1/const.go b/api/v1alpha1/const.go index 0fe9e0ec..ab8803c1 100644 --- a/api/v1alpha1/const.go +++ b/api/v1alpha1/const.go @@ -9,11 +9,12 @@ const ( DefaultDomainName = "cluster.local" DNSDomainAnnotation = "dns.domain" - GRPCPort = 2135 - GRPCServicePortName = "grpc" - GRPCProto = "grpc://" - GRPCSProto = "grpcs://" - GRPCServiceFQDNFormat = "%s-grpc.%s.svc.%s" + GRPCPort = 2135 + GRPCServicePortName = "grpc" + GRPCServiceInsecurePortName = "insecure-grpc" + GRPCProto = "grpc://" + GRPCSProto = "grpcs://" + GRPCServiceFQDNFormat = "%s-grpc.%s.svc.%s" InterconnectPort = 19001 InterconnectServicePortName = "interconnect" diff --git a/api/v1alpha1/service_types.go b/api/v1alpha1/service_types.go index 8c540c74..bec3d1b7 100644 --- a/api/v1alpha1/service_types.go +++ b/api/v1alpha1/service_types.go @@ -20,6 +20,9 @@ type TLSConfiguration struct { type GRPCService struct { Service `json:""` + InsecurePort int32 `json:"insecurePort,omitempty"` + Port int32 `json:"port,omitempty"` + TLSConfiguration *TLSConfiguration `json:"tls,omitempty"` ExternalHost string `json:"externalHost,omitempty"` ExternalPort int32 `json:"externalPort,omitempty"` diff --git a/api/v1alpha1/storage_webhook.go b/api/v1alpha1/storage_webhook.go index 816f4ba4..e99eddec 100644 --- a/api/v1alpha1/storage_webhook.go +++ b/api/v1alpha1/storage_webhook.go @@ -67,15 +67,7 @@ func (r *Storage) GetGRPCServiceEndpoint() string { } func (r *Storage) GetHostFromConfigEndpoint() string { - var rawYamlConfiguration string - // skip handle error because we already checked in webhook - success, dynConfig, _ := ParseDynConfig(r.Spec.Configuration) - if success { - config, _ := yaml.Marshal(dynConfig.Config) - rawYamlConfiguration = string(config) - } else { - rawYamlConfiguration = r.Spec.Configuration - } + rawYamlConfiguration := r.getRawYamlConfiguration() configuration, _ := ParseConfiguration(rawYamlConfiguration) randNum := rand.Intn(len(configuration.Hosts)) // #nosec G404 @@ -443,6 +435,115 @@ func (r *Storage) ValidateUpdate(old runtime.Object) error { return crdCheckError } + if err := r.validateGrpcPorts(); err != nil { + return err + } + + return nil +} + +func (r *Storage) getRawYamlConfiguration() string { + var rawYamlConfiguration string + // skip handle error because we already checked in webhook + success, dynConfig, _ := ParseDynConfig(r.Spec.Configuration) + if success { + config, _ := yaml.Marshal(dynConfig.Config) + rawYamlConfiguration = string(config) + } else { + rawYamlConfiguration = r.Spec.Configuration + } + + return rawYamlConfiguration +} + +func (r *Storage) validateGrpcPorts() error { + // There are three possible ways to configure grpc ports: + + // service: + // grpc: == this means one insecure port, tls is disabled + // port: 2135 + + // service: + // grpc: + // port: 2136 == this means one secure port, tls is enabled + // tls: + // enabled: true + + // service: + // grpc: + // insecurePort: 2135 == this means two ports, one secure \ one insecure + // port: 2136 + // tls: + // enabled: true + + rawYamlConfiguration := r.getRawYamlConfiguration() + configuration, err := ParseConfiguration(rawYamlConfiguration) + if err != nil { + return fmt.Errorf("failed to parse configuration immediately after building it, should not happen, %w", err) + } + configurationPort := int32(GRPCPort) + if configuration.GrpcConfig.Port != 0 { + configurationPort = configuration.GrpcConfig.Port + } + configurationSslPort := int32(0) + if configuration.GrpcConfig.SslPort != 0 { + configurationSslPort = configuration.GrpcConfig.SslPort + } + + if !r.Spec.Service.GRPC.TLSConfiguration.Enabled { + // there should be only 1 port, both in service and in config, insecure + servicePort := int32(GRPCPort) + if r.Spec.Service.GRPC.Port != 0 { + servicePort = r.Spec.Service.GRPC.Port + } + if configurationPort != servicePort { + return fmt.Errorf( + "inconsistent grpc ports: spec.service.grpc.port (%v) != configuration.grpc_config.port (%v)", + servicePort, + configurationPort, + ) + } + + if r.Spec.Service.GRPC.InsecurePort != 0 { + return fmt.Errorf( + "spec.service.grpc.tls.enabled is false, use `port` instead of `insecurePort` field to assign non-tls grpc port", + ) + } + return nil + } + + // otherwise, there might be 1 (secure only) port... + servicePort := int32(GRPCPort) + if r.Spec.Service.GRPC.Port != 0 { + servicePort = r.Spec.Service.GRPC.Port + } + if configurationSslPort == 0 { + return fmt.Errorf( + "configuration.grpc_config.ssl_port is absent in cluster configuration, but spec.service.grpc has tls enabled and port %v", + servicePort, + ) + } + if configurationSslPort != servicePort { + return fmt.Errorf( + "inconsistent grpc ports: spec.service.grpc.port (%v) != configuration.grpc_config.ssl_port (%v)", + servicePort, + configurationSslPort, + ) + } + + // or, optionally, one more: insecure port + if r.Spec.Service.GRPC.InsecurePort != 0 { + serviceInsecurePort := r.Spec.Service.GRPC.InsecurePort + + if configurationPort != serviceInsecurePort { + return fmt.Errorf( + "inconsistent grpc insecure ports: spec.service.grpc.insecure_port (%v) != configuration.grpc_config.port (%v)", + serviceInsecurePort, + configurationPort, + ) + } + } + return nil } diff --git a/deploy/ydb-operator/crds/database.yaml b/deploy/ydb-operator/crds/database.yaml index 7493ceec..eee4bff0 100644 --- a/deploy/ydb-operator/crds/database.yaml +++ b/deploy/ydb-operator/crds/database.yaml @@ -4078,6 +4078,9 @@ spec: externalPort: format: int32 type: integer + insecurePort: + format: int32 + type: integer ipDiscovery: properties: enabled: @@ -4103,6 +4106,9 @@ spec: description: IPFamilyPolicy represents the dual-stack-ness requested or required by a Service type: string + port: + format: int32 + type: integer tls: properties: CA: diff --git a/deploy/ydb-operator/crds/databasenodeset.yaml b/deploy/ydb-operator/crds/databasenodeset.yaml index d5164b28..504ec3a9 100644 --- a/deploy/ydb-operator/crds/databasenodeset.yaml +++ b/deploy/ydb-operator/crds/databasenodeset.yaml @@ -2690,6 +2690,9 @@ spec: externalPort: format: int32 type: integer + insecurePort: + format: int32 + type: integer ipDiscovery: properties: enabled: @@ -2715,6 +2718,9 @@ spec: description: IPFamilyPolicy represents the dual-stack-ness requested or required by a Service type: string + port: + format: int32 + type: integer tls: properties: CA: diff --git a/deploy/ydb-operator/crds/remotedatabasenodeset.yaml b/deploy/ydb-operator/crds/remotedatabasenodeset.yaml index ac65fe56..36207bcf 100644 --- a/deploy/ydb-operator/crds/remotedatabasenodeset.yaml +++ b/deploy/ydb-operator/crds/remotedatabasenodeset.yaml @@ -2691,6 +2691,9 @@ spec: externalPort: format: int32 type: integer + insecurePort: + format: int32 + type: integer ipDiscovery: properties: enabled: @@ -2716,6 +2719,9 @@ spec: description: IPFamilyPolicy represents the dual-stack-ness requested or required by a Service type: string + port: + format: int32 + type: integer tls: properties: CA: diff --git a/deploy/ydb-operator/crds/remotestoragenodeset.yaml b/deploy/ydb-operator/crds/remotestoragenodeset.yaml index ce30bb27..f2c3f0a5 100644 --- a/deploy/ydb-operator/crds/remotestoragenodeset.yaml +++ b/deploy/ydb-operator/crds/remotestoragenodeset.yaml @@ -2718,6 +2718,9 @@ spec: externalPort: format: int32 type: integer + insecurePort: + format: int32 + type: integer ipDiscovery: properties: enabled: @@ -2743,6 +2746,9 @@ spec: description: IPFamilyPolicy represents the dual-stack-ness requested or required by a Service type: string + port: + format: int32 + type: integer tls: properties: CA: diff --git a/deploy/ydb-operator/crds/storage.yaml b/deploy/ydb-operator/crds/storage.yaml index 1fed259e..bda09bad 100644 --- a/deploy/ydb-operator/crds/storage.yaml +++ b/deploy/ydb-operator/crds/storage.yaml @@ -5219,6 +5219,9 @@ spec: externalPort: format: int32 type: integer + insecurePort: + format: int32 + type: integer ipDiscovery: properties: enabled: @@ -5244,6 +5247,9 @@ spec: description: IPFamilyPolicy represents the dual-stack-ness requested or required by a Service type: string + port: + format: int32 + type: integer tls: properties: CA: diff --git a/deploy/ydb-operator/crds/storagenodeset.yaml b/deploy/ydb-operator/crds/storagenodeset.yaml index cf3ea5ee..b4c3e8ce 100644 --- a/deploy/ydb-operator/crds/storagenodeset.yaml +++ b/deploy/ydb-operator/crds/storagenodeset.yaml @@ -2717,6 +2717,9 @@ spec: externalPort: format: int32 type: integer + insecurePort: + format: int32 + type: integer ipDiscovery: properties: enabled: @@ -2742,6 +2745,9 @@ spec: description: IPFamilyPolicy represents the dual-stack-ness requested or required by a Service type: string + port: + format: int32 + type: integer tls: properties: CA: diff --git a/internal/configuration/schema/configuration.go b/internal/configuration/schema/configuration.go index 086c2d16..e8bf09da 100644 --- a/internal/configuration/schema/configuration.go +++ b/internal/configuration/schema/configuration.go @@ -10,6 +10,7 @@ type Configuration struct { DomainsConfig *DomainsConfig `yaml:"domains_config"` Hosts []Host `yaml:"hosts,omitempty"` KeyConfig *KeyConfig `yaml:"key_config,omitempty"` + GrpcConfig *GrpcConfig `yaml:"grpc_config,omitempty"` } type Metadata struct { diff --git a/internal/configuration/schema/grpc.go b/internal/configuration/schema/grpc.go new file mode 100644 index 00000000..e1128b52 --- /dev/null +++ b/internal/configuration/schema/grpc.go @@ -0,0 +1,6 @@ +package schema + +type GrpcConfig struct { + Port int32 `yaml:"port,omitempty"` + SslPort int32 `yaml:"ssl_port,omitempty"` +} diff --git a/internal/controllers/databasenodeset/controller_test.go b/internal/controllers/databasenodeset/controller_test.go index 26f7aeb7..e4596b32 100644 --- a/internal/controllers/databasenodeset/controller_test.go +++ b/internal/controllers/databasenodeset/controller_test.go @@ -100,10 +100,11 @@ var _ = Describe("DatabaseNodeSet controller medium tests", func() { }, test.Timeout, test.Interval).ShouldNot(HaveOccurred()) databaseSample = *testobjects.DefaultDatabase() + databaseSample.Spec.DatabaseNodeSpec.Nodes = 4 databaseSample.Spec.NodeSets = append(databaseSample.Spec.NodeSets, v1alpha1.DatabaseNodeSetSpecInline{ Name: testNodeSetName, DatabaseNodeSpec: v1alpha1.DatabaseNodeSpec{ - Nodes: 1, + Nodes: 4, }, }) @@ -154,15 +155,15 @@ var _ = Describe("DatabaseNodeSet controller medium tests", func() { v1alpha1.AnnotationUpdateStrategyOnDelete: "true", } - foundDatabase.Spec.NodeSets = append(foundDatabase.Spec.NodeSets, v1alpha1.DatabaseNodeSetSpecInline{ + foundDatabase.Spec.NodeSets = []v1alpha1.DatabaseNodeSetSpecInline{{ Name: testNodeSetName + "-labeled", Labels: map[string]string{ testNodeSetLabel: "true", }, DatabaseNodeSpec: v1alpha1.DatabaseNodeSpec{ - Nodes: 1, + Nodes: 2, }, - }) + }} foundDatabase.Spec.NodeSets = append(foundDatabase.Spec.NodeSets, v1alpha1.DatabaseNodeSetSpecInline{ Name: testNodeSetName + "-annotated", @@ -170,7 +171,7 @@ var _ = Describe("DatabaseNodeSet controller medium tests", func() { v1alpha1.AnnotationDataCenter: "envtest", }, DatabaseNodeSpec: v1alpha1.DatabaseNodeSpec{ - Nodes: 1, + Nodes: 2, }, }) diff --git a/internal/controllers/monitoring/monitoring_test.go b/internal/controllers/monitoring/monitoring_test.go index be2651e0..fbdf7a54 100644 --- a/internal/controllers/monitoring/monitoring_test.go +++ b/internal/controllers/monitoring/monitoring_test.go @@ -107,6 +107,8 @@ func createMockSvc(name string, parentKind string, parent client.Object) { func createMockDBAndSvc() { GinkgoHelper() + createMockStorageAndSvc() + db := testobjects.DefaultDatabase() Expect(k8sClient.Create(ctx, db)).Should(Succeed()) @@ -128,6 +130,40 @@ func createMockStorageAndSvc() { createMockSvc("storage-svc-status", "Storage", stor) } +func cleanupMockStorageAndSvc() { + storage := api.Storage{} + Expect(k8sClient.Get(ctx, types.NamespacedName{ + Name: testobjects.StorageName, + Namespace: testobjects.YdbNamespace, + }, &storage)) + Expect(k8sClient.Delete(ctx, &storage)).Should(Succeed()) + + svc := corev1.Service{} + Expect(k8sClient.Get(ctx, types.NamespacedName{ + Name: "storage-svc-status", + Namespace: testobjects.YdbNamespace, + }, &svc)) + Expect(k8sClient.Delete(ctx, &svc)).Should(Succeed()) +} + +func cleanupMockDatabaseAndSvc() { + cleanupMockStorageAndSvc() + + database := api.Database{} + Expect(k8sClient.Get(ctx, types.NamespacedName{ + Name: testobjects.DatabaseName, + Namespace: testobjects.YdbNamespace, + }, &database)) + Expect(k8sClient.Delete(ctx, &database)).Should(Succeed()) + + svc := corev1.Service{} + Expect(k8sClient.Get(ctx, types.NamespacedName{ + Name: "database-svc-status", + Namespace: testobjects.YdbNamespace, + }, &svc)) + Expect(k8sClient.Delete(ctx, &svc)).Should(Succeed()) +} + var _ = Describe("Create DatabaseMonitoring", func() { When("Database is already ready", func() { It("We must create ServiceMonitor", func() { @@ -162,12 +198,14 @@ var _ = Describe("Create DatabaseMonitoring", func() { } return false }, test.Timeout, test.Interval).Should(BeTrue()) + + cleanupMockDatabaseAndSvc() }) }) }) var _ = Describe("StorageMonitoring tests", func() { - When("Database is already ready", func() { + When("Storage is already ready", func() { It("We must create ServiceMonitor", func() { createMockStorageAndSvc() @@ -200,6 +238,8 @@ var _ = Describe("StorageMonitoring tests", func() { } return false }, test.Timeout, test.Interval).Should(BeTrue()) + + cleanupMockStorageAndSvc() }) }) }) diff --git a/internal/controllers/storage/controller_test.go b/internal/controllers/storage/controller_test.go index b23e2498..f0218e55 100644 --- a/internal/controllers/storage/controller_test.go +++ b/internal/controllers/storage/controller_test.go @@ -7,6 +7,8 @@ import ( "strings" "testing" + "gopkg.in/yaml.v3" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" @@ -305,5 +307,151 @@ var _ = Describe("Storage controller medium tests", func() { By("check that --auth-token-file arg was added to Statefulset template...") Eventually(checkAuthTokenArgs, test.Timeout, test.Interval).ShouldNot(HaveOccurred()) }) + + By("Checking overriding port value in GRPC Service...", func() { + storage := v1alpha1.Storage{} + Expect(k8sClient.Get(ctx, types.NamespacedName{ + Name: testobjects.StorageName, + Namespace: testobjects.YdbNamespace, + }, &storage)).Should(Succeed()) + + storage.Spec.Service.GRPC.Port = 2137 + + configWithNewPorts, err := patchGRPCPortsInConfiguration(storage.Spec.Configuration, -1, 2137) + Expect(err).To(BeNil()) + storage.Spec.Configuration = configWithNewPorts + + Expect(k8sClient.Update(ctx, &storage)).Should(Succeed()) + + var svc corev1.Service + serviceName := fmt.Sprintf("%v-grpc", testobjects.StorageName) + + Eventually(func(g Gomega) bool { + err := k8sClient.Get(ctx, + client.ObjectKey{ + Name: serviceName, + Namespace: testobjects.YdbNamespace, + }, + &svc, + ) + if err != nil { + return false + } + + ports := svc.Spec.Ports + g.Expect(len(ports)).To(Equal(1), "expected 1 port but got %d", len(ports)) + g.Expect(ports[0].Name).To(Equal(v1alpha1.GRPCServicePortName)) + g.Expect(ports[0].Port).To(Equal(storage.Spec.Service.GRPC.Port)) + return true + }, test.Timeout, test.Interval).Should(BeTrue(), + "Service %s/%s should eventually have proper ports", testobjects.YdbNamespace, serviceName, + ) + }) + + By("Checking insecurePort propagation in GRPC Service...", func() { + storage := v1alpha1.Storage{} + Expect(k8sClient.Get(ctx, types.NamespacedName{ + Name: testobjects.StorageName, + Namespace: testobjects.YdbNamespace, + }, &storage)).Should(Succeed()) + + storage.Spec.Service.GRPC.Port = v1alpha1.GRPCPort + storage.Spec.Service.GRPC.InsecurePort = 2136 + storage.Spec.Service.GRPC.TLSConfiguration.Enabled = true + + configWithNewPorts, err := patchGRPCPortsInConfiguration( + storage.Spec.Configuration, + storage.Spec.Service.GRPC.Port, + storage.Spec.Service.GRPC.InsecurePort, + ) + + Expect(err).To(BeNil()) + storage.Spec.Configuration = configWithNewPorts + + Expect(k8sClient.Update(ctx, &storage)).Should(Succeed()) + + var svc corev1.Service + serviceName := fmt.Sprintf("%v-grpc", testobjects.StorageName) + Eventually(func(g Gomega) error { + err := k8sClient.Get(ctx, + client.ObjectKey{ + Name: serviceName, + Namespace: testobjects.YdbNamespace, + }, + &svc, + ) + if err != nil { + return err + } + + ports := svc.Spec.Ports + g.Expect(len(ports)).To(Equal(2), "expected 2 ports but got %d", len(ports)) + g.Expect(ports[0].Port).To(Equal(int32(v1alpha1.GRPCPort))) + g.Expect(ports[0].Name).To(Equal(v1alpha1.GRPCServicePortName)) + g.Expect(ports[1].Port).To(Equal(storage.Spec.Service.GRPC.InsecurePort)) + g.Expect(ports[1].Name).To(Equal(v1alpha1.GRPCServiceInsecurePortName)) + g.Expect(ports[1].TargetPort.IntVal).To(Equal(storage.Spec.Service.GRPC.InsecurePort)) + return nil + }, test.Timeout, test.Interval).Should(Succeed(), + "Service %s/%s should eventually have proper ports", testobjects.YdbNamespace, serviceName, + ) + }) + + By("Forbid to edit grpc ports, when out of sync with YDB config...", func() { + storage := v1alpha1.Storage{} + Expect(k8sClient.Get(ctx, types.NamespacedName{ + Name: testobjects.StorageName, + Namespace: testobjects.YdbNamespace, + }, &storage)).Should(Succeed()) + + storage.Spec.Service.GRPC.TLSConfiguration.Enabled = true + + storage.Spec.Service.GRPC.Port = v1alpha1.GRPCPort + By("Specify 2136 in manifest spec...") + storage.Spec.Service.GRPC.InsecurePort = 2136 + + By("And then specify 2137 in manifest spec...") + configWithNewPorts, err := patchGRPCPortsInConfiguration(storage.Spec.Configuration, v1alpha1.GRPCPort, 2137) + Expect(err).To(BeNil()) + storage.Spec.Configuration = configWithNewPorts + + err = k8sClient.Update(ctx, &storage) + Expect(err).To(MatchError(ContainSubstring( + "inconsistent grpc insecure ports: spec.service.grpc.insecure_port (2136) != configuration.grpc_config.port (2137)", + ))) + }) }) }) + +func patchGRPCPortsInConfiguration(in string, sslPort, port int32) (string, error) { + m := make(map[string]any) + if err := yaml.Unmarshal([]byte(in), &m); err != nil { + return "", err + } + + cfg, _ := m["grpc_config"].(map[string]any) + if cfg == nil { + cfg = make(map[string]any) + } + + if sslPort != -1 { + cfg["ssl_port"] = sslPort + } else { + delete(cfg, "ssl_port") + } + + if port != -1 { + cfg["port"] = port + } else { + delete(cfg, "port") + } + + m["grpc_config"] = cfg + + res, err := yaml.Marshal(m) + if err != nil { + return "", err + } + + return string(res), nil +} diff --git a/internal/resources/storage.go b/internal/resources/storage.go index 77238adb..69c2787c 100644 --- a/internal/resources/storage.go +++ b/internal/resources/storage.go @@ -177,10 +177,7 @@ func (b *StorageClusterBuilder) GetResourceBuilders(restConfig *rest.Config) []R Labels: grpcServiceLabels, SelectorLabels: storageSelectorLabels, Annotations: b.Spec.Service.GRPC.AdditionalAnnotations, - Ports: []corev1.ServicePort{{ - Name: api.GRPCServicePortName, - Port: api.GRPCPort, - }}, + Ports: b.buildGrpcServicePorts(), IPFamilies: b.Spec.Service.GRPC.IPFamilies, IPFamilyPolicy: b.Spec.Service.GRPC.IPFamilyPolicy, }, @@ -214,6 +211,28 @@ func (b *StorageClusterBuilder) GetResourceBuilders(restConfig *rest.Config) []R ) } +func (b *StorageClusterBuilder) buildGrpcServicePorts() []corev1.ServicePort { + firstPort := int32(api.GRPCPort) + + if b.Spec.Service.GRPC.Port != 0 { + firstPort = b.Spec.Service.GRPC.Port + } + + ports := []corev1.ServicePort{{ + Name: api.GRPCServicePortName, + Port: firstPort, + }} + + if b.Spec.Service.GRPC.InsecurePort != 0 { + ports = append(ports, corev1.ServicePort{ + Name: api.GRPCServiceInsecurePortName, + Port: b.Spec.Service.GRPC.InsecurePort, + }) + } + + return ports +} + func (b *StorageClusterBuilder) getNodeSetBuilders() []ResourceBuilder { var nodeSetBuilders []ResourceBuilder diff --git a/internal/resources/storage_statefulset.go b/internal/resources/storage_statefulset.go index 5fb87671..11d1cf0e 100644 --- a/internal/resources/storage_statefulset.go +++ b/internal/resources/storage_statefulset.go @@ -356,6 +356,36 @@ func (b *StorageStatefulSetBuilder) buildCaStorePatchingInitContainerVolumeMount return volumeMounts } +func (b *StorageStatefulSetBuilder) buildContainerPorts() []corev1.ContainerPort { + podPorts := []corev1.ContainerPort{{ + Name: "interconnect", ContainerPort: api.InterconnectPort, + }, { + Name: "status", ContainerPort: api.StatusPort, + }} + + firstGRPCPort := corev1.ContainerPort{ + Name: api.GRPCServicePortName, + ContainerPort: api.GRPCPort, + } + + overrideGRPCPort := b.Spec.StorageClusterSpec.Service.GRPC.Port + if overrideGRPCPort != 0 { + firstGRPCPort.ContainerPort = overrideGRPCPort + } + + podPorts = append(podPorts, firstGRPCPort) + + insecurePort := b.Spec.StorageClusterSpec.Service.GRPC.InsecurePort + if insecurePort != 0 { + podPorts = append(podPorts, corev1.ContainerPort{ + Name: api.GRPCServiceInsecurePortName, + ContainerPort: insecurePort, + }) + } + + return podPorts +} + func (b *StorageStatefulSetBuilder) buildContainer() corev1.Container { // todo add init container for sparse files? command, args := b.buildContainerArgs() containerResources := corev1.ResourceRequirements{} @@ -376,23 +406,22 @@ func (b *StorageStatefulSetBuilder) buildContainer() corev1.Container { // todo SecurityContext: mergeSecurityContextWithDefaults(b.Spec.SecurityContext), - Ports: []corev1.ContainerPort{{ - Name: "grpc", ContainerPort: api.GRPCPort, - }, { - Name: "interconnect", ContainerPort: api.InterconnectPort, - }, { - Name: "status", ContainerPort: api.StatusPort, - }}, + Ports: b.buildContainerPorts(), VolumeMounts: b.buildVolumeMounts(), Resources: containerResources, } + livenessProbePort := api.GRPCPort + if b.Spec.Service.GRPC.Port != 0 { + livenessProbePort = int(b.Spec.Service.GRPC.Port) + } + if value, ok := b.ObjectMeta.Annotations[api.AnnotationDisableLivenessProbe]; !ok || value != api.AnnotationValueTrue { container.LivenessProbe = &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ TCPSocket: &corev1.TCPSocketAction{ - Port: intstr.FromInt(api.GRPCPort), + Port: intstr.FromInt(livenessProbePort), }, }, } diff --git a/internal/test/k8s_helpers.go b/internal/test/k8s_helpers.go index b135bd05..268eb5a3 100644 --- a/internal/test/k8s_helpers.go +++ b/internal/test/k8s_helpers.go @@ -149,6 +149,7 @@ func SetupK8STestManager(testCtx *context.Context, k8sClient *client.Client, con // FIXME: find a better way? _, curfile, _, _ := runtime.Caller(0) //nolint:dogsled + webhookDir := filepath.Join(curfile, "..", "..", "..", "config", "webhook") testEnv := &envtest.Environment{ CRDDirectoryPaths: []string{ filepath.Join(curfile, "..", "..", "..", "deploy", "ydb-operator", "crds"), @@ -156,6 +157,12 @@ func SetupK8STestManager(testCtx *context.Context, k8sClient *client.Client, con }, ErrorIfCRDPathMissing: true, UseExistingCluster: &useExistingCluster, + WebhookInstallOptions: envtest.WebhookInstallOptions{ + Paths: []string{webhookDir}, + LocalServingHost: "127.0.0.1", + LocalServingPort: 9443, + LocalServingCertDir: "", + }, } BeforeSuite(func() { @@ -174,6 +181,9 @@ func SetupK8STestManager(testCtx *context.Context, k8sClient *client.Client, con mgr, err := ctrl.NewManager(cfg, ctrl.Options{ MetricsBindAddress: "0", Scheme: scheme.Scheme, + Host: testEnv.WebhookInstallOptions.LocalServingHost, + Port: testEnv.WebhookInstallOptions.LocalServingPort, + CertDir: testEnv.WebhookInstallOptions.LocalServingCertDir, }) Expect(err).ToNot(HaveOccurred()) @@ -183,6 +193,11 @@ func SetupK8STestManager(testCtx *context.Context, k8sClient *client.Client, con Expect(c.SetupWithManager(mgr)).To(Succeed()) } + // Setup webhooks + Expect((&v1alpha1.Storage{}).SetupWebhookWithManager(mgr)).To(Succeed()) + Expect((&v1alpha1.Database{}).SetupWebhookWithManager(mgr)).To(Succeed()) + Expect(v1alpha1.RegisterMonitoringValidatingWebhook(mgr, true)).To(Succeed()) + go func() { defer GinkgoRecover() err = mgr.Start(ctx) From 000f685598f2ec100bdce214b32707f8b046cee0 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 21 May 2025 13:12:23 +0200 Subject: [PATCH 21/21] Release v0.6.4 (#302) Co-authored-by: Jorres --- .changes/unreleased/Added-20250512-140119.yaml | 3 --- .changes/unreleased/Added-20250512-143551.yaml | 3 --- .changes/unreleased/Fixed-20250513-113445.yaml | 3 --- .changes/v0.6.4.md | 6 ++++++ CHANGELOG.md | 7 +++++++ deploy/ydb-operator/Chart.yaml | 4 ++-- 6 files changed, 15 insertions(+), 11 deletions(-) delete mode 100644 .changes/unreleased/Added-20250512-140119.yaml delete mode 100644 .changes/unreleased/Added-20250512-143551.yaml delete mode 100644 .changes/unreleased/Fixed-20250513-113445.yaml create mode 100644 .changes/v0.6.4.md diff --git a/.changes/unreleased/Added-20250512-140119.yaml b/.changes/unreleased/Added-20250512-140119.yaml deleted file mode 100644 index d9ea0343..00000000 --- a/.changes/unreleased/Added-20250512-140119.yaml +++ /dev/null @@ -1,3 +0,0 @@ -kind: Added -body: '`insecurePort` can be specified in GRPC Service spec to create a Service with a second port (for non-tls grpc port in storage)' -time: 2025-05-12T14:01:19.020259248+02:00 diff --git a/.changes/unreleased/Added-20250512-143551.yaml b/.changes/unreleased/Added-20250512-143551.yaml deleted file mode 100644 index 9fa427b4..00000000 --- a/.changes/unreleased/Added-20250512-143551.yaml +++ /dev/null @@ -1,3 +0,0 @@ -kind: Added -body: Default 2135 port on the GRPC service can now be overridden -time: 2025-05-12T14:35:51.255373275+02:00 diff --git a/.changes/unreleased/Fixed-20250513-113445.yaml b/.changes/unreleased/Fixed-20250513-113445.yaml deleted file mode 100644 index 0daedb3f..00000000 --- a/.changes/unreleased/Fixed-20250513-113445.yaml +++ /dev/null @@ -1,3 +0,0 @@ -kind: Fixed -body: '[development] mutating\validating webhooks now run during medium tests (previously e2e only)' -time: 2025-05-13T11:34:45.560660001+02:00 diff --git a/.changes/v0.6.4.md b/.changes/v0.6.4.md new file mode 100644 index 00000000..edb26643 --- /dev/null +++ b/.changes/v0.6.4.md @@ -0,0 +1,6 @@ +## v0.6.4 - 2025-05-21 +### Added +* `insecurePort` can be specified in GRPC Service spec to create a Service with a second port (for non-tls grpc port in storage) +* Default 2135 port on the GRPC service can now be overridden +### Fixed +* [development] mutating\validating webhooks now run during medium tests (previously e2e only) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f93cff9..eef3eca7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,13 @@ # Changelog +## v0.6.4 - 2025-05-21 +### Added +* `insecurePort` can be specified in GRPC Service spec to create a Service with a second port (for non-tls grpc port in storage) +* Default 2135 port on the GRPC service can now be overridden +### Fixed +* [development] mutating\validating webhooks now run during medium tests (previously e2e only) + ## v0.6.3 - 2025-05-07 ## v0.6.2 - 2025-02-24 diff --git a/deploy/ydb-operator/Chart.yaml b/deploy/ydb-operator/Chart.yaml index f44e072f..80d5cc24 100644 --- a/deploy/ydb-operator/Chart.yaml +++ b/deploy/ydb-operator/Chart.yaml @@ -15,10 +15,10 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: "0.6.3" +version: "0.6.4" # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "0.6.3" +appVersion: "0.6.4"