Skip to content

Commit 20bfc74

Browse files
kobzonegakobzonega
authored
YDBOPS-8536 Do not trigger Reconcile on CREATE event for Owns (#229)
* ignore CREATE event from owns objects * unit-test detect StatefulSet delete event * compare only lastApplied annotation * fix misspell lint * bump helm chart version
1 parent 7245f02 commit 20bfc74

File tree

17 files changed

+220
-135
lines changed

17 files changed

+220
-135
lines changed

‎Makefile

+2-2
Original file line numberDiff line numberDiff line change
@@ -79,11 +79,11 @@ kind-load:
7979

8080
.PHONY: unit-test
8181
unit-test: manifests generate fmt vet envtest ## Run unit tests
82-
KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use --arch=amd64 $(ENVTEST_K8S_VERSION) -p path)" go test -v -timeout 1800s -p 1 ./internal/... -ginkgo.v -coverprofile cover.out
82+
KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use --arch=amd64 $(ENVTEST_K8S_VERSION) -p path)" go test -v -timeout 900s -p 1 ./internal/... -ginkgo.v -coverprofile cover.out
8383

8484
.PHONY: e2e-test
8585
e2e-test: manifests generate fmt vet docker-build kind-init kind-load ## Run e2e tests
86-
go test -v -timeout 1800s -p 1 ./e2e/... -args -ginkgo.v
86+
go test -v -timeout 3600s -p 1 ./e2e/... -args -ginkgo.v
8787

8888
.PHONY: test
8989
test: unit-test e2e-test ## Run all tests

‎deploy/ydb-operator/Chart.yaml

+2-2
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,10 @@ type: application
1515
# This is the chart version. This version number should be incremented each time you make changes
1616
# to the chart and its templates, including the app version.
1717
# Versions are expected to follow Semantic Versioning (https://semver.org/)
18-
version: 0.5.18
18+
version: 0.5.19
1919

2020
# This is the version number of the application being deployed. This version number should be
2121
# incremented each time you make changes to the application. Versions are not expected to
2222
# follow Semantic Versioning. They should reflect the version the application is using.
2323
# It is recommended to use it with quotes.
24-
appVersion: "0.5.18"
24+
appVersion: "0.5.19"

‎internal/annotations/annotations.go

+9-24
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
11
package annotations
22

3-
import "strings"
4-
53
const (
64
PrimaryResourceStorageAnnotation = "ydb.tech/primary-resource-storage"
75
PrimaryResourceDatabaseAnnotation = "ydb.tech/primary-resource-database"
@@ -11,30 +9,17 @@ const (
119
LastAppliedAnnotation = "ydb.tech/last-applied"
1210
)
1311

14-
func GetYdbTechAnnotations(annotations map[string]string) map[string]string {
15-
result := make(map[string]string)
16-
for key, value := range annotations {
17-
if strings.HasPrefix(key, "ydb.tech/") {
18-
result[key] = value
19-
}
20-
}
21-
return result
12+
func CompareLastAppliedAnnotation(map1, map2 map[string]string) bool {
13+
value1 := getLastAppliedAnnotation(map1)
14+
value2 := getLastAppliedAnnotation(map2)
15+
return value1 == value2
2216
}
2317

24-
func CompareMaps(map1, map2 map[string]string) bool {
25-
if len(map1) != len(map2) {
26-
return false
27-
}
28-
for key1, value1 := range map1 {
29-
if value2, ok := map2[key1]; !ok || value2 != value1 {
30-
return false
18+
func getLastAppliedAnnotation(annotations map[string]string) string {
19+
for key, value := range annotations {
20+
if key == LastAppliedAnnotation {
21+
return value
3122
}
3223
}
33-
return true
34-
}
35-
36-
func CompareYdbTechAnnotations(map1, map2 map[string]string) bool {
37-
map1 = GetYdbTechAnnotations(map1)
38-
map2 = GetYdbTechAnnotations(map2)
39-
return CompareMaps(map1, map2)
24+
return ""
4025
}

‎internal/controllers/database/controller.go

+23-13
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313
"k8s.io/client-go/rest"
1414
"k8s.io/client-go/tools/record"
1515
ctrl "sigs.k8s.io/controller-runtime"
16+
"sigs.k8s.io/controller-runtime/pkg/builder"
1617
"sigs.k8s.io/controller-runtime/pkg/client"
1718
"sigs.k8s.io/controller-runtime/pkg/handler"
1819
"sigs.k8s.io/controller-runtime/pkg/log"
@@ -69,6 +70,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
6970
r.Log.Error(err, "unexpected Get error")
7071
return ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err
7172
}
73+
7274
result, err := r.Sync(ctx, resource)
7375
if err != nil {
7476
r.Log.Error(err, "unexpected Sync error")
@@ -152,23 +154,31 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error {
152154
}
153155

154156
return controller.
155-
For(&v1alpha1.Database{}).
156-
Owns(&v1alpha1.RemoteDatabaseNodeSet{}).
157-
Owns(&v1alpha1.DatabaseNodeSet{}).
158-
Owns(&appsv1.StatefulSet{}).
159-
Owns(&corev1.ConfigMap{}).
160-
Owns(&corev1.Service{}).
157+
For(&v1alpha1.Database{},
158+
builder.WithPredicates(predicate.GenerationChangedPredicate{}),
159+
).
160+
Owns(&v1alpha1.RemoteDatabaseNodeSet{},
161+
builder.WithPredicates(resources.LastAppliedAnnotationPredicate()), // TODO: YDBOPS-9194
162+
).
163+
Owns(&v1alpha1.DatabaseNodeSet{},
164+
builder.WithPredicates(resources.LastAppliedAnnotationPredicate()), // TODO: YDBOPS-9194
165+
).
166+
Owns(&appsv1.StatefulSet{},
167+
builder.WithPredicates(predicate.GenerationChangedPredicate{}),
168+
).
169+
Owns(&corev1.ConfigMap{},
170+
builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}),
171+
).
172+
Owns(&corev1.Service{},
173+
builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}),
174+
).
161175
Watches(
162176
&source.Kind{Type: &corev1.Secret{}},
163177
handler.EnqueueRequestsFromMapFunc(r.findDatabasesForSecret),
178+
builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}),
164179
).
165-
WithEventFilter(predicate.Or(
166-
predicate.GenerationChangedPredicate{},
167-
resources.LastAppliedAnnotationPredicate(),
168-
resources.IsServicePredicate(),
169-
resources.IsSecretPredicate(),
170-
)).
171-
WithEventFilter(resources.IgnoreDeletetionPredicate()).
180+
WithEventFilter(resources.IsDatabaseCreatePredicate()).
181+
WithEventFilter(resources.IgnoreDeleteStateUnknownPredicate()).
172182
Complete(r)
173183
}
174184

‎internal/controllers/databasenodeset/controller.go

+8-6
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"k8s.io/client-go/rest"
1111
"k8s.io/client-go/tools/record"
1212
ctrl "sigs.k8s.io/controller-runtime"
13+
"sigs.k8s.io/controller-runtime/pkg/builder"
1314
"sigs.k8s.io/controller-runtime/pkg/client"
1415
"sigs.k8s.io/controller-runtime/pkg/log"
1516
"sigs.k8s.io/controller-runtime/pkg/predicate"
@@ -66,12 +67,13 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error {
6667
controller := ctrl.NewControllerManagedBy(mgr)
6768

6869
return controller.
69-
For(&v1alpha1.DatabaseNodeSet{}).
70-
Owns(&appsv1.StatefulSet{}).
71-
WithEventFilter(predicate.Or(
72-
predicate.GenerationChangedPredicate{},
73-
resources.LastAppliedAnnotationPredicate()),
70+
For(&v1alpha1.DatabaseNodeSet{},
71+
builder.WithPredicates(predicate.GenerationChangedPredicate{}),
7472
).
75-
WithEventFilter(resources.IgnoreDeletetionPredicate()).
73+
Owns(&appsv1.StatefulSet{},
74+
builder.WithPredicates(predicate.GenerationChangedPredicate{}),
75+
).
76+
WithEventFilter(resources.IsDatabaseNodeSetCreatePredicate()).
77+
WithEventFilter(resources.IgnoreDeleteStateUnknownPredicate()).
7678
Complete(r)
7779
}

‎internal/controllers/remotedatabasenodeset/controller.go

+9-10
Original file line numberDiff line numberDiff line change
@@ -165,31 +165,30 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager, remoteCluster *cluster.C
165165
Watches(
166166
source.NewKindWithCache(&v1alpha1.RemoteDatabaseNodeSet{}, cluster.GetCache()),
167167
&handler.EnqueueRequestForObject{},
168-
builder.WithPredicates(predicate.Or(
169-
predicate.GenerationChangedPredicate{},
170-
resources.LastAppliedAnnotationPredicate(),
171-
)),
168+
builder.WithPredicates(predicate.GenerationChangedPredicate{}),
172169
).
173170
Watches(
174171
&source.Kind{Type: &v1alpha1.DatabaseNodeSet{}},
175172
&handler.EnqueueRequestForObject{},
176-
builder.WithPredicates(
177-
resources.LabelExistsPredicate(isNodeSetFromMgmt),
178-
),
173+
builder.WithPredicates(resources.LabelExistsPredicate(isNodeSetFromMgmt)),
179174
).
180175
Watches(
181176
&source.Kind{Type: &corev1.Service{}},
182177
handler.EnqueueRequestsFromMapFunc(annotationFilter),
178+
builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}),
183179
).
184180
Watches(
185-
&source.Kind{Type: &corev1.Secret{}},
181+
&source.Kind{Type: &corev1.ConfigMap{}},
186182
handler.EnqueueRequestsFromMapFunc(annotationFilter),
183+
builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}),
187184
).
188185
Watches(
189-
&source.Kind{Type: &corev1.ConfigMap{}},
186+
&source.Kind{Type: &corev1.Secret{}},
190187
handler.EnqueueRequestsFromMapFunc(annotationFilter),
188+
builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}),
191189
).
192-
WithEventFilter(resources.IgnoreDeletetionPredicate()).
190+
WithEventFilter(resources.IsRemoteDatabaseNodeSetCreatePredicate()).
191+
WithEventFilter(resources.IgnoreDeleteStateUnknownPredicate()).
193192
Complete(r)
194193
}
195194

‎internal/controllers/remotestoragenodeset/controller.go

+9-10
Original file line numberDiff line numberDiff line change
@@ -165,31 +165,30 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager, remoteCluster *cluster.C
165165
Watches(
166166
source.NewKindWithCache(&v1alpha1.RemoteStorageNodeSet{}, cluster.GetCache()),
167167
&handler.EnqueueRequestForObject{},
168-
builder.WithPredicates(predicate.Or(
169-
predicate.GenerationChangedPredicate{},
170-
resources.LastAppliedAnnotationPredicate(),
171-
)),
168+
builder.WithPredicates(predicate.GenerationChangedPredicate{}),
172169
).
173170
Watches(
174171
&source.Kind{Type: &v1alpha1.StorageNodeSet{}},
175172
&handler.EnqueueRequestForObject{},
176-
builder.WithPredicates(
177-
resources.LabelExistsPredicate(isNodeSetFromMgmt),
178-
),
173+
builder.WithPredicates(resources.LabelExistsPredicate(isNodeSetFromMgmt)),
179174
).
180175
Watches(
181176
&source.Kind{Type: &corev1.Service{}},
182177
handler.EnqueueRequestsFromMapFunc(annotationFilter),
178+
builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}),
183179
).
184180
Watches(
185-
&source.Kind{Type: &corev1.Secret{}},
181+
&source.Kind{Type: &corev1.ConfigMap{}},
186182
handler.EnqueueRequestsFromMapFunc(annotationFilter),
183+
builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}),
187184
).
188185
Watches(
189-
&source.Kind{Type: &corev1.ConfigMap{}},
186+
&source.Kind{Type: &corev1.Secret{}},
190187
handler.EnqueueRequestsFromMapFunc(annotationFilter),
188+
builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}),
191189
).
192-
WithEventFilter(resources.IgnoreDeletetionPredicate()).
190+
WithEventFilter(resources.IsRemoteStorageNodeSetCreatePredicate()).
191+
WithEventFilter(resources.IgnoreDeleteStateUnknownPredicate()).
193192
Complete(r)
194193
}
195194

‎internal/controllers/storage/controller.go

+22-13
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import (
1414
"k8s.io/client-go/rest"
1515
"k8s.io/client-go/tools/record"
1616
ctrl "sigs.k8s.io/controller-runtime"
17+
"sigs.k8s.io/controller-runtime/pkg/builder"
1718
"sigs.k8s.io/controller-runtime/pkg/client"
1819
"sigs.k8s.io/controller-runtime/pkg/handler"
1920
"sigs.k8s.io/controller-runtime/pkg/log"
@@ -160,23 +161,31 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error {
160161
}
161162

162163
return controller.
163-
For(&v1alpha1.Storage{}).
164-
Owns(&v1alpha1.RemoteStorageNodeSet{}).
165-
Owns(&v1alpha1.StorageNodeSet{}).
166-
Owns(&appsv1.StatefulSet{}).
167-
Owns(&corev1.ConfigMap{}).
168-
Owns(&corev1.Service{}).
164+
For(&v1alpha1.Storage{},
165+
builder.WithPredicates(predicate.GenerationChangedPredicate{}),
166+
).
167+
Owns(&v1alpha1.RemoteStorageNodeSet{},
168+
builder.WithPredicates(resources.LastAppliedAnnotationPredicate()), // TODO: YDBOPS-9194
169+
).
170+
Owns(&v1alpha1.StorageNodeSet{},
171+
builder.WithPredicates(resources.LastAppliedAnnotationPredicate()), // TODO: YDBOPS-9194
172+
).
173+
Owns(&appsv1.StatefulSet{},
174+
builder.WithPredicates(predicate.GenerationChangedPredicate{}),
175+
).
176+
Owns(&corev1.ConfigMap{},
177+
builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}),
178+
).
179+
Owns(&corev1.Service{},
180+
builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}),
181+
).
169182
Watches(
170183
&source.Kind{Type: &corev1.Secret{}},
171184
handler.EnqueueRequestsFromMapFunc(r.findStoragesForSecret),
185+
builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}),
172186
).
173-
WithEventFilter(predicate.Or(
174-
predicate.GenerationChangedPredicate{},
175-
resources.LastAppliedAnnotationPredicate(),
176-
resources.IsServicePredicate(),
177-
resources.IsSecretPredicate(),
178-
)).
179-
WithEventFilter(resources.IgnoreDeletetionPredicate()).
187+
WithEventFilter(resources.IsStorageCreatePredicate()).
188+
WithEventFilter(resources.IgnoreDeleteStateUnknownPredicate()).
180189
Complete(r)
181190
}
182191

‎internal/controllers/storage/controller_test.go

+9
Original file line numberDiff line numberDiff line change
@@ -186,6 +186,15 @@ var _ = Describe("Storage controller medium tests", func() {
186186
},
187187
},
188188
))
189+
190+
By("check that delete StatefulSet event was detected...")
191+
Expect(k8sClient.List(ctx, &foundStatefulSets, client.InNamespace(testobjects.YdbNamespace))).ShouldNot(HaveOccurred())
192+
Expect(len(foundStatefulSets.Items)).Should(Equal(1))
193+
Expect(k8sClient.Delete(ctx, &foundStatefulSets.Items[0])).ShouldNot(HaveOccurred())
194+
Eventually(func() int {
195+
Expect(k8sClient.List(ctx, &foundStatefulSets, client.InNamespace(testobjects.YdbNamespace))).ShouldNot(HaveOccurred())
196+
return len(foundStatefulSets.Items)
197+
}, test.Timeout, test.Interval).Should(Equal(1))
189198
})
190199
})
191200
})

‎internal/controllers/storage/init.go

+12
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,12 @@ func (r *Reconciler) initializeBlobstorage(
119119
return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err
120120
}
121121

122+
r.Recorder.Event(
123+
storage,
124+
corev1.EventTypeNormal,
125+
"InitializingStorage",
126+
fmt.Sprintf("Successfully created Job %s", fmt.Sprintf(resources.InitJobNameFormat, storage.Name)),
127+
)
122128
return Stop, ctrl.Result{RequeueAfter: StorageInitializationRequeueDelay}, nil
123129
}
124130

@@ -207,6 +213,12 @@ func (r *Reconciler) initializeBlobstorage(
207213
return r.updateStatus(ctx, storage, StatusUpdateRequeueDelay)
208214
}
209215

216+
r.Recorder.Event(
217+
storage,
218+
corev1.EventTypeNormal,
219+
"InitializingStorage",
220+
fmt.Sprintf("Waiting for Job %s status update", initJob.Name),
221+
)
210222
return Stop, ctrl.Result{RequeueAfter: StorageInitializationRequeueDelay}, nil
211223
}
212224

‎internal/controllers/storagenodeset/controller.go

+8-6
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"k8s.io/client-go/rest"
1111
"k8s.io/client-go/tools/record"
1212
ctrl "sigs.k8s.io/controller-runtime"
13+
"sigs.k8s.io/controller-runtime/pkg/builder"
1314
"sigs.k8s.io/controller-runtime/pkg/client"
1415
"sigs.k8s.io/controller-runtime/pkg/log"
1516
"sigs.k8s.io/controller-runtime/pkg/predicate"
@@ -65,12 +66,13 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error {
6566
controller := ctrl.NewControllerManagedBy(mgr)
6667

6768
return controller.
68-
For(&v1alpha1.StorageNodeSet{}).
69-
Owns(&appsv1.StatefulSet{}).
70-
WithEventFilter(predicate.Or(
71-
predicate.GenerationChangedPredicate{},
72-
resources.LastAppliedAnnotationPredicate()),
69+
For(&v1alpha1.StorageNodeSet{},
70+
builder.WithPredicates(predicate.GenerationChangedPredicate{}),
7371
).
74-
WithEventFilter(resources.IgnoreDeletetionPredicate()).
72+
Owns(&appsv1.StatefulSet{},
73+
builder.WithPredicates(predicate.GenerationChangedPredicate{}),
74+
).
75+
WithEventFilter(resources.IsStorageNodeSetCreatePredicate()).
76+
WithEventFilter(resources.IgnoreDeleteStateUnknownPredicate()).
7577
Complete(r)
7678
}

‎internal/resources/databasenodeset.go

+4
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,10 @@ func (b *DatabaseNodeSetResource) GetResourceBuilders(restConfig *rest.Config) [
8888
func NewDatabaseNodeSet(databaseNodeSet *api.DatabaseNodeSet) DatabaseNodeSetResource {
8989
crDatabaseNodeSet := databaseNodeSet.DeepCopy()
9090

91+
if crDatabaseNodeSet.Spec.Service.Status.TLSConfiguration == nil {
92+
crDatabaseNodeSet.Spec.Service.Status.TLSConfiguration = &api.TLSConfiguration{Enabled: false}
93+
}
94+
9195
return DatabaseNodeSetResource{DatabaseNodeSet: crDatabaseNodeSet}
9296
}
9397

0 commit comments

Comments
 (0)