Skip to content

Commit df75d2b

Browse files
nikitkanikitka
committed
add ip discovery tests, extract deleteAll function to k8s_helpers
1 parent f83de6c commit df75d2b

File tree

5 files changed

+198
-225
lines changed

5 files changed

+198
-225
lines changed

‎internal/controllers/database/controller_test.go

+78-1
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import (
1717
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1818
"k8s.io/apimachinery/pkg/types"
1919
"sigs.k8s.io/controller-runtime/pkg/client"
20+
"sigs.k8s.io/controller-runtime/pkg/envtest"
2021
"sigs.k8s.io/controller-runtime/pkg/manager"
2122

2223
"github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1"
@@ -30,12 +31,13 @@ import (
3031
var (
3132
k8sClient client.Client
3233
ctx context.Context
34+
env *envtest.Environment
3335
)
3436

3537
func TestAPIs(t *testing.T) {
3638
RegisterFailHandler(Fail)
3739

38-
test.SetupK8STestManager(&ctx, &k8sClient, func(mgr *manager.Manager) []test.Reconciler {
40+
env = test.SetupK8STestManager(&ctx, &k8sClient, func(mgr *manager.Manager) []test.Reconciler {
3941
return []test.Reconciler{
4042
&storage.Reconciler{
4143
Client: k8sClient,
@@ -94,6 +96,7 @@ var _ = Describe("Database controller medium tests", func() {
9496
AfterEach(func() {
9597
Expect(k8sClient.Delete(ctx, &storageSample)).Should(Succeed())
9698
Expect(k8sClient.Delete(ctx, &namespace)).Should(Succeed())
99+
test.DeleteAllObjects(env, k8sClient, &namespace)
97100
})
98101

99102
It("Checking field propagation to objects", func() {
@@ -217,4 +220,78 @@ var _ = Describe("Database controller medium tests", func() {
217220
return reflect.DeepEqual(encryptionData, encryptionSecret.Data)
218221
}, test.Timeout, test.Interval).Should(BeTrue())
219222
})
223+
224+
It("Check iPDiscovery flag works", func() {
225+
getDBSts := func(generation int64) appsv1.StatefulSet {
226+
sts := appsv1.StatefulSet{}
227+
228+
Eventually(
229+
func() error {
230+
objectKey := types.NamespacedName{
231+
Name: testobjects.DatabaseName,
232+
Namespace: testobjects.YdbNamespace,
233+
}
234+
err := k8sClient.Get(ctx, objectKey, &sts)
235+
if err != nil {
236+
return err
237+
}
238+
239+
if sts.Generation <= generation {
240+
return fmt.Errorf("sts is too old (generation=%d)", sts.Generation)
241+
}
242+
return nil
243+
},
244+
).WithTimeout(test.Timeout).WithPolling(test.Interval).Should(Succeed())
245+
246+
return sts
247+
}
248+
getDB := func() v1alpha1.Database {
249+
found := v1alpha1.Database{}
250+
Expect(k8sClient.Get(ctx, types.NamespacedName{
251+
Name: testobjects.DatabaseName,
252+
Namespace: testobjects.YdbNamespace,
253+
}, &found)).Should(Succeed())
254+
255+
return found
256+
}
257+
258+
db := *testobjects.DefaultDatabase()
259+
260+
Expect(k8sClient.Create(ctx, &db)).Should(Succeed())
261+
262+
sts := getDBSts(0)
263+
args := sts.Spec.Template.Spec.Containers[0].Args
264+
265+
Expect(args).To(ContainElements([]string{"--grpc-public-host"}))
266+
Expect(args).ToNot(ContainElements([]string{"--grpc-public-address-v6", "--grpc-public-address-v4", "--grpc-public-target-name-override"}))
267+
268+
db = getDB()
269+
db.Spec.Service.GRPC.IPDiscovery = &v1alpha1.IPDiscovery{
270+
Enabled: true,
271+
IPFamily: corev1.IPv6Protocol,
272+
}
273+
274+
Expect(k8sClient.Update(ctx, &db)).Should(Succeed())
275+
276+
sts = getDBSts(sts.Generation)
277+
args = sts.Spec.Template.Spec.Containers[0].Args
278+
279+
Expect(args).To(ContainElements([]string{"--grpc-public-address-v6"}))
280+
Expect(args).ToNot(ContainElements([]string{"--grpc-public-target-name-override"}))
281+
282+
db = getDB()
283+
284+
db.Spec.Service.GRPC.IPDiscovery = &v1alpha1.IPDiscovery{
285+
Enabled: true,
286+
IPFamily: corev1.IPv4Protocol,
287+
TargetNameOverride: "a.b.c.d",
288+
}
289+
290+
Expect(k8sClient.Update(ctx, &db)).Should(Succeed())
291+
292+
sts = getDBSts(sts.Generation)
293+
args = sts.Spec.Template.Spec.Containers[0].Args
294+
295+
Expect(args).To(ContainElements([]string{"--grpc-public-address-v4", "--grpc-public-target-name-override"}))
296+
})
220297
})

‎internal/controllers/remotedatabasenodeset/controller_test.go

+2-110
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,7 @@ import (
1414
apierrors "k8s.io/apimachinery/pkg/api/errors"
1515
"k8s.io/apimachinery/pkg/api/meta"
1616
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
17-
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
18-
"k8s.io/apimachinery/pkg/runtime/schema"
1917
"k8s.io/apimachinery/pkg/types"
20-
"k8s.io/client-go/kubernetes"
2118
"k8s.io/kubectl/pkg/scheme"
2219
ctrl "sigs.k8s.io/controller-runtime"
2320
"sigs.k8s.io/controller-runtime/pkg/cache"
@@ -349,8 +346,8 @@ var _ = Describe("RemoteDatabaseNodeSet controller tests", func() {
349346
AfterEach(func() {
350347
Expect(localClient.Delete(ctx, databaseSample)).Should(Succeed())
351348
Expect(localClient.Delete(ctx, storageSample)).Should(Succeed())
352-
deleteAll(localEnv, localClient, &localNamespace)
353-
deleteAll(remoteEnv, remoteClient, &localNamespace)
349+
test.DeleteAllObjects(localEnv, localClient, &localNamespace)
350+
test.DeleteAllObjects(remoteEnv, remoteClient, &localNamespace)
354351
})
355352

356353
When("Created RemoteDatabaseNodeSet in k8s-mgmt-cluster", func() {
@@ -892,108 +889,3 @@ var _ = Describe("RemoteDatabaseNodeSet controller tests", func() {
892889
})
893890
})
894891
})
895-
896-
func deleteAll(env *envtest.Environment, k8sClient client.Client, objs ...client.Object) {
897-
for _, obj := range objs {
898-
ctx := context.Background()
899-
clientGo, err := kubernetes.NewForConfig(env.Config)
900-
Expect(err).ShouldNot(HaveOccurred())
901-
Expect(client.IgnoreNotFound(k8sClient.Delete(ctx, obj))).Should(Succeed())
902-
903-
//nolint:nestif
904-
if ns, ok := obj.(*corev1.Namespace); ok {
905-
// Normally the kube-controller-manager would handle finalization
906-
// and garbage collection of namespaces, but with envtest, we aren't
907-
// running a kube-controller-manager. Instead we're gonna approximate
908-
// (poorly) the kube-controller-manager by explicitly deleting some
909-
// resources within the namespace and then removing the `kubernetes`
910-
// finalizer from the namespace resource so it can finish deleting.
911-
// Note that any resources within the namespace that we don't
912-
// successfully delete could reappear if the namespace is ever
913-
// recreated with the same name.
914-
915-
// Look up all namespaced resources under the discovery API
916-
_, apiResources, err := clientGo.Discovery().ServerGroupsAndResources()
917-
Expect(err).ShouldNot(HaveOccurred())
918-
namespacedGVKs := make(map[string]schema.GroupVersionKind)
919-
for _, apiResourceList := range apiResources {
920-
defaultGV, err := schema.ParseGroupVersion(apiResourceList.GroupVersion)
921-
Expect(err).ShouldNot(HaveOccurred())
922-
for _, r := range apiResourceList.APIResources {
923-
if !r.Namespaced || strings.Contains(r.Name, "/") {
924-
// skip non-namespaced and subresources
925-
continue
926-
}
927-
gvk := schema.GroupVersionKind{
928-
Group: defaultGV.Group,
929-
Version: defaultGV.Version,
930-
Kind: r.Kind,
931-
}
932-
if r.Group != "" {
933-
gvk.Group = r.Group
934-
}
935-
if r.Version != "" {
936-
gvk.Version = r.Version
937-
}
938-
namespacedGVKs[gvk.String()] = gvk
939-
}
940-
}
941-
942-
// Delete all namespaced resources in this namespace
943-
for _, gvk := range namespacedGVKs {
944-
var u unstructured.Unstructured
945-
u.SetGroupVersionKind(gvk)
946-
err := k8sClient.DeleteAllOf(ctx, &u, client.InNamespace(ns.Name))
947-
Expect(client.IgnoreNotFound(ignoreMethodNotAllowed(err))).ShouldNot(HaveOccurred())
948-
}
949-
950-
// Delete all Services in this namespace
951-
serviceList := corev1.ServiceList{}
952-
err = k8sClient.List(ctx, &serviceList, client.InNamespace(ns.Name))
953-
Expect(err).ShouldNot(HaveOccurred())
954-
for idx := range serviceList.Items {
955-
policy := metav1.DeletePropagationForeground
956-
err = k8sClient.Delete(ctx, &serviceList.Items[idx], &client.DeleteOptions{PropagationPolicy: &policy})
957-
Expect(err).ShouldNot(HaveOccurred())
958-
}
959-
960-
Eventually(func() error {
961-
key := client.ObjectKeyFromObject(ns)
962-
if err := k8sClient.Get(ctx, key, ns); err != nil {
963-
return client.IgnoreNotFound(err)
964-
}
965-
// remove `kubernetes` finalizer
966-
const kubernetes = "kubernetes"
967-
finalizers := []corev1.FinalizerName{}
968-
for _, f := range ns.Spec.Finalizers {
969-
if f != kubernetes {
970-
finalizers = append(finalizers, f)
971-
}
972-
}
973-
ns.Spec.Finalizers = finalizers
974-
975-
// We have to use the k8s.io/client-go library here to expose
976-
// ability to patch the /finalize subresource on the namespace
977-
_, err = clientGo.CoreV1().Namespaces().Finalize(ctx, ns, metav1.UpdateOptions{})
978-
return err
979-
}, test.Timeout, test.Interval).Should(Succeed())
980-
}
981-
982-
Eventually(func() metav1.StatusReason {
983-
key := client.ObjectKeyFromObject(obj)
984-
if err := k8sClient.Get(ctx, key, obj); err != nil {
985-
return apierrors.ReasonForError(err)
986-
}
987-
return ""
988-
}, test.Timeout, test.Interval).Should(Equal(metav1.StatusReasonNotFound))
989-
}
990-
}
991-
992-
func ignoreMethodNotAllowed(err error) error {
993-
if err != nil {
994-
if apierrors.ReasonForError(err) == metav1.StatusReasonMethodNotAllowed {
995-
return nil
996-
}
997-
}
998-
return err
999-
}

‎internal/controllers/remotestoragenodeset/controller_test.go

+2-111
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@ import (
55
"fmt"
66
"path/filepath"
77
"reflect"
8-
"strings"
98
"testing"
109

1110
. "github.com/onsi/ginkgo/v2"
@@ -14,10 +13,7 @@ import (
1413
apierrors "k8s.io/apimachinery/pkg/api/errors"
1514
"k8s.io/apimachinery/pkg/api/meta"
1615
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
17-
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
18-
"k8s.io/apimachinery/pkg/runtime/schema"
1916
"k8s.io/apimachinery/pkg/types"
20-
"k8s.io/client-go/kubernetes"
2117
"k8s.io/kubectl/pkg/scheme"
2218
ctrl "sigs.k8s.io/controller-runtime"
2319
"sigs.k8s.io/controller-runtime/pkg/cache"
@@ -280,8 +276,8 @@ var _ = Describe("RemoteStorageNodeSet controller tests", func() {
280276

281277
AfterEach(func() {
282278
Expect(localClient.Delete(ctx, storageSample)).Should(Succeed())
283-
deleteAll(localEnv, localClient, &localNamespace)
284-
deleteAll(remoteEnv, remoteClient, &localNamespace)
279+
test.DeleteAllObjects(localEnv, localClient, &localNamespace)
280+
test.DeleteAllObjects(remoteEnv, remoteClient, &localNamespace)
285281
})
286282

287283
When("Created RemoteStorageNodeSet in k8s-mgmt-cluster", func() {
@@ -635,108 +631,3 @@ var _ = Describe("RemoteStorageNodeSet controller tests", func() {
635631
})
636632
})
637633
})
638-
639-
func deleteAll(env *envtest.Environment, k8sClient client.Client, objs ...client.Object) {
640-
for _, obj := range objs {
641-
ctx := context.Background()
642-
clientGo, err := kubernetes.NewForConfig(env.Config)
643-
Expect(err).ShouldNot(HaveOccurred())
644-
Expect(client.IgnoreNotFound(k8sClient.Delete(ctx, obj))).Should(Succeed())
645-
646-
//nolint:nestif
647-
if ns, ok := obj.(*corev1.Namespace); ok {
648-
// Normally the kube-controller-manager would handle finalization
649-
// and garbage collection of namespaces, but with envtest, we aren't
650-
// running a kube-controller-manager. Instead we're gonna approximate
651-
// (poorly) the kube-controller-manager by explicitly deleting some
652-
// resources within the namespace and then removing the `kubernetes`
653-
// finalizer from the namespace resource so it can finish deleting.
654-
// Note that any resources within the namespace that we don't
655-
// successfully delete could reappear if the namespace is ever
656-
// recreated with the same name.
657-
658-
// Look up all namespaced resources under the discovery API
659-
_, apiResources, err := clientGo.Discovery().ServerGroupsAndResources()
660-
Expect(err).ShouldNot(HaveOccurred())
661-
namespacedGVKs := make(map[string]schema.GroupVersionKind)
662-
for _, apiResourceList := range apiResources {
663-
defaultGV, err := schema.ParseGroupVersion(apiResourceList.GroupVersion)
664-
Expect(err).ShouldNot(HaveOccurred())
665-
for _, r := range apiResourceList.APIResources {
666-
if !r.Namespaced || strings.Contains(r.Name, "/") {
667-
// skip non-namespaced and subresources
668-
continue
669-
}
670-
gvk := schema.GroupVersionKind{
671-
Group: defaultGV.Group,
672-
Version: defaultGV.Version,
673-
Kind: r.Kind,
674-
}
675-
if r.Group != "" {
676-
gvk.Group = r.Group
677-
}
678-
if r.Version != "" {
679-
gvk.Version = r.Version
680-
}
681-
namespacedGVKs[gvk.String()] = gvk
682-
}
683-
}
684-
685-
// Delete all namespaced resources in this namespace
686-
for _, gvk := range namespacedGVKs {
687-
var u unstructured.Unstructured
688-
u.SetGroupVersionKind(gvk)
689-
err := k8sClient.DeleteAllOf(ctx, &u, client.InNamespace(ns.Name))
690-
Expect(client.IgnoreNotFound(ignoreMethodNotAllowed(err))).ShouldNot(HaveOccurred())
691-
}
692-
693-
// Delete all Services in this namespace
694-
serviceList := corev1.ServiceList{}
695-
err = k8sClient.List(ctx, &serviceList, client.InNamespace(ns.Name))
696-
Expect(err).ShouldNot(HaveOccurred())
697-
for idx := range serviceList.Items {
698-
policy := metav1.DeletePropagationForeground
699-
err = k8sClient.Delete(ctx, &serviceList.Items[idx], &client.DeleteOptions{PropagationPolicy: &policy})
700-
Expect(err).ShouldNot(HaveOccurred())
701-
}
702-
703-
Eventually(func() error {
704-
key := client.ObjectKeyFromObject(ns)
705-
if err := k8sClient.Get(ctx, key, ns); err != nil {
706-
return client.IgnoreNotFound(err)
707-
}
708-
// remove `kubernetes` finalizer
709-
const kubernetes = "kubernetes"
710-
finalizers := []corev1.FinalizerName{}
711-
for _, f := range ns.Spec.Finalizers {
712-
if f != kubernetes {
713-
finalizers = append(finalizers, f)
714-
}
715-
}
716-
ns.Spec.Finalizers = finalizers
717-
718-
// We have to use the k8s.io/client-go library here to expose
719-
// ability to patch the /finalize subresource on the namespace
720-
_, err = clientGo.CoreV1().Namespaces().Finalize(ctx, ns, metav1.UpdateOptions{})
721-
return err
722-
}, test.Timeout, test.Interval).Should(Succeed())
723-
}
724-
725-
Eventually(func() metav1.StatusReason {
726-
key := client.ObjectKeyFromObject(obj)
727-
if err := k8sClient.Get(ctx, key, obj); err != nil {
728-
return apierrors.ReasonForError(err)
729-
}
730-
return ""
731-
}, test.Timeout, test.Interval).Should(Equal(metav1.StatusReasonNotFound))
732-
}
733-
}
734-
735-
func ignoreMethodNotAllowed(err error) error {
736-
if err != nil {
737-
if apierrors.ReasonForError(err) == metav1.StatusReasonMethodNotAllowed {
738-
return nil
739-
}
740-
}
741-
return err
742-
}

‎internal/resources/database_statefulset.go

+1-2
Original file line numberDiff line numberDiff line change
@@ -642,7 +642,7 @@ func (b *DatabaseStatefulSetBuilder) buildContainerArgs() ([]string, []string) {
642642
publicHost = value
643643
}
644644

645-
if b.Spec.Service.GRPC.IPDiscovery != nil && b.Spec.Service.GRPC.IPDiscovery.Enabled == true {
645+
if b.Spec.Service.GRPC.IPDiscovery != nil && b.Spec.Service.GRPC.IPDiscovery.Enabled {
646646
targetNameOverride := b.Spec.Service.GRPC.IPDiscovery.TargetNameOverride
647647
ipFamilyArg := "--grpc-public-address-v4"
648648

@@ -663,7 +663,6 @@ func (b *DatabaseStatefulSetBuilder) buildContainerArgs() ([]string, []string) {
663663
fmt.Sprintf("%s.%s", "$(NODE_NAME)", targetNameOverride),
664664
)
665665
}
666-
667666
}
668667

669668
publicPortOption := "--grpc-public-port"

0 commit comments

Comments
 (0)