From b14b8b35ad5d0c3a2d4c4d27a7c781beca39e2f9 Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Tue, 30 Jun 2020 12:13:41 +0100 Subject: [PATCH 01/34] testing --- deploy/crds/mongodb.com_v1_mongodb_cr.yaml | 6 ++++++ pkg/controller/mongodb/mongodb_controller.go | 3 +++ 2 files changed, 9 insertions(+) diff --git a/deploy/crds/mongodb.com_v1_mongodb_cr.yaml b/deploy/crds/mongodb.com_v1_mongodb_cr.yaml index a1bc63db2..f25014ad0 100644 --- a/deploy/crds/mongodb.com_v1_mongodb_cr.yaml +++ b/deploy/crds/mongodb.com_v1_mongodb_cr.yaml @@ -6,3 +6,9 @@ spec: members: 3 type: ReplicaSet version: "4.2.6" + statefulset: + spec: + template: + metadata: + annotations: + key1: value1 diff --git a/pkg/controller/mongodb/mongodb_controller.go b/pkg/controller/mongodb/mongodb_controller.go index d397ebf9c..383bbc10d 100644 --- a/pkg/controller/mongodb/mongodb_controller.go +++ b/pkg/controller/mongodb/mongodb_controller.go @@ -245,6 +245,8 @@ func (r *ReplicaSetReconciler) createOrUpdateStatefulSet(mdb mdbv1.MongoDB) erro if err != nil { return fmt.Errorf("error getting StatefulSet: %s", err) } + + fmt.Println("resource: ", mdb) buildStatefulSetModificationFunction(mdb)(&set) if err = r.client.CreateOrUpdate(&set); err != nil { return fmt.Errorf("error creating/updating StatefulSet: %s", err) @@ -399,6 +401,7 @@ func getUpdateStrategyType(mdb mdbv1.MongoDB) appsv1.StatefulSetUpdateStrategyTy func buildStatefulSet(mdb mdbv1.MongoDB) (appsv1.StatefulSet, error) { sts := appsv1.StatefulSet{} buildStatefulSetModificationFunction(mdb)(&sts) + return sts, nil } From fab49ac7880cf6eaf4afd9c5dc0b07f6c3fc527e Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Wed, 15 Jul 2020 09:08:09 +0100 Subject: [PATCH 02/34] temp --- deploy/crds/mongodb.com_v1_mongodb_cr.yaml | 9 +- deploy/operator.yaml | 2 +- pkg/apis/mongodb/v1/mongodb_types.go | 10 + pkg/controller/mongodb/mongodb_controller.go | 14 +- pkg/kube/statefulset/statefulset.go | 504 +++++++++++++++++++ 5 files changed, 533 insertions(+), 6 deletions(-) diff --git a/deploy/crds/mongodb.com_v1_mongodb_cr.yaml b/deploy/crds/mongodb.com_v1_mongodb_cr.yaml index f25014ad0..9493d22d0 100644 --- a/deploy/crds/mongodb.com_v1_mongodb_cr.yaml +++ b/deploy/crds/mongodb.com_v1_mongodb_cr.yaml @@ -8,7 +8,8 @@ spec: version: "4.2.6" statefulset: spec: - template: - metadata: - annotations: - key1: value1 + volumeClaimTemplates: + - metadata: + name: data-volume-example-mongodb-0 + spec: + storageClassName: manual diff --git a/deploy/operator.yaml b/deploy/operator.yaml index 64b2d56a7..027ab2365 100644 --- a/deploy/operator.yaml +++ b/deploy/operator.yaml @@ -15,7 +15,7 @@ spec: serviceAccountName: mongodb-kubernetes-operator containers: - name: mongodb-kubernetes-operator - image: quay.io/mongodb/mongodb-kubernetes-operator:0.0.7 + image: localhost:5000/mongodb-kubernetes-operator command: - mongodb-kubernetes-operator imagePullPolicy: Always diff --git a/pkg/apis/mongodb/v1/mongodb_types.go b/pkg/apis/mongodb/v1/mongodb_types.go index cde03a16d..959058b79 100644 --- a/pkg/apis/mongodb/v1/mongodb_types.go +++ b/pkg/apis/mongodb/v1/mongodb_types.go @@ -4,6 +4,7 @@ import ( "fmt" "strings" + appsv1 "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/types" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -40,6 +41,15 @@ type MongoDBSpec struct { // Security configures security features, such as TLS, and authentication settings for a deployment // +optional Security Security `json:"security"` + + // +optional + StatefulSetConfiguration StatefulSetConfiguration `json:"statefulset,omitempty"` +} + +// StatefulSetConfiguration holds the optional custom StatefulSet +// that should be merged into the operator created one. +type StatefulSetConfiguration struct { + Spec appsv1.StatefulSetSpec `json:"spec"` } type Security struct { diff --git a/pkg/controller/mongodb/mongodb_controller.go b/pkg/controller/mongodb/mongodb_controller.go index 301ffbca8..3f14f1615 100644 --- a/pkg/controller/mongodb/mongodb_controller.go +++ b/pkg/controller/mongodb/mongodb_controller.go @@ -9,6 +9,7 @@ import ( "os" "time" + "github.com/kylelemons/godebug/pretty" "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/persistentvolumeclaim" "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/probes" @@ -243,6 +244,8 @@ func (r *ReplicaSetReconciler) resetStatefulSetUpdateStrategy(mdb mdbv1.MongoDB) // is currently ready. func (r *ReplicaSetReconciler) isStatefulSetReady(mdb mdbv1.MongoDB, existingStatefulSet *appsv1.StatefulSet) (bool, error) { stsFunc := buildStatefulSetModificationFunction(mdb) + fmt.Fprintf(os.Stderr, "\n\nExisting STS Spec: %+v\n\n", existingStatefulSet.Spec) + spec1 := existingStatefulSet.Spec stsCopy := existingStatefulSet.DeepCopyObject() stsFunc(existingStatefulSet) stsCopyBytes, err := json.Marshal(stsCopy) @@ -255,6 +258,9 @@ func (r *ReplicaSetReconciler) isStatefulSetReady(mdb mdbv1.MongoDB, existingSta return false, err } + fmt.Fprintf(os.Stderr, "\n\nModified STS Spec: %+v\n\n", existingStatefulSet.Spec) + + fmt.Fprintf(os.Stderr, "\n\nDifferences: %+v", pretty.Compare(spec1, existingStatefulSet.Spec)) //comparison is done with bytes instead of reflect.DeepEqual as there are //some issues with nil/empty maps not being compared correctly otherwise areEqual := bytes.Equal(stsCopyBytes, stsBytes) @@ -293,13 +299,17 @@ func (r *ReplicaSetReconciler) ensureService(mdb mdbv1.MongoDB) error { func (r *ReplicaSetReconciler) createOrUpdateStatefulSet(mdb mdbv1.MongoDB) error { set := appsv1.StatefulSet{} err := r.client.Get(context.TODO(), mdb.NamespacedName(), &set) + + newMdb := &mdbv1.MongoDB{} + _ = r.client.Get(context.TODO(), mdb.NamespacedName(), newMdb) + fmt.Fprintf(os.Stderr, "\n\nSPEC: %+v\n\n", mdb.Spec.StatefulSetConfiguration.Spec) err = k8sClient.IgnoreNotFound(err) if err != nil { return fmt.Errorf("error getting StatefulSet: %s", err) } - fmt.Println("resource: ", mdb) buildStatefulSetModificationFunction(mdb)(&set) + fmt.Fprintf(os.Stderr, "\n\nSPEC: %+v\n\n", set.Spec) if err = statefulset.CreateOrUpdate(r.client, set); err != nil { return fmt.Errorf("error creating/updating StatefulSet: %s", err) } @@ -575,6 +585,7 @@ mongod -f /data/automation-mongod.conf ; } func buildStatefulSetModificationFunction(mdb mdbv1.MongoDB) statefulset.Modification { + labels := map[string]string{ "app": mdb.ServiceName(), } @@ -627,6 +638,7 @@ func buildStatefulSetModificationFunction(mdb mdbv1.MongoDB) statefulset.Modific buildScramPodSpecModification(mdb), ), ), + statefulset.WithStatefulSetSpec(mdb.Spec.StatefulSetConfiguration.Spec), ) } diff --git a/pkg/kube/statefulset/statefulset.go b/pkg/kube/statefulset/statefulset.go index c49e03d91..a35af2afa 100644 --- a/pkg/kube/statefulset/statefulset.go +++ b/pkg/kube/statefulset/statefulset.go @@ -1,6 +1,9 @@ package statefulset import ( + "sort" + + "github.com/imdario/mergo" apiErrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -250,6 +253,417 @@ func WithVolumeClaim(name string, f func(*corev1.PersistentVolumeClaim)) Modific } } +func mergedValue(original, modified interface{}) interface{} { + if modified != nil { + return modified + } + return original +} + +func mergedString(original, modified string) string { + if modified != "" { + return modified + } + return original +} + +func mergedLabelSelector(original, modified *metav1.LabelSelector) *metav1.LabelSelector { + if modified == nil { + return original + } + for key, val := range modified.MatchLabels { + original.MatchLabels[key] = val + } + // Selector.MatchExpressions + for _, expression := range modified.MatchExpressions { + if index := findMatchExpressionByKey(expression.Key, original.MatchExpressions); index == notFound { + // Add + original.MatchExpressions = append(original.MatchExpressions, expression) + } else { + original.MatchExpressions[index] = expression + } + } + return original +} + +func mergedContainer(original, modified corev1.Container) corev1.Container { + // args + if len(modified.Args) > 0 { + original.Args = modified.Args + } + //command + if len(modified.Command) > 0 { + original.Command = modified.Command + } + + // env + for _, envVar := range modified.Env { + if index := findEnvVarByName(envVar.Name, original.Env); index == notFound { + original.Env = append(original.Env, envVar) + } else { + original.Env[index] = envVar + } + } + + // envFrom + if len(modified.EnvFrom) > 0 { + original.EnvFrom = modified.EnvFrom + } + + // image and imagepullpolicy + original.Image = mergedString(original.Image, modified.Image) + if modified.ImagePullPolicy != "" { + original.ImagePullPolicy = modified.ImagePullPolicy + } + + // lifecycle + if val, ok := mergedValue(original.Lifecycle, modified.Lifecycle).(*corev1.Lifecycle); ok { + original.Lifecycle = val + } + + //lifevenessprobe + if val, ok := mergedValue(original.LivenessProbe, modified.LivenessProbe).(*corev1.Probe); ok { + original.LivenessProbe = val + } + + //ports + for _, port := range modified.Ports { + if index := findContainerPortByPort(port.ContainerPort, original.Ports); index == notFound { + original.Ports = append(original.Ports, port) + } else { + original.Ports[index].HostIP = mergedString(original.Ports[index].HostIP, port.HostIP) + original.Ports[index].Name = mergedString(original.Ports[index].Name, port.Name) + if port.Protocol != "" { + original.Ports[index].Protocol = port.Protocol + } + if port.HostPort != 0 { + original.Ports[index].HostPort = port.HostPort + } + } + } + + //ReadinessProbe + if val, ok := mergedValue(original.ReadinessProbe, modified.ReadinessProbe).(*corev1.Probe); ok { + original.ReadinessProbe = val + } + + // resources - limits + if len(modified.Resources.Limits) > 0 { + original.Resources.Limits = modified.Resources.Limits + } + + // resources - requests + if len(modified.Resources.Requests) > 0 { + original.Resources.Requests = modified.Resources.Requests + } + + //SecurityContext + if val, ok := mergedValue(original.SecurityContext, modified.SecurityContext).(*corev1.SecurityContext); ok { + original.SecurityContext = val + } + + //StartupProbe + if val, ok := mergedValue(original.StartupProbe, modified.StartupProbe).(*corev1.Probe); ok { + original.StartupProbe = val + } + + // stdin + original.Stdin = modified.Stdin + //stdinOnce + original.StdinOnce = modified.StdinOnce + + // terminationMessagePath + original.TerminationMessagePath = mergedString(original.TerminationMessagePath, modified.TerminationMessagePath) + + // terminationMEssagePolicy + if modified.TerminationMessagePolicy != "" { + original.TerminationMessagePolicy = modified.TerminationMessagePolicy + } + + //TTY + original.TTY = modified.TTY + + //volumedevices + for _, device := range modified.VolumeDevices { + if index := findVolumeDeviceByName(device.Name, original.VolumeDevices); index == notFound { + original.VolumeDevices = append(original.VolumeDevices, device) + } else { + original.VolumeDevices[index].DevicePath = mergedString(original.VolumeDevices[index].DevicePath, device.DevicePath) + } + } + + //volumemounts + for _, mount := range modified.VolumeMounts { + if index := findVolumeMountsByMountPath(mount.MountPath, original.VolumeMounts); index == notFound { + original.VolumeMounts = append(original.VolumeMounts, mount) + } else { + original.VolumeMounts[index].Name = mergedString(original.VolumeMounts[index].Name, mount.Name) + original.VolumeMounts[index].SubPath = mergedString(original.VolumeMounts[index].SubPath, mount.SubPath) + original.VolumeMounts[index].SubPathExpr = mergedString(original.VolumeMounts[index].SubPathExpr, mount.SubPathExpr) + original.VolumeMounts[index].ReadOnly = mount.ReadOnly + if val, ok := mergedValue(original.VolumeMounts[index].MountPropagation, mount.MountPropagation).(*corev1.MountPropagationMode); ok { + original.VolumeMounts[index].MountPropagation = val + } + } + } + + //workingdir + original.WorkingDir = mergedString(original.WorkingDir, modified.WorkingDir) + + return original +} + +func createVolumeClaimMap(volumeMounts []corev1.PersistentVolumeClaim) map[string]corev1.PersistentVolumeClaim { + mountMap := make(map[string]corev1.PersistentVolumeClaim) + for _, m := range volumeMounts { + mountMap[m.Name] = m + } + return mountMap +} + +func mergeVolumeClaimTemplates(original, modified []corev1.PersistentVolumeClaim) []corev1.PersistentVolumeClaim { + defaultMountsMap := createVolumeClaimMap(original) + customMountsMap := createVolumeClaimMap(modified) + var mergedVolumes []corev1.PersistentVolumeClaim + for _, defaultMount := range defaultMountsMap { + if customMount, ok := customMountsMap[defaultMount.Name]; ok { + // needs merge + if err := mergo.Merge(&defaultMount, customMount, mergo.WithAppendSlice, mergo.WithOverride); err != nil { + return nil + } + } + mergedVolumes = append(mergedVolumes, defaultMount) + } + for _, customMount := range customMountsMap { + if _, ok := defaultMountsMap[customMount.Name]; ok { + // already merged + continue + } + mergedVolumes = append(mergedVolumes, customMount) + } + sort.SliceStable(mergedVolumes, func(i, j int) bool { + return mergedVolumes[i].Name < mergedVolumes[j].Name + }) + return mergedVolumes +} + +// Merges the change provided by the user on top of what we already built +func WithStatefulSetSpec(spec appsv1.StatefulSetSpec) Modification { + return func(sts *appsv1.StatefulSet) { + // Replicas + if spec.Replicas != nil { + sts.Spec.Replicas = spec.Replicas + } + // Selector + sts.Spec.Selector = mergedLabelSelector(sts.Spec.Selector, spec.Selector) + + // ServiceName + if spec.ServiceName != "" { + sts.Spec.ServiceName = spec.ServiceName + } + // RevisionHistoryLimit + if spec.RevisionHistoryLimit != nil { + sts.Spec.RevisionHistoryLimit = spec.RevisionHistoryLimit + } + // UpdateStrategy + if spec.UpdateStrategy.Type != "" { + sts.Spec.UpdateStrategy = spec.UpdateStrategy + } + if spec.UpdateStrategy.RollingUpdate != nil { + sts.Spec.UpdateStrategy.RollingUpdate = spec.UpdateStrategy.RollingUpdate + } + + // PodManagementPolicy + if spec.PodManagementPolicy != "" { + sts.Spec.PodManagementPolicy = spec.PodManagementPolicy + } + + sts.Spec.VolumeClaimTemplates = mergeVolumeClaimTemplates(sts.Spec.VolumeClaimTemplates, spec.VolumeClaimTemplates) + + //VolumeClaimTemplates + if len(spec.VolumeClaimTemplates) != 0 { + sts.Spec.VolumeClaimTemplates = spec.VolumeClaimTemplates + } + + //PodTemplateSpec + // - ObjectMeta + // TODO + // - PodSpec + podSpec := spec.Template.Spec + // -- actieveDeadLineSeconds + if podSpec.ActiveDeadlineSeconds != nil { + sts.Spec.Template.Spec.ActiveDeadlineSeconds = podSpec.ActiveDeadlineSeconds + } + // -- Affinity + if val, ok := mergedValue(sts.Spec.Template.Spec.Affinity, podSpec.Affinity).(*corev1.Affinity); ok { + sts.Spec.Template.Spec.Affinity = val + } + // -- automountService + if val, ok := mergedValue(sts.Spec.Template.Spec.AutomountServiceAccountToken, podSpec.AutomountServiceAccountToken).(*bool); ok { + sts.Spec.Template.Spec.AutomountServiceAccountToken = val + } + // -- Containers + for _, container := range podSpec.Containers { + if index := findContainerByName(container.Name, sts.Spec.Template.Spec.Containers); index == notFound { + sts.Spec.Template.Spec.Containers = append(sts.Spec.Template.Spec.Containers, container) + } else { + sts.Spec.Template.Spec.Containers[index] = mergedContainer(sts.Spec.Template.Spec.Containers[index], container) + } + } + + //DNSConfig + if val, ok := mergedValue(sts.Spec.Template.Spec.DNSConfig, podSpec.DNSConfig).(*corev1.PodDNSConfig); ok { + sts.Spec.Template.Spec.DNSConfig = val + } + // EnableServiceLinks + if val, ok := mergedValue(sts.Spec.Template.Spec.EnableServiceLinks, podSpec.EnableServiceLinks).(*bool); ok { + sts.Spec.Template.Spec.EnableServiceLinks = val + } + //EphemeralContainers + for _, container := range podSpec.EphemeralContainers { + if index := findEphemeralContainerByName(container.Name, sts.Spec.Template.Spec.EphemeralContainers); index == notFound { + sts.Spec.Template.Spec.EphemeralContainers = append(sts.Spec.Template.Spec.EphemeralContainers, container) + } else { + sts.Spec.Template.Spec.EphemeralContainers[index].EphemeralContainerCommon = corev1.EphemeralContainerCommon(mergedContainer(corev1.Container(sts.Spec.Template.Spec.EphemeralContainers[index].EphemeralContainerCommon), corev1.Container(container.EphemeralContainerCommon))) + sts.Spec.Template.Spec.EphemeralContainers[index].TargetContainerName = mergedString(sts.Spec.Template.Spec.EphemeralContainers[index].TargetContainerName, container.TargetContainerName) + } + } + + // hostAliases + for _, hostalias := range podSpec.HostAliases { + if index := findHostAliasByIp(hostalias.IP, sts.Spec.Template.Spec.HostAliases); index == notFound { + sts.Spec.Template.Spec.HostAliases = append(sts.Spec.Template.Spec.HostAliases, hostalias) + } else { + sts.Spec.Template.Spec.HostAliases[index] = hostalias + } + + } + + // host + sts.Spec.Template.Spec.HostIPC = podSpec.HostIPC + sts.Spec.Template.Spec.HostNetwork = podSpec.HostNetwork + sts.Spec.Template.Spec.HostPID = podSpec.HostPID + + sts.Spec.Template.Spec.Hostname = mergedString(sts.Spec.Template.Spec.Hostname, podSpec.Hostname) + + // ImagePullSecrets + for _, ips := range podSpec.ImagePullSecrets { + if index := findImagePullSecretByName(ips.Name, sts.Spec.Template.Spec.ImagePullSecrets); index == notFound { + sts.Spec.Template.Spec.ImagePullSecrets = append(sts.Spec.Template.Spec.ImagePullSecrets, ips) + } else { + sts.Spec.Template.Spec.ImagePullSecrets[index] = ips + } + + } + + // InitContainers + for _, container := range podSpec.InitContainers { + if index := findContainerByName(container.Name, sts.Spec.Template.Spec.InitContainers); index == notFound { + sts.Spec.Template.Spec.InitContainers = append(sts.Spec.Template.Spec.InitContainers, container) + } else { + sts.Spec.Template.Spec.InitContainers[index] = mergedContainer(sts.Spec.Template.Spec.InitContainers[index], container) + } + } + + //nodename + sts.Spec.Template.Spec.NodeName = mergedString(sts.Spec.Template.Spec.NodeName, podSpec.NodeName) + + //nodeselector + if len(podSpec.NodeSelector) != 0 { + sts.Spec.Template.Spec.NodeSelector = podSpec.NodeSelector + } + //overhead + if len(podSpec.Overhead) != 0 { + sts.Spec.Template.Spec.Overhead = podSpec.Overhead + } + // preemption policy + if val, ok := mergedValue(sts.Spec.Template.Spec.PreemptionPolicy, podSpec.PreemptionPolicy).(*corev1.PreemptionPolicy); ok { + sts.Spec.Template.Spec.PreemptionPolicy = val + } + //priority + if val, ok := mergedValue(sts.Spec.Template.Spec.Priority, podSpec.Priority).(*int32); ok { + sts.Spec.Template.Spec.Priority = val + } + // PriorityClassName + sts.Spec.Template.Spec.PriorityClassName = mergedString(sts.Spec.Template.Spec.PriorityClassName, podSpec.PriorityClassName) + + // Readiness gates + if len(podSpec.ReadinessGates) != 0 { + sts.Spec.Template.Spec.ReadinessGates = podSpec.ReadinessGates + } + + //restartpolicy + if podSpec.RestartPolicy != "" { + sts.Spec.Template.Spec.RestartPolicy = podSpec.RestartPolicy + } + // RunTimeClassName + if val, ok := mergedValue(sts.Spec.Template.Spec.RuntimeClassName, podSpec.RuntimeClassName).(*string); ok { + sts.Spec.Template.Spec.RuntimeClassName = val + } + + //SchedulerName + sts.Spec.Template.Spec.SchedulerName = mergedString(sts.Spec.Template.Spec.SchedulerName, podSpec.SchedulerName) + + //Security Context + if podSpec.SecurityContext != nil { + sts.Spec.Template.Spec.SecurityContext = podSpec.SecurityContext + } + + // ServiceAccountName + sts.Spec.Template.Spec.ServiceAccountName = mergedString(sts.Spec.Template.Spec.ServiceAccountName, podSpec.ServiceAccountName) + + //shareProcessNamespace + if val, ok := mergedValue(sts.Spec.Template.Spec.ShareProcessNamespace, podSpec.ShareProcessNamespace).(*bool); ok { + sts.Spec.Template.Spec.ShareProcessNamespace = val + } + + //subdomain + sts.Spec.Template.Spec.Subdomain = mergedString(sts.Spec.Template.Spec.Subdomain, podSpec.Subdomain) + + if podSpec.TerminationGracePeriodSeconds != nil { + sts.Spec.Template.Spec.TerminationGracePeriodSeconds = podSpec.TerminationGracePeriodSeconds + } + + // tolerations + if len(podSpec.Tolerations) > 0 { + sts.Spec.Template.Spec.Tolerations = podSpec.Tolerations + } + + // TopologySpreadContraints + for _, constraint := range podSpec.TopologySpreadConstraints { + if index := findTopologySpreadConstraintByKey(constraint.TopologyKey, sts.Spec.Template.Spec.TopologySpreadConstraints); index == notFound { + sts.Spec.Template.Spec.TopologySpreadConstraints = append(sts.Spec.Template.Spec.TopologySpreadConstraints, constraint) + } else { + // merge + sts.Spec.Template.Spec.TopologySpreadConstraints[index].MaxSkew = constraint.MaxSkew + sts.Spec.Template.Spec.TopologySpreadConstraints[index].WhenUnsatisfiable = constraint.WhenUnsatisfiable + sts.Spec.Template.Spec.TopologySpreadConstraints[index].LabelSelector = mergedLabelSelector(sts.Spec.Template.Spec.TopologySpreadConstraints[index].LabelSelector, constraint.LabelSelector) + + } + + } + + // volumes + for _, volume := range podSpec.Volumes { + if index := findVolumeByName(volume.Name, sts.Spec.Template.Spec.Volumes); index == notFound { + sts.Spec.Template.Spec.Volumes = append(sts.Spec.Template.Spec.Volumes, volume) + } else { + sts.Spec.Template.Spec.Volumes[index] = volume + } + + } + } +} + +func findMatchExpressionByKey(key string, matchExpressions []metav1.LabelSelectorRequirement) int { + for idx, expression := range matchExpressions { + if expression.Key == key { + return idx + } + } + return notFound +} + func findVolumeClaimIndexByName(name string, pvcs []corev1.PersistentVolumeClaim) int { for idx, pvc := range pvcs { if pvc.Name == name { @@ -258,3 +672,93 @@ func findVolumeClaimIndexByName(name string, pvcs []corev1.PersistentVolumeClaim } return notFound } + +func findHostAliasByIp(ip string, hostAliases []corev1.HostAlias) int { + for idx, alias := range hostAliases { + if alias.IP == ip { + return idx + } + } + return notFound +} + +func findImagePullSecretByName(name string, ips []corev1.LocalObjectReference) int { + for idx, localObjectReference := range ips { + if localObjectReference.Name == name { + return idx + } + } + return notFound +} + +func findTopologySpreadConstraintByKey(key string, tsc []corev1.TopologySpreadConstraint) int { + for idx, constraint := range tsc { + if constraint.TopologyKey == key { + return idx + } + } + return notFound +} + +func findVolumeByName(name string, volumes []corev1.Volume) int { + for idx, volume := range volumes { + if volume.Name == name { + return idx + } + } + return notFound +} + +func findContainerByName(name string, containers []corev1.Container) int { + for idx, container := range containers { + if container.Name == name { + return idx + } + } + return notFound +} + +func findEphemeralContainerByName(name string, containers []corev1.EphemeralContainer) int { + for idx, container := range containers { + if container.Name == name { + return idx + } + } + return notFound +} + +func findEnvVarByName(name string, vars []corev1.EnvVar) int { + for idx, val := range vars { + if val.Name == name { + return idx + } + } + return notFound +} + +func findContainerPortByPort(port int32, ports []corev1.ContainerPort) int { + for idx, containerPort := range ports { + if containerPort.ContainerPort == port { + return idx + } + } + return notFound +} + +func findVolumeDeviceByName(name string, volumeDevices []corev1.VolumeDevice) int { + for idx, volumeDevice := range volumeDevices { + if volumeDevice.Name == name { + return idx + } + } + return notFound +} + +func findVolumeMountsByMountPath(path string, volumeMounts []corev1.VolumeMount) int { + for idx, volumeMount := range volumeMounts { + if volumeMount.MountPath == path { + return idx + } + } + return notFound +} From 63216d2affc11109c8b40b6a3d7d61204fc786c0 Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Thu, 16 Jul 2020 14:10:41 +0100 Subject: [PATCH 03/34] Revert --- pkg/controller/mongodb/mongodb_controller.go | 9 +- pkg/kube/statefulset/statefulset.go | 506 +------------------ 2 files changed, 6 insertions(+), 509 deletions(-) diff --git a/pkg/controller/mongodb/mongodb_controller.go b/pkg/controller/mongodb/mongodb_controller.go index 3f14f1615..303d80daa 100644 --- a/pkg/controller/mongodb/mongodb_controller.go +++ b/pkg/controller/mongodb/mongodb_controller.go @@ -9,7 +9,6 @@ import ( "os" "time" - "github.com/kylelemons/godebug/pretty" "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/persistentvolumeclaim" "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/probes" @@ -244,8 +243,6 @@ func (r *ReplicaSetReconciler) resetStatefulSetUpdateStrategy(mdb mdbv1.MongoDB) // is currently ready. func (r *ReplicaSetReconciler) isStatefulSetReady(mdb mdbv1.MongoDB, existingStatefulSet *appsv1.StatefulSet) (bool, error) { stsFunc := buildStatefulSetModificationFunction(mdb) - fmt.Fprintf(os.Stderr, "\n\nExisting STS Spec: %+v\n\n", existingStatefulSet.Spec) - spec1 := existingStatefulSet.Spec stsCopy := existingStatefulSet.DeepCopyObject() stsFunc(existingStatefulSet) stsCopyBytes, err := json.Marshal(stsCopy) @@ -258,9 +255,6 @@ func (r *ReplicaSetReconciler) isStatefulSetReady(mdb mdbv1.MongoDB, existingSta return false, err } - fmt.Fprintf(os.Stderr, "\n\nModified STS Spec: %+v\n\n", existingStatefulSet.Spec) - - fmt.Fprintf(os.Stderr, "\n\nDifferences: %+v", pretty.Compare(spec1, existingStatefulSet.Spec)) //comparison is done with bytes instead of reflect.DeepEqual as there are //some issues with nil/empty maps not being compared correctly otherwise areEqual := bytes.Equal(stsCopyBytes, stsBytes) @@ -302,7 +296,6 @@ func (r *ReplicaSetReconciler) createOrUpdateStatefulSet(mdb mdbv1.MongoDB) erro newMdb := &mdbv1.MongoDB{} _ = r.client.Get(context.TODO(), mdb.NamespacedName(), newMdb) - fmt.Fprintf(os.Stderr, "\n\nSPEC: %+v\n\n", mdb.Spec.StatefulSetConfiguration.Spec) err = k8sClient.IgnoreNotFound(err) if err != nil { return fmt.Errorf("error getting StatefulSet: %s", err) @@ -638,8 +631,8 @@ func buildStatefulSetModificationFunction(mdb mdbv1.MongoDB) statefulset.Modific buildScramPodSpecModification(mdb), ), ), - statefulset.WithStatefulSetSpec(mdb.Spec.StatefulSetConfiguration.Spec), ) + } func getDomain(service, namespace, clusterName string) string { diff --git a/pkg/kube/statefulset/statefulset.go b/pkg/kube/statefulset/statefulset.go index a35af2afa..927ed58a0 100644 --- a/pkg/kube/statefulset/statefulset.go +++ b/pkg/kube/statefulset/statefulset.go @@ -1,9 +1,9 @@ package statefulset import ( - "sort" + "fmt" + "os" - "github.com/imdario/mergo" apiErrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -253,415 +253,9 @@ func WithVolumeClaim(name string, f func(*corev1.PersistentVolumeClaim)) Modific } } -func mergedValue(original, modified interface{}) interface{} { - if modified != nil { - return modified - } - return original -} - -func mergedString(original, modified string) string { - if modified != "" { - return modified - } - return original -} - -func mergedLabelSelector(original, modified *metav1.LabelSelector) *metav1.LabelSelector { - if modified == nil { - return original - } - for key, val := range modified.MatchLabels { - original.MatchLabels[key] = val - } - // Selector.MatchExpressions - for _, expression := range modified.MatchExpressions { - if index := findMatchExpressionByKey(expression.Key, original.MatchExpressions); index == notFound { - // Add - original.MatchExpressions = append(original.MatchExpressions, expression) - } else { - original.MatchExpressions[index] = expression - } - } - return original -} - -func mergedContainer(original, modified corev1.Container) corev1.Container { - // args - if len(modified.Args) > 0 { - original.Args = modified.Args - } - //command - if len(modified.Command) > 0 { - original.Command = modified.Command - } - - // env - for _, envVar := range modified.Env { - if index := findEnvVarByName(envVar.Name, original.Env); index == notFound { - original.Env = append(original.Env, envVar) - } else { - original.Env[index] = envVar - } - } - - // envFrom - if len(modified.EnvFrom) > 0 { - original.EnvFrom = modified.EnvFrom - } - - // image and imagepullpolicy - original.Image = mergedString(original.Image, modified.Image) - if modified.ImagePullPolicy != "" { - original.ImagePullPolicy = modified.ImagePullPolicy - } - - // lifecycle - if val, ok := mergedValue(original.Lifecycle, modified.Lifecycle).(*corev1.Lifecycle); ok { - original.Lifecycle = val - } - - //lifevenessprobe - if val, ok := mergedValue(original.LivenessProbe, modified.LivenessProbe).(*corev1.Probe); ok { - original.LivenessProbe = val - } - - //ports - for _, port := range modified.Ports { - if index := findContainerPortByPort(port.ContainerPort, original.Ports); index == notFound { - original.Ports = append(original.Ports, port) - } else { - original.Ports[index].HostIP = mergedString(original.Ports[index].HostIP, port.HostIP) - original.Ports[index].Name = mergedString(original.Ports[index].Name, port.Name) - if port.Protocol != "" { - original.Ports[index].Protocol = port.Protocol - } - if port.HostPort != 0 { - original.Ports[index].HostPort = port.HostPort - } - } - } - - //ReadinessProbe - if val, ok := mergedValue(original.ReadinessProbe, modified.ReadinessProbe).(*corev1.Probe); ok { - original.ReadinessProbe = val - } - - // resources - limits - if len(modified.Resources.Limits) > 0 { - original.Resources.Limits = modified.Resources.Limits - } - - // resources - requests - if len(modified.Resources.Requests) > 0 { - original.Resources.Requests = modified.Resources.Requests - } - - //SecurityContext - if val, ok := mergedValue(original.SecurityContext, modified.SecurityContext).(*corev1.SecurityContext); ok { - original.SecurityContext = val - } - - //StartupProbe - if val, ok := mergedValue(original.StartupProbe, modified.StartupProbe).(*corev1.Probe); ok { - original.StartupProbe = val - } - - // stdin - original.Stdin = modified.Stdin - //stdinOnce - original.StdinOnce = modified.StdinOnce - - // terminationMessagePath - original.TerminationMessagePath = mergedString(original.TerminationMessagePath, modified.TerminationMessagePath) - - // terminationMEssagePolicy - if modified.TerminationMessagePolicy != "" { - original.TerminationMessagePolicy = modified.TerminationMessagePolicy - } - - //TTY - original.TTY = modified.TTY - - //volumedevices - for _, device := range modified.VolumeDevices { - if index := findVolumeDeviceByName(device.Name, original.VolumeDevices); index == notFound { - original.VolumeDevices = append(original.VolumeDevices, device) - } else { - original.VolumeDevices[index].DevicePath = mergedString(original.VolumeDevices[index].DevicePath, device.DevicePath) - } - } - - //volumemounts - for _, mount := range modified.VolumeMounts { - if index := findVolumeMountsByMountPath(mount.MountPath, original.VolumeMounts); index == notFound { - original.VolumeMounts = append(original.VolumeMounts, mount) - } else { - original.VolumeMounts[index].Name = mergedString(original.VolumeMounts[index].Name, mount.Name) - original.VolumeMounts[index].SubPath = mergedString(original.VolumeMounts[index].SubPath, mount.SubPath) - original.VolumeMounts[index].SubPathExpr = mergedString(original.VolumeMounts[index].SubPathExpr, mount.SubPathExpr) - original.VolumeMounts[index].ReadOnly = mount.ReadOnly - if val, ok := mergedValue(original.VolumeMounts[index].MountPropagation, mount.MountPropagation).(*corev1.MountPropagationMode); ok { - original.VolumeMounts[index].MountPropagation = val - } - } - } - - //workingdir - original.WorkingDir = mergedString(original.WorkingDir, modified.WorkingDir) - - return original -} - -func createVolumeClaimMap(volumeMounts []corev1.PersistentVolumeClaim) map[string]corev1.PersistentVolumeClaim { - mountMap := make(map[string]corev1.PersistentVolumeClaim) - for _, m := range volumeMounts { - mountMap[m.Name] = m - } - return mountMap -} - -func mergeVolumeClaimTemplates(original, modified []corev1.PersistentVolumeClaim) []corev1.PersistentVolumeClaim { - defaultMountsMap := createVolumeClaimMap(original) - customMountsMap := createVolumeClaimMap(modified) - var mergedVolumes []corev1.PersistentVolumeClaim - for _, defaultMount := range defaultMountsMap { - if customMount, ok := customMountsMap[defaultMount.Name]; ok { - // needs merge - if err := mergo.Merge(&defaultMount, customMount, mergo.WithAppendSlice, mergo.WithOverride); err != nil { - return nil - } - } - mergedVolumes = append(mergedVolumes, defaultMount) - } - for _, customMount := range customMountsMap { - if _, ok := defaultMountsMap[customMount.Name]; ok { - // already merged - continue - } - mergedVolumes = append(mergedVolumes, customMount) - } - sort.SliceStable(mergedVolumes, func(i, j int) bool { - return mergedVolumes[i].Name < mergedVolumes[j].Name - }) - return mergedVolumes -} - -// Merges the change provided by the user on top of what we already built -func WithStatefulSetSpec(spec appsv1.StatefulSetSpec) Modification { - return func(sts *appsv1.StatefulSet) { - // Replicas - if spec.Replicas != nil { - sts.Spec.Replicas = spec.Replicas - } - // Selector - sts.Spec.Selector = mergedLabelSelector(sts.Spec.Selector, spec.Selector) - - // ServiceName - if spec.ServiceName != "" { - sts.Spec.ServiceName = spec.ServiceName - } - // RevisionHistoryLimit - if spec.RevisionHistoryLimit != nil { - sts.Spec.RevisionHistoryLimit = spec.RevisionHistoryLimit - } - // UpdateStrategy - if spec.UpdateStrategy.Type != "" { - sts.Spec.UpdateStrategy = spec.UpdateStrategy - } - if spec.UpdateStrategy.RollingUpdate != nil { - sts.Spec.UpdateStrategy.RollingUpdate = spec.UpdateStrategy.RollingUpdate - } - - // PodManagementPolicy - if spec.PodManagementPolicy != "" { - sts.Spec.PodManagementPolicy = spec.PodManagementPolicy - } - - sts.Spec.VolumeClaimTemplates = mergeVolumeClaimTemplates(sts.Spec.VolumeClaimTemplates, spec.VolumeClaimTemplates) - - //VolumeClaimTemplates - if len(spec.VolumeClaimTemplates) != 0 { - sts.Spec.VolumeClaimTemplates = spec.VolumeClaimTemplates - } - - //PodTemplateSpec - // - ObjectMeta - // TODO - // - PodSpec - podSpec := spec.Template.Spec - // -- actieveDeadLineSeconds - if podSpec.ActiveDeadlineSeconds != nil { - sts.Spec.Template.Spec.ActiveDeadlineSeconds = podSpec.ActiveDeadlineSeconds - } - // -- Affinity - if val, ok := mergedValue(sts.Spec.Template.Spec.Affinity, podSpec.Affinity).(*corev1.Affinity); ok { - sts.Spec.Template.Spec.Affinity = val - } - // -- automountService - if val, ok := mergedValue(sts.Spec.Template.Spec.AutomountServiceAccountToken, podSpec.AutomountServiceAccountToken).(*bool); ok { - sts.Spec.Template.Spec.AutomountServiceAccountToken = val - } - // -- Containers - for _, container := range podSpec.Containers { - if index := findContainerByName(container.Name, sts.Spec.Template.Spec.Containers); index == notFound { - sts.Spec.Template.Spec.Containers = append(sts.Spec.Template.Spec.Containers, container) - } else { - sts.Spec.Template.Spec.Containers[index] = mergedContainer(sts.Spec.Template.Spec.Containers[index], container) - } - } - - //DNSConfig - if val, ok := mergedValue(sts.Spec.Template.Spec.DNSConfig, podSpec.DNSConfig).(*corev1.PodDNSConfig); ok { - sts.Spec.Template.Spec.DNSConfig = val - } - // EnableServiceLinks - if val, ok := mergedValue(sts.Spec.Template.Spec.EnableServiceLinks, podSpec.EnableServiceLinks).(*bool); ok { - sts.Spec.Template.Spec.EnableServiceLinks = val - } - //EphemeralContainers - for _, container := range podSpec.EphemeralContainers { - if index := findEphemeralContainerByName(container.Name, sts.Spec.Template.Spec.EphemeralContainers); index == notFound { - sts.Spec.Template.Spec.EphemeralContainers = append(sts.Spec.Template.Spec.EphemeralContainers, container) - } else { - sts.Spec.Template.Spec.EphemeralContainers[index].EphemeralContainerCommon = corev1.EphemeralContainerCommon(mergedContainer(corev1.Container(sts.Spec.Template.Spec.EphemeralContainers[index].EphemeralContainerCommon), corev1.Container(container.EphemeralContainerCommon))) - sts.Spec.Template.Spec.EphemeralContainers[index].TargetContainerName = mergedString(sts.Spec.Template.Spec.EphemeralContainers[index].TargetContainerName, container.TargetContainerName) - } - } - - // hostAliases - for _, hostalias := range podSpec.HostAliases { - if index := findHostAliasByIp(hostalias.IP, sts.Spec.Template.Spec.HostAliases); index == notFound { - sts.Spec.Template.Spec.HostAliases = append(sts.Spec.Template.Spec.HostAliases, hostalias) - } else { - sts.Spec.Template.Spec.HostAliases[index] = hostalias - } - - } - - // host - sts.Spec.Template.Spec.HostIPC = podSpec.HostIPC - sts.Spec.Template.Spec.HostNetwork = podSpec.HostNetwork - sts.Spec.Template.Spec.HostPID = podSpec.HostPID - - sts.Spec.Template.Spec.Hostname = mergedString(sts.Spec.Template.Spec.Hostname, podSpec.Hostname) - - // ImagePullSecrets - for _, ips := range podSpec.ImagePullSecrets { - if index := findImagePullSecretByName(ips.Name, sts.Spec.Template.Spec.ImagePullSecrets); index == notFound { - sts.Spec.Template.Spec.ImagePullSecrets = append(sts.Spec.Template.Spec.ImagePullSecrets, ips) - } else { - sts.Spec.Template.Spec.ImagePullSecrets[index] = ips - } - - } - - // InitContainers - for _, container := range podSpec.InitContainers { - if index := findContainerByName(container.Name, sts.Spec.Template.Spec.InitContainers); index == notFound { - sts.Spec.Template.Spec.InitContainers = append(sts.Spec.Template.Spec.InitContainers, container) - } else { - sts.Spec.Template.Spec.InitContainers[index] = mergedContainer(sts.Spec.Template.Spec.InitContainers[index], container) - } - } - - //nodename - sts.Spec.Template.Spec.NodeName = mergedString(sts.Spec.Template.Spec.NodeName, podSpec.NodeName) - - //nodeselector - if len(podSpec.NodeSelector) != 0 { - sts.Spec.Template.Spec.NodeSelector = podSpec.NodeSelector - } - //overhead - if len(podSpec.Overhead) != 0 { - sts.Spec.Template.Spec.Overhead = podSpec.Overhead - } - // preemption policy - if val, ok := mergedValue(sts.Spec.Template.Spec.PreemptionPolicy, podSpec.PreemptionPolicy).(*corev1.PreemptionPolicy); ok { - sts.Spec.Template.Spec.PreemptionPolicy = val - } - //priority - if val, ok := mergedValue(sts.Spec.Template.Spec.Priority, podSpec.Priority).(*int32); ok { - sts.Spec.Template.Spec.Priority = val - } - // PriorityClassName - sts.Spec.Template.Spec.PriorityClassName = mergedString(sts.Spec.Template.Spec.PriorityClassName, podSpec.PriorityClassName) - - // Readiness gates - if len(podSpec.ReadinessGates) != 0 { - sts.Spec.Template.Spec.ReadinessGates = podSpec.ReadinessGates - } - - //restartpolicy - if podSpec.RestartPolicy != "" { - sts.Spec.Template.Spec.RestartPolicy = podSpec.RestartPolicy - } - // RunTimeClassName - if val, ok := mergedValue(sts.Spec.Template.Spec.RuntimeClassName, podSpec.RuntimeClassName).(*string); ok { - sts.Spec.Template.Spec.RuntimeClassName = val - } - - //SchedulerName - sts.Spec.Template.Spec.SchedulerName = mergedString(sts.Spec.Template.Spec.SchedulerName, podSpec.SchedulerName) - - //Security Context - if podSpec.SecurityContext != nil { - sts.Spec.Template.Spec.SecurityContext = podSpec.SecurityContext - } - - // ServiceAccountName - sts.Spec.Template.Spec.ServiceAccountName = mergedString(sts.Spec.Template.Spec.ServiceAccountName, podSpec.ServiceAccountName) - - //shareProcessNamespace - if val, ok := mergedValue(sts.Spec.Template.Spec.ShareProcessNamespace, podSpec.ShareProcessNamespace).(*bool); ok { - sts.Spec.Template.Spec.ShareProcessNamespace = val - } - - //subdomain - sts.Spec.Template.Spec.Subdomain = mergedString(sts.Spec.Template.Spec.Subdomain, podSpec.Subdomain) - - if podSpec.TerminationGracePeriodSeconds != nil { - sts.Spec.Template.Spec.TerminationGracePeriodSeconds = podSpec.TerminationGracePeriodSeconds - } - - // tolerations - if len(podSpec.Tolerations) > 0 { - sts.Spec.Template.Spec.Tolerations = podSpec.Tolerations - } - - // TopologySpreadContraints - for _, constraint := range podSpec.TopologySpreadConstraints { - if index := findTopologySpreadConstraintByKey(constraint.TopologyKey, sts.Spec.Template.Spec.TopologySpreadConstraints); index == notFound { - sts.Spec.Template.Spec.TopologySpreadConstraints = append(sts.Spec.Template.Spec.TopologySpreadConstraints, constraint) - } else { - // merge - sts.Spec.Template.Spec.TopologySpreadConstraints[index].MaxSkew = constraint.MaxSkew - sts.Spec.Template.Spec.TopologySpreadConstraints[index].WhenUnsatisfiable = constraint.WhenUnsatisfiable - sts.Spec.Template.Spec.TopologySpreadConstraints[index].LabelSelector = mergedLabelSelector(sts.Spec.Template.Spec.TopologySpreadConstraints[index].LabelSelector, constraint.LabelSelector) - - } - - } - - // volumes - for _, volume := range podSpec.Volumes { - if index := findVolumeByName(volume.Name, sts.Spec.Template.Spec.Volumes); index == notFound { - sts.Spec.Template.Spec.Volumes = append(sts.Spec.Template.Spec.Volumes, volume) - } else { - sts.Spec.Template.Spec.Volumes[index] = volume - } - - } - } -} - -func findMatchExpressionByKey(key string, matchExpressions []metav1.LabelSelectorRequirement) int { - for idx, expression := range matchExpressions { - if expression.Key == key { - return idx - } - } - return notFound +func MergeStsSpecs(original, modification appsv1.StatefulSet) { + fmt.Fprintf(os.Stderr, "Original: %+v", original) + fmt.Fprintf(os.Stderr, "modification: %+v", modification) } func findVolumeClaimIndexByName(name string, pvcs []corev1.PersistentVolumeClaim) int { @@ -672,93 +266,3 @@ func findVolumeClaimIndexByName(name string, pvcs []corev1.PersistentVolumeClaim } return notFound } - -func findHostAliasByIp(ip string, hostAliases []corev1.HostAlias) int { - for idx, alias := range hostAliases { - if alias.IP == ip { - return idx - } - } - return notFound -} - -func findImagePullSecretByName(name string, ips []corev1.LocalObjectReference) int { - for idx, localObjectReference := range ips { - if localObjectReference.Name == name { - return idx - } - } - return notFound -} - -func findTopologySpreadConstraintByKey(key string, tsc []corev1.TopologySpreadConstraint) int { - for idx, constraint := range tsc { - if constraint.TopologyKey == key { - return idx - } - } - return notFound -} - -func findVolumeByName(name string, volumes []corev1.Volume) int { - for idx, volume := range volumes { - if volume.Name == name { - return idx - } - } - return notFound -} - -func findContainerByName(name string, containers []corev1.Container) int { - for idx, container := range containers { - if container.Name == name { - return idx - } - } - return notFound -} - -func findEphemeralContainerByName(name string, containers []corev1.EphemeralContainer) int { - for idx, container := range containers { - if container.Name == name { - return idx - } - } - return notFound -} - -func findEnvVarByName(name string, vars []corev1.EnvVar) int { - for idx, val := range vars { - if val.Name == name { - return idx - } - } - return notFound -} - -func findContainerPortByPort(port int32, ports []corev1.ContainerPort) int { - for idx, containerPort := range ports { - if containerPort.ContainerPort == port { - return idx - } - } - return notFound -} - -func findVolumeDeviceByName(name string, volumeDevices []corev1.VolumeDevice) int { - for idx, volumeDevice := range volumeDevices { - if volumeDevice.Name == name { - return idx - } - } - return notFound -} - -func findVolumeMountsByMountPath(path string, volumeMounts []corev1.VolumeMount) int { - for idx, volumeMount := range volumeMounts { - if volumeMount.MountPath == path { - return idx - } - } - return notFound -} From 4e27141ddf55388394e4c040df352262bd4f1338 Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Fri, 17 Jul 2020 13:48:15 +0100 Subject: [PATCH 04/34] initial merging approach --- deploy/crds/mongodb.com_v1_mongodb_cr.yaml | 7 -- pkg/controller/mongodb/mongodb_controller.go | 1 + pkg/kube/podtemplatespec/podspec_template.go | 119 +++++++++++++++++++ pkg/kube/statefulset/statefulset.go | 77 +++++++++++- 4 files changed, 194 insertions(+), 10 deletions(-) diff --git a/deploy/crds/mongodb.com_v1_mongodb_cr.yaml b/deploy/crds/mongodb.com_v1_mongodb_cr.yaml index 9493d22d0..a1bc63db2 100644 --- a/deploy/crds/mongodb.com_v1_mongodb_cr.yaml +++ b/deploy/crds/mongodb.com_v1_mongodb_cr.yaml @@ -6,10 +6,3 @@ spec: members: 3 type: ReplicaSet version: "4.2.6" - statefulset: - spec: - volumeClaimTemplates: - - metadata: - name: data-volume-example-mongodb-0 - spec: - storageClassName: manual diff --git a/pkg/controller/mongodb/mongodb_controller.go b/pkg/controller/mongodb/mongodb_controller.go index 145228408..d62713a9b 100644 --- a/pkg/controller/mongodb/mongodb_controller.go +++ b/pkg/controller/mongodb/mongodb_controller.go @@ -611,6 +611,7 @@ func buildStatefulSetModificationFunction(mdb mdbv1.MongoDB) statefulset.Modific buildScramPodSpecModification(mdb), ), ), + // statefulset.WithCustomSpecs(mdb.Spec.StatefulSetConfiguration.Spec), ) } diff --git a/pkg/kube/podtemplatespec/podspec_template.go b/pkg/kube/podtemplatespec/podspec_template.go index 623bc4a16..7db97b791 100644 --- a/pkg/kube/podtemplatespec/podspec_template.go +++ b/pkg/kube/podtemplatespec/podspec_template.go @@ -1,6 +1,7 @@ package podtemplatespec import ( + "github.com/imdario/mergo" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -222,6 +223,124 @@ func WithVolumeMounts(containerName string, volumeMounts ...corev1.VolumeMount) } } +func mergeVolumeMounts(defaultMounts, overrideMounts []corev1.VolumeMount) ([]corev1.VolumeMount, error) { + defaultMountsMap := createMountsMap(defaultMounts) + overrideMountsMap := createMountsMap(overrideMounts) + mergedVolumeMounts := []corev1.VolumeMount{} + for _, defaultMount := range defaultMounts { + if overrideMount, ok := overrideMountsMap[defaultMount.Name]; ok { + // needs merge + if err := mergo.Merge(&defaultMount, overrideMount, mergo.WithAppendSlice); err != nil { + return nil, err + } + } + mergedVolumeMounts = append(mergedVolumeMounts, defaultMount) + } + for _, overrideMount := range overrideMounts { + if _, ok := defaultMountsMap[overrideMount.Name]; ok { + // already merged + continue + } + mergedVolumeMounts = append(mergedVolumeMounts, overrideMount) + } + return mergedVolumeMounts, nil +} + +func createMountsMap(volumeMounts []corev1.VolumeMount) map[string]corev1.VolumeMount { + mountMap := make(map[string]corev1.VolumeMount) + for _, m := range volumeMounts { + mountMap[m.Name] = m + } + return mountMap +} + +func mergeContainers(defaultContainers, overrideContainers []corev1.Container) ([]corev1.Container, error) { + + defaultMap := createContainerMap(defaultContainers) + overrideMap := createContainerMap(overrideContainers) + mergedContainers := []corev1.Container{} + for _, defaultContainer := range defaultContainers { + if overrideContainer, ok := overrideMap[defaultContainer.Name]; ok { + // Merge + // Merge mounts + mergedMounts, err := mergeVolumeMounts(defaultContainer.VolumeMounts, overrideContainer.VolumeMounts) + if err != nil { + return nil, err + } + if err := mergo.Merge(&defaultContainer, overrideContainer, mergo.WithOverride); err != nil { + return nil, err + } + // completely override any resources that were provided + // this prevents issues with custom requests giving errors due + // to the defaulted limits + defaultContainer.Resources = overrideContainer.Resources + defaultContainer.VolumeMounts = mergedMounts + } + // Need to add it + mergedContainers = append(mergedContainers, defaultContainer) + } + + // Look for overrideContainers that were not merged + for _, overrideContainer := range overrideContainers { + if _, ok := defaultMap[overrideContainer.Name]; ok { + continue + } + // Need to add it + mergedContainers = append(mergedContainers, overrideContainer) + } + + return defaultContainers, nil +} + +func createContainerMap(containers []corev1.Container) map[string]corev1.Container { + containerMap := make(map[string]corev1.Container) + for _, c := range containers { + containerMap[c.Name] = c + } + return containerMap +} + +func mergeAffinity(defaultAffinity, overrideAffinity *corev1.Affinity) (*corev1.Affinity, error) { + if defaultAffinity == nil { + return overrideAffinity, nil + } + if overrideAffinity == nil { + return defaultAffinity, nil + } + mergedAffinity := defaultAffinity.DeepCopy() + if err := mergo.Merge(mergedAffinity, *overrideAffinity, mergo.WithOverride); err != nil { + return nil, err + } + return mergedAffinity, nil +} + +func MergePodTemplateSpecs(defaultTemplate, overrideTemplate corev1.PodTemplateSpec) (corev1.PodTemplateSpec, error) { + mergedContainers, err := mergeContainers(defaultTemplate.Spec.Containers, overrideTemplate.Spec.Containers) + if err != nil { + return corev1.PodTemplateSpec{}, err + } + + mergedInitContainers, err := mergeContainers(defaultTemplate.Spec.InitContainers, overrideTemplate.Spec.InitContainers) + if err != nil { + return corev1.PodTemplateSpec{}, err + } + + mergedAffinity, err := mergeAffinity(defaultTemplate.Spec.Affinity, overrideTemplate.Spec.Affinity) + if err != nil { + return corev1.PodTemplateSpec{}, err + } + + mergedPodTemplateSpec := *defaultTemplate.DeepCopy() + if err = mergo.Merge(&mergedPodTemplateSpec, overrideTemplate, mergo.WithOverride, mergo.WithAppendSlice); err != nil { + return corev1.PodTemplateSpec{}, err + } + + mergedPodTemplateSpec.Spec.Containers = mergedContainers + mergedPodTemplateSpec.Spec.InitContainers = mergedInitContainers + mergedPodTemplateSpec.Spec.Affinity = mergedAffinity + return mergedPodTemplateSpec, nil +} + // findContainerByName will find either a container or init container by name in a pod template spec func findContainerByName(name string, podTemplateSpec *corev1.PodTemplateSpec) *corev1.Container { containerIdx := findIndexByName(name, podTemplateSpec.Spec.Containers) diff --git a/pkg/kube/statefulset/statefulset.go b/pkg/kube/statefulset/statefulset.go index 927ed58a0..4dbc8c366 100644 --- a/pkg/kube/statefulset/statefulset.go +++ b/pkg/kube/statefulset/statefulset.go @@ -3,7 +3,10 @@ package statefulset import ( "fmt" "os" + "sort" + "github.com/imdario/mergo" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/podtemplatespec" apiErrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -180,6 +183,10 @@ func Apply(funcs ...Modification) func(*appsv1.StatefulSet) { } } +func NOOP() Modification { + return func(sts *appsv1.StatefulSet) {} +} + func WithName(name string) Modification { return func(sts *appsv1.StatefulSet) { sts.Name = name @@ -253,9 +260,73 @@ func WithVolumeClaim(name string, f func(*corev1.PersistentVolumeClaim)) Modific } } -func MergeStsSpecs(original, modification appsv1.StatefulSet) { - fmt.Fprintf(os.Stderr, "Original: %+v", original) - fmt.Fprintf(os.Stderr, "modification: %+v", modification) +func createVolumeClaimMap(volumeMounts []corev1.PersistentVolumeClaim) map[string]corev1.PersistentVolumeClaim { + mountMap := make(map[string]corev1.PersistentVolumeClaim) + for _, m := range volumeMounts { + mountMap[m.Name] = m + } + return mountMap +} + +func mergeVolumeClaimTemplates(defaultTemplates []corev1.PersistentVolumeClaim, overrideTemplates []corev1.PersistentVolumeClaim) ([]corev1.PersistentVolumeClaim, error) { + defaultMountsMap := createVolumeClaimMap(defaultTemplates) + overrideMountsMap := createVolumeClaimMap(overrideTemplates) + var mergedVolumes []corev1.PersistentVolumeClaim + for _, defaultMount := range defaultMountsMap { + if overrideMount, ok := overrideMountsMap[defaultMount.Name]; ok { + // needs merge + if err := mergo.Merge(&defaultMount, overrideMount, mergo.WithAppendSlice, mergo.WithOverride); err != nil { + return nil, err + } + } + mergedVolumes = append(mergedVolumes, defaultMount) + } + for _, overrideMount := range overrideMountsMap { + if _, ok := defaultMountsMap[overrideMount.Name]; ok { + // already merged + continue + } + mergedVolumes = append(mergedVolumes, overrideMount) + } + sort.SliceStable(mergedVolumes, func(i, j int) bool { + return mergedVolumes[i].Name < mergedVolumes[j].Name + }) + return mergedVolumes, nil +} + +func mergeStatefulSetSpecs(defaultSpec, overrideSpec appsv1.StatefulSetSpec) (appsv1.StatefulSetSpec, error) { + + fmt.Fprintf(os.Stderr, "\n\nDefaultSpec: %+v\n\n", defaultSpec) + + fmt.Fprintf(os.Stderr, "\n\nOverrideSpec: %+v\n\n", overrideSpec) + + // PodTemplateSpec needs to be manually merged + mergedPodTemplateSpec, err := podtemplatespec.MergePodTemplateSpecs(defaultSpec.Template, overrideSpec.Template) + if err != nil { + return appsv1.StatefulSetSpec{}, err + } + defaultSpec.Template = mergedPodTemplateSpec + // VolumeClaimTemplates needs to be manually merged + mergedVolumeClaimTemplates, err := mergeVolumeClaimTemplates(defaultSpec.VolumeClaimTemplates, overrideSpec.VolumeClaimTemplates) + + defaultSpec.VolumeClaimTemplates = mergedVolumeClaimTemplates + // Merging the rest with mergo + + if err := mergo.Merge(&defaultSpec, overrideSpec, mergo.WithOverride); err != nil { + return appsv1.StatefulSetSpec{}, err + } + fmt.Fprintf(os.Stderr, "\n\nAfterMerge: %+v\n\n", defaultSpec) + return defaultSpec, nil +} + +func WithCustomSpecs(spec appsv1.StatefulSetSpec) Modification { + return func(set *appsv1.StatefulSet) { + m, err := mergeStatefulSetSpecs(set.Spec, spec) + if err != nil { + return + } + set.Spec = m + } } func findVolumeClaimIndexByName(name string, pvcs []corev1.PersistentVolumeClaim) int { From bab53b9846ad8acc35d92786ddbde61c14a78601 Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Fri, 17 Jul 2020 13:48:30 +0100 Subject: [PATCH 05/34] linting --- pkg/controller/mongodb/mongodb_tls.go | 2 +- pkg/controller/mongodb/replicaset_controller_test.go | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pkg/controller/mongodb/mongodb_tls.go b/pkg/controller/mongodb/mongodb_tls.go index 4507dd11f..fd802ff30 100644 --- a/pkg/controller/mongodb/mongodb_tls.go +++ b/pkg/controller/mongodb/mongodb_tls.go @@ -94,7 +94,7 @@ func getTLSConfigModification(mdb mdbv1.MongoDB) automationconfig.Modification { // Configure CA certificate for agent config.TLS.CAFilePath = caCertificatePath - for i, _ := range config.Processes { + for i := range config.Processes { config.Processes[i].Args26.Net.TLS = automationconfig.MongoDBTLS{ Mode: mode, CAFile: caCertificatePath, diff --git a/pkg/controller/mongodb/replicaset_controller_test.go b/pkg/controller/mongodb/replicaset_controller_test.go index 38d2be43a..a7054150d 100644 --- a/pkg/controller/mongodb/replicaset_controller_test.go +++ b/pkg/controller/mongodb/replicaset_controller_test.go @@ -369,7 +369,8 @@ func TestStatefulSet_IsCorrectlyConfiguredWithTLS(t *testing.T) { SetField("tls.key", "KEY"). Build() - mgr.GetClient().Create(context.TODO(), &s) + err := mgr.GetClient().Create(context.TODO(), &s) + assert.NoError(t, err) configMap := configmap.Builder(). SetName(mdb.Spec.Security.TLS.CaConfigMap.Name). @@ -377,7 +378,8 @@ func TestStatefulSet_IsCorrectlyConfiguredWithTLS(t *testing.T) { SetField("ca.crt", "CERT"). Build() - mgr.GetClient().Create(context.TODO(), &configMap) + err = mgr.GetClient().Create(context.TODO(), &configMap) + assert.NoError(t, err) r := newReconciler(mgr, mockManifestProvider(mdb.Spec.Version)) res, err := r.Reconcile(reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) From 468370acfc5e0fbd3f0ba557bcc4ee4b7d110b61 Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Fri, 17 Jul 2020 14:57:51 +0100 Subject: [PATCH 06/34] reintroduced change --- pkg/controller/mongodb/mongodb_controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/controller/mongodb/mongodb_controller.go b/pkg/controller/mongodb/mongodb_controller.go index d62713a9b..7ac3ec584 100644 --- a/pkg/controller/mongodb/mongodb_controller.go +++ b/pkg/controller/mongodb/mongodb_controller.go @@ -611,7 +611,7 @@ func buildStatefulSetModificationFunction(mdb mdbv1.MongoDB) statefulset.Modific buildScramPodSpecModification(mdb), ), ), - // statefulset.WithCustomSpecs(mdb.Spec.StatefulSetConfiguration.Spec), + statefulset.WithCustomSpecs(mdb.Spec.StatefulSetConfiguration.Spec), ) } From 716d23725ff03184647174ecd0bb7682fa9ac78f Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Tue, 21 Jul 2020 17:39:28 +0100 Subject: [PATCH 07/34] wip --- .evergreen.yml | 7 + deploy/operator.yaml | 4 +- pkg/controller/mongodb/mongodb_controller.go | 3 - pkg/kube/podtemplatespec/podspec_template.go | 32 +- pkg/kube/statefulset/statefulset.go | 15 +- pkg/kube/statefulset/statefulset_test.go | 310 ++++++++++++++++++ test/e2e/e2eutil.go | 14 + test/e2e/mongodbtests/mongodbtests.go | 10 + .../statefulset_arbitrary_config_test.go | 63 ++++ 9 files changed, 426 insertions(+), 32 deletions(-) create mode 100644 test/e2e/statefulset_arbitrary_config/statefulset_arbitrary_config_test.go diff --git a/.evergreen.yml b/.evergreen.yml index eae6effda..7ee0c936c 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -151,6 +151,7 @@ task_groups: - e2e_test_replica_set_multiple - e2e_test_replica_set_tls - e2e_test_replica_set_tls_upgrade + - e2e_test_statefulset_arbitrary_config teardown_task: - func: upload_e2e_logs @@ -277,6 +278,12 @@ tasks: vars: test: replica_set_tls_upgrade + - name: e2e_test_statefulset_arbitrary_config + commands: + - func: run_e2e_test + vars: + test: statefulset_arbitrary_config + buildvariants: - name: go_unit_tests display_name: go_unit_tests diff --git a/deploy/operator.yaml b/deploy/operator.yaml index 027ab2365..4fb2b66ff 100644 --- a/deploy/operator.yaml +++ b/deploy/operator.yaml @@ -32,5 +32,5 @@ spec: value: "mongodb-kubernetes-operator" - name: AGENT_IMAGE # The MongoDB Agent the operator will deploy to manage MongoDB deployments value: quay.io/mongodb/mongodb-agent:10.15.1.6468-1 - - name: PRE_STOP_HOOK_IMAGE - value: quay.io/mongodb/mongodb-kubernetes-operator-pre-stop-hook:1.0.1 + - name: VERSION_UPGRADE_HOOK_IMAGE + value: quay.io/mongodb/mongodb-kubernetes-operator-version-upgrade-post-start-hook:1.0.2 diff --git a/pkg/controller/mongodb/mongodb_controller.go b/pkg/controller/mongodb/mongodb_controller.go index 7ac3ec584..f4d5e1120 100644 --- a/pkg/controller/mongodb/mongodb_controller.go +++ b/pkg/controller/mongodb/mongodb_controller.go @@ -293,15 +293,12 @@ func (r *ReplicaSetReconciler) createOrUpdateStatefulSet(mdb mdbv1.MongoDB) erro set := appsv1.StatefulSet{} err := r.client.Get(context.TODO(), mdb.NamespacedName(), &set) - newMdb := &mdbv1.MongoDB{} - _ = r.client.Get(context.TODO(), mdb.NamespacedName(), newMdb) err = k8sClient.IgnoreNotFound(err) if err != nil { return fmt.Errorf("error getting StatefulSet: %s", err) } buildStatefulSetModificationFunction(mdb)(&set) - fmt.Fprintf(os.Stderr, "\n\nSPEC: %+v\n\n", set.Spec) if err = statefulset.CreateOrUpdate(r.client, set); err != nil { return fmt.Errorf("error creating/updating StatefulSet: %s", err) } diff --git a/pkg/kube/podtemplatespec/podspec_template.go b/pkg/kube/podtemplatespec/podspec_template.go index 7db97b791..8b765af9d 100644 --- a/pkg/kube/podtemplatespec/podspec_template.go +++ b/pkg/kube/podtemplatespec/podspec_template.go @@ -223,7 +223,7 @@ func WithVolumeMounts(containerName string, volumeMounts ...corev1.VolumeMount) } } -func mergeVolumeMounts(defaultMounts, overrideMounts []corev1.VolumeMount) ([]corev1.VolumeMount, error) { +func MergeVolumeMounts(defaultMounts, overrideMounts []corev1.VolumeMount) ([]corev1.VolumeMount, error) { defaultMountsMap := createMountsMap(defaultMounts) overrideMountsMap := createMountsMap(overrideMounts) mergedVolumeMounts := []corev1.VolumeMount{} @@ -254,42 +254,41 @@ func createMountsMap(volumeMounts []corev1.VolumeMount) map[string]corev1.Volume return mountMap } -func mergeContainers(defaultContainers, overrideContainers []corev1.Container) ([]corev1.Container, error) { - +func MergeContainers(defaultContainers, customContainers []corev1.Container) ([]corev1.Container, error) { defaultMap := createContainerMap(defaultContainers) - overrideMap := createContainerMap(overrideContainers) + customMap := createContainerMap(customContainers) mergedContainers := []corev1.Container{} for _, defaultContainer := range defaultContainers { - if overrideContainer, ok := overrideMap[defaultContainer.Name]; ok { + if customContainer, ok := customMap[defaultContainer.Name]; ok { // Merge // Merge mounts - mergedMounts, err := mergeVolumeMounts(defaultContainer.VolumeMounts, overrideContainer.VolumeMounts) + mergedMounts, err := MergeVolumeMounts(defaultContainer.VolumeMounts, customContainer.VolumeMounts) if err != nil { return nil, err } - if err := mergo.Merge(&defaultContainer, overrideContainer, mergo.WithOverride); err != nil { + if err := mergo.Merge(&defaultContainer, customContainer, mergo.WithOverride); err != nil { return nil, err } - // completely override any resources that were provided + // completely custom any resources that were provided // this prevents issues with custom requests giving errors due // to the defaulted limits - defaultContainer.Resources = overrideContainer.Resources + defaultContainer.Resources = customContainer.Resources defaultContainer.VolumeMounts = mergedMounts } // Need to add it mergedContainers = append(mergedContainers, defaultContainer) } - // Look for overrideContainers that were not merged - for _, overrideContainer := range overrideContainers { - if _, ok := defaultMap[overrideContainer.Name]; ok { + // Look for customContainers that were not merged + for _, customContainer := range customContainers { + if _, ok := defaultMap[customContainer.Name]; ok { continue } // Need to add it - mergedContainers = append(mergedContainers, overrideContainer) + mergedContainers = append(mergedContainers, customContainer) } - return defaultContainers, nil + return mergedContainers, nil } func createContainerMap(containers []corev1.Container) map[string]corev1.Container { @@ -315,12 +314,13 @@ func mergeAffinity(defaultAffinity, overrideAffinity *corev1.Affinity) (*corev1. } func MergePodTemplateSpecs(defaultTemplate, overrideTemplate corev1.PodTemplateSpec) (corev1.PodTemplateSpec, error) { - mergedContainers, err := mergeContainers(defaultTemplate.Spec.Containers, overrideTemplate.Spec.Containers) + + mergedContainers, err := MergeContainers(defaultTemplate.Spec.Containers, overrideTemplate.Spec.Containers) if err != nil { return corev1.PodTemplateSpec{}, err } - mergedInitContainers, err := mergeContainers(defaultTemplate.Spec.InitContainers, overrideTemplate.Spec.InitContainers) + mergedInitContainers, err := MergeContainers(defaultTemplate.Spec.InitContainers, overrideTemplate.Spec.InitContainers) if err != nil { return corev1.PodTemplateSpec{}, err } diff --git a/pkg/kube/statefulset/statefulset.go b/pkg/kube/statefulset/statefulset.go index 4dbc8c366..564a4b04d 100644 --- a/pkg/kube/statefulset/statefulset.go +++ b/pkg/kube/statefulset/statefulset.go @@ -1,8 +1,6 @@ package statefulset import ( - "fmt" - "os" "sort" "github.com/imdario/mergo" @@ -268,7 +266,7 @@ func createVolumeClaimMap(volumeMounts []corev1.PersistentVolumeClaim) map[strin return mountMap } -func mergeVolumeClaimTemplates(defaultTemplates []corev1.PersistentVolumeClaim, overrideTemplates []corev1.PersistentVolumeClaim) ([]corev1.PersistentVolumeClaim, error) { +func MergeVolumeClaimTemplates(defaultTemplates []corev1.PersistentVolumeClaim, overrideTemplates []corev1.PersistentVolumeClaim) ([]corev1.PersistentVolumeClaim, error) { defaultMountsMap := createVolumeClaimMap(defaultTemplates) overrideMountsMap := createVolumeClaimMap(overrideTemplates) var mergedVolumes []corev1.PersistentVolumeClaim @@ -296,26 +294,21 @@ func mergeVolumeClaimTemplates(defaultTemplates []corev1.PersistentVolumeClaim, func mergeStatefulSetSpecs(defaultSpec, overrideSpec appsv1.StatefulSetSpec) (appsv1.StatefulSetSpec, error) { - fmt.Fprintf(os.Stderr, "\n\nDefaultSpec: %+v\n\n", defaultSpec) - - fmt.Fprintf(os.Stderr, "\n\nOverrideSpec: %+v\n\n", overrideSpec) - // PodTemplateSpec needs to be manually merged mergedPodTemplateSpec, err := podtemplatespec.MergePodTemplateSpecs(defaultSpec.Template, overrideSpec.Template) if err != nil { return appsv1.StatefulSetSpec{}, err } - defaultSpec.Template = mergedPodTemplateSpec // VolumeClaimTemplates needs to be manually merged - mergedVolumeClaimTemplates, err := mergeVolumeClaimTemplates(defaultSpec.VolumeClaimTemplates, overrideSpec.VolumeClaimTemplates) + mergedVolumeClaimTemplates, err := MergeVolumeClaimTemplates(defaultSpec.VolumeClaimTemplates, overrideSpec.VolumeClaimTemplates) - defaultSpec.VolumeClaimTemplates = mergedVolumeClaimTemplates // Merging the rest with mergo if err := mergo.Merge(&defaultSpec, overrideSpec, mergo.WithOverride); err != nil { return appsv1.StatefulSetSpec{}, err } - fmt.Fprintf(os.Stderr, "\n\nAfterMerge: %+v\n\n", defaultSpec) + defaultSpec.Template = mergedPodTemplateSpec + defaultSpec.VolumeClaimTemplates = mergedVolumeClaimTemplates return defaultSpec, nil } diff --git a/pkg/kube/statefulset/statefulset_test.go b/pkg/kube/statefulset/statefulset_test.go index d194e5775..0ec154f2b 100644 --- a/pkg/kube/statefulset/statefulset_test.go +++ b/pkg/kube/statefulset/statefulset_test.go @@ -2,8 +2,10 @@ package statefulset import ( "fmt" + "reflect" "testing" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/podtemplatespec" "github.com/stretchr/testify/assert" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -204,3 +206,311 @@ func TestCreateVolumeMountWithMultipleOptions(t *testing.T) { assert.Equal(t, mount.SubPath, "our-subpath") assert.True(t, mount.ReadOnly) } + +func TestMergeVolumeMounts(t *testing.T) { + vol0 := corev1.VolumeMount{Name: "container-0.volume-mount-0"} + vol1 := corev1.VolumeMount{Name: "another-mount"} + volumeMounts := []corev1.VolumeMount{vol0, vol1} + var mergedVolumeMounts []corev1.VolumeMount + var err error + + mergedVolumeMounts, err = podtemplatespec.MergeVolumeMounts(nil, volumeMounts) + assert.NoError(t, err) + assert.Equal(t, []corev1.VolumeMount{vol0, vol1}, mergedVolumeMounts) + + vol2 := vol1 + vol2.MountPath = "/somewhere" + mergedVolumeMounts, err = podtemplatespec.MergeVolumeMounts([]corev1.VolumeMount{vol2}, []corev1.VolumeMount{vol0, vol1}) + assert.NoError(t, err) + assert.Equal(t, []corev1.VolumeMount{vol2, vol0}, mergedVolumeMounts) +} + +func TestMergeContainer(t *testing.T) { + vol0 := corev1.VolumeMount{Name: "container-0.volume-mount-0"} + sideCarVol := corev1.VolumeMount{Name: "container-1.volume-mount-0"} + + anotherVol := corev1.VolumeMount{Name: "another-mount"} + + overrideDefaultContainer := corev1.Container{Name: "container-0"} + overrideDefaultContainer.Image = "overridden" + overrideDefaultContainer.ReadinessProbe = &corev1.Probe{PeriodSeconds: 20} + + otherDefaultContainer := getDefaultContainer() + otherDefaultContainer.Name = "default-side-car" + otherDefaultContainer.VolumeMounts = []corev1.VolumeMount{sideCarVol} + + overrideOtherDefaultContainer := otherDefaultContainer + overrideOtherDefaultContainer.Env = []corev1.EnvVar{{Name: "env_var", Value: "xxx"}} + overrideOtherDefaultContainer.VolumeMounts = []corev1.VolumeMount{anotherVol} + + mergedContainers, err := podtemplatespec.MergeContainers( + []corev1.Container{getDefaultContainer(), otherDefaultContainer}, + []corev1.Container{getCustomContainer(), overrideDefaultContainer, overrideOtherDefaultContainer}, + ) + + assert.NoError(t, err) + assert.Len(t, mergedContainers, 3) + + assert.Equal(t, getCustomContainer(), mergedContainers[2]) + + mergedDefaultContainer := mergedContainers[0] + assert.Equal(t, "container-0", mergedDefaultContainer.Name) + assert.Equal(t, []corev1.VolumeMount{vol0}, mergedDefaultContainer.VolumeMounts) + assert.Equal(t, "overridden", mergedDefaultContainer.Image) + // only "periodSeconds" was overwritten - other fields stayed untouched + assert.Equal(t, corev1.Handler{HTTPGet: &corev1.HTTPGetAction{Path: "/foo"}}, mergedDefaultContainer.ReadinessProbe.Handler) + assert.Equal(t, int32(20), mergedDefaultContainer.ReadinessProbe.PeriodSeconds) + + mergedOtherContainer := mergedContainers[1] + assert.Equal(t, "default-side-car", mergedOtherContainer.Name) + assert.Equal(t, []corev1.VolumeMount{sideCarVol, anotherVol}, mergedOtherContainer.VolumeMounts) + assert.Len(t, mergedOtherContainer.Env, 1) + assert.Equal(t, "env_var", mergedOtherContainer.Env[0].Name) + assert.Equal(t, "xxx", mergedOtherContainer.Env[0].Value) +} + +func TestMergePodSpecsEmptyCustom(t *testing.T) { + + defaultPodSpec := getDefaultPodSpec() + customPodSpecTemplate := corev1.PodTemplateSpec{} + + mergedPodTemplateSpec, err := podtemplatespec.MergePodTemplateSpecs(defaultPodSpec, customPodSpecTemplate) + + assert.NoError(t, err) + assert.Equal(t, "my-default-service-account", mergedPodTemplateSpec.Spec.ServiceAccountName) + assert.Equal(t, int64Ref(12), mergedPodTemplateSpec.Spec.TerminationGracePeriodSeconds) + + assert.Equal(t, "my-default-name", mergedPodTemplateSpec.ObjectMeta.Name) + assert.Equal(t, "my-default-namespace", mergedPodTemplateSpec.ObjectMeta.Namespace) + assert.Equal(t, int64Ref(10), mergedPodTemplateSpec.Spec.ActiveDeadlineSeconds) + + // ensure collections have been merged + assert.Contains(t, mergedPodTemplateSpec.Spec.NodeSelector, "node-0") + assert.Len(t, mergedPodTemplateSpec.Spec.Containers, 1) + assert.Equal(t, "container-0", mergedPodTemplateSpec.Spec.Containers[0].Name) + assert.Equal(t, "image-0", mergedPodTemplateSpec.Spec.Containers[0].Image) + assert.Equal(t, "container-0.volume-mount-0", mergedPodTemplateSpec.Spec.Containers[0].VolumeMounts[0].Name) + assert.Len(t, mergedPodTemplateSpec.Spec.InitContainers, 1) + assert.Equal(t, "init-container-default", mergedPodTemplateSpec.Spec.InitContainers[0].Name) +} + +func TestMergePodSpecsEmptyDefault(t *testing.T) { + + defaultPodSpec := corev1.PodTemplateSpec{} + customPodSpecTemplate := getCustomPodSpec() + + mergedPodTemplateSpec, err := podtemplatespec.MergePodTemplateSpecs(defaultPodSpec, customPodSpecTemplate) + + assert.NoError(t, err) + assert.Equal(t, "my-service-account-override", mergedPodTemplateSpec.Spec.ServiceAccountName) + assert.Equal(t, int64Ref(11), mergedPodTemplateSpec.Spec.TerminationGracePeriodSeconds) + assert.Equal(t, "my-node-name", mergedPodTemplateSpec.Spec.NodeName) + assert.Equal(t, corev1.RestartPolicy("Always"), mergedPodTemplateSpec.Spec.RestartPolicy) + + assert.Len(t, mergedPodTemplateSpec.Spec.Containers, 1) + assert.Equal(t, "container-1", mergedPodTemplateSpec.Spec.Containers[0].Name) + assert.Equal(t, "image-1", mergedPodTemplateSpec.Spec.Containers[0].Image) + assert.Len(t, mergedPodTemplateSpec.Spec.InitContainers, 1) + assert.Equal(t, "init-container-custom", mergedPodTemplateSpec.Spec.InitContainers[0].Name) + +} + +func TestMergePodSpecsBoth(t *testing.T) { + + defaultPodSpec := getDefaultPodSpec() + customPodSpecTemplate := getCustomPodSpec() + + var mergedPodTemplateSpec corev1.PodTemplateSpec + var err error + + // multiple merges must give the same result + for i := 0; i < 3; i++ { + mergedPodTemplateSpec, err = podtemplatespec.MergePodTemplateSpecs(defaultPodSpec, customPodSpecTemplate) + + assert.NoError(t, err) + // ensure values that were specified in the custom pod spec template remain unchanged + assert.Equal(t, "my-service-account-override", mergedPodTemplateSpec.Spec.ServiceAccountName) + assert.Equal(t, int64Ref(11), mergedPodTemplateSpec.Spec.TerminationGracePeriodSeconds) + assert.Equal(t, "my-node-name", mergedPodTemplateSpec.Spec.NodeName) + assert.Equal(t, corev1.RestartPolicy("Always"), mergedPodTemplateSpec.Spec.RestartPolicy) + + // ensure values from the default pod spec template have been merged in + assert.Equal(t, "my-default-name", mergedPodTemplateSpec.ObjectMeta.Name) + assert.Equal(t, "my-default-namespace", mergedPodTemplateSpec.ObjectMeta.Namespace) + assert.Equal(t, int64Ref(10), mergedPodTemplateSpec.Spec.ActiveDeadlineSeconds) + + // ensure collections have been merged + assert.Contains(t, mergedPodTemplateSpec.Spec.NodeSelector, "node-0") + assert.Contains(t, mergedPodTemplateSpec.Spec.NodeSelector, "node-1") + assert.Len(t, mergedPodTemplateSpec.Spec.Containers, 2) + assert.Equal(t, "container-0", mergedPodTemplateSpec.Spec.Containers[0].Name) + assert.Equal(t, "image-0", mergedPodTemplateSpec.Spec.Containers[0].Image) + assert.Equal(t, "container-0.volume-mount-0", mergedPodTemplateSpec.Spec.Containers[0].VolumeMounts[0].Name) + assert.Equal(t, "container-1", mergedPodTemplateSpec.Spec.Containers[1].Name) + assert.Equal(t, "image-1", mergedPodTemplateSpec.Spec.Containers[1].Image) + assert.Len(t, mergedPodTemplateSpec.Spec.InitContainers, 2) + assert.Equal(t, "init-container-default", mergedPodTemplateSpec.Spec.InitContainers[0].Name) + assert.Equal(t, "init-container-custom", mergedPodTemplateSpec.Spec.InitContainers[1].Name) + + // ensure labels were appended + assert.Len(t, mergedPodTemplateSpec.Labels, 2) + assert.Contains(t, mergedPodTemplateSpec.Labels, "app") + assert.Contains(t, mergedPodTemplateSpec.Labels, "custom") + + // ensure the pointers are not the same + assert.NotEqual(t, mergedPodTemplateSpec.Spec.Affinity, defaultPodSpec.Spec.Affinity) + + // ensure the affinity rules slices were overridden + assert.Equal(t, affinity("zone", "custom"), mergedPodTemplateSpec.Spec.Affinity) + } +} + +func TestGetMergedDefaultPodSpecTemplate(t *testing.T) { + var err error + + dbPodSpecTemplate := getDefaultPodSpec() + var mergedPodSpecTemplate corev1.PodTemplateSpec + + // nothing to merge + mergedPodSpecTemplate, err = podtemplatespec.MergePodTemplateSpecs(corev1.PodTemplateSpec{}, dbPodSpecTemplate) + assert.NoError(t, err) + assert.Equal(t, mergedPodSpecTemplate, dbPodSpecTemplate) + assert.Len(t, mergedPodSpecTemplate.Spec.Containers, 1) + assertContainersEqualBarResources(t, mergedPodSpecTemplate.Spec.Containers[0], dbPodSpecTemplate.Spec.Containers[0]) + + extraContainer := corev1.Container{ + Name: "extra-container", + Image: "container-image", + } + + newPodSpecTemplate := corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{extraContainer}, + }, + } + + // with a side car container + mergedPodSpecTemplate, err = podtemplatespec.MergePodTemplateSpecs(newPodSpecTemplate, dbPodSpecTemplate) + assert.NoError(t, err) + assert.Len(t, mergedPodSpecTemplate.Spec.Containers, 2) + assertContainersEqualBarResources(t, mergedPodSpecTemplate.Spec.Containers[1], dbPodSpecTemplate.Spec.Containers[0]) + assertContainersEqualBarResources(t, mergedPodSpecTemplate.Spec.Containers[0], extraContainer) +} + +func assertContainersEqualBarResources(t *testing.T, self corev1.Container, other corev1.Container) { + // Copied fields from k8s.io/api/core/v1/types.go + assert.Equal(t, self.Name, other.Name) + assert.Equal(t, self.Image, other.Image) + assert.True(t, reflect.DeepEqual(self.Command, other.Command)) + assert.True(t, reflect.DeepEqual(self.Args, other.Args)) + assert.Equal(t, self.WorkingDir, other.WorkingDir) + assert.True(t, reflect.DeepEqual(self.Ports, other.Ports)) + assert.True(t, reflect.DeepEqual(self.EnvFrom, other.EnvFrom)) + assert.True(t, reflect.DeepEqual(self.Env, other.Env)) + assert.True(t, reflect.DeepEqual(self.Resources, other.Resources)) + assert.True(t, reflect.DeepEqual(self.VolumeMounts, other.VolumeMounts)) + assert.True(t, reflect.DeepEqual(self.VolumeDevices, other.VolumeDevices)) + assert.Equal(t, self.LivenessProbe, other.LivenessProbe) + assert.Equal(t, self.ReadinessProbe, other.ReadinessProbe) + assert.Equal(t, self.Lifecycle, other.Lifecycle) + assert.Equal(t, self.TerminationMessagePath, other.TerminationMessagePath) + assert.Equal(t, self.TerminationMessagePolicy, other.TerminationMessagePolicy) + assert.Equal(t, self.ImagePullPolicy, other.ImagePullPolicy) + assert.Equal(t, self.SecurityContext, other.SecurityContext) + assert.Equal(t, self.Stdin, other.Stdin) + assert.Equal(t, self.StdinOnce, other.StdinOnce) + assert.Equal(t, self.TTY, other.TTY) +} + +func int64Ref(i int64) *int64 { + return &i +} + +func getDefaultPodSpec() corev1.PodTemplateSpec { + initContainer := getDefaultContainer() + initContainer.Name = "init-container-default" + return corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-default-name", + Namespace: "my-default-namespace", + Labels: map[string]string{"app": "operator"}, + }, + Spec: corev1.PodSpec{ + NodeSelector: map[string]string{ + "node-0": "node-0", + }, + ServiceAccountName: "my-default-service-account", + TerminationGracePeriodSeconds: int64Ref(12), + ActiveDeadlineSeconds: int64Ref(10), + Containers: []corev1.Container{getDefaultContainer()}, + InitContainers: []corev1.Container{initContainer}, + Affinity: affinity("hostname", "default"), + }, + } +} + +func getCustomPodSpec() corev1.PodTemplateSpec { + initContainer := getCustomContainer() + initContainer.Name = "init-container-custom" + return corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"custom": "some"}, + }, + Spec: corev1.PodSpec{ + NodeSelector: map[string]string{ + "node-1": "node-1", + }, + ServiceAccountName: "my-service-account-override", + TerminationGracePeriodSeconds: int64Ref(11), + NodeName: "my-node-name", + RestartPolicy: corev1.RestartPolicyAlways, + Containers: []corev1.Container{getCustomContainer()}, + InitContainers: []corev1.Container{initContainer}, + Affinity: affinity("zone", "custom"), + }, + } +} + +func affinity(antiAffinityKey, nodeAffinityKey string) *corev1.Affinity { + return &corev1.Affinity{ + PodAntiAffinity: &corev1.PodAntiAffinity{ + PreferredDuringSchedulingIgnoredDuringExecution: []corev1.WeightedPodAffinityTerm{{ + PodAffinityTerm: corev1.PodAffinityTerm{ + TopologyKey: antiAffinityKey, + }, + }}, + }, + NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{NodeSelectorTerms: []corev1.NodeSelectorTerm{{ + MatchFields: []corev1.NodeSelectorRequirement{{ + Key: nodeAffinityKey, + }}, + }}}, + }, + } +} + +func getDefaultContainer() corev1.Container { + return corev1.Container{ + Name: "container-0", + Image: "image-0", + ReadinessProbe: &corev1.Probe{ + Handler: corev1.Handler{HTTPGet: &corev1.HTTPGetAction{ + Path: "/foo", + }}, + PeriodSeconds: 10, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "container-0.volume-mount-0", + }, + }, + } +} + +func getCustomContainer() corev1.Container { + return corev1.Container{ + Name: "container-1", + Image: "image-1", + } +} diff --git a/test/e2e/e2eutil.go b/test/e2e/e2eutil.go index ed6518250..0ba7ee142 100644 --- a/test/e2e/e2eutil.go +++ b/test/e2e/e2eutil.go @@ -1,7 +1,9 @@ package e2eutil import ( + "bytes" "context" + "encoding/json" "fmt" "testing" "time" @@ -9,6 +11,7 @@ import ( mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/pkg/apis/mongodb/v1" "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/statefulset" f "github.com/operator-framework/operator-sdk/pkg/test" + "github.com/stretchr/testify/assert" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -74,6 +77,17 @@ func WaitForStatefulSetToHaveUpdateStrategy(t *testing.T, mdb *mdbv1.MongoDB, st }) } +// WaitForStatefulSetToHaveExpectedContainers waits until the statefulSet has the exptected Containers +func WaitForStatefulSetToHaveExpectedContainers(t *testing.T, mdb *mdbv1.MongoDB, containers []corev1.Container, retryInterval, timeout time.Duration) error { + return waitForStatefulSetCondition(t, mdb, retryInterval, timeout, func(sts appsv1.StatefulSet) bool { + currentStsContainersBytes, err := json.Marshal(sts.Spec.Template.Spec.Containers) + assert.NoError(t, err) + expectedContainersBytes, err := json.Marshal(containers) + assert.NoError(t, err) + return bytes.Equal(currentStsContainersBytes, expectedContainersBytes) + }) +} + // WaitForStatefulSetToBeReady waits until all replicas of the StatefulSet with the given name // have reached the ready status func WaitForStatefulSetToBeReady(t *testing.T, mdb *mdbv1.MongoDB, retryInterval, timeout time.Duration) error { diff --git a/test/e2e/mongodbtests/mongodbtests.go b/test/e2e/mongodbtests/mongodbtests.go index 398d277a8..88d7287a3 100644 --- a/test/e2e/mongodbtests/mongodbtests.go +++ b/test/e2e/mongodbtests/mongodbtests.go @@ -289,3 +289,13 @@ func IsReachableDuringWithConnection(mdb *mdbv1.MongoDB, interval time.Duration, testFunc() } } + +func StatefulSetHasExpectedContainers(mdb *mdbv1.MongoDB, containers []corev1.Container) func(*testing.T) { + return func(t *testing.T) { + err := e2eutil.WaitForStatefulSetToHaveExpectedContainers(t, mdb, containers, time.Second*5, time.Minute*5) + if err != nil { + t.Fatal(err) + } + t.Logf("StatefulSet %s/%s has the expected containers!", mdb.Namespace, mdb.Name) + } +} diff --git a/test/e2e/statefulset_arbitrary_config/statefulset_arbitrary_config_test.go b/test/e2e/statefulset_arbitrary_config/statefulset_arbitrary_config_test.go new file mode 100644 index 000000000..12ddd23cd --- /dev/null +++ b/test/e2e/statefulset_arbitrary_config/statefulset_arbitrary_config_test.go @@ -0,0 +1,63 @@ +package statefulset_arbitrary_config + +import ( + "context" + "testing" + + "github.com/golangplus/testing/assert" + v1 "github.com/mongodb/mongodb-kubernetes-operator/pkg/apis/mongodb/v1" + e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" + "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/mongodbtests" + setup "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" + f "github.com/operator-framework/operator-sdk/pkg/test" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" +) + +func TestMain(m *testing.M) { + f.MainEntry(m) +} + +func findIndexByName(name string, containers []corev1.Container) int { + for idx, c := range containers { + if c.Name == name { + return idx + } + } + return -1 +} + +func TestStatefulSetArbitraryConfig(t *testing.T) { + ctx, shouldCleanup := setup.InitTest(t) + + if shouldCleanup { + defer ctx.Cleanup() + } + mdb, user := e2eutil.NewTestMongoDB("mdb0") + + _, err := setup.GeneratePasswordForUser(user, ctx) + if err != nil { + t.Fatal(err) + } + + t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) + t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb)) + t.Run("Test Basic Connectivity", mongodbtests.Connectivity(&mdb)) + t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 1)) + + // Get the original containers + originalSts := &appsv1.StatefulSet{} + err = f.Global.Client.Get(context.TODO(), mdb.NamespacedName(), originalSts) + assert.NoError(t, err) + expectedContainers := originalSts.Spec.Template.Spec.Containers + overrideSpec := v1.StatefulSetConfiguration{} + overrideSpec.Spec.Template.Spec.Containers = []corev1.Container{ + {Name: "mongodb-agent", ReadinessProbe: &corev1.Probe{TimeoutSeconds: 100}}} + + idx := findIndexByName("mongodb-agent", expectedContainers) + expectedContainers[idx].ReadinessProbe.TimeoutSeconds = 100 + e2eutil.UpdateMongoDBResource(&mdb, func(mdb *v1.MongoDB) { mdb.Spec.StatefulSetConfiguration = overrideSpec }) + + t.Run("Container has been merged by name", mongodbtests.StatefulSetHasExpectedContainers(&mdb, expectedContainers)) + +} From d8476eda0d6fb539dc226ed735fa4ed77f28b5da Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Tue, 21 Jul 2020 17:48:59 +0100 Subject: [PATCH 08/34] wip --- pkg/kube/podtemplatespec/podspec_template.go | 20 ++++++++++++-------- pkg/kube/statefulset/statefulset.go | 4 +++- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/pkg/kube/podtemplatespec/podspec_template.go b/pkg/kube/podtemplatespec/podspec_template.go index 8b765af9d..7ebaff6d3 100644 --- a/pkg/kube/podtemplatespec/podspec_template.go +++ b/pkg/kube/podtemplatespec/podspec_template.go @@ -227,10 +227,10 @@ func MergeVolumeMounts(defaultMounts, overrideMounts []corev1.VolumeMount) ([]co defaultMountsMap := createMountsMap(defaultMounts) overrideMountsMap := createMountsMap(overrideMounts) mergedVolumeMounts := []corev1.VolumeMount{} - for _, defaultMount := range defaultMounts { + for idx, defaultMount := range defaultMounts { if overrideMount, ok := overrideMountsMap[defaultMount.Name]; ok { // needs merge - if err := mergo.Merge(&defaultMount, overrideMount, mergo.WithAppendSlice); err != nil { + if err := mergo.Merge(defaultMounts[idx], overrideMount, mergo.WithAppendSlice); err != nil { return nil, err } } @@ -258,28 +258,28 @@ func MergeContainers(defaultContainers, customContainers []corev1.Container) ([] defaultMap := createContainerMap(defaultContainers) customMap := createContainerMap(customContainers) mergedContainers := []corev1.Container{} - for _, defaultContainer := range defaultContainers { + for idx, defaultContainer := range defaultContainers { if customContainer, ok := customMap[defaultContainer.Name]; ok { - // Merge + // The container is present in both maps, so we need to merge // Merge mounts mergedMounts, err := MergeVolumeMounts(defaultContainer.VolumeMounts, customContainer.VolumeMounts) if err != nil { return nil, err } - if err := mergo.Merge(&defaultContainer, customContainer, mergo.WithOverride); err != nil { + if err := mergo.Merge(&defaultContainers[idx], customContainer, mergo.WithOverride); err != nil { return nil, err } - // completely custom any resources that were provided + // completely override any resources that were provided // this prevents issues with custom requests giving errors due // to the defaulted limits defaultContainer.Resources = customContainer.Resources defaultContainer.VolumeMounts = mergedMounts } - // Need to add it + // The default container was not modified by the override, so just add it mergedContainers = append(mergedContainers, defaultContainer) } - // Look for customContainers that were not merged + // Look for customContainers that were not merged into existing ones for _, customContainer := range customContainers { if _, ok := defaultMap[customContainer.Name]; ok { continue @@ -315,21 +315,25 @@ func mergeAffinity(defaultAffinity, overrideAffinity *corev1.Affinity) (*corev1. func MergePodTemplateSpecs(defaultTemplate, overrideTemplate corev1.PodTemplateSpec) (corev1.PodTemplateSpec, error) { + // Containers need to be merged manually mergedContainers, err := MergeContainers(defaultTemplate.Spec.Containers, overrideTemplate.Spec.Containers) if err != nil { return corev1.PodTemplateSpec{}, err } + // InitContainers need to be merged manually mergedInitContainers, err := MergeContainers(defaultTemplate.Spec.InitContainers, overrideTemplate.Spec.InitContainers) if err != nil { return corev1.PodTemplateSpec{}, err } + // Affinity needs to be merged manually mergedAffinity, err := mergeAffinity(defaultTemplate.Spec.Affinity, overrideTemplate.Spec.Affinity) if err != nil { return corev1.PodTemplateSpec{}, err } + // Everything else can be merged with mergo mergedPodTemplateSpec := *defaultTemplate.DeepCopy() if err = mergo.Merge(&mergedPodTemplateSpec, overrideTemplate, mergo.WithOverride, mergo.WithAppendSlice); err != nil { return corev1.PodTemplateSpec{}, err diff --git a/pkg/kube/statefulset/statefulset.go b/pkg/kube/statefulset/statefulset.go index 564a4b04d..c6fb20590 100644 --- a/pkg/kube/statefulset/statefulset.go +++ b/pkg/kube/statefulset/statefulset.go @@ -299,14 +299,16 @@ func mergeStatefulSetSpecs(defaultSpec, overrideSpec appsv1.StatefulSetSpec) (ap if err != nil { return appsv1.StatefulSetSpec{}, err } + // VolumeClaimTemplates needs to be manually merged mergedVolumeClaimTemplates, err := MergeVolumeClaimTemplates(defaultSpec.VolumeClaimTemplates, overrideSpec.VolumeClaimTemplates) // Merging the rest with mergo - if err := mergo.Merge(&defaultSpec, overrideSpec, mergo.WithOverride); err != nil { return appsv1.StatefulSetSpec{}, err } + + // Assigning merged vales AFTER the merge with mergo or they would be overwritten defaultSpec.Template = mergedPodTemplateSpec defaultSpec.VolumeClaimTemplates = mergedVolumeClaimTemplates return defaultSpec, nil From 7c1d559af19c32e6f63422a64ee5cea8e0d5e40c Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Tue, 21 Jul 2020 18:02:42 +0100 Subject: [PATCH 09/34] refactored tests --- deploy/operator.yaml | 6 ++--- test/e2e/e2eutil.go | 23 +++++++++++-------- test/e2e/mongodbtests/mongodbtests.go | 6 ++--- .../statefulset_arbitrary_config_test.go | 15 ++---------- 4 files changed, 21 insertions(+), 29 deletions(-) diff --git a/deploy/operator.yaml b/deploy/operator.yaml index 4fb2b66ff..3fe97bcfe 100644 --- a/deploy/operator.yaml +++ b/deploy/operator.yaml @@ -15,7 +15,7 @@ spec: serviceAccountName: mongodb-kubernetes-operator containers: - name: mongodb-kubernetes-operator - image: localhost:5000/mongodb-kubernetes-operator + image: quay.io/mongodb/mongodb-kubernetes-operator:0.0.8 command: - mongodb-kubernetes-operator imagePullPolicy: Always @@ -32,5 +32,5 @@ spec: value: "mongodb-kubernetes-operator" - name: AGENT_IMAGE # The MongoDB Agent the operator will deploy to manage MongoDB deployments value: quay.io/mongodb/mongodb-agent:10.15.1.6468-1 - - name: VERSION_UPGRADE_HOOK_IMAGE - value: quay.io/mongodb/mongodb-kubernetes-operator-version-upgrade-post-start-hook:1.0.2 + - name: PRE_STOP_HOOK_IMAGE + value: quay.io/mongodb/mongodb-kubernetes-operator-pre-stop-hook:1.0.1 diff --git a/test/e2e/e2eutil.go b/test/e2e/e2eutil.go index 0ba7ee142..5ba73d256 100644 --- a/test/e2e/e2eutil.go +++ b/test/e2e/e2eutil.go @@ -1,9 +1,7 @@ package e2eutil import ( - "bytes" "context" - "encoding/json" "fmt" "testing" "time" @@ -11,7 +9,6 @@ import ( mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/pkg/apis/mongodb/v1" "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/statefulset" f "github.com/operator-framework/operator-sdk/pkg/test" - "github.com/stretchr/testify/assert" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -77,14 +74,11 @@ func WaitForStatefulSetToHaveUpdateStrategy(t *testing.T, mdb *mdbv1.MongoDB, st }) } -// WaitForStatefulSetToHaveExpectedContainers waits until the statefulSet has the exptected Containers -func WaitForStatefulSetToHaveExpectedContainers(t *testing.T, mdb *mdbv1.MongoDB, containers []corev1.Container, retryInterval, timeout time.Duration) error { +// WaitForStatefulSetToHaveExpectedContainerValue waits until the passed +func WaitForStatefulSetToHaveExpectedContainerCondition(t *testing.T, mdb *mdbv1.MongoDB, containerName string, condition func(container corev1.Container) bool, retryInterval, timeout time.Duration) error { return waitForStatefulSetCondition(t, mdb, retryInterval, timeout, func(sts appsv1.StatefulSet) bool { - currentStsContainersBytes, err := json.Marshal(sts.Spec.Template.Spec.Containers) - assert.NoError(t, err) - expectedContainersBytes, err := json.Marshal(containers) - assert.NoError(t, err) - return bytes.Equal(currentStsContainersBytes, expectedContainersBytes) + idx := findIndexByName(containerName, sts.Spec.Template.Spec.Containers) + return idx != -1 && condition(sts.Spec.Template.Spec.Containers[idx]) }) } @@ -189,3 +183,12 @@ func NewTestTLSConfig(optional bool) mdbv1.TLS { }, } } + +func findIndexByName(name string, containers []corev1.Container) int { + for idx, c := range containers { + if c.Name == name { + return idx + } + } + return -1 +} diff --git a/test/e2e/mongodbtests/mongodbtests.go b/test/e2e/mongodbtests/mongodbtests.go index 88d7287a3..7c4b0ec50 100644 --- a/test/e2e/mongodbtests/mongodbtests.go +++ b/test/e2e/mongodbtests/mongodbtests.go @@ -290,12 +290,12 @@ func IsReachableDuringWithConnection(mdb *mdbv1.MongoDB, interval time.Duration, } } -func StatefulSetHasExpectedContainers(mdb *mdbv1.MongoDB, containers []corev1.Container) func(*testing.T) { +func StatefulSetContainerConditionIsTrue(mdb *mdbv1.MongoDB, containerName string, condition func(container corev1.Container) bool) func(*testing.T) { return func(t *testing.T) { - err := e2eutil.WaitForStatefulSetToHaveExpectedContainers(t, mdb, containers, time.Second*5, time.Minute*5) + err := e2eutil.WaitForStatefulSetToHaveExpectedContainerCondition(t, mdb, containerName, condition, time.Second*5, time.Minute*5) if err != nil { t.Fatal(err) } - t.Logf("StatefulSet %s/%s has the expected containers!", mdb.Namespace, mdb.Name) + t.Logf("StatefulSet %s/%s has the expected container value!", mdb.Namespace, mdb.Name) } } diff --git a/test/e2e/statefulset_arbitrary_config/statefulset_arbitrary_config_test.go b/test/e2e/statefulset_arbitrary_config/statefulset_arbitrary_config_test.go index 12ddd23cd..90571ff37 100644 --- a/test/e2e/statefulset_arbitrary_config/statefulset_arbitrary_config_test.go +++ b/test/e2e/statefulset_arbitrary_config/statefulset_arbitrary_config_test.go @@ -18,15 +18,6 @@ func TestMain(m *testing.M) { f.MainEntry(m) } -func findIndexByName(name string, containers []corev1.Container) int { - for idx, c := range containers { - if c.Name == name { - return idx - } - } - return -1 -} - func TestStatefulSetArbitraryConfig(t *testing.T) { ctx, shouldCleanup := setup.InitTest(t) @@ -49,15 +40,13 @@ func TestStatefulSetArbitraryConfig(t *testing.T) { originalSts := &appsv1.StatefulSet{} err = f.Global.Client.Get(context.TODO(), mdb.NamespacedName(), originalSts) assert.NoError(t, err) - expectedContainers := originalSts.Spec.Template.Spec.Containers + overrideSpec := v1.StatefulSetConfiguration{} overrideSpec.Spec.Template.Spec.Containers = []corev1.Container{ {Name: "mongodb-agent", ReadinessProbe: &corev1.Probe{TimeoutSeconds: 100}}} - idx := findIndexByName("mongodb-agent", expectedContainers) - expectedContainers[idx].ReadinessProbe.TimeoutSeconds = 100 e2eutil.UpdateMongoDBResource(&mdb, func(mdb *v1.MongoDB) { mdb.Spec.StatefulSetConfiguration = overrideSpec }) - t.Run("Container has been merged by name", mongodbtests.StatefulSetHasExpectedContainers(&mdb, expectedContainers)) + t.Run("Container has been merged by name", mongodbtests.StatefulSetContainerConditionIsTrue(mdb, "mongodb-agent", func(container corev1.Container) { return container.ReadinessProbe.TimeoutSeconds == 100 })) } From ee8d748519112dfb8e003649f49d7943e91b981f Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Tue, 21 Jul 2020 18:05:33 +0100 Subject: [PATCH 10/34] reverted change --- pkg/controller/mongodb/replica_set_controller.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pkg/controller/mongodb/replica_set_controller.go b/pkg/controller/mongodb/replica_set_controller.go index 90b2476da..9ff650845 100644 --- a/pkg/controller/mongodb/replica_set_controller.go +++ b/pkg/controller/mongodb/replica_set_controller.go @@ -293,12 +293,10 @@ func (r *ReplicaSetReconciler) ensureService(mdb mdbv1.MongoDB) error { func (r *ReplicaSetReconciler) createOrUpdateStatefulSet(mdb mdbv1.MongoDB) error { set := appsv1.StatefulSet{} err := r.client.Get(context.TODO(), mdb.NamespacedName(), &set) - err = k8sClient.IgnoreNotFound(err) if err != nil { return fmt.Errorf("error getting StatefulSet: %s", err) } - buildStatefulSetModificationFunction(mdb)(&set) if err = statefulset.CreateOrUpdate(r.client, set); err != nil { return fmt.Errorf("error creating/updating StatefulSet: %s", err) @@ -477,7 +475,6 @@ func getUpdateStrategyType(mdb mdbv1.MongoDB) appsv1.StatefulSetUpdateStrategyTy func buildStatefulSet(mdb mdbv1.MongoDB) (appsv1.StatefulSet, error) { sts := appsv1.StatefulSet{} buildStatefulSetModificationFunction(mdb)(&sts) - return sts, nil } @@ -556,7 +553,6 @@ exec mongod -f /data/automation-mongod.conf ; } func buildStatefulSetModificationFunction(mdb mdbv1.MongoDB) statefulset.Modification { - labels := map[string]string{ "app": mdb.ServiceName(), } @@ -609,9 +605,7 @@ func buildStatefulSetModificationFunction(mdb mdbv1.MongoDB) statefulset.Modific buildScramPodSpecModification(mdb), ), ), - statefulset.WithCustomSpecs(mdb.Spec.StatefulSetConfiguration.Spec), ) - } func getDomain(service, namespace, clusterName string) string { From 520833b15f614e27ea68a5c4d938849b73df055e Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Tue, 21 Jul 2020 18:09:33 +0100 Subject: [PATCH 11/34] removed unused code --- .../statefulset_arbitrary_config_test.go | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/test/e2e/statefulset_arbitrary_config/statefulset_arbitrary_config_test.go b/test/e2e/statefulset_arbitrary_config/statefulset_arbitrary_config_test.go index 90571ff37..e0c4463da 100644 --- a/test/e2e/statefulset_arbitrary_config/statefulset_arbitrary_config_test.go +++ b/test/e2e/statefulset_arbitrary_config/statefulset_arbitrary_config_test.go @@ -1,16 +1,14 @@ package statefulset_arbitrary_config import ( - "context" "testing" - "github.com/golangplus/testing/assert" v1 "github.com/mongodb/mongodb-kubernetes-operator/pkg/apis/mongodb/v1" e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/mongodbtests" setup "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" f "github.com/operator-framework/operator-sdk/pkg/test" - appsv1 "k8s.io/api/apps/v1" + "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" ) @@ -36,17 +34,13 @@ func TestStatefulSetArbitraryConfig(t *testing.T) { t.Run("Test Basic Connectivity", mongodbtests.Connectivity(&mdb)) t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 1)) - // Get the original containers - originalSts := &appsv1.StatefulSet{} - err = f.Global.Client.Get(context.TODO(), mdb.NamespacedName(), originalSts) - assert.NoError(t, err) - overrideSpec := v1.StatefulSetConfiguration{} overrideSpec.Spec.Template.Spec.Containers = []corev1.Container{ {Name: "mongodb-agent", ReadinessProbe: &corev1.Probe{TimeoutSeconds: 100}}} - e2eutil.UpdateMongoDBResource(&mdb, func(mdb *v1.MongoDB) { mdb.Spec.StatefulSetConfiguration = overrideSpec }) + err = e2eutil.UpdateMongoDBResource(&mdb, func(mdb *v1.MongoDB) { mdb.Spec.StatefulSetConfiguration = overrideSpec }) + assert.NoError(t, err) - t.Run("Container has been merged by name", mongodbtests.StatefulSetContainerConditionIsTrue(mdb, "mongodb-agent", func(container corev1.Container) { return container.ReadinessProbe.TimeoutSeconds == 100 })) + t.Run("Container has been merged by name", mongodbtests.StatefulSetContainerConditionIsTrue(&mdb, "mongodb-agent", func(container corev1.Container) bool { return container.ReadinessProbe.TimeoutSeconds == 100 })) } From b5a720a91d614c7ecd61c54a057071e40c13cf7d Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Tue, 21 Jul 2020 18:16:40 +0100 Subject: [PATCH 12/34] unused function and linting --- pkg/kube/statefulset/statefulset.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pkg/kube/statefulset/statefulset.go b/pkg/kube/statefulset/statefulset.go index c6fb20590..c45f3ad9f 100644 --- a/pkg/kube/statefulset/statefulset.go +++ b/pkg/kube/statefulset/statefulset.go @@ -181,10 +181,6 @@ func Apply(funcs ...Modification) func(*appsv1.StatefulSet) { } } -func NOOP() Modification { - return func(sts *appsv1.StatefulSet) {} -} - func WithName(name string) Modification { return func(sts *appsv1.StatefulSet) { sts.Name = name @@ -270,10 +266,10 @@ func MergeVolumeClaimTemplates(defaultTemplates []corev1.PersistentVolumeClaim, defaultMountsMap := createVolumeClaimMap(defaultTemplates) overrideMountsMap := createVolumeClaimMap(overrideTemplates) var mergedVolumes []corev1.PersistentVolumeClaim - for _, defaultMount := range defaultMountsMap { + for idx, defaultMount := range defaultMountsMap { if overrideMount, ok := overrideMountsMap[defaultMount.Name]; ok { // needs merge - if err := mergo.Merge(&defaultMount, overrideMount, mergo.WithAppendSlice, mergo.WithOverride); err != nil { + if err := mergo.Merge(defaultMountsMap[idx], overrideMount, mergo.WithAppendSlice, mergo.WithOverride); err != nil { return nil, err } } @@ -302,6 +298,9 @@ func mergeStatefulSetSpecs(defaultSpec, overrideSpec appsv1.StatefulSetSpec) (ap // VolumeClaimTemplates needs to be manually merged mergedVolumeClaimTemplates, err := MergeVolumeClaimTemplates(defaultSpec.VolumeClaimTemplates, overrideSpec.VolumeClaimTemplates) + if err != nil { + return appsv1.StatefulSetSpec{}, err + } // Merging the rest with mergo if err := mergo.Merge(&defaultSpec, overrideSpec, mergo.WithOverride); err != nil { From d8d10e8765527477d25d355a262eb85e35258f59 Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Tue, 21 Jul 2020 18:26:08 +0100 Subject: [PATCH 13/34] fixed tests broken by linting --- pkg/kube/podtemplatespec/podspec_template.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/kube/podtemplatespec/podspec_template.go b/pkg/kube/podtemplatespec/podspec_template.go index 53e20d401..2231b8c20 100644 --- a/pkg/kube/podtemplatespec/podspec_template.go +++ b/pkg/kube/podtemplatespec/podspec_template.go @@ -232,10 +232,10 @@ func MergeVolumeMounts(defaultMounts, overrideMounts []corev1.VolumeMount) ([]co defaultMountsMap := createMountsMap(defaultMounts) overrideMountsMap := createMountsMap(overrideMounts) mergedVolumeMounts := []corev1.VolumeMount{} - for idx, defaultMount := range defaultMounts { + for _, defaultMount := range defaultMounts { if overrideMount, ok := overrideMountsMap[defaultMount.Name]; ok { // needs merge - if err := mergo.Merge(defaultMounts[idx], overrideMount, mergo.WithAppendSlice); err != nil { + if err := mergo.Merge(&defaultMount, overrideMount, mergo.WithAppendSlice); err != nil { //nolint return nil, err } } @@ -263,7 +263,7 @@ func MergeContainers(defaultContainers, customContainers []corev1.Container) ([] defaultMap := createContainerMap(defaultContainers) customMap := createContainerMap(customContainers) mergedContainers := []corev1.Container{} - for idx, defaultContainer := range defaultContainers { + for _, defaultContainer := range defaultContainers { if customContainer, ok := customMap[defaultContainer.Name]; ok { // The container is present in both maps, so we need to merge // Merge mounts @@ -271,7 +271,7 @@ func MergeContainers(defaultContainers, customContainers []corev1.Container) ([] if err != nil { return nil, err } - if err := mergo.Merge(&defaultContainers[idx], customContainer, mergo.WithOverride); err != nil { + if err := mergo.Merge(&defaultContainer, customContainer, mergo.WithOverride); err != nil { //nolint return nil, err } // completely override any resources that were provided From 8d2c4395064c7b2eac4841b32971a17d34f3165f Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Tue, 21 Jul 2020 18:27:14 +0100 Subject: [PATCH 14/34] improved logging --- test/e2e/mongodbtests/mongodbtests.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/mongodbtests/mongodbtests.go b/test/e2e/mongodbtests/mongodbtests.go index 7c4b0ec50..b7ef25c37 100644 --- a/test/e2e/mongodbtests/mongodbtests.go +++ b/test/e2e/mongodbtests/mongodbtests.go @@ -296,6 +296,6 @@ func StatefulSetContainerConditionIsTrue(mdb *mdbv1.MongoDB, containerName strin if err != nil { t.Fatal(err) } - t.Logf("StatefulSet %s/%s has the expected container value!", mdb.Namespace, mdb.Name) + t.Logf("StatefulSet %s/%s - container %s satisfies the given condition!", mdb.Namespace, mdb.Name, containerName) } } From 28d0202cd2c8f3fef2fe16eb70f7a42d7410b155 Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Tue, 21 Jul 2020 20:02:41 +0100 Subject: [PATCH 15/34] improved e2e test --- .../statefulset_arbitrary_config_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/e2e/statefulset_arbitrary_config/statefulset_arbitrary_config_test.go b/test/e2e/statefulset_arbitrary_config/statefulset_arbitrary_config_test.go index e0c4463da..391c39e79 100644 --- a/test/e2e/statefulset_arbitrary_config/statefulset_arbitrary_config_test.go +++ b/test/e2e/statefulset_arbitrary_config/statefulset_arbitrary_config_test.go @@ -41,6 +41,10 @@ func TestStatefulSetArbitraryConfig(t *testing.T) { err = e2eutil.UpdateMongoDBResource(&mdb, func(mdb *v1.MongoDB) { mdb.Spec.StatefulSetConfiguration = overrideSpec }) assert.NoError(t, err) + t.Run("Stateful Set Reaches Ready State", mongodbtests.StatefulSetIsReady(&mdb)) + t.Run("Container has been merged by name", mongodbtests.StatefulSetContainerConditionIsTrue(&mdb, "mongodb-agent", func(container corev1.Container) bool { return container.ReadinessProbe.TimeoutSeconds == 100 })) + t.Run("MongoDB Reaches Running Phase", mongodbtests.MongoDBReachesRunningPhase(&mdb)) + t.Run("AutomationConfig's version has been increased", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 2)) } From 040c3ab1ca560ce26e240a059e54ef197a0ccd51 Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Wed, 22 Jul 2020 08:57:15 +0100 Subject: [PATCH 16/34] added new e2e test --- .evergreen.yml | 8 +++ .../statefulset_arbitrary_config_test.go | 18 +++---- ...tatefulset_arbitrary_config_update_test.go | 50 +++++++++++++++++++ 3 files changed, 64 insertions(+), 12 deletions(-) create mode 100644 test/e2e/statefulset_arbitrary_config_update/statefulset_arbitrary_config_update_test.go diff --git a/.evergreen.yml b/.evergreen.yml index 7ee0c936c..83a4d49c4 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -152,6 +152,7 @@ task_groups: - e2e_test_replica_set_tls - e2e_test_replica_set_tls_upgrade - e2e_test_statefulset_arbitrary_config + - e2e_test_statefulset_arbitrary_config_update teardown_task: - func: upload_e2e_logs @@ -284,6 +285,13 @@ tasks: vars: test: statefulset_arbitrary_config + - name: e2e_test_statefulset_arbitrary_config_update + commands: + - func: run_e2e_test + vars: + test: statefulset_arbitrary_config_update + + buildvariants: - name: go_unit_tests display_name: go_unit_tests diff --git a/test/e2e/statefulset_arbitrary_config/statefulset_arbitrary_config_test.go b/test/e2e/statefulset_arbitrary_config/statefulset_arbitrary_config_test.go index 391c39e79..42f6be085 100644 --- a/test/e2e/statefulset_arbitrary_config/statefulset_arbitrary_config_test.go +++ b/test/e2e/statefulset_arbitrary_config/statefulset_arbitrary_config_test.go @@ -1,4 +1,4 @@ -package statefulset_arbitrary_config +package statefulset_arbitrary_config_update import ( "testing" @@ -8,7 +8,6 @@ import ( "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/mongodbtests" setup "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" f "github.com/operator-framework/operator-sdk/pkg/test" - "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" ) @@ -29,22 +28,17 @@ func TestStatefulSetArbitraryConfig(t *testing.T) { t.Fatal(err) } - t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) - t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb)) - t.Run("Test Basic Connectivity", mongodbtests.Connectivity(&mdb)) - t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 1)) - overrideSpec := v1.StatefulSetConfiguration{} overrideSpec.Spec.Template.Spec.Containers = []corev1.Container{ {Name: "mongodb-agent", ReadinessProbe: &corev1.Probe{TimeoutSeconds: 100}}} - err = e2eutil.UpdateMongoDBResource(&mdb, func(mdb *v1.MongoDB) { mdb.Spec.StatefulSetConfiguration = overrideSpec }) - assert.NoError(t, err) - - t.Run("Stateful Set Reaches Ready State", mongodbtests.StatefulSetIsReady(&mdb)) + mdb.Spec.StatefulSetConfiguration = overrideSpec + t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) + t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb)) + t.Run("Test Basic Connectivity", mongodbtests.Connectivity(&mdb)) + t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 1)) t.Run("Container has been merged by name", mongodbtests.StatefulSetContainerConditionIsTrue(&mdb, "mongodb-agent", func(container corev1.Container) bool { return container.ReadinessProbe.TimeoutSeconds == 100 })) t.Run("MongoDB Reaches Running Phase", mongodbtests.MongoDBReachesRunningPhase(&mdb)) - t.Run("AutomationConfig's version has been increased", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 2)) } diff --git a/test/e2e/statefulset_arbitrary_config_update/statefulset_arbitrary_config_update_test.go b/test/e2e/statefulset_arbitrary_config_update/statefulset_arbitrary_config_update_test.go new file mode 100644 index 000000000..391c39e79 --- /dev/null +++ b/test/e2e/statefulset_arbitrary_config_update/statefulset_arbitrary_config_update_test.go @@ -0,0 +1,50 @@ +package statefulset_arbitrary_config + +import ( + "testing" + + v1 "github.com/mongodb/mongodb-kubernetes-operator/pkg/apis/mongodb/v1" + e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" + "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/mongodbtests" + setup "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" + f "github.com/operator-framework/operator-sdk/pkg/test" + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" +) + +func TestMain(m *testing.M) { + f.MainEntry(m) +} + +func TestStatefulSetArbitraryConfig(t *testing.T) { + ctx, shouldCleanup := setup.InitTest(t) + + if shouldCleanup { + defer ctx.Cleanup() + } + mdb, user := e2eutil.NewTestMongoDB("mdb0") + + _, err := setup.GeneratePasswordForUser(user, ctx) + if err != nil { + t.Fatal(err) + } + + t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) + t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb)) + t.Run("Test Basic Connectivity", mongodbtests.Connectivity(&mdb)) + t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 1)) + + overrideSpec := v1.StatefulSetConfiguration{} + overrideSpec.Spec.Template.Spec.Containers = []corev1.Container{ + {Name: "mongodb-agent", ReadinessProbe: &corev1.Probe{TimeoutSeconds: 100}}} + + err = e2eutil.UpdateMongoDBResource(&mdb, func(mdb *v1.MongoDB) { mdb.Spec.StatefulSetConfiguration = overrideSpec }) + assert.NoError(t, err) + + t.Run("Stateful Set Reaches Ready State", mongodbtests.StatefulSetIsReady(&mdb)) + + t.Run("Container has been merged by name", mongodbtests.StatefulSetContainerConditionIsTrue(&mdb, "mongodb-agent", func(container corev1.Container) bool { return container.ReadinessProbe.TimeoutSeconds == 100 })) + t.Run("MongoDB Reaches Running Phase", mongodbtests.MongoDBReachesRunningPhase(&mdb)) + t.Run("AutomationConfig's version has been increased", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 2)) + +} From 319eaccf3c4c0aeda794a0d73539f955e561123c Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Wed, 22 Jul 2020 09:01:58 +0100 Subject: [PATCH 17/34] removed unneeded test --- .../statefulset_arbitrary_config_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/test/e2e/statefulset_arbitrary_config/statefulset_arbitrary_config_test.go b/test/e2e/statefulset_arbitrary_config/statefulset_arbitrary_config_test.go index 42f6be085..eb451492a 100644 --- a/test/e2e/statefulset_arbitrary_config/statefulset_arbitrary_config_test.go +++ b/test/e2e/statefulset_arbitrary_config/statefulset_arbitrary_config_test.go @@ -39,6 +39,5 @@ func TestStatefulSetArbitraryConfig(t *testing.T) { t.Run("Test Basic Connectivity", mongodbtests.Connectivity(&mdb)) t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 1)) t.Run("Container has been merged by name", mongodbtests.StatefulSetContainerConditionIsTrue(&mdb, "mongodb-agent", func(container corev1.Container) bool { return container.ReadinessProbe.TimeoutSeconds == 100 })) - t.Run("MongoDB Reaches Running Phase", mongodbtests.MongoDBReachesRunningPhase(&mdb)) } From 43bf744d8f34f418ddbb32aed6ba0ff282986fa2 Mon Sep 17 00:00:00 2001 From: Fabian Lindfors Date: Wed, 29 Jul 2020 13:41:15 +0200 Subject: [PATCH 18/34] Add custom specs during statefulset creation --- pkg/controller/mongodb/replica_set_controller.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/controller/mongodb/replica_set_controller.go b/pkg/controller/mongodb/replica_set_controller.go index 9ff650845..ceec3f339 100644 --- a/pkg/controller/mongodb/replica_set_controller.go +++ b/pkg/controller/mongodb/replica_set_controller.go @@ -605,6 +605,7 @@ func buildStatefulSetModificationFunction(mdb mdbv1.MongoDB) statefulset.Modific buildScramPodSpecModification(mdb), ), ), + statefulset.WithCustomSpecs(mdb.Spec.StatefulSetConfiguration.Spec), ) } From 7e420f122475e6ccf460f04dfb3f17264d40ad00 Mon Sep 17 00:00:00 2001 From: Fabian Lindfors Date: Wed, 29 Jul 2020 14:07:07 +0200 Subject: [PATCH 19/34] Remove unnecessary Automation Config version test --- .../statefulset_arbitrary_config_update_test.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/test/e2e/statefulset_arbitrary_config_update/statefulset_arbitrary_config_update_test.go b/test/e2e/statefulset_arbitrary_config_update/statefulset_arbitrary_config_update_test.go index 391c39e79..c3223790e 100644 --- a/test/e2e/statefulset_arbitrary_config_update/statefulset_arbitrary_config_update_test.go +++ b/test/e2e/statefulset_arbitrary_config_update/statefulset_arbitrary_config_update_test.go @@ -42,9 +42,8 @@ func TestStatefulSetArbitraryConfig(t *testing.T) { assert.NoError(t, err) t.Run("Stateful Set Reaches Ready State", mongodbtests.StatefulSetIsReady(&mdb)) - - t.Run("Container has been merged by name", mongodbtests.StatefulSetContainerConditionIsTrue(&mdb, "mongodb-agent", func(container corev1.Container) bool { return container.ReadinessProbe.TimeoutSeconds == 100 })) + t.Run("Container has been merged by name", mongodbtests.StatefulSetContainerConditionIsTrue(&mdb, "mongodb-agent", func(container corev1.Container) bool { + return container.ReadinessProbe.TimeoutSeconds == 100 + })) t.Run("MongoDB Reaches Running Phase", mongodbtests.MongoDBReachesRunningPhase(&mdb)) - t.Run("AutomationConfig's version has been increased", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 2)) - } From ea83c067a2ff1a08ea7a28aeb3c068bcf3ab6e23 Mon Sep 17 00:00:00 2001 From: Fabian Lindfors Date: Wed, 29 Jul 2020 15:45:32 +0200 Subject: [PATCH 20/34] Update tests to not use a retry --- test/e2e/e2eutil.go | 17 -------------- test/e2e/mongodbtests/mongodbtests.go | 23 +++++++++++++++++-- .../statefulset_arbitrary_config_test.go | 5 ++-- ...tatefulset_arbitrary_config_update_test.go | 6 ++--- 4 files changed, 27 insertions(+), 24 deletions(-) diff --git a/test/e2e/e2eutil.go b/test/e2e/e2eutil.go index 5ba73d256..ed6518250 100644 --- a/test/e2e/e2eutil.go +++ b/test/e2e/e2eutil.go @@ -74,14 +74,6 @@ func WaitForStatefulSetToHaveUpdateStrategy(t *testing.T, mdb *mdbv1.MongoDB, st }) } -// WaitForStatefulSetToHaveExpectedContainerValue waits until the passed -func WaitForStatefulSetToHaveExpectedContainerCondition(t *testing.T, mdb *mdbv1.MongoDB, containerName string, condition func(container corev1.Container) bool, retryInterval, timeout time.Duration) error { - return waitForStatefulSetCondition(t, mdb, retryInterval, timeout, func(sts appsv1.StatefulSet) bool { - idx := findIndexByName(containerName, sts.Spec.Template.Spec.Containers) - return idx != -1 && condition(sts.Spec.Template.Spec.Containers[idx]) - }) -} - // WaitForStatefulSetToBeReady waits until all replicas of the StatefulSet with the given name // have reached the ready status func WaitForStatefulSetToBeReady(t *testing.T, mdb *mdbv1.MongoDB, retryInterval, timeout time.Duration) error { @@ -183,12 +175,3 @@ func NewTestTLSConfig(optional bool) mdbv1.TLS { }, } } - -func findIndexByName(name string, containers []corev1.Container) int { - for idx, c := range containers { - if c.Name == name { - return idx - } - } - return -1 -} diff --git a/test/e2e/mongodbtests/mongodbtests.go b/test/e2e/mongodbtests/mongodbtests.go index b7ef25c37..aeb1f5874 100644 --- a/test/e2e/mongodbtests/mongodbtests.go +++ b/test/e2e/mongodbtests/mongodbtests.go @@ -292,10 +292,29 @@ func IsReachableDuringWithConnection(mdb *mdbv1.MongoDB, interval time.Duration, func StatefulSetContainerConditionIsTrue(mdb *mdbv1.MongoDB, containerName string, condition func(container corev1.Container) bool) func(*testing.T) { return func(t *testing.T) { - err := e2eutil.WaitForStatefulSetToHaveExpectedContainerCondition(t, mdb, containerName, condition, time.Second*5, time.Minute*5) + sts := appsv1.StatefulSet{} + err := f.Global.Client.Get(context.TODO(), types.NamespacedName{Name: mdb.Name, Namespace: f.Global.OperatorNamespace}, &sts) if err != nil { t.Fatal(err) } - t.Logf("StatefulSet %s/%s - container %s satisfies the given condition!", mdb.Namespace, mdb.Name, containerName) + + container := findContainerByName(containerName, sts.Spec.Template.Spec.Containers) + if container == nil { + t.Fatalf(`No container found with name "%s" in StatefulSet pod template`, containerName) + } + + if !condition(*container) { + t.Fatalf(`Container "%s" does not satisfy condition`, containerName) + } } } + +func findContainerByName(name string, containers []corev1.Container) *corev1.Container { + for _, c := range containers { + if c.Name == name { + return &c + } + } + + return nil +} diff --git a/test/e2e/statefulset_arbitrary_config/statefulset_arbitrary_config_test.go b/test/e2e/statefulset_arbitrary_config/statefulset_arbitrary_config_test.go index eb451492a..5c24cc884 100644 --- a/test/e2e/statefulset_arbitrary_config/statefulset_arbitrary_config_test.go +++ b/test/e2e/statefulset_arbitrary_config/statefulset_arbitrary_config_test.go @@ -38,6 +38,7 @@ func TestStatefulSetArbitraryConfig(t *testing.T) { t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb)) t.Run("Test Basic Connectivity", mongodbtests.Connectivity(&mdb)) t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 1)) - t.Run("Container has been merged by name", mongodbtests.StatefulSetContainerConditionIsTrue(&mdb, "mongodb-agent", func(container corev1.Container) bool { return container.ReadinessProbe.TimeoutSeconds == 100 })) - + t.Run("Container has been merged by name", mongodbtests.StatefulSetContainerConditionIsTrue(&mdb, "mongodb-agent", func(container corev1.Container) bool { + return container.ReadinessProbe.TimeoutSeconds == 100 + })) } diff --git a/test/e2e/statefulset_arbitrary_config_update/statefulset_arbitrary_config_update_test.go b/test/e2e/statefulset_arbitrary_config_update/statefulset_arbitrary_config_update_test.go index c3223790e..3b3893ea4 100644 --- a/test/e2e/statefulset_arbitrary_config_update/statefulset_arbitrary_config_update_test.go +++ b/test/e2e/statefulset_arbitrary_config_update/statefulset_arbitrary_config_update_test.go @@ -31,7 +31,7 @@ func TestStatefulSetArbitraryConfig(t *testing.T) { t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb)) - t.Run("Test Basic Connectivity", mongodbtests.Connectivity(&mdb)) + t.Run("Test basic connectivity", mongodbtests.Connectivity(&mdb)) t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 1)) overrideSpec := v1.StatefulSetConfiguration{} @@ -41,9 +41,9 @@ func TestStatefulSetArbitraryConfig(t *testing.T) { err = e2eutil.UpdateMongoDBResource(&mdb, func(mdb *v1.MongoDB) { mdb.Spec.StatefulSetConfiguration = overrideSpec }) assert.NoError(t, err) - t.Run("Stateful Set Reaches Ready State", mongodbtests.StatefulSetIsReady(&mdb)) + t.Run("Basic tests after update", mongodbtests.BasicFunctionality(&mdb)) + t.Run("Test basic connectivity after update", mongodbtests.Connectivity(&mdb)) t.Run("Container has been merged by name", mongodbtests.StatefulSetContainerConditionIsTrue(&mdb, "mongodb-agent", func(container corev1.Container) bool { return container.ReadinessProbe.TimeoutSeconds == 100 })) - t.Run("MongoDB Reaches Running Phase", mongodbtests.MongoDBReachesRunningPhase(&mdb)) } From 9bf3b6800eea17506307e4c7cd6a6a7d4de07a6b Mon Sep 17 00:00:00 2001 From: Fabian Lindfors Date: Wed, 29 Jul 2020 16:40:11 +0200 Subject: [PATCH 21/34] Refactored unit tests --- pkg/kube/podtemplatespec/podspec_template.go | 11 +- .../podtemplatespec/podspec_template_test.go | 226 +++++++++++++ pkg/kube/statefulset/statefulset.go | 5 +- pkg/kube/statefulset/statefulset_test.go | 310 ------------------ 4 files changed, 233 insertions(+), 319 deletions(-) diff --git a/pkg/kube/podtemplatespec/podspec_template.go b/pkg/kube/podtemplatespec/podspec_template.go index 2231b8c20..cf7bdc83f 100644 --- a/pkg/kube/podtemplatespec/podspec_template.go +++ b/pkg/kube/podtemplatespec/podspec_template.go @@ -228,7 +228,7 @@ func WithVolumeMounts(containerName string, volumeMounts ...corev1.VolumeMount) } } -func MergeVolumeMounts(defaultMounts, overrideMounts []corev1.VolumeMount) ([]corev1.VolumeMount, error) { +func mergeVolumeMounts(defaultMounts, overrideMounts []corev1.VolumeMount) ([]corev1.VolumeMount, error) { defaultMountsMap := createMountsMap(defaultMounts) overrideMountsMap := createMountsMap(overrideMounts) mergedVolumeMounts := []corev1.VolumeMount{} @@ -259,7 +259,7 @@ func createMountsMap(volumeMounts []corev1.VolumeMount) map[string]corev1.Volume return mountMap } -func MergeContainers(defaultContainers, customContainers []corev1.Container) ([]corev1.Container, error) { +func mergeContainers(defaultContainers, customContainers []corev1.Container) ([]corev1.Container, error) { defaultMap := createContainerMap(defaultContainers) customMap := createContainerMap(customContainers) mergedContainers := []corev1.Container{} @@ -267,7 +267,7 @@ func MergeContainers(defaultContainers, customContainers []corev1.Container) ([] if customContainer, ok := customMap[defaultContainer.Name]; ok { // The container is present in both maps, so we need to merge // Merge mounts - mergedMounts, err := MergeVolumeMounts(defaultContainer.VolumeMounts, customContainer.VolumeMounts) + mergedMounts, err := mergeVolumeMounts(defaultContainer.VolumeMounts, customContainer.VolumeMounts) if err != nil { return nil, err } @@ -319,15 +319,14 @@ func mergeAffinity(defaultAffinity, overrideAffinity *corev1.Affinity) (*corev1. } func MergePodTemplateSpecs(defaultTemplate, overrideTemplate corev1.PodTemplateSpec) (corev1.PodTemplateSpec, error) { - // Containers need to be merged manually - mergedContainers, err := MergeContainers(defaultTemplate.Spec.Containers, overrideTemplate.Spec.Containers) + mergedContainers, err := mergeContainers(defaultTemplate.Spec.Containers, overrideTemplate.Spec.Containers) if err != nil { return corev1.PodTemplateSpec{}, err } // InitContainers need to be merged manually - mergedInitContainers, err := MergeContainers(defaultTemplate.Spec.InitContainers, overrideTemplate.Spec.InitContainers) + mergedInitContainers, err := mergeContainers(defaultTemplate.Spec.InitContainers, overrideTemplate.Spec.InitContainers) if err != nil { return corev1.PodTemplateSpec{}, err } diff --git a/pkg/kube/podtemplatespec/podspec_template_test.go b/pkg/kube/podtemplatespec/podspec_template_test.go index 2c89bbfa4..d845c1457 100644 --- a/pkg/kube/podtemplatespec/podspec_template_test.go +++ b/pkg/kube/podtemplatespec/podspec_template_test.go @@ -3,6 +3,8 @@ package podtemplatespec import ( "testing" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/container" "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" @@ -96,3 +98,227 @@ func TestPodTemplateSpec_MultipleEditsToContainer(t *testing.T) { assert.Equal(t, corev1.PullAlways, c.ImagePullPolicy) assert.Equal(t, "cmd", c.Command[0]) } + +func TestMergeFromEmpty(t *testing.T) { + defaultPodSpec := corev1.PodTemplateSpec{} + customPodSpecTemplate := getCustomPodSpec() + + mergedPodTemplateSpec, err := MergePodTemplateSpecs(defaultPodSpec, customPodSpecTemplate) + + assert.NoError(t, err) + assert.Equal(t, customPodSpecTemplate, mergedPodTemplateSpec) +} + +func TestMergeWithEmpty(t *testing.T) { + defaultPodSpec := getDefaultPodSpec() + customPodSpecTemplate := corev1.PodTemplateSpec{} + + mergedPodTemplateSpec, err := MergePodTemplateSpecs(defaultPodSpec, customPodSpecTemplate) + + assert.NoError(t, err) + assert.Equal(t, defaultPodSpec, mergedPodTemplateSpec) +} + +func TestMultipleMerges(t *testing.T) { + defaultPodSpec := getDefaultPodSpec() + customPodSpecTemplate := getCustomPodSpec() + + referenceSpec, err := MergePodTemplateSpecs(defaultPodSpec, customPodSpecTemplate) + assert.NoError(t, err) + + mergedSpec := defaultPodSpec + + // multiple merges must give the same result + for i := 0; i < 3; i++ { + mergedSpec, err := MergePodTemplateSpecs(mergedSpec, customPodSpecTemplate) + assert.NoError(t, err) + assert.Equal(t, referenceSpec, mergedSpec) + } +} + +func TestMergeContainer(t *testing.T) { + vol0 := corev1.VolumeMount{Name: "container-0.volume-mount-0"} + sideCarVol := corev1.VolumeMount{Name: "container-1.volume-mount-0"} + + anotherVol := corev1.VolumeMount{Name: "another-mount"} + + overrideDefaultContainer := corev1.Container{Name: "container-0"} + overrideDefaultContainer.Image = "overridden" + overrideDefaultContainer.ReadinessProbe = &corev1.Probe{PeriodSeconds: 20} + + otherDefaultContainer := getDefaultContainer() + otherDefaultContainer.Name = "default-side-car" + otherDefaultContainer.VolumeMounts = []corev1.VolumeMount{sideCarVol} + + overrideOtherDefaultContainer := otherDefaultContainer + overrideOtherDefaultContainer.Env = []corev1.EnvVar{{Name: "env_var", Value: "xxx"}} + overrideOtherDefaultContainer.VolumeMounts = []corev1.VolumeMount{anotherVol} + + mergedContainers, err := mergeContainers( + []corev1.Container{getDefaultContainer(), otherDefaultContainer}, + []corev1.Container{getCustomContainer(), overrideDefaultContainer, overrideOtherDefaultContainer}, + ) + + assert.NoError(t, err) + assert.Len(t, mergedContainers, 3) + + assert.Equal(t, getCustomContainer(), mergedContainers[2]) + + mergedDefaultContainer := mergedContainers[0] + assert.Equal(t, "container-0", mergedDefaultContainer.Name) + assert.Equal(t, []corev1.VolumeMount{vol0}, mergedDefaultContainer.VolumeMounts) + assert.Equal(t, "overridden", mergedDefaultContainer.Image) + // only "periodSeconds" was overwritten - other fields stayed untouched + assert.Equal(t, corev1.Handler{HTTPGet: &corev1.HTTPGetAction{Path: "/foo"}}, mergedDefaultContainer.ReadinessProbe.Handler) + assert.Equal(t, int32(20), mergedDefaultContainer.ReadinessProbe.PeriodSeconds) + + mergedOtherContainer := mergedContainers[1] + assert.Equal(t, "default-side-car", mergedOtherContainer.Name) + assert.Equal(t, []corev1.VolumeMount{sideCarVol, anotherVol}, mergedOtherContainer.VolumeMounts) + assert.Len(t, mergedOtherContainer.Env, 1) + assert.Equal(t, "env_var", mergedOtherContainer.Env[0].Name) + assert.Equal(t, "xxx", mergedOtherContainer.Env[0].Value) +} + +func TestMergeVolumeMounts(t *testing.T) { + vol0 := corev1.VolumeMount{Name: "container-0.volume-mount-0"} + vol1 := corev1.VolumeMount{Name: "another-mount"} + volumeMounts := []corev1.VolumeMount{vol0, vol1} + var mergedVolumeMounts []corev1.VolumeMount + var err error + + mergedVolumeMounts, err = mergeVolumeMounts(nil, volumeMounts) + assert.NoError(t, err) + assert.Equal(t, []corev1.VolumeMount{vol0, vol1}, mergedVolumeMounts) + + vol2 := vol1 + vol2.MountPath = "/somewhere" + mergedVolumeMounts, err = mergeVolumeMounts([]corev1.VolumeMount{vol2}, []corev1.VolumeMount{vol0, vol1}) + assert.NoError(t, err) + assert.Equal(t, []corev1.VolumeMount{vol2, vol0}, mergedVolumeMounts) +} + +func TestGetMergedDefaultPodSpecTemplate(t *testing.T) { + var err error + + dbPodSpecTemplate := getDefaultPodSpec() + var mergedPodSpecTemplate corev1.PodTemplateSpec + + // nothing to merge + mergedPodSpecTemplate, err = MergePodTemplateSpecs(corev1.PodTemplateSpec{}, dbPodSpecTemplate) + assert.NoError(t, err) + assert.Equal(t, mergedPodSpecTemplate, dbPodSpecTemplate) + assert.Len(t, mergedPodSpecTemplate.Spec.Containers, 1) + assert.Equal(t, mergedPodSpecTemplate.Spec.Containers[0], dbPodSpecTemplate.Spec.Containers[0]) + + extraContainer := corev1.Container{ + Name: "extra-container", + Image: "container-image", + } + + newPodSpecTemplate := corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{extraContainer}, + }, + } + + // with a side car container + mergedPodSpecTemplate, err = MergePodTemplateSpecs(newPodSpecTemplate, dbPodSpecTemplate) + assert.NoError(t, err) + assert.Len(t, mergedPodSpecTemplate.Spec.Containers, 2) + assert.Equal(t, mergedPodSpecTemplate.Spec.Containers[1], dbPodSpecTemplate.Spec.Containers[0]) + assert.Equal(t, mergedPodSpecTemplate.Spec.Containers[0], extraContainer) +} + +func int64Ref(i int64) *int64 { + return &i +} + +func getDefaultPodSpec() corev1.PodTemplateSpec { + initContainer := getDefaultContainer() + initContainer.Name = "init-container-default" + return corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-default-name", + Namespace: "my-default-namespace", + Labels: map[string]string{"app": "operator"}, + }, + Spec: corev1.PodSpec{ + NodeSelector: map[string]string{ + "node-0": "node-0", + }, + ServiceAccountName: "my-default-service-account", + TerminationGracePeriodSeconds: int64Ref(12), + ActiveDeadlineSeconds: int64Ref(10), + Containers: []corev1.Container{getDefaultContainer()}, + InitContainers: []corev1.Container{initContainer}, + Affinity: affinity("hostname", "default"), + }, + } +} + +func getCustomPodSpec() corev1.PodTemplateSpec { + initContainer := getCustomContainer() + initContainer.Name = "init-container-custom" + return corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"custom": "some"}, + }, + Spec: corev1.PodSpec{ + NodeSelector: map[string]string{ + "node-1": "node-1", + }, + ServiceAccountName: "my-service-account-override", + TerminationGracePeriodSeconds: int64Ref(11), + NodeName: "my-node-name", + RestartPolicy: corev1.RestartPolicyAlways, + Containers: []corev1.Container{getCustomContainer()}, + InitContainers: []corev1.Container{initContainer}, + Affinity: affinity("zone", "custom"), + }, + } +} + +func affinity(antiAffinityKey, nodeAffinityKey string) *corev1.Affinity { + return &corev1.Affinity{ + PodAntiAffinity: &corev1.PodAntiAffinity{ + PreferredDuringSchedulingIgnoredDuringExecution: []corev1.WeightedPodAffinityTerm{{ + PodAffinityTerm: corev1.PodAffinityTerm{ + TopologyKey: antiAffinityKey, + }, + }}, + }, + NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{NodeSelectorTerms: []corev1.NodeSelectorTerm{{ + MatchFields: []corev1.NodeSelectorRequirement{{ + Key: nodeAffinityKey, + }}, + }}}, + }, + } +} + +func getDefaultContainer() corev1.Container { + return corev1.Container{ + Name: "container-0", + Image: "image-0", + ReadinessProbe: &corev1.Probe{ + Handler: corev1.Handler{HTTPGet: &corev1.HTTPGetAction{ + Path: "/foo", + }}, + PeriodSeconds: 10, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "container-0.volume-mount-0", + }, + }, + } +} + +func getCustomContainer() corev1.Container { + return corev1.Container{ + Name: "container-1", + Image: "image-1", + } +} diff --git a/pkg/kube/statefulset/statefulset.go b/pkg/kube/statefulset/statefulset.go index c45f3ad9f..ce8e17a95 100644 --- a/pkg/kube/statefulset/statefulset.go +++ b/pkg/kube/statefulset/statefulset.go @@ -262,7 +262,7 @@ func createVolumeClaimMap(volumeMounts []corev1.PersistentVolumeClaim) map[strin return mountMap } -func MergeVolumeClaimTemplates(defaultTemplates []corev1.PersistentVolumeClaim, overrideTemplates []corev1.PersistentVolumeClaim) ([]corev1.PersistentVolumeClaim, error) { +func mergeVolumeClaimTemplates(defaultTemplates []corev1.PersistentVolumeClaim, overrideTemplates []corev1.PersistentVolumeClaim) ([]corev1.PersistentVolumeClaim, error) { defaultMountsMap := createVolumeClaimMap(defaultTemplates) overrideMountsMap := createVolumeClaimMap(overrideTemplates) var mergedVolumes []corev1.PersistentVolumeClaim @@ -289,7 +289,6 @@ func MergeVolumeClaimTemplates(defaultTemplates []corev1.PersistentVolumeClaim, } func mergeStatefulSetSpecs(defaultSpec, overrideSpec appsv1.StatefulSetSpec) (appsv1.StatefulSetSpec, error) { - // PodTemplateSpec needs to be manually merged mergedPodTemplateSpec, err := podtemplatespec.MergePodTemplateSpecs(defaultSpec.Template, overrideSpec.Template) if err != nil { @@ -297,7 +296,7 @@ func mergeStatefulSetSpecs(defaultSpec, overrideSpec appsv1.StatefulSetSpec) (ap } // VolumeClaimTemplates needs to be manually merged - mergedVolumeClaimTemplates, err := MergeVolumeClaimTemplates(defaultSpec.VolumeClaimTemplates, overrideSpec.VolumeClaimTemplates) + mergedVolumeClaimTemplates, err := mergeVolumeClaimTemplates(defaultSpec.VolumeClaimTemplates, overrideSpec.VolumeClaimTemplates) if err != nil { return appsv1.StatefulSetSpec{}, err } diff --git a/pkg/kube/statefulset/statefulset_test.go b/pkg/kube/statefulset/statefulset_test.go index 0ec154f2b..d194e5775 100644 --- a/pkg/kube/statefulset/statefulset_test.go +++ b/pkg/kube/statefulset/statefulset_test.go @@ -2,10 +2,8 @@ package statefulset import ( "fmt" - "reflect" "testing" - "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/podtemplatespec" "github.com/stretchr/testify/assert" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -206,311 +204,3 @@ func TestCreateVolumeMountWithMultipleOptions(t *testing.T) { assert.Equal(t, mount.SubPath, "our-subpath") assert.True(t, mount.ReadOnly) } - -func TestMergeVolumeMounts(t *testing.T) { - vol0 := corev1.VolumeMount{Name: "container-0.volume-mount-0"} - vol1 := corev1.VolumeMount{Name: "another-mount"} - volumeMounts := []corev1.VolumeMount{vol0, vol1} - var mergedVolumeMounts []corev1.VolumeMount - var err error - - mergedVolumeMounts, err = podtemplatespec.MergeVolumeMounts(nil, volumeMounts) - assert.NoError(t, err) - assert.Equal(t, []corev1.VolumeMount{vol0, vol1}, mergedVolumeMounts) - - vol2 := vol1 - vol2.MountPath = "/somewhere" - mergedVolumeMounts, err = podtemplatespec.MergeVolumeMounts([]corev1.VolumeMount{vol2}, []corev1.VolumeMount{vol0, vol1}) - assert.NoError(t, err) - assert.Equal(t, []corev1.VolumeMount{vol2, vol0}, mergedVolumeMounts) -} - -func TestMergeContainer(t *testing.T) { - vol0 := corev1.VolumeMount{Name: "container-0.volume-mount-0"} - sideCarVol := corev1.VolumeMount{Name: "container-1.volume-mount-0"} - - anotherVol := corev1.VolumeMount{Name: "another-mount"} - - overrideDefaultContainer := corev1.Container{Name: "container-0"} - overrideDefaultContainer.Image = "overridden" - overrideDefaultContainer.ReadinessProbe = &corev1.Probe{PeriodSeconds: 20} - - otherDefaultContainer := getDefaultContainer() - otherDefaultContainer.Name = "default-side-car" - otherDefaultContainer.VolumeMounts = []corev1.VolumeMount{sideCarVol} - - overrideOtherDefaultContainer := otherDefaultContainer - overrideOtherDefaultContainer.Env = []corev1.EnvVar{{Name: "env_var", Value: "xxx"}} - overrideOtherDefaultContainer.VolumeMounts = []corev1.VolumeMount{anotherVol} - - mergedContainers, err := podtemplatespec.MergeContainers( - []corev1.Container{getDefaultContainer(), otherDefaultContainer}, - []corev1.Container{getCustomContainer(), overrideDefaultContainer, overrideOtherDefaultContainer}, - ) - - assert.NoError(t, err) - assert.Len(t, mergedContainers, 3) - - assert.Equal(t, getCustomContainer(), mergedContainers[2]) - - mergedDefaultContainer := mergedContainers[0] - assert.Equal(t, "container-0", mergedDefaultContainer.Name) - assert.Equal(t, []corev1.VolumeMount{vol0}, mergedDefaultContainer.VolumeMounts) - assert.Equal(t, "overridden", mergedDefaultContainer.Image) - // only "periodSeconds" was overwritten - other fields stayed untouched - assert.Equal(t, corev1.Handler{HTTPGet: &corev1.HTTPGetAction{Path: "/foo"}}, mergedDefaultContainer.ReadinessProbe.Handler) - assert.Equal(t, int32(20), mergedDefaultContainer.ReadinessProbe.PeriodSeconds) - - mergedOtherContainer := mergedContainers[1] - assert.Equal(t, "default-side-car", mergedOtherContainer.Name) - assert.Equal(t, []corev1.VolumeMount{sideCarVol, anotherVol}, mergedOtherContainer.VolumeMounts) - assert.Len(t, mergedOtherContainer.Env, 1) - assert.Equal(t, "env_var", mergedOtherContainer.Env[0].Name) - assert.Equal(t, "xxx", mergedOtherContainer.Env[0].Value) -} - -func TestMergePodSpecsEmptyCustom(t *testing.T) { - - defaultPodSpec := getDefaultPodSpec() - customPodSpecTemplate := corev1.PodTemplateSpec{} - - mergedPodTemplateSpec, err := podtemplatespec.MergePodTemplateSpecs(defaultPodSpec, customPodSpecTemplate) - - assert.NoError(t, err) - assert.Equal(t, "my-default-service-account", mergedPodTemplateSpec.Spec.ServiceAccountName) - assert.Equal(t, int64Ref(12), mergedPodTemplateSpec.Spec.TerminationGracePeriodSeconds) - - assert.Equal(t, "my-default-name", mergedPodTemplateSpec.ObjectMeta.Name) - assert.Equal(t, "my-default-namespace", mergedPodTemplateSpec.ObjectMeta.Namespace) - assert.Equal(t, int64Ref(10), mergedPodTemplateSpec.Spec.ActiveDeadlineSeconds) - - // ensure collections have been merged - assert.Contains(t, mergedPodTemplateSpec.Spec.NodeSelector, "node-0") - assert.Len(t, mergedPodTemplateSpec.Spec.Containers, 1) - assert.Equal(t, "container-0", mergedPodTemplateSpec.Spec.Containers[0].Name) - assert.Equal(t, "image-0", mergedPodTemplateSpec.Spec.Containers[0].Image) - assert.Equal(t, "container-0.volume-mount-0", mergedPodTemplateSpec.Spec.Containers[0].VolumeMounts[0].Name) - assert.Len(t, mergedPodTemplateSpec.Spec.InitContainers, 1) - assert.Equal(t, "init-container-default", mergedPodTemplateSpec.Spec.InitContainers[0].Name) -} - -func TestMergePodSpecsEmptyDefault(t *testing.T) { - - defaultPodSpec := corev1.PodTemplateSpec{} - customPodSpecTemplate := getCustomPodSpec() - - mergedPodTemplateSpec, err := podtemplatespec.MergePodTemplateSpecs(defaultPodSpec, customPodSpecTemplate) - - assert.NoError(t, err) - assert.Equal(t, "my-service-account-override", mergedPodTemplateSpec.Spec.ServiceAccountName) - assert.Equal(t, int64Ref(11), mergedPodTemplateSpec.Spec.TerminationGracePeriodSeconds) - assert.Equal(t, "my-node-name", mergedPodTemplateSpec.Spec.NodeName) - assert.Equal(t, corev1.RestartPolicy("Always"), mergedPodTemplateSpec.Spec.RestartPolicy) - - assert.Len(t, mergedPodTemplateSpec.Spec.Containers, 1) - assert.Equal(t, "container-1", mergedPodTemplateSpec.Spec.Containers[0].Name) - assert.Equal(t, "image-1", mergedPodTemplateSpec.Spec.Containers[0].Image) - assert.Len(t, mergedPodTemplateSpec.Spec.InitContainers, 1) - assert.Equal(t, "init-container-custom", mergedPodTemplateSpec.Spec.InitContainers[0].Name) - -} - -func TestMergePodSpecsBoth(t *testing.T) { - - defaultPodSpec := getDefaultPodSpec() - customPodSpecTemplate := getCustomPodSpec() - - var mergedPodTemplateSpec corev1.PodTemplateSpec - var err error - - // multiple merges must give the same result - for i := 0; i < 3; i++ { - mergedPodTemplateSpec, err = podtemplatespec.MergePodTemplateSpecs(defaultPodSpec, customPodSpecTemplate) - - assert.NoError(t, err) - // ensure values that were specified in the custom pod spec template remain unchanged - assert.Equal(t, "my-service-account-override", mergedPodTemplateSpec.Spec.ServiceAccountName) - assert.Equal(t, int64Ref(11), mergedPodTemplateSpec.Spec.TerminationGracePeriodSeconds) - assert.Equal(t, "my-node-name", mergedPodTemplateSpec.Spec.NodeName) - assert.Equal(t, corev1.RestartPolicy("Always"), mergedPodTemplateSpec.Spec.RestartPolicy) - - // ensure values from the default pod spec template have been merged in - assert.Equal(t, "my-default-name", mergedPodTemplateSpec.ObjectMeta.Name) - assert.Equal(t, "my-default-namespace", mergedPodTemplateSpec.ObjectMeta.Namespace) - assert.Equal(t, int64Ref(10), mergedPodTemplateSpec.Spec.ActiveDeadlineSeconds) - - // ensure collections have been merged - assert.Contains(t, mergedPodTemplateSpec.Spec.NodeSelector, "node-0") - assert.Contains(t, mergedPodTemplateSpec.Spec.NodeSelector, "node-1") - assert.Len(t, mergedPodTemplateSpec.Spec.Containers, 2) - assert.Equal(t, "container-0", mergedPodTemplateSpec.Spec.Containers[0].Name) - assert.Equal(t, "image-0", mergedPodTemplateSpec.Spec.Containers[0].Image) - assert.Equal(t, "container-0.volume-mount-0", mergedPodTemplateSpec.Spec.Containers[0].VolumeMounts[0].Name) - assert.Equal(t, "container-1", mergedPodTemplateSpec.Spec.Containers[1].Name) - assert.Equal(t, "image-1", mergedPodTemplateSpec.Spec.Containers[1].Image) - assert.Len(t, mergedPodTemplateSpec.Spec.InitContainers, 2) - assert.Equal(t, "init-container-default", mergedPodTemplateSpec.Spec.InitContainers[0].Name) - assert.Equal(t, "init-container-custom", mergedPodTemplateSpec.Spec.InitContainers[1].Name) - - // ensure labels were appended - assert.Len(t, mergedPodTemplateSpec.Labels, 2) - assert.Contains(t, mergedPodTemplateSpec.Labels, "app") - assert.Contains(t, mergedPodTemplateSpec.Labels, "custom") - - // ensure the pointers are not the same - assert.NotEqual(t, mergedPodTemplateSpec.Spec.Affinity, defaultPodSpec.Spec.Affinity) - - // ensure the affinity rules slices were overridden - assert.Equal(t, affinity("zone", "custom"), mergedPodTemplateSpec.Spec.Affinity) - } -} - -func TestGetMergedDefaultPodSpecTemplate(t *testing.T) { - var err error - - dbPodSpecTemplate := getDefaultPodSpec() - var mergedPodSpecTemplate corev1.PodTemplateSpec - - // nothing to merge - mergedPodSpecTemplate, err = podtemplatespec.MergePodTemplateSpecs(corev1.PodTemplateSpec{}, dbPodSpecTemplate) - assert.NoError(t, err) - assert.Equal(t, mergedPodSpecTemplate, dbPodSpecTemplate) - assert.Len(t, mergedPodSpecTemplate.Spec.Containers, 1) - assertContainersEqualBarResources(t, mergedPodSpecTemplate.Spec.Containers[0], dbPodSpecTemplate.Spec.Containers[0]) - - extraContainer := corev1.Container{ - Name: "extra-container", - Image: "container-image", - } - - newPodSpecTemplate := corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{extraContainer}, - }, - } - - // with a side car container - mergedPodSpecTemplate, err = podtemplatespec.MergePodTemplateSpecs(newPodSpecTemplate, dbPodSpecTemplate) - assert.NoError(t, err) - assert.Len(t, mergedPodSpecTemplate.Spec.Containers, 2) - assertContainersEqualBarResources(t, mergedPodSpecTemplate.Spec.Containers[1], dbPodSpecTemplate.Spec.Containers[0]) - assertContainersEqualBarResources(t, mergedPodSpecTemplate.Spec.Containers[0], extraContainer) -} - -func assertContainersEqualBarResources(t *testing.T, self corev1.Container, other corev1.Container) { - // Copied fields from k8s.io/api/core/v1/types.go - assert.Equal(t, self.Name, other.Name) - assert.Equal(t, self.Image, other.Image) - assert.True(t, reflect.DeepEqual(self.Command, other.Command)) - assert.True(t, reflect.DeepEqual(self.Args, other.Args)) - assert.Equal(t, self.WorkingDir, other.WorkingDir) - assert.True(t, reflect.DeepEqual(self.Ports, other.Ports)) - assert.True(t, reflect.DeepEqual(self.EnvFrom, other.EnvFrom)) - assert.True(t, reflect.DeepEqual(self.Env, other.Env)) - assert.True(t, reflect.DeepEqual(self.Resources, other.Resources)) - assert.True(t, reflect.DeepEqual(self.VolumeMounts, other.VolumeMounts)) - assert.True(t, reflect.DeepEqual(self.VolumeDevices, other.VolumeDevices)) - assert.Equal(t, self.LivenessProbe, other.LivenessProbe) - assert.Equal(t, self.ReadinessProbe, other.ReadinessProbe) - assert.Equal(t, self.Lifecycle, other.Lifecycle) - assert.Equal(t, self.TerminationMessagePath, other.TerminationMessagePath) - assert.Equal(t, self.TerminationMessagePolicy, other.TerminationMessagePolicy) - assert.Equal(t, self.ImagePullPolicy, other.ImagePullPolicy) - assert.Equal(t, self.SecurityContext, other.SecurityContext) - assert.Equal(t, self.Stdin, other.Stdin) - assert.Equal(t, self.StdinOnce, other.StdinOnce) - assert.Equal(t, self.TTY, other.TTY) -} - -func int64Ref(i int64) *int64 { - return &i -} - -func getDefaultPodSpec() corev1.PodTemplateSpec { - initContainer := getDefaultContainer() - initContainer.Name = "init-container-default" - return corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: "my-default-name", - Namespace: "my-default-namespace", - Labels: map[string]string{"app": "operator"}, - }, - Spec: corev1.PodSpec{ - NodeSelector: map[string]string{ - "node-0": "node-0", - }, - ServiceAccountName: "my-default-service-account", - TerminationGracePeriodSeconds: int64Ref(12), - ActiveDeadlineSeconds: int64Ref(10), - Containers: []corev1.Container{getDefaultContainer()}, - InitContainers: []corev1.Container{initContainer}, - Affinity: affinity("hostname", "default"), - }, - } -} - -func getCustomPodSpec() corev1.PodTemplateSpec { - initContainer := getCustomContainer() - initContainer.Name = "init-container-custom" - return corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{"custom": "some"}, - }, - Spec: corev1.PodSpec{ - NodeSelector: map[string]string{ - "node-1": "node-1", - }, - ServiceAccountName: "my-service-account-override", - TerminationGracePeriodSeconds: int64Ref(11), - NodeName: "my-node-name", - RestartPolicy: corev1.RestartPolicyAlways, - Containers: []corev1.Container{getCustomContainer()}, - InitContainers: []corev1.Container{initContainer}, - Affinity: affinity("zone", "custom"), - }, - } -} - -func affinity(antiAffinityKey, nodeAffinityKey string) *corev1.Affinity { - return &corev1.Affinity{ - PodAntiAffinity: &corev1.PodAntiAffinity{ - PreferredDuringSchedulingIgnoredDuringExecution: []corev1.WeightedPodAffinityTerm{{ - PodAffinityTerm: corev1.PodAffinityTerm{ - TopologyKey: antiAffinityKey, - }, - }}, - }, - NodeAffinity: &corev1.NodeAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{NodeSelectorTerms: []corev1.NodeSelectorTerm{{ - MatchFields: []corev1.NodeSelectorRequirement{{ - Key: nodeAffinityKey, - }}, - }}}, - }, - } -} - -func getDefaultContainer() corev1.Container { - return corev1.Container{ - Name: "container-0", - Image: "image-0", - ReadinessProbe: &corev1.Probe{ - Handler: corev1.Handler{HTTPGet: &corev1.HTTPGetAction{ - Path: "/foo", - }}, - PeriodSeconds: 10, - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "container-0.volume-mount-0", - }, - }, - } -} - -func getCustomContainer() corev1.Container { - return corev1.Container{ - Name: "container-1", - Image: "image-1", - } -} From d24b7c0668a6ab3fd62ac8008c6d29565858f6f2 Mon Sep 17 00:00:00 2001 From: Fabian Lindfors Date: Wed, 29 Jul 2020 16:41:40 +0200 Subject: [PATCH 22/34] Rearrange functions --- pkg/kube/podtemplatespec/podspec_template.go | 62 ++++++++-------- pkg/kube/statefulset/statefulset.go | 75 ++++++++++---------- 2 files changed, 69 insertions(+), 68 deletions(-) diff --git a/pkg/kube/podtemplatespec/podspec_template.go b/pkg/kube/podtemplatespec/podspec_template.go index cf7bdc83f..5f6deb8d8 100644 --- a/pkg/kube/podtemplatespec/podspec_template.go +++ b/pkg/kube/podtemplatespec/podspec_template.go @@ -228,6 +228,37 @@ func WithVolumeMounts(containerName string, volumeMounts ...corev1.VolumeMount) } } +func MergePodTemplateSpecs(defaultTemplate, overrideTemplate corev1.PodTemplateSpec) (corev1.PodTemplateSpec, error) { + // Containers need to be merged manually + mergedContainers, err := mergeContainers(defaultTemplate.Spec.Containers, overrideTemplate.Spec.Containers) + if err != nil { + return corev1.PodTemplateSpec{}, err + } + + // InitContainers need to be merged manually + mergedInitContainers, err := mergeContainers(defaultTemplate.Spec.InitContainers, overrideTemplate.Spec.InitContainers) + if err != nil { + return corev1.PodTemplateSpec{}, err + } + + // Affinity needs to be merged manually + mergedAffinity, err := mergeAffinity(defaultTemplate.Spec.Affinity, overrideTemplate.Spec.Affinity) + if err != nil { + return corev1.PodTemplateSpec{}, err + } + + // Everything else can be merged with mergo + mergedPodTemplateSpec := *defaultTemplate.DeepCopy() + if err = mergo.Merge(&mergedPodTemplateSpec, overrideTemplate, mergo.WithOverride, mergo.WithAppendSlice); err != nil { + return corev1.PodTemplateSpec{}, err + } + + mergedPodTemplateSpec.Spec.Containers = mergedContainers + mergedPodTemplateSpec.Spec.InitContainers = mergedInitContainers + mergedPodTemplateSpec.Spec.Affinity = mergedAffinity + return mergedPodTemplateSpec, nil +} + func mergeVolumeMounts(defaultMounts, overrideMounts []corev1.VolumeMount) ([]corev1.VolumeMount, error) { defaultMountsMap := createMountsMap(defaultMounts) overrideMountsMap := createMountsMap(overrideMounts) @@ -318,37 +349,6 @@ func mergeAffinity(defaultAffinity, overrideAffinity *corev1.Affinity) (*corev1. return mergedAffinity, nil } -func MergePodTemplateSpecs(defaultTemplate, overrideTemplate corev1.PodTemplateSpec) (corev1.PodTemplateSpec, error) { - // Containers need to be merged manually - mergedContainers, err := mergeContainers(defaultTemplate.Spec.Containers, overrideTemplate.Spec.Containers) - if err != nil { - return corev1.PodTemplateSpec{}, err - } - - // InitContainers need to be merged manually - mergedInitContainers, err := mergeContainers(defaultTemplate.Spec.InitContainers, overrideTemplate.Spec.InitContainers) - if err != nil { - return corev1.PodTemplateSpec{}, err - } - - // Affinity needs to be merged manually - mergedAffinity, err := mergeAffinity(defaultTemplate.Spec.Affinity, overrideTemplate.Spec.Affinity) - if err != nil { - return corev1.PodTemplateSpec{}, err - } - - // Everything else can be merged with mergo - mergedPodTemplateSpec := *defaultTemplate.DeepCopy() - if err = mergo.Merge(&mergedPodTemplateSpec, overrideTemplate, mergo.WithOverride, mergo.WithAppendSlice); err != nil { - return corev1.PodTemplateSpec{}, err - } - - mergedPodTemplateSpec.Spec.Containers = mergedContainers - mergedPodTemplateSpec.Spec.InitContainers = mergedInitContainers - mergedPodTemplateSpec.Spec.Affinity = mergedAffinity - return mergedPodTemplateSpec, nil -} - // findContainerByName will find either a container or init container by name in a pod template spec func findContainerByName(name string, podTemplateSpec *corev1.PodTemplateSpec) *corev1.Container { containerIdx := findIndexByName(name, podTemplateSpec.Spec.Containers) diff --git a/pkg/kube/statefulset/statefulset.go b/pkg/kube/statefulset/statefulset.go index ce8e17a95..fd6889efe 100644 --- a/pkg/kube/statefulset/statefulset.go +++ b/pkg/kube/statefulset/statefulset.go @@ -3,8 +3,9 @@ package statefulset import ( "sort" - "github.com/imdario/mergo" "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/podtemplatespec" + + "github.com/imdario/mergo" apiErrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -254,38 +255,14 @@ func WithVolumeClaim(name string, f func(*corev1.PersistentVolumeClaim)) Modific } } -func createVolumeClaimMap(volumeMounts []corev1.PersistentVolumeClaim) map[string]corev1.PersistentVolumeClaim { - mountMap := make(map[string]corev1.PersistentVolumeClaim) - for _, m := range volumeMounts { - mountMap[m.Name] = m - } - return mountMap -} - -func mergeVolumeClaimTemplates(defaultTemplates []corev1.PersistentVolumeClaim, overrideTemplates []corev1.PersistentVolumeClaim) ([]corev1.PersistentVolumeClaim, error) { - defaultMountsMap := createVolumeClaimMap(defaultTemplates) - overrideMountsMap := createVolumeClaimMap(overrideTemplates) - var mergedVolumes []corev1.PersistentVolumeClaim - for idx, defaultMount := range defaultMountsMap { - if overrideMount, ok := overrideMountsMap[defaultMount.Name]; ok { - // needs merge - if err := mergo.Merge(defaultMountsMap[idx], overrideMount, mergo.WithAppendSlice, mergo.WithOverride); err != nil { - return nil, err - } - } - mergedVolumes = append(mergedVolumes, defaultMount) - } - for _, overrideMount := range overrideMountsMap { - if _, ok := defaultMountsMap[overrideMount.Name]; ok { - // already merged - continue +func WithCustomSpecs(spec appsv1.StatefulSetSpec) Modification { + return func(set *appsv1.StatefulSet) { + m, err := mergeStatefulSetSpecs(set.Spec, spec) + if err != nil { + return } - mergedVolumes = append(mergedVolumes, overrideMount) + set.Spec = m } - sort.SliceStable(mergedVolumes, func(i, j int) bool { - return mergedVolumes[i].Name < mergedVolumes[j].Name - }) - return mergedVolumes, nil } func mergeStatefulSetSpecs(defaultSpec, overrideSpec appsv1.StatefulSetSpec) (appsv1.StatefulSetSpec, error) { @@ -312,14 +289,38 @@ func mergeStatefulSetSpecs(defaultSpec, overrideSpec appsv1.StatefulSetSpec) (ap return defaultSpec, nil } -func WithCustomSpecs(spec appsv1.StatefulSetSpec) Modification { - return func(set *appsv1.StatefulSet) { - m, err := mergeStatefulSetSpecs(set.Spec, spec) - if err != nil { - return +func mergeVolumeClaimTemplates(defaultTemplates []corev1.PersistentVolumeClaim, overrideTemplates []corev1.PersistentVolumeClaim) ([]corev1.PersistentVolumeClaim, error) { + defaultMountsMap := createVolumeClaimMap(defaultTemplates) + overrideMountsMap := createVolumeClaimMap(overrideTemplates) + var mergedVolumes []corev1.PersistentVolumeClaim + for idx, defaultMount := range defaultMountsMap { + if overrideMount, ok := overrideMountsMap[defaultMount.Name]; ok { + // needs merge + if err := mergo.Merge(defaultMountsMap[idx], overrideMount, mergo.WithAppendSlice, mergo.WithOverride); err != nil { + return nil, err + } } - set.Spec = m + mergedVolumes = append(mergedVolumes, defaultMount) } + for _, overrideMount := range overrideMountsMap { + if _, ok := defaultMountsMap[overrideMount.Name]; ok { + // already merged + continue + } + mergedVolumes = append(mergedVolumes, overrideMount) + } + sort.SliceStable(mergedVolumes, func(i, j int) bool { + return mergedVolumes[i].Name < mergedVolumes[j].Name + }) + return mergedVolumes, nil +} + +func createVolumeClaimMap(volumeMounts []corev1.PersistentVolumeClaim) map[string]corev1.PersistentVolumeClaim { + mountMap := make(map[string]corev1.PersistentVolumeClaim) + for _, m := range volumeMounts { + mountMap[m.Name] = m + } + return mountMap } func findVolumeClaimIndexByName(name string, pvcs []corev1.PersistentVolumeClaim) int { From 05605f279a954307f9e00d00d997456b847fa44f Mon Sep 17 00:00:00 2001 From: Fabian Lindfors Date: Thu, 30 Jul 2020 10:36:58 +0200 Subject: [PATCH 23/34] Refactor unit tests --- .../podtemplatespec/podspec_template_test.go | 141 +++++++++++------- 1 file changed, 84 insertions(+), 57 deletions(-) diff --git a/pkg/kube/podtemplatespec/podspec_template_test.go b/pkg/kube/podtemplatespec/podspec_template_test.go index d845c1457..9f4770a6c 100644 --- a/pkg/kube/podtemplatespec/podspec_template_test.go +++ b/pkg/kube/podtemplatespec/podspec_template_test.go @@ -99,6 +99,52 @@ func TestPodTemplateSpec_MultipleEditsToContainer(t *testing.T) { assert.Equal(t, "cmd", c.Command[0]) } +func TestMerge(t *testing.T) { + defaultSpec := getDefaultPodSpec() + customSpec := getCustomPodSpec() + + mergedSpec, err := MergePodTemplateSpecs(defaultSpec, customSpec) + assert.NoError(t, err) + + initContainerDefault := getDefaultContainer() + initContainerDefault.Name = "init-container-default" + + initContainerCustom := getCustomContainer() + initContainerCustom.Name = "init-container-custom" + + expected := corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-default-name", + Namespace: "my-default-namespace", + Labels: map[string]string{ + "app": "operator", + "custom": "some", + }, + }, + Spec: corev1.PodSpec{ + NodeSelector: map[string]string{ + "node-0": "node-0", + "node-1": "node-1", + }, + ServiceAccountName: "my-service-account-override", + TerminationGracePeriodSeconds: int64Ref(11), + ActiveDeadlineSeconds: int64Ref(10), + NodeName: "my-node-name", + RestartPolicy: corev1.RestartPolicyAlways, + Containers: []corev1.Container{ + getDefaultContainer(), + getCustomContainer(), + }, + InitContainers: []corev1.Container{ + initContainerDefault, + initContainerCustom, + }, + Affinity: affinity("zone", "custom"), + }, + } + assert.Equal(t, expected, mergedSpec) +} + func TestMergeFromEmpty(t *testing.T) { defaultPodSpec := corev1.PodTemplateSpec{} customPodSpecTemplate := getCustomPodSpec() @@ -154,40 +200,53 @@ func TestMergeContainer(t *testing.T) { overrideOtherDefaultContainer.Env = []corev1.EnvVar{{Name: "env_var", Value: "xxx"}} overrideOtherDefaultContainer.VolumeMounts = []corev1.VolumeMount{anotherVol} - mergedContainers, err := mergeContainers( - []corev1.Container{getDefaultContainer(), otherDefaultContainer}, - []corev1.Container{getCustomContainer(), overrideDefaultContainer, overrideOtherDefaultContainer}, - ) + defaultSpec := getDefaultPodSpec() + defaultSpec.Spec.Containers = []corev1.Container{getDefaultContainer(), otherDefaultContainer} + customSpec := getCustomPodSpec() + customSpec.Spec.Containers = []corev1.Container{getCustomContainer(), overrideDefaultContainer, overrideOtherDefaultContainer} + + mergedSpec, err := MergePodTemplateSpecs(defaultSpec, customSpec) assert.NoError(t, err) - assert.Len(t, mergedContainers, 3) - - assert.Equal(t, getCustomContainer(), mergedContainers[2]) - - mergedDefaultContainer := mergedContainers[0] - assert.Equal(t, "container-0", mergedDefaultContainer.Name) - assert.Equal(t, []corev1.VolumeMount{vol0}, mergedDefaultContainer.VolumeMounts) - assert.Equal(t, "overridden", mergedDefaultContainer.Image) - // only "periodSeconds" was overwritten - other fields stayed untouched - assert.Equal(t, corev1.Handler{HTTPGet: &corev1.HTTPGetAction{Path: "/foo"}}, mergedDefaultContainer.ReadinessProbe.Handler) - assert.Equal(t, int32(20), mergedDefaultContainer.ReadinessProbe.PeriodSeconds) - - mergedOtherContainer := mergedContainers[1] - assert.Equal(t, "default-side-car", mergedOtherContainer.Name) - assert.Equal(t, []corev1.VolumeMount{sideCarVol, anotherVol}, mergedOtherContainer.VolumeMounts) - assert.Len(t, mergedOtherContainer.Env, 1) - assert.Equal(t, "env_var", mergedOtherContainer.Env[0].Name) - assert.Equal(t, "xxx", mergedOtherContainer.Env[0].Value) + + assert.Len(t, mergedSpec.Spec.Containers, 3) + assert.Equal(t, getCustomContainer(), mergedSpec.Spec.Containers[2]) + + firstExpected := corev1.Container{ + Name: "container-0", + VolumeMounts: []corev1.VolumeMount{vol0}, + Image: "overridden", + ReadinessProbe: &corev1.Probe{ + // only "periodSeconds" was overwritten - other fields stayed untouched + Handler: corev1.Handler{ + HTTPGet: &corev1.HTTPGetAction{Path: "/foo"}, + }, + PeriodSeconds: 20, + }, + } + assert.Equal(t, firstExpected, mergedSpec.Spec.Containers[0]) + + secondExpected := corev1.Container{ + Name: "default-side-car", + Image: "image-0", + VolumeMounts: []corev1.VolumeMount{sideCarVol, anotherVol}, + Env: []corev1.EnvVar{ + corev1.EnvVar{ + Name: "env_var", + Value: "xxx", + }, + }, + ReadinessProbe: otherDefaultContainer.ReadinessProbe, + } + assert.Equal(t, secondExpected, mergedSpec.Spec.Containers[1]) } func TestMergeVolumeMounts(t *testing.T) { vol0 := corev1.VolumeMount{Name: "container-0.volume-mount-0"} vol1 := corev1.VolumeMount{Name: "another-mount"} volumeMounts := []corev1.VolumeMount{vol0, vol1} - var mergedVolumeMounts []corev1.VolumeMount - var err error - mergedVolumeMounts, err = mergeVolumeMounts(nil, volumeMounts) + mergedVolumeMounts, err := mergeVolumeMounts(nil, volumeMounts) assert.NoError(t, err) assert.Equal(t, []corev1.VolumeMount{vol0, vol1}, mergedVolumeMounts) @@ -198,38 +257,6 @@ func TestMergeVolumeMounts(t *testing.T) { assert.Equal(t, []corev1.VolumeMount{vol2, vol0}, mergedVolumeMounts) } -func TestGetMergedDefaultPodSpecTemplate(t *testing.T) { - var err error - - dbPodSpecTemplate := getDefaultPodSpec() - var mergedPodSpecTemplate corev1.PodTemplateSpec - - // nothing to merge - mergedPodSpecTemplate, err = MergePodTemplateSpecs(corev1.PodTemplateSpec{}, dbPodSpecTemplate) - assert.NoError(t, err) - assert.Equal(t, mergedPodSpecTemplate, dbPodSpecTemplate) - assert.Len(t, mergedPodSpecTemplate.Spec.Containers, 1) - assert.Equal(t, mergedPodSpecTemplate.Spec.Containers[0], dbPodSpecTemplate.Spec.Containers[0]) - - extraContainer := corev1.Container{ - Name: "extra-container", - Image: "container-image", - } - - newPodSpecTemplate := corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{extraContainer}, - }, - } - - // with a side car container - mergedPodSpecTemplate, err = MergePodTemplateSpecs(newPodSpecTemplate, dbPodSpecTemplate) - assert.NoError(t, err) - assert.Len(t, mergedPodSpecTemplate.Spec.Containers, 2) - assert.Equal(t, mergedPodSpecTemplate.Spec.Containers[1], dbPodSpecTemplate.Spec.Containers[0]) - assert.Equal(t, mergedPodSpecTemplate.Spec.Containers[0], extraContainer) -} - func int64Ref(i int64) *int64 { return &i } From 6981bd3997772658319aed4848f52b1c8ec4f05e Mon Sep 17 00:00:00 2001 From: Fabian Lindfors Date: Thu, 30 Jul 2020 10:42:03 +0200 Subject: [PATCH 24/34] Change to make Evergreen run --- pkg/kube/podtemplatespec/podspec_template_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/kube/podtemplatespec/podspec_template_test.go b/pkg/kube/podtemplatespec/podspec_template_test.go index 9f4770a6c..3715aa9af 100644 --- a/pkg/kube/podtemplatespec/podspec_template_test.go +++ b/pkg/kube/podtemplatespec/podspec_template_test.go @@ -264,6 +264,7 @@ func int64Ref(i int64) *int64 { func getDefaultPodSpec() corev1.PodTemplateSpec { initContainer := getDefaultContainer() initContainer.Name = "init-container-default" + return corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Name: "my-default-name", @@ -287,6 +288,7 @@ func getDefaultPodSpec() corev1.PodTemplateSpec { func getCustomPodSpec() corev1.PodTemplateSpec { initContainer := getCustomContainer() initContainer.Name = "init-container-custom" + return corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{"custom": "some"}, From 28eca5b443165acd721b2081e820b098cc8cc287 Mon Sep 17 00:00:00 2001 From: Fabian Lindfors Date: Thu, 30 Jul 2020 15:44:49 +0200 Subject: [PATCH 25/34] Change to make Evergreen run --- pkg/kube/statefulset/statefulset.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/kube/statefulset/statefulset.go b/pkg/kube/statefulset/statefulset.go index fd6889efe..a9f34df6f 100644 --- a/pkg/kube/statefulset/statefulset.go +++ b/pkg/kube/statefulset/statefulset.go @@ -292,6 +292,7 @@ func mergeStatefulSetSpecs(defaultSpec, overrideSpec appsv1.StatefulSetSpec) (ap func mergeVolumeClaimTemplates(defaultTemplates []corev1.PersistentVolumeClaim, overrideTemplates []corev1.PersistentVolumeClaim) ([]corev1.PersistentVolumeClaim, error) { defaultMountsMap := createVolumeClaimMap(defaultTemplates) overrideMountsMap := createVolumeClaimMap(overrideTemplates) + var mergedVolumes []corev1.PersistentVolumeClaim for idx, defaultMount := range defaultMountsMap { if overrideMount, ok := overrideMountsMap[defaultMount.Name]; ok { @@ -302,6 +303,7 @@ func mergeVolumeClaimTemplates(defaultTemplates []corev1.PersistentVolumeClaim, } mergedVolumes = append(mergedVolumes, defaultMount) } + for _, overrideMount := range overrideMountsMap { if _, ok := defaultMountsMap[overrideMount.Name]; ok { // already merged @@ -309,9 +311,11 @@ func mergeVolumeClaimTemplates(defaultTemplates []corev1.PersistentVolumeClaim, } mergedVolumes = append(mergedVolumes, overrideMount) } + sort.SliceStable(mergedVolumes, func(i, j int) bool { return mergedVolumes[i].Name < mergedVolumes[j].Name }) + return mergedVolumes, nil } From 6831a61f73134fdf6bc0b7ab05a1723809acb759 Mon Sep 17 00:00:00 2001 From: Fabian Lindfors Date: Thu, 30 Jul 2020 15:46:47 +0200 Subject: [PATCH 26/34] Evergreen From a0b1e5cdda41550647d689fe7ca63efe64c0a823 Mon Sep 17 00:00:00 2001 From: Fabian Lindfors Date: Thu, 30 Jul 2020 15:48:01 +0200 Subject: [PATCH 27/34] Evergreen re-trigger From 7894026360039dfcaefb5e841dfa0b2ecd67881b Mon Sep 17 00:00:00 2001 From: Fabian Lindfors Date: Thu, 30 Jul 2020 15:48:27 +0200 Subject: [PATCH 28/34] Evergreen re-trigger From 375000985a3ba238510b473b0f3fe9fc5a434520 Mon Sep 17 00:00:00 2001 From: Fabian Lindfors Date: Thu, 30 Jul 2020 15:48:38 +0200 Subject: [PATCH 29/34] Evergreen re-trigger From 3ecda9bfc3b5246761af626c48a4fd01e2f3b22c Mon Sep 17 00:00:00 2001 From: Fabian Lindfors Date: Thu, 30 Jul 2020 16:30:35 +0200 Subject: [PATCH 30/34] Evergreen re-trigger From 8f67c227b53b20288d0f8fb93789af24982bf5f0 Mon Sep 17 00:00:00 2001 From: Fabian Lindfors Date: Fri, 31 Jul 2020 13:43:55 +0200 Subject: [PATCH 31/34] Merge master --- .evergreen.yml | 8 +- pkg/apis/mongodb/v1/mongodb_types.go | 7 +- pkg/controller/mongodb/mongodb_tls.go | 130 +++++++--- pkg/controller/mongodb/mongodb_tls_test.go | 245 ++++++++++++++++++ .../mongodb/replica_set_controller.go | 55 ++-- .../mongodb/replicaset_controller_test.go | 185 ------------- pkg/controller/watch/watch.go | 72 +++++ pkg/controller/watch/watch_test.go | 163 ++++++++++++ pkg/kube/client/client.go | 13 +- pkg/kube/client/client_test.go | 19 ++ pkg/kube/client/mocked_client.go | 8 +- pkg/kube/configmap/configmap.go | 11 + pkg/kube/secret/secret.go | 4 +- pkg/util/contains/contains.go | 14 +- .../replica_set_tls_rotate_test.go | 49 ++++ test/e2e/tlstests/tlstests.go | 58 +++++ testdata/tls/server.crt | 30 +-- testdata/tls/server.key | 50 ++-- testdata/tls/server_rotated.crt | 17 ++ testdata/tls/server_rotated.key | 27 ++ 20 files changed, 868 insertions(+), 297 deletions(-) create mode 100644 pkg/controller/mongodb/mongodb_tls_test.go create mode 100644 pkg/controller/watch/watch.go create mode 100644 pkg/controller/watch/watch_test.go create mode 100644 test/e2e/replica_set_tls_rotate/replica_set_tls_rotate_test.go create mode 100644 testdata/tls/server_rotated.crt create mode 100644 testdata/tls/server_rotated.key diff --git a/.evergreen.yml b/.evergreen.yml index 83a4d49c4..2d560f21f 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -151,6 +151,7 @@ task_groups: - e2e_test_replica_set_multiple - e2e_test_replica_set_tls - e2e_test_replica_set_tls_upgrade + - e2e_test_replica_set_tls_rotate - e2e_test_statefulset_arbitrary_config - e2e_test_statefulset_arbitrary_config_update teardown_task: @@ -279,6 +280,12 @@ tasks: vars: test: replica_set_tls_upgrade + - name: e2e_test_replica_set_tls_rotate + commands: + - func: run_e2e_test + vars: + test: replica_set_tls_rotate + - name: e2e_test_statefulset_arbitrary_config commands: - func: run_e2e_test @@ -291,7 +298,6 @@ tasks: vars: test: statefulset_arbitrary_config_update - buildvariants: - name: go_unit_tests display_name: go_unit_tests diff --git a/pkg/apis/mongodb/v1/mongodb_types.go b/pkg/apis/mongodb/v1/mongodb_types.go index 38954d1c9..bce102920 100644 --- a/pkg/apis/mongodb/v1/mongodb_types.go +++ b/pkg/apis/mongodb/v1/mongodb_types.go @@ -200,11 +200,16 @@ func (m MongoDB) TLSConfigMapNamespacedName() types.NamespacedName { } // TLSSecretNamespacedName will get the namespaced name of the Secret containing the server certificate and key -// As the Secret will be mounted to our pods, it has to be in the same namespace as the MongoDB resource func (m MongoDB) TLSSecretNamespacedName() types.NamespacedName { return types.NamespacedName{Name: m.Spec.Security.TLS.CertificateKeySecret.Name, Namespace: m.Namespace} } +// TLSOperatorSecretNamespacedName will get the namespaced name of the Secret created by the operator +// containing the combined certificate and key. +func (m MongoDB) TLSOperatorSecretNamespacedName() types.NamespacedName { + return types.NamespacedName{Name: m.Name + "-server-certificate-key", Namespace: m.Namespace} +} + func (m MongoDB) NamespacedName() types.NamespacedName { return types.NamespacedName{Name: m.Name, Namespace: m.Namespace} } diff --git a/pkg/controller/mongodb/mongodb_tls.go b/pkg/controller/mongodb/mongodb_tls.go index fd802ff30..e6d583b62 100644 --- a/pkg/controller/mongodb/mongodb_tls.go +++ b/pkg/controller/mongodb/mongodb_tls.go @@ -1,29 +1,34 @@ package mongodb import ( + "crypto/sha256" "fmt" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/automationconfig" "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/secret" "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/configmap" - "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/container" "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/podtemplatespec" "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/statefulset" - corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/pkg/apis/mongodb/v1" ) +const ( + tlsCAMountPath = "/var/lib/tls/ca/" + tlsCACertName = "ca.crt" + tlsOperatorSecretMountPath = "/var/lib/tls/server/" //nolint + tlsSecretCertName = "tls.crt" //nolint + tlsSecretKeyName = "tls.key" +) + // validateTLSConfig will check that the configured ConfigMap and Secret exist and that they have the correct fields. -// The possible return values are: -// - (true, nil) if the config is valid -// - (false, nil) if the config is not valid -// - (_, err) if an error occured when validating the config func (r *ReplicaSetReconciler) validateTLSConfig(mdb mdbv1.MongoDB) (bool, error) { if !mdb.Spec.Security.TLS.Enabled { return true, nil @@ -69,20 +74,85 @@ func (r *ReplicaSetReconciler) validateTLSConfig(mdb mdbv1.MongoDB) (bool, error return false, nil } + // Watch certificate-key secret to handle rotations + r.secretWatcher.Watch(mdb.TLSSecretNamespacedName(), mdb.NamespacedName()) + return true, nil } // getTLSConfigModification creates a modification function which enables TLS in the automation config. -// The config is only updated after the certs and keys have been rolled out to all pods. -// The agent needs these to be in place before the config is updated. -// Once the config is updated, the agents will gradually enable TLS in accordance with: https://docs.mongodb.com/manual/tutorial/upgrade-cluster-to-ssl/ -func getTLSConfigModification(mdb mdbv1.MongoDB) automationconfig.Modification { - if !(mdb.Spec.Security.TLS.Enabled && hasRolledOutTLS(mdb)) { - return automationconfig.NOOP() +// It will also ensure that the combined cert-key secret is created. +func getTLSConfigModification(getUpdateCreator secret.GetUpdateCreator, mdb mdbv1.MongoDB) (automationconfig.Modification, error) { + if !mdb.Spec.Security.TLS.Enabled { + return automationconfig.NOOP(), nil } + cert, key, err := getCertAndKey(getUpdateCreator, mdb) + if err != nil { + return automationconfig.NOOP(), err + } + + err = ensureTLSSecret(getUpdateCreator, mdb, cert, key) + if err != nil { + return automationconfig.NOOP(), err + } + + // The config is only updated after the certs and keys have been rolled out to all pods. + // The agent needs these to be in place before the config is updated. + // Once the config is updated, the agents will gradually enable TLS in accordance with: https://docs.mongodb.com/manual/tutorial/upgrade-cluster-to-ssl/ + if hasRolledOutTLS(mdb) { + return tlsConfigModification(mdb, cert, key), nil + } + + return automationconfig.NOOP(), nil +} + +// getCertAndKey will fetch the certificate and key from the user-provided Secret. +func getCertAndKey(getter secret.Getter, mdb mdbv1.MongoDB) (string, string, error) { + cert, err := secret.ReadKey(getter, tlsSecretCertName, mdb.TLSSecretNamespacedName()) + if err != nil { + return "", "", err + } + + key, err := secret.ReadKey(getter, tlsSecretKeyName, mdb.TLSSecretNamespacedName()) + if err != nil { + return "", "", err + } + + return cert, key, nil +} + +// ensureTLSSecret will create or update the operator-managed Secret containing +// the concatenated certificate and key from the user-provided Secret. +func ensureTLSSecret(getUpdateCreator secret.GetUpdateCreator, mdb mdbv1.MongoDB, cert, key string) error { + // Calculate file name from certificate and key + fileName := tlsOperatorSecretFileName(cert, key) + + operatorSecret := secret.Builder(). + SetName(mdb.TLSOperatorSecretNamespacedName().Name). + SetNamespace(mdb.TLSOperatorSecretNamespacedName().Namespace). + SetField(fileName, cert+key). + SetOwnerReferences([]metav1.OwnerReference{getOwnerReference(mdb)}). + Build() + + return secret.CreateOrUpdate(getUpdateCreator, operatorSecret) +} + +// tlsOperatorSecretFileName calculates the file name to use for the mounted +// certificate-key file. The name is based on the hash of the combined cert and key. +// If the certificate or key changes, the file path changes as well which will trigger +// the agent to perform a restart. +// The user-provided secret is being watched and will trigger a reconciliation +// on changes. This enables the operator to automatically handle cert rotations. +func tlsOperatorSecretFileName(cert, key string) string { + hash := sha256.Sum256([]byte(cert + key)) + return fmt.Sprintf("%x.pem", hash) +} + +// tlsConfigModification will enable TLS in the automation config. +func tlsConfigModification(mdb mdbv1.MongoDB, cert, key string) automationconfig.Modification { caCertificatePath := tlsCAMountPath + tlsCACertName - certificateKeyPath := tlsServerMountPath + tlsServerFileName + certificateKeyPath := tlsOperatorSecretMountPath + tlsOperatorSecretFileName(cert, key) mode := automationconfig.TLSModeRequired if mdb.Spec.Security.TLS.Optional { @@ -108,7 +178,7 @@ func getTLSConfigModification(mdb mdbv1.MongoDB) automationconfig.Modification { // hasRolledOutTLS determines if the TLS key and certs have been mounted to all pods. // These must be mounted before TLS can be enabled in the automation config. func hasRolledOutTLS(mdb mdbv1.MongoDB) bool { - _, completedRollout := mdb.Annotations[tLSRolledOutAnnotationKey] + _, completedRollout := mdb.Annotations[tlsRolledOutAnnotationKey] return completedRollout } @@ -122,7 +192,7 @@ func (r *ReplicaSetReconciler) completeTLSRollout(mdb mdbv1.MongoDB) error { r.log.Debug("Completing TLS rollout") - mdb.Annotations[tLSRolledOutAnnotationKey] = trueAnnotation + mdb.Annotations[tlsRolledOutAnnotationKey] = trueAnnotation if err := r.ensureAutomationConfig(mdb); err != nil { return fmt.Errorf("error updating automation config after TLS rollout: %+v", err) } @@ -140,10 +210,6 @@ func buildTLSPodSpecModification(mdb mdbv1.MongoDB) podtemplatespec.Modification return podtemplatespec.NOOP() } - // Configure an empty volume into which the TLS init container will write the certificate and key file - tlsVolume := statefulset.CreateVolumeFromEmptyDir("tls") - tlsVolumeMount := statefulset.CreateVolumeMount(tlsVolume.Name, tlsServerMountPath, statefulset.WithReadOnly(false)) - // Configure a volume which mounts the CA certificate from a ConfigMap // The certificate is used by both mongod and the agent caVolume := statefulset.CreateVolumeFromConfigMap("tls-ca", mdb.Spec.Security.TLS.CaConfigMap.Name) @@ -151,34 +217,16 @@ func buildTLSPodSpecModification(mdb mdbv1.MongoDB) podtemplatespec.Modification // Configure a volume which mounts the secret holding the server key and certificate // The same key-certificate pair is used for all servers - tlsSecretVolume := statefulset.CreateVolumeFromSecret("tls-secret", mdb.Spec.Security.TLS.CertificateKeySecret.Name) - tlsSecretVolumeMount := statefulset.CreateVolumeMount(tlsSecretVolume.Name, tlsSecretMountPath, statefulset.WithReadOnly(true)) + tlsSecretVolume := statefulset.CreateVolumeFromSecret("tls-secret", mdb.TLSOperatorSecretNamespacedName().Name) + tlsSecretVolumeMount := statefulset.CreateVolumeMount(tlsSecretVolume.Name, tlsOperatorSecretMountPath, statefulset.WithReadOnly(true)) // MongoDB expects both key and certificate to be provided in a single PEM file // We are using a secret format where they are stored in separate fields, tls.crt and tls.key // Because of this we need to use an init container which reads the two files mounted from the secret and combines them into one return podtemplatespec.Apply( - podtemplatespec.WithInitContainer("tls-init", tlsInit(tlsVolumeMount, tlsSecretVolumeMount)), - podtemplatespec.WithVolume(tlsVolume), podtemplatespec.WithVolume(caVolume), podtemplatespec.WithVolume(tlsSecretVolume), - podtemplatespec.WithVolumeMounts(agentName, tlsVolumeMount, caVolumeMount), - podtemplatespec.WithVolumeMounts(mongodbName, tlsVolumeMount, caVolumeMount), - ) -} - -// tlsInit creates an init container which combines the mounted tls.key and tls.crt into a single PEM file -func tlsInit(tlsMount, tlsSecretMount corev1.VolumeMount) container.Modification { - command := fmt.Sprintf( - "cat %s %s > %s", - tlsSecretMountPath+tlsSecretCertName, - tlsSecretMountPath+tlsSecretKeyName, - tlsServerMountPath+tlsServerFileName) - - return container.Apply( - container.WithName("tls-init"), - container.WithImage("busybox"), - container.WithCommand([]string{"sh", "-c", command}), - container.WithVolumeMounts([]corev1.VolumeMount{tlsMount, tlsSecretMount}), + podtemplatespec.WithVolumeMounts(agentName, tlsSecretVolumeMount, caVolumeMount), + podtemplatespec.WithVolumeMounts(mongodbName, tlsSecretVolumeMount, caVolumeMount), ) } diff --git a/pkg/controller/mongodb/mongodb_tls_test.go b/pkg/controller/mongodb/mongodb_tls_test.go new file mode 100644 index 000000000..0bbcb2eae --- /dev/null +++ b/pkg/controller/mongodb/mongodb_tls_test.go @@ -0,0 +1,245 @@ +package mongodb + +import ( + "context" + "testing" + + corev1 "k8s.io/api/core/v1" + + mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/pkg/apis/mongodb/v1" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/automationconfig" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/client" + mdbClient "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/client" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/configmap" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/secret" + "github.com/stretchr/testify/assert" + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/types" + k8sClient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +func TestStatefulSet_IsCorrectlyConfiguredWithTLS(t *testing.T) { + mdb := newTestReplicaSetWithTLS() + mgr := client.NewManager(&mdb) + + err := createTLSSecretAndConfigMap(mgr.GetClient(), mdb) + assert.NoError(t, err) + + r := newReconciler(mgr, mockManifestProvider(mdb.Spec.Version)) + res, err := r.Reconcile(reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) + assertReconciliationSuccessful(t, res, err) + + sts := appsv1.StatefulSet{} + err = mgr.GetClient().Get(context.TODO(), types.NamespacedName{Name: mdb.Name, Namespace: mdb.Namespace}, &sts) + assert.NoError(t, err) + + // Assert that all TLS volumes have been added. + assert.Len(t, sts.Spec.Template.Spec.Volumes, 5) + assert.Contains(t, sts.Spec.Template.Spec.Volumes, corev1.Volume{ + Name: "tls-ca", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: mdb.Spec.Security.TLS.CaConfigMap.Name, + }, + }, + }, + }) + assert.Contains(t, sts.Spec.Template.Spec.Volumes, corev1.Volume{ + Name: "tls-secret", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: mdb.TLSOperatorSecretNamespacedName().Name, + }, + }, + }) + + tlsSecretVolumeMount := corev1.VolumeMount{ + Name: "tls-secret", + ReadOnly: true, + MountPath: tlsOperatorSecretMountPath, + } + tlsCAVolumeMount := corev1.VolumeMount{ + Name: "tls-ca", + ReadOnly: true, + MountPath: tlsCAMountPath, + } + + assert.Len(t, sts.Spec.Template.Spec.InitContainers, 1) + + agentContainer := sts.Spec.Template.Spec.Containers[0] + assert.Contains(t, agentContainer.VolumeMounts, tlsSecretVolumeMount) + assert.Contains(t, agentContainer.VolumeMounts, tlsCAVolumeMount) + + mongodbContainer := sts.Spec.Template.Spec.Containers[1] + assert.Contains(t, mongodbContainer.VolumeMounts, tlsSecretVolumeMount) + assert.Contains(t, mongodbContainer.VolumeMounts, tlsCAVolumeMount) +} + +func TestAutomationConfig_IsCorrectlyConfiguredWithTLS(t *testing.T) { + createAC := func(mdb mdbv1.MongoDB) automationconfig.AutomationConfig { + client := mdbClient.NewClient(client.NewManager(&mdb).GetClient()) + err := createTLSSecretAndConfigMap(client, mdb) + assert.NoError(t, err) + + manifest, err := mockManifestProvider(mdb.Spec.Version)() + assert.NoError(t, err) + versionConfig := manifest.BuildsForVersion(mdb.Spec.Version) + + tlsModification, err := getTLSConfigModification(client, mdb) + assert.NoError(t, err) + + ac, err := buildAutomationConfig(mdb, versionConfig, automationconfig.AutomationConfig{}, tlsModification) + assert.NoError(t, err) + + return ac + } + + t.Run("With TLS disabled", func(t *testing.T) { + mdb := newTestReplicaSet() + ac := createAC(mdb) + + assert.Equal(t, automationconfig.TLS{ + CAFilePath: "", + ClientCertificateMode: automationconfig.ClientCertificateModeOptional, + }, ac.TLS) + + for _, process := range ac.Processes { + assert.Equal(t, automationconfig.MongoDBTLS{ + Mode: automationconfig.TLSModeDisabled, + }, process.Args26.Net.TLS) + } + }) + + t.Run("With TLS enabled, during rollout", func(t *testing.T) { + mdb := newTestReplicaSetWithTLS() + ac := createAC(mdb) + + assert.Equal(t, automationconfig.TLS{ + CAFilePath: "", + ClientCertificateMode: automationconfig.ClientCertificateModeOptional, + }, ac.TLS) + + for _, process := range ac.Processes { + assert.Equal(t, automationconfig.MongoDBTLS{ + Mode: automationconfig.TLSModeDisabled, + }, process.Args26.Net.TLS) + } + }) + + t.Run("With TLS enabled and required, rollout completed", func(t *testing.T) { + mdb := newTestReplicaSetWithTLS() + mdb.Annotations[tlsRolledOutAnnotationKey] = "true" + ac := createAC(mdb) + + assert.Equal(t, automationconfig.TLS{ + CAFilePath: tlsCAMountPath + tlsCACertName, + ClientCertificateMode: automationconfig.ClientCertificateModeOptional, + }, ac.TLS) + + for _, process := range ac.Processes { + operatorSecretFileName := tlsOperatorSecretFileName("CERT", "KEY") + + assert.Equal(t, automationconfig.MongoDBTLS{ + Mode: automationconfig.TLSModeRequired, + PEMKeyFile: tlsOperatorSecretMountPath + operatorSecretFileName, + CAFile: tlsCAMountPath + tlsCACertName, + AllowConnectionsWithoutCertificate: true, + }, process.Args26.Net.TLS) + } + }) + + t.Run("With TLS enabled and optional, rollout completed", func(t *testing.T) { + mdb := newTestReplicaSetWithTLS() + mdb.Annotations[tlsRolledOutAnnotationKey] = "true" + mdb.Spec.Security.TLS.Optional = true + ac := createAC(mdb) + + assert.Equal(t, automationconfig.TLS{ + CAFilePath: tlsCAMountPath + tlsCACertName, + ClientCertificateMode: automationconfig.ClientCertificateModeOptional, + }, ac.TLS) + + for _, process := range ac.Processes { + operatorSecretFileName := tlsOperatorSecretFileName("CERT", "KEY") + + assert.Equal(t, automationconfig.MongoDBTLS{ + Mode: automationconfig.TLSModePreferred, + PEMKeyFile: tlsOperatorSecretMountPath + operatorSecretFileName, + CAFile: tlsCAMountPath + tlsCACertName, + AllowConnectionsWithoutCertificate: true, + }, process.Args26.Net.TLS) + } + }) +} + +func TestTLSOperatorSecret(t *testing.T) { + t.Run("Secret is created if it doesn't exist", func(t *testing.T) { + mdb := newTestReplicaSetWithTLS() + client := mdbClient.NewClient(client.NewManager(&mdb).GetClient()) + err := createTLSSecretAndConfigMap(client, mdb) + assert.NoError(t, err) + + _, err = getTLSConfigModification(client, mdb) + assert.NoError(t, err) + + // Operator-managed secret should have been created and contain the + // concatenated certificate and key. + certificateKey, err := secret.ReadKey(client, tlsOperatorSecretFileName("CERT", "KEY"), mdb.TLSOperatorSecretNamespacedName()) + assert.NoError(t, err) + assert.Equal(t, "CERTKEY", certificateKey) + }) + + t.Run("Secret is updated if it already exists", func(t *testing.T) { + mdb := newTestReplicaSetWithTLS() + client := mdbClient.NewClient(client.NewManager(&mdb).GetClient()) + err := createTLSSecretAndConfigMap(client, mdb) + assert.NoError(t, err) + + // Create operator-managed secret + s := secret.Builder(). + SetName(mdb.TLSOperatorSecretNamespacedName().Name). + SetNamespace(mdb.TLSOperatorSecretNamespacedName().Namespace). + SetField(tlsOperatorSecretFileName("", ""), ""). + Build() + err = client.CreateSecret(s) + assert.NoError(t, err) + + _, err = getTLSConfigModification(client, mdb) + assert.NoError(t, err) + + // Operator-managed secret should have been updated with the concatenated + // certificate and key. + certificateKey, err := secret.ReadKey(client, tlsOperatorSecretFileName("CERT", "KEY"), mdb.TLSOperatorSecretNamespacedName()) + assert.NoError(t, err) + assert.Equal(t, "CERTKEY", certificateKey) + }) +} + +func createTLSSecretAndConfigMap(c k8sClient.Client, mdb mdbv1.MongoDB) error { + s := secret.Builder(). + SetName(mdb.Spec.Security.TLS.CertificateKeySecret.Name). + SetNamespace(mdb.Namespace). + SetField("tls.crt", "CERT"). + SetField("tls.key", "KEY"). + Build() + + err := c.Create(context.TODO(), &s) + if err != nil { + return err + } + + configMap := configmap.Builder(). + SetName(mdb.Spec.Security.TLS.CaConfigMap.Name). + SetNamespace(mdb.Namespace). + SetField("ca.crt", "CERT"). + Build() + + err = c.Create(context.TODO(), &configMap) + if err != nil { + return err + } + + return nil +} diff --git a/pkg/controller/mongodb/replica_set_controller.go b/pkg/controller/mongodb/replica_set_controller.go index ceec3f339..fe0851238 100644 --- a/pkg/controller/mongodb/replica_set_controller.go +++ b/pkg/controller/mongodb/replica_set_controller.go @@ -9,6 +9,8 @@ import ( "os" "time" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/controller/watch" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/persistentvolumeclaim" "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/probes" @@ -57,19 +59,11 @@ const ( operatorServiceAccountName = "mongodb-kubernetes-operator" agentHealthStatusFilePathValue = "/var/log/mongodb-mms-automation/healthstatus/agent-health-status.json" - tlsCAMountPath = "/var/lib/tls/ca/" - tlsCACertName = "ca.crt" - tlsSecretMountPath = "/var/lib/tls/secret/" //nolint - tlsSecretCertName = "tls.crt" //nolint - tlsSecretKeyName = "tls.key" - tlsServerMountPath = "/var/lib/tls/server/" - tlsServerFileName = "server.pem" - // lastVersionAnnotationKey should indicate which version of MongoDB was last // configured lastVersionAnnotationKey = "mongodb.com/v1.lastVersion" - // TLSRolledOutKey indicates if TLS has been fully rolled out - tLSRolledOutAnnotationKey = "mongodb.com/v1.tlsRolledOut" + // tlsRolledOutAnnotationKey indicates if TLS has been fully rolled out + tlsRolledOutAnnotationKey = "mongodb.com/v1.tlsRolledOut" hasLeftReadyStateAnnotationKey = "mongodb.com/v1.hasLeftReadyStateAnnotationKey" trueAnnotation = "true" @@ -85,19 +79,22 @@ func Add(mgr manager.Manager) error { // contains the list of all available MongoDB versions type ManifestProvider func() (automationconfig.VersionManifest, error) -// newReconciler returns a new reconcile.Reconciler -func newReconciler(mgr manager.Manager, manifestProvider ManifestProvider) reconcile.Reconciler { +func newReconciler(mgr manager.Manager, manifestProvider ManifestProvider) *ReplicaSetReconciler { mgrClient := mgr.GetClient() + secretWatcher := watch.New() + return &ReplicaSetReconciler{ client: kubernetesClient.NewClient(mgrClient), scheme: mgr.GetScheme(), manifestProvider: manifestProvider, log: zap.S(), + secretWatcher: &secretWatcher, } } -// add adds a new Controller to mgr with r as the reconcile.Reconciler -func add(mgr manager.Manager, r reconcile.Reconciler) error { +// add sets up a controller for the Reconciler on the manager. It will +// also configure the necessary watches. +func add(mgr manager.Manager, r *ReplicaSetReconciler) error { // Create a new controller c, err := controller.New("replicaset-controller", mgr, controller.Options{Reconciler: r}) if err != nil { @@ -109,6 +106,12 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { if err != nil { return err } + + err = c.Watch(&source.Kind{Type: &corev1.Secret{}}, r.secretWatcher) + if err != nil { + return err + } + return nil } @@ -123,6 +126,7 @@ type ReplicaSetReconciler struct { scheme *runtime.Scheme manifestProvider func() (automationconfig.VersionManifest, error) log *zap.SugaredLogger + secretWatcher *watch.ResourceWatcher } // Reconcile reads that state of the cluster for a MongoDB object and makes changes based on the state read @@ -438,7 +442,10 @@ func (r ReplicaSetReconciler) buildAutomationConfigConfigMap(mdb mdbv1.MongoDB) return corev1.ConfigMap{}, err } - tlsModification := getTLSConfigModification(mdb) + tlsModification, err := getTLSConfigModification(r.client, mdb) + if err != nil { + return corev1.ConfigMap{}, err + } currentAC, err := getCurrentAutomationConfig(r.client, mdb) if err != nil { @@ -557,14 +564,6 @@ func buildStatefulSetModificationFunction(mdb mdbv1.MongoDB) statefulset.Modific "app": mdb.ServiceName(), } - ownerReferences := []metav1.OwnerReference{ - *metav1.NewControllerRef(&mdb, schema.GroupVersionKind{ - Group: mdbv1.SchemeGroupVersion.Group, - Version: mdbv1.SchemeGroupVersion.Version, - Kind: mdb.Kind, - }), - } - // the health status volume is required in both agent and mongod pods. // the mongod requires it to determine if an upgrade is happening and needs to kill the pod // to prevent agent deadlock @@ -587,7 +586,7 @@ func buildStatefulSetModificationFunction(mdb mdbv1.MongoDB) statefulset.Modific statefulset.WithServiceName(mdb.ServiceName()), statefulset.WithLabels(labels), statefulset.WithMatchLabels(labels), - statefulset.WithOwnerReference(ownerReferences), + statefulset.WithOwnerReference([]metav1.OwnerReference{getOwnerReference(mdb)}), statefulset.WithReplicas(mdb.Spec.Members), statefulset.WithUpdateStrategyType(getUpdateStrategyType(mdb)), statefulset.WithVolumeClaim(dataVolumeName, defaultPvc()), @@ -609,6 +608,14 @@ func buildStatefulSetModificationFunction(mdb mdbv1.MongoDB) statefulset.Modific ) } +func getOwnerReference(mdb mdbv1.MongoDB) metav1.OwnerReference { + return *metav1.NewControllerRef(&mdb, schema.GroupVersionKind{ + Group: mdbv1.SchemeGroupVersion.Group, + Version: mdbv1.SchemeGroupVersion.Version, + Kind: mdb.Kind, + }) +} + func getDomain(service, namespace, clusterName string) string { if clusterName == "" { clusterName = "cluster.local" diff --git a/pkg/controller/mongodb/replicaset_controller_test.go b/pkg/controller/mongodb/replicaset_controller_test.go index f470f4757..5eb59cb67 100644 --- a/pkg/controller/mongodb/replicaset_controller_test.go +++ b/pkg/controller/mongodb/replicaset_controller_test.go @@ -7,8 +7,6 @@ import ( "testing" "time" - "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/configmap" - k8sClient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/mongodb/mongodb-kubernetes-operator/pkg/authentication/scram" @@ -358,189 +356,6 @@ func TestScramIsConfigured(t *testing.T) { }) } -func TestStatefulSet_IsCorrectlyConfiguredWithTLS(t *testing.T) { - mdb := newTestReplicaSetWithTLS() - mgr := client.NewManager(&mdb) - - s := secret.Builder(). - SetName(mdb.Spec.Security.TLS.CertificateKeySecret.Name). - SetNamespace(mdb.Namespace). - SetField("tls.crt", "CERT"). - SetField("tls.key", "KEY"). - Build() - - err := mgr.GetClient().Create(context.TODO(), &s) - assert.NoError(t, err) - - configMap := configmap.Builder(). - SetName(mdb.Spec.Security.TLS.CaConfigMap.Name). - SetNamespace(mdb.Namespace). - SetField("ca.crt", "CERT"). - Build() - - err = mgr.GetClient().Create(context.TODO(), &configMap) - assert.NoError(t, err) - - r := newReconciler(mgr, mockManifestProvider(mdb.Spec.Version)) - res, err := r.Reconcile(reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) - assertReconciliationSuccessful(t, res, err) - - sts := appsv1.StatefulSet{} - err = mgr.GetClient().Get(context.TODO(), types.NamespacedName{Name: mdb.Name, Namespace: mdb.Namespace}, &sts) - assert.NoError(t, err) - - // Assert that all TLS volumes have been added. - assert.Len(t, sts.Spec.Template.Spec.Volumes, 6) - assert.Contains(t, sts.Spec.Template.Spec.Volumes, corev1.Volume{ - Name: "tls", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }) - assert.Contains(t, sts.Spec.Template.Spec.Volumes, corev1.Volume{ - Name: "tls-ca", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: mdb.Spec.Security.TLS.CaConfigMap.Name, - }, - }, - }, - }) - assert.Contains(t, sts.Spec.Template.Spec.Volumes, corev1.Volume{ - Name: "tls-secret", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: mdb.Spec.Security.TLS.CertificateKeySecret.Name, - }, - }, - }) - - // Assert that the TLS init container has been added. - tlsVolumeMount := corev1.VolumeMount{ - Name: "tls", - ReadOnly: false, - MountPath: tlsServerMountPath, - } - tlsSecretVolumeMount := corev1.VolumeMount{ - Name: "tls-secret", - ReadOnly: true, - MountPath: tlsSecretMountPath, - } - tlsCAVolumeMount := corev1.VolumeMount{ - Name: "tls-ca", - ReadOnly: true, - MountPath: tlsCAMountPath, - } - - assert.Len(t, sts.Spec.Template.Spec.InitContainers, 2) - tlsInitContainer := sts.Spec.Template.Spec.InitContainers[1] - assert.Equal(t, "tls-init", tlsInitContainer.Name) - - // Assert that all containers have the correct volumes mounted. - assert.Len(t, tlsInitContainer.VolumeMounts, 2) - assert.Contains(t, tlsInitContainer.VolumeMounts, tlsVolumeMount) - assert.Contains(t, tlsInitContainer.VolumeMounts, tlsSecretVolumeMount) - - agentContainer := sts.Spec.Template.Spec.Containers[0] - assert.Contains(t, agentContainer.VolumeMounts, tlsVolumeMount) - assert.Contains(t, agentContainer.VolumeMounts, tlsCAVolumeMount) - - mongodbContainer := sts.Spec.Template.Spec.Containers[1] - assert.Contains(t, mongodbContainer.VolumeMounts, tlsVolumeMount) - assert.Contains(t, mongodbContainer.VolumeMounts, tlsCAVolumeMount) -} - -func TestAutomationConfig_IsCorrectlyConfiguredWithTLS(t *testing.T) { - createAC := func(mdb mdbv1.MongoDB) automationconfig.AutomationConfig { - manifest, err := mockManifestProvider(mdb.Spec.Version)() - assert.NoError(t, err) - versionConfig := manifest.BuildsForVersion(mdb.Spec.Version) - - ac, err := buildAutomationConfig( - mdb, - versionConfig, - automationconfig.AutomationConfig{}, - getTLSConfigModification(mdb), - ) - assert.NoError(t, err) - return ac - } - - t.Run("With TLS disabled", func(t *testing.T) { - mdb := newTestReplicaSet() - ac := createAC(mdb) - - assert.Equal(t, automationconfig.TLS{ - CAFilePath: "", - ClientCertificateMode: automationconfig.ClientCertificateModeOptional, - }, ac.TLS) - - for _, process := range ac.Processes { - assert.Equal(t, automationconfig.MongoDBTLS{ - Mode: automationconfig.TLSModeDisabled, - }, process.Args26.Net.TLS) - } - }) - - t.Run("With TLS enabled, during rollout", func(t *testing.T) { - mdb := newTestReplicaSetWithTLS() - ac := createAC(mdb) - - assert.Equal(t, automationconfig.TLS{ - CAFilePath: "", - ClientCertificateMode: automationconfig.ClientCertificateModeOptional, - }, ac.TLS) - - for _, process := range ac.Processes { - assert.Equal(t, automationconfig.MongoDBTLS{ - Mode: automationconfig.TLSModeDisabled, - }, process.Args26.Net.TLS) - } - }) - - t.Run("With TLS enabled and required, rollout completed", func(t *testing.T) { - mdb := newTestReplicaSetWithTLS() - mdb.Annotations[tLSRolledOutAnnotationKey] = "true" - ac := createAC(mdb) - - assert.Equal(t, automationconfig.TLS{ - CAFilePath: tlsCAMountPath + tlsCACertName, - ClientCertificateMode: automationconfig.ClientCertificateModeOptional, - }, ac.TLS) - - for _, process := range ac.Processes { - assert.Equal(t, automationconfig.MongoDBTLS{ - Mode: automationconfig.TLSModeRequired, - PEMKeyFile: tlsServerMountPath + tlsServerFileName, - CAFile: tlsCAMountPath + tlsCACertName, - AllowConnectionsWithoutCertificate: true, - }, process.Args26.Net.TLS) - } - }) - - t.Run("With TLS enabled and optional, rollout completed", func(t *testing.T) { - mdb := newTestReplicaSetWithTLS() - mdb.Annotations[tLSRolledOutAnnotationKey] = "true" - mdb.Spec.Security.TLS.Optional = true - ac := createAC(mdb) - - assert.Equal(t, automationconfig.TLS{ - CAFilePath: tlsCAMountPath + tlsCACertName, - ClientCertificateMode: automationconfig.ClientCertificateModeOptional, - }, ac.TLS) - - for _, process := range ac.Processes { - assert.Equal(t, automationconfig.MongoDBTLS{ - Mode: automationconfig.TLSModePreferred, - PEMKeyFile: tlsServerMountPath + tlsServerFileName, - CAFile: tlsCAMountPath + tlsCACertName, - AllowConnectionsWithoutCertificate: true, - }, process.Args26.Net.TLS) - } - }) -} - func assertReconciliationSuccessful(t *testing.T, result reconcile.Result, err error) { assert.NoError(t, err) assert.Equal(t, false, result.Requeue) diff --git a/pkg/controller/watch/watch.go b/pkg/controller/watch/watch.go new file mode 100644 index 000000000..183433ebb --- /dev/null +++ b/pkg/controller/watch/watch.go @@ -0,0 +1,72 @@ +package watch + +import ( + "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/contains" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/workqueue" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +// ResourceWatcher implements handler.EventHandler and is used to trigger reconciliation when +// a watched object changes. It's designed to only be used for a single type of object. +// If multiple types should be watched, one ResourceWatcher for each type should be used. +type ResourceWatcher struct { + watched map[types.NamespacedName][]types.NamespacedName +} + +// New will create a new ResourceWatcher with no watched objects. +func New() ResourceWatcher { + return ResourceWatcher{ + watched: make(map[types.NamespacedName][]types.NamespacedName), + } +} + +// Watch will add a new object to watch. +func (w ResourceWatcher) Watch(watchedName, dependentName types.NamespacedName) { + existing, hasExisting := w.watched[watchedName] + if !hasExisting { + existing = []types.NamespacedName{} + } + + // Check if resource is already being watched. + if contains.NamespacedName(existing, dependentName) { + return + } + + w.watched[watchedName] = append(existing, dependentName) +} + +func (w ResourceWatcher) Create(event event.CreateEvent, queue workqueue.RateLimitingInterface) { + w.handleEvent(event.Meta, queue) +} + +func (w ResourceWatcher) Update(event event.UpdateEvent, queue workqueue.RateLimitingInterface) { + w.handleEvent(event.MetaOld, queue) +} + +func (w ResourceWatcher) Delete(event event.DeleteEvent, queue workqueue.RateLimitingInterface) { + w.handleEvent(event.Meta, queue) +} + +func (w ResourceWatcher) Generic(event event.GenericEvent, queue workqueue.RateLimitingInterface) { + w.handleEvent(event.Meta, queue) +} + +// handleEvent is called when an event is received for an object. +// It will check if the object is being watched and trigger a reconciliation for +// the dependent object. +func (w ResourceWatcher) handleEvent(meta metav1.Object, queue workqueue.RateLimitingInterface) { + changedObjectName := types.NamespacedName{ + Name: meta.GetName(), + Namespace: meta.GetNamespace(), + } + + // Enqueue reconciliation for each dependent object. + for _, reconciledObjectName := range w.watched[changedObjectName] { + queue.Add(reconcile.Request{ + NamespacedName: reconciledObjectName, + }) + } +} diff --git a/pkg/controller/watch/watch_test.go b/pkg/controller/watch/watch_test.go new file mode 100644 index 000000000..091966b49 --- /dev/null +++ b/pkg/controller/watch/watch_test.go @@ -0,0 +1,163 @@ +package watch + +import ( + "testing" + + "k8s.io/apimachinery/pkg/types" + + mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/pkg/apis/mongodb/v1" + + "github.com/stretchr/testify/assert" + + "sigs.k8s.io/controller-runtime/pkg/controller/controllertest" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/util/workqueue" + + "sigs.k8s.io/controller-runtime/pkg/event" +) + +func TestWatcher(t *testing.T) { + obj := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod", + Namespace: "namespace", + }, + } + objNsName := types.NamespacedName{Name: obj.Name, Namespace: obj.Namespace} + + mdb1 := mdbv1.MongoDB{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mdb1", + Namespace: "namespace", + }, + } + + mdb2 := mdbv1.MongoDB{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mdb2", + Namespace: "namespace", + }, + } + + t.Run("Non-watched object", func(t *testing.T) { + watcher := New() + queue := controllertest.Queue{Interface: workqueue.New()} + + watcher.Create(event.CreateEvent{ + Meta: obj.GetObjectMeta(), + Object: obj, + }, queue) + + // Ensure no reconciliation is queued if object is not watched. + assert.Equal(t, 0, queue.Len()) + }) + + t.Run("Multiple objects to reconile", func(t *testing.T) { + watcher := New() + queue := controllertest.Queue{Interface: workqueue.New()} + watcher.Watch(objNsName, mdb1.NamespacedName()) + watcher.Watch(objNsName, mdb2.NamespacedName()) + + watcher.Create(event.CreateEvent{ + Meta: obj.GetObjectMeta(), + Object: obj, + }, queue) + + // Ensure multiple reconciliations are enqueued. + assert.Equal(t, 2, queue.Len()) + }) + + t.Run("Create event", func(t *testing.T) { + watcher := New() + queue := controllertest.Queue{Interface: workqueue.New()} + watcher.Watch(objNsName, mdb1.NamespacedName()) + + watcher.Create(event.CreateEvent{ + Meta: obj.GetObjectMeta(), + Object: obj, + }, queue) + + assert.Equal(t, 1, queue.Len()) + }) + + t.Run("Update event", func(t *testing.T) { + watcher := New() + queue := controllertest.Queue{Interface: workqueue.New()} + watcher.Watch(objNsName, mdb1.NamespacedName()) + + watcher.Update(event.UpdateEvent{ + MetaOld: obj.GetObjectMeta(), + ObjectOld: obj, + MetaNew: obj.GetObjectMeta(), + ObjectNew: obj, + }, queue) + + assert.Equal(t, 1, queue.Len()) + }) + + t.Run("Delete event", func(t *testing.T) { + watcher := New() + queue := controllertest.Queue{Interface: workqueue.New()} + watcher.Watch(objNsName, mdb1.NamespacedName()) + + watcher.Delete(event.DeleteEvent{ + Meta: obj.GetObjectMeta(), + Object: obj, + }, queue) + + assert.Equal(t, 1, queue.Len()) + }) + + t.Run("Generic event", func(t *testing.T) { + watcher := New() + queue := controllertest.Queue{Interface: workqueue.New()} + watcher.Watch(objNsName, mdb1.NamespacedName()) + + watcher.Generic(event.GenericEvent{ + Meta: obj.GetObjectMeta(), + Object: obj, + }, queue) + + assert.Equal(t, 1, queue.Len()) + }) +} + +func TestWatcherAdd(t *testing.T) { + watcher := New() + assert.Empty(t, watcher.watched) + + watchedName := types.NamespacedName{Name: "object", Namespace: "namespace"} + + mdb1 := mdbv1.MongoDB{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mdb1", + Namespace: "namespace", + }, + } + mdb2 := mdbv1.MongoDB{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mdb2", + Namespace: "namespace", + }, + } + + // Ensure single object can be added to empty watchlist. + watcher.Watch(watchedName, mdb1.NamespacedName()) + assert.Len(t, watcher.watched, 1) + assert.Equal(t, []types.NamespacedName{mdb1.NamespacedName()}, watcher.watched[watchedName]) + + // Ensure object can only be watched once. + watcher.Watch(watchedName, mdb1.NamespacedName()) + assert.Len(t, watcher.watched, 1) + assert.Equal(t, []types.NamespacedName{mdb1.NamespacedName()}, watcher.watched[watchedName]) + + // Ensure a single object can be watched for multiple reconciliations. + watcher.Watch(watchedName, mdb2.NamespacedName()) + assert.Len(t, watcher.watched, 1) + assert.Equal(t, []types.NamespacedName{ + mdb1.NamespacedName(), + mdb2.NamespacedName(), + }, watcher.watched[watchedName]) +} diff --git a/pkg/kube/client/client.go b/pkg/kube/client/client.go index 6fc928270..c72e7609e 100644 --- a/pkg/kube/client/client.go +++ b/pkg/kube/client/client.go @@ -26,7 +26,7 @@ type Client interface { k8sClient.Client // TODO: remove this function, add mongodb package which has GetAndUpdate function GetAndUpdate(nsName types.NamespacedName, obj runtime.Object, updateFunc func()) error - configmap.GetUpdateCreator + configmap.GetUpdateCreateDeleter service.GetUpdateCreator secret.GetUpdateCreateDeleter statefulset.GetUpdateCreateDeleter @@ -68,6 +68,17 @@ func (c client) CreateConfigMap(cm corev1.ConfigMap) error { return c.Create(context.TODO(), &cm) } +// DeleteConfigMap deletes the configmap of the given object key +func (c client) DeleteConfigMap(key k8sClient.ObjectKey) error { + cm := corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: key.Name, + Namespace: key.Namespace, + }, + } + return c.Delete(context.TODO(), &cm) +} + // GetSecret provides a thin wrapper and client.Client to access corev1.Secret types func (c client) GetSecret(objectKey k8sClient.ObjectKey) (corev1.Secret, error) { s := corev1.Secret{} diff --git a/pkg/kube/client/client_test.go b/pkg/kube/client/client_test.go index 4544f4285..07757579f 100644 --- a/pkg/kube/client/client_test.go +++ b/pkg/kube/client/client_test.go @@ -4,6 +4,8 @@ import ( "context" "testing" + "k8s.io/apimachinery/pkg/types" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/configmap" "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" @@ -62,3 +64,20 @@ func TestAddingDataField_ModifiesExistingObject(t *testing.T) { assert.Contains(t, cm.Data, "new-field") assert.Equal(t, cm.Data["new-field"], "value") } + +func TestDeleteConfigMap(t *testing.T) { + cm := configmap.Builder(). + SetName("config-map"). + SetNamespace("default"). + Build() + + client := NewClient(NewMockedClient()) + err := client.CreateConfigMap(cm) + assert.NoError(t, err) + + err = client.DeleteConfigMap(types.NamespacedName{Name: "config-map", Namespace: "default"}) + assert.NoError(t, err) + + _, err = client.GetConfigMap(types.NamespacedName{Name: "config-map", Namespace: "default"}) + assert.Equal(t, err, notFoundError()) +} diff --git a/pkg/kube/client/mocked_client.go b/pkg/kube/client/mocked_client.go index f4166c657..fa489134b 100644 --- a/pkg/kube/client/mocked_client.go +++ b/pkg/kube/client/mocked_client.go @@ -76,7 +76,13 @@ func (m *mockedClient) List(_ context.Context, _ runtime.Object, _ ...k8sClient. return nil } -func (m *mockedClient) Delete(_ context.Context, _ runtime.Object, _ ...k8sClient.DeleteOption) error { +func (m *mockedClient) Delete(_ context.Context, obj runtime.Object, _ ...k8sClient.DeleteOption) error { + relevantMap := m.ensureMapFor(obj) + objKey, err := k8sClient.ObjectKeyFromObject(obj) + if err != nil { + return err + } + delete(relevantMap, objKey) return nil } diff --git a/pkg/kube/configmap/configmap.go b/pkg/kube/configmap/configmap.go index 3677b48c8..23462ac10 100644 --- a/pkg/kube/configmap/configmap.go +++ b/pkg/kube/configmap/configmap.go @@ -21,6 +21,10 @@ type Creator interface { CreateConfigMap(cm corev1.ConfigMap) error } +type Deleter interface { + DeleteConfigMap(key client.ObjectKey) error +} + type GetUpdater interface { Getter Updater @@ -32,6 +36,13 @@ type GetUpdateCreator interface { Creator } +type GetUpdateCreateDeleter interface { + Getter + Updater + Creator + Deleter +} + // ReadKey accepts a ConfigMap Getter, the object of the ConfigMap to get, and the key within // the config map to read. It returns the string value, and an error if one occurred. func ReadKey(getter Getter, key string, objectKey client.ObjectKey) (string, error) { diff --git a/pkg/kube/secret/secret.go b/pkg/kube/secret/secret.go index 65ca9cb3a..f5b109e72 100644 --- a/pkg/kube/secret/secret.go +++ b/pkg/kube/secret/secret.go @@ -89,14 +89,14 @@ func UpdateField(getUpdater GetUpdater, objectKey client.ObjectKey, key, value s // CreateOrUpdate creates the Secret if it doesn't exist, other wise it updates it func CreateOrUpdate(getUpdateCreator GetUpdateCreator, secret corev1.Secret) error { - newSecret, err := getUpdateCreator.GetSecret(types.NamespacedName{Name: secret.Name, Namespace: secret.Namespace}) + _, err := getUpdateCreator.GetSecret(types.NamespacedName{Name: secret.Name, Namespace: secret.Namespace}) if err != nil { if apiErrors.IsNotFound(err) { return getUpdateCreator.CreateSecret(secret) } return err } - return getUpdateCreator.UpdateSecret(newSecret) + return getUpdateCreator.UpdateSecret(secret) } // HasAllKeys returns true if the provided secret contains an element for every diff --git a/pkg/util/contains/contains.go b/pkg/util/contains/contains.go index f0fdabbfa..3bdccc921 100644 --- a/pkg/util/contains/contains.go +++ b/pkg/util/contains/contains.go @@ -1,6 +1,9 @@ package contains -import mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/pkg/apis/mongodb/v1" +import ( + mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/pkg/apis/mongodb/v1" + "k8s.io/apimachinery/pkg/types" +) func String(slice []string, s string) bool { for _, elem := range slice { @@ -19,3 +22,12 @@ func AuthMode(slice []mdbv1.AuthMode, s mdbv1.AuthMode) bool { } return false } + +func NamespacedName(nsNames []types.NamespacedName, nsName types.NamespacedName) bool { + for _, elem := range nsNames { + if elem == nsName { + return true + } + } + return false +} diff --git a/test/e2e/replica_set_tls_rotate/replica_set_tls_rotate_test.go b/test/e2e/replica_set_tls_rotate/replica_set_tls_rotate_test.go new file mode 100644 index 000000000..c0680ee34 --- /dev/null +++ b/test/e2e/replica_set_tls_rotate/replica_set_tls_rotate_test.go @@ -0,0 +1,49 @@ +package replica_set_tls + +import ( + "testing" + "time" + + "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/tlstests" + + e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" + "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/mongodbtests" + setup "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" + f "github.com/operator-framework/operator-sdk/pkg/test" +) + +func TestMain(m *testing.M) { + f.MainEntry(m) +} + +func TestReplicaSetTLSRotate(t *testing.T) { + ctx, shouldCleanup := setup.InitTest(t) + if shouldCleanup { + defer ctx.Cleanup() + } + + mdb, user := e2eutil.NewTestMongoDB("mdb-tls") + mdb.Spec.Security.TLS = e2eutil.NewTestTLSConfig(false) + + _, err := setup.GeneratePasswordForUser(user, ctx) + if err != nil { + t.Fatal(err) + } + + if err := setup.CreateTLSResources(mdb.Namespace, ctx); err != nil { + t.Fatalf("Failed to set up TLS resources: %+v", err) + } + + t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) + t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb)) + t.Run("Wait for TLS to be enabled", tlstests.WaitForTLSMode(&mdb, "requireSSL")) + t.Run("Test Basic TLS Connectivity", tlstests.ConnectivityWithTLS(&mdb)) + t.Run("Test TLS required", tlstests.ConnectivityWithoutTLSShouldFail(&mdb)) + + t.Run("MongoDB is reachable while certificate is rotated", tlstests.IsReachableOverTLSDuring(&mdb, time.Second*10, + func() { + t.Run("Update certificate secret", tlstests.RotateCertificate(&mdb)) + t.Run("Wait for certificate to be rotated", tlstests.WaitForRotatedCertificate(&mdb)) + }, + )) +} diff --git a/test/e2e/tlstests/tlstests.go b/test/e2e/tlstests/tlstests.go index a3cfda880..2d7328b80 100644 --- a/test/e2e/tlstests/tlstests.go +++ b/test/e2e/tlstests/tlstests.go @@ -6,9 +6,14 @@ import ( "crypto/x509" "fmt" "io/ioutil" + "math/big" "testing" "time" + f "github.com/operator-framework/operator-sdk/pkg/test" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/secret" + v1 "github.com/mongodb/mongodb-kubernetes-operator/pkg/apis/mongodb/v1" e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/mongodbtests" @@ -136,6 +141,59 @@ func getAdminSetting(uri, key string) (interface{}, error) { return value, nil } +func RotateCertificate(mdb *v1.MongoDB) func(*testing.T) { + return func(t *testing.T) { + // Load new certificate and key + cert, err := ioutil.ReadFile("testdata/tls/server_rotated.crt") + assert.NoError(t, err) + key, err := ioutil.ReadFile("testdata/tls/server_rotated.key") + assert.NoError(t, err) + + certKeySecret := secret.Builder(). + SetName(mdb.Spec.Security.TLS.CertificateKeySecret.Name). + SetNamespace(mdb.Namespace). + SetField("tls.crt", string(cert)). + SetField("tls.key", string(key)). + Build() + + err = f.Global.Client.Update(context.TODO(), &certKeySecret) + assert.NoError(t, err) + } +} + +func WaitForRotatedCertificate(mdb *v1.MongoDB) func(*testing.T) { + return func(t *testing.T) { + // The rotated certificate has serial number 2 + expectedSerial := big.NewInt(2) + + tlsConfig, err := getClientTLSConfig() + assert.NoError(t, err) + + // Reject all server certificates that don't have the expected serial number + tlsConfig.VerifyPeerCertificate = func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error { + cert := verifiedChains[0][0] + if expectedSerial.Cmp(cert.SerialNumber) != 0 { + return fmt.Errorf("expected certificate serial number %s, got %s", expectedSerial, cert.SerialNumber) + } + + return nil + } + + opts := options.Client().SetTLSConfig(tlsConfig).ApplyURI(mdb.MongoURI()) + mongoClient, err := mongo.Connect(context.TODO(), opts) + assert.NoError(t, err) + + // Ping the cluster until it succeeds. The ping will only suceed with the right certificate. + err = wait.Poll(5*time.Second, 5*time.Minute, func() (done bool, err error) { + if err := mongoClient.Ping(context.TODO(), nil); err != nil { + return false, nil + } + return true, nil + }) + assert.NoError(t, err) + } +} + func getClientTLSConfig() (*tls.Config, error) { // Read the CA certificate from test data caPEM, err := ioutil.ReadFile("testdata/tls/ca.crt") diff --git a/testdata/tls/server.crt b/testdata/tls/server.crt index 7f16f0f8f..7aaaca225 100644 --- a/testdata/tls/server.crt +++ b/testdata/tls/server.crt @@ -1,17 +1,17 @@ -----BEGIN CERTIFICATE----- -MIICuzCCAaMCCQCZuwR+Czvh4zANBgkqhkiG9w0BAQsFADANMQswCQYDVQQDDAJD -QTAeFw0yMDA3MTAwOTUyNTZaFw0zMDA3MDgwOTUyNTZaMDIxMDAuBgNVBAMMJyou -bWRiLXRscy1zdmMuZGVmYXVsdC5zdmMuY2x1c3Rlci5sb2NhbDCCASIwDQYJKoZI -hvcNAQEBBQADggEPADCCAQoCggEBANzc4+o9BABoRhKi6cikBKGYVHtd3ypGCM27 -V8xjILdCK0hS1kwMWDHrGP4u1S0uxpqdt4UNiblGGPvEWJRRg0pbCrEkRB7+d+BA -iqE5g4/kTISl3CliZkptUDFoIaVoXCXILf3nUVkofMqiGvDZvxIfOr3pnizuTQDV -2E2ifbI1GBRxxPSlaokta8vKrdOl3QZMmzkHdxY+meOX8SH6zA2pnwDQ90GuTVsv -a4j62PjO1eoKHc8GpuITvWFllHIDrniTsW7Ipxzt/gpfdDaA8Qt+Yvfq3mjDF13S -kcciCY93j0LFlPRPepyqcvUhpgzP5b/3zf5czC2ZxqnAWudqz5kCAwEAATANBgkq -hkiG9w0BAQsFAAOCAQEA809PxVEtUyyYCmxPQGQWgt5m5eGQ3dkCrZJ1E4+IKF8w -2i5MqfxjfigOUtWsSFN6v+qNwYC1/ss3qOQ0ojr0vlh/hQ2DpI3hul8tWTGER5dC -RgnzUuVvdsj4vu1lVZ9d6zuq7ED42zoV0fSfveBdKQ/BOwlc3O7izmhmhLSE1dEz -4UWfsRuuU5CaAbKcb9ztM/jClYDkNAN4h+46f9hA1T36WCa7+W+kyoIQah9506wK -L9FCamgzWiHPYkC+2S9ZfHFtwFxPAXNPcd4R4IFId6OR8g81zOYnJCPK/qBfKgQK -i9am5PJxKcsm1dStEBlMnKBLjN/8nWsjEz1mAjFRBQ== +MIICszCCAZsCAQEwDQYJKoZIhvcNAQELBQAwDTELMAkGA1UEAwwCQ0EwHhcNMjAw +NzIyMDgxMDQzWhcNMzAwNzIwMDgxMDQzWjAyMTAwLgYDVQQDDCcqLm1kYi10bHMt +c3ZjLmRlZmF1bHQuc3ZjLmNsdXN0ZXIubG9jYWwwggEiMA0GCSqGSIb3DQEBAQUA +A4IBDwAwggEKAoIBAQDsJftQQU+5IyP7SrJw5lydbNjtdfDH6LP5nd58rUh0KxLJ +iQmsQBoePQpTh2uF9bzKzCuglrqHjTRszUJiziO5/eTMucH26AlsT9bR6md8w42I +mnLQ+UNnt89k+tWsr0lXrV31IHkeOSAmme5mK10T9r0YtMQojOept5s5vt4zsNKs +pdS5cv8eSPQu7+GzwEA0QrC75MIXRyVbuSf0HKZJET1Ic36mL/ucwK2LjEQbUT/Q +Daf/WbzVvPMochvMPxwEiFMIDQyzJJO/WZbUX1jGLn09JJJQWNYbrv0wRJst0H11 +5zVlN1mYgYpVtaCSPOEaZRT0Qe4sjn2aG8NtIgLhAgMBAAEwDQYJKoZIhvcNAQEL +BQADggEBAOdhoqeMzFn5c4m71V9RhJlGGQ75mlmWqZZFBZFEHwZl0eZz8YS/SXDZ +xG1rurkvpM/dRHbJRt4x1rc300dRLaeJ3jbIXnYpEt3vlWmfpJmoReyHvH0e5jGc +AFBn+bRo/Gl59NlKUh3h2dXfqZ7bI63sYUYp4dUAH2b/jhKp+onMkdT5KiglOKIP +1ZwmttTVZo/QIUgQzS2VZSisAHQnt8tHhh0byQJiKK29Lm5ghzsAs2BIvajsCKva +2LkaDUONaVkFn1S3xD7JMJfM5YgYOmANsQYa5CSUzrEz6BT1idJAwekWRDRaEuHj +M0D03eMd66Soh8UvwQnLT3U4vd3aGNk= -----END CERTIFICATE----- diff --git a/testdata/tls/server.key b/testdata/tls/server.key index c8f22c1b9..e653a77b1 100644 --- a/testdata/tls/server.key +++ b/testdata/tls/server.key @@ -1,27 +1,27 @@ -----BEGIN RSA PRIVATE KEY----- -MIIEpAIBAAKCAQEA3Nzj6j0EAGhGEqLpyKQEoZhUe13fKkYIzbtXzGMgt0IrSFLW -TAxYMesY/i7VLS7Gmp23hQ2JuUYY+8RYlFGDSlsKsSREHv534ECKoTmDj+RMhKXc -KWJmSm1QMWghpWhcJcgt/edRWSh8yqIa8Nm/Eh86vemeLO5NANXYTaJ9sjUYFHHE -9KVqiS1ry8qt06XdBkybOQd3Fj6Z45fxIfrMDamfAND3Qa5NWy9riPrY+M7V6god -zwam4hO9YWWUcgOueJOxbsinHO3+Cl90NoDxC35i9+reaMMXXdKRxyIJj3ePQsWU -9E96nKpy9SGmDM/lv/fN/lzMLZnGqcBa52rPmQIDAQABAoIBADKT4x2hG2l8d4+M -/zyTUvwuhio6NQDZSOrUHCcSDEvrOz40uh5wNUI5NGABGZBfRdvigkRV9VQYWWBO -dnqAsDA26Rvn0jHG1GzmWtjQ8u/guCUoVmKjxgkFP3gRAKOAYpnEXcPnVRm1y94b -gU/mjXr469rRxQdFCx1F22Lr8xz73r0RQyispRUX8U5dQDx9y4//4XzpdoM0C26d -fFhdXzEeWgWHrsl9a8JD7RUcjJqtfqyyOi0sIp0CBFO3AQTQOyF5d2H0JDjK6TBp -tAa/FSpq2DqaIW/zJ7sO9eYDxEsry9f73yDodlXTb7D2c2+JzvFnWp3RX23jeOxS -e10/6gECgYEA8RZNBj8LNw+qSNPJ0zaX/kB2RTwHH2tjjlT54f/YuNk0+JFMPwQ5 -79SZu9eXTWlTefG57OeLeJjgDoAM8iiod8XMnhAkRsWrvea9EpKn4kQdJyZXCpZP -QZCr48/STKCYFVhM60ULbBlZSX//avQbsOEr0J2aDNIW4a8sFPm0UOUCgYEA6oZU -xnRgpyneupr2O+FRatGmjKCRNIR3c8SA8Uj+vUKwgLTPswjYcBb3IVjMUU4U2rRX -K3S/JE4JFax6qxdj8rY/SGT+4XweMO91ADMUFBNAcCBpUb1b9cN8Fmj0aBKfxBYB -0iHDYyXPtAbE0RnXtlELGjgp9WiP4ON1kyt5PKUCgYEAiCG9kH9cx9SSpNjiJ9+1 -551iqymAJB+xcZdOGm0rZQVRQeJZmWO7i08TF9xe1RlaR4tVHw6H1KsOKWHo7XxD -1I3eQop+0W+g0HRP4wLoxX0MsSSXFaVWT4Fvbg3Vg+tStQrNNDQihQGzRyt9gFki -obgj04KKkB65SOpvC9EMHnkCgYBnkMytGsV37ICELVxhkx3OKyj+XwXgiELr57H9 -W3vK/kFNhQtXh6d00F9v5XPMz8tbzVnVQegK0z8lyBYfqeOkUiDeCmOYFaSLfaNQ -ZD4Qo3PIUDfSNen/PMV2bADKWLce84z4S5qqFQ+E85xErsHYrC9X88drrTIkYK4S -QJd4VQKBgQCI4roSBDGcCRtKbARqtlJaLs7UYORBbK7535/TZp/gMyU9xWJ3JrTU -tX0o6jOe9TMo1IVFbw0RVThN0KOMRRAzwe21pSLN2d84lRLo2tIqUjGnQwzIhbdG -/BQy8aUsxDjnEnvXci4asSoBAv3OiV8mJBgT4i1KdNQgkhokMXTupA== +MIIEpAIBAAKCAQEA7CX7UEFPuSMj+0qycOZcnWzY7XXwx+iz+Z3efK1IdCsSyYkJ +rEAaHj0KU4drhfW8yswroJa6h400bM1CYs4juf3kzLnB9ugJbE/W0epnfMONiJpy +0PlDZ7fPZPrVrK9JV61d9SB5HjkgJpnuZitdE/a9GLTEKIznqbebOb7eM7DSrKXU +uXL/Hkj0Lu/hs8BANEKwu+TCF0clW7kn9BymSRE9SHN+pi/7nMCti4xEG1E/0A2n +/1m81bzzKHIbzD8cBIhTCA0MsySTv1mW1F9Yxi59PSSSUFjWG679MESbLdB9dec1 +ZTdZmIGKVbWgkjzhGmUU9EHuLI59mhvDbSIC4QIDAQABAoIBAEnqBYR0PODk9+Ey +2zFtWTXJGQkSbmAUHSkXWclKb7A0vzenlgh9M++dCXtlmqkeZo5PY6RrKU0+TFd1 +076baSFRL+lIh0aiEDj/sGyZ4vRxPP6x4Rg5vPhc1yRzQqg/YUR5NjyAgoiMNtz+ +N1lxXzvdcgimo/NRTz2XA5YKgQBKUVLw92wXKwCi6rrEuTuaxr0OdHEdoQzLWv/f +gRVeXLf9eHkhYwh3Wu9Si2Yl4lUKcx7OdFvZr5TzY3y9IgxE+gfT+U4z3zGfwC2y +euml+A/7KYnt2/oA+BBrWbb0W7enOLJ4T5jbsHQgGLk03fe4sALIo4T1z8a2t2HR +wZusUAECgYEA/rkvJxWoIzU2LO4j66NLOAz+N62zKPWdQlYL25LSLz/5IJgAFpNA +r9k1p3GKRYfX7p9tyHS4/hhyo2AR+IXwJePOJvG97/U3lu3EkfY7Lb9Vuu3n6VK2 +gPgax30n5LkYJjTxIPd4sYSuIfeDew7xs1hB1x6sCNhP6qIMGI82m2ECgYEA7VT3 +IRcvxS/fbnZt/NMEvvkwdh2Un8DG9H29XEsiLcYv58q/aDNaR/LyvDZFbgW+FE9R +X2TpwnAgosW8Ctiyf08qM9jxdU402WhbHgAptncWilxsWNJXkJA5RGJ6imbbDjZ6 +YNdImvnSxenWp37+Dm2Np16JJQhJIYHzJUOoF4ECgYEAhXBFf1gdCLSreMYYEy0s +DmTgGBLqtB5XD5U8CP7VFOOSgryd7zWcwYIsVVdpdBtBx0PFoylib9om0+dUArlH +oNHCASzKr5XqVSqhu0Ueo0yEgLR4tQYbjVxryu2JpIxCVmGNoBOEKpqzDiA3xJOD +ksw9UZBD1y5aTzQs2gDMPoECgYA/oW8ctR2+rYYnFKOKjH0SQrdGg8nMRyBQfsHd +U9uXEDLZ35cP9ey3q6B+68ITrIB464cyn1i5I9zsJz2yXsUEsxHqkriyLcSnoX4E +fiCw5h7p+7uk2MhXXwOrnQejwc3rcpm/CxlRS7fCDl7Zy4eMEL1Q6Vy1zBHnZPLu +w8P2gQKBgQCtBUlZzsy2MNK78gWtflLZZl01FZDjMd8ucyezvTkEx7WWvi1p8l0v +mpUGF43d1zcKAbVBPY0gbaD1aPYK4aMyRkS0lN6k44tW3qKjmIhA7SGdqHSX66Tr +8/sqbf/7LPoWdaB/l1T4jBHLNur6Iy+lLWx8psbTqNhocaTkNI6M6Q== -----END RSA PRIVATE KEY----- diff --git a/testdata/tls/server_rotated.crt b/testdata/tls/server_rotated.crt new file mode 100644 index 000000000..d6079114e --- /dev/null +++ b/testdata/tls/server_rotated.crt @@ -0,0 +1,17 @@ +-----BEGIN CERTIFICATE----- +MIICszCCAZsCAQIwDQYJKoZIhvcNAQELBQAwDTELMAkGA1UEAwwCQ0EwHhcNMjAw +NzIyMTYwOTE4WhcNMzAwNzIwMTYwOTE4WjAyMTAwLgYDVQQDDCcqLm1kYi10bHMt +c3ZjLmRlZmF1bHQuc3ZjLmNsdXN0ZXIubG9jYWwwggEiMA0GCSqGSIb3DQEBAQUA +A4IBDwAwggEKAoIBAQDS9+ePhIjG9lDMo/UYov1MYo/bVUZ5dourMwUKAFnhjZYH +Z0HWqboaD+THsqfA6eoRcPadavucV+J1fg+ZUo4srbcGEQTNjBfogusLYwVdPm+/ +C0kyEc6nU10AVvhNrB4egALBSD6/nEyzd5Ua4bscQT1Hj3q8834cCJRcEr7MxLyY +Pr8vnBzILyOf7JOoJ148JwJjs1WuSh0fVmVSNLJ76GLbd3X0s1zMPPkl2bv+IZqb +Far3oLvCEyyKxeum0PmzTGzBAefyQs2dPwaGgws3Tfjxvlb8vZyCa4N/pW87QhC6 +AbSsheGk05WxVyIb8EWQ2LpBJnmFyLzDdlRNcra1AgMBAAEwDQYJKoZIhvcNAQEL +BQADggEBAL3oBfO7malbfuLvApwnzFFwWHHqe0GtUGsy6GK2v0wlA0hVmMvkB0RK +jfUQyEI911bF/ll3V32l52fkBM9p6OEEuJvS09FnqQVFWDSkZtLiyog5L6IUo5I8 +4Yq/MHKnw/ziDLjrfRfkEFeLTjDthONVebERYOWEFPpRQrW6H9U2ToNvc91/wZ7H +XhtgdukSYDCG39X+SnTgXCCJXuPcaqc1na6FzigNGmJ4IrI2VGU6MRnLt6m969VN +3Wkc/Gx+qyWobpmI8hfTARlP8jPjG3DBIjNHPeF7R2hy0ppSrn+4LVdjf/DMD7EH +9OH2mOsVXeSLUu9TzjvavZwrB6QMKyU= +-----END CERTIFICATE----- diff --git a/testdata/tls/server_rotated.key b/testdata/tls/server_rotated.key new file mode 100644 index 000000000..ca6db81d7 --- /dev/null +++ b/testdata/tls/server_rotated.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEA0vfnj4SIxvZQzKP1GKL9TGKP21VGeXaLqzMFCgBZ4Y2WB2dB +1qm6Gg/kx7KnwOnqEXD2nWr7nFfidX4PmVKOLK23BhEEzYwX6ILrC2MFXT5vvwtJ +MhHOp1NdAFb4TaweHoACwUg+v5xMs3eVGuG7HEE9R496vPN+HAiUXBK+zMS8mD6/ +L5wcyC8jn+yTqCdePCcCY7NVrkodH1ZlUjSye+hi23d19LNczDz5Jdm7/iGamxWq +96C7whMsisXrptD5s0xswQHn8kLNnT8GhoMLN0348b5W/L2cgmuDf6VvO0IQugG0 +rIXhpNOVsVciG/BFkNi6QSZ5hci8w3ZUTXK2tQIDAQABAoIBAFOEAA84kEzzRZk9 +btGJ9GxAuGJDp9p9q1zinaQP74Ty3+meXtnz5tBaal8DwpUEBL1S0s4Og/yobeXF +ObegjtfxDPtB0XztcKmC6jlfwcff56zhcuB3XC+xOKfhxfo5oNLCKlUJnI2N+m8x +b71sjv3odbHWgug6HDpMyy6H3TCCGSu7UoWtjA8hmRW/xMOactdAxck6NdfYWvHA +REXs+SQEPkk+T73It8QDHGVqKAXddmF2CDhQOe0BMLkuwqAzeekO3eAulE/WKQMj +y12I79tTTb2Ms5tl89drOeYzs1ux8/lxWtCtN3P12Gis8wa4rswbY4jLPZwgIMRG +U5tcQiECgYEA91KjFC2WVeg3Y2FJv31eE+Z52dIsKbq2/hkl2WeVbL+Inc8zZBds +7aOFO2U/REWJr45EZ3jNkSu5AzAjYXKIjNIk0+SVvDa851+2wAZbQXdxS5+ruiRu +zsDlw48OT3ynMEIIuwiEsPKfq2FWae+UliWxwLuKw2/K5KuDn5/TXqcCgYEA2l6+ +0C2s38YFFu0pjaQfqk7aGqfJ9qWt7tm3FBrDRyWgPYi1LhWN5f6xlHZBw+WxS3IN +joEH4CzHmADi1nhEsb+tQHVc5CZ2GmgIfMw13XE1FBdRW6PgdEPIC7UYBDNh9wMC +zPDkI6jV7zipsRyIoEs9PTXkRa7C6DuPAbTOp0MCgYEAwFW9iPWi0hAS8vA3v/ko +7mTwIdr2iUUxBg5chuOtKrMQ9ViraI1nIq9l7zjfqKJDXwlOXQFvLBRKfxYyjZfa +ZVkPVtGPOJ2A7pZasp6+3PycWOlFTS8EFTmh9SENSfdwtXDFBV6sgkdMsKSz5RJy +BQovX+j5Et+fc5GGfN54LEUCgYBV4RkN6kiooMnzoEXNTJSfd+9SuFY3SCVFYB4e +LABMhMGmMZN/kj6CC05vYqqujjDRyQMH3jrosPO2FfMgAaCSfx110jI8D9w2ul9M +JUux0Qnc4ua+MY7eaqHL6OaPEF4gtPBvBPXUCFxKfnBOFTiuQajN39nshbRlfLbb +Ju523QKBgQCO1uWOdZFrMKejK8lS+OLJBZ0IQl4SKATIce4xppLGyV/p837uAhet +n0LuuzjwC+8DG8ZPeC+bhAgO0Sow7NHAfWnxL3kVExVFm3qVX0uiw3+rRMu1bRuU +OUIoHZf7nUmYEXqCAwMecQDZpGtPyMKGuOx8EFlYQGo7I0JNF1cl3w== +-----END RSA PRIVATE KEY----- From 5403f87914e392c7dd632d99d5911347b7ab00c6 Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Fri, 31 Jul 2020 13:56:59 +0100 Subject: [PATCH 32/34] evergreen retrigger From cf8e93b55226ca129ec3e1c95ae3032be49ea8e5 Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Sat, 1 Aug 2020 16:22:03 +0100 Subject: [PATCH 33/34] evergreen retrigger From 118527f8b8d64dce6da6f59019901037f4371246 Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Sat, 1 Aug 2020 19:24:42 +0100 Subject: [PATCH 34/34] fixed evergreen duplicated entry --- .evergreen.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.evergreen.yml b/.evergreen.yml index b4624765c..2d560f21f 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -154,7 +154,6 @@ task_groups: - e2e_test_replica_set_tls_rotate - e2e_test_statefulset_arbitrary_config - e2e_test_statefulset_arbitrary_config_update - - e2e_test_replica_set_tls_rotate teardown_task: - func: upload_e2e_logs