From 30266fbe8b272404e1edfda9856ac340bdfaf7cc Mon Sep 17 00:00:00 2001 From: Rodrigo Valin Date: Tue, 12 May 2020 21:56:11 +0100 Subject: [PATCH 001/790] CLOUDP-59610: Pre-stop hook --- .evergreen.yml | 9 + agent/Dockerfile | 4 +- cmd/prestop/main.go | 226 ++++++++++++++++++ cmd/testrunner/main.go | 2 + deploy/operator.yaml | 2 + pkg/agenthealth/agenthealth.go | 40 ++++ pkg/apis/mongodb/v1/mongodb_types.go | 2 +- pkg/controller/mongodb/mongodb_controller.go | 125 ++++++++-- .../mongodb/replicaset_controller_test.go | 2 +- pkg/kube/statefulset/statefulset.go | 7 +- requirements.txt | 3 + scripts/dev/build_and_deploy_operator.py | 13 +- scripts/dev/dev_config.py | 9 +- scripts/dev/dockerutil.py | 22 +- scripts/dev/e2e.py | 5 +- scripts/dev/requirements.txt | 3 - scripts/dev/templates/Dockerfile.prehook | 18 ++ test/e2e/e2eutil.go | 16 ++ test/e2e/mongodbtests/mongodbtests.go | 37 +++ .../replica_set_test.go | 59 +++++ 20 files changed, 563 insertions(+), 41 deletions(-) create mode 100644 cmd/prestop/main.go create mode 100644 pkg/agenthealth/agenthealth.go create mode 100644 requirements.txt delete mode 100644 scripts/dev/requirements.txt create mode 100644 scripts/dev/templates/Dockerfile.prehook create mode 100644 test/e2e/replica_set_change_version/replica_set_test.go diff --git a/.evergreen.yml b/.evergreen.yml index 2ee7c863d..26366997f 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -150,6 +150,14 @@ tasks: vars: test: replica_set_scale + - name: e2e_test_replica_set_change_version + commands: + - func: clone + - func: setup_kubernetes_environment + - func: run_e2e_test + vars: + test: replica_set_change_version + buildvariants: - name: go_unit_tests display_name: go_unit_tests @@ -173,6 +181,7 @@ buildvariants: - name: e2e_test_replica_set - name: e2e_test_replica_set_readiness_probe - name: e2e_test_replica_set_scale + - name: e2e_test_replica_set_change_version - name: init_test_run display_name: init_test_run diff --git a/agent/Dockerfile b/agent/Dockerfile index 91ce29d42..0aaa78924 100644 --- a/agent/Dockerfile +++ b/agent/Dockerfile @@ -26,9 +26,9 @@ RUN curl -LO http://downloads.mongodb.org/linux/mongodb-linux-x86_64-ubuntu1604- mv mongodb-linux-x86_64-ubuntu1604-4.0.6/bin/mongo /usr/bin && \ rm -rf mongodb-linux-x86_64-ubuntu1604-4.0.6.tgz mongodb-linux-x86_64-ubuntu1604-4.0.6 - RUN mkdir -p /var/lib/mongodb-mms-automation/probes/ \ - && curl --retry 3 https://readinessprobe.s3-us-west-1.amazonaws.com/readinessprobe -o /var/lib/mongodb-mms-automation/probes/readinessprobe \ +# && curl --retry 3 https://readinessprobe.s3-us-west-1.amazonaws.com/readinessprobe -o /var/lib/mongodb-mms-automation/probes/readinessprobe \ + && curl --retry 3 https://readinessprobe-test.s3-us-west-1.amazonaws.com/readiness -o /var/lib/mongodb-mms-automation/probes/readinessprobe \ && chmod +x /var/lib/mongodb-mms-automation/probes/readinessprobe \ && mkdir -p /var/log/mongodb-mms-automation/ \ && chmod -R +wr /var/log/mongodb-mms-automation/ diff --git a/cmd/prestop/main.go b/cmd/prestop/main.go new file mode 100644 index 000000000..95aad9236 --- /dev/null +++ b/cmd/prestop/main.go @@ -0,0 +1,226 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "os" + "strings" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/agenthealth" + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var logger *zap.SugaredLogger + +const ( + agentStatusFilePathEnv = "AGENT_STATUS_FILEPATH" + logFilePathEnv = "PRE_STOP_HOOK_LOG_PATH" + defaultNamespace = "default" +) + +func getNamespace() (string, error) { + data, err := ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace") + if err != nil { + return "", err + } + if ns := strings.TrimSpace(string(data)); len(ns) > 0 { + return ns, nil + } + return defaultNamespace, nil +} + +// deletePod attempts to delete the pod this mongod is running in +func deletePod() error { + thisPod, err := getThisPod() + if err != nil { + return fmt.Errorf("error getting this pod: %s", err) + } + k8sClient, err := inClusterClient() + if err != nil { + return fmt.Errorf("error getting client: %s", err) + } + + if err := k8sClient.Delete(context.TODO(), &thisPod); err != nil { + return fmt.Errorf("error deleting pod: %s", err) + } + return nil +} + +func inClusterClient() (client.Client, error) { + config, err := rest.InClusterConfig() + if err != nil { + return nil, fmt.Errorf("error getting cluster config: %+v", err) + } + + k8sClient, err := client.New(config, client.Options{}) + if err != nil { + return nil, fmt.Errorf("error creating client: %+v", err) + } + return k8sClient, nil +} + +func prettyPrint(i interface{}) { + b, err := json.MarshalIndent(i, "", " ") + if err != nil { + fmt.Println("error:", err) + } + fmt.Println(string(b)) +} + +// shouldDeletePod returns a boolean value indicating if this pod should be deleted +// this would be the case if the agent is currently trying to upgrade the version +// of mongodb. +func shouldDeletePod(health agenthealth.Health) (bool, error) { + hostname := os.Getenv("HOSTNAME") + status, ok := health.ProcessPlans[hostname] + if !ok { + return false, fmt.Errorf("hostname %s was not in the process plans", hostname) + } + return isWaitingToBeDeleted(status), nil +} + +// getAgentHealthStatus returns an instance of agenthealth.Health read +// from the health file on disk +func getAgentHealthStatus() (agenthealth.Health, error) { + f, err := os.Open(os.Getenv(agentStatusFilePathEnv)) + if err != nil { + return agenthealth.Health{}, fmt.Errorf("error opening file: %s", err) + } + defer f.Close() + + h, err := readAgentHealthStatus(f) + if err != nil { + return agenthealth.Health{}, fmt.Errorf("error reading health status: %s", err) + } + return h, err + +} + +// getThisPod returns an instance of corev1.Pod that points to the current pod +func getThisPod() (corev1.Pod, error) { + podName := os.Getenv("HOSTNAME") + if podName == "" { + return corev1.Pod{}, fmt.Errorf("environment variable HOSTNAME was not present") + } + + ns, err := getNamespace() + if err != nil { + return corev1.Pod{}, fmt.Errorf("error reading namespace: %+v", err) + } + + return corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + Namespace: ns, + }, + }, nil +} + +// readAgentHealthStatus reads an instance of health.Health from the provided +// io.Reader +func readAgentHealthStatus(reader io.Reader) (agenthealth.Health, error) { + var h agenthealth.Health + data, err := ioutil.ReadAll(reader) + if err != nil { + return h, err + } + err = json.Unmarshal(data, &h) + return h, err +} + +func ensureEnvironmentVariables(requiredEnvVars ...string) error { + var missingEnvVars []string + for _, envVar := range requiredEnvVars { + if val := os.Getenv(envVar); val == "" { + missingEnvVars = append(missingEnvVars, envVar) + } + } + if len(missingEnvVars) > 0 { + return fmt.Errorf("missing envars: %s", strings.Join(missingEnvVars, ",")) + } + return nil +} + +func main() { + fmt.Println("Calling pre-stop hook!") + cfg := zap.NewDevelopmentConfig() + if err := ensureEnvironmentVariables(logFilePathEnv, agentStatusFilePathEnv); err != nil { + zap.S().Fatal("Not all required environment variables are present: %s", err) + os.Exit(1) + } + cfg.OutputPaths = []string{ + os.Getenv(logFilePathEnv), + } + log, err := cfg.Build() + if err != nil { + zap.S().Errorf("Error building logger config: %s", err) + os.Exit(1) + } + logger = log.Sugar() + health, err := getAgentHealthStatus() + if err != nil { + logger.Errorf("Error getting the agent health file: %s", err) + } + + shouldDelete, err := shouldDeletePod(health) + logger.Debugf("shouldDeletePod=%t", shouldDelete) + if err != nil { + logger.Errorf("Error in shouldDeletePod: %s", err) + } + + if shouldDelete { + if err := deletePod(); err != nil { + // We should not raise an error if the Pod could not be deleted. It can have even + // worst consequences: Pod being restarted with the same version, and the agent + // killing it immediately after. + logger.Errorf("Could not manually trigger restart of this Pod because of: %s", err) + logger.Errorf("Make sure the Pod is restarted in order for the upgrade process to continue") + } + + // If the Pod needs to be killed, we'll wait until the Pod + // is killed by Kubernetes, bringing the new container image + // into play. + var quit = make(chan struct{}) + logger.Info("A Pod killed itself, waiting...") + <-quit + } +} + +// isWaitingToBeDeleted determines if the agent is currently waiting +// on the mongod pod to be restarted. In order to do this, we need to check the agent +// status file and determine if the mongod has been stopped and if we are in the process +// of a version change. +func isWaitingToBeDeleted(healthStatus agenthealth.MmsDirectorStatus) bool { + if len(healthStatus.Plans) == 0 { + return false + } + lastPlan := healthStatus.Plans[len(healthStatus.Plans)-1] + for _, m := range lastPlan.Moves { + + // The next conditions are based on observations on the outcome + // of the agent after they have stopped the mongo server. + + switch m.Move { + case "WaitFeatureCompatibilityVersionCorrect": + // First condition observed. This is the Plan reported by the + // agent on the first MongoD stopped. + for _, s := range m.Steps { + if s.Step == "WaitFeatureCompatibilityVersionCorrect" && + s.Result == "" { + return true + } + } + case "ChangeVersion": + // This is the condition observed in the 2nd and 3rd Pods. + return true + } + } + return false +} diff --git a/cmd/testrunner/main.go b/cmd/testrunner/main.go index 3425d8a35..242e53221 100644 --- a/cmd/testrunner/main.go +++ b/cmd/testrunner/main.go @@ -227,6 +227,8 @@ func withTest(test string) func(obj runtime.Object) { "--verbose", "--kubeconfig", "/etc/config/kubeconfig", + "--go-test-flags", + "-timeout=20m", } } } diff --git a/deploy/operator.yaml b/deploy/operator.yaml index a4572f685..5c2a1001a 100644 --- a/deploy/operator.yaml +++ b/deploy/operator.yaml @@ -32,3 +32,5 @@ spec: value: "mongodb-kubernetes-operator" - name: AGENT_IMAGE # The MongoDB Agent the operator will deploy to manage MongoDB deployments value: quay.io/chatton/mongodb-agent + - name: PRE_STOP_HOOK_IMAGE + value: quay.io/mongodb/community-operator-pre-stop-hook diff --git a/pkg/agenthealth/agenthealth.go b/pkg/agenthealth/agenthealth.go new file mode 100644 index 000000000..56d96f44a --- /dev/null +++ b/pkg/agenthealth/agenthealth.go @@ -0,0 +1,40 @@ +package agenthealth + +import ( + "time" +) + +type Health struct { + Healthiness map[string]ProcessHealth `json:"statuses"` + ProcessPlans map[string]MmsDirectorStatus `json:"mmsStatus"` +} + +type ProcessHealth struct { + IsInGoalState bool `json:"IsInGoalState"` + LastMongoUpTime int64 `json:"LastMongoUpTime"` + ExpectedToBeUp bool `json:"ExpectedToBeUp"` +} + +type MmsDirectorStatus struct { + Name string `json:"name"` + LastGoalStateClusterConfigVersion int64 `json:"lastGoalVersionAchieved"` + Plans []*PlanStatus `json:"plans"` +} + +type PlanStatus struct { + Moves []*MoveStatus `json:"moves"` + Started *time.Time `json:"started"` + Completed *time.Time `json:"completed"` +} + +type MoveStatus struct { + Move string `json:"move"` + Steps []*StepStatus `json:"steps"` +} + +type StepStatus struct { + Step string `json:"step"` + Started *time.Time `json:"started"` + Completed *time.Time `json:"completed"` + Result string `json:"result"` +} diff --git a/pkg/apis/mongodb/v1/mongodb_types.go b/pkg/apis/mongodb/v1/mongodb_types.go index f0db540a7..b5d6e1676 100644 --- a/pkg/apis/mongodb/v1/mongodb_types.go +++ b/pkg/apis/mongodb/v1/mongodb_types.go @@ -65,7 +65,7 @@ func (m *MongoDB) UpdateSuccess() { m.Status.Phase = Running } -func (m MongoDB) ChangingVersion() bool { +func (m MongoDB) IsChangingVersion() bool { if lastVersion, ok := m.Annotations[LastVersionAnnotationKey]; ok { return (m.Spec.Version != lastVersion) && lastVersion != "" } diff --git a/pkg/controller/mongodb/mongodb_controller.go b/pkg/controller/mongodb/mongodb_controller.go index 618f162f7..3c44eabec 100644 --- a/pkg/controller/mongodb/mongodb_controller.go +++ b/pkg/controller/mongodb/mongodb_controller.go @@ -32,16 +32,28 @@ import ( ) const ( - AutomationConfigKey = "automation-config" - agentName = "mongodb-agent" - mongodbName = "mongod" - agentImageEnvVariable = "AGENT_IMAGE" - versionManifestFilePath = "/usr/local/version_manifest.json" - readinessProbePath = "/var/lib/mongodb-mms-automation/probes/readinessprobe" - agentHealthStatusFilePath = "/var/log/mongodb-mms-automation/agent-health-status.json" - clusterFilePath = "/var/lib/automation/config/automation-config" + agentImageEnv = "AGENT_IMAGE" + preStopHookImageEnv = "PRE_STOP_HOOK_IMAGE" + agentHealthStatusFilePathEnv = "AGENT_STATUS_FILEPATH" + preStopHookLogFilePathEnv = "PRE_STOP_HOOK_LOG_PATH" + + AutomationConfigKey = "automation-config" + agentName = "mongodb-agent" + mongodbName = "mongod" + versionManifestFilePath = "/usr/local/version_manifest.json" + readinessProbePath = "/var/lib/mongodb-mms-automation/probes/readinessprobe" + clusterFilePath = "/var/lib/automation/config/automation-config" + operatorServiceAccountName = "mongodb-kubernetes-operator" + agentHealthStatusFilePathValue = "/var/log/mongodb-mms-automation/healthstatus/agent-health-status.json" ) +func RequiredOperatorEnvVars() []string { + return []string{ + agentImageEnv, + preStopHookImageEnv, + } +} + // Add creates a new MongoDB Controller and adds it to the Manager. The Manager will set fields on the Controller // and Start it when the Manager is Started. func Add(mgr manager.Manager) error { @@ -123,17 +135,20 @@ func (r *ReplicaSetReconciler) Reconcile(request reconcile.Request) (reconcile.R return reconcile.Result{}, err } + r.log.Debug("Building service") svc := buildService(mdb) if err = r.client.CreateOrUpdate(&svc); err != nil { r.log.Infof("The service already exists... moving forward: %s", err) } + r.log.Debug("Creating/Updating StatefulSet") if err := r.createOrUpdateStatefulSet(mdb); err != nil { r.log.Infof("Error creating/updating StatefulSet: %+v", err) return reconcile.Result{}, err } - if ready, err := r.isStatefulSetReady(mdb); err != nil { + r.log.Debugf("Ensuring StatefulSet is ready, with type: %s", getUpdateStrategyType(mdb)) + if ready, err := r.isStatefulSetReady(mdb, getUpdateStrategyType(mdb)); err != nil { r.log.Infof("error checking StatefulSet status: %+v", err) return reconcile.Result{}, err } else if !ready { @@ -141,29 +156,32 @@ func (r *ReplicaSetReconciler) Reconcile(request reconcile.Request) (reconcile.R return reconcile.Result{RequeueAfter: time.Second * 10}, nil } + r.log.Debug("Resetting StatefulSet UpdateStrategy") if err := r.resetStatefulSetUpdateStrategy(mdb); err != nil { r.log.Infof("error resetting StatefulSet UpdateStrategyType: %+v", err) return reconcile.Result{}, err } + r.log.Debug("Setting MongoDB Annotations") if err := r.setAnnotation(types.NamespacedName{Name: mdb.Name, Namespace: mdb.Namespace}, mdbv1.LastVersionAnnotationKey, mdb.Spec.Version); err != nil { r.log.Infof("Error setting annotation: %+v", err) return reconcile.Result{}, err } + r.log.Debug("Updating MongoDB Status") if err := r.updateStatusSuccess(&mdb); err != nil { r.log.Infof("Error updating the status of the MongoDB resource: %+v", err) return reconcile.Result{}, err } - r.log.Info("Successfully finished reconciliation", "MongoDB.Spec:", mdb.Spec, "MongoDB.Status", mdb.Status) + r.log.Infow("Successfully finished reconciliation", "MongoDB.Spec:", mdb.Spec, "MongoDB.Status", mdb.Status) return reconcile.Result{}, nil } // resetStatefulSetUpdateStrategy ensures the stateful set is configured back to using RollingUpdateStatefulSetStrategyType // and does not keep using OnDelete func (r *ReplicaSetReconciler) resetStatefulSetUpdateStrategy(mdb mdbv1.MongoDB) error { - if !mdb.ChangingVersion() { + if !mdb.IsChangingVersion() { return nil } // if we changed the version, we need to reset the UpdatePolicy back to OnUpdate @@ -175,12 +193,14 @@ func (r *ReplicaSetReconciler) resetStatefulSetUpdateStrategy(mdb mdbv1.MongoDB) // isStatefulSetReady checks to see if the stateful set corresponding to the given MongoDB resource // is currently in the ready state -func (r *ReplicaSetReconciler) isStatefulSetReady(mdb mdbv1.MongoDB) (bool, error) { +func (r *ReplicaSetReconciler) isStatefulSetReady(mdb mdbv1.MongoDB, expectedUpdateStrategy appsv1.StatefulSetUpdateStrategyType) (bool, error) { + // TODO: This sleep will be addressed as part of https://jira.mongodb.org/browse/CLOUDP-62444 + time.Sleep(time.Second * 5) set := appsv1.StatefulSet{} if err := r.client.Get(context.TODO(), types.NamespacedName{Name: mdb.Name, Namespace: mdb.Namespace}, &set); err != nil { return false, fmt.Errorf("error getting StatefulSet: %s", err) } - return statefulset.IsReady(set), nil + return statefulset.IsReady(set, mdb.Spec.Members) && expectedUpdateStrategy == set.Spec.UpdateStrategy.Type, nil } func (r *ReplicaSetReconciler) createOrUpdateStatefulSet(mdb mdbv1.MongoDB) error { @@ -311,34 +331,78 @@ func buildContainers(mdb mdbv1.MongoDB) []corev1.Container { "-cluster=" + clusterFilePath, "-skipMongoStart", "-noDaemonize", - "-healthCheckFilePath=" + agentHealthStatusFilePath, + "-healthCheckFilePath=" + agentHealthStatusFilePathValue, "-serveStatusPort=5000", } readinessProbe := defaultReadinessProbe() agentContainer := corev1.Container{ Name: agentName, - Image: os.Getenv(agentImageEnvVariable), + Image: os.Getenv(agentImageEnv), ImagePullPolicy: corev1.PullAlways, Resources: resourcerequirements.Defaults(), Command: agentCommand, ReadinessProbe: &readinessProbe, + + // EnvVar to configure the health file path in the readiness probe + Env: []corev1.EnvVar{ + { + Name: agentHealthStatusFilePathEnv, + Value: agentHealthStatusFilePathValue, + }, + }, } mongoDbCommand := []string{ "/bin/sh", "-c", - `while [ ! -f /data/automation-mongod.conf ]; do sleep 3 ; done ; sleep 2; mongod -f /data/automation-mongod.conf`, + // we execute the pre-stop hook once the mongod has been gracefully shut down by the agent. + `while [ ! -f /data/automation-mongod.conf ]; do sleep 3 ; done ; sleep 2 ; +# start mongod with this configuration +mongod -f /data/automation-mongod.conf ; + +# start the pre-stop-hook to restart the Pod when needed +# If the Pod does not require to be restarted, the pre-stop-hook will +# exit(0) for Kubernetes to restart the container. +/hooks/pre-stop-hook ; +`, } + mongodbContainer := corev1.Container{ Name: mongodbName, Image: fmt.Sprintf("mongo:%s", mdb.Spec.Version), Command: mongoDbCommand, Resources: resourcerequirements.Defaults(), + // the mongod container needs access to the agent health status file + // for the pre-stop hook + Env: []corev1.EnvVar{ + { + Name: agentHealthStatusFilePathEnv, + Value: "/healthstatus/agent-health-status.json", + }, + { + Name: preStopHookLogFilePathEnv, + Value: "/hooks/pre-stop-hook.log", + }, + }, } return []corev1.Container{agentContainer, mongodbContainer} } +func buildInitContainers(volumeMount corev1.VolumeMount) []corev1.Container { + return []corev1.Container{ + { + Name: "mongod-prehook", + Image: os.Getenv("PRE_STOP_HOOK_IMAGE"), + Command: []string{ + "cp", "pre-stop-hook", "/hooks/pre-stop-hook", + }, + VolumeMounts: []corev1.VolumeMount{volumeMount}, + ImagePullPolicy: corev1.PullAlways, + }, + } +} + func defaultReadinessProbe() corev1.Probe { return corev1.Probe{ Handler: corev1.Handler{ @@ -355,7 +419,7 @@ func defaultReadinessProbe() corev1.Probe { // getUpdateStrategyType returns the type of RollingUpgradeStrategy that the StatefulSet // should be configured with func getUpdateStrategyType(mdb mdbv1.MongoDB) appsv1.StatefulSetUpdateStrategyType { - if !mdb.ChangingVersion() { + if !mdb.IsChangingVersion() { return appsv1.RollingUpdateStatefulSetStrategyType } return appsv1.OnDeleteStatefulSetStrategyType @@ -368,12 +432,18 @@ func buildStatefulSet(mdb mdbv1.MongoDB) (appsv1.StatefulSet, error) { "app": mdb.ServiceName(), } + // Configure an empty volume on the mongod container into which the initContainer will copy over the pre-stop hook + hooksVolume := statefulset.CreateVolumeFromEmptyDir("hooks") + hooksVolumeMount := statefulset.CreateVolumeMount(hooksVolume.Name, "/hooks", statefulset.WithReadOnly(false)) + podSpecTemplate := corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: labels, }, Spec: corev1.PodSpec{ - Containers: buildContainers(mdb), + ServiceAccountName: operatorServiceAccountName, + Containers: buildContainers(mdb), + InitContainers: buildInitContainers(hooksVolumeMount), }, } @@ -399,12 +469,25 @@ func buildStatefulSet(mdb mdbv1.MongoDB) (appsv1.StatefulSet, error) { AddVolumeMount(mongodbName, dataVolume). AddVolumeMount(agentName, dataVolume). AddVolumeClaimTemplates(dataVolumeClaim) + // the automation config is only mounted, as read only, on the agent container - automationConfigVolume := statefulset.CreateVolumeFromConfigMap("automation-config", "example-mongodb-config") - automationConfigVolumeMount := statefulset.CreateVolumeMount("automation-config", "/var/lib/automation/config", statefulset.WithReadOnly(true)) + automationConfigVolume := statefulset.CreateVolumeFromConfigMap("automation-config", mdb.ConfigMapName()) + automationConfigVolumeMount := statefulset.CreateVolumeMount(automationConfigVolume.Name, "/var/lib/automation/config", statefulset.WithReadOnly(true)) builder. AddVolume(automationConfigVolume). - AddVolumeMount(agentName, automationConfigVolumeMount) + AddVolumeMount(agentName, automationConfigVolumeMount). + AddVolumeMount(mongodbName, automationConfigVolumeMount) + + // share the agent-health-status.json file in both containers + // mongod container needs access to the health status to check to see if an upgrade + // is being performed and should call the pre-stop hook. + healthStatusVolume := statefulset.CreateVolumeFromEmptyDir("healthstatus") + builder.AddVolume(healthStatusVolume). + AddVolumeMount(agentName, statefulset.CreateVolumeMount(healthStatusVolume.Name, "/var/log/mongodb-mms-automation/healthstatus")). + AddVolumeMount(mongodbName, statefulset.CreateVolumeMount(healthStatusVolume.Name, "/healthstatus")) + + builder.AddVolume(hooksVolume). + AddVolumeMount(mongodbName, hooksVolumeMount) return builder.Build() } diff --git a/pkg/controller/mongodb/replicaset_controller_test.go b/pkg/controller/mongodb/replicaset_controller_test.go index 8948f1e29..363529b37 100644 --- a/pkg/controller/mongodb/replicaset_controller_test.go +++ b/pkg/controller/mongodb/replicaset_controller_test.go @@ -93,7 +93,7 @@ func TestStatefulSet_IsCorrectlyConfigured(t *testing.T) { agentContainer := sts.Spec.Template.Spec.Containers[0] assert.Equal(t, agentName, agentContainer.Name) - assert.Equal(t, os.Getenv(agentImageEnvVariable), agentContainer.Image) + assert.Equal(t, os.Getenv(agentImageEnv), agentContainer.Image) expectedProbe := defaultReadinessProbe() assert.True(t, reflect.DeepEqual(&expectedProbe, agentContainer.ReadinessProbe)) diff --git a/pkg/kube/statefulset/statefulset.go b/pkg/kube/statefulset/statefulset.go index 3455f4eb6..e8518d34f 100644 --- a/pkg/kube/statefulset/statefulset.go +++ b/pkg/kube/statefulset/statefulset.go @@ -75,10 +75,9 @@ func WithReadOnly(readonly bool) func(*corev1.VolumeMount) { } } -func IsReady(sts appsv1.StatefulSet) bool { - replicas := *sts.Spec.Replicas - allUpdated := replicas == sts.Status.UpdatedReplicas - allReady := replicas == sts.Status.ReadyReplicas +func IsReady(sts appsv1.StatefulSet, expectedReplicas int) bool { + allUpdated := int32(expectedReplicas) == sts.Status.UpdatedReplicas + allReady := int32(expectedReplicas) == sts.Status.ReadyReplicas return allUpdated && allReady } diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..d899554ae --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +docker==4.2.0 +kubernetes==11.0.0 +jinja2==2.11.2 diff --git a/scripts/dev/build_and_deploy_operator.py b/scripts/dev/build_and_deploy_operator.py index afeed5f7b..3517b3662 100644 --- a/scripts/dev/build_and_deploy_operator.py +++ b/scripts/dev/build_and_deploy_operator.py @@ -24,8 +24,10 @@ def _load_operator_role_binding() -> Optional[Dict]: return load_yaml_from_file("deploy/role_binding.yaml") -def _load_operator_deployment() -> Optional[Dict]: - return load_yaml_from_file("deploy/operator.yaml") +def _load_operator_deployment(operator_image: str) -> Optional[Dict]: + operator = load_yaml_from_file("deploy/operator.yaml") + operator["spec"]["template"]["spec"]["containers"][0]["image"] = operator_image + return operator def _load_mongodb_crd() -> Optional[Dict]: @@ -46,7 +48,7 @@ def _ensure_crds(): crd = _load_mongodb_crd() ignore_if_doesnt_exist( - lambda: crdv1.delete_custom_resource_definition("mongodbs.mongodb.com") + lambda: crdv1.delete_custom_resource_definition("mongodb.mongodb.com") ) # TODO: fix this, when calling create_custom_resource_definition, we get the error @@ -122,7 +124,10 @@ def deploy_operator(): ) ignore_if_already_exists( lambda: appsv1.create_namespaced_deployment( - dev_config.namespace, _load_operator_deployment() + dev_config.namespace, + _load_operator_deployment( + f"{dev_config.repo_url}/mongodb-kubernetes-operator" + ), ) ) diff --git a/scripts/dev/dev_config.py b/scripts/dev/dev_config.py index c1fa476a5..9cd09c9b4 100644 --- a/scripts/dev/dev_config.py +++ b/scripts/dev/dev_config.py @@ -6,6 +6,10 @@ FULL_CONFIG_PATH = os.path.expanduser(CONFIG_PATH) +def get_config_path() -> str: + return os.getenv("MONGODB_COMMUNITY_CONFIG", FULL_CONFIG_PATH) + + class DevConfig: """ DevConfig is a wrapper around the developer configuration file @@ -24,10 +28,11 @@ def repo_url(self): def load_config() -> Optional[DevConfig]: - with open(FULL_CONFIG_PATH, "r") as f: + config_file_path = get_config_path() + with open(config_file_path, "r") as f: return DevConfig(json.loads(f.read())) print( - f"No DevConfig found. Please ensure that the configuration file exists at '{FULL_CONFIG_PATH}'" + f"No DevConfig found. Please ensure that the configuration file exists at '{config_file_path}'" ) return None diff --git a/scripts/dev/dockerutil.py b/scripts/dev/dockerutil.py index 69bb174ac..7ce92b2b9 100644 --- a/scripts/dev/dockerutil.py +++ b/scripts/dev/dockerutil.py @@ -1,6 +1,7 @@ import docker from dockerfile_generator import render import os +import json def build_image(repo_url: str, tag: str, path): @@ -20,8 +21,27 @@ def push_image(tag: str): """ client = docker.from_env() print(f"Pushing image: {tag}") + progress = "" for line in client.images.push(tag, stream=True): - print(line.decode("utf-8").rstrip()) + print(push_image_formatted(line), end="") + + +def push_image_formatted(line) -> str: + try: + line = json.loads(line.strip()) + except ValueError: + return "" + + to_skip = ("Preparing", "Waiting", "Layer already exists") + if "status" in line: + if line["status"] in to_skip: + return "" + if line["status"] == "Pushing": + current = int(line["progressDetail"]["current"]) + total = int(line["progressDetail"]["total"]) + return "Complete: {:.2f}\n".format(current / total) + + return "" def build_and_push_image(repo_url: str, tag: str, path: str, image_type: str): diff --git a/scripts/dev/e2e.py b/scripts/dev/e2e.py index 3abd00dc0..9156d9db6 100644 --- a/scripts/dev/e2e.py +++ b/scripts/dev/e2e.py @@ -1,3 +1,6 @@ +#!/usr/bin/env python + +from kubernetes.client.rest import ApiException from build_and_deploy_operator import ( ignore_if_doesnt_exist, ignore_if_already_exists, @@ -168,8 +171,6 @@ def main(): corev1 = client.CoreV1Api() wait_for_pod_to_be_running(corev1, TEST_RUNNER_NAME, dev_config.namespace) - print(f"Running test: {args.test}") - # stream all of the pod output as the pod is running for line in corev1.read_namespaced_pod_log( TEST_RUNNER_NAME, dev_config.namespace, follow=True, _preload_content=False diff --git a/scripts/dev/requirements.txt b/scripts/dev/requirements.txt deleted file mode 100644 index 386dd234d..000000000 --- a/scripts/dev/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -docker=4.2.0 -kubernetes=10.0.1 -jinja2==2.11.2 diff --git a/scripts/dev/templates/Dockerfile.prehook b/scripts/dev/templates/Dockerfile.prehook new file mode 100644 index 000000000..14bd00544 --- /dev/null +++ b/scripts/dev/templates/Dockerfile.prehook @@ -0,0 +1,18 @@ +# TODO: template this +FROM golang AS builder + +ENV GO111MODULE=on +ENV GOFLAGS="-mod=vendor" +ENV GOPATH "" + +COPY go.mod go.sum ./ +RUN go mod download + +ADD . . + +RUN go mod vendor && \ + go build -o build/_output/pre-stop-hook -mod=vendor github.com/mongodb/mongodb-kubernetes-operator/cmd/prestop + +FROM busybox + +COPY --from=builder /go/build/_output/pre-stop-hook /pre-stop-hook diff --git a/test/e2e/e2eutil.go b/test/e2e/e2eutil.go index a2f091f8e..99a19937a 100644 --- a/test/e2e/e2eutil.go +++ b/test/e2e/e2eutil.go @@ -75,6 +75,14 @@ func WaitForStatefulSetToExist(stsName string, retryInterval, timeout time.Durat return sts, waitForRuntimeObjectToExist(stsName, retryInterval, timeout, &sts) } +// WaitForStatefulSetToHaveUpdateStrategy waits until all replicas of the StatefulSet with the given name +// have reached the ready status +func WaitForStatefulSetToHaveUpdateStrategy(t *testing.T, mdb *mdbv1.MongoDB, strategy appsv1.StatefulSetUpdateStrategyType, retryInterval, timeout time.Duration) error { + return waitForStatefulSetCondition(t, mdb, retryInterval, timeout, func(sts appsv1.StatefulSet) bool { + return sts.Spec.UpdateStrategy.Type == strategy + }) +} + // WaitForStatefulSetToBeReady waits until all replicas of the StatefulSet with the given name // have reached the ready status func WaitForStatefulSetToBeReady(t *testing.T, mdb *mdbv1.MongoDB, retryInterval, timeout time.Duration) error { @@ -83,6 +91,14 @@ func WaitForStatefulSetToBeReady(t *testing.T, mdb *mdbv1.MongoDB, retryInterval }) } +// waitForStatefulSetToBeUpdated waits until all replicas of the StatefulSet with the given name +// are updated. +func WaitForStatefulSetToBeUpdated(t *testing.T, mdb *mdbv1.MongoDB, retryInterval, timeout time.Duration) error { + return waitForStatefulSetCondition(t, mdb, retryInterval, timeout, func(sts appsv1.StatefulSet) bool { + return sts.Status.UpdatedReplicas == int32(mdb.Spec.Members) + }) +} + func waitForStatefulSetCondition(t *testing.T, mdb *mdbv1.MongoDB, retryInterval, timeout time.Duration, condition func(set appsv1.StatefulSet) bool) error { _, err := WaitForStatefulSetToExist(mdb.Name, retryInterval, timeout) if err != nil { diff --git a/test/e2e/mongodbtests/mongodbtests.go b/test/e2e/mongodbtests/mongodbtests.go index acf72e17d..94f7e5a6f 100644 --- a/test/e2e/mongodbtests/mongodbtests.go +++ b/test/e2e/mongodbtests/mongodbtests.go @@ -18,6 +18,7 @@ import ( "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -34,6 +35,30 @@ func StatefulSetIsReady(mdb *mdbv1.MongoDB) func(t *testing.T) { } } +// StatefulSetIsUpdated ensures that all the Pods of the StatefulSet are +// in "Updated" condition. +func StatefulSetIsUpdated(mdb *mdbv1.MongoDB) func(t *testing.T) { + return func(t *testing.T) { + err := e2eutil.WaitForStatefulSetToBeUpdated(t, mdb, time.Second*15, time.Minute*5) + if err != nil { + t.Fatal(err) + } + t.Logf("StatefulSet %s/%s is updated!", mdb.Namespace, mdb.Name) + } +} + +// StatefulSetHasUpdateStrategy verifies that the StatefulSet holding this MongoDB +// resource has the correct Update Strategy +func StatefulSetHasUpdateStrategy(mdb *mdbv1.MongoDB, strategy appsv1.StatefulSetUpdateStrategyType) func(t *testing.T) { + return func(t *testing.T) { + err := e2eutil.WaitForStatefulSetToHaveUpdateStrategy(t, mdb, appsv1.RollingUpdateStatefulSetStrategyType, time.Second*15, time.Minute*5) + if err != nil { + t.Fatal(err) + } + t.Logf("StatefulSet %s/%s is ready!", mdb.Namespace, mdb.Name) + } +} + // MongoDBReachesRunningPhase ensure the MongoDB resource reaches the Running phase func MongoDBReachesRunningPhase(mdb *mdbv1.MongoDB) func(t *testing.T) { return func(t *testing.T) { @@ -117,6 +142,18 @@ func Scale(mdb *mdbv1.MongoDB, newMembers int) func(*testing.T) { } } +func ChangeVersion(mdb *mdbv1.MongoDB, newVersion string) func(*testing.T) { + return func(t *testing.T) { + t.Logf("Changing versions from: %s to %s", mdb.Spec.Version, newVersion) + err := e2eutil.UpdateMongoDBResource(mdb, func(db *mdbv1.MongoDB) { + db.Spec.Version = newVersion + }) + if err != nil { + t.Fatal(err) + } + } +} + // Connect performs a connectivity check by initializing a mongo client // and inserting a document into the MongoDB resource func Connect(mdb *mdbv1.MongoDB) error { diff --git a/test/e2e/replica_set_change_version/replica_set_test.go b/test/e2e/replica_set_change_version/replica_set_test.go new file mode 100644 index 000000000..0510d78d8 --- /dev/null +++ b/test/e2e/replica_set_change_version/replica_set_test.go @@ -0,0 +1,59 @@ +package replica_set + +import ( + "testing" + "time" + + mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/pkg/apis/mongodb/v1" + e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" + "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/mongodbtests" + f "github.com/operator-framework/operator-sdk/pkg/test" + + appsv1 "k8s.io/api/apps/v1" +) + +func TestMain(m *testing.M) { + f.MainEntry(m) +} + +func TestReplicaSetUpgradeVersion(t *testing.T) { + ctx := f.NewContext(t) + defer ctx.Cleanup() + + // register our types with the testing framework + if err := e2eutil.RegisterTypesWithFramework(&mdbv1.MongoDB{}); err != nil { + t.Fatal(err) + } + + mdb := e2eutil.NewTestMongoDB() + t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) + t.Run("Config Map Was Correctly Created", mongodbtests.AutomationConfigConfigMapExists(&mdb)) + t.Run("Stateful Set Reaches Ready State", mongodbtests.StatefulSetIsReady(&mdb)) + t.Run("MongoDB Reaches Running Phase", mongodbtests.MongoDBReachesRunningPhase(&mdb)) + t.Run("Test Basic Connectivity", mongodbtests.BasicConnectivity(&mdb)) + t.Run("Test Status Was Updated", mongodbtests.Status(&mdb, + mdbv1.MongoDBStatus{ + MongoURI: mdb.MongoURI(), + Phase: mdbv1.Running, + })) + + // Upgrade version to 4.0.8 + t.Run("MongoDB is reachable while version is upgraded", mongodbtests.IsReachableDuring(&mdb, time.Second*10, + func() { + t.Run("Test Version can be upgraded", mongodbtests.ChangeVersion(&mdb, "4.0.8")) + t.Run("StatefulSet has OnDelete update strategy", mongodbtests.StatefulSetHasUpdateStrategy(&mdb, appsv1.OnDeleteStatefulSetStrategyType)) + t.Run("Stateful Set Reaches Ready State, after Upgrading", mongodbtests.StatefulSetIsUpdated(&mdb)) + }, + )) + t.Run("StatefulSet has RollingUpgrade restart strategy", mongodbtests.StatefulSetHasUpdateStrategy(&mdb, appsv1.RollingUpdateStatefulSetStrategyType)) + + // Downgrade version back to 4.0.6 + t.Run("MongoDB is reachable while version is downgraded", mongodbtests.IsReachableDuring(&mdb, time.Second*10, + func() { + t.Run("Test Version can be downgraded", mongodbtests.ChangeVersion(&mdb, "4.0.6")) + t.Run("StatefulSet has OnDelete restart strategy", mongodbtests.StatefulSetHasUpdateStrategy(&mdb, appsv1.OnDeleteStatefulSetStrategyType)) + t.Run("Stateful Set Reaches Ready State, after Upgrading", mongodbtests.StatefulSetIsUpdated(&mdb)) + }, + )) + t.Run("StatefulSet has RollingUpgrade restart strategy", mongodbtests.StatefulSetHasUpdateStrategy(&mdb, appsv1.RollingUpdateStatefulSetStrategyType)) +} From 8a47f115916892892807079fc0d8a2e639912880 Mon Sep 17 00:00:00 2001 From: Rodrigo Valin Date: Wed, 13 May 2020 17:19:32 +0100 Subject: [PATCH 002/790] CLOUDP-62533: Build pre-stop image (#43) --- .evergreen.yml | 13 +++++++++++ cmd/testrunner/main.go | 35 +++++++++++++++++++++++++++-- scripts/ci/build_and_push_image.sh | 10 ++++----- scripts/ci/run_test.sh | 14 +++++++----- scripts/dev/dockerfile_generator.py | 11 ++------- scripts/dev/e2e.py | 10 +++++++++ 6 files changed, 72 insertions(+), 21 deletions(-) diff --git a/.evergreen.yml b/.evergreen.yml index 26366997f..771b2efff 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -111,6 +111,16 @@ tasks: image: quay.io/mongodb/community-operator-e2e:${version_id} image_type: e2e + - name: build_prehook_image + priority: 60 + exec_timeout_secs: 600 + commands: + - func: clone + - func: build_and_push_image + vars: + image: quay.io/mongodb/community-operator-pre-stop-hook:${version_id} + image_type: prehook + - name: build_testrunner_image priority: 60 exec_timeout_secs: 600 @@ -175,6 +185,8 @@ buildvariants: variant: init_test_run - name: build_e2e_image variant: init_test_run + - name: build_prehook_image + variant: init_test_run - name: build_testrunner_image variant: init_test_run tasks: @@ -191,3 +203,4 @@ buildvariants: - name: build_operator_image - name: build_e2e_image - name: build_testrunner_image + - name: build_prehook_image diff --git a/cmd/testrunner/main.go b/cmd/testrunner/main.go index 242e53221..7895b8486 100644 --- a/cmd/testrunner/main.go +++ b/cmd/testrunner/main.go @@ -33,15 +33,17 @@ type flags struct { deployDir string namespace string operatorImage string + preHookImage string testImage string test string } func parseFlags() flags { - var namespace, deployDir, operatorImage, testImage, test *string + var namespace, deployDir, operatorImage, preHookImage, testImage, test *string namespace = flag.String("namespace", "default", "the namespace the operator and tests should be deployed in") deployDir = flag.String("deployDir", "deploy/", "the path to the directory which contains the yaml deployment files") operatorImage = flag.String("operatorImage", "quay.io/mongodb/community-operator-dev:latest", "the image which should be used for the operator deployment") + preHookImage = flag.String("preHookImage", "quay.io/mongodb/community-operator-prehook:latest", "the prestophook image") testImage = flag.String("testImage", "quay.io/mongodb/community-operator-e2e:latest", "the image which should be used for the operator e2e tests") test = flag.String("test", "", "test e2e test that should be run. (name of folder containing the test)") flag.Parse() @@ -50,6 +52,7 @@ func parseFlags() flags { deployDir: *deployDir, namespace: *namespace, operatorImage: *operatorImage, + preHookImage: *preHookImage, testImage: *testImage, test: *test, } @@ -164,7 +167,11 @@ func deployOperator(f flags, c client.Client) error { return fmt.Errorf("error building operator role binding: %v", err) } fmt.Println("Successfully created the operator Role Binding") - if err := buildKubernetesResourceFromYamlFile(c, path.Join(f.deployDir, "operator.yaml"), &appsv1.Deployment{}, withNamespace(f.namespace), withOperatorImage(f.operatorImage)); err != nil { + if err := buildKubernetesResourceFromYamlFile(c, path.Join(f.deployDir, "operator.yaml"), + &appsv1.Deployment{}, + withNamespace(f.namespace), + withOperatorImage(f.operatorImage), + withPreHookImage(f.preHookImage)); err != nil { return fmt.Errorf("error building operator deployment: %v", err) } fmt.Println("Successfully created the operator Deployment") @@ -201,6 +208,30 @@ func withTestImage(image string) func(obj runtime.Object) { } } +// withPreHookImage sets the value of the PRE_STOP_HOOK_IMAGE +// EnvVar from first container to `image`. The EnvVar is updated +// if it exists. Or appended if there is no EnvVar with this `Name`. +func withPreHookImage(image string) func(runtime.Object) { + return func(obj runtime.Object) { + if dep, ok := obj.(*appsv1.Deployment); ok { + preHookEnv := corev1.EnvVar{ + Name: "PRE_STOP_HOOK_IMAGE", + Value: image, + } + found := false + for idx := range dep.Spec.Template.Spec.Containers[0].Env { + if dep.Spec.Template.Spec.Containers[0].Env[idx].Name == preHookEnv.Name { + dep.Spec.Template.Spec.Containers[0].Env[idx].Value = preHookEnv.Value + found = true + } + } + if !found { + dep.Spec.Template.Spec.Containers[0].Env = append(dep.Spec.Template.Spec.Containers[0].Env, preHookEnv) + } + } + } +} + // withOperatorImage assumes that the underlying type is an appsv1.Deployment // which has the operator container as the first container. There will be // no effect when used with a non-deployment type diff --git a/scripts/ci/build_and_push_image.sh b/scripts/ci/build_and_push_image.sh index b18a1a9d7..8a37d555e 100755 --- a/scripts/ci/build_and_push_image.sh +++ b/scripts/ci/build_and_push_image.sh @@ -1,7 +1,7 @@ -#!/bin/sh +#!/usr/bin/env bash -echo ${quay_password} | docker login -u=${quay_user_name} quay.io --password-stdin +echo "${quay_password:?}" | docker login "-u=${quay_user_name:?}" quay.io --password-stdin -python scripts/dev/dockerfile_generator.py ${image_type} > Dockerfile -docker build . -f Dockerfile -t ${image} -docker push ${image} +python scripts/dev/dockerfile_generator.py "${image_type:?}" > Dockerfile +docker build . -f Dockerfile -t "${image:?}" +docker push "${image:?}" diff --git a/scripts/ci/run_test.sh b/scripts/ci/run_test.sh index f1c9ee26b..2550d4d24 100755 --- a/scripts/ci/run_test.sh +++ b/scripts/ci/run_test.sh @@ -7,10 +7,10 @@ # See: https://github.com/operator-framework/operator-sdk/issues/2618 KUBERNETES_SERVICE_HOST="$(kubectl get service kubernetes -o jsonpath='{.spec.clusterIP }')" temp=$(mktemp) -cat ${KUBECONFIG} | sed "s/server: https.*/server: https:\/\/${KUBERNETES_SERVICE_HOST}/g" > ${temp} -contents="$(cat ${temp})" +cat "${KUBECONFIG}" | sed "s/server: https.*/server: https:\/\/${KUBERNETES_SERVICE_HOST}/g" > "${temp}" +contents=$(<"${temp}") kubectl create cm kube-config --from-literal=kubeconfig="${contents}" -rm ${temp} +rm "${temp}" # create roles and service account required for the test runner kubectl apply -f deploy/testrunner @@ -19,9 +19,13 @@ kubectl apply -f deploy/testrunner kubectl run test-runner --generator=run-pod/v1 \ --restart=Never \ --image-pull-policy=Always \ - --image=quay.io/mongodb/community-operator-testrunner:${version_id} \ + "--image=quay.io/mongodb/community-operator-testrunner:${version_id:?}" \ --serviceaccount=test-runner \ - --command -- ./runner --operatorImage quay.io/mongodb/community-operator-dev:${version_id} --testImage quay.io/mongodb/community-operator-e2e:${version_id} --test=${test} + --command -- \ + ./runner --operatorImage "quay.io/mongodb/community-operator-dev:${version_id}" \ + --preHookImage "quay.io/mongodb/community-operator-pre-stop-hook:${version_id}" \ + --testImage "quay.io/mongodb/community-operator-e2e:${version_id}" \ + "--test=${test:?}" echo "Test pod is ready to begin" diff --git a/scripts/dev/dockerfile_generator.py b/scripts/dev/dockerfile_generator.py index f981c17d7..2452a149a 100755 --- a/scripts/dev/dockerfile_generator.py +++ b/scripts/dev/dockerfile_generator.py @@ -38,18 +38,11 @@ def render(image_name): "operator": operator_params(), } - if image_name not in param_dict: - raise ValueError( - "Image name: {} is invalid. Valid values are {}".format( - image_name, param_dict.keys() - ) - ) + render_values = param_dict.get(image_name, dict()) env = jinja2.Environment() env.loader = jinja2.FileSystemLoader(searchpath="scripts/dev/templates") - return env.get_template("Dockerfile.{}".format(image_name)).render( - param_dict[image_name] - ) + return env.get_template("Dockerfile.{}".format(image_name)).render(render_values) def parse_args(): diff --git a/scripts/dev/e2e.py b/scripts/dev/e2e.py index 9156d9db6..9cb988b0f 100644 --- a/scripts/dev/e2e.py +++ b/scripts/dev/e2e.py @@ -87,6 +87,13 @@ def build_and_push_e2e(repo_url: str, tag: str, path: str): return build_and_push_image(repo_url, tag, path, "e2e") +def build_and_push_prehook(repo_url: str, tag: str, path: str): + """ + build_and_push_prehook builds and pushes the pre-stop-hook image. + """ + return build_and_push_image(repo_url, tag, path, "prehook") + + def _delete_testrunner_pod() -> None: """ _delete_testrunner_pod deletes the test runner pod @@ -126,6 +133,8 @@ def _get_testrunner_pod_body(test: str) -> Dict: "./runner", "--operatorImage", f"{dev_config.repo_url}/mongodb-kubernetes-operator", + "--preHookImage", + f"{dev_config.repo_url}/prehook", "--testImage", f"{dev_config.repo_url}/e2e", f"--test={test}", @@ -164,6 +173,7 @@ def main(): dev_config.repo_url, f"{dev_config.repo_url}/{TEST_RUNNER_NAME}", "." ) build_and_push_e2e(dev_config.repo_url, f"{dev_config.repo_url}/e2e", ".") + build_and_push_prehook(dev_config.repo_url, f"{dev_config.repo_url}/prehook", ".") _prepare_testrunner_environment() From c63bfbf916a8bd03d81de87e97f4e7460d3c1af9 Mon Sep 17 00:00:00 2001 From: Rodrigo Valin Date: Thu, 14 May 2020 20:47:07 +0100 Subject: [PATCH 003/790] Added FeatureCompatibilityVersion e2e test (#45) --- .evergreen.yml | 18 +++++++ .../feature_compatibility_version_test.go | 47 ++++++++++++++++ ...ture_compatibility_version_upgrade_test.go | 54 +++++++++++++++++++ test/e2e/mongodbtests/mongodbtests.go | 54 +++++++++++++++++++ 4 files changed, 173 insertions(+) create mode 100644 test/e2e/feature_compatibility_version/feature_compatibility_version_test.go create mode 100644 test/e2e/feature_compatibility_version_upgrade/feature_compatibility_version_upgrade_test.go diff --git a/.evergreen.yml b/.evergreen.yml index 771b2efff..fbbfbb844 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -136,6 +136,22 @@ tasks: - func: clone - func: go_test + - name: e2e_test_feature_compatibility_version + commands: + - func: clone + - func: setup_kubernetes_environment + - func: run_e2e_test + vars: + test: feature_compatibility_version + + - name: e2e_test_feature_compatibility_version_upgrade + commands: + - func: clone + - func: setup_kubernetes_environment + - func: run_e2e_test + vars: + test: feature_compatibility_version + - name: e2e_test_replica_set commands: - func: clone @@ -194,6 +210,8 @@ buildvariants: - name: e2e_test_replica_set_readiness_probe - name: e2e_test_replica_set_scale - name: e2e_test_replica_set_change_version + - name: e2e_test_feature_compatibility_version + - name: e2e_test_feature_compatibility_version_upgrade - name: init_test_run display_name: init_test_run diff --git a/test/e2e/feature_compatibility_version/feature_compatibility_version_test.go b/test/e2e/feature_compatibility_version/feature_compatibility_version_test.go new file mode 100644 index 000000000..a5869da02 --- /dev/null +++ b/test/e2e/feature_compatibility_version/feature_compatibility_version_test.go @@ -0,0 +1,47 @@ +package feature_compatibility_version + +import ( + "testing" + "time" + + mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/pkg/apis/mongodb/v1" + e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" + "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/mongodbtests" + f "github.com/operator-framework/operator-sdk/pkg/test" +) + +func TestMain(m *testing.M) { + f.MainEntry(m) +} + +func TestFeatureCompatibilityVersion(t *testing.T) { + ctx := f.NewContext(t) + defer ctx.Cleanup() + if err := e2eutil.RegisterTypesWithFramework(&mdbv1.MongoDB{}); err != nil { + t.Fatal(err) + } + + mdb := e2eutil.NewTestMongoDB() + t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) + t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb)) + + t.Run("Test FeatureCompatibilityVersion is 4.0", mongodbtests.HasFeatureCompatibilityVersion(&mdb, "4.0", 1)) + // Upgrade version to 4.2.6 while keeping the FCV set to 4.0 + t.Run("MongoDB is reachable while version is upgraded", mongodbtests.IsReachableDuring(&mdb, time.Second*10, + func() { + t.Run("Test Version can be upgraded", mongodbtests.ChangeVersion(&mdb, "4.2.6")) + t.Run("Stateful Set Reaches Ready State, after Upgrading", mongodbtests.StatefulSetIsUpdated(&mdb)) + }, + )) + t.Run("Test Basic Connectivity after upgrade has completed", mongodbtests.BasicConnectivity(&mdb)) + t.Run("Test FeatureCompatibilityVersion, after upgrade, is 4.0", mongodbtests.HasFeatureCompatibilityVersion(&mdb, "4.0", 1)) + + // Downgrade version back to 4.0.6, checks that the FeatureCompatibilityVersion stayed at 4.0 + t.Run("MongoDB is reachable while version is downgraded", mongodbtests.IsReachableDuring(&mdb, time.Second*10, + func() { + t.Run("Test Version can be downgraded", mongodbtests.ChangeVersion(&mdb, "4.0.6")) + t.Run("Stateful Set Reaches Ready State, after Upgrading", mongodbtests.StatefulSetIsUpdated(&mdb)) + }, + )) + t.Run("Test FeatureCompatibilityVersion, after downgrade, is 4.0", mongodbtests.HasFeatureCompatibilityVersion(&mdb, "4.0", 1)) +} diff --git a/test/e2e/feature_compatibility_version_upgrade/feature_compatibility_version_upgrade_test.go b/test/e2e/feature_compatibility_version_upgrade/feature_compatibility_version_upgrade_test.go new file mode 100644 index 000000000..31b0c0e28 --- /dev/null +++ b/test/e2e/feature_compatibility_version_upgrade/feature_compatibility_version_upgrade_test.go @@ -0,0 +1,54 @@ +package feature_compatibility_version_upgrade + +import ( + "testing" + "time" + + mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/pkg/apis/mongodb/v1" + e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" + "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/mongodbtests" + f "github.com/operator-framework/operator-sdk/pkg/test" + "github.com/stretchr/testify/assert" +) + +func TestMain(m *testing.M) { + f.MainEntry(m) +} + +func TestFeatureCompatibilityVersionUpgrade(t *testing.T) { + ctx := f.NewContext(t) + defer ctx.Cleanup() + if err := e2eutil.RegisterTypesWithFramework(&mdbv1.MongoDB{}); err != nil { + t.Fatal(err) + } + + mdb := e2eutil.NewTestMongoDB() + t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) + t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb)) + + t.Run("Test FeatureCompatibilityVersion is 4.0", mongodbtests.HasFeatureCompatibilityVersion(&mdb, "4.0", 1)) + // Upgrade version to 4.2.6 while keeping the FCV set to 4.0 + t.Run("MongoDB is reachable", mongodbtests.IsReachableDuring(&mdb, time.Second*10, + func() { + t.Run("Test Version can be upgraded", mongodbtests.ChangeVersion(&mdb, "4.2.6")) + t.Run("Stateful Set Reaches Ready State, after Upgrading", mongodbtests.StatefulSetIsUpdated(&mdb)) + t.Run("Test Basic Connectivity after upgrade has completed", mongodbtests.BasicConnectivity(&mdb)) + }, + )) + t.Run("Test FeatureCompatibilityVersion, after upgrade, is 4.0", mongodbtests.HasFeatureCompatibilityVersion(&mdb, "4.0", 1)) + + t.Run("MongoDB is reachable", mongodbtests.IsReachableDuring(&mdb, time.Second*10, + func() { + t.Run("Test FCV can be upgraded", func(t *testing.T) { + err := e2eutil.UpdateMongoDBResource(&mdb, func(db *mdbv1.MongoDB) { + db.Spec.FeatureCompatibilityVersion = "4.2" + }) + assert.NoError(t, err) + }) + t.Run("Stateful Set Reaches Ready State", mongodbtests.StatefulSetIsUpdated(&mdb)) + t.Run("MongoDB Reaches Running Phase", mongodbtests.MongoDBReachesRunningPhase(&mdb)) + }, + )) + + t.Run("Test FeatureCompatibilityVersion, after upgrade, is 4.2", mongodbtests.HasFeatureCompatibilityVersion(&mdb, "4.2", 3)) +} diff --git a/test/e2e/mongodbtests/mongodbtests.go b/test/e2e/mongodbtests/mongodbtests.go index 94f7e5a6f..09156a5e9 100644 --- a/test/e2e/mongodbtests/mongodbtests.go +++ b/test/e2e/mongodbtests/mongodbtests.go @@ -3,6 +3,7 @@ package mongodbtests import ( "context" "fmt" + "reflect" "testing" "time" @@ -16,6 +17,7 @@ import ( f "github.com/operator-framework/operator-sdk/pkg/test" "github.com/stretchr/testify/assert" "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" appsv1 "k8s.io/api/apps/v1" @@ -82,6 +84,44 @@ func AutomationConfigConfigMapExists(mdb *mdbv1.MongoDB) func(t *testing.T) { } } +// HasFeatureCompatibilityVersion verifies that the FeatureCompatibilityVersion is +// set to `version`. The FCV parameter is not signaled as a non Running state, for +// this reason, this function checks the value of the parameter many times, based +// on the value of `tries`. +func HasFeatureCompatibilityVersion(mdb *mdbv1.MongoDB, fcv string, tries int) func(t *testing.T) { + return func(t *testing.T) { + ctx, _ := context.WithTimeout(context.Background(), 10*time.Minute) + mongoClient, err := mongo.Connect(ctx, options.Client().ApplyURI(mdb.MongoURI())) + assert.NoError(t, err) + + database := mongoClient.Database("admin") + assert.NotNil(t, database) + + runCommand := bson.D{ + primitive.E{Key: "getParameter", Value: 1}, + primitive.E{Key: "featureCompatibilityVersion", Value: 1}, + } + found := false + for !found && tries > 0 { + select { + case <-time.After(10 * time.Second): + var result bson.M + err = database.RunCommand(ctx, runCommand).Decode(&result) + assert.NoError(t, err) + + expected := primitive.M{"version": fcv} + if reflect.DeepEqual(expected, result["featureCompatibilityVersion"]) { + found = true + } + } + + tries -= 1 + } + + assert.True(t, found) + } +} + // CreateMongoDBResource creates the MongoDB resource func CreateMongoDBResource(mdb *mdbv1.MongoDB, ctx *f.TestCtx) func(*testing.T) { return func(t *testing.T) { @@ -92,6 +132,20 @@ func CreateMongoDBResource(mdb *mdbv1.MongoDB, ctx *f.TestCtx) func(*testing.T) } } +func BasicFunctionality(mdb *mdbv1.MongoDB) func(*testing.T) { + return func(t *testing.T) { + t.Run("Config Map Was Correctly Created", AutomationConfigConfigMapExists(mdb)) + t.Run("Stateful Set Reaches Ready State", StatefulSetIsReady(mdb)) + t.Run("MongoDB Reaches Running Phase", MongoDBReachesRunningPhase(mdb)) + t.Run("Test Basic Connectivity", BasicConnectivity(mdb)) + t.Run("Test Status Was Updated", Status(mdb, + mdbv1.MongoDBStatus{ + MongoURI: mdb.MongoURI(), + Phase: mdbv1.Running, + })) + } +} + // DeletePod will delete a pod that belongs to this MongoDB resource's StatefulSet func DeletePod(mdb *mdbv1.MongoDB, podNum int) func(*testing.T) { return func(t *testing.T) { From 11ae0792d3f3002c950e7a78c7533daa6342beeb Mon Sep 17 00:00:00 2001 From: Rodrigo Valin Date: Fri, 15 May 2020 01:09:11 +0100 Subject: [PATCH 004/790] E2E for multiple resources (#46) --- .evergreen.yml | 9 +++ test/e2e/e2eutil.go | 4 +- .../feature_compatibility_version_test.go | 8 +- ...ture_compatibility_version_upgrade_test.go | 6 +- test/e2e/replica_set/replica_set_test.go | 6 +- .../replica_set_test.go | 4 +- .../replica_set_multiple_test.go | 77 +++++++++++++++++++ .../replica_set_readiness_probe_test.go | 6 +- .../replica_set_scaling_test.go | 6 +- 9 files changed, 102 insertions(+), 24 deletions(-) create mode 100644 test/e2e/replica_set_multiple/replica_set_multiple_test.go diff --git a/.evergreen.yml b/.evergreen.yml index fbbfbb844..d6d9f4690 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -184,6 +184,14 @@ tasks: vars: test: replica_set_change_version + - name: e2e_test_replica_set_multiple + commands: + - func: clone + - func: setup_kubernetes_environment + - func: run_e2e_test + vars: + test: replica_set_multiple + buildvariants: - name: go_unit_tests display_name: go_unit_tests @@ -212,6 +220,7 @@ buildvariants: - name: e2e_test_replica_set_change_version - name: e2e_test_feature_compatibility_version - name: e2e_test_feature_compatibility_version_upgrade + - name: e2e_test_replica_set_multiple - name: init_test_run display_name: init_test_run diff --git a/test/e2e/e2eutil.go b/test/e2e/e2eutil.go index 99a19937a..ed267e551 100644 --- a/test/e2e/e2eutil.go +++ b/test/e2e/e2eutil.go @@ -129,10 +129,10 @@ func waitForRuntimeObjectToExist(name string, retryInterval, timeout time.Durati }) } -func NewTestMongoDB() mdbv1.MongoDB { +func NewTestMongoDB(name string) mdbv1.MongoDB { return mdbv1.MongoDB{ ObjectMeta: metav1.ObjectMeta{ - Name: "example-mongodb", + Name: name, Namespace: f.Global.OperatorNamespace, }, Spec: mdbv1.MongoDBSpec{ diff --git a/test/e2e/feature_compatibility_version/feature_compatibility_version_test.go b/test/e2e/feature_compatibility_version/feature_compatibility_version_test.go index a5869da02..bfeacbb42 100644 --- a/test/e2e/feature_compatibility_version/feature_compatibility_version_test.go +++ b/test/e2e/feature_compatibility_version/feature_compatibility_version_test.go @@ -21,11 +21,11 @@ func TestFeatureCompatibilityVersion(t *testing.T) { t.Fatal(err) } - mdb := e2eutil.NewTestMongoDB() + mdb := e2eutil.NewTestMongoDB("mdb0") t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb)) - t.Run("Test FeatureCompatibilityVersion is 4.0", mongodbtests.HasFeatureCompatibilityVersion(&mdb, "4.0", 1)) + t.Run("Test FeatureCompatibilityVersion is 4.0", mongodbtests.HasFeatureCompatibilityVersion(&mdb, "4.0", 3)) // Upgrade version to 4.2.6 while keeping the FCV set to 4.0 t.Run("MongoDB is reachable while version is upgraded", mongodbtests.IsReachableDuring(&mdb, time.Second*10, func() { @@ -34,7 +34,7 @@ func TestFeatureCompatibilityVersion(t *testing.T) { }, )) t.Run("Test Basic Connectivity after upgrade has completed", mongodbtests.BasicConnectivity(&mdb)) - t.Run("Test FeatureCompatibilityVersion, after upgrade, is 4.0", mongodbtests.HasFeatureCompatibilityVersion(&mdb, "4.0", 1)) + t.Run("Test FeatureCompatibilityVersion, after upgrade, is 4.0", mongodbtests.HasFeatureCompatibilityVersion(&mdb, "4.0", 3)) // Downgrade version back to 4.0.6, checks that the FeatureCompatibilityVersion stayed at 4.0 t.Run("MongoDB is reachable while version is downgraded", mongodbtests.IsReachableDuring(&mdb, time.Second*10, @@ -43,5 +43,5 @@ func TestFeatureCompatibilityVersion(t *testing.T) { t.Run("Stateful Set Reaches Ready State, after Upgrading", mongodbtests.StatefulSetIsUpdated(&mdb)) }, )) - t.Run("Test FeatureCompatibilityVersion, after downgrade, is 4.0", mongodbtests.HasFeatureCompatibilityVersion(&mdb, "4.0", 1)) + t.Run("Test FeatureCompatibilityVersion, after downgrade, is 4.0", mongodbtests.HasFeatureCompatibilityVersion(&mdb, "4.0", 3)) } diff --git a/test/e2e/feature_compatibility_version_upgrade/feature_compatibility_version_upgrade_test.go b/test/e2e/feature_compatibility_version_upgrade/feature_compatibility_version_upgrade_test.go index 31b0c0e28..394ae8412 100644 --- a/test/e2e/feature_compatibility_version_upgrade/feature_compatibility_version_upgrade_test.go +++ b/test/e2e/feature_compatibility_version_upgrade/feature_compatibility_version_upgrade_test.go @@ -22,11 +22,11 @@ func TestFeatureCompatibilityVersionUpgrade(t *testing.T) { t.Fatal(err) } - mdb := e2eutil.NewTestMongoDB() + mdb := e2eutil.NewTestMongoDB("mdb0") t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb)) - t.Run("Test FeatureCompatibilityVersion is 4.0", mongodbtests.HasFeatureCompatibilityVersion(&mdb, "4.0", 1)) + t.Run("Test FeatureCompatibilityVersion is 4.0", mongodbtests.HasFeatureCompatibilityVersion(&mdb, "4.0", 3)) // Upgrade version to 4.2.6 while keeping the FCV set to 4.0 t.Run("MongoDB is reachable", mongodbtests.IsReachableDuring(&mdb, time.Second*10, func() { @@ -35,7 +35,7 @@ func TestFeatureCompatibilityVersionUpgrade(t *testing.T) { t.Run("Test Basic Connectivity after upgrade has completed", mongodbtests.BasicConnectivity(&mdb)) }, )) - t.Run("Test FeatureCompatibilityVersion, after upgrade, is 4.0", mongodbtests.HasFeatureCompatibilityVersion(&mdb, "4.0", 1)) + t.Run("Test FeatureCompatibilityVersion, after upgrade, is 4.0", mongodbtests.HasFeatureCompatibilityVersion(&mdb, "4.0", 3)) t.Run("MongoDB is reachable", mongodbtests.IsReachableDuring(&mdb, time.Second*10, func() { diff --git a/test/e2e/replica_set/replica_set_test.go b/test/e2e/replica_set/replica_set_test.go index 02f900ac1..bbe602e20 100644 --- a/test/e2e/replica_set/replica_set_test.go +++ b/test/e2e/replica_set/replica_set_test.go @@ -14,15 +14,13 @@ func TestMain(m *testing.M) { } func TestReplicaSet(t *testing.T) { - ctx := f.NewTestCtx(t) + ctx := f.NewContext(t) defer ctx.Cleanup() - - // register our types with the testing framework if err := e2eutil.RegisterTypesWithFramework(&mdbv1.MongoDB{}); err != nil { t.Fatal(err) } - mdb := e2eutil.NewTestMongoDB() + mdb := e2eutil.NewTestMongoDB("mdb0") t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) t.Run("Config Map Was Correctly Created", mongodbtests.AutomationConfigConfigMapExists(&mdb)) t.Run("Stateful Set Reaches Ready State", mongodbtests.StatefulSetIsReady(&mdb)) diff --git a/test/e2e/replica_set_change_version/replica_set_test.go b/test/e2e/replica_set_change_version/replica_set_test.go index 0510d78d8..c63b50414 100644 --- a/test/e2e/replica_set_change_version/replica_set_test.go +++ b/test/e2e/replica_set_change_version/replica_set_test.go @@ -19,13 +19,11 @@ func TestMain(m *testing.M) { func TestReplicaSetUpgradeVersion(t *testing.T) { ctx := f.NewContext(t) defer ctx.Cleanup() - - // register our types with the testing framework if err := e2eutil.RegisterTypesWithFramework(&mdbv1.MongoDB{}); err != nil { t.Fatal(err) } - mdb := e2eutil.NewTestMongoDB() + mdb := e2eutil.NewTestMongoDB("mdb0") t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) t.Run("Config Map Was Correctly Created", mongodbtests.AutomationConfigConfigMapExists(&mdb)) t.Run("Stateful Set Reaches Ready State", mongodbtests.StatefulSetIsReady(&mdb)) diff --git a/test/e2e/replica_set_multiple/replica_set_multiple_test.go b/test/e2e/replica_set_multiple/replica_set_multiple_test.go new file mode 100644 index 000000000..876f491c4 --- /dev/null +++ b/test/e2e/replica_set_multiple/replica_set_multiple_test.go @@ -0,0 +1,77 @@ +package replica_set_multiple + +import ( + "testing" + "time" + + mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/pkg/apis/mongodb/v1" + e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" + "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/mongodbtests" + f "github.com/operator-framework/operator-sdk/pkg/test" +) + +func TestMain(m *testing.M) { + f.MainEntry(m) +} + +// TestReplicaSet creates two MongoDB resources that are handled by the Operator at the +// same time. One of them is scaled to 5 and then back to 3 +func TestReplicaSet(t *testing.T) { + ctx := f.NewContext(t) + defer ctx.Cleanup() + if err := e2eutil.RegisterTypesWithFramework(&mdbv1.MongoDB{}); err != nil { + t.Fatal(err) + } + + mdb0 := e2eutil.NewTestMongoDB("mdb0") + mdb1 := e2eutil.NewTestMongoDB("mdb1") + t.Run("Create MongoDB Resource mdb0", mongodbtests.CreateMongoDBResource(&mdb0, ctx)) + t.Run("Create MongoDB Resource mdb1", mongodbtests.CreateMongoDBResource(&mdb1, ctx)) + + t.Run("mdb0: Config Map Was Correctly Created", mongodbtests.AutomationConfigConfigMapExists(&mdb0)) + t.Run("mdb1: Config Map Was Correctly Created", mongodbtests.AutomationConfigConfigMapExists(&mdb1)) + + t.Run("mdb0: Stateful Set Reaches Ready State", mongodbtests.StatefulSetIsReady(&mdb0)) + t.Run("mdb1: Stateful Set Reaches Ready State", mongodbtests.StatefulSetIsReady(&mdb1)) + + t.Run("mdb0: MongoDB Reaches Running Phase", mongodbtests.MongoDBReachesRunningPhase(&mdb0)) + t.Run("mdb1: MongoDB Reaches Running Phase", mongodbtests.MongoDBReachesRunningPhase(&mdb1)) + + t.Run("mdb0: Test Basic Connectivity", mongodbtests.BasicConnectivity(&mdb0)) + t.Run("mdb1: Test Basic Connectivity", mongodbtests.BasicConnectivity(&mdb1)) + + t.Run("mdb0: Test Status Was Updated", mongodbtests.Status(&mdb0, + mdbv1.MongoDBStatus{ + MongoURI: mdb0.MongoURI(), + Phase: mdbv1.Running, + })) + t.Run("mdb1: Test Status Was Updated", mongodbtests.Status(&mdb1, + mdbv1.MongoDBStatus{ + MongoURI: mdb1.MongoURI(), + Phase: mdbv1.Running, + })) + + t.Run("MongoDB is reachable while being scaled up", mongodbtests.IsReachableDuring(&mdb0, time.Second*10, + func() { + t.Run("Scale MongoDB Resource Up", mongodbtests.Scale(&mdb0, 5)) + t.Run("Stateful Set Scaled Up Correctly", mongodbtests.StatefulSetIsReady(&mdb0)) + t.Run("MongoDB Reaches Running Phase", mongodbtests.MongoDBReachesRunningPhase(&mdb0)) + t.Run("Test Status Was Updated", mongodbtests.Status(&mdb0, + mdbv1.MongoDBStatus{ + MongoURI: mdb0.MongoURI(), + Phase: mdbv1.Running, + })) + t.Run("Scale MongoDB Resource Down", mongodbtests.Scale(&mdb0, 3)) + t.Run("Stateful Set Scaled Down Correctly", mongodbtests.StatefulSetIsReady(&mdb0)) + t.Run("MongoDB Reaches Running Phase", mongodbtests.MongoDBReachesRunningPhase(&mdb0)) + t.Run("Test Status Was Updated", mongodbtests.Status(&mdb0, + mdbv1.MongoDBStatus{ + MongoURI: mdb0.MongoURI(), + Phase: mdbv1.Running, + })) + }, + )) + + // One last check that mdb1 was not altered. + t.Run("mdb1: Test Basic Connectivity", mongodbtests.BasicConnectivity(&mdb1)) +} diff --git a/test/e2e/replica_set_readiness_probe/replica_set_readiness_probe_test.go b/test/e2e/replica_set_readiness_probe/replica_set_readiness_probe_test.go index a7f7830e6..4d8b10a5f 100644 --- a/test/e2e/replica_set_readiness_probe/replica_set_readiness_probe_test.go +++ b/test/e2e/replica_set_readiness_probe/replica_set_readiness_probe_test.go @@ -16,15 +16,13 @@ func TestMain(m *testing.M) { } func TestReplicaSetReadinessProbeScaling(t *testing.T) { - ctx := f.NewTestCtx(t) + ctx := f.NewContext(t) defer ctx.Cleanup() - - // register our types with the testing framework if err := e2eutil.RegisterTypesWithFramework(&mdbv1.MongoDB{}); err != nil { t.Fatal(err) } - mdb := e2eutil.NewTestMongoDB() + mdb := e2eutil.NewTestMongoDB("mdb0") t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) t.Run("Config Map Was Correctly Created", mongodbtests.AutomationConfigConfigMapExists(&mdb)) t.Run("Stateful Set Reaches Ready State", mongodbtests.StatefulSetIsReady(&mdb)) diff --git a/test/e2e/replica_set_scale/replica_set_scaling_test.go b/test/e2e/replica_set_scale/replica_set_scaling_test.go index b2ea42af3..724f3c27f 100644 --- a/test/e2e/replica_set_scale/replica_set_scaling_test.go +++ b/test/e2e/replica_set_scale/replica_set_scaling_test.go @@ -15,15 +15,13 @@ func TestMain(m *testing.M) { } func TestReplicaSetScale(t *testing.T) { - ctx := f.NewTestCtx(t) + ctx := f.NewContext(t) defer ctx.Cleanup() - - // register our types with the testing framework if err := e2eutil.RegisterTypesWithFramework(&mdbv1.MongoDB{}); err != nil { t.Fatal(err) } - mdb := e2eutil.NewTestMongoDB() + mdb := e2eutil.NewTestMongoDB("mdb0") t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) t.Run("Config Map Was Correctly Created", mongodbtests.AutomationConfigConfigMapExists(&mdb)) t.Run("Stateful Set Reaches Ready State", mongodbtests.StatefulSetIsReady(&mdb)) From 44283c9957e5ccc114e6f9b93ac5e5985439a500 Mon Sep 17 00:00:00 2001 From: Rodrigo Valin Date: Tue, 26 May 2020 15:50:08 +0100 Subject: [PATCH 005/790] Dev start guide (#48) --- README.md | 5 ++ architecture.md | 11 +++ contributing.md | 142 ++++++++++++++++++++++++++++++ requirements.txt | 1 + scripts/ci/run_unit_tests.sh | 4 +- scripts/dev/dockerutil.py | 14 ++- scripts/dev/e2e.py | 45 +++++++++- scripts/dev/setup_kind_cluster.sh | 11 +-- 8 files changed, 216 insertions(+), 17 deletions(-) create mode 100644 architecture.md create mode 100644 contributing.md diff --git a/README.md b/README.md index 92e4f6196..a6cc8879b 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ To install the MongoDB Community Kubernetes Operator: 1. Change to the directory in which you cloned the repository. 2. Install the [Custom Resource Definitions](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/). + a. Invoke the following `kubectl` command: ``` kubectl create -f deploy/crds/mongodb.com_mongodb_crd.yaml @@ -39,6 +40,7 @@ To install the MongoDB Community Kubernetes Operator: kubectl get crd/mongodb.mongodb.com ``` 3. Install the Operator. + a. Invoke the following `kubectl` command to install the Operator in the specified namespace: ``` kubectl create -f deploy --namespace @@ -67,6 +69,9 @@ To deploy your first replica set: ## Contributing +Please get familiar with the architecture.md document and then go ahead and read +the [contributing guide](contributing.md) guide. + Please file issues before filing PRs. For PRs to be accepted, contributors must sign our [CLA](https://www.mongodb.com/legal/contributor-agreement). Reviewers, please ensure that the CLA has been signed by referring to [the contributors tool](https://contributors.corp.mongodb.com/) (internal link). diff --git a/architecture.md b/architecture.md new file mode 100644 index 000000000..3d5801169 --- /dev/null +++ b/architecture.md @@ -0,0 +1,11 @@ +# MongoDB Community Operator Architecture + +**Work in Progress** + +## Topics + +* Automation Agent +* Cluster Configuration and ConfigMap +* MongoDB Docker Images +* Example case: MongoDB version upgrade + diff --git a/contributing.md b/contributing.md new file mode 100644 index 000000000..5e1fada4c --- /dev/null +++ b/contributing.md @@ -0,0 +1,142 @@ +# Contributing to MongoDB Kubernetes Operator + +First you need to get familiar with the [Architecture guide](architecture.md), which explains +from a high perspective how everything works together. + +After our experience building the [Enterprise MongoDB Kubernetes +Operator](https://github.com/mongodb/mongodb-enterprise-operator), we have +realized that is is very important to have a clean environment to work, and as such we have +adopted a strategy that makes it easier for everyone to contribute. + +This strategy is based on using +[`operator-sdk`](https://github.com/operator-framework/operator-sdk) for running +the tests, and making the test-runner itself run as a Kubernetes Pod. This +makes it easier to run the tests in environments with access to a Kubernetes +cluster with no go toolchain installed locally, making it easier to reproduce +our local working environments in CI/CD systems. + +# High-Perspective Architecture + +The operator itself consists of 1 image, that has all the operational logic to deploy and +maintain the MongoDB resources in your cluster. + +The operator deploys MongoDB in Pods (via a higher-level resource, a +StatefulSet), on each Pod there will be multiple images coexisting during the +lifetime of the MongoDB server. + +* Agent image: This image includes a binary provided by MongoDB that handles +the local operation of a MongoDB server given a series of configurations +provided by the operator. The configuration exists as a ConfigMap that's created +by the operator and mounted in the Agent's Pod. + +* MongoDB image: Docker image that includes the MongoDB server. + +* Pre-stop Hook: This image includes a binary that helps orchestrate the + restarts of the MongoDB Replica Set members, in particular, when dealing with + version upgrades, which requires a very precise set of operations to allow for + seamless upgrades and downgrades, with no downtime. + +Each Pod holds a member of a Replica Set, and each Pod has different components, +each one of them in charge of some part of the lifecycle of the MongoDB database. + +# Developing locally + +The operator is built using the latest stable `operator-sdk` and `golang`. We use a simple +json file that describe some local options that you need to set for the testing environment +to be able to run properly. Create a json file with the following content: + +```json +{ + "namespace": "default", + "repo_url": "localhost:5000" +} +``` + +The `namespace` attribute sets the Kubernetes namespace to be used for running +your tests. The `repo_url` sets the Docker registry. In my case I have a +`local-config.json` file in the root of this repo. For the e2e tests to pick +this file, set the `MONGODB_COMMUNITY_CONFIG` env variable to the absolute path +of this file. + +## Configure Docker registry + +The build process consist in multiple Docker images being built, you need to specify +where you want the locally build images to be pushed. The Docker registry needs to be +accessible from your Kubernetes cluster. + +## Test Namespace + +You can change the namespace used for tests, if you are using `Kind`, for +instance, you can leave this as `default`. + +## Python Environment + +The test runner is a Python script, in order to use it a virtualenv needs to be +created. The dependencies of the Python environment are described, as usual, in +a `requirements.txt` file: + +```sh +python -m venv venv +source venv/bin/activate +python -m pip install -r requirements.txt +``` + +# Running Unit tests + +Unit tests should be run from the root of the project with: + +```sh +go test ./pkg/... +``` + +# Running E2E Tests + +## Running an E2E test + +We have built a simple mechanism to run E2E tests on your cluster using a runner +that deploys a series of Kubernetes objects, runs them, and awaits for their +completion. If the objects complete with a Success status, it means that the +tests were run successfully. + +The available tests can be found in the `tests/e2e` directory, at the time of this +writting we have: + +```sh +$ ls -l test/e2e +replica_set +replica_set_change_version +replica_set_readiness_probe +replica_set_scale +... +``` + +The tests should run individually using the runner like this: + +```sh +# python scripts/dev/e2e.py +# for example +python scripts/dev/e2e.py --test replica_set +``` + +This will run the `replica_set` E2E test which is a simple test that installs a +MongoDB Replica Set and asserts that the deployed server can be connected to. + +# Writing new E2E tests + +You can start with `replica_set` test as an starting point to write a new test. +The tests are written using `operator-sdk` and so you can find more information +about how to write tests in the [official operator-sdk +docs](https://sdk.operatorframework.io/docs/golang/e2e-tests/). + +Adding a new test is as easy as to create a new directory in `test/e2e` with the +new E2E test, and to run them: + +```sh +python scripts/dev/e2e.py --test +``` + +# Before Committing your code + +Please make sure you sign our Contributor Agreement +[here](https://www.mongodb.com/legal/contributor-agreement). This will be +required when creating a PR against this repo! diff --git a/requirements.txt b/requirements.txt index d899554ae..d5975d23b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ docker==4.2.0 kubernetes==11.0.0 jinja2==2.11.2 +PyYAML==5.3.1 diff --git a/scripts/ci/run_unit_tests.sh b/scripts/ci/run_unit_tests.sh index c2998647f..ec91f7357 100755 --- a/scripts/ci/run_unit_tests.sh +++ b/scripts/ci/run_unit_tests.sh @@ -1,5 +1,5 @@ #!/bin/sh python scripts/dev/dockerfile_generator.py "unittest" > Dockerfile -docker build . -f Dockerfile -t unit-tests:${version_id} -docker run unit-tests:${version_id} +docker build . -f Dockerfile -t "unit-tests:${version_id:?}" +docker run "unit-tests:${version_id:?}" diff --git a/scripts/dev/dockerutil.py b/scripts/dev/dockerutil.py index 7ce92b2b9..98f4eed40 100644 --- a/scripts/dev/dockerutil.py +++ b/scripts/dev/dockerutil.py @@ -23,7 +23,7 @@ def push_image(tag: str): print(f"Pushing image: {tag}") progress = "" for line in client.images.push(tag, stream=True): - print(push_image_formatted(line), end="") + print("\r" + push_image_formatted(line), end="", flush=True) def push_image_formatted(line) -> str: @@ -37,9 +37,15 @@ def push_image_formatted(line) -> str: if line["status"] in to_skip: return "" if line["status"] == "Pushing": - current = int(line["progressDetail"]["current"]) - total = int(line["progressDetail"]["total"]) - return "Complete: {:.2f}\n".format(current / total) + try: + current = int(line["progressDetail"]["current"]) + total = int(line["progressDetail"]["total"]) + except KeyError: + return "" + progress = current / total + if progress > 1.0: + progress == 1.0 + return "Complete: {:.1%}\n".format(progress) return "" diff --git a/scripts/dev/e2e.py b/scripts/dev/e2e.py index 9cb988b0f..28aa78061 100644 --- a/scripts/dev/e2e.py +++ b/scripts/dev/e2e.py @@ -2,6 +2,8 @@ from kubernetes.client.rest import ApiException from build_and_deploy_operator import ( + build_and_push_operator, + deploy_operator, ignore_if_doesnt_exist, ignore_if_already_exists, load_yaml_from_file, @@ -12,6 +14,8 @@ from kubernetes import client, config import argparse import time +import os +import yaml TEST_RUNNER_NAME = "test-runner" @@ -72,6 +76,29 @@ def _prepare_testrunner_environment(): ) +def create_kube_config(): + """Replicates the local kubeconfig file (pointed at by KUBECONFIG), + as a ConfigMap.""" + corev1 = client.CoreV1Api() + print("Creating kube-config ConfigMap") + + svc = corev1.read_namespaced_service("kubernetes", "default") + kube_config = os.getenv("KUBECONFIG") + with open(kube_config) as fd: + kube_config = yaml.safe_load(fd.read()) + + kube_config["clusters"][0]["cluster"]["server"] = "https://" + svc.spec.cluster_ip + kube_config = yaml.safe_dump(kube_config) + data = {"kubeconfig": kube_config} + config_map = client.V1ConfigMap( + metadata=client.V1ObjectMeta(name="kube-config"), data=data + ) + + ignore_if_already_exists( + lambda: corev1.create_namespaced_config_map("default", config_map) + ) + + def build_and_push_testrunner(repo_url: str, tag: str, path: str): """ build_and_push_testrunner builds and pushes the test runner @@ -161,7 +188,13 @@ def wait_for_pod_to_be_running(corev1, name, namespace): def parse_args(): parser = argparse.ArgumentParser() - parser.add_argument("test", help="Name of the test to run") + parser.add_argument("--test", help="Name of the test to run") + parser.add_argument( + "--skip-operator-install", + help="Do not install the Operator, assumes one is installed already", + type=bool, + default=False, + ) return parser.parse_args() @@ -169,6 +202,16 @@ def main(): args = parse_args() config.load_kube_config() dev_config = load_config() + create_kube_config() + + if not args.skip_operator_install: + build_and_push_operator( + dev_config.repo_url, + f"{dev_config.repo_url}/mongodb-kubernetes-operator", + ".", + ) + deploy_operator() + build_and_push_testrunner( dev_config.repo_url, f"{dev_config.repo_url}/{TEST_RUNNER_NAME}", "." ) diff --git a/scripts/dev/setup_kind_cluster.sh b/scripts/dev/setup_kind_cluster.sh index b974a076a..15e616f2e 100755 --- a/scripts/dev/setup_kind_cluster.sh +++ b/scripts/dev/setup_kind_cluster.sh @@ -13,7 +13,7 @@ if [ "${running}" != 'true' ]; then registry:2 fi -ip="$(docker inspect kind-registry -f {{.NetworkSettings.IPAddress}})" +ip="$(docker inspect kind-registry -f '{{.NetworkSettings.IPAddress}}')" # create a cluster with the local registry enabled in containerd cat <${temp} -contents="$(cat ${temp})" -kubectl create cm kube-config --from-literal=kubeconfig="${contents}" -rm ${temp} From c3da323b22872f57ff1d9ad1146e1982eaa49ee6 Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Fri, 29 May 2020 08:58:17 +0100 Subject: [PATCH 006/790] Cloudp 62906 set owner reference (#49) * CLOUDP-62906: added SetOwnerReference to StatefulSet Builder * CLOUDP-62906: added e2e tests for OwnerReference in StatefulSet * CLOUDP-62906: better testing of OwnerReference in StatefulSet * removed dead code * Improved check of OwnerReference * CLOUDP-62906: fixed plural of variable name * CLOUDP-62906: refactored tests * CLOUDP-62906: Improved e2e testing output * CLOUDP-62906: Fixed test failure instead of test pass * CLOUDP-62906: replaced if-fatal with assert --- pkg/controller/mongodb/mongodb_controller.go | 13 +++++++++- test/e2e/mongodbtests/mongodbtests.go | 27 ++++++++++++++++++++ test/e2e/replica_set/replica_set_test.go | 8 ++++++ 3 files changed, 47 insertions(+), 1 deletion(-) diff --git a/pkg/controller/mongodb/mongodb_controller.go b/pkg/controller/mongodb/mongodb_controller.go index 3c44eabec..1e4c27458 100644 --- a/pkg/controller/mongodb/mongodb_controller.go +++ b/pkg/controller/mongodb/mongodb_controller.go @@ -23,6 +23,7 @@ import ( "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/handler" @@ -149,6 +150,7 @@ func (r *ReplicaSetReconciler) Reconcile(request reconcile.Request) (reconcile.R r.log.Debugf("Ensuring StatefulSet is ready, with type: %s", getUpdateStrategyType(mdb)) if ready, err := r.isStatefulSetReady(mdb, getUpdateStrategyType(mdb)); err != nil { + r.log.Infof("error checking StatefulSet status: %+v", err) return reconcile.Result{}, err } else if !ready { @@ -447,6 +449,14 @@ func buildStatefulSet(mdb mdbv1.MongoDB) (appsv1.StatefulSet, error) { }, } + ownerReferences := []metav1.OwnerReference{ + *metav1.NewControllerRef(&mdb, schema.GroupVersionKind{ + Group: mdbv1.SchemeGroupVersion.Group, + Version: mdbv1.SchemeGroupVersion.Version, + Kind: mdb.Kind, + }), + } + builder := statefulset.NewBuilder(). SetPodTemplateSpec(podSpecTemplate). SetNamespace(mdb.Namespace). @@ -455,7 +465,8 @@ func buildStatefulSet(mdb mdbv1.MongoDB) (appsv1.StatefulSet, error) { SetLabels(labels). SetMatchLabels(labels). SetServiceName(mdb.ServiceName()). - SetUpdateStrategy(getUpdateStrategyType(mdb)) + SetUpdateStrategy(getUpdateStrategyType(mdb)). + SetOwnerReference(ownerReferences) // TODO: Add this section to architecture document. // The design of the multi-container and the different volumes mounted to them is as follows: diff --git a/test/e2e/mongodbtests/mongodbtests.go b/test/e2e/mongodbtests/mongodbtests.go index 09156a5e9..919eb7179 100644 --- a/test/e2e/mongodbtests/mongodbtests.go +++ b/test/e2e/mongodbtests/mongodbtests.go @@ -23,6 +23,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" ) // StatefulSetIsReady ensures that the underlying stateful set @@ -37,6 +38,26 @@ func StatefulSetIsReady(mdb *mdbv1.MongoDB) func(t *testing.T) { } } +func StatefulSetHasOwnerReference(mdb *mdbv1.MongoDB, expectedOwnerReference metav1.OwnerReference) func(t *testing.T) { + return func(t *testing.T) { + sts := appsv1.StatefulSet{} + err := f.Global.Client.Get(context.TODO(), types.NamespacedName{Name: mdb.Name, Namespace: f.Global.OperatorNamespace}, &sts) + if err != nil { + t.Fatal(err) + } + ownerReferences := sts.GetOwnerReferences() + + assert.Len(t, ownerReferences, 1, "StatefulSet doesn't have OwnerReferences") + + assert.Equal(t, expectedOwnerReference.APIVersion, ownerReferences[0].APIVersion) + assert.Equal(t, "MongoDB", ownerReferences[0].Kind) + assert.Equal(t, expectedOwnerReference.Name, ownerReferences[0].Name) + assert.Equal(t, expectedOwnerReference.UID, ownerReferences[0].UID) + + t.Logf("StatefulSet %s/%s has the correct OwnerReference!", mdb.Namespace, mdb.Name) + } +} + // StatefulSetIsUpdated ensures that all the Pods of the StatefulSet are // in "Updated" condition. func StatefulSetIsUpdated(mdb *mdbv1.MongoDB) func(t *testing.T) { @@ -137,6 +158,12 @@ func BasicFunctionality(mdb *mdbv1.MongoDB) func(*testing.T) { t.Run("Config Map Was Correctly Created", AutomationConfigConfigMapExists(mdb)) t.Run("Stateful Set Reaches Ready State", StatefulSetIsReady(mdb)) t.Run("MongoDB Reaches Running Phase", MongoDBReachesRunningPhase(mdb)) + t.Run("Stateful Set has OwnerReference", StatefulSetHasOwnerReference(mdb, + *metav1.NewControllerRef(mdb, schema.GroupVersionKind{ + Group: mdbv1.SchemeGroupVersion.Group, + Version: mdbv1.SchemeGroupVersion.Version, + Kind: mdb.Kind, + }))) t.Run("Test Basic Connectivity", BasicConnectivity(mdb)) t.Run("Test Status Was Updated", Status(mdb, mdbv1.MongoDBStatus{ diff --git a/test/e2e/replica_set/replica_set_test.go b/test/e2e/replica_set/replica_set_test.go index bbe602e20..54bb1a784 100644 --- a/test/e2e/replica_set/replica_set_test.go +++ b/test/e2e/replica_set/replica_set_test.go @@ -7,6 +7,8 @@ import ( e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/mongodbtests" f "github.com/operator-framework/operator-sdk/pkg/test" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" ) func TestMain(m *testing.M) { @@ -24,6 +26,12 @@ func TestReplicaSet(t *testing.T) { t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) t.Run("Config Map Was Correctly Created", mongodbtests.AutomationConfigConfigMapExists(&mdb)) t.Run("Stateful Set Reaches Ready State", mongodbtests.StatefulSetIsReady(&mdb)) + t.Run("Stateful Set has OwnerReference", mongodbtests.StatefulSetHasOwnerReference(&mdb, + *metav1.NewControllerRef(&mdb, schema.GroupVersionKind{ + Group: mdbv1.SchemeGroupVersion.Group, + Version: mdbv1.SchemeGroupVersion.Version, + Kind: mdb.Kind, + }))) t.Run("MongoDB Reaches Running Phase", mongodbtests.MongoDBReachesRunningPhase(&mdb)) t.Run("Test Basic Connectivity", mongodbtests.BasicConnectivity(&mdb)) t.Run("Test Status Was Updated", mongodbtests.Status(&mdb, From 07f847e18ab299828843967c1e533d3b3ec3d0b2 Mon Sep 17 00:00:00 2001 From: Melissa Mahoney Date: Tue, 2 Jun 2020 08:34:11 -0700 Subject: [PATCH 007/790] (DOCSP-10428): Upgrade procedures (#50) * (DOCSP-10428): Upgrade procedures * Remove namespace from cluster-wide command --- README.md | 92 ++++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 85 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index a6cc8879b..0b82895ee 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # MongoDB Community Kubernetes Operator # - + This is a [Kubernetes Operator](https://coreos.com/operators/) which deploys MongoDB Community into Kubernetes clusters. @@ -8,7 +8,20 @@ This codebase is currently _pre-alpha_, and is not ready for use. If you are a MongoDB Enterprise customer, or need Enterprise features such as Backup, you can use the [MongoDB Enterprise Operator for Kubernetes](https://github.com/mongodb/mongodb-enterprise-kubernetes). -## Installation +## Table of Contents + +- [Install the Operator](#install-the-operator) + - [Prerequisites](#prerequisites) + - [Procedure](#procedure) +- [Upgrade the Operator](#upgrade-the-operator) +- [Deploy & Configure MongoDB Resources](#deploy-and-configure-a-mongodb-resource) + - [Deploy a Replica Set](#deploy-a-replica-set) + - [Upgrade MongoDB Version & FCV](#upgrade-your-mongodb-resource-version-and-feature-compatibility-version) +- [Contribute](#contribute) +- [License](#license) + + +## Install the Operator ### Prerequisites @@ -22,7 +35,7 @@ Before you install the MongoDB Community Kubernetes Operator, you must: git clone https://github.com/mongodb/mongodb-kubernetes-operator.git ``` -### Installing the MongoDB Community Kubernetes Operator +### Procedure The MongoDB Community Kubernetes Operator is a [Custom Resource Definition](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/) and a controller. @@ -50,11 +63,23 @@ To install the MongoDB Community Kubernetes Operator: kubectl get pods --namespace ``` -## Usage + +## Upgrade the Operator + +To upgrade the MongoDB Community Kubernetes Operator: + +1. Change to the directory in which you cloned the repository. +2. Invoke the following `kubectl` command to upgrade the [Custom Resource Definitions](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/). + ``` + kubectl apply -f deploy/crds/mongodb.com_mongodb_crd.yaml + ``` + + +## Deploy and Configure a MongoDB Resource The `/deploy/crds` directory contains example MongoDB resources that you can modify and deploy. -### Deploying a MongoDB Resource +### Deploy a Replica Set To deploy your first replica set: @@ -67,15 +92,68 @@ To deploy your first replica set: kubectl get mongodb --namespace ``` -## Contributing +### Upgrade your MongoDB Resource Version and Feature Compatibility Version + +You can upgrade the major, minor, and/or feature compatibility versions of your MongoDB resource. These settings are configured in your resource definition YAML file. + +- To upgrade your resource's major and/or minor versions, set the `spec.version` setting to the desired MongoDB version. + +- To modify your resource's [feature compatibility version](https://docs.mongodb.com/manual/reference/command/setFeatureCompatibilityVersion/), set the `spec.featureCompatibilityVersion` setting to the desired version. + +If you update `spec.version` to a later version, consider setting `spec.featureCompatibilityVersion` to the current working MongoDB version to give yourself the option to downgrade if necessary. To learn more about feature compatibility, see [`setFeatureCompatibilityVersion`](https://docs.mongodb.com/manual/reference/command/setFeatureCompatibilityVersion/) in the MongoDB Manual. + +#### Example + +Consider the following example MongoDB resource definition: + +```yaml +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: example-mongodb +spec: + members: 3 + type: ReplicaSet + version: "4.0.6" +``` +To upgrade this resource from `4.0.6` to `4.2.7`: + +1. Edit the resource definition. + + a. Update `spec.version` to `4.2.7`. + + b. Update `spec.featureCompatibilityVersion` to `4.0`. + + ```yaml + apiVersion: mongodb.com/v1 + kind: MongoDB + metadata: + name: example-mongodb + spec: + members: 3 + type: ReplicaSet + version: "4.2.7" + featureCompatibilityVersion: "4.0" + ``` + + **NOTE:** Setting `featureCompatibilityVersion` to `4.0` disables [4.2 features incompatible with MongoDB 4.0](https://docs.mongodb.com/manual/release-notes/4.2-compatibility/#compatibility-enabled). + +2. Reapply the configuration to Kubernetes: + ``` + kubectl apply -f .yaml --namespace + ``` + + +## Contribute Please get familiar with the architecture.md document and then go ahead and read -the [contributing guide](contributing.md) guide. +the [contributing guide](contributing.md). Please file issues before filing PRs. For PRs to be accepted, contributors must sign our [CLA](https://www.mongodb.com/legal/contributor-agreement). Reviewers, please ensure that the CLA has been signed by referring to [the contributors tool](https://contributors.corp.mongodb.com/) (internal link). + ## License The source code of this Operator is available under the Apache v2 license. From 602f5a2d74f32c9ebe127d3ed90191d8e49cd668 Mon Sep 17 00:00:00 2001 From: Rodrigo Valin Date: Wed, 3 Jun 2020 09:57:26 +0100 Subject: [PATCH 008/790] Release 0.0.5 (#47) --- README.md | 4 ++-- agent/Dockerfile | 16 ++++------------ deploy/crds/mongodb.com_mongodb_crd.yaml | 9 +++++++++ deploy/crds/mongodb.com_v1_mongodb_cr.yaml | 2 +- deploy/operator.yaml | 6 +++--- pkg/apis/mongodb/v1/mongodb_types.go | 2 ++ 6 files changed, 21 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 0b82895ee..530449c48 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ This is a [Kubernetes Operator](https://coreos.com/operators/) which deploys MongoDB Community into Kubernetes clusters. -This codebase is currently _pre-alpha_, and is not ready for use. +This codebase is currently _alpha_, and is not ready for production use. If you are a MongoDB Enterprise customer, or need Enterprise features such as Backup, you can use the [MongoDB Enterprise Operator for Kubernetes](https://github.com/mongodb/mongodb-enterprise-kubernetes). @@ -56,7 +56,7 @@ To install the MongoDB Community Kubernetes Operator: a. Invoke the following `kubectl` command to install the Operator in the specified namespace: ``` - kubectl create -f deploy --namespace + kubectl create -f deploy/ --namespace ``` b. Verify that the Operator installed successsfully: ``` diff --git a/agent/Dockerfile b/agent/Dockerfile index 0aaa78924..e1fb0ff13 100644 --- a/agent/Dockerfile +++ b/agent/Dockerfile @@ -1,9 +1,7 @@ FROM ubuntu:16.04 -# https://jira.mongodb.org/browse/CLOUDP-58488 -# Adds -noDaemonize option -ARG agent_image=https://s3.amazonaws.com/mciuploads/mms-automation/mongodb-mms-build-agent/builds/patches/5e600292d6d80a561311da80/automation-agent/dev/mongodb-mms-automation-agent-10.12.0.6191-1.rhel7_x86_64.tar.gz -ARG agent_version=10.12.0.6191 +ARG agent_image=https://s3.amazonaws.com/mciuploads/mms-automation/mongodb-mms-build-agent/builds/automation-agent/prod/mongodb-mms-automation-agent-10.13.0.6199-1.rhel7_x86_64.tar.gz +ARG agent_version=10.13.0.6199-1 RUN apt-get -qq update \ && apt-get -y -qq install \ @@ -13,19 +11,13 @@ RUN apt-get -qq update \ && rm -rf /var/lib/apt/lists/* RUN mkdir -p agent \ - && curl $agent_image -o agent/mongodb-agent.tar.gz \ + && curl --fail --retry 3 --silent $agent_image -o agent/mongodb-agent.tar.gz \ && tar xfz agent/mongodb-agent.tar.gz \ - && mv mongodb-mms-automation-agent-$agent_version-1.rhel7_x86_64/mongodb-mms-automation-agent agent/mongodb-agent \ + && mv mongodb-mms-automation-agent-$agent_version.rhel7_x86_64/mongodb-mms-automation-agent agent/mongodb-agent \ && chmod +x agent/mongodb-agent \ && mkdir -p /var/lib/automation/config \ && chmod -R +r /var/lib/automation/config -# TODO: this mongo client is used for testing purposes and should be removed in the future -RUN curl -LO http://downloads.mongodb.org/linux/mongodb-linux-x86_64-ubuntu1604-4.0.6.tgz && \ - tar xfz mongodb-linux-x86_64-ubuntu1604-4.0.6.tgz && \ - mv mongodb-linux-x86_64-ubuntu1604-4.0.6/bin/mongo /usr/bin && \ - rm -rf mongodb-linux-x86_64-ubuntu1604-4.0.6.tgz mongodb-linux-x86_64-ubuntu1604-4.0.6 - RUN mkdir -p /var/lib/mongodb-mms-automation/probes/ \ # && curl --retry 3 https://readinessprobe.s3-us-west-1.amazonaws.com/readinessprobe -o /var/lib/mongodb-mms-automation/probes/readinessprobe \ && curl --retry 3 https://readinessprobe-test.s3-us-west-1.amazonaws.com/readiness -o /var/lib/mongodb-mms-automation/probes/readinessprobe \ diff --git a/deploy/crds/mongodb.com_mongodb_crd.yaml b/deploy/crds/mongodb.com_mongodb_crd.yaml index 110682d32..15d439d14 100644 --- a/deploy/crds/mongodb.com_mongodb_crd.yaml +++ b/deploy/crds/mongodb.com_mongodb_crd.yaml @@ -3,6 +3,15 @@ kind: CustomResourceDefinition metadata: name: mongodb.mongodb.com spec: + additionalPrinterColumns: + - JSONPath: .status.phase + description: Current state of the MongoDB deployment + name: Phase + type: string + - JSONPath: .status.version + description: Version of MongoDB server + name: Version + type: string group: mongodb.com names: kind: MongoDB diff --git a/deploy/crds/mongodb.com_v1_mongodb_cr.yaml b/deploy/crds/mongodb.com_v1_mongodb_cr.yaml index 3464de36a..a1bc63db2 100644 --- a/deploy/crds/mongodb.com_v1_mongodb_cr.yaml +++ b/deploy/crds/mongodb.com_v1_mongodb_cr.yaml @@ -5,4 +5,4 @@ metadata: spec: members: 3 type: ReplicaSet - version: "4.0.6" + version: "4.2.6" diff --git a/deploy/operator.yaml b/deploy/operator.yaml index 5c2a1001a..effa0efea 100644 --- a/deploy/operator.yaml +++ b/deploy/operator.yaml @@ -15,7 +15,7 @@ spec: serviceAccountName: mongodb-kubernetes-operator containers: - name: mongodb-kubernetes-operator - image: quay.io/chatton/mongodb-kubernetes-operator + image: quay.io/mongodb/mongodb-kubernetes-operator command: - mongodb-kubernetes-operator imagePullPolicy: Always @@ -31,6 +31,6 @@ spec: - name: OPERATOR_NAME value: "mongodb-kubernetes-operator" - name: AGENT_IMAGE # The MongoDB Agent the operator will deploy to manage MongoDB deployments - value: quay.io/chatton/mongodb-agent + value: quay.io/mongodb/mongodb-agent:10.13.0.6199-1 - name: PRE_STOP_HOOK_IMAGE - value: quay.io/mongodb/community-operator-pre-stop-hook + value: quay.io/mongodb/mongodb-kubernetes-operator-pre-stop-hook:1.0.0 diff --git a/pkg/apis/mongodb/v1/mongodb_types.go b/pkg/apis/mongodb/v1/mongodb_types.go index b5d6e1676..6ec3a21ec 100644 --- a/pkg/apis/mongodb/v1/mongodb_types.go +++ b/pkg/apis/mongodb/v1/mongodb_types.go @@ -52,6 +52,8 @@ type MongoDBStatus struct { // MongoDB is the Schema for the mongodbs API // +kubebuilder:subresource:status // +kubebuilder:resource:path=mongodb,scope=Namespaced,shortName=mdb +// +kubebuilder:printcolumn:name="Phase",type="string",JSONPath=".status.phase",description="Current state of the MongoDB deployment" +// +kubebuilder:printcolumn:name="Version",type="string",JSONPath=".status.version",description="Version of MongoDB server" type MongoDB struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` From b7ece020edba239f0ac4cd20e0e2421f4cd02a88 Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Thu, 4 Jun 2020 08:55:28 +0100 Subject: [PATCH 009/790] CLOUDP-62351: update service (#52) * CLOUDP-62531: update service now sets correctly the resoruce version * CLOUDP-62531: better error messages and removed update of service * CLOUDP-62531: simplified code to ensure a service is running * CLOUDP-62531: Fixed missing return statement * CLOUDP-62531: Added unit test of service creation/update * amend * fixed typo --- pkg/controller/mongodb/mongodb_controller.go | 18 ++++++++++++---- .../mongodb/replicaset_controller_test.go | 21 +++++++++++++++++++ pkg/kube/client/client.go | 15 ++++++++++--- 3 files changed, 47 insertions(+), 7 deletions(-) diff --git a/pkg/controller/mongodb/mongodb_controller.go b/pkg/controller/mongodb/mongodb_controller.go index 1e4c27458..9d429626a 100644 --- a/pkg/controller/mongodb/mongodb_controller.go +++ b/pkg/controller/mongodb/mongodb_controller.go @@ -136,10 +136,10 @@ func (r *ReplicaSetReconciler) Reconcile(request reconcile.Request) (reconcile.R return reconcile.Result{}, err } - r.log.Debug("Building service") - svc := buildService(mdb) - if err = r.client.CreateOrUpdate(&svc); err != nil { - r.log.Infof("The service already exists... moving forward: %s", err) + r.log.Debug("Ensuring the service exists") + if err := r.ensureService(mdb); err != nil { + r.log.Infof("Error ensuring the service exists: %s", err) + return reconcile.Result{}, err } r.log.Debug("Creating/Updating StatefulSet") @@ -205,6 +205,16 @@ func (r *ReplicaSetReconciler) isStatefulSetReady(mdb mdbv1.MongoDB, expectedUpd return statefulset.IsReady(set, mdb.Spec.Members) && expectedUpdateStrategy == set.Spec.UpdateStrategy.Type, nil } +func (r *ReplicaSetReconciler) ensureService(mdb mdbv1.MongoDB) error { + svc := buildService(mdb) + err := r.client.Create(context.TODO(), &svc) + if err != nil && errors.IsAlreadyExists(err) { + r.log.Infof("The service already exists... moving forward: %s", err) + return nil + } + return err +} + func (r *ReplicaSetReconciler) createOrUpdateStatefulSet(mdb mdbv1.MongoDB) error { sts, err := buildStatefulSet(mdb) if err != nil { diff --git a/pkg/controller/mongodb/replicaset_controller_test.go b/pkg/controller/mongodb/replicaset_controller_test.go index 363529b37..c2c5f6ac4 100644 --- a/pkg/controller/mongodb/replicaset_controller_test.go +++ b/pkg/controller/mongodb/replicaset_controller_test.go @@ -162,3 +162,24 @@ func TestBuildStatefulSet_ConfiguresUpdateStrategyCorrectly(t *testing.T) { assert.Equal(t, appsv1.OnDeleteStatefulSetStrategyType, sts.Spec.UpdateStrategy.Type) }) } + +func TestService_isCorrectlyCreatedAndUpdated(t *testing.T) { + mdb := newTestReplicaSet() + + mgr := client.NewManager(&mdb) + r := newReconciler(mgr, mockManifestProvider(mdb.Spec.Version)) + res, err := r.Reconcile(reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) + assertReconciliationSuccessful(t, res, err) + + svc := corev1.Service{} + err = mgr.GetClient().Get(context.TODO(), types.NamespacedName{Name: mdb.ServiceName(), Namespace: mdb.Namespace}, &svc) + assert.NoError(t, err) + assert.Equal(t, svc.Spec.Type, corev1.ServiceTypeClusterIP) + assert.Equal(t, svc.Spec.Selector["app"], mdb.ServiceName()) + assert.Len(t, svc.Spec.Ports, 1) + assert.Equal(t, svc.Spec.Ports[0], corev1.ServicePort{Port: 27017}) + + res, err = r.Reconcile(reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) + assertReconciliationSuccessful(t, res, err) + +} diff --git a/pkg/kube/client/client.go b/pkg/kube/client/client.go index b68e4ea05..cc39c48c0 100644 --- a/pkg/kube/client/client.go +++ b/pkg/kube/client/client.go @@ -2,6 +2,7 @@ package client import ( "context" + "fmt" "reflect" "k8s.io/apimachinery/pkg/api/errors" @@ -33,11 +34,19 @@ func (c client) CreateOrUpdate(obj runtime.Object) error { err := c.Get(context.TODO(), namespacedNameFromObject(obj), objCopy) if err != nil { if errors.IsNotFound(err) { - return c.Create(context.TODO(), obj) + err = c.Create(context.TODO(), obj) + if err != nil { + return fmt.Errorf("Error creating the object: %s", err) + } + return err } - return err + return fmt.Errorf("Error getting the object: %s", err) } - return c.Update(context.TODO(), obj) + err = c.Update(context.TODO(), obj) + if err != nil { + return fmt.Errorf("Error updating the object: %s", err) + } + return err } // GetAndUpdate fetches the most recent version of the runtime.Object with the provided From 7e3e50a8a2a970a2091231e2a11cc0bd8036315c Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Thu, 4 Jun 2020 14:06:24 +0100 Subject: [PATCH 010/790] CLOUDP-64702: fix status log message (#54) * CLOUDP-64702: fixed status in log message * CLOUDP-64702: Updated function name --- pkg/controller/mongodb/mongodb_controller.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/pkg/controller/mongodb/mongodb_controller.go b/pkg/controller/mongodb/mongodb_controller.go index 9d429626a..8b8b03c53 100644 --- a/pkg/controller/mongodb/mongodb_controller.go +++ b/pkg/controller/mongodb/mongodb_controller.go @@ -171,12 +171,13 @@ func (r *ReplicaSetReconciler) Reconcile(request reconcile.Request) (reconcile.R } r.log.Debug("Updating MongoDB Status") - if err := r.updateStatusSuccess(&mdb); err != nil { + newStatus, err := r.updateAndReturnStatusSuccess(&mdb) + if err != nil { r.log.Infof("Error updating the status of the MongoDB resource: %+v", err) return reconcile.Result{}, err } - r.log.Infow("Successfully finished reconciliation", "MongoDB.Spec:", mdb.Spec, "MongoDB.Status", mdb.Status) + r.log.Infow("Successfully finished reconciliation", "MongoDB.Spec:", mdb.Spec, "MongoDB.Status", newStatus) return reconcile.Result{}, nil } @@ -244,19 +245,19 @@ func (r ReplicaSetReconciler) setAnnotation(nsName types.NamespacedName, key, va }) } -// updateStatusSuccess should be called after a successful reconciliation +// updateAndReturnStatusSuccess should be called after a successful reconciliation // the resource's status is updated to reflect to the state, and any other cleanup // operators should be performed here -func (r ReplicaSetReconciler) updateStatusSuccess(mdb *mdbv1.MongoDB) error { +func (r ReplicaSetReconciler) updateAndReturnStatusSuccess(mdb *mdbv1.MongoDB) (mdbv1.MongoDBStatus, error) { newMdb := &mdbv1.MongoDB{} if err := r.client.Get(context.TODO(), types.NamespacedName{Name: mdb.Name, Namespace: mdb.Namespace}, newMdb); err != nil { - return fmt.Errorf("error getting resource: %+v", err) + return mdbv1.MongoDBStatus{}, fmt.Errorf("error getting resource: %+v", err) } newMdb.UpdateSuccess() if err := r.client.Status().Update(context.TODO(), newMdb); err != nil { - return fmt.Errorf("error updating status: %+v", err) + return mdbv1.MongoDBStatus{}, fmt.Errorf("error updating status: %+v", err) } - return nil + return newMdb.Status, nil } func (r ReplicaSetReconciler) ensureAutomationConfig(mdb mdbv1.MongoDB) error { From 4184c5a4d6e525913058ab6e38d4a7a7ec299ab0 Mon Sep 17 00:00:00 2001 From: Melissa Mahoney Date: Thu, 4 Jun 2020 07:26:47 -0700 Subject: [PATCH 011/790] (DOCSP-10429): Add supported features (#51) * (DOCSP-10429): Add supported features * Add to TOC * Add link to CRD --- README.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/README.md b/README.md index 530449c48..2c8755f90 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ If you are a MongoDB Enterprise customer, or need Enterprise features such as Ba - [Deploy & Configure MongoDB Resources](#deploy-and-configure-a-mongodb-resource) - [Deploy a Replica Set](#deploy-a-replica-set) - [Upgrade MongoDB Version & FCV](#upgrade-your-mongodb-resource-version-and-feature-compatibility-version) +- [Supported Features](#supported-features) - [Contribute](#contribute) - [License](#license) @@ -143,6 +144,22 @@ To upgrade this resource from `4.0.6` to `4.2.7`: kubectl apply -f .yaml --namespace ``` +## Supported Features + +The MongoDB Community Kubernetes Operator supports the following features: + +- MongoDB Topology: [replica sets](https://docs.mongodb.com/manual/replication/) +- Upgrading and downgrading MongoDB server version +- Scaling replica sets up and down +- Reading from and writing to the replica set while scaling, upgrading, and downgrading. These operations are done in an "always up" manner. +- Reporting of MongoDB server state via the [MongoDB resource](/deploy/crds/mongodb.com_mongodb_crd.yaml) `status` field +- Use of any of the available [Docker MongoDB images](https://hub.docker.com/_/mongo/) +- Clients inside the Kubernetes cluster can connect to the replica set (no external connectivity) + +### Planned Features +- TLS support for client/server communication +- Server internal authentication via keyfile +- Creating users with SCRAM-SHA authentication ## Contribute From 643db47559eba788df522b36300c40f2bc75c17f Mon Sep 17 00:00:00 2001 From: Melissa Mahoney Date: Fri, 5 Jun 2020 10:32:18 -0700 Subject: [PATCH 012/790] (DOCSP-10035): Add architecture docs (#53) * (DOCSP-10035): Add architecture docs * Cleanup * Tech review * Copy review --- README.md | 12 +++---- architecture.md | 89 +++++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 87 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 2c8755f90..2d3477f05 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,6 @@ If you are a MongoDB Enterprise customer, or need Enterprise features such as Ba - [Contribute](#contribute) - [License](#license) - ## Install the Operator ### Prerequisites @@ -64,7 +63,6 @@ To install the MongoDB Community Kubernetes Operator: kubectl get pods --namespace ``` - ## Upgrade the Operator To upgrade the MongoDB Community Kubernetes Operator: @@ -75,10 +73,9 @@ To upgrade the MongoDB Community Kubernetes Operator: kubectl apply -f deploy/crds/mongodb.com_mongodb_crd.yaml ``` - ## Deploy and Configure a MongoDB Resource -The `/deploy/crds` directory contains example MongoDB resources that you can modify and deploy. +The [`/deploy/crds`](deploy/crds) directory contains example MongoDB resources that you can modify and deploy. ### Deploy a Replica Set @@ -163,14 +160,15 @@ The MongoDB Community Kubernetes Operator supports the following features: ## Contribute -Please get familiar with the architecture.md document and then go ahead and read -the [contributing guide](contributing.md). +Before you contribute to the MongoDB Community Kubernetes Operator, please read: + +- [MongoDB Community Kubernetes Operator Architecture](architecture.md) +- [Contributing to MongoDB Community Kubernetes Operator](contributing.md) Please file issues before filing PRs. For PRs to be accepted, contributors must sign our [CLA](https://www.mongodb.com/legal/contributor-agreement). Reviewers, please ensure that the CLA has been signed by referring to [the contributors tool](https://contributors.corp.mongodb.com/) (internal link). - ## License The source code of this Operator is available under the Apache v2 license. diff --git a/architecture.md b/architecture.md index 3d5801169..e5081bcdb 100644 --- a/architecture.md +++ b/architecture.md @@ -1,11 +1,86 @@ -# MongoDB Community Operator Architecture +# MongoDB Community Kubernetes Operator Architecture -**Work in Progress** +The MongoDB Community Kubernetes Operator is a [Custom Resource Definition](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/) and a [Controller](https://kubernetes.io/docs/concepts/architecture/controller/). -## Topics +## Table of Contents -* Automation Agent -* Cluster Configuration and ConfigMap -* MongoDB Docker Images -* Example case: MongoDB version upgrade +- [Cluster Configuration](#cluster-configuration) +- [Example: MongoDB Version Upgrade](#example-mongodb-version-upgrade) +- [MongoDB Docker Images](#mongodb-docker-images) +## Cluster Configuration + +You create and update MongoDB resources by defining a MongoDB resource definition. When you apply the MongoDB resource definition to your Kubernetes environment, the Operator: + +1. Creates a [StatefulSet](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/) that contains one [pod](https://kubernetes.io/docs/concepts/workloads/pods/pod-overview/) for each [replica set](https://docs.mongodb.com/manual/replication/) member. +1. Creates two [containers](https://kubernetes.io/docs/concepts/containers/overview/) in each pod: + + - A container for the [`mongod`](https://docs.mongodb.com/manual/reference/program/mongod/index.html) process binary.
+ `mongod` is the primary daemon process for the MongoDB system. It handles data requests, manages data access, and performs background management operations. + + - A container for the MongoDB Agent.
+ The Automation function of the MongoDB Agent handles configuring, stopping, and restarting the `mongod` process. The MongoDB Agent periodically polls the `mongod` to determine status and can deploy changes as needed. +1. Writes the Automation configuration as a [ConfigMap](https://kubernetes.io/docs/concepts/configuration/configmap/) and mounts it to each pod. +1. Initiates the MongoDB Agent, which in turn creates the database configuration and launches the `mongod` process according to your [MongoDB resource definition](deploy/crds/mongodb.com_v1_mongodb_cr.yaml). + + + +This architecture maximizes use of the MongoDB Agent while integrating naturally with Kubernetes to produce a number of benefits. + +- The database container is not tied to the lifecycle of the Agent container or to the Operator, so you can: + - Use your preferred Linux distribution inside the container. + - Update operating system packages on your own schedule. + - Upgrade the Operator or Agent without affecting the database image or uptime of the MongoDB servers. +- Containers are immutable and have a single responsibility or process, so you can: + - Describe and understand each container. + - Configure resources independently for easier debugging and triage. + - Inspect resources independently, including tailing the logs. + - Expose the state of each container. +- Pods are defined as StatefulSets so they benefit from stable identities. +- You can upgrade the Operator without restarting either the database or the MongoDB Agent containers. +- You can set up a MongoDB Kubernetes cluster offline once you download the Docker containers for the database and MongoDB Agent. + +## Example: MongoDB Version Upgrade + +The MongoDB Community Kubernetes Operator uses the Automation function of the MongoDB Agent to efficiently handle rolling upgrades. The Operator configures the StatefulSet to block Kubernetes from performing native rolling upgrades because the native process can trigger multiple re-elections in your MongoDB cluster. + +When you update the MongoDB version in your resource definition and reapply it to your Kubernetes environment, the Operator initiates a rolling upgrade: + +1. The Operator changes the StatefulSet [update strategy](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#update-strategies) from `RollingUpdate` to `OnDelete`. + +1. The Operator updates the [image](https://kubernetes.io/docs/concepts/containers/images/) specification to the new version of MongoDB and writes a new Automation configuration ConfigMap to each pod. + +1. The MongoDB Agent chooses the first pod to upgrade and stops the `mongod` process using a local connection and [`db.shutdownServer`](https://docs.mongodb.com/manual/reference/method/db.shutdownServer/#db.shutdownServer). + +1. A pre-stop hook on the database container checks the state of the MongoDB Agent. If the MongoDB Agent expects the `mongod` process to start with a new version, the hook uses a Kubernetes API call to delete the pod. + +1. The Kubernetes Controller downloads the target version of MongoDB from its default docker registry and restarts the pod with the target version of `mongod` in the database container. + +1. The MongoDB Agent starts. It checks the target version of the new `mongod`, then generates the configuration file for the `mongod` process. + +1. The `mongod` process receives the configuration file from the MongoDB Agent and starts. + +1. The MongoDB Agent reaches goal state. + +1. The MongoDB Agent chooses the next pod to upgrade and repeats the process until all pods are upgraded. + +1. The Operator changes the StatefulSet update strategy from `OnDelete` back to `RollingUpdate`. + + + +This upgrade process allows the MongoDB Agent to: + +- Perform pre-conditions. +- Upgrade the secondaries first. +- Wait for the secondaries' oplogs to catch up before triggering an election. +- Upgrade quickly for large replica sets. +- Consider voting nodes. +- Ensure a replica set is always available throughout the entire upgrade process. + +## MongoDB Docker Images + +MongoDB images are available on [Docker Hub](https://hub.docker.com/_/mongo?tab=tags&page=1&ordering=last_updated). From fcb640e7d117b12e6a2bff5a7fc4a1d9c44a225a Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Mon, 8 Jun 2020 12:27:28 +0100 Subject: [PATCH 013/790] CLOUDP-64640: fix deploy operator (#55) --- scripts/dev/build_and_deploy_operator.py | 40 +++++--------- scripts/dev/e2e.py | 47 +++++++++++----- scripts/dev/k8sutil.py | 70 ++++++++++++++++++++++++ 3 files changed, 115 insertions(+), 42 deletions(-) create mode 100644 scripts/dev/k8sutil.py diff --git a/scripts/dev/build_and_deploy_operator.py b/scripts/dev/build_and_deploy_operator.py index 3517b3662..d2412ba2e 100644 --- a/scripts/dev/build_and_deploy_operator.py +++ b/scripts/dev/build_and_deploy_operator.py @@ -1,16 +1,16 @@ import io import os -import time from typing import Dict, Optional import yaml from kubernetes import client, config -from kubernetes.client.rest import ApiException from dev_config import DevConfig, load_config from dockerfile_generator import render from dockerutil import build_and_push_image +from k8sutil import wait_for_condition, ignore_if_already_exists, ignore_if_doesnt_exist + def _load_operator_service_account() -> Optional[Dict]: return load_yaml_from_file("deploy/service_account.yaml") @@ -51,6 +51,17 @@ def _ensure_crds(): lambda: crdv1.delete_custom_resource_definition("mongodb.mongodb.com") ) + # Make sure that the CRD has being deleted before trying to create it again + if not wait_for_condition( + lambda: crdv1.list_custom_resource_definition( + field_selector="metadata.name==mongodb.mongodb.com" + ), + lambda crd_list: len(crd_list.items) == 0, + timeout=5, + sleep_time=0.5, + ): + raise Exception("Execution timed out while waiting for the CRD to be deleted") + # TODO: fix this, when calling create_custom_resource_definition, we get the error # ValueError("Invalid value for `conditions`, must not be `None`") # but the crd is still successfully created @@ -70,31 +81,6 @@ def build_and_push_operator(repo_url: str, tag: str, path: str): return build_and_push_image(repo_url, tag, path, "operator") -def _ignore_error_codes(fn, codes): - try: - fn() - except ApiException as e: - if e.status not in codes: - raise - - -def ignore_if_already_exists(fn): - """ - ignore_if_already_exists accepts a function and calls it, - ignoring an Kubernetes API conflict errors - """ - - return _ignore_error_codes(fn, [409]) - - -def ignore_if_doesnt_exist(fn): - """ - ignore_if_doesnt_exist accepts a function and calls it, - ignoring an Kubernetes API not found errors - """ - return _ignore_error_codes(fn, [404]) - - def deploy_operator(): """ deploy_operator ensures the CRDs are created, and als creates all the required ServiceAccounts, Roles diff --git a/scripts/dev/e2e.py b/scripts/dev/e2e.py index 28aa78061..09e58ea95 100644 --- a/scripts/dev/e2e.py +++ b/scripts/dev/e2e.py @@ -4,10 +4,13 @@ from build_and_deploy_operator import ( build_and_push_operator, deploy_operator, + load_yaml_from_file, +) +from k8sutil import ( + wait_for_condition, ignore_if_doesnt_exist, ignore_if_already_exists, - load_yaml_from_file, -) # TODO: put these function somewhere else +) from dockerutil import build_and_push_image from typing import Dict, Optional from dev_config import load_config @@ -140,9 +143,35 @@ def create_test_runner_pod(test: str): dev_config = load_config() corev1 = client.CoreV1Api() pod_body = _get_testrunner_pod_body(test) + + if not wait_for_condition( + lambda: corev1.list_namespaced_pod( + dev_config.namespace, field_selector=f"metadata.name=={TEST_RUNNER_NAME}" + ), + lambda pod_list: len(pod_list.items) == 0, + timeout=10, + sleep_time=0.5, + ): + + raise Exception( + "Execution timed out while waiting for the existing pod to be deleted" + ) + return corev1.create_namespaced_pod(dev_config.namespace, body=pod_body) +def wait_for_pod_to_be_running(corev1, name, namespace): + print("Waiting for pod to be running") + if not wait_for_condition( + lambda: corev1.read_namespaced_pod(name, namespace), + lambda pod: pod.status.phase == "Running", + sleep_time=5, + timeout=50, + exceptions_to_ignore=ApiException, + ): + raise Exception("Pod never got into Running state!") + + def _get_testrunner_pod_body(test: str) -> Dict: dev_config = load_config() return { @@ -173,19 +202,6 @@ def _get_testrunner_pod_body(test: str) -> Dict: } -def wait_for_pod_to_be_running(corev1, name, namespace): - print("Waiting for pod to be running") - for i in range(10): - try: - pod = corev1.read_namespaced_pod(name, namespace) - if pod.status.phase == "Running": - return True - except ApiException as e: - pass - time.sleep(5) - raise Exception("Pod never got into Running state!") - - def parse_args(): parser = argparse.ArgumentParser() parser.add_argument("--test", help="Name of the test to run") @@ -222,6 +238,7 @@ def main(): pod = create_test_runner_pod(args.test) corev1 = client.CoreV1Api() + wait_for_pod_to_be_running(corev1, TEST_RUNNER_NAME, dev_config.namespace) # stream all of the pod output as the pod is running diff --git a/scripts/dev/k8sutil.py b/scripts/dev/k8sutil.py new file mode 100644 index 000000000..8bb6dc637 --- /dev/null +++ b/scripts/dev/k8sutil.py @@ -0,0 +1,70 @@ +import time + +from kubernetes.client.rest import ApiException + +# time to sleep between retries +SLEEP_TIME = 2 +# no timeout (loop forever) +INFINITY = -1 + + +def _current_milliseconds() -> int: + return int(round(time.time() * 1000)) + + +def wait_for_condition( + fn, + condition, + exceptions_to_ignore=None, + codes_to_ignore=None, + sleep_time=SLEEP_TIME, + timeout=INFINITY, +) -> bool: + """ + wait_for_condition accepts a function fn and a function condition, + it periodically calls the function fn and then applies the condition function on the result + until it returns True or we reach timeout + + exceptions_to_ignore is a tuple of Exceptions to ignore is raised by the call to fn + If ApiException is not ignored, if raised by the call to fn codes in codes_to_ignore are ignored + """ + start_time = _current_milliseconds() + end = start_time + (timeout * 1000) + + while _current_milliseconds() < end or timeout <= 0: + res = None + try: + res = _ignore_error_codes(fn, codes_to_ignore) + except exceptions_to_ignore: + pass + if res is not None and condition(res): + return True + + time.sleep(sleep_time) + + return False + + +def _ignore_error_codes(fn, codes): + try: + return fn() + except ApiException as e: + if e.status not in codes: + raise + + +def ignore_if_already_exists(fn): + """ + ignore_if_already_exists accepts a function and calls it, + ignoring an Kubernetes API conflict errors + """ + + return _ignore_error_codes(fn, [409]) + + +def ignore_if_doesnt_exist(fn): + """ + ignore_if_doesnt_exist accepts a function and calls it, + ignoring an Kubernetes API not found errors + """ + return _ignore_error_codes(fn, [404]) From eeff12152ceaa10d536254fb384ef5f2a79aca05 Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Mon, 8 Jun 2020 16:15:25 +0100 Subject: [PATCH 014/790] CLOUDP-64640: refactoring of k8sutils (#56) * CLOUDP-64640: refactoring of k8sutils --- scripts/dev/build_and_deploy_operator.py | 14 ++++++------ scripts/dev/e2e.py | 22 ++++++++----------- scripts/dev/{k8sutil.py => k8s_conditions.py} | 2 +- 3 files changed, 17 insertions(+), 21 deletions(-) rename scripts/dev/{k8sutil.py => k8s_conditions.py} (98%) diff --git a/scripts/dev/build_and_deploy_operator.py b/scripts/dev/build_and_deploy_operator.py index d2412ba2e..94cd79d8d 100644 --- a/scripts/dev/build_and_deploy_operator.py +++ b/scripts/dev/build_and_deploy_operator.py @@ -9,7 +9,7 @@ from dockerfile_generator import render from dockerutil import build_and_push_image -from k8sutil import wait_for_condition, ignore_if_already_exists, ignore_if_doesnt_exist +import k8s_conditions def _load_operator_service_account() -> Optional[Dict]: @@ -47,12 +47,12 @@ def _ensure_crds(): crdv1 = client.ApiextensionsV1beta1Api() crd = _load_mongodb_crd() - ignore_if_doesnt_exist( + k8s_conditions.ignore_if_doesnt_exist( lambda: crdv1.delete_custom_resource_definition("mongodb.mongodb.com") ) # Make sure that the CRD has being deleted before trying to create it again - if not wait_for_condition( + if not k8s_conditions.wait( lambda: crdv1.list_custom_resource_definition( field_selector="metadata.name==mongodb.mongodb.com" ), @@ -93,22 +93,22 @@ def deploy_operator(): dev_config = load_config() _ensure_crds() - ignore_if_already_exists( + k8s_conditions.ignore_if_already_exists( lambda: rbacv1.create_namespaced_role( dev_config.namespace, _load_operator_role() ) ) - ignore_if_already_exists( + k8s_conditions.ignore_if_already_exists( lambda: rbacv1.create_namespaced_role_binding( dev_config.namespace, _load_operator_role_binding() ) ) - ignore_if_already_exists( + k8s_conditions.ignore_if_already_exists( lambda: corev1.create_namespaced_service_account( dev_config.namespace, _load_operator_service_account() ) ) - ignore_if_already_exists( + k8s_conditions.ignore_if_already_exists( lambda: appsv1.create_namespaced_deployment( dev_config.namespace, _load_operator_deployment( diff --git a/scripts/dev/e2e.py b/scripts/dev/e2e.py index 09e58ea95..38b7a5892 100644 --- a/scripts/dev/e2e.py +++ b/scripts/dev/e2e.py @@ -6,11 +6,7 @@ deploy_operator, load_yaml_from_file, ) -from k8sutil import ( - wait_for_condition, - ignore_if_doesnt_exist, - ignore_if_already_exists, -) +import k8s_conditions from dockerutil import build_and_push_image from typing import Dict, Optional from dev_config import load_config @@ -51,28 +47,28 @@ def _prepare_testrunner_environment(): _delete_testrunner_pod() print("Creating Role") - ignore_if_already_exists( + k8s_conditions.ignore_if_already_exists( lambda: rbacv1.create_namespaced_role( dev_config.namespace, _load_testrunner_role() ) ) print("Creating Role Binding") - ignore_if_already_exists( + k8s_conditions.ignore_if_already_exists( lambda: rbacv1.create_namespaced_role_binding( dev_config.namespace, _load_testrunner_role_binding() ) ) print("Creating Cluster Role Binding") - ignore_if_already_exists( + k8s_conditions.ignore_if_already_exists( lambda: rbacv1.create_cluster_role_binding( _load_testrunner_cluster_role_binding() ) ) print("Creating ServiceAccount") - ignore_if_already_exists( + k8s_conditions.ignore_if_already_exists( lambda: corev1.create_namespaced_service_account( dev_config.namespace, _load_testrunner_service_account() ) @@ -97,7 +93,7 @@ def create_kube_config(): metadata=client.V1ObjectMeta(name="kube-config"), data=data ) - ignore_if_already_exists( + k8s_conditions.ignore_if_already_exists( lambda: corev1.create_namespaced_config_map("default", config_map) ) @@ -131,7 +127,7 @@ def _delete_testrunner_pod() -> None: """ dev_config = load_config() corev1 = client.CoreV1Api() - ignore_if_doesnt_exist( + k8s_conditions.ignore_if_doesnt_exist( lambda: corev1.delete_namespaced_pod(TEST_RUNNER_NAME, dev_config.namespace) ) @@ -144,7 +140,7 @@ def create_test_runner_pod(test: str): corev1 = client.CoreV1Api() pod_body = _get_testrunner_pod_body(test) - if not wait_for_condition( + if not k8s_conditions.wait( lambda: corev1.list_namespaced_pod( dev_config.namespace, field_selector=f"metadata.name=={TEST_RUNNER_NAME}" ), @@ -162,7 +158,7 @@ def create_test_runner_pod(test: str): def wait_for_pod_to_be_running(corev1, name, namespace): print("Waiting for pod to be running") - if not wait_for_condition( + if not k8s_conditions.wait( lambda: corev1.read_namespaced_pod(name, namespace), lambda pod: pod.status.phase == "Running", sleep_time=5, diff --git a/scripts/dev/k8sutil.py b/scripts/dev/k8s_conditions.py similarity index 98% rename from scripts/dev/k8sutil.py rename to scripts/dev/k8s_conditions.py index 8bb6dc637..4c3a90b7f 100644 --- a/scripts/dev/k8sutil.py +++ b/scripts/dev/k8s_conditions.py @@ -12,7 +12,7 @@ def _current_milliseconds() -> int: return int(round(time.time() * 1000)) -def wait_for_condition( +def wait( fn, condition, exceptions_to_ignore=None, From 865b24c11837135392ec669d1a001e7a7b4b5633 Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Tue, 9 Jun 2020 14:49:40 +0100 Subject: [PATCH 015/790] CLOUDP-65077: Release 0.0.6 (#57) --- README.md | 2 -- deploy/operator.yaml | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index 2d3477f05..96e9c7ad4 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,6 @@ This is a [Kubernetes Operator](https://coreos.com/operators/) which deploys MongoDB Community into Kubernetes clusters. -This codebase is currently _alpha_, and is not ready for production use. - If you are a MongoDB Enterprise customer, or need Enterprise features such as Backup, you can use the [MongoDB Enterprise Operator for Kubernetes](https://github.com/mongodb/mongodb-enterprise-kubernetes). ## Table of Contents diff --git a/deploy/operator.yaml b/deploy/operator.yaml index effa0efea..c77e87bb6 100644 --- a/deploy/operator.yaml +++ b/deploy/operator.yaml @@ -15,7 +15,7 @@ spec: serviceAccountName: mongodb-kubernetes-operator containers: - name: mongodb-kubernetes-operator - image: quay.io/mongodb/mongodb-kubernetes-operator + image: quay.io/mongodb/mongodb-kubernetes-operator:0.0.6 command: - mongodb-kubernetes-operator imagePullPolicy: Always From 58f59111536b37afd23b49c7fef6d2ac841ce224 Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Tue, 16 Jun 2020 10:36:10 +0100 Subject: [PATCH 016/790] Ensure enum values for type (#59) --- deploy/crds/mongodb.com_mongodb_crd.yaml | 2 ++ pkg/apis/mongodb/v1/mongodb_types.go | 1 + 2 files changed, 3 insertions(+) diff --git a/deploy/crds/mongodb.com_mongodb_crd.yaml b/deploy/crds/mongodb.com_mongodb_crd.yaml index 15d439d14..c5ad64486 100644 --- a/deploy/crds/mongodb.com_mongodb_crd.yaml +++ b/deploy/crds/mongodb.com_mongodb_crd.yaml @@ -52,6 +52,8 @@ spec: type: description: Type defines which type of MongoDB deployment the resource should create + enum: + - ReplicaSet type: string version: description: Version defines which version of MongoDB will be used diff --git a/pkg/apis/mongodb/v1/mongodb_types.go b/pkg/apis/mongodb/v1/mongodb_types.go index 6ec3a21ec..b853baa77 100644 --- a/pkg/apis/mongodb/v1/mongodb_types.go +++ b/pkg/apis/mongodb/v1/mongodb_types.go @@ -31,6 +31,7 @@ type MongoDBSpec struct { // +optional Members int `json:"members"` // Type defines which type of MongoDB deployment the resource should create + // +kubebuilder:validation:Enum=ReplicaSet Type Type `json:"type"` // Version defines which version of MongoDB will be used Version string `json:"version"` From 85ad151e3fc114ceee706dcb61d19e2bb7f5d581 Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Tue, 16 Jun 2020 19:23:29 +0100 Subject: [PATCH 017/790] CLOUDP-64701: ensure automation config version bump (#60) --- .../automation_config_builder.go | 35 ++++++++++++--- .../automation_config_test.go | 24 +++++------ pkg/controller/mongodb/mongodb_controller.go | 40 ++++++++++++++--- .../mongodb/replicaset_controller_test.go | 43 +++++++++++++++++++ test/e2e/mongodbtests/mongodbtests.go | 14 ++++++ test/e2e/replica_set/replica_set_test.go | 1 + .../replica_set_test.go | 5 +++ .../replica_set_multiple_test.go | 5 +++ .../replica_set_readiness_probe_test.go | 1 + .../replica_set_scaling_test.go | 3 ++ 10 files changed, 147 insertions(+), 24 deletions(-) diff --git a/pkg/automationconfig/automation_config_builder.go b/pkg/automationconfig/automation_config_builder.go index 71562c8c9..7758ee646 100644 --- a/pkg/automationconfig/automation_config_builder.go +++ b/pkg/automationconfig/automation_config_builder.go @@ -1,6 +1,8 @@ package automationconfig import ( + "bytes" + "encoding/json" "fmt" ) @@ -21,7 +23,7 @@ type Builder struct { fcv string topology Topology mongodbVersion string - + previousAC AutomationConfig // MongoDB installable versions versions []MongoDbVersionConfig } @@ -74,12 +76,11 @@ func (b *Builder) SetMongoDBVersion(version string) *Builder { return b } -func (b *Builder) SetAutomationConfigVersion(version int) *Builder { - b.version = version +func (b *Builder) SetPreviousAutomationConfig(previousAC AutomationConfig) *Builder { + b.previousAC = previousAC return b } - -func (b *Builder) Build() AutomationConfig { +func (b *Builder) Build() (AutomationConfig, error) { hostnames := make([]string, b.members) for i := 0; i < b.members; i++ { hostnames[i] = fmt.Sprintf("%s-%d.%s", b.name, i, b.domain) @@ -93,8 +94,8 @@ func (b *Builder) Build() AutomationConfig { members[i] = newReplicaSetMember(process, i) } - return AutomationConfig{ - Version: b.version, + currentAc := AutomationConfig{ + Version: b.previousAC.Version, Processes: processes, ReplicaSets: []ReplicaSet{ { @@ -107,6 +108,26 @@ func (b *Builder) Build() AutomationConfig { Options: Options{DownloadBase: "/var/lib/mongodb-mms-automation"}, Auth: DisabledAuth(), } + + // Here we compare the bytes of the two automationconfigs, + // we can't use reflect.DeepEqual() as it treats nil entries as different from empty ones, + // and in the AutomationConfig Struct we use omitempty to set empty field to nil + // The agent requires the nil value we provide, otherwise the agent attempts to configure authentication. + + newAcBytes, err := json.Marshal(b.previousAC) + if err != nil { + return AutomationConfig{}, err + } + + currentAcBytes, err := json.Marshal(currentAc) + if err != nil { + return AutomationConfig{}, err + } + + if bytes.Compare(newAcBytes, currentAcBytes) != 0 { + currentAc.Version += 1 + } + return currentAc, nil } func toHostName(name string, index int) string { diff --git a/pkg/automationconfig/automation_config_test.go b/pkg/automationconfig/automation_config_test.go index 7a9bb9079..2d51ae867 100644 --- a/pkg/automationconfig/automation_config_test.go +++ b/pkg/automationconfig/automation_config_test.go @@ -26,15 +26,15 @@ func defaultMongoDbVersion(version string) MongoDbVersionConfig { func TestBuildAutomationConfig(t *testing.T) { - ac := NewBuilder(). + ac, err := NewBuilder(). SetName("my-rs"). SetDomain("my-ns.svc.cluster.local"). SetMongoDBVersion("4.2.0"). - SetAutomationConfigVersion(1). SetMembers(3). SetFCV("4.0"). Build() + assert.NoError(t, err) assert.Len(t, ac.Processes, 3) assert.Equal(t, 1, ac.Version) @@ -63,15 +63,15 @@ func TestBuildAutomationConfig(t *testing.T) { func TestMongoDbVersions(t *testing.T) { - ac := NewBuilder(). + ac, err := NewBuilder(). SetName("my-rs"). SetDomain("my-ns.svc.cluster.local"). SetMongoDBVersion("4.2.0"). - SetAutomationConfigVersion(1). SetMembers(3). AddVersion(defaultMongoDbVersion("4.2.0")). Build() + assert.NoError(t, err) assert.Len(t, ac.Processes, 3) assert.Len(t, ac.Versions, 1) assert.Len(t, ac.Versions[0].Builds, 1) @@ -90,16 +90,16 @@ func TestMongoDbVersions(t *testing.T) { }, ) - ac = NewBuilder(). + ac, err = NewBuilder(). SetName("my-rs"). SetDomain("my-ns.svc.cluster.local"). SetMongoDBVersion("4.2.0"). - SetAutomationConfigVersion(1). SetMembers(3). AddVersion(defaultMongoDbVersion("4.2.0")). AddVersion(version2). Build() + assert.NoError(t, err) assert.Len(t, ac.Processes, 3) assert.Len(t, ac.Versions, 2) assert.Len(t, ac.Versions[0].Builds, 1) @@ -107,42 +107,42 @@ func TestMongoDbVersions(t *testing.T) { } func TestHasOptions(t *testing.T) { - ac := NewBuilder(). + ac, err := NewBuilder(). SetName("my-rs"). SetDomain("my-ns.svc.cluster.local"). SetMongoDBVersion("4.2.0"). - SetAutomationConfigVersion(1). SetMembers(3). Build() + assert.NoError(t, err) assert.Equal(t, ac.Options.DownloadBase, "/var/lib/mongodb-mms-automation") } func TestModulesNotNil(t *testing.T) { // We make sure the .Modules is initialized as an empty list of strings // or it will dumped as null attribute in json. - ac := NewBuilder(). + ac, err := NewBuilder(). SetName("my-rs"). SetDomain("my-ns.svc.cluster.local"). SetMongoDBVersion("4.2.0"). - SetAutomationConfigVersion(1). SetMembers(3). AddVersion(defaultMongoDbVersion("4.3.2")). Build() + assert.NoError(t, err) assert.NotNil(t, ac.Versions[0].Builds[0].Modules) } func TestProcessHasPortSetToDefault(t *testing.T) { - ac := NewBuilder(). + ac, err := NewBuilder(). SetName("my-rs"). SetDomain("my-ns.svc.cluster.local"). SetMongoDBVersion("4.2.0"). - SetAutomationConfigVersion(1). SetMembers(3). AddVersion(defaultMongoDbVersion("4.3.2")). Build() + assert.NoError(t, err) assert.Len(t, ac.Processes, 3) assert.Equal(t, ac.Processes[0].Args26.Net.Port, 27017) assert.Equal(t, ac.Processes[1].Args26.Net.Port, 27017) diff --git a/pkg/controller/mongodb/mongodb_controller.go b/pkg/controller/mongodb/mongodb_controller.go index 8b8b03c53..91cf99eaa 100644 --- a/pkg/controller/mongodb/mongodb_controller.go +++ b/pkg/controller/mongodb/mongodb_controller.go @@ -25,6 +25,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" + k8sClient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -130,7 +131,6 @@ func (r *ReplicaSetReconciler) Reconcile(request reconcile.Request) (reconcile.R return reconcile.Result{}, err } - // TODO: Read current automation config version from config map if err := r.ensureAutomationConfig(mdb); err != nil { r.log.Infof("error creating automation config config map: %s", err) return reconcile.Result{}, err @@ -265,24 +265,36 @@ func (r ReplicaSetReconciler) ensureAutomationConfig(mdb mdbv1.MongoDB) error { if err != nil { return err } + if err := r.client.CreateOrUpdate(&cm); err != nil { return err } return nil } -func buildAutomationConfig(mdb mdbv1.MongoDB, mdbVersionConfig automationconfig.MongoDbVersionConfig) automationconfig.AutomationConfig { +func buildAutomationConfig(mdb mdbv1.MongoDB, mdbVersionConfig automationconfig.MongoDbVersionConfig, client mdbClient.Client) (automationconfig.AutomationConfig, error) { domain := getDomain(mdb.ServiceName(), mdb.Namespace, "") - return automationconfig.NewBuilder(). + + currentAc, err := getCurrentAutomationConfig(client, mdb) + if err != nil { + return automationconfig.AutomationConfig{}, err + } + newAc, err := automationconfig.NewBuilder(). SetTopology(automationconfig.ReplicaSetTopology). SetName(mdb.Name). SetDomain(domain). SetMembers(mdb.Spec.Members). + SetPreviousAutomationConfig(currentAc). SetMongoDBVersion(mdb.Spec.Version). - SetAutomationConfigVersion(1). // TODO: Correctly set the version SetFCV(mdb.GetFCV()). AddVersion(mdbVersionConfig). Build() + + if err != nil { + return automationconfig.AutomationConfig{}, err + } + + return newAc, nil } func readVersionManifestFromDisk() (automationconfig.VersionManifest, error) { @@ -318,12 +330,30 @@ func buildService(mdb mdbv1.MongoDB) corev1.Service { Build() } +func getCurrentAutomationConfig(client mdbClient.Client, mdb mdbv1.MongoDB) (automationconfig.AutomationConfig, error) { + currentCm := corev1.ConfigMap{} + currentAc := automationconfig.AutomationConfig{} + if err := client.Get(context.TODO(), types.NamespacedName{Name: mdb.ConfigMapName(), Namespace: mdb.Namespace}, ¤tCm); err != nil { + // If the AC was not found we don't surface it as an error + return automationconfig.AutomationConfig{}, k8sClient.IgnoreNotFound(err) + + } + if err := json.Unmarshal([]byte(currentCm.Data[AutomationConfigKey]), ¤tAc); err != nil { + return automationconfig.AutomationConfig{}, err + } + return currentAc, nil +} + func (r ReplicaSetReconciler) buildAutomationConfigConfigMap(mdb mdbv1.MongoDB) (corev1.ConfigMap, error) { manifest, err := r.manifestProvider() if err != nil { return corev1.ConfigMap{}, fmt.Errorf("error reading version manifest from disk: %+v", err) } - ac := buildAutomationConfig(mdb, manifest.BuildsForVersion(mdb.Spec.Version)) + + ac, err := buildAutomationConfig(mdb, manifest.BuildsForVersion(mdb.Spec.Version), r.client) + if err != nil { + return corev1.ConfigMap{}, err + } acBytes, err := json.Marshal(ac) if err != nil { return corev1.ConfigMap{}, err diff --git a/pkg/controller/mongodb/replicaset_controller_test.go b/pkg/controller/mongodb/replicaset_controller_test.go index c2c5f6ac4..81701e426 100644 --- a/pkg/controller/mongodb/replicaset_controller_test.go +++ b/pkg/controller/mongodb/replicaset_controller_test.go @@ -181,5 +181,48 @@ func TestService_isCorrectlyCreatedAndUpdated(t *testing.T) { res, err = r.Reconcile(reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) +} + +func TestAutomationConfig_versionIsBumpedOnChange(t *testing.T) { + mdb := newTestReplicaSet() + + mgr := client.NewManager(&mdb) + r := newReconciler(mgr, mockManifestProvider(mdb.Spec.Version)) + res, err := r.Reconcile(reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) + assertReconciliationSuccessful(t, res, err) + + currentAc, err := getCurrentAutomationConfig(client.NewClient(mgr.GetClient()), mdb) + assert.NoError(t, err) + assert.Equal(t, 1, currentAc.Version) + + mdb.Spec.Members += 1 + + _ = mgr.GetClient().Update(context.TODO(), &mdb) + res, err = r.Reconcile(reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) + assertReconciliationSuccessful(t, res, err) + + currentAc, err = getCurrentAutomationConfig(client.NewClient(mgr.GetClient()), mdb) + assert.NoError(t, err) + assert.Equal(t, 2, currentAc.Version) } + +func TestAutomationConfig_versionIsNotBumpedWithNoChanges(t *testing.T) { + mdb := newTestReplicaSet() + + mgr := client.NewManager(&mdb) + r := newReconciler(mgr, mockManifestProvider(mdb.Spec.Version)) + res, err := r.Reconcile(reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) + assertReconciliationSuccessful(t, res, err) + + currentAc, err := getCurrentAutomationConfig(client.NewClient(mgr.GetClient()), mdb) + assert.NoError(t, err) + assert.Equal(t, currentAc.Version, 1) + + res, err = r.Reconcile(reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) + assertReconciliationSuccessful(t, res, err) + + currentAc, err = getCurrentAutomationConfig(client.NewClient(mgr.GetClient()), mdb) + assert.NoError(t, err) + assert.Equal(t, currentAc.Version, 1) +} diff --git a/test/e2e/mongodbtests/mongodbtests.go b/test/e2e/mongodbtests/mongodbtests.go index 919eb7179..cc5115513 100644 --- a/test/e2e/mongodbtests/mongodbtests.go +++ b/test/e2e/mongodbtests/mongodbtests.go @@ -2,6 +2,7 @@ package mongodbtests import ( "context" + "encoding/json" "fmt" "reflect" "testing" @@ -12,6 +13,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/pkg/apis/mongodb/v1" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/automationconfig" "github.com/mongodb/mongodb-kubernetes-operator/pkg/controller/mongodb" e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" f "github.com/operator-framework/operator-sdk/pkg/test" @@ -105,6 +107,18 @@ func AutomationConfigConfigMapExists(mdb *mdbv1.MongoDB) func(t *testing.T) { } } +func AutomationConfigVersionHasTheExpectedVersion(mdb *mdbv1.MongoDB, expectedVersion int) func(t *testing.T) { + return func(t *testing.T) { + currentCm := corev1.ConfigMap{} + currentAc := automationconfig.AutomationConfig{} + err := f.Global.Client.Get(context.TODO(), types.NamespacedName{Name: mdb.ConfigMapName(), Namespace: mdb.Namespace}, ¤tCm) + assert.NoError(t, err) + err = json.Unmarshal([]byte(currentCm.Data[mongodb.AutomationConfigKey]), ¤tAc) + assert.NoError(t, err) + assert.Equal(t, expectedVersion, currentAc.Version) + } +} + // HasFeatureCompatibilityVersion verifies that the FeatureCompatibilityVersion is // set to `version`. The FCV parameter is not signaled as a non Running state, for // this reason, this function checks the value of the parameter many times, based diff --git a/test/e2e/replica_set/replica_set_test.go b/test/e2e/replica_set/replica_set_test.go index 54bb1a784..22fcf2506 100644 --- a/test/e2e/replica_set/replica_set_test.go +++ b/test/e2e/replica_set/replica_set_test.go @@ -34,6 +34,7 @@ func TestReplicaSet(t *testing.T) { }))) t.Run("MongoDB Reaches Running Phase", mongodbtests.MongoDBReachesRunningPhase(&mdb)) t.Run("Test Basic Connectivity", mongodbtests.BasicConnectivity(&mdb)) + t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 1)) t.Run("Test Status Was Updated", mongodbtests.Status(&mdb, mdbv1.MongoDBStatus{ MongoURI: mdb.MongoURI(), diff --git a/test/e2e/replica_set_change_version/replica_set_test.go b/test/e2e/replica_set_change_version/replica_set_test.go index c63b50414..caf83ae6e 100644 --- a/test/e2e/replica_set_change_version/replica_set_test.go +++ b/test/e2e/replica_set_change_version/replica_set_test.go @@ -28,6 +28,7 @@ func TestReplicaSetUpgradeVersion(t *testing.T) { t.Run("Config Map Was Correctly Created", mongodbtests.AutomationConfigConfigMapExists(&mdb)) t.Run("Stateful Set Reaches Ready State", mongodbtests.StatefulSetIsReady(&mdb)) t.Run("MongoDB Reaches Running Phase", mongodbtests.MongoDBReachesRunningPhase(&mdb)) + t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 1)) t.Run("Test Basic Connectivity", mongodbtests.BasicConnectivity(&mdb)) t.Run("Test Status Was Updated", mongodbtests.Status(&mdb, mdbv1.MongoDBStatus{ @@ -41,6 +42,8 @@ func TestReplicaSetUpgradeVersion(t *testing.T) { t.Run("Test Version can be upgraded", mongodbtests.ChangeVersion(&mdb, "4.0.8")) t.Run("StatefulSet has OnDelete update strategy", mongodbtests.StatefulSetHasUpdateStrategy(&mdb, appsv1.OnDeleteStatefulSetStrategyType)) t.Run("Stateful Set Reaches Ready State, after Upgrading", mongodbtests.StatefulSetIsUpdated(&mdb)) + t.Run("AutomationConfig's version has been increased", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 2)) + }, )) t.Run("StatefulSet has RollingUpgrade restart strategy", mongodbtests.StatefulSetHasUpdateStrategy(&mdb, appsv1.RollingUpdateStatefulSetStrategyType)) @@ -51,6 +54,8 @@ func TestReplicaSetUpgradeVersion(t *testing.T) { t.Run("Test Version can be downgraded", mongodbtests.ChangeVersion(&mdb, "4.0.6")) t.Run("StatefulSet has OnDelete restart strategy", mongodbtests.StatefulSetHasUpdateStrategy(&mdb, appsv1.OnDeleteStatefulSetStrategyType)) t.Run("Stateful Set Reaches Ready State, after Upgrading", mongodbtests.StatefulSetIsUpdated(&mdb)) + t.Run("AutomationConfig's version has been increased", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 3)) + }, )) t.Run("StatefulSet has RollingUpgrade restart strategy", mongodbtests.StatefulSetHasUpdateStrategy(&mdb, appsv1.RollingUpdateStatefulSetStrategyType)) diff --git a/test/e2e/replica_set_multiple/replica_set_multiple_test.go b/test/e2e/replica_set_multiple/replica_set_multiple_test.go index 876f491c4..7431202d3 100644 --- a/test/e2e/replica_set_multiple/replica_set_multiple_test.go +++ b/test/e2e/replica_set_multiple/replica_set_multiple_test.go @@ -31,6 +31,9 @@ func TestReplicaSet(t *testing.T) { t.Run("mdb0: Config Map Was Correctly Created", mongodbtests.AutomationConfigConfigMapExists(&mdb0)) t.Run("mdb1: Config Map Was Correctly Created", mongodbtests.AutomationConfigConfigMapExists(&mdb1)) + t.Run("mdb0: AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb0, 1)) + t.Run("mdb1: AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb1, 1)) + t.Run("mdb0: Stateful Set Reaches Ready State", mongodbtests.StatefulSetIsReady(&mdb0)) t.Run("mdb1: Stateful Set Reaches Ready State", mongodbtests.StatefulSetIsReady(&mdb1)) @@ -56,6 +59,7 @@ func TestReplicaSet(t *testing.T) { t.Run("Scale MongoDB Resource Up", mongodbtests.Scale(&mdb0, 5)) t.Run("Stateful Set Scaled Up Correctly", mongodbtests.StatefulSetIsReady(&mdb0)) t.Run("MongoDB Reaches Running Phase", mongodbtests.MongoDBReachesRunningPhase(&mdb0)) + t.Run("AutomationConfig's version has been increased", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb0, 2)) t.Run("Test Status Was Updated", mongodbtests.Status(&mdb0, mdbv1.MongoDBStatus{ MongoURI: mdb0.MongoURI(), @@ -64,6 +68,7 @@ func TestReplicaSet(t *testing.T) { t.Run("Scale MongoDB Resource Down", mongodbtests.Scale(&mdb0, 3)) t.Run("Stateful Set Scaled Down Correctly", mongodbtests.StatefulSetIsReady(&mdb0)) t.Run("MongoDB Reaches Running Phase", mongodbtests.MongoDBReachesRunningPhase(&mdb0)) + t.Run("AutomationConfig's version has been increased", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb0, 3)) t.Run("Test Status Was Updated", mongodbtests.Status(&mdb0, mdbv1.MongoDBStatus{ MongoURI: mdb0.MongoURI(), diff --git a/test/e2e/replica_set_readiness_probe/replica_set_readiness_probe_test.go b/test/e2e/replica_set_readiness_probe/replica_set_readiness_probe_test.go index 4d8b10a5f..4096405a9 100644 --- a/test/e2e/replica_set_readiness_probe/replica_set_readiness_probe_test.go +++ b/test/e2e/replica_set_readiness_probe/replica_set_readiness_probe_test.go @@ -25,6 +25,7 @@ func TestReplicaSetReadinessProbeScaling(t *testing.T) { mdb := e2eutil.NewTestMongoDB("mdb0") t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) t.Run("Config Map Was Correctly Created", mongodbtests.AutomationConfigConfigMapExists(&mdb)) + t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 1)) t.Run("Stateful Set Reaches Ready State", mongodbtests.StatefulSetIsReady(&mdb)) t.Run("MongoDB is reachable", mongodbtests.IsReachableDuring(&mdb, time.Second*10, func() { diff --git a/test/e2e/replica_set_scale/replica_set_scaling_test.go b/test/e2e/replica_set_scale/replica_set_scaling_test.go index 724f3c27f..3e504c76f 100644 --- a/test/e2e/replica_set_scale/replica_set_scaling_test.go +++ b/test/e2e/replica_set_scale/replica_set_scaling_test.go @@ -24,12 +24,14 @@ func TestReplicaSetScale(t *testing.T) { mdb := e2eutil.NewTestMongoDB("mdb0") t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) t.Run("Config Map Was Correctly Created", mongodbtests.AutomationConfigConfigMapExists(&mdb)) + t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 1)) t.Run("Stateful Set Reaches Ready State", mongodbtests.StatefulSetIsReady(&mdb)) t.Run("MongoDB is reachable", mongodbtests.IsReachableDuring(&mdb, time.Second*10, func() { t.Run("Scale MongoDB Resource Up", mongodbtests.Scale(&mdb, 5)) t.Run("Stateful Set Scaled Up Correctly", mongodbtests.StatefulSetIsReady(&mdb)) t.Run("MongoDB Reaches Running Phase", mongodbtests.MongoDBReachesRunningPhase(&mdb)) + t.Run("AutomationConfig's version has been increased", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 2)) t.Run("Test Status Was Updated", mongodbtests.Status(&mdb, mdbv1.MongoDBStatus{ MongoURI: mdb.MongoURI(), @@ -38,6 +40,7 @@ func TestReplicaSetScale(t *testing.T) { t.Run("Scale MongoDB Resource Down", mongodbtests.Scale(&mdb, 3)) t.Run("Stateful Set Scaled Down Correctly", mongodbtests.StatefulSetIsReady(&mdb)) t.Run("MongoDB Reaches Running Phase", mongodbtests.MongoDBReachesRunningPhase(&mdb)) + t.Run("AutomationConfig's version has been increased", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 3)) t.Run("Test Status Was Updated", mongodbtests.Status(&mdb, mdbv1.MongoDBStatus{ MongoURI: mdb.MongoURI(), From 588f8a04c12d1c0fdc7400cc5e7408519e85a18f Mon Sep 17 00:00:00 2001 From: Fabian Lindfors Date: Tue, 16 Jun 2020 20:52:49 +0200 Subject: [PATCH 018/790] Fix broken setup script for kind 0.8 (#62) --- scripts/dev/setup_kind_cluster.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/scripts/dev/setup_kind_cluster.sh b/scripts/dev/setup_kind_cluster.sh index 15e616f2e..40a38c9d6 100755 --- a/scripts/dev/setup_kind_cluster.sh +++ b/scripts/dev/setup_kind_cluster.sh @@ -1,6 +1,10 @@ #!/usr/bin/env bash set -Eeou pipefail +# create the kind network early unless it already exists. +# it would normally be created automatically by kind but we +# need it earlier to get the IP address of our registry. +docker network create kind || true # adapted from https://kind.sigs.k8s.io/docs/user/local-registry/ # create registry container unless it already exists @@ -9,11 +13,12 @@ reg_port='5000' running="$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)" if [ "${running}" != 'true' ]; then docker run \ - -d --restart=always -p "${reg_port}:${reg_port}" --name "${reg_name}" \ + -d --restart=always -p "${reg_port}:${reg_port}" --name "${reg_name}" --network kind \ registry:2 fi -ip="$(docker inspect kind-registry -f '{{.NetworkSettings.IPAddress}}')" +# find registry IP inside the kind network +ip="$(docker inspect kind-registry -f '{{.NetworkSettings.Networks.kind.IPAddress}}')" # create a cluster with the local registry enabled in containerd cat < Date: Fri, 19 Jun 2020 13:14:01 +0200 Subject: [PATCH 019/790] CLOUDP-65697: Add init container and volumes to architecture doc (#64) --- architecture.md | 19 +++++++++++++------ pkg/controller/mongodb/mongodb_controller.go | 1 - 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/architecture.md b/architecture.md index e5081bcdb..f4e6e9b05 100644 --- a/architecture.md +++ b/architecture.md @@ -13,14 +13,21 @@ The MongoDB Community Kubernetes Operator is a [Custom Resource Definition](http You create and update MongoDB resources by defining a MongoDB resource definition. When you apply the MongoDB resource definition to your Kubernetes environment, the Operator: 1. Creates a [StatefulSet](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/) that contains one [pod](https://kubernetes.io/docs/concepts/workloads/pods/pod-overview/) for each [replica set](https://docs.mongodb.com/manual/replication/) member. -1. Creates two [containers](https://kubernetes.io/docs/concepts/containers/overview/) in each pod: +1. Writes the Automation configuration as a [ConfigMap](https://kubernetes.io/docs/concepts/configuration/configmap/) and mounts it to each pod. +1. Creates one [init container](https://kubernetes.io/docs/concepts/workloads/pods/init-containers/) and two [containers](https://kubernetes.io/docs/concepts/containers/overview/) in each pod: - - A container for the [`mongod`](https://docs.mongodb.com/manual/reference/program/mongod/index.html) process binary.
- `mongod` is the primary daemon process for the MongoDB system. It handles data requests, manages data access, and performs background management operations. + - An init container which copies the `cmd/prestop` binary to the main `mongod` container. [This pre-stop hook](https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/) is used during [version upgrades](#example-mongodb-version-upgrade). - - A container for the MongoDB Agent.
- The Automation function of the MongoDB Agent handles configuring, stopping, and restarting the `mongod` process. The MongoDB Agent periodically polls the `mongod` to determine status and can deploy changes as needed. -1. Writes the Automation configuration as a [ConfigMap](https://kubernetes.io/docs/concepts/configuration/configmap/) and mounts it to each pod. + - A container for the [`mongod`](https://docs.mongodb.com/manual/reference/program/mongod/index.html) process binary. `mongod` is the primary daemon process for the MongoDB system. It handles data requests, manages data access, and performs background management operations. + + - A container for the MongoDB Agent. The Automation function of the MongoDB Agent handles configuring, stopping, and restarting the `mongod` process. The MongoDB Agent periodically polls the `mongod` to determine status and can deploy changes as needed. + +1. Creates several volumes: + + - `data-volume` which is [persistent](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) and mounts to `/data` on both the server and agent containers. Stores server data as well as `automation-mongod.conf` written by the agent and some locks the agent needs. + - `automation-config` which is mounted from the previously generated ConfigMap to both the server and agent. Only lives as long as the pod. + - `healthstatus` which contains the agent's current status. This is shared with the `mongod` container where it's used by the pre-stop hook. Only lives as long as the pod. + 1. Initiates the MongoDB Agent, which in turn creates the database configuration and launches the `mongod` process according to your [MongoDB resource definition](deploy/crds/mongodb.com_v1_mongodb_cr.yaml). + +# MongoDB Kubernetes Operator 0.5.2 +## Kubernetes Operator +* Changes + * Readiness probe has been moved into an init container from the Agent image. + * Security context is now added when the `MANAGED_SECURITY_CONTEXT` environment variable is not set. +* Bug fixes + * Removed unnecessary environment variable configuration in the openshift samples. + * Fixed an issue where the operator would perform unnecessary reconcilliations when Secrets were modified. + * Fixed an issue where a race condition could cause the deployment to get into a bad state when TLS + settings when being changed at the same time as a scaling operation was happening. + * Fixed an issue where the agent pod would panic when running as a non-root user. + +## MongoDBCommunity Resource +* Changes + * Added `spec.security.authentication.ignoreUnknownUsers` field. This value defaults to `true`. When enabled, + any MongoDB users added through external sources will not be removed. + + +## Miscellaneous +* Changes + * Internal code refactorings to allow libraries to be imported into other projects. + + + ## Updated Image Tags + * mongodb-kubernetes-operator:0.5.2 + * mongodb-agent:10.27.0.6772-1 + * mongodb-kubernetes-readinessprobe:1.0.1 [new image] + + diff --git a/dev_notes/release-notes-template.md b/dev_notes/release-notes-template.md index a8ea96728..58a9e7986 100644 --- a/dev_notes/release-notes-template.md +++ b/dev_notes/release-notes-template.md @@ -25,6 +25,10 @@ * Issue 1 * Issue 2 +## Miscellaneous +* Item 1 +* Item 2 + ## Updated Image Tags * mongodb-kubernetes-operator:0.3.0 * mongodb-agent:10.19.0.6562-1 diff --git a/release.json b/release.json index 30951aada..b4b01ae8d 100644 --- a/release.json +++ b/release.json @@ -1,5 +1,5 @@ { - "mongodb-kubernetes-operator": "0.5.1", + "mongodb-kubernetes-operator": "0.5.2", "version-upgrade-hook": "1.0.2", "readiness-probe": "1.0.1", "agent": { From 3312bebe8aee29f1adbe30f7110289d92ca29d27 Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Wed, 31 Mar 2021 09:18:34 +0100 Subject: [PATCH 214/790] Extract agent command common to monitoring and automation (#405) --- controllers/construct/mongodbstatefulset.go | 52 ++++++++++----------- 1 file changed, 24 insertions(+), 28 deletions(-) diff --git a/controllers/construct/mongodbstatefulset.go b/controllers/construct/mongodbstatefulset.go index 3ddbc6f45..d15757cad 100644 --- a/controllers/construct/mongodbstatefulset.go +++ b/controllers/construct/mongodbstatefulset.go @@ -36,18 +36,30 @@ const ( MongodbRepoUrl = "MONGODB_REPO_URL" - headlessAgentEnv = "HEADLESS_AGENT" - podNamespaceEnv = "POD_NAMESPACE" - automationConfigEnv = "AUTOMATION_CONFIG_MAP" - - automationconfFilePath = "/data/automation-mongod.conf" - keyfileFilePath = "/var/lib/mongodb-mms-automation/authentication/keyfile" - + headlessAgentEnv = "HEADLESS_AGENT" + podNamespaceEnv = "POD_NAMESPACE" + automationConfigEnv = "AUTOMATION_CONFIG_MAP" AgentImageEnv = "AGENT_IMAGE" MongodbImageEnv = "MONGODB_IMAGE" VersionUpgradeHookImageEnv = "VERSION_UPGRADE_HOOK_IMAGE" ReadinessProbeImageEnv = "READINESS_PROBE_IMAGE" ManagedSecurityContextEnv = "MANAGED_SECURITY_CONTEXT" + + automationconfFilePath = "/data/automation-mongod.conf" + keyfileFilePath = "/var/lib/mongodb-mms-automation/authentication/keyfile" + + automationAgentOptions = " -skipMongoStart -noDaemonize -useLocalMongoDbTools" + + MongodbUserCommand = `current_uid=$(id -u) +declare -r current_uid +if ! grep -q "${current_uid}" /etc/passwd ; then +sed -e "s/^mongodb:/builder:/" /etc/passwd > /tmp/passwd +echo "mongodb:x:$(id -u):$(id -g):,,,:/:/bin/bash" >> /tmp/passwd +export NSS_WRAPPER_PASSWD=/tmp/passwd +export LD_PRELOAD=libnss_wrapper.so +export NSS_WRAPPER_GROUP=/etc/group +fi +` ) // MongoDBStatefulSetOwner is an interface which any resource which generates a MongoDB StatefulSet should implement. @@ -162,16 +174,11 @@ func BuildMongoDBReplicaSetStatefulSetModificationFunction(mdb MongoDBStatefulSe )) } -func mongodbAgentContainer(automationConfigSecretName string, volumeMounts []corev1.VolumeMount) container.Modification { - agentCommand := strings.Join([]string{ - "agent/mongodb-agent", - "-cluster=" + clusterFilePath, - "-skipMongoStart", - "-noDaemonize", - "-healthCheckFilePath=" + agentHealthStatusFilePathValue, - "-serveStatusPort=5000", - "-useLocalMongoDbTools"}, " ") +func BaseAgentCommand() string { + return "agent/mongodb-agent -cluster=" + clusterFilePath + " -healthCheckFilePath=" + agentHealthStatusFilePathValue + " -serveStatusPort=5000" +} +func mongodbAgentContainer(automationConfigSecretName string, volumeMounts []corev1.VolumeMount) container.Modification { securityContext := container.NOOP() managedSecurityContext := envvar.ReadBool(ManagedSecurityContextEnv) if !managedSecurityContext { @@ -185,18 +192,7 @@ func mongodbAgentContainer(automationConfigSecretName string, volumeMounts []cor container.WithResourceRequirements(resourcerequirements.Defaults()), container.WithVolumeMounts(volumeMounts), securityContext, - container.WithCommand([]string{"/bin/bash", "-c", `current_uid=$(id -u) -echo $current_uid -declare -r current_uid -if ! grep -q "${current_uid}" /etc/passwd ; then -sed -e "s/^mongodb:/builder:/" /etc/passwd > /tmp/passwd -echo "mongodb:x:$(id -u):$(id -g):,,,:/:/bin/bash" >> /tmp/passwd -cat /tmp/passwd -export NSS_WRAPPER_PASSWD=/tmp/passwd -export LD_PRELOAD=libnss_wrapper.so -export NSS_WRAPPER_GROUP=/etc/group -fi -` + agentCommand}), + container.WithCommand([]string{"/bin/bash", "-c", MongodbUserCommand + BaseAgentCommand() + automationAgentOptions}), container.WithEnvs( corev1.EnvVar{ Name: headlessAgentEnv, From 357bd36b63d316cc66dab6597a0fefd62ba11f5c Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Wed, 31 Mar 2021 10:51:39 +0100 Subject: [PATCH 215/790] Create ubi buildvariant (#378) --- .evergreen.yml | 50 +++++++++++++++++++++--- scripts/ci/build_and_push_agent_image.sh | 2 +- scripts/ci/config.json | 3 +- scripts/ci/run_test.sh | 4 +- scripts/dev/dev_config.py | 28 +++++++++++-- scripts/dev/e2e.py | 10 ++++- 6 files changed, 81 insertions(+), 16 deletions(-) diff --git a/.evergreen.yml b/.evergreen.yml index 8eda63bd9..a6cf9e96e 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -89,6 +89,7 @@ functions: - version_id - test - clusterwide + - distro binary: scripts/ci/run_test.sh # TODO: can be removed once the agent is included in the Dockerfile generator. @@ -102,6 +103,7 @@ functions: - quay_password - expire_after - image + - dockerfile_path working_dir: mongodb-kubernetes-operator binary: scripts/ci/build_and_push_agent_image.sh @@ -187,7 +189,7 @@ tasks: image_type: e2e expire_after: 48h - - name: build_agent_image + - name: build_agent_image_ubuntu priority: 60 exec_timeout_secs: 600 commands: @@ -197,6 +199,19 @@ tasks: vars: image: quay.io/mongodb/mongodb-agent-dev:${version_id} expire_after: 48h + dockerfile_path: agent/Dockerfile + + - name: build_agent_image_ubi + priority: 60 + exec_timeout_secs: 600 + commands: + - func: clone + - func: setup_virtualenv + - func: build_and_push_agent_image + vars: + image: quay.io/mongodb/mongodb-agent-ubi-dev:${version_id} + expire_after: 48h + dockerfile_path: agent/Dockerfile.ubi - name: build_prehook_image priority: 60 @@ -335,8 +350,10 @@ tasks: buildvariants: - - name: e2e_tests - display_name: e2e_tests + - name: e2e_tests_ubuntu + display_name: e2e_tests_ubuntu + expansions: + distro: ubuntu run_on: - ubuntu1604-build depends_on: @@ -346,7 +363,25 @@ buildvariants: variant: init_test_run - name: build_prehook_image variant: init_test_run - - name: build_agent_image + - name: build_agent_image_ubuntu + variant: init_test_run + tasks: + - name: e2e_test_group + + - name: e2e_tests_ubi + display_name: e2e_tests_ubi + expansions: + distro: ubi + run_on: + - ubuntu1604-build + depends_on: + - name: build_operator_image + variant: init_test_run + - name: build_e2e_image + variant: init_test_run + - name: build_prehook_image + variant: init_test_run + - name: build_agent_image_ubi variant: init_test_run tasks: - name: e2e_test_group @@ -359,7 +394,8 @@ buildvariants: - name: build_operator_image - name: build_e2e_image - name: build_prehook_image - - name: build_agent_image + - name: build_agent_image_ubi + - name: build_agent_image_ubuntu - name: release_blocker display_name: release_blocker @@ -377,7 +413,9 @@ buildvariants: variant: init_test_run - name: build_prehook_image variant: init_test_run - - name: build_agent_image + - name: build_agent_image_ubuntu + variant: init_test_run + - name: build_agent_image_ubi variant: init_test_run run_on: - ubuntu1604-test diff --git a/scripts/ci/build_and_push_agent_image.sh b/scripts/ci/build_and_push_agent_image.sh index 6d886aa38..af8a07c5b 100755 --- a/scripts/ci/build_and_push_agent_image.sh +++ b/scripts/ci/build_and_push_agent_image.sh @@ -7,5 +7,5 @@ echo "${quay_password:?}" | docker login "-u=${quay_user_name:?}" quay.io --pass # Providing the quay.expires-after configures quay to delete this image after the provided amount of time -docker build . --label "quay.expires-after=${expire_after:-never}" -f agent/Dockerfile -t "${image:?}" --build-arg tools_version="100.2.0" --build-arg agent_version="10.27.0.6772-1" +docker build . --label "quay.expires-after=${expire_after:-never}" -f "${dockerfile_path:?}" -t "${image:?}" --build-arg tools_version="100.2.0" --build-arg agent_version="10.27.0.6772-1" docker push "${image:?}" diff --git a/scripts/ci/config.json b/scripts/ci/config.json index e7f64a4ed..de612ceba 100644 --- a/scripts/ci/config.json +++ b/scripts/ci/config.json @@ -5,5 +5,6 @@ "e2e_image": "community-operator-e2e", "version_upgrade_hook_image": "community-operator-version-upgrade-post-start-hook", "testrunner_image": "community-operator-testrunner", - "agent_image": "mongodb-agent-dev" + "agent_image_ubuntu": "mongodb-agent-dev", + "agent_image_ubi": "mongodb-agent-ubi-dev" } diff --git a/scripts/ci/run_test.sh b/scripts/ci/run_test.sh index d5ec36665..c8cd77937 100755 --- a/scripts/ci/run_test.sh +++ b/scripts/ci/run_test.sh @@ -3,7 +3,7 @@ # shellcheck disable=SC1091 . venv/bin/activate if [ -z "${clusterwide:-}" ]; then - python3 ./scripts/dev/e2e.py --test "${test:?}" --tag "${version_id:?}" --config_file ./scripts/ci/config.json + python3 ./scripts/dev/e2e.py --test "${test:?}" --tag "${version_id:?}" --config_file ./scripts/ci/config.json --distro "${distro:?}" else - python3 ./scripts/dev/e2e.py --test "${test:?}" --tag "${version_id:?}" --config_file ./scripts/ci/config.json --cluster-wide + python3 ./scripts/dev/e2e.py --test "${test:?}" --tag "${version_id:?}" --config_file ./scripts/ci/config.json --cluster-wide --distro "${distro:?}" fi diff --git a/scripts/dev/dev_config.py b/scripts/dev/dev_config.py index 1bb3d1bfa..5c6944466 100644 --- a/scripts/dev/dev_config.py +++ b/scripts/dev/dev_config.py @@ -1,4 +1,6 @@ +from __future__ import annotations from typing import Dict, Optional +from enum import Enum import json import os @@ -6,6 +8,19 @@ FULL_CONFIG_PATH = os.path.expanduser(CONFIG_PATH) +class Distro(Enum): + UBUNTU = 0 + UBI = 1 + + @staticmethod + def from_string(distro_name: str) -> Distro: + distro_name = distro_name.lower() + return { + "ubuntu": Distro.UBUNTU, + "ubi": Distro.UBI, + }[distro_name] + + def get_config_path() -> str: return os.getenv("MONGODB_COMMUNITY_CONFIG", FULL_CONFIG_PATH) @@ -15,8 +30,9 @@ class DevConfig: DevConfig is a wrapper around the developer configuration file """ - def __init__(self, config: Dict): + def __init__(self, config: Dict, distro: Distro): self._config = config + self._distro = distro @property def namespace(self) -> str: @@ -40,16 +56,20 @@ def version_upgrade_hook_image(self) -> str: @property def agent_image(self) -> str: - return self._config["agent_image"] + if self._distro == Distro.UBI: + return self._config["agent_image_ubi"] + return self._config["agent_image_ubuntu"] -def load_config(config_file_path: str = None) -> DevConfig: +def load_config( + config_file_path: Optional[str] = None, distro: Distro = Distro.UBUNTU +) -> DevConfig: if config_file_path is None: config_file_path = get_config_path() try: with open(config_file_path, "r") as f: - return DevConfig(json.loads(f.read())) + return DevConfig(json.loads(f.read()), distro=distro) except FileNotFoundError: print( f"No DevConfig found. Please ensure that the configuration file exists at '{config_file_path}'" diff --git a/scripts/dev/e2e.py b/scripts/dev/e2e.py index 46df9b40a..11cc8779a 100644 --- a/scripts/dev/e2e.py +++ b/scripts/dev/e2e.py @@ -10,7 +10,7 @@ import dump_diagnostic from dockerutil import build_and_push_image from typing import Dict -from dev_config import load_config, DevConfig +from dev_config import load_config, DevConfig, Distro from kubernetes import client, config import argparse import os @@ -255,6 +255,12 @@ def parse_args() -> argparse.Namespace: help="Watch all namespaces", action="store_true", ) + parser.add_argument( + "--distro", + help="The distro of images that should be used", + type=str, + default="ubuntu", + ) parser.add_argument("--config_file", help="Path to the config file") return parser.parse_args() @@ -305,7 +311,7 @@ def main() -> int: args = parse_args() config.load_kube_config() - dev_config = load_config(args.config_file) + dev_config = load_config(args.config_file, Distro.from_string(args.distro)) create_kube_config(args.config_file) try: From 2329b89f612ed8d43c7ab16e3e334e823cab44bb Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Wed, 31 Mar 2021 11:22:33 +0100 Subject: [PATCH 216/790] Changed agent version (#407) --- agent/Dockerfile.ubi | 1 + config/manager/manager.yaml | 2 +- deploy/openshift/operator_openshift.yaml | 2 +- release.json | 4 ++-- scripts/ci/build_and_push_agent_image.sh | 2 +- test/e2e/setup/test_config.go | 2 +- 6 files changed, 7 insertions(+), 6 deletions(-) diff --git a/agent/Dockerfile.ubi b/agent/Dockerfile.ubi index 0841376bb..c3d8d7d1c 100644 --- a/agent/Dockerfile.ubi +++ b/agent/Dockerfile.ubi @@ -32,4 +32,5 @@ RUN curl --fail --retry 3 --silent https://downloads.mongodb.org/tools/db/mongod && tar xfz mongodb-tools.tgz --directory /var/lib/mongodb-mms-automation/ \ && rm mongodb-tools.tgz +USER 2000 CMD ["agent/mongodb-agent", "-cluster=/var/lib/automation/config/automation-config.json"] diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 80bde49c8..1077cb544 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -32,7 +32,7 @@ spec: - name: OPERATOR_NAME value: "mongodb-kubernetes-operator" - name: AGENT_IMAGE # The MongoDB Agent the operator will deploy to manage MongoDB deployments - value: quay.io/mongodb/mongodb-agent:10.27.0.6772-1 + value: quay.io/mongodb/mongodb-agent:10.29.0.6830-1 - name: VERSION_UPGRADE_HOOK_IMAGE value: quay.io/mongodb/mongodb-kubernetes-operator-version-upgrade-post-start-hook:1.0.2 - name: READINESS_PROBE_IMAGE diff --git a/deploy/openshift/operator_openshift.yaml b/deploy/openshift/operator_openshift.yaml index 51cac7c7e..7ca027261 100644 --- a/deploy/openshift/operator_openshift.yaml +++ b/deploy/openshift/operator_openshift.yaml @@ -33,7 +33,7 @@ spec: - name: OPERATOR_NAME value: "mongodb-kubernetes-operator" - name: AGENT_IMAGE # The MongoDB Agent the operator will deploy to manage MongoDB deployments - value: quay.io/mongodb/mongodb-agent:10.27.0.6772-1 + value: quay.io/mongodb/mongodb-agent:10.29.0.6830-1 - name: READINESS_PROBE_IMAGE value: quay.io/mongodb/mongodb-kubernetes-readinessprobe:1.0.1 - name: VERSION_UPGRADE_HOOK_IMAGE diff --git a/release.json b/release.json index b4b01ae8d..8fe9acdbf 100644 --- a/release.json +++ b/release.json @@ -3,7 +3,7 @@ "version-upgrade-hook": "1.0.2", "readiness-probe": "1.0.1", "agent": { - "version": "10.27.0.6772-1", - "tools_version": "100.2.0" + "version": "10.29.0.6830-1", + "tools_version": "100.2.0" } } diff --git a/scripts/ci/build_and_push_agent_image.sh b/scripts/ci/build_and_push_agent_image.sh index af8a07c5b..4a65485d1 100755 --- a/scripts/ci/build_and_push_agent_image.sh +++ b/scripts/ci/build_and_push_agent_image.sh @@ -7,5 +7,5 @@ echo "${quay_password:?}" | docker login "-u=${quay_user_name:?}" quay.io --pass # Providing the quay.expires-after configures quay to delete this image after the provided amount of time -docker build . --label "quay.expires-after=${expire_after:-never}" -f "${dockerfile_path:?}" -t "${image:?}" --build-arg tools_version="100.2.0" --build-arg agent_version="10.27.0.6772-1" +docker build . --label "quay.expires-after=${expire_after:-never}" -f agent/Dockerfile -t "${image:?}" --build-arg tools_version="100.2.0" --build-arg agent_version="10.29.0.6830-1" docker push "${image:?}" diff --git a/test/e2e/setup/test_config.go b/test/e2e/setup/test_config.go index 265b7df37..e4fdfb500 100644 --- a/test/e2e/setup/test_config.go +++ b/test/e2e/setup/test_config.go @@ -26,7 +26,7 @@ func loadTestConfigFromEnv() testConfig { namespace: envvar.GetEnvOrDefault(testNamespaceEnvName, "default"), operatorImage: envvar.GetEnvOrDefault(operatorImageEnvName, "quay.io/mongodb/community-operator-dev:latest"), versionUpgradeHookImage: envvar.GetEnvOrDefault(construct.VersionUpgradeHookImageEnv, "quay.io/mongodb/mongodb-kubernetes-operator-version-upgrade-post-start-hook:1.0.2"), - agentImage: envvar.GetEnvOrDefault(construct.AgentImageEnv, "quay.io/mongodb/mongodb-agent:10.27.0.6772-1"), // TODO: better way to decide default agent image. + agentImage: envvar.GetEnvOrDefault(construct.AgentImageEnv, "quay.io/mongodb/mongodb-agent:10.29.0.6830-1"), // TODO: better way to decide default agent image. clusterWide: envvar.ReadBool(clusterWideEnvName), performCleanup: envvar.ReadBool(performCleanupEnvName), } From a8c750812dce2fd1d6cc9f02acde022e91daa6d7 Mon Sep 17 00:00:00 2001 From: Rodrigo Valin Date: Wed, 31 Mar 2021 14:46:01 -0300 Subject: [PATCH 217/790] CLOUDP-65923: Replication state aware Readinessprobe (#381) --- cmd/readiness/main.go | 82 ++++++++++++++++++- cmd/readiness/readiness_test.go | 30 +++++++ .../health-status-not-readable-state.json | 82 +++++++++++++++++++ .../health-status-not-started-yet.json | 82 +++++++++++++++++++ .../health-status-ok-no-replica-status.json | 82 +++++++++++++++++++ .../health-status-readable-state.json | 82 +++++++++++++++++++ pkg/readiness/health/health.go | 22 ++++- release.json | 2 +- 8 files changed, 457 insertions(+), 7 deletions(-) create mode 100644 cmd/readiness/testdata/health-status-not-readable-state.json create mode 100644 cmd/readiness/testdata/health-status-not-started-yet.json create mode 100644 cmd/readiness/testdata/health-status-ok-no-replica-status.json create mode 100644 cmd/readiness/testdata/health-status-readable-state.json diff --git a/cmd/readiness/main.go b/cmd/readiness/main.go index 8ebbd41b5..209f3d5f4 100644 --- a/cmd/readiness/main.go +++ b/cmd/readiness/main.go @@ -66,14 +66,14 @@ func isPodReady(conf config.Config) bool { } // If the agent has reached the goal state - returning true - ok, err := isInGoalState(health, conf) - + inGoalState, err := isInGoalState(health, conf) if err != nil { logger.Errorf("There was problem checking the health status: %s", err) panic(err) } - if ok { + inReadyState := isInReadyState(health) + if inGoalState && inReadyState { logger.Info("Agent has reached goal state") return true } @@ -198,6 +198,7 @@ func kubernetesClientset() (kubernetes.Interface, error) { } return clientset, nil } + func main() { clientSet, err := kubernetesClientset() if err != nil { @@ -222,3 +223,78 @@ func main() { os.Exit(1) } } + +// isInReadyState checks the MongoDB Server state. It returns true if the state +// is PRIMARY or SECONDARY. +// This function will always return true if the agent doesn't publish this state. +func isInReadyState(health health.Status) bool { + if len(health.Healthiness) == 0 { + return true + } + for _, processHealth := range health.Healthiness { + // We know this loop should run only once, in Kubernetes there's + // only 1 server managed per host. + if processHealth.ReplicaStatus == nil { + // We always return true if the Agent does not publish mongodb + // server state + return true + } + + if mongoDbServerHasStarted(health) { + // There should be only one entry reported for this Pod. + return processHealth.IsReadyState() + } + } + return false +} + +// mongoDbServerHasStarted checks if the current plan includes a Move and a Step +// of type "StartFresh" with a Result of "success". +// +// This function will return true if the agent has been able to successfully +// start the MongoDB server. +func mongoDbServerHasStarted(health health.Status) bool { + plan := findCurrentPlan(health.ProcessPlans) + if plan == nil { + return false + } + + for _, move := range plan.Moves { + for _, step := range move.Steps { + if step.Step == "StartFresh" && step.Result == "success" { + return true + } + } + } + + return false +} + +// findCurrentPlan returns the current plan as informed by the Agent. +// +// The current plan is the last plan from the `processStatuses` parameter, this +// is, the plan that's currently being processed by the agent. +func findCurrentPlan(processStatuses map[string]health.MmsDirectorStatus) *health.PlanStatus { + var currentPlan *health.PlanStatus + if len(processStatuses) == 0 { + // Seems shouldn't happen but let's check anyway - may be needs to be + // changed to Info if this happens. + logger.Warnf("There is no information about Agent process plans") + return nil + } + if len(processStatuses) > 1 { + logger.Errorf("Only one process status is expected but got %d!", len(processStatuses)) + return nil + } + // There is only one process managed by the Agent - so will only check one + // iteration. + for k, v := range processStatuses { + if len(v.Plans) == 0 { + logger.Errorf("The process %s doesn't contain any plans!", k) + return nil + } + currentPlan = v.Plans[len(v.Plans)-1] + } + + return currentPlan +} diff --git a/cmd/readiness/readiness_test.go b/cmd/readiness/readiness_test.go index 5887287e9..a11a2607d 100644 --- a/cmd/readiness/readiness_test.go +++ b/cmd/readiness/readiness_test.go @@ -100,6 +100,8 @@ func TestHeadlessAgentHasntReachedGoal(t *testing.T) { assert.False(t, isPodReady(c)) thePod, _ := c.ClientSet.CoreV1().Pods(c.Namespace).Get(context.TODO(), c.Hostname, metav1.GetOptions{}) assert.Equal(t, map[string]string{"agent.mongodb.com/version": "5"}, thePod.Annotations) + + os.Unsetenv(headlessAgent) } // TestHeadlessAgentReachedGoal verifies that the probe reports "true" if the config version is equal to the @@ -111,6 +113,34 @@ func TestHeadlessAgentReachedGoal(t *testing.T) { assert.True(t, isPodReady(c)) thePod, _ := c.ClientSet.CoreV1().Pods(c.Namespace).Get(context.TODO(), c.Hostname, metav1.GetOptions{}) assert.Equal(t, map[string]string{"agent.mongodb.com/version": "5"}, thePod.Annotations) + + os.Unsetenv(headlessAgent) +} + +func TestPodReadiness(t *testing.T) { + t.Run("MongoDB replication state is reported by agents", func(t *testing.T) { + assert.True(t, isPodReady(testConfig("testdata/health-status-ok-no-replica-status.json"))) + }) + + t.Run("If replication state is not PRIMARY or SECONDARY, Pod is not ready", func(t *testing.T) { + assert.False(t, isPodReady(testConfig("testdata/health-status-not-readable-state.json"))) + }) + + t.Run("If replication state is readable", func(t *testing.T) { + assert.True(t, isPodReady(testConfig("testdata/health-status-readable-state.json"))) + }) +} + +func TestServerHasAlreadyStarted(t *testing.T) { + t.Run("Agent should report server has started", func(t *testing.T) { + healthDoc := readHealthinessFile("testdata/health-status-readable-state.json") + assert.True(t, mongoDbServerHasStarted(healthDoc)) + }) + + t.Run("Agent should report server has not started", func(t *testing.T) { + healthDoc := readHealthinessFile("testdata/health-status-not-started-yet.json") + assert.False(t, mongoDbServerHasStarted(healthDoc)) + }) } func readHealthinessFile(path string) health.Status { diff --git a/cmd/readiness/testdata/health-status-not-readable-state.json b/cmd/readiness/testdata/health-status-not-readable-state.json new file mode 100644 index 000000000..82449bbc0 --- /dev/null +++ b/cmd/readiness/testdata/health-status-not-readable-state.json @@ -0,0 +1,82 @@ +{ + "mmsStatus": { + "bar": { + "errorString": "", + "errorCode": 0, + "plans": [ + { + "moves": [ + { + "steps": [ + { + "result": "success", + "completed": "2019-09-11T14:20:55.645615846Z", + "started": "2019-09-11T14:20:40.631404367Z", + "isWaitStep": false, + "stepDoc": "Download mongodb binaries (may take a while)", + "step": "Download" + } + ], + "moveDoc": "Download mongodb binaries", + "move": "Download" + }, + { + "steps": [ + { + "result": "success", + "completed": "2019-09-11T14:20:59.325129842Z", + "started": "2019-09-11T14:20:55.645743003Z", + "isWaitStep": false, + "stepDoc": "Start a mongo instance (start fresh)", + "step": "StartFresh" + } + ], + "moveDoc": "Start the process", + "move": "Start" + }, + { + "steps": [ + { + "result": "wait", + "completed": null, + "started": "2019-09-11T14:20:59.325272608Z", + "isWaitStep": true, + "stepDoc": "Wait for the replica set to be initialized by another member", + "step": "WaitRsInit" + } + ], + "moveDoc": "Wait for the replica set to be initialized by another member", + "move": "WaitRsInit" + }, + { + "steps": [ + { + "result": "", + "completed": null, + "started": null, + "isWaitStep": true, + "stepDoc": "Wait for featureCompatibilityVersion to be right", + "step": "WaitFeatureCompatibilityVersionCorrect" + } + ], + "moveDoc": "Wait for featureCompatibilityVersion to be right", + "move": "WaitFeatureCompatibilityVersionCorrect" + } + ], + "completed": "2019-09-11T14:21:42.034934358Z", + "started": "2019-09-11T14:20:40.631348806Z" + } + ], + "lastGoalVersionAchieved": 5, + "name": "bar" + } + }, + "statuses": { + "bar": { + "ReplicationStatus": 3, + "ExpectedToBeUp": true, + "LastMongoUpTime": 1568222195, + "IsInGoalState": true + } + } +} diff --git a/cmd/readiness/testdata/health-status-not-started-yet.json b/cmd/readiness/testdata/health-status-not-started-yet.json new file mode 100644 index 000000000..4f3598a8c --- /dev/null +++ b/cmd/readiness/testdata/health-status-not-started-yet.json @@ -0,0 +1,82 @@ +{ + "mmsStatus": { + "bar": { + "errorString": "", + "errorCode": 0, + "plans": [ + { + "moves": [ + { + "steps": [ + { + "result": "success", + "completed": "2019-09-11T14:20:55.645615846Z", + "started": "2019-09-11T14:20:40.631404367Z", + "isWaitStep": false, + "stepDoc": "Download mongodb binaries (may take a while)", + "step": "Download" + } + ], + "moveDoc": "Download mongodb binaries", + "move": "Download" + }, + { + "steps": [ + { + "result": "wait", + "completed": "2019-09-11T14:20:59.325129842Z", + "started": "2019-09-11T14:20:55.645743003Z", + "isWaitStep": false, + "stepDoc": "Start a mongo instance (start fresh)", + "step": "StartFresh" + } + ], + "moveDoc": "Start the process", + "move": "Start" + }, + { + "steps": [ + { + "result": "wait", + "completed": null, + "started": "2019-09-11T14:20:59.325272608Z", + "isWaitStep": true, + "stepDoc": "Wait for the replica set to be initialized by another member", + "step": "WaitRsInit" + } + ], + "moveDoc": "Wait for the replica set to be initialized by another member", + "move": "WaitRsInit" + }, + { + "steps": [ + { + "result": "", + "completed": null, + "started": null, + "isWaitStep": true, + "stepDoc": "Wait for featureCompatibilityVersion to be right", + "step": "WaitFeatureCompatibilityVersionCorrect" + } + ], + "moveDoc": "Wait for featureCompatibilityVersion to be right", + "move": "WaitFeatureCompatibilityVersionCorrect" + } + ], + "completed": "2019-09-11T14:21:42.034934358Z", + "started": "2019-09-11T14:20:40.631348806Z" + } + ], + "lastGoalVersionAchieved": 5, + "name": "bar" + } + }, + "statuses": { + "bar": { + "ReplicationStatus": 1, + "ExpectedToBeUp": true, + "LastMongoUpTime": 1568222195, + "IsInGoalState": true + } + } +} diff --git a/cmd/readiness/testdata/health-status-ok-no-replica-status.json b/cmd/readiness/testdata/health-status-ok-no-replica-status.json new file mode 100644 index 000000000..caaf67f56 --- /dev/null +++ b/cmd/readiness/testdata/health-status-ok-no-replica-status.json @@ -0,0 +1,82 @@ +{ + "mmsStatus": { + "bar": { + "errorString": "", + "errorCode": 0, + "plans": [ + { + "moves": [ + { + "steps": [ + { + "result": "success", + "completed": "2019-09-11T14:20:55.645615846Z", + "started": "2019-09-11T14:20:40.631404367Z", + "isWaitStep": false, + "stepDoc": "Download mongodb binaries (may take a while)", + "step": "Download" + } + ], + "moveDoc": "Download mongodb binaries", + "move": "Download" + }, + { + "steps": [ + { + "result": "success", + "completed": "2019-09-11T14:20:59.325129842Z", + "started": "2019-09-11T14:20:55.645743003Z", + "isWaitStep": false, + "stepDoc": "Start a mongo instance (start fresh)", + "step": "StartFresh" + } + ], + "moveDoc": "Start the process", + "move": "Start" + }, + { + "steps": [ + { + "result": "wait", + "completed": null, + "started": "2019-09-11T14:20:59.325272608Z", + "isWaitStep": true, + "stepDoc": "Wait for the replica set to be initialized by another member", + "step": "WaitRsInit" + } + ], + "moveDoc": "Wait for the replica set to be initialized by another member", + "move": "WaitRsInit" + }, + { + "steps": [ + { + "result": "", + "completed": null, + "started": null, + "isWaitStep": true, + "stepDoc": "Wait for featureCompatibilityVersion to be right", + "step": "WaitFeatureCompatibilityVersionCorrect" + } + ], + "moveDoc": "Wait for featureCompatibilityVersion to be right", + "move": "WaitFeatureCompatibilityVersionCorrect" + } + ], + "completed": "2019-09-11T14:21:42.034934358Z", + "started": "2019-09-11T14:20:40.631348806Z" + } + ], + "lastGoalVersionAchieved": 5, + "name": "bar" + } + }, + "statuses": { + "bar": { + "ReplicationStatus": null, + "ExpectedToBeUp": true, + "LastMongoUpTime": 1568222195, + "IsInGoalState": true + } + } +} diff --git a/cmd/readiness/testdata/health-status-readable-state.json b/cmd/readiness/testdata/health-status-readable-state.json new file mode 100644 index 000000000..ab2dfadb3 --- /dev/null +++ b/cmd/readiness/testdata/health-status-readable-state.json @@ -0,0 +1,82 @@ +{ + "mmsStatus": { + "bar": { + "errorString": "", + "errorCode": 0, + "plans": [ + { + "moves": [ + { + "steps": [ + { + "result": "success", + "completed": "2019-09-11T14:20:55.645615846Z", + "started": "2019-09-11T14:20:40.631404367Z", + "isWaitStep": false, + "stepDoc": "Download mongodb binaries (may take a while)", + "step": "Download" + } + ], + "moveDoc": "Download mongodb binaries", + "move": "Download" + }, + { + "steps": [ + { + "result": "success", + "completed": "2019-09-11T14:20:59.325129842Z", + "started": "2019-09-11T14:20:55.645743003Z", + "isWaitStep": false, + "stepDoc": "Start a mongo instance (start fresh)", + "step": "StartFresh" + } + ], + "moveDoc": "Start the process", + "move": "Start" + }, + { + "steps": [ + { + "result": "wait", + "completed": null, + "started": "2019-09-11T14:20:59.325272608Z", + "isWaitStep": true, + "stepDoc": "Wait for the replica set to be initialized by another member", + "step": "WaitRsInit" + } + ], + "moveDoc": "Wait for the replica set to be initialized by another member", + "move": "WaitRsInit" + }, + { + "steps": [ + { + "result": "", + "completed": null, + "started": null, + "isWaitStep": true, + "stepDoc": "Wait for featureCompatibilityVersion to be right", + "step": "WaitFeatureCompatibilityVersionCorrect" + } + ], + "moveDoc": "Wait for featureCompatibilityVersion to be right", + "move": "WaitFeatureCompatibilityVersionCorrect" + } + ], + "completed": "2019-09-11T14:21:42.034934358Z", + "started": "2019-09-11T14:20:40.631348806Z" + } + ], + "lastGoalVersionAchieved": 5, + "name": "bar" + } + }, + "statuses": { + "bar": { + "ReplicationStatus": 1, + "ExpectedToBeUp": true, + "LastMongoUpTime": 1568222195, + "IsInGoalState": true + } + } +} diff --git a/pkg/readiness/health/health.go b/pkg/readiness/health/health.go index c23262882..8a14e8168 100644 --- a/pkg/readiness/health/health.go +++ b/pkg/readiness/health/health.go @@ -5,15 +5,23 @@ import ( "time" ) +type replicationStatus int + +const ( + replicationStatusPrimary replicationStatus = 1 + replicationStatusSecondary replicationStatus = 2 +) + type Status struct { Healthiness map[string]processHealth `json:"statuses"` ProcessPlans map[string]MmsDirectorStatus `json:"mmsStatus"` } type processHealth struct { - IsInGoalState bool `json:"IsInGoalState"` - LastMongoUpTime int64 `json:"LastMongoUpTime"` - ExpectedToBeUp bool `json:"ExpectedToBeUp"` + IsInGoalState bool `json:"IsInGoalState"` + LastMongoUpTime int64 `json:"LastMongoUpTime"` + ExpectedToBeUp bool `json:"ExpectedToBeUp"` + ReplicaStatus *replicationStatus `json:"ReplicationStatus,omitempty"` } func (h processHealth) String() string { @@ -43,3 +51,11 @@ type StepStatus struct { Completed *time.Time `json:"completed"` Result string `json:"result"` } + +// isReadyState will return true, meaning a *ready state* in the sense that this Process can +// accept read operations. There are no other states in which the MongoDB server could that +// would mean a Ready State. +func (h processHealth) IsReadyState() bool { + return *h.ReplicaStatus == replicationStatusPrimary || + *h.ReplicaStatus == replicationStatusSecondary +} diff --git a/release.json b/release.json index 8fe9acdbf..243360fe3 100644 --- a/release.json +++ b/release.json @@ -1,7 +1,7 @@ { "mongodb-kubernetes-operator": "0.5.2", "version-upgrade-hook": "1.0.2", - "readiness-probe": "1.0.1", + "readiness-probe": "1.0.2", "agent": { "version": "10.29.0.6830-1", "tools_version": "100.2.0" From 57e37ba42664ea5129197180b4e4c831d4e2b639 Mon Sep 17 00:00:00 2001 From: axelbarjon <47013790+axelbarjon@users.noreply.github.com> Date: Thu, 1 Apr 2021 07:01:14 -0600 Subject: [PATCH 218/790] CLOUD-84084 Added - mongodbcommunity/finalizers to role. Fixes the StatefulSet creation on OpenShift 4.5 (#402) --- config/rbac/role.yaml | 1 + deploy/e2e/role.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index fad40ea2b..4cb3c74c1 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -73,6 +73,7 @@ rules: - mongodbcommunity - mongodbcommunity/status - mongodbcommunity/spec + - mongodbcommunity/finalizers verbs: - create - delete diff --git a/deploy/e2e/role.yaml b/deploy/e2e/role.yaml index ec56d7ba6..0f2178e42 100644 --- a/deploy/e2e/role.yaml +++ b/deploy/e2e/role.yaml @@ -74,6 +74,7 @@ rules: - mongodbcommunity - mongodbcommunity/status - mongodbcommunity/spec + - mongodbcommunity/finalizers verbs: - create - delete From 021ffaccaa6433f3099334b6efdfa7a32e4941dd Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Thu, 1 Apr 2021 14:59:42 +0100 Subject: [PATCH 219/790] Changed FCV handling when upgrading (#412) Co-authored-by: Cian Hatton --- api/v1/mongodbcommunity_types.go | 12 ------ api/v1/mongodbcommunity_types_test.go | 12 ------ controllers/replica_set_controller.go | 3 +- controllers/replicaset_controller_test.go | 26 ++++++++++++ dev_notes/RELEASE_NOTES.md | 19 +++++---- .../automation_config_builder.go | 40 +++++++++++++++++-- .../automation_config_test.go | 3 +- test/e2e/e2eutil.go | 7 ++-- .../replica_set_test.go | 16 ++++---- 9 files changed, 86 insertions(+), 52 deletions(-) diff --git a/api/v1/mongodbcommunity_types.go b/api/v1/mongodbcommunity_types.go index eaa39b78e..2fe0cc5cc 100644 --- a/api/v1/mongodbcommunity_types.go +++ b/api/v1/mongodbcommunity_types.go @@ -483,18 +483,6 @@ func (m MongoDBCommunity) GetAgentScramCredentialsNamespacedName() types.Namespa return types.NamespacedName{Name: fmt.Sprintf("%s-agent-scram-credentials", m.Name), Namespace: m.Namespace} } -// GetFCV returns the feature compatibility version. If no FeatureCompatibilityVersion is specified. -// It uses the major and minor version for whichever version of MongoDB is configured. -func (m MongoDBCommunity) GetFCV() string { - versionToSplit := m.Spec.FeatureCompatibilityVersion - if versionToSplit == "" { - versionToSplit = m.Spec.Version - } - minorIndex := 1 - parts := strings.Split(versionToSplit, ".") - return strings.Join(parts[:minorIndex+1], ".") -} - func (m MongoDBCommunity) DesiredReplicas() int { return m.Spec.Members } diff --git a/api/v1/mongodbcommunity_types_test.go b/api/v1/mongodbcommunity_types_test.go index 3903221f2..692e9732a 100644 --- a/api/v1/mongodbcommunity_types_test.go +++ b/api/v1/mongodbcommunity_types_test.go @@ -16,18 +16,6 @@ func TestMongoDB_MongoURI(t *testing.T) { assert.Equal(t, mdb.MongoURI(), "mongodb://my-big-rs-0.my-big-rs-svc.my-big-namespace.svc.cluster.local:27017,my-big-rs-1.my-big-rs-svc.my-big-namespace.svc.cluster.local:27017,my-big-rs-2.my-big-rs-svc.my-big-namespace.svc.cluster.local:27017,my-big-rs-3.my-big-rs-svc.my-big-namespace.svc.cluster.local:27017,my-big-rs-4.my-big-rs-svc.my-big-namespace.svc.cluster.local:27017") } -func TestGetFCV(t *testing.T) { - mdb := newReplicaSet(3, "my-rs", "my-ns") - mdb.Spec.Version = "4.2.0" - assert.Equal(t, "4.2", mdb.GetFCV()) - - mdb.Spec.FeatureCompatibilityVersion = "4.0" - assert.Equal(t, "4.0", mdb.GetFCV()) - - mdb.Spec.FeatureCompatibilityVersion = "" - assert.Equal(t, "4.2", mdb.GetFCV()) -} - func TestGetScramCredentialsSecretName(t *testing.T) { testusers := []struct { in MongoDBUser diff --git a/controllers/replica_set_controller.go b/controllers/replica_set_controller.go index 9e815dc01..f062e3bea 100644 --- a/controllers/replica_set_controller.go +++ b/controllers/replica_set_controller.go @@ -413,7 +413,6 @@ func buildAutomationConfig(mdb mdbv1.MongoDBCommunity, auth automationconfig.Aut domain := getDomain(mdb.ServiceName(), mdb.Namespace, os.Getenv(clusterDNSName)) zap.S().Debugw("AutomationConfigMembersThisReconciliation", "mdb.AutomationConfigMembersThisReconciliation()", mdb.AutomationConfigMembersThisReconciliation()) - fcv := mdb.GetFCV() return automationconfig.NewBuilder(). SetTopology(automationconfig.ReplicaSetTopology). SetName(mdb.Name). @@ -422,7 +421,7 @@ func buildAutomationConfig(mdb mdbv1.MongoDBCommunity, auth automationconfig.Aut SetReplicaSetHorizons(mdb.Spec.ReplicaSetHorizons). SetPreviousAutomationConfig(currentAc). SetMongoDBVersion(mdb.Spec.Version). - SetFCV(&fcv). + SetFCV(mdb.Spec.FeatureCompatibilityVersion). SetOptions(automationconfig.Options{DownloadBase: "/var/lib/mongodb-mms-automation"}). SetAuth(auth). AddModifications(getMongodConfigModification(mdb)). diff --git a/controllers/replicaset_controller_test.go b/controllers/replicaset_controller_test.go index 241159ec0..9b612043b 100644 --- a/controllers/replicaset_controller_test.go +++ b/controllers/replicaset_controller_test.go @@ -306,6 +306,32 @@ func TestAutomationConfig_versionIsNotBumpedWithNoChanges(t *testing.T) { assert.Equal(t, currentAc.Version, 1) } +func TestAutomationConfigFCVIsNotIncreasedWhenUpgradingMinorVersion(t *testing.T) { + mdb := newTestReplicaSet() + mgr := client.NewManager(&mdb) + r := NewReconciler(mgr) + res, err := r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) + assertReconciliationSuccessful(t, res, err) + + currentAc, err := automationconfig.ReadFromSecret(mgr.Client, types.NamespacedName{Name: mdb.AutomationConfigSecretName(), Namespace: mdb.Namespace}) + assert.NoError(t, err) + assert.Len(t, currentAc.Processes, 3) + assert.Equal(t, currentAc.Processes[0].FeatureCompatibilityVersion, "4.2") + + // Upgrading minor version does not change the FCV on the automationConfig + mdbRef := &mdb + mdbRef.Spec.Version = "4.4.0" + _ = mgr.Client.Update(context.TODO(), mdbRef) + res, err = r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) + assertReconciliationSuccessful(t, res, err) + + currentAc, err = automationconfig.ReadFromSecret(mgr.Client, types.NamespacedName{Name: mdb.AutomationConfigSecretName(), Namespace: mdb.Namespace}) + assert.NoError(t, err) + assert.Len(t, currentAc.Processes, 3) + assert.Equal(t, currentAc.Processes[0].FeatureCompatibilityVersion, "4.2") + +} + func TestAutomationConfig_CustomMongodConfig(t *testing.T) { mdb := newTestReplicaSet() diff --git a/dev_notes/RELEASE_NOTES.md b/dev_notes/RELEASE_NOTES.md index d45a8a768..89ac7e77f 100644 --- a/dev_notes/RELEASE_NOTES.md +++ b/dev_notes/RELEASE_NOTES.md @@ -1,11 +1,16 @@ *(Please use the [release template](release-notes-template.md) as the template for this document)* +# MongoDB Kubernetes Operator 0.5.3 +## Kubernetes Operator +* Bug fixes + * Fixes an issue that prevented the agents from reaching goal state when upgrading minor version of MongoDB. + # MongoDB Kubernetes Operator 0.5.2 ## Kubernetes Operator * Changes * Readiness probe has been moved into an init container from the Agent image. - * Security context is now added when the `MANAGED_SECURITY_CONTEXT` environment variable is not set. + * Security context is now added when the `MANAGED_SECURITY_CONTEXT` environment variable is not set. * Bug fixes * Removed unnecessary environment variable configuration in the openshift samples. * Fixed an issue where the operator would perform unnecessary reconcilliations when Secrets were modified. @@ -15,18 +20,16 @@ ## MongoDBCommunity Resource * Changes - * Added `spec.security.authentication.ignoreUnknownUsers` field. This value defaults to `true`. When enabled, + * Added `spec.security.authentication.ignoreUnknownUsers` field. This value defaults to `true`. When enabled, any MongoDB users added through external sources will not be removed. - - + + ## Miscellaneous * Changes * Internal code refactorings to allow libraries to be imported into other projects. - - + + ## Updated Image Tags * mongodb-kubernetes-operator:0.5.2 * mongodb-agent:10.27.0.6772-1 * mongodb-kubernetes-readinessprobe:1.0.1 [new image] - - diff --git a/pkg/automationconfig/automation_config_builder.go b/pkg/automationconfig/automation_config_builder.go index 92314c600..bdc8413bc 100644 --- a/pkg/automationconfig/automation_config_builder.go +++ b/pkg/automationconfig/automation_config_builder.go @@ -4,6 +4,7 @@ import ( "fmt" "path" + "github.com/blang/semver" "github.com/pkg/errors" "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/versions" @@ -29,7 +30,7 @@ type Builder struct { members int domain string name string - fcv *string + fcv string topology Topology mongodbVersion string previousAC AutomationConfig @@ -100,7 +101,7 @@ func (b *Builder) SetName(name string) *Builder { return b } -func (b *Builder) SetFCV(fcv *string) *Builder { +func (b *Builder) SetFCV(fcv string) *Builder { b.fcv = fcv return b } @@ -155,6 +156,33 @@ func (b *Builder) AddModifications(mod ...Modification) *Builder { return b } +func (b *Builder) setFeatureCompatibilityVersionIfUpgradeIsHappening() error { + // If we are upgrading, we can't increase featureCompatibilityVersion + // as that will make the agent never reach goal state + if len(b.previousAC.Processes) > 0 && b.fcv == "" { + + // Create a x.y.0 version from FCV x.y + previousFCV := b.previousAC.Processes[0].FeatureCompatibilityVersion + previousFCVsemver, err := semver.Make(fmt.Sprintf("%s.0", previousFCV)) + if err != nil { + return errors.Errorf("can't compute semver version from previous FeatureCompatibilityVersion %s", previousFCV) + } + + currentVersionSemver, err := semver.Make(b.mongodbVersion) + if err != nil { + return errors.Errorf("current MongoDB version is not a valid semver version: %s", b.mongodbVersion) + } + + // We would increase FCV here. + // Note: in theory this will also catch upgrade like 4.2.0 -> 4.2.1 but we don't care about those + // as they would not change the FCV + if currentVersionSemver.GT(previousFCVsemver) { + b.fcv = previousFCV + } + } + return nil +} + func (b *Builder) Build() (AutomationConfig, error) { hostnames := make([]string, b.members) for i := 0; i < b.members; i++ { @@ -163,6 +191,10 @@ func (b *Builder) Build() (AutomationConfig, error) { members := make([]ReplicaSetMember, b.members) processes := make([]Process, b.members) + + if err := b.setFeatureCompatibilityVersionIfUpgradeIsHappening(); err != nil { + return AutomationConfig{}, errors.Errorf("can't build the automation config: %s", err) + } for i, h := range hostnames { process := &Process{ @@ -178,8 +210,8 @@ func (b *Builder) Build() (AutomationConfig, error) { Path: path.Join(DefaultAgentLogPath, "/mongodb.log"), }) - if b.fcv != nil { - process.FeatureCompatibilityVersion = *b.fcv + if b.fcv != "" { + process.FeatureCompatibilityVersion = b.fcv } process.SetPort(27017) diff --git a/pkg/automationconfig/automation_config_test.go b/pkg/automationconfig/automation_config_test.go index 3ae18b5c5..00ca5457e 100644 --- a/pkg/automationconfig/automation_config_test.go +++ b/pkg/automationconfig/automation_config_test.go @@ -25,13 +25,12 @@ func defaultMongoDbVersion(version string) MongoDbVersionConfig { } func TestBuildAutomationConfig(t *testing.T) { - fcv := "4.0" ac, err := NewBuilder(). SetName("my-rs"). SetDomain("my-ns.svc.cluster.local"). SetMongoDBVersion("4.2.0"). SetMembers(3). - SetFCV(&fcv). + SetFCV("4.0"). Build() assert.NoError(t, err) diff --git a/test/e2e/e2eutil.go b/test/e2e/e2eutil.go index e83f9023f..bdac62c47 100644 --- a/test/e2e/e2eutil.go +++ b/test/e2e/e2eutil.go @@ -153,10 +153,9 @@ func NewTestMongoDB(name string, namespace string) (mdbv1.MongoDBCommunity, mdbv Namespace: mongodbNamespace, }, Spec: mdbv1.MongoDBCommunitySpec{ - Members: 3, - Type: "ReplicaSet", - Version: "4.4.0", - FeatureCompatibilityVersion: "4.4", + Members: 3, + Type: "ReplicaSet", + Version: "4.4.0", Security: mdbv1.Security{ Authentication: mdbv1.Authentication{ Modes: []mdbv1.AuthMode{"SCRAM"}, diff --git a/test/e2e/replica_set_change_version/replica_set_test.go b/test/e2e/replica_set_change_version/replica_set_test.go index 1b4d08107..1aa6f88d2 100644 --- a/test/e2e/replica_set_change_version/replica_set_test.go +++ b/test/e2e/replica_set_change_version/replica_set_test.go @@ -32,7 +32,7 @@ func TestReplicaSetUpgradeVersion(t *testing.T) { } mdb, user := e2eutil.NewTestMongoDB("mdb0", "") - mdb.Spec.Version = "4.4.0" + mdb.Spec.Version = "4.2.0" _, err := setup.GeneratePasswordForUser(user, ctx, "") if err != nil { @@ -50,10 +50,10 @@ func TestReplicaSetUpgradeVersion(t *testing.T) { t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 1)) t.Run("Ensure Authentication", tester.EnsureAuthenticationIsConfigured(3)) - // Upgrade version to 4.4.1 - t.Run("MongoDB is reachable while version is upgraded", func(t *testing.T) { + // Upgrade minor version to 4.4.0 + t.Run("MongoDB is reachable while minor version is upgraded", func(t *testing.T) { defer tester.StartBackgroundConnectivityTest(t, time.Second*10)() - t.Run("Test Version can be upgraded", mongodbtests.ChangeVersion(&mdb, "4.4.1")) + t.Run("Test Minor Version can be upgraded", mongodbtests.ChangeVersion(&mdb, "4.4.0")) t.Run("StatefulSet has OnDelete update strategy", mongodbtests.StatefulSetHasUpdateStrategy(&mdb, appsv1.OnDeleteStatefulSetStrategyType)) t.Run("Stateful Set Reaches Ready State, after Upgrading", mongodbtests.StatefulSetBecomesReady(&mdb)) t.Run("AutomationConfig's version has been increased", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 2)) @@ -61,12 +61,12 @@ func TestReplicaSetUpgradeVersion(t *testing.T) { t.Run("StatefulSet has RollingUpgrade restart strategy", mongodbtests.StatefulSetHasUpdateStrategy(&mdb, appsv1.RollingUpdateStatefulSetStrategyType)) - // Downgrade version back to 4.4.0 - t.Run("MongoDB is reachable while version is downgraded", func(t *testing.T) { + // Upgrade patch version to 4.4.1 + t.Run("MongoDB is reachable while patch version is upgraded", func(t *testing.T) { defer tester.StartBackgroundConnectivityTest(t, time.Second*10)() - t.Run("Test Version can be downgraded", mongodbtests.ChangeVersion(&mdb, "4.4.0")) + t.Run("Test Patch Version can be upgraded", mongodbtests.ChangeVersion(&mdb, "4.4.1")) t.Run("StatefulSet has OnDelete restart strategy", mongodbtests.StatefulSetHasUpdateStrategy(&mdb, appsv1.OnDeleteStatefulSetStrategyType)) - t.Run("Stateful Set Reaches Ready State, after downgrading", mongodbtests.StatefulSetBecomesReady(&mdb)) + t.Run("Stateful Set Reaches Ready State, after upgrading", mongodbtests.StatefulSetBecomesReady(&mdb)) t.Run("AutomationConfig's version has been increased", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 3)) }) t.Run("StatefulSet has RollingUpgrade restart strategy", mongodbtests.StatefulSetHasUpdateStrategy(&mdb, appsv1.RollingUpdateStatefulSetStrategyType)) From a2712e306b797098774d53c4cd1c57c6044fa709 Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Thu, 1 Apr 2021 16:32:42 +0100 Subject: [PATCH 220/790] Export full automation command (#413) --- controllers/construct/mongodbstatefulset.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/controllers/construct/mongodbstatefulset.go b/controllers/construct/mongodbstatefulset.go index d15757cad..ec2695374 100644 --- a/controllers/construct/mongodbstatefulset.go +++ b/controllers/construct/mongodbstatefulset.go @@ -178,6 +178,10 @@ func BaseAgentCommand() string { return "agent/mongodb-agent -cluster=" + clusterFilePath + " -healthCheckFilePath=" + agentHealthStatusFilePathValue + " -serveStatusPort=5000" } +func AutomationAgentCommand() []string { + return []string{"/bin/bash", "-c", MongodbUserCommand + BaseAgentCommand() + automationAgentOptions} +} + func mongodbAgentContainer(automationConfigSecretName string, volumeMounts []corev1.VolumeMount) container.Modification { securityContext := container.NOOP() managedSecurityContext := envvar.ReadBool(ManagedSecurityContextEnv) @@ -192,7 +196,7 @@ func mongodbAgentContainer(automationConfigSecretName string, volumeMounts []cor container.WithResourceRequirements(resourcerequirements.Defaults()), container.WithVolumeMounts(volumeMounts), securityContext, - container.WithCommand([]string{"/bin/bash", "-c", MongodbUserCommand + BaseAgentCommand() + automationAgentOptions}), + container.WithCommand(AutomationAgentCommand()), container.WithEnvs( corev1.EnvVar{ Name: headlessAgentEnv, From f634a76d4214c12e9d8f19eee05f1bc6fdb9efaa Mon Sep 17 00:00:00 2001 From: Rodrigo Valin Date: Tue, 6 Apr 2021 10:57:51 +0100 Subject: [PATCH 221/790] Adds replicationStatusUndefined. (#411) Used with standalones and sharded clusters. --- pkg/readiness/health/health.go | 22 ++++++++++++++++---- pkg/readiness/health/health_test.go | 32 +++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 4 deletions(-) create mode 100644 pkg/readiness/health/health_test.go diff --git a/pkg/readiness/health/health.go b/pkg/readiness/health/health.go index 8a14e8168..e3f72f931 100644 --- a/pkg/readiness/health/health.go +++ b/pkg/readiness/health/health.go @@ -8,8 +8,17 @@ import ( type replicationStatus int const ( - replicationStatusPrimary replicationStatus = 1 - replicationStatusSecondary replicationStatus = 2 + replicationStatusStartup replicationStatus = 0 + replicationStatusPrimary replicationStatus = 1 + replicationStatusSecondary replicationStatus = 2 + replicationStatusRecovering replicationStatus = 3 + replicationStatusStartup2 replicationStatus = 5 + replicationStatusUnknown replicationStatus = 6 + replicationStatusArbiter replicationStatus = 7 + replicationStatusDown replicationStatus = 8 + replicationStatusRollback replicationStatus = 9 + replicationStatusRemoved replicationStatus = 10 + replicationStatusUndefined replicationStatus = -1 ) type Status struct { @@ -55,7 +64,12 @@ type StepStatus struct { // isReadyState will return true, meaning a *ready state* in the sense that this Process can // accept read operations. There are no other states in which the MongoDB server could that // would mean a Ready State. +// It returns true if the managed process is mongos or standalone (replicationStatusUndefined). func (h processHealth) IsReadyState() bool { - return *h.ReplicaStatus == replicationStatusPrimary || - *h.ReplicaStatus == replicationStatusSecondary + status := *h.ReplicaStatus + if status == replicationStatusUndefined { + return true + } + + return status == replicationStatusPrimary || status == replicationStatusSecondary } diff --git a/pkg/readiness/health/health_test.go b/pkg/readiness/health/health_test.go new file mode 100644 index 000000000..6938b53bc --- /dev/null +++ b/pkg/readiness/health/health_test.go @@ -0,0 +1,32 @@ +package health + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +// TestIsReadyState checks that Primary, Secondary and Undefined always result +// in Ready State. +func TestIsReadyStateNotPrimaryNorSecondary(t *testing.T) { + status := []replicationStatus{replicationStatusUndefined, replicationStatusPrimary, replicationStatusSecondary} + + for _, st := range status { + h := processHealth{ReplicaStatus: &st} + assert.True(t, h.IsReadyState()) + } +} + +// TestIsNotReady any of these states will result on a Database not being ready. +func TestIsNotReady(t *testing.T) { + status := []replicationStatus{ + replicationStatusStartup, replicationStatusRecovering, replicationStatusStartup2, + replicationStatusUnknown, replicationStatusArbiter, replicationStatusDown, + replicationStatusRollback, replicationStatusRemoved, + } + + for _, st := range status { + h := processHealth{ReplicaStatus: &st} + assert.False(t, h.IsReadyState()) + } +} From 7cf843f4a5b29e255d24d54f4f28169244d06185 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 8 Apr 2021 09:12:16 +0000 Subject: [PATCH 222/790] Bump go.mongodb.org/mongo-driver from 1.5.0 to 1.5.1 (#418) --- go.mod | 2 +- go.sum | 16 ++++++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 446ec41bf..d8a3c04a9 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/stretchr/objx v0.3.0 github.com/stretchr/testify v1.7.0 github.com/xdg/stringprep v1.0.0 - go.mongodb.org/mongo-driver v1.5.0 + go.mongodb.org/mongo-driver v1.5.1 go.uber.org/zap v1.16.0 k8s.io/api v0.19.2 k8s.io/apimachinery v0.20.5 diff --git a/go.sum b/go.sum index accddff2f..b6acd0ea5 100644 --- a/go.sum +++ b/go.sum @@ -417,9 +417,12 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= -github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk= -github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= -github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= +github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.0.2 h1:akYIkZ28e6A96dkWNJQu3nmCzH3YfwMPQExUYDaRv7w= +github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= +github.com/xdg-go/stringprep v1.0.2 h1:6iq84/ryjjeRmMJwxutI51F2GIPlP5BfTvXHeYjyhBc= +github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= github.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0= github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= @@ -434,8 +437,8 @@ go.etcd.io/etcd v0.5.0-alpha.5.0.20200819165624-17cef6e3e9d5/go.mod h1:skWido08r go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= -go.mongodb.org/mongo-driver v1.5.0 h1:REddm85e1Nl0JPXGGhgZkgJdG/yOe6xvpXUcYK5WLt0= -go.mongodb.org/mongo-driver v1.5.0/go.mod h1:boiGPFqyBs5R0R5qf2ErokGRekMfwn+MqKaUyHs7wy0= +go.mongodb.org/mongo-driver v1.5.1 h1:9nOVLGDfOaZ9R0tBumx/BcuqkbFpyTCU2r/Po7A2azI= +go.mongodb.org/mongo-driver v1.5.1/go.mod h1:gRXCHX4Jo7J0IJ1oDQyUxF7jfy19UfxniMS4xxMmUqw= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -570,8 +573,9 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From f288c2846928c2a43abbab05c3b172e1741384de Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Thu, 8 Apr 2021 16:58:20 +0100 Subject: [PATCH 223/790] Cloudp 84351 add agent images to dockerfile generator (#420) --- .evergreen.yml | 22 ++------- agent/Dockerfile.ubi | 36 --------------- scripts/ci/build_and_push_agent_image.sh | 11 ----- scripts/dev/dockerfile_generator.py | 46 ++++++++++++++++++- .../dev/templates/agent/Dockerfile.agent_ubi | 8 ++++ .../templates/agent/Dockerfile.agent_ubuntu | 12 +++++ .../dev/templates/agent/Dockerfile.template | 18 ++------ 7 files changed, 73 insertions(+), 80 deletions(-) delete mode 100644 agent/Dockerfile.ubi delete mode 100755 scripts/ci/build_and_push_agent_image.sh create mode 100644 scripts/dev/templates/agent/Dockerfile.agent_ubi create mode 100644 scripts/dev/templates/agent/Dockerfile.agent_ubuntu rename agent/Dockerfile => scripts/dev/templates/agent/Dockerfile.template (71%) diff --git a/.evergreen.yml b/.evergreen.yml index a6cf9e96e..c24463bfd 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -92,20 +92,6 @@ functions: - distro binary: scripts/ci/run_test.sh - # TODO: can be removed once the agent is included in the Dockerfile generator. - build_and_push_agent_image: - - command: subprocess.exec - type: setup - params: - include_expansions_in_env: - - version_id - - quay_user_name - - quay_password - - expire_after - - image - - dockerfile_path - working_dir: mongodb-kubernetes-operator - binary: scripts/ci/build_and_push_agent_image.sh build_and_push_image: - command: subprocess.exec @@ -195,11 +181,11 @@ tasks: commands: - func: clone - func: setup_virtualenv - - func: build_and_push_agent_image + - func: build_and_push_image vars: + image_type: agent_ubuntu image: quay.io/mongodb/mongodb-agent-dev:${version_id} expire_after: 48h - dockerfile_path: agent/Dockerfile - name: build_agent_image_ubi priority: 60 @@ -207,11 +193,11 @@ tasks: commands: - func: clone - func: setup_virtualenv - - func: build_and_push_agent_image + - func: build_and_push_image vars: + image_type: agent_ubi image: quay.io/mongodb/mongodb-agent-ubi-dev:${version_id} expire_after: 48h - dockerfile_path: agent/Dockerfile.ubi - name: build_prehook_image priority: 60 diff --git a/agent/Dockerfile.ubi b/agent/Dockerfile.ubi deleted file mode 100644 index c3d8d7d1c..000000000 --- a/agent/Dockerfile.ubi +++ /dev/null @@ -1,36 +0,0 @@ -FROM registry.access.redhat.com/ubi7/ubi - -ARG agent_version -ARG tools_version - -RUN yum update -y && rm -rf /var/cache/yum - -RUN yum install -y --disableplugin=subscription-manager -q curl \ - hostname nss_wrapper --exclude perl-IO-Socket-SSL procps \ - && yum upgrade -y -q \ - && rm -rf /var/lib/apt/lists/* - -RUN mkdir -p agent \ - && curl --fail --retry 3 --silent https://mciuploads.s3.amazonaws.com/mms-automation/mongodb-mms-build-agent/builds/automation-agent/prod/mongodb-mms-automation-agent-${agent_version}.rhel7_x86_64.tar.gz -o agent/mongodb-agent.tar.gz \ - && tar xfz agent/mongodb-agent.tar.gz \ - && mv mongodb-mms-automation-agent-*/mongodb-mms-automation-agent agent/mongodb-agent \ - && chmod +x agent/mongodb-agent \ - && mkdir -p /var/lib/automation/config \ - && chmod -R +r /var/lib/automation/config \ - && rm agent/mongodb-agent.tar.gz \ - && rm -r mongodb-mms-automation-agent-* - -RUN mkdir -p /var/lib/mongodb-mms-automation \ - && mkdir -p /var/log/mongodb-mms-automation/ \ - && chmod -R +wr /var/log/mongodb-mms-automation/ \ - # ensure that the agent user can write the logs in OpenShift - && touch /var/log/mongodb-mms-automation/readiness.log \ - && chmod ugo+rw /var/log/mongodb-mms-automation/readiness.log - -## Install MongoDB tools. The agent will automatically search the folder and find the binaries. -RUN curl --fail --retry 3 --silent https://downloads.mongodb.org/tools/db/mongodb-database-tools-rhel70-x86_64-${tools_version}.tgz -o mongodb-tools.tgz \ - && tar xfz mongodb-tools.tgz --directory /var/lib/mongodb-mms-automation/ \ - && rm mongodb-tools.tgz - -USER 2000 -CMD ["agent/mongodb-agent", "-cluster=/var/lib/automation/config/automation-config.json"] diff --git a/scripts/ci/build_and_push_agent_image.sh b/scripts/ci/build_and_push_agent_image.sh deleted file mode 100755 index 4a65485d1..000000000 --- a/scripts/ci/build_and_push_agent_image.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash - - -# shellcheck disable=SC1091 -. venv/bin/activate -echo "${quay_password:?}" | docker login "-u=${quay_user_name:?}" quay.io --password-stdin - - -# Providing the quay.expires-after configures quay to delete this image after the provided amount of time -docker build . --label "quay.expires-after=${expire_after:-never}" -f agent/Dockerfile -t "${image:?}" --build-arg tools_version="100.2.0" --build-arg agent_version="10.29.0.6830-1" -docker push "${image:?}" diff --git a/scripts/dev/dockerfile_generator.py b/scripts/dev/dockerfile_generator.py index b1dc08995..5d2f1b852 100755 --- a/scripts/dev/dockerfile_generator.py +++ b/scripts/dev/dockerfile_generator.py @@ -1,3 +1,5 @@ +import json + import jinja2 import argparse import os @@ -9,6 +11,41 @@ GOLANG_TAG = "1.14" +def _shared_agent_params() -> DockerParameters: + with open("release.json", "r") as f: + release = json.loads(f.read()) + + return { + "template_path": "scripts/dev/templates/agent", + "agent_version": release["agent"]["version"], + "tools_version": release["agent"]["tools_version"], + } + + +def agent_ubuntu_params() -> DockerParameters: + params = _shared_agent_params() + params.update( + { + "base_image": "ubuntu:16.04", + "tools_distro": "ubuntu1604-x86_64", + "agent_distro": "linux_x86_64", + } + ) + return params + + +def agent_ubi_params() -> DockerParameters: + params = _shared_agent_params() + params.update( + { + "base_image": "registry.access.redhat.com/ubi7/ubi", + "tools_distro": "rhel70-x86_64", + "agent_distro": "rhel7_x86_64", + } + ) + return params + + def operator_params(files_to_add: List[str]) -> DockerParameters: return { "builder": True, @@ -20,7 +57,8 @@ def operator_params(files_to_add: List[str]) -> DockerParameters: def e2e_params(files_to_add: List[str]) -> DockerParameters: return { - "base_image": f"golang:{GOLANG_TAG}", # TODO: make this image smaller, error: 'exec: "gcc": executable file not found in $PATH' with golang:alpine + "base_image": f"golang:{GOLANG_TAG}", + # TODO: make this image smaller, error: 'exec: "gcc": executable file not found in $PATH' with golang:alpine "files_to_add": files_to_add, } @@ -29,12 +67,16 @@ def render(image_name: str, files_to_add: List[str]) -> str: param_dict = { "e2e": e2e_params(files_to_add), "operator": operator_params(files_to_add), + "agent_ubi": agent_ubi_params(), + "agent_ubuntu": agent_ubuntu_params(), } render_values = param_dict.get(image_name, dict()) + search_path = str(render_values.get("template_path", "scripts/dev/templates")) + env = jinja2.Environment() - env.loader = jinja2.FileSystemLoader(searchpath="scripts/dev/templates") + env.loader = jinja2.FileSystemLoader(searchpath=search_path) return env.get_template(f"Dockerfile.{image_name}").render(render_values) diff --git a/scripts/dev/templates/agent/Dockerfile.agent_ubi b/scripts/dev/templates/agent/Dockerfile.agent_ubi new file mode 100644 index 000000000..a00bee4de --- /dev/null +++ b/scripts/dev/templates/agent/Dockerfile.agent_ubi @@ -0,0 +1,8 @@ +{% extends "Dockerfile.template" %} + +{% block packages -%} +RUN yum install -y --disableplugin=subscription-manager -q curl \ + hostname nss_wrapper --exclude perl-IO-Socket-SSL procps \ + && yum upgrade -y -q \ + && rm -rf /var/lib/apt/lists/* +{% endblock -%} diff --git a/scripts/dev/templates/agent/Dockerfile.agent_ubuntu b/scripts/dev/templates/agent/Dockerfile.agent_ubuntu new file mode 100644 index 000000000..ddbea8125 --- /dev/null +++ b/scripts/dev/templates/agent/Dockerfile.agent_ubuntu @@ -0,0 +1,12 @@ +{% extends "Dockerfile.template" %} + + +{% block packages -%} +RUN apt-get -qq update \ + && apt-get -y -qq install \ + curl \ + libnss-wrapper \ + && apt-get upgrade -y -qq \ + && apt-get dist-upgrade -y -qq \ + && rm -rf /var/lib/apt/lists/* +{% endblock -%} diff --git a/agent/Dockerfile b/scripts/dev/templates/agent/Dockerfile.template similarity index 71% rename from agent/Dockerfile rename to scripts/dev/templates/agent/Dockerfile.template index 815c7fd9c..260dfe9bb 100644 --- a/agent/Dockerfile +++ b/scripts/dev/templates/agent/Dockerfile.template @@ -1,18 +1,10 @@ -FROM ubuntu:16.04 +FROM {{base_image}} -ARG agent_version -ARG tools_version - -RUN apt-get -qq update \ - && apt-get -y -qq install \ - curl \ - libnss-wrapper \ - && apt-get upgrade -y -qq \ - && apt-get dist-upgrade -y -qq \ - && rm -rf /var/lib/apt/lists/* +{% block packages -%} +{% endblock -%} RUN mkdir -p agent \ - && curl --fail --retry 3 --silent https://mciuploads.s3.amazonaws.com/mms-automation/mongodb-mms-build-agent/builds/automation-agent/prod/mongodb-mms-automation-agent-${agent_version}.linux_x86_64.tar.gz -o agent/mongodb-agent.tar.gz \ + && curl --fail --retry 3 --silent https://mciuploads.s3.amazonaws.com/mms-automation/mongodb-mms-build-agent/builds/automation-agent/prod/mongodb-mms-automation-agent-{{agent_version}}.{{agent_distro}}.tar.gz -o agent/mongodb-agent.tar.gz \ && tar xfz agent/mongodb-agent.tar.gz \ && mv mongodb-mms-automation-agent-*/mongodb-mms-automation-agent agent/mongodb-agent \ && chmod +x agent/mongodb-agent \ @@ -29,7 +21,7 @@ RUN mkdir -p /var/lib/mongodb-mms-automation \ && chmod ugo+rw /var/log/mongodb-mms-automation/readiness.log # Install MongoDB tools. The agent will automatically search the folder and find the binaries. -RUN curl --fail --retry 3 --silent https://downloads.mongodb.org/tools/db/mongodb-database-tools-ubuntu1604-x86_64-${tools_version}.tgz -o mongodb-tools.tgz \ +RUN curl --fail --retry 3 --silent https://downloads.mongodb.org/tools/db/mongodb-database-tools-{{tools_distro}}-{{tools_version}}.tgz -o mongodb-tools.tgz \ && tar xfz mongodb-tools.tgz --directory /var/lib/mongodb-mms-automation/ \ && rm mongodb-tools.tgz From 1114a52bc9660efb3644f8dd9f8c979144611812 Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Tue, 20 Apr 2021 12:02:40 +0100 Subject: [PATCH 224/790] CLOUDP-87241: Fix readiness probe check for MongoDB up (#429) --- cmd/readiness/main.go | 66 ++------------- cmd/readiness/readiness_test.go | 16 +--- ...json => health-status-no-replication.json} | 1 - .../health-status-not-started-yet.json | 82 ------------------- pkg/readiness/health/health.go | 8 +- pkg/readiness/health/health_test.go | 8 +- 6 files changed, 20 insertions(+), 161 deletions(-) rename cmd/readiness/testdata/{health-status-readable-state.json => health-status-no-replication.json} (99%) delete mode 100644 cmd/readiness/testdata/health-status-not-started-yet.json diff --git a/cmd/readiness/main.go b/cmd/readiness/main.go index 209f3d5f4..a01eddcb6 100644 --- a/cmd/readiness/main.go +++ b/cmd/readiness/main.go @@ -226,7 +226,6 @@ func main() { // isInReadyState checks the MongoDB Server state. It returns true if the state // is PRIMARY or SECONDARY. -// This function will always return true if the agent doesn't publish this state. func isInReadyState(health health.Status) bool { if len(health.Healthiness) == 0 { return true @@ -234,67 +233,14 @@ func isInReadyState(health health.Status) bool { for _, processHealth := range health.Healthiness { // We know this loop should run only once, in Kubernetes there's // only 1 server managed per host. - if processHealth.ReplicaStatus == nil { - // We always return true if the Agent does not publish mongodb - // server state - return true - } - if mongoDbServerHasStarted(health) { - // There should be only one entry reported for this Pod. - return processHealth.IsReadyState() - } - } - return false -} + // Every time the process health is created by the agent, + // it checks if the MongoDB process is up and populates this field + // (https://github.com/10gen/mms-automation/blob/bb72f74a22d98cfa635c1317e623386b089dc69f/go_planner/src/com.tengen/cm/healthcheck/status.go#L43) + // So it's enough to check that this value is not the zero-value for int64 -// mongoDbServerHasStarted checks if the current plan includes a Move and a Step -// of type "StartFresh" with a Result of "success". -// -// This function will return true if the agent has been able to successfully -// start the MongoDB server. -func mongoDbServerHasStarted(health health.Status) bool { - plan := findCurrentPlan(health.ProcessPlans) - if plan == nil { - return false + // The case in which the agent is too old to publish replication status is handled inside "IsReadyState" + return processHealth.LastMongoUpTime != 0 && processHealth.IsReadyState() } - - for _, move := range plan.Moves { - for _, step := range move.Steps { - if step.Step == "StartFresh" && step.Result == "success" { - return true - } - } - } - return false } - -// findCurrentPlan returns the current plan as informed by the Agent. -// -// The current plan is the last plan from the `processStatuses` parameter, this -// is, the plan that's currently being processed by the agent. -func findCurrentPlan(processStatuses map[string]health.MmsDirectorStatus) *health.PlanStatus { - var currentPlan *health.PlanStatus - if len(processStatuses) == 0 { - // Seems shouldn't happen but let's check anyway - may be needs to be - // changed to Info if this happens. - logger.Warnf("There is no information about Agent process plans") - return nil - } - if len(processStatuses) > 1 { - logger.Errorf("Only one process status is expected but got %d!", len(processStatuses)) - return nil - } - // There is only one process managed by the Agent - so will only check one - // iteration. - for k, v := range processStatuses { - if len(v.Plans) == 0 { - logger.Errorf("The process %s doesn't contain any plans!", k) - return nil - } - currentPlan = v.Plans[len(v.Plans)-1] - } - - return currentPlan -} diff --git a/cmd/readiness/readiness_test.go b/cmd/readiness/readiness_test.go index a11a2607d..988acb571 100644 --- a/cmd/readiness/readiness_test.go +++ b/cmd/readiness/readiness_test.go @@ -118,6 +118,10 @@ func TestHeadlessAgentReachedGoal(t *testing.T) { } func TestPodReadiness(t *testing.T) { + t.Run("Pod readiness is correctly checked when no ReplicationStatus is present on the file ", func(t *testing.T) { + assert.True(t, isPodReady(testConfig("testdata/health-status-no-replication.json"))) + }) + t.Run("MongoDB replication state is reported by agents", func(t *testing.T) { assert.True(t, isPodReady(testConfig("testdata/health-status-ok-no-replica-status.json"))) }) @@ -131,18 +135,6 @@ func TestPodReadiness(t *testing.T) { }) } -func TestServerHasAlreadyStarted(t *testing.T) { - t.Run("Agent should report server has started", func(t *testing.T) { - healthDoc := readHealthinessFile("testdata/health-status-readable-state.json") - assert.True(t, mongoDbServerHasStarted(healthDoc)) - }) - - t.Run("Agent should report server has not started", func(t *testing.T) { - healthDoc := readHealthinessFile("testdata/health-status-not-started-yet.json") - assert.False(t, mongoDbServerHasStarted(healthDoc)) - }) -} - func readHealthinessFile(path string) health.Status { fd, _ := os.Open(path) health, _ := readAgentHealthStatus(fd) diff --git a/cmd/readiness/testdata/health-status-readable-state.json b/cmd/readiness/testdata/health-status-no-replication.json similarity index 99% rename from cmd/readiness/testdata/health-status-readable-state.json rename to cmd/readiness/testdata/health-status-no-replication.json index ab2dfadb3..0398e8e4e 100644 --- a/cmd/readiness/testdata/health-status-readable-state.json +++ b/cmd/readiness/testdata/health-status-no-replication.json @@ -73,7 +73,6 @@ }, "statuses": { "bar": { - "ReplicationStatus": 1, "ExpectedToBeUp": true, "LastMongoUpTime": 1568222195, "IsInGoalState": true diff --git a/cmd/readiness/testdata/health-status-not-started-yet.json b/cmd/readiness/testdata/health-status-not-started-yet.json deleted file mode 100644 index 4f3598a8c..000000000 --- a/cmd/readiness/testdata/health-status-not-started-yet.json +++ /dev/null @@ -1,82 +0,0 @@ -{ - "mmsStatus": { - "bar": { - "errorString": "", - "errorCode": 0, - "plans": [ - { - "moves": [ - { - "steps": [ - { - "result": "success", - "completed": "2019-09-11T14:20:55.645615846Z", - "started": "2019-09-11T14:20:40.631404367Z", - "isWaitStep": false, - "stepDoc": "Download mongodb binaries (may take a while)", - "step": "Download" - } - ], - "moveDoc": "Download mongodb binaries", - "move": "Download" - }, - { - "steps": [ - { - "result": "wait", - "completed": "2019-09-11T14:20:59.325129842Z", - "started": "2019-09-11T14:20:55.645743003Z", - "isWaitStep": false, - "stepDoc": "Start a mongo instance (start fresh)", - "step": "StartFresh" - } - ], - "moveDoc": "Start the process", - "move": "Start" - }, - { - "steps": [ - { - "result": "wait", - "completed": null, - "started": "2019-09-11T14:20:59.325272608Z", - "isWaitStep": true, - "stepDoc": "Wait for the replica set to be initialized by another member", - "step": "WaitRsInit" - } - ], - "moveDoc": "Wait for the replica set to be initialized by another member", - "move": "WaitRsInit" - }, - { - "steps": [ - { - "result": "", - "completed": null, - "started": null, - "isWaitStep": true, - "stepDoc": "Wait for featureCompatibilityVersion to be right", - "step": "WaitFeatureCompatibilityVersionCorrect" - } - ], - "moveDoc": "Wait for featureCompatibilityVersion to be right", - "move": "WaitFeatureCompatibilityVersionCorrect" - } - ], - "completed": "2019-09-11T14:21:42.034934358Z", - "started": "2019-09-11T14:20:40.631348806Z" - } - ], - "lastGoalVersionAchieved": 5, - "name": "bar" - } - }, - "statuses": { - "bar": { - "ReplicationStatus": 1, - "ExpectedToBeUp": true, - "LastMongoUpTime": 1568222195, - "IsInGoalState": true - } - } -} diff --git a/pkg/readiness/health/health.go b/pkg/readiness/health/health.go index e3f72f931..d538d2a91 100644 --- a/pkg/readiness/health/health.go +++ b/pkg/readiness/health/health.go @@ -30,7 +30,7 @@ type processHealth struct { IsInGoalState bool `json:"IsInGoalState"` LastMongoUpTime int64 `json:"LastMongoUpTime"` ExpectedToBeUp bool `json:"ExpectedToBeUp"` - ReplicaStatus *replicationStatus `json:"ReplicationStatus,omitempty"` + ReplicaStatus *replicationStatus `json:"ReplicationStatus"` } func (h processHealth) String() string { @@ -64,8 +64,12 @@ type StepStatus struct { // isReadyState will return true, meaning a *ready state* in the sense that this Process can // accept read operations. There are no other states in which the MongoDB server could that // would mean a Ready State. -// It returns true if the managed process is mongos or standalone (replicationStatusUndefined). +// It returns true if the managed process is mongos or standalone (replicationStatusUndefined) +// or if the agent doesn't publish the replica status (older agents) func (h processHealth) IsReadyState() bool { + if h.ReplicaStatus == nil { + return true + } status := *h.ReplicaStatus if status == replicationStatusUndefined { return true diff --git a/pkg/readiness/health/health_test.go b/pkg/readiness/health/health_test.go index 6938b53bc..6690736b1 100644 --- a/pkg/readiness/health/health_test.go +++ b/pkg/readiness/health/health_test.go @@ -11,8 +11,8 @@ import ( func TestIsReadyStateNotPrimaryNorSecondary(t *testing.T) { status := []replicationStatus{replicationStatusUndefined, replicationStatusPrimary, replicationStatusSecondary} - for _, st := range status { - h := processHealth{ReplicaStatus: &st} + for i := range status { + h := processHealth{ReplicaStatus: &status[i]} assert.True(t, h.IsReadyState()) } } @@ -25,8 +25,8 @@ func TestIsNotReady(t *testing.T) { replicationStatusRollback, replicationStatusRemoved, } - for _, st := range status { - h := processHealth{ReplicaStatus: &st} + for i := range status { + h := processHealth{ReplicaStatus: &status[i]} assert.False(t, h.IsReadyState()) } } From 9038ebdffcd26e1950b68e86caa59a8d198d5544 Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Tue, 20 Apr 2021 16:06:34 +0100 Subject: [PATCH 225/790] CLOUDP-83092: Add sonar inventory (#422) --- .evergreen.yml | 28 ++++-- .gitignore | 4 + inventory.yaml | 65 +++++++++++++ pipeline.py | 91 +++++++++++++++++++ requirements.txt | 3 +- scripts/ci/build_and_push_image.sh | 1 + scripts/ci/build_and_push_image_sonar.sh | 10 ++ scripts/ci/config.json | 2 +- scripts/ci/setup_virtualenv.sh | 4 + scripts/dev/__init__.py | 0 scripts/dev/dev_config.py | 4 + .../{Dockerfile.agent_ubi => Dockerfile.ubi} | 2 + ...kerfile.agent_ubuntu => Dockerfile.ubuntu} | 1 + 13 files changed, 205 insertions(+), 10 deletions(-) create mode 100644 inventory.yaml create mode 100644 pipeline.py create mode 100755 scripts/ci/build_and_push_image_sonar.sh create mode 100644 scripts/dev/__init__.py rename scripts/dev/templates/agent/{Dockerfile.agent_ubi => Dockerfile.ubi} (81%) rename scripts/dev/templates/agent/{Dockerfile.agent_ubuntu => Dockerfile.ubuntu} (88%) diff --git a/.evergreen.yml b/.evergreen.yml index c24463bfd..e2bcdd0e3 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -8,6 +8,8 @@ functions: params: working_dir: mongodb-kubernetes-operator binary: scripts/ci/setup_virtualenv.sh + include_expansions_in_env: + - sonar_github_token clone: - command: subprocess.exec @@ -107,6 +109,20 @@ functions: working_dir: mongodb-kubernetes-operator binary: scripts/ci/build_and_push_image.sh + build_and_push_image_sonar: + - command: subprocess.exec + type: setup + params: + env: + MONGODB_COMMUNITY_CONFIG: ${workdir}/mongodb-kubernetes-operator/scripts/ci/config.json + include_expansions_in_env: + - version_id + - quay_user_name + - quay_password + - image_name + working_dir: mongodb-kubernetes-operator + binary: scripts/ci/build_and_push_image_sonar.sh + release_docker_image: - command: subprocess.exec type: system @@ -181,11 +197,9 @@ tasks: commands: - func: clone - func: setup_virtualenv - - func: build_and_push_image + - func: build_and_push_image_sonar vars: - image_type: agent_ubuntu - image: quay.io/mongodb/mongodb-agent-dev:${version_id} - expire_after: 48h + image_name: agent-ubuntu - name: build_agent_image_ubi priority: 60 @@ -193,11 +207,9 @@ tasks: commands: - func: clone - func: setup_virtualenv - - func: build_and_push_image + - func: build_and_push_image_sonar vars: - image_type: agent_ubi - image: quay.io/mongodb/mongodb-agent-ubi-dev:${version_id} - expire_after: 48h + image_name: agent-ubi - name: build_prehook_image priority: 60 diff --git a/.gitignore b/.gitignore index 96ffc511f..d1e3f2356 100644 --- a/.gitignore +++ b/.gitignore @@ -88,3 +88,7 @@ logs/* testbin/bin # OSX Trash .DS_Store + +# ignore files generated by sonar +Dockerfile.ubi-* +Dockerfile.ubuntu-* diff --git a/inventory.yaml b/inventory.yaml new file mode 100644 index 000000000..3ad59ff7a --- /dev/null +++ b/inventory.yaml @@ -0,0 +1,65 @@ +vars: + registry: + +images: + - name: agent-ubuntu + vars: + context: . + template_context: scripts/dev/templates/agent + + stages: + - name: agent-template-ubuntu + task_type: dockerfile_template + distro: ubuntu + + inputs: + - agent_version + - tools_version + - tools_distro + - agent_distro + + output: + - dockerfile: scripts/dev/templates/agent/Dockerfile.ubuntu-$(inputs.params.version_id) + + - name: agent-ubuntu-build + task_type: docker_build + dockerfile: scripts/dev/templates/agent/Dockerfile.ubuntu-$(inputs.params.version_id) + + labels: + quay.expires-after: 48h + + output: + - registry: $(inputs.params.registry)/mongodb-agent-ubuntu-dev + tag: $(inputs.params.version_id) + + + - name: agent-ubi + vars: + context: . + template_context: scripts/dev/templates/agent + + stages: + - name: agent-template-ubi + task_type: dockerfile_template + distro: ubi + + inputs: + - agent_version + - tools_version + - tools_distro + - agent_distro + + output: + - dockerfile: scripts/dev/templates/agent/Dockerfile.ubi-$(inputs.params.version_id) + + + - name: agent-ubi-build + task_type: docker_build + dockerfile: scripts/dev/templates/agent/Dockerfile.ubi-$(inputs.params.version_id) + + labels: + quay.expires-after: 48h + + output: + - registry: $(inputs.params.registry)/mongodb-agent-ubi-dev + tag: $(inputs.params.version_id) diff --git a/pipeline.py b/pipeline.py new file mode 100644 index 000000000..2440388ec --- /dev/null +++ b/pipeline.py @@ -0,0 +1,91 @@ +import argparse +import json +import sys +from typing import Dict, Optional + +from sonar.sonar import process_image + +from scripts.dev.dev_config import load_config, DevConfig + +VALID_IMAGE_NAMES = frozenset(["agent-ubi", "agent-ubuntu"]) + +DEFAULT_IMAGE_TYPE = "ubuntu" +DEFAULT_NAMESPACE = "default" + + +def build_agent_image_ubi(config: DevConfig) -> None: + image_name = "agent-ubi" + with open("release.json") as f: + release = json.loads(f.read()) + args = { + "agent_version": release["agent"]["version"], + "tools_version": release["agent"]["tools_version"], + "tools_distro": "ubuntu1604-x86_64", + "agent_distro": "linux_x86_64", + "registry": config.repo_url, + } + sonar_build_image( + image_name, + args=args, + ) + + +def build_agent_image_ubuntu(config: DevConfig) -> None: + image_name = "agent-ubuntu" + with open("release.json") as f: + release = json.loads(f.read()) + args = { + "agent_version": release["agent"]["version"], + "tools_version": release["agent"]["tools_version"], + "tools_distro": "rhel70-x86_64", + "agent_distro": "rhel7_x86_64", + "registry": config.repo_url, + } + sonar_build_image( + image_name, + args=args, + ) + + +def sonar_build_image( + image_name: str, + args: Optional[Dict[str, str]] = None, + inventory: str = "inventory.yaml", +) -> None: + """Calls sonar to build `image_name` with arguments defined in `args`.""" + process_image( + image_name, + build_args=args, + inventory=inventory, + include_tags=[], + skip_tags=[], + ) + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--image-name", type=str) + return parser.parse_args() + + +def main() -> int: + args = _parse_args() + + image_name = args.image_name + if image_name not in VALID_IMAGE_NAMES: + print( + f"Image name [{image_name}] is not valid. Must be one of [{', '.join(VALID_IMAGE_NAMES)}]" + ) + return 1 + + agent_build_function = { + "agent-ubi": build_agent_image_ubi, + "agent-ubuntu": build_agent_image_ubuntu, + }[image_name] + + agent_build_function(load_config()) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/requirements.txt b/requirements.txt index 2dc8a16f6..574ca74b9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,8 @@ -docker==4.2.0 +docker==4.3.1 kubernetes==11.0.0 jinja2==2.11.3 PyYAML==5.4.1 black==20.8b1 mypy==0.782 tqdm==v4.49.0 +boto3==1.16.21 diff --git a/scripts/ci/build_and_push_image.sh b/scripts/ci/build_and_push_image.sh index 4fae59328..30006795b 100755 --- a/scripts/ci/build_and_push_image.sh +++ b/scripts/ci/build_and_push_image.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash +set +x # shellcheck disable=SC1091 . venv/bin/activate diff --git a/scripts/ci/build_and_push_image_sonar.sh b/scripts/ci/build_and_push_image_sonar.sh new file mode 100755 index 000000000..ea9f52548 --- /dev/null +++ b/scripts/ci/build_and_push_image_sonar.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +set +x + +# shellcheck disable=SC1091 +. venv/bin/activate +echo "${quay_password:?}" | docker login "-u=${quay_user_name:?}" quay.io --password-stdin + +# shellcheck disable=SC2154 +python3 pipeline.py --image-name "${image_name}" diff --git a/scripts/ci/config.json b/scripts/ci/config.json index de612ceba..565aa58d2 100644 --- a/scripts/ci/config.json +++ b/scripts/ci/config.json @@ -5,6 +5,6 @@ "e2e_image": "community-operator-e2e", "version_upgrade_hook_image": "community-operator-version-upgrade-post-start-hook", "testrunner_image": "community-operator-testrunner", - "agent_image_ubuntu": "mongodb-agent-dev", + "agent_image_ubuntu": "mongodb-agent-ubuntu-dev", "agent_image_ubi": "mongodb-agent-ubi-dev" } diff --git a/scripts/ci/setup_virtualenv.sh b/scripts/ci/setup_virtualenv.sh index cbbf71bee..af8cf9b6a 100755 --- a/scripts/ci/setup_virtualenv.sh +++ b/scripts/ci/setup_virtualenv.sh @@ -3,4 +3,8 @@ # shellcheck disable=SC1091 virtualenv --python /opt/python/3.7/bin/python3 ./venv . venv/bin/activate + pip3 install -r ./requirements.txt + +# shellcheck disable=SC2154 +pip3 install "git+https://${sonar_github_token}@github.com/10gen/sonar.git@0.0.9" diff --git a/scripts/dev/__init__.py b/scripts/dev/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/dev/dev_config.py b/scripts/dev/dev_config.py index 5c6944466..badc6004e 100644 --- a/scripts/dev/dev_config.py +++ b/scripts/dev/dev_config.py @@ -42,6 +42,10 @@ def namespace(self) -> str: def repo_url(self) -> str: return self._config["repo_url"] + @property + def expire_after(self) -> str: + return self._config.get("expire_after", "never") + @property def operator_image(self) -> str: return self._config["operator_image"] diff --git a/scripts/dev/templates/agent/Dockerfile.agent_ubi b/scripts/dev/templates/agent/Dockerfile.ubi similarity index 81% rename from scripts/dev/templates/agent/Dockerfile.agent_ubi rename to scripts/dev/templates/agent/Dockerfile.ubi index a00bee4de..a70db5ef2 100644 --- a/scripts/dev/templates/agent/Dockerfile.agent_ubi +++ b/scripts/dev/templates/agent/Dockerfile.ubi @@ -1,5 +1,7 @@ {% extends "Dockerfile.template" %} +{% set base_image = "registry.access.redhat.com/ubi7/ubi" %} + {% block packages -%} RUN yum install -y --disableplugin=subscription-manager -q curl \ hostname nss_wrapper --exclude perl-IO-Socket-SSL procps \ diff --git a/scripts/dev/templates/agent/Dockerfile.agent_ubuntu b/scripts/dev/templates/agent/Dockerfile.ubuntu similarity index 88% rename from scripts/dev/templates/agent/Dockerfile.agent_ubuntu rename to scripts/dev/templates/agent/Dockerfile.ubuntu index ddbea8125..c0cca8bb7 100644 --- a/scripts/dev/templates/agent/Dockerfile.agent_ubuntu +++ b/scripts/dev/templates/agent/Dockerfile.ubuntu @@ -1,5 +1,6 @@ {% extends "Dockerfile.template" %} +{% set base_image = "ubuntu:16.04" %} {% block packages -%} RUN apt-get -qq update \ From 7ee21edea034bc31b12e52ab82fa77373d376744 Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Tue, 20 Apr 2021 17:43:44 +0100 Subject: [PATCH 226/790] State Machine Design #1 (#427) --- pkg/util/result/reconciliationresults.go | 22 +- pkg/util/state/statemachine.go | 185 ++++++++++++ pkg/util/state/statemachine_test.go | 349 +++++++++++++++++++++++ 3 files changed, 554 insertions(+), 2 deletions(-) create mode 100644 pkg/util/state/statemachine.go create mode 100644 pkg/util/state/statemachine_test.go diff --git a/pkg/util/result/reconciliationresults.go b/pkg/util/result/reconciliationresults.go index 2bfbc8b71..51f9a3268 100644 --- a/pkg/util/result/reconciliationresults.go +++ b/pkg/util/result/reconciliationresults.go @@ -6,8 +6,26 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" ) -func ShouldRequeue(result reconcile.Result, err error) bool { - return err != nil || result.Requeue || result.RequeueAfter > 0 +// StateComplete returns the result required for the State Machine +// to execute the next State in the next reconciliation. +func StateComplete() (reconcile.Result, error, bool) { + return retry(0, true) +} + +// RetryState returns the result required for the State Machine to +// execute this state in the next reconciliation. +func RetryState(after int) (reconcile.Result, error, bool) { + return retry(after, false) +} + +// FailedState returns the result required for the State to retry +// the current State. +func FailedState() (reconcile.Result, error, bool) { + return RetryState(1) +} + +func retry(after int, isComplete bool) (reconcile.Result, error, bool) { + return reconcile.Result{Requeue: true, RequeueAfter: time.Second * time.Duration(after)}, nil, isComplete } func OK() (reconcile.Result, error) { diff --git a/pkg/util/state/statemachine.go b/pkg/util/state/statemachine.go new file mode 100644 index 000000000..feb5bcf52 --- /dev/null +++ b/pkg/util/state/statemachine.go @@ -0,0 +1,185 @@ +package state + +import ( + "github.com/pkg/errors" + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +// State should provide a unique name, and a Reconcile function. +// This function gets called by the Machine. The first two returned values +// are returned to the caller, while the 3rd value is used to indicate if the +// State completed successfully. A value of true will move onto the next State, +// a value of false will repeat this State until true is returned. +type State struct { + // Name should be a unique identifier of the State + Name string + + // Reconcile should perform the actual reconciliation of the State. + // The reconcile.Result and error should be returned from the controller. + // the boolean value indicates that the State has been successfully completed. + Reconcile func() (reconcile.Result, error, bool) + + // OnEnter executes before the Reconcile function is called. + OnEnter func() error +} + +// transition represents a transition between two states. +type transition struct { + from, to State + predicate TransitionPredicate +} + +// Saver saves the next state name that should be reconciled. +// If a transition is A -> B, after A finishes reconciling `SaveNextState("B")` will be called. +type Saver interface { + SaveNextState(nsName types.NamespacedName, stateName string) error +} + +// Loader should return the value saved by Saver. +type Loader interface { + LoadNextState(nsName types.NamespacedName) (string, error) +} + +// SaveLoader can both load and save the name of a state. +type SaveLoader interface { + Saver + Loader +} + +// TransitionPredicate is used to indicate if two States should be connected. +type TransitionPredicate func() bool + +var FromBool = func(b bool) TransitionPredicate { + return func() bool { + return b + } +} + +// directTransition can be used to ensure two states are directly linked. +var directTransition = FromBool(true) + +// Machine allows for several States to be registered via "AddTransition" +// When calling Reconcile, the corresponding State will be used based on the values +// stored/loaded from the SaveLoader. A Machine corresponds to a single Kubernetes resource. +type Machine struct { + allTransitions map[string][]transition + currentState *State + logger *zap.SugaredLogger + saveLoader SaveLoader + states map[string]State + nsName types.NamespacedName +} + +// NewStateMachine returns a Machine, it must be set up with calls to "AddTransition(s1, s2, predicate)" +// before Reconcile is called. +func NewStateMachine(saver SaveLoader, nsName types.NamespacedName, logger *zap.SugaredLogger) *Machine { + return &Machine{ + allTransitions: map[string][]transition{}, + logger: logger, + saveLoader: saver, + states: map[string]State{}, + nsName: nsName, + } +} + +// Reconcile will reconcile the currently active State. This method should be called +// from the controllers. +func (m *Machine) Reconcile() (reconcile.Result, error) { + + if err := m.determineState(); err != nil { + m.logger.Errorf("error initializing starting state: %s", err) + return reconcile.Result{}, err + } + + m.logger.Infof("Reconciling state: [%s]", m.currentState.Name) + + if m.currentState.OnEnter != nil { + if err := m.currentState.OnEnter(); err != nil { + m.logger.Debugf("Error reconciling state [%s]: %s", m.currentState.Name, err) + return reconcile.Result{}, err + } + } + + res, err, isComplete := m.currentState.Reconcile() + + if err != nil { + m.logger.Debugf("Error reconciling state [%s]: %s", m.currentState.Name, err) + return res, err + } + + if isComplete { + m.logger.Debugf("Completed state: [%s]", m.currentState.Name) + + transition := m.getTransitionForState(*m.currentState) + nextState := "" + if transition != nil { + nextState = transition.to.Name + } + + if nextState != "" { + m.logger.Debugf("preparing transition [%s] -> [%s]", m.currentState.Name, nextState) + } + + if err := m.saveLoader.SaveNextState(m.nsName, nextState); err != nil { + m.logger.Debugf("Error marking state: [%s] as complete: %s", m.currentState.Name, err) + return reconcile.Result{}, err + } + return res, err + } + + m.logger.Debugf("State [%s] is not yet complete", m.currentState.Name) + + return res, err +} + +// determineState ensures that "currentState" has a valid value. +// the state that is loaded comes from the Loader. +func (m *Machine) determineState() error { + currentStateName, err := m.saveLoader.LoadNextState(m.nsName) + if err != nil { + return errors.Errorf("could not load starting state: %s", err) + } + nextState, ok := m.states[currentStateName] + if !ok { + return errors.Errorf("could not determine state %s as it was not added to the State Machine", currentStateName) + } + m.currentState = &nextState + return nil +} + +// AddDirectTransition creates a transition between the two +// provided states which will always be valid. +func (m *Machine) AddDirectTransition(from, to State) { + m.AddTransition(from, to, directTransition) +} + +// AddTransition creates a transition between the two states if the given +// predicate returns true. +func (m *Machine) AddTransition(from, to State, predicate TransitionPredicate) { + _, ok := m.allTransitions[from.Name] + if !ok { + m.allTransitions[from.Name] = []transition{} + } + m.allTransitions[from.Name] = append(m.allTransitions[from.Name], transition{ + from: from, + to: to, + predicate: predicate, + }) + + m.states[from.Name] = from + m.states[to.Name] = to +} + +// getTransitionForState returns the first transition it finds that is available +// from the current state. +func (m *Machine) getTransitionForState(s State) *transition { + transitions := m.allTransitions[s.Name] + for _, t := range transitions { + if t.predicate() { + return &t + } + } + return nil +} diff --git a/pkg/util/state/statemachine_test.go b/pkg/util/state/statemachine_test.go new file mode 100644 index 000000000..650749711 --- /dev/null +++ b/pkg/util/state/statemachine_test.go @@ -0,0 +1,349 @@ +package state + +import ( + "errors" + "os" + "testing" + "time" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/result" + "github.com/stretchr/testify/assert" + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +func init() { + logger, err := zap.NewDevelopment() + if err != nil { + os.Exit(1) + } + zap.ReplaceGlobals(logger) +} + +// inMemorySaveLoader stores and loads states to member fields +// and maintains a history of all the fields saved. +type inMemorySaveLoader struct { + stateHistory []string + nextState string + startingState string +} + +func (s *inMemorySaveLoader) SaveNextState(_ types.NamespacedName, stateName string) error { + if stateName == "" { + return nil + } + s.stateHistory = append(s.stateHistory, stateName) + s.nextState = stateName + return nil +} + +func (s *inMemorySaveLoader) LoadNextState(_ types.NamespacedName) (string, error) { + return s.nextState, nil +} + +func newInMemorySaveLoader(startingState string) *inMemorySaveLoader { + s := &inMemorySaveLoader{} + s.startingState = startingState + _ = s.SaveNextState(types.NamespacedName{}, startingState) + return s +} + +func TestOrderOfStatesIsCorrect(t *testing.T) { + in := newInMemorySaveLoader("State0") + s := NewStateMachine(in, types.NamespacedName{}, zap.S()) + + state0 := newAlwaysCompletingState("State0") + state1 := newAlwaysCompletingState("State1") + state2 := newAlwaysCompletingState("State2") + + s.AddDirectTransition(state0, state1) + s.AddDirectTransition(state1, state2) + + _, _ = s.Reconcile() + _, _ = s.Reconcile() + _, _ = s.Reconcile() + + assert.Equal(t, []string{"State0", "State1", "State2"}, in.stateHistory) +} + +func TestOrderOfStatesIsCorrectIfAddedInDifferentOrder(t *testing.T) { + in := newInMemorySaveLoader("State0") + s := NewStateMachine(in, types.NamespacedName{}, zap.S()) + + state0 := newAlwaysCompletingState("State0") + state1 := newAlwaysCompletingState("State1") + state2 := newAlwaysCompletingState("State2") + + s.AddDirectTransition(state1, state2) + s.AddDirectTransition(state0, state1) + + _, _ = s.Reconcile() + assert.Equal(t, "State1", in.nextState) + + _, _ = s.Reconcile() + assert.Equal(t, "State2", in.nextState) + + _, _ = s.Reconcile() + + assert.Equal(t, []string{"State0", "State1", "State2"}, in.stateHistory) +} + +func TestPredicateReturningFalse_PreventsStateTransition(t *testing.T) { + in := newInMemorySaveLoader("State0") + s := NewStateMachine(in, types.NamespacedName{}, zap.S()) + + state0 := newAlwaysCompletingState("State0") + state1 := newAlwaysCompletingState("State1") + state2 := newAlwaysCompletingState("State2") + state3 := newAlwaysCompletingState("State3") + + s.AddDirectTransition(state0, state1) + + // there is no transition from state1 to state2 + s.AddTransition(state1, state2, func() bool { + return false + }) + s.AddDirectTransition(state1, state3) + + _, _ = s.Reconcile() + _, _ = s.Reconcile() + _, _ = s.Reconcile() + + assert.Equal(t, []string{"State0", "State1", "State3"}, in.stateHistory) +} + +func TestAddTransition(t *testing.T) { + in := newInMemorySaveLoader("State0") + s := NewStateMachine(in, types.NamespacedName{}, zap.S()) + + state0 := newAlwaysCompletingState("State0") + state1 := newAlwaysCompletingState("State1") + + s.AddDirectTransition(state0, state1) + + t.Run("Adds both states to internal map", func(t *testing.T) { + assert.Contains(t, s.states, "State0") + assert.Contains(t, s.states, "State1") + }) + + t.Run("Creates transition for first state", func(t *testing.T) { + assert.Len(t, s.allTransitions["State0"], 1) + assert.Equal(t, s.allTransitions["State0"][0].from.Name, "State0") + assert.Equal(t, s.allTransitions["State0"][0].to.Name, "State1") + }) + + t.Run("Does not create transition for second state", func(t *testing.T) { + assert.Len(t, s.allTransitions["State1"], 0) + }) +} + +func TestIfStateFails_ItIsRunAgain(t *testing.T) { + fails := newAlwaysFailsState("FailsState") + succeeds := newAlwaysCompletingState("SucceedsState") + + in := newInMemorySaveLoader(fails.Name) + s := NewStateMachine(in, types.NamespacedName{}, zap.S()) + + s.AddDirectTransition(fails, succeeds) + + t.Run("Any number of runs will not change the next state to be run", func(t *testing.T) { + _, _ = s.Reconcile() + assert.Equal(t, fails.Name, in.nextState) + + _, _ = s.Reconcile() + assert.Equal(t, fails.Name, in.nextState) + + _, _ = s.Reconcile() + assert.Equal(t, fails.Name, in.nextState) + }) + + t.Run("When the state passes, the next one will run", func(t *testing.T) { + + // the state will now succeed + s.states["FailsState"] = newAlwaysCompletingState(fails.Name) + + _, _ = s.Reconcile() + assert.Equal(t, succeeds.Name, in.nextState) + }) +} + +func TestStateReconcileValue_IsReturnedFromStateMachine(t *testing.T) { + t.Run("When State is Completed", func(t *testing.T) { + s0 := State{ + Name: "State0", + Reconcile: func() (reconcile.Result, error, bool) { + return reconcile.Result{RequeueAfter: time.Duration(15000)}, errors.New("error"), true + }, + } + + s1 := newAlwaysCompletingState("State1") + + in := newInMemorySaveLoader(s0.Name) + s := NewStateMachine(in, types.NamespacedName{}, zap.S()) + + s.AddDirectTransition(s0, s1) + + res, err := s.Reconcile() + assert.False(t, res.Requeue) + assert.Equal(t, time.Duration(15000), res.RequeueAfter) + assert.Equal(t, errors.New("error"), err) + }) + + t.Run("When State is not Completed", func(t *testing.T) { + s0 := State{ + Name: "State0", + Reconcile: func() (reconcile.Result, error, bool) { + return reconcile.Result{Requeue: true, RequeueAfter: time.Duration(5000)}, errors.New("error"), false + }, + } + + s1 := newAlwaysCompletingState("State1") + + in := newInMemorySaveLoader(s0.Name) + s := NewStateMachine(in, types.NamespacedName{}, zap.S()) + + s.AddDirectTransition(s0, s1) + + res, err := s.Reconcile() + assert.True(t, res.Requeue) + assert.Equal(t, time.Duration(5000), res.RequeueAfter) + assert.Equal(t, errors.New("error"), err) + }) +} + +func TestCycleInStateMachine(t *testing.T) { + s0 := newAlwaysCompletingState("State0") + s1 := newAlwaysCompletingState("State1") + s2 := newAlwaysCompletingState("State2") + s3 := newAlwaysCompletingState("State3") + s4 := newAlwaysCompletingState("State4") + + in := newInMemorySaveLoader("State0") + s := NewStateMachine(in, types.NamespacedName{}, zap.S()) + + flag := true + s.AddDirectTransition(s0, s1) + s.AddDirectTransition(s1, s2) + s.AddDirectTransition(s2, s3) + + // create a one time cycle back to s1 + s.AddTransition(s3, s1, func() bool { + res := flag + flag = !flag + return res + }) + + s.AddDirectTransition(s3, s4) + + _, _ = s.Reconcile() + _, _ = s.Reconcile() + _, _ = s.Reconcile() + _, _ = s.Reconcile() + _, _ = s.Reconcile() + _, _ = s.Reconcile() + _, _ = s.Reconcile() + _, _ = s.Reconcile() + + assert.Equal(t, []string{"State0", "State1", "State2", "State3", "State1", "State2", "State3", "State4"}, in.stateHistory) +} + +func TestBranchingPath(t *testing.T) { + root := newAlwaysCompletingState("Root") + left0 := newAlwaysCompletingState("Left0") + left1 := newAlwaysCompletingState("Left1") + left2 := newAlwaysCompletingState("Left2") + + right0 := newAlwaysCompletingState("Right0") + right1 := newAlwaysCompletingState("Right1") + right2 := newAlwaysCompletingState("Right2") + + in := newInMemorySaveLoader(root.Name) + s := NewStateMachine(in, types.NamespacedName{}, zap.S()) + + goLeft := true + + s.AddTransition(root, left0, func() bool { + return goLeft + }) + s.AddDirectTransition(left0, left1) + s.AddDirectTransition(left1, left2) + + s.AddTransition(root, right0, func() bool { + return !goLeft + }) + + s.AddDirectTransition(right0, right1) + s.AddDirectTransition(right1, right2) + + t.Run("Left Path", func(t *testing.T) { + + _, _ = s.Reconcile() + _, _ = s.Reconcile() + _, _ = s.Reconcile() + _, _ = s.Reconcile() + + assert.Equal(t, []string{"Root", "Left0", "Left1", "Left2"}, in.stateHistory) + }) + + t.Run("Right Path", func(t *testing.T) { + goLeft = false + // reset save loader state + in.stateHistory = nil + _ = in.SaveNextState(types.NamespacedName{}, root.Name) + + _, _ = s.Reconcile() + _, _ = s.Reconcile() + _, _ = s.Reconcile() + _, _ = s.Reconcile() + + assert.Equal(t, []string{"Root", "Right0", "Right1", "Right2"}, in.stateHistory) + }) +} + +func TestDetermineStartingState_ReadsFromLoader(t *testing.T) { + t.Run("State Can be determined once added", func(t *testing.T) { + s0 := newAlwaysCompletingState("State0") + s1 := newAlwaysCompletingState("State1") + + in := newInMemorySaveLoader(s0.Name) + s := NewStateMachine(in, types.NamespacedName{}, zap.S()) + + // State must be added before it can be returned in determine state + s.AddDirectTransition(s0, s1) + + assert.Nil(t, s.currentState) + err := s.determineState() + assert.NoError(t, err) + assert.Equal(t, "State0", s.currentState.Name) + }) + + t.Run("State cannot be determined if not added", func(t *testing.T) { + s0 := newAlwaysCompletingState("State0") + + in := newInMemorySaveLoader(s0.Name) + s := NewStateMachine(in, types.NamespacedName{}, zap.S()) + + assert.Nil(t, s.currentState) + err := s.determineState() + assert.Error(t, err) + assert.Nil(t, s.currentState) + }) + +} + +// newAlwaysCompletingState returns a State that will always succeed. +func newAlwaysCompletingState(name string) State { + return State{ + Name: name, + Reconcile: result.StateComplete, + } +} + +// newAlwaysFailsState returns a State that will always fail. +func newAlwaysFailsState(name string) State { + return State{ + Name: name, + Reconcile: result.FailedState, + } +} From 8f99457809b3d4b54d6e647d33d5e8758240896b Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Tue, 20 Apr 2021 21:04:05 +0100 Subject: [PATCH 227/790] Use readinessProbe 1.0.3 (#430) --- release.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release.json b/release.json index 243360fe3..26a022111 100644 --- a/release.json +++ b/release.json @@ -1,7 +1,7 @@ { "mongodb-kubernetes-operator": "0.5.2", "version-upgrade-hook": "1.0.2", - "readiness-probe": "1.0.2", + "readiness-probe": "1.0.3", "agent": { "version": "10.29.0.6830-1", "tools_version": "100.2.0" From 56a8d85be4b063244f445310c22bf1085c886f0f Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Wed, 21 Apr 2021 14:49:31 +0100 Subject: [PATCH 228/790] CLOUDP-83092: Add init container images to inventory file (#432) --- .evergreen.yml | 22 +++++++++-- inventory.yaml | 33 +++++++++++++++++ pipeline.py | 33 +++++++++++++++-- scripts/dev/dockerfile_generator.py | 37 ------------------- .../Dockerfile.versionhook | 1 - 5 files changed, 81 insertions(+), 45 deletions(-) rename scripts/dev/{templates => dockerfiles}/Dockerfile.versionhook (94%) diff --git a/.evergreen.yml b/.evergreen.yml index e2bcdd0e3..8a4112c42 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -217,11 +217,20 @@ tasks: commands: - func: clone - func: setup_virtualenv - - func: build_and_push_image + - func: build_and_push_image_sonar vars: - image: quay.io/mongodb/community-operator-version-upgrade-post-start-hook:${version_id} - image_type: versionhook - expire_after: 48h + image_name: version-post-start-hook-init + + - name: build_readiness_probe_image + priority: 60 + exec_timeout_secs: 600 + commands: + - func: clone + - func: setup_virtualenv + - func: build_and_push_image_sonar + vars: + image_name: readiness-probe-init + - name: e2e_test_feature_compatibility_version commands: @@ -361,6 +370,8 @@ buildvariants: variant: init_test_run - name: build_prehook_image variant: init_test_run + - name: build_readiness_probe_image + variant: init_test_run - name: build_agent_image_ubuntu variant: init_test_run tasks: @@ -379,6 +390,8 @@ buildvariants: variant: init_test_run - name: build_prehook_image variant: init_test_run + - name: build_readiness_probe_image + variant: init_test_run - name: build_agent_image_ubi variant: init_test_run tasks: @@ -394,6 +407,7 @@ buildvariants: - name: build_prehook_image - name: build_agent_image_ubi - name: build_agent_image_ubuntu + - name: build_readiness_probe_image - name: release_blocker display_name: release_blocker diff --git a/inventory.yaml b/inventory.yaml index 3ad59ff7a..f1182f9dc 100644 --- a/inventory.yaml +++ b/inventory.yaml @@ -63,3 +63,36 @@ images: output: - registry: $(inputs.params.registry)/mongodb-agent-ubi-dev tag: $(inputs.params.version_id) + + + - name: readiness-probe-init + vars: + context: . + + stages: + - name: readiness-probe-init-build + task_type: docker_build + dockerfile: scripts/dev/dockerfiles/Dockerfile.readiness + + labels: + quay.expires-after: 48h + + output: + - registry: $(inputs.params.registry)/mongodb-kubernetes-readinessprobe-dev + tag: $(inputs.params.version_id) + + - name: version-post-start-hook-init + vars: + context: . + + stages: + - name: version-post-start-hook-init-build + task_type: docker_build + dockerfile: scripts/dev/dockerfiles/Dockerfile.versionhook + + labels: + quay.expires-after: 48h + + output: + - registry: $(inputs.params.registry)/mongodb-kubernetes-operator-version-upgrade-post-start-hook-dev + tag: $(inputs.params.version_id) diff --git a/pipeline.py b/pipeline.py index 2440388ec..9f7d38305 100644 --- a/pipeline.py +++ b/pipeline.py @@ -7,7 +7,14 @@ from scripts.dev.dev_config import load_config, DevConfig -VALID_IMAGE_NAMES = frozenset(["agent-ubi", "agent-ubuntu"]) +VALID_IMAGE_NAMES = frozenset( + [ + "agent-ubi", + "agent-ubuntu", + "readiness-probe-init", + "version-post-start-hook-init", + ] +) DEFAULT_IMAGE_TYPE = "ubuntu" DEFAULT_NAMESPACE = "default" @@ -47,6 +54,24 @@ def build_agent_image_ubuntu(config: DevConfig) -> None: ) +def build_readiness_probe_image(config: DevConfig) -> None: + sonar_build_image( + "readiness-probe-init", + args={ + "registry": config.repo_url, + }, + ) + + +def build_version_post_start_hook_image(config: DevConfig) -> None: + sonar_build_image( + "version-post-start-hook-init", + args={ + "registry": config.repo_url, + }, + ) + + def sonar_build_image( image_name: str, args: Optional[Dict[str, str]] = None, @@ -78,12 +103,14 @@ def main() -> int: ) return 1 - agent_build_function = { + image_build_function = { "agent-ubi": build_agent_image_ubi, "agent-ubuntu": build_agent_image_ubuntu, + "readiness-probe-init": build_readiness_probe_image, + "version-post-start-hook-init": build_version_post_start_hook_image, }[image_name] - agent_build_function(load_config()) + image_build_function(load_config()) return 0 diff --git a/scripts/dev/dockerfile_generator.py b/scripts/dev/dockerfile_generator.py index 5d2f1b852..6bf824378 100755 --- a/scripts/dev/dockerfile_generator.py +++ b/scripts/dev/dockerfile_generator.py @@ -11,41 +11,6 @@ GOLANG_TAG = "1.14" -def _shared_agent_params() -> DockerParameters: - with open("release.json", "r") as f: - release = json.loads(f.read()) - - return { - "template_path": "scripts/dev/templates/agent", - "agent_version": release["agent"]["version"], - "tools_version": release["agent"]["tools_version"], - } - - -def agent_ubuntu_params() -> DockerParameters: - params = _shared_agent_params() - params.update( - { - "base_image": "ubuntu:16.04", - "tools_distro": "ubuntu1604-x86_64", - "agent_distro": "linux_x86_64", - } - ) - return params - - -def agent_ubi_params() -> DockerParameters: - params = _shared_agent_params() - params.update( - { - "base_image": "registry.access.redhat.com/ubi7/ubi", - "tools_distro": "rhel70-x86_64", - "agent_distro": "rhel7_x86_64", - } - ) - return params - - def operator_params(files_to_add: List[str]) -> DockerParameters: return { "builder": True, @@ -67,8 +32,6 @@ def render(image_name: str, files_to_add: List[str]) -> str: param_dict = { "e2e": e2e_params(files_to_add), "operator": operator_params(files_to_add), - "agent_ubi": agent_ubi_params(), - "agent_ubuntu": agent_ubuntu_params(), } render_values = param_dict.get(image_name, dict()) diff --git a/scripts/dev/templates/Dockerfile.versionhook b/scripts/dev/dockerfiles/Dockerfile.versionhook similarity index 94% rename from scripts/dev/templates/Dockerfile.versionhook rename to scripts/dev/dockerfiles/Dockerfile.versionhook index a3df35627..4180e2b0e 100644 --- a/scripts/dev/templates/Dockerfile.versionhook +++ b/scripts/dev/dockerfiles/Dockerfile.versionhook @@ -1,4 +1,3 @@ -# TODO: template this FROM golang AS builder ENV GO111MODULE=on From af9674ab7447efac0e1fe378b7ce47a6fb853c42 Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Thu, 22 Apr 2021 10:21:50 +0100 Subject: [PATCH 229/790] CLOUDP-83092: Add release task for init containers (#433) --- .evergreen.yml | 24 +++++++++--- inventory.yaml | 37 ++++++++++++++++++ pipeline.py | 48 ++++++++++++++++++++---- scripts/ci/build_and_push_image_sonar.sh | 2 +- scripts/dev/dev_config.py | 14 ++++++- 5 files changed, 110 insertions(+), 15 deletions(-) diff --git a/.evergreen.yml b/.evergreen.yml index 8a4112c42..ba9eff144 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -120,6 +120,7 @@ functions: - quay_user_name - quay_password - image_name + - release working_dir: mongodb-kubernetes-operator binary: scripts/ci/build_and_push_image_sonar.sh @@ -345,15 +346,25 @@ tasks: new_image: quay.io/mongodb/mongodb-kubernetes-operator image_type: mongodb-kubernetes-operator - - name: release_version_upgrade_hook + + - name: release_version_upgrade_post_start_hook commands: - func: clone - func: setup_virtualenv - - func: release_docker_image + - func: build_and_push_image_sonar + vars: + image_type: version-post-start-hook-init + release: true + + + - name: release_readiness_probe + commands: + - func: clone + - func: setup_virtualenv + - func: build_and_push_image_sonar vars: - old_image: quay.io/mongodb/community-operator-version-upgrade-post-start-hook - new_image: quay.io/mongodb/mongodb-kubernetes-operator-version-upgrade-post-start-hook - image_type: version-upgrade-hook + image_type: readiness-probe-init + release: true buildvariants: @@ -433,4 +444,5 @@ buildvariants: - ubuntu1604-test tasks: - name: release_operator - - name: release_version_upgrade_hook + - name: release_version_upgrade_post_start_hook + - name: release_readiness_probe diff --git a/inventory.yaml b/inventory.yaml index f1182f9dc..38c626401 100644 --- a/inventory.yaml +++ b/inventory.yaml @@ -11,6 +11,7 @@ images: - name: agent-template-ubuntu task_type: dockerfile_template distro: ubuntu + tags: ["ubuntu"] inputs: - agent_version @@ -24,6 +25,7 @@ images: - name: agent-ubuntu-build task_type: docker_build dockerfile: scripts/dev/templates/agent/Dockerfile.ubuntu-$(inputs.params.version_id) + tags: ["ubuntu"] labels: quay.expires-after: 48h @@ -42,6 +44,7 @@ images: - name: agent-template-ubi task_type: dockerfile_template distro: ubi + tags: ["ubi"] inputs: - agent_version @@ -56,6 +59,7 @@ images: - name: agent-ubi-build task_type: docker_build dockerfile: scripts/dev/templates/agent/Dockerfile.ubi-$(inputs.params.version_id) + tags: ["ubi"] labels: quay.expires-after: 48h @@ -73,6 +77,7 @@ images: - name: readiness-probe-init-build task_type: docker_build dockerfile: scripts/dev/dockerfiles/Dockerfile.readiness + tags: ["readiness-probe"] labels: quay.expires-after: 48h @@ -81,6 +86,22 @@ images: - registry: $(inputs.params.registry)/mongodb-kubernetes-readinessprobe-dev tag: $(inputs.params.version_id) + - name: readiness-probe-init-release + task_type: tag_image + tags: ["release", "readiness-probe"] + + inputs: + - release_version + + source: + registry: $(inputs.params.registry)/mongodb-kubernetes-readinessprobe-dev + tag: $(inputs.params.version_id) + + destination: + - registry: $(inputs.params.registry)/mongodb-kubernetes-readinessprobe + tag: $(inputs.params.release_version) + + - name: version-post-start-hook-init vars: context: . @@ -89,6 +110,7 @@ images: - name: version-post-start-hook-init-build task_type: docker_build dockerfile: scripts/dev/dockerfiles/Dockerfile.versionhook + tags: ["post-start-hook"] labels: quay.expires-after: 48h @@ -96,3 +118,18 @@ images: output: - registry: $(inputs.params.registry)/mongodb-kubernetes-operator-version-upgrade-post-start-hook-dev tag: $(inputs.params.version_id) + + - name: version-post-start-hook-init-build-release + task_type: tag_image + tags: ["release", "post-start-hook"] + + inputs: + - release_version + + source: + registry: $(inputs.params.registry)/mongodb-kubernetes-operator-version-upgrade-post-start-hook-dev + tag: $(inputs.params.version_id) + + destination: + - registry: $(inputs.params.registry)/mongodb-kubernetes-operator-version-upgrade-post-start-hook + tag: $(inputs.params.release_version) diff --git a/pipeline.py b/pipeline.py index 9f7d38305..4ed1d919b 100644 --- a/pipeline.py +++ b/pipeline.py @@ -27,12 +27,16 @@ def build_agent_image_ubi(config: DevConfig) -> None: args = { "agent_version": release["agent"]["version"], "tools_version": release["agent"]["tools_version"], - "tools_distro": "ubuntu1604-x86_64", - "agent_distro": "linux_x86_64", + "tools_distro": "rhel70-x86_64", + "agent_distro": "rhel7_x86_64", "registry": config.repo_url, } + + config.ensure_tag_is_run("ubi") + sonar_build_image( image_name, + config, args=args, ) @@ -44,36 +48,55 @@ def build_agent_image_ubuntu(config: DevConfig) -> None: args = { "agent_version": release["agent"]["version"], "tools_version": release["agent"]["tools_version"], - "tools_distro": "rhel70-x86_64", - "agent_distro": "rhel7_x86_64", + "tools_distro": "ubuntu1604-x86_64", + "agent_distro": "linux_x86_64", "registry": config.repo_url, } + + config.ensure_tag_is_run("ubuntu") + sonar_build_image( image_name, + config, args=args, ) def build_readiness_probe_image(config: DevConfig) -> None: + with open("release.json") as f: + release = json.loads(f.read()) + + config.ensure_tag_is_run("readiness-probe") + sonar_build_image( "readiness-probe-init", + config, args={ "registry": config.repo_url, + "release_version": release["readiness-probe"], }, ) def build_version_post_start_hook_image(config: DevConfig) -> None: + with open("release.json") as f: + release = json.loads(f.read()) + + config.ensure_tag_is_run("post-start-hook") + sonar_build_image( "version-post-start-hook-init", + config, args={ "registry": config.repo_url, + "release_version": release["version-upgrade-hook"], }, ) def sonar_build_image( image_name: str, + config: DevConfig, args: Optional[Dict[str, str]] = None, inventory: str = "inventory.yaml", ) -> None: @@ -82,14 +105,15 @@ def sonar_build_image( image_name, build_args=args, inventory=inventory, - include_tags=[], - skip_tags=[], + include_tags=config.include_tags, + skip_tags=config.skip_tags, ) def _parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser() parser.add_argument("--image-name", type=str) + parser.add_argument("--release", type=bool) return parser.parse_args() @@ -103,6 +127,16 @@ def main() -> int: ) return 1 + config = load_config() + + # by default we do not want to run any release tasks. We must explicitly + # use the --release flag to run them. + config.ensure_skip_tag("release") + + # specify --release to release the image + if args.release: + config.ensure_tag_is_run("release") + image_build_function = { "agent-ubi": build_agent_image_ubi, "agent-ubuntu": build_agent_image_ubuntu, @@ -110,7 +144,7 @@ def main() -> int: "version-post-start-hook-init": build_version_post_start_hook_image, }[image_name] - image_build_function(load_config()) + image_build_function(config) return 0 diff --git a/scripts/ci/build_and_push_image_sonar.sh b/scripts/ci/build_and_push_image_sonar.sh index ea9f52548..a9c760a23 100755 --- a/scripts/ci/build_and_push_image_sonar.sh +++ b/scripts/ci/build_and_push_image_sonar.sh @@ -7,4 +7,4 @@ set +x echo "${quay_password:?}" | docker login "-u=${quay_user_name:?}" quay.io --password-stdin # shellcheck disable=SC2154 -python3 pipeline.py --image-name "${image_name}" +python3 pipeline.py --image-name "${image_name}" --release "${release:-false}" diff --git a/scripts/dev/dev_config.py b/scripts/dev/dev_config.py index badc6004e..1bcaa0cb3 100644 --- a/scripts/dev/dev_config.py +++ b/scripts/dev/dev_config.py @@ -1,5 +1,5 @@ from __future__ import annotations -from typing import Dict, Optional +from typing import Dict, Optional, List from enum import Enum import json import os @@ -33,6 +33,14 @@ class DevConfig: def __init__(self, config: Dict, distro: Distro): self._config = config self._distro = distro + self.include_tags: List[str] = [] + self.skip_tags: List[str] = [] + + def ensure_tag_is_run(self, tag: str) -> None: + if tag not in self.include_tags: + self.include_tags.append(tag) + if tag in self.skip_tags: + self.skip_tags.remove(tag) @property def namespace(self) -> str: @@ -64,6 +72,10 @@ def agent_image(self) -> str: return self._config["agent_image_ubi"] return self._config["agent_image_ubuntu"] + def ensure_skip_tag(self, tag: str) -> None: + if tag not in self.skip_tags: + self.skip_tags.append(tag) + def load_config( config_file_path: Optional[str] = None, distro: Distro = Distro.UBUNTU From 0f07b20b87a3475ceb2821bfeb8b6f0d70742d8b Mon Sep 17 00:00:00 2001 From: James Broadhead Date: Thu, 22 Apr 2021 12:38:44 +0100 Subject: [PATCH 230/790] CLOUDP-85550: mark github issues stale (#434) Co-authored-by: Nikolas De Giorgis --- .github/workflows/close-stale-issues.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .github/workflows/close-stale-issues.yml diff --git a/.github/workflows/close-stale-issues.yml b/.github/workflows/close-stale-issues.yml new file mode 100644 index 000000000..e0592e2e5 --- /dev/null +++ b/.github/workflows/close-stale-issues.yml @@ -0,0 +1,24 @@ +# +# Docs: https://github.com/marketplace/actions/close-stale-issues +# +name: Close Stale Issues +on: + schedule: + - cron: '30 1 * * *' + +jobs: + stale: + runs-on: ubuntu-latest + steps: + - uses: actions/stale@v3 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + stale-issue-message: 'This issue is being marked stale because it has been open for 60 days with no activity. Please comment if this issue is still affecting you. If there is no change, this issue will be closed in 30 days.' + stale-pr-message: 'This PR is being marked stale because it has been open for 60 days with no activity. Please update the PR or ask for a fresh review.' + close-issue-message: 'This issue was closed because it became stale and did not receive further updates. If the issue is still affecting you, please re-open it, or file a fresh Issue with updated information.' + + days-before-stale: 60 + days-before-close: 30 + days-before-pr-close: -1 # never close PRs + + exempt-issue-labels: 'bug,feature-request' From dcddbfb1a19e3fa65454e3963a8741d2446590c5 Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Fri, 23 Apr 2021 11:03:54 +0100 Subject: [PATCH 231/790] CLOUDP-87698: perform check for readiness probe init container already existing (#431) --- README.md | 3 +++ controllers/construct/mongodbstatefulset.go | 6 ++--- controllers/replica_set_controller.go | 13 +++++++++++ dev_notes/RELEASE_NOTES.md | 26 ++++++++++++++++++++- 4 files changed, 44 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index c58d3368f..5473481a3 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,9 @@ +###v0.6.0 has introduced breaking changes. If you are upgrading from a previous version, follow the upgrade instructions outlined [in the release notes](https://github.com/mongodb/mongodb-kubernetes-operator/releases/tag/v0.6.0) + + This is a [Kubernetes Operator](https://coreos.com/operators/) which deploys MongoDB Community into Kubernetes clusters. If you are a MongoDB Enterprise customer, or need Enterprise features such as Backup, you can use the [MongoDB Enterprise Operator for Kubernetes](https://github.com/mongodb/mongodb-enterprise-kubernetes). diff --git a/controllers/construct/mongodbstatefulset.go b/controllers/construct/mongodbstatefulset.go index ec2695374..4adb1e6ca 100644 --- a/controllers/construct/mongodbstatefulset.go +++ b/controllers/construct/mongodbstatefulset.go @@ -25,7 +25,7 @@ const ( MongodbName = "mongod" versionUpgradeHookName = "mongod-posthook" - readinessProbeContainerName = "mongodb-agent-readinessprobe" + ReadinessProbeContainerName = "mongodb-agent-readinessprobe" dataVolumeName = "data-volume" logVolumeName = "logs-volume" readinessProbePath = "/opt/scripts/readinessprobe" @@ -169,7 +169,7 @@ func BuildMongoDBReplicaSetStatefulSetModificationFunction(mdb MongoDBStatefulSe podtemplatespec.WithContainer(AgentName, mongodbAgentContainer(mdb.AutomationConfigSecretName(), mongodbAgentVolumeMounts)), podtemplatespec.WithContainer(MongodbName, mongodbContainer(mdb.GetMongoDBVersion(), mongodVolumeMounts)), podtemplatespec.WithInitContainer(versionUpgradeHookName, versionUpgradeHookInit([]corev1.VolumeMount{hooksVolumeMount})), - podtemplatespec.WithInitContainer(readinessProbeContainerName, readinessProbeInit([]corev1.VolumeMount{scriptsVolumeMount})), + podtemplatespec.WithInitContainer(ReadinessProbeContainerName, readinessProbeInit([]corev1.VolumeMount{scriptsVolumeMount})), ), )) } @@ -261,7 +261,7 @@ func logsPvc() persistentvolumeclaim.Modification { // this container will copy the readiness probe binary into the /opt/scripts directory. func readinessProbeInit(volumeMount []corev1.VolumeMount) container.Modification { return container.Apply( - container.WithName(readinessProbeContainerName), + container.WithName(ReadinessProbeContainerName), container.WithCommand([]string{"cp", "/probes/readinessprobe", "/opt/scripts/readinessprobe"}), container.WithImage(os.Getenv(ReadinessProbeImageEnv)), container.WithImagePullPolicy(corev1.PullAlways), diff --git a/controllers/replica_set_controller.go b/controllers/replica_set_controller.go index f062e3bea..c97ba8217 100644 --- a/controllers/replica_set_controller.go +++ b/controllers/replica_set_controller.go @@ -6,6 +6,8 @@ import ( "fmt" "os" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/container" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/functions" "github.com/mongodb/mongodb-kubernetes-operator/pkg/agent" @@ -313,6 +315,11 @@ func (r *ReplicaSetReconciler) deployAutomationConfig(mdb mdbv1.MongoDBCommunity return true, nil } + if isPreReadinessInitContainerStatefulSet(sts) { + r.log.Debugf("The existing StatefulSet did not have the readiness probe init container, skipping pod annotation check.") + return true, nil + } + r.log.Debugf("Waiting for agents to reach version %d", ac.Version) // Note: we pass in the expected number of replicas this reconciliation as we scale members one at a time. If we were // to pass in the final member count, we would be waiting for agents that do not exist yet to be ready. @@ -556,3 +563,9 @@ func getDomain(service, namespace, clusterName string) string { } return fmt.Sprintf("%s.%s.svc.%s", service, namespace, clusterName) } + +// isPreReadinessInitContainerStatefulSet determines if the existing StatefulSet has been configured with the readiness probe init container. +// if this is not the case, then we should ensure to skip past the annotation check otherwise the pods will remain in pending state forever. +func isPreReadinessInitContainerStatefulSet(sts appsv1.StatefulSet) bool { + return container.GetByName(construct.ReadinessProbeContainerName, sts.Spec.Template.Spec.InitContainers) == nil +} diff --git a/dev_notes/RELEASE_NOTES.md b/dev_notes/RELEASE_NOTES.md index 89ac7e77f..8af1407c3 100644 --- a/dev_notes/RELEASE_NOTES.md +++ b/dev_notes/RELEASE_NOTES.md @@ -1,10 +1,34 @@ *(Please use the [release template](release-notes-template.md) as the template for this document)* -# MongoDB Kubernetes Operator 0.5.3 +# MongoDB Kubernetes Operator 0.6.0 ## Kubernetes Operator + +* Breaking Changes + * A new VolumeClaimTemplate has been added `logs-volume`. When you deploy the operator, if there is an existing StatefulSet the operator will attempt to perform an invalid update. The existing StatefulSet must be deleted before upgrading the operator. + + * The user of the mongod and mongodb-agent containers has changed. This means that there will be permissions + issues when upgrading from an earlier version of the operator. In order to update the permissions in the volume, you can use an init container. + +* Upgrade instructions + + Remove the current operator deployment + - `kubectl delete deployment ` + Delete the existing StatefulSet for the MongoDBCommunity resource + Note: to ensure existing data is not lost, ensure that the retain policy of your Persistent Volumes is configured correctly. + - `kubectl delete statefulset ` + Install the new operator + - follow the regular [installation instruction](https://github.com/mongodb/mongodb-kubernetes-operator/blob/master/docs/install-upgrade.md) + Patch the StatefulSet once it has been created. This will add an init container that will update the permissions of the existing volume. + - `kubectl patch statefulset --type='json' --patch '[ {"op":"add","path":"/spec/template/spec/initContainers/-", "value": { "name": "change-data-dir-permissions", "image": "busybox", "command": [ "chown", "-R", "2000", "/data" ], "securityContext": { "runAsNonRoot": false, "runAsUser": 0, "runAsGroup":0 }, "volumeMounts": [ { "mountPath": "/data", "name" : "data-volume" } ] } } ]'` + * Bug fixes * Fixes an issue that prevented the agents from reaching goal state when upgrading minor version of MongoDB. + ## Updated Image Tags + * mongodb-kubernetes-operator:0.6.0 + * mongodb-agent:0.29.0.6830-1 + + # MongoDB Kubernetes Operator 0.5.2 ## Kubernetes Operator From 2021ae6b6424a3c9a6809be7885e5322a067c470 Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Fri, 23 Apr 2021 15:50:00 +0100 Subject: [PATCH 232/790] Add script to create draft github release (#439) --- dev_notes/RELEASE_NOTES.md | 32 -------------------------- dev_notes/how-to-release.md | 9 ++++++-- dev_notes/past_release_notes/v0.5.2.md | 27 ++++++++++++++++++++++ scripts/dev/create_github_release.sh | 7 ++++++ 4 files changed, 41 insertions(+), 34 deletions(-) create mode 100644 dev_notes/past_release_notes/v0.5.2.md create mode 100755 scripts/dev/create_github_release.sh diff --git a/dev_notes/RELEASE_NOTES.md b/dev_notes/RELEASE_NOTES.md index 8af1407c3..a903802e2 100644 --- a/dev_notes/RELEASE_NOTES.md +++ b/dev_notes/RELEASE_NOTES.md @@ -1,5 +1,3 @@ -*(Please use the [release template](release-notes-template.md) as the template for this document)* - # MongoDB Kubernetes Operator 0.6.0 ## Kubernetes Operator @@ -27,33 +25,3 @@ ## Updated Image Tags * mongodb-kubernetes-operator:0.6.0 * mongodb-agent:0.29.0.6830-1 - - - -# MongoDB Kubernetes Operator 0.5.2 -## Kubernetes Operator -* Changes - * Readiness probe has been moved into an init container from the Agent image. - * Security context is now added when the `MANAGED_SECURITY_CONTEXT` environment variable is not set. -* Bug fixes - * Removed unnecessary environment variable configuration in the openshift samples. - * Fixed an issue where the operator would perform unnecessary reconcilliations when Secrets were modified. - * Fixed an issue where a race condition could cause the deployment to get into a bad state when TLS - settings when being changed at the same time as a scaling operation was happening. - * Fixed an issue where the agent pod would panic when running as a non-root user. - -## MongoDBCommunity Resource -* Changes - * Added `spec.security.authentication.ignoreUnknownUsers` field. This value defaults to `true`. When enabled, - any MongoDB users added through external sources will not be removed. - - -## Miscellaneous -* Changes - * Internal code refactorings to allow libraries to be imported into other projects. - - - ## Updated Image Tags - * mongodb-kubernetes-operator:0.5.2 - * mongodb-agent:10.27.0.6772-1 - * mongodb-kubernetes-readinessprobe:1.0.1 [new image] diff --git a/dev_notes/how-to-release.md b/dev_notes/how-to-release.md index 92c62ad3d..19205d2a0 100644 --- a/dev_notes/how-to-release.md +++ b/dev_notes/how-to-release.md @@ -3,9 +3,14 @@ * Update any finished tickets in [kube-community-next](https://jira.mongodb.org/issues?jql=project%20%3D%20CLOUDP%20AND%20component%20%3D%20%22Kubernetes%20Community%22%20%20AND%20status%20in%20(Resolved%2C%20Closed)%20and%20fixVersion%3D%20kube-community-next%20%20ORDER%20BY%20resolved) to have the version of the release you're doing (kube-community-x.y) -* Create github [draft release](https://github.com/mongodb/mongodb-kubernetes-operator/releases/new) +* Prepare the release PR + 1. Increment any image version changes. + 2. Create a github draft release `./scripts/dev/create_github_release.sh`. + 3. Commit changes. + +* Create release PR + 1. Reconfigure the Evergreen run to add the relevant release task(s). -* Create release PR & and reconfigure the Evergreen run to add the release task * Unblock release task once everything is green diff --git a/dev_notes/past_release_notes/v0.5.2.md b/dev_notes/past_release_notes/v0.5.2.md new file mode 100644 index 000000000..f333ce5fa --- /dev/null +++ b/dev_notes/past_release_notes/v0.5.2.md @@ -0,0 +1,27 @@ +# MongoDB Kubernetes Operator 0.5.2 +## Kubernetes Operator +* Changes + * Readiness probe has been moved into an init container from the Agent image. + * Security context is now added when the `MANAGED_SECURITY_CONTEXT` environment variable is not set. +* Bug fixes + * Removed unnecessary environment variable configuration in the openshift samples. + * Fixed an issue where the operator would perform unnecessary reconcilliations when Secrets were modified. + * Fixed an issue where a race condition could cause the deployment to get into a bad state when TLS + settings when being changed at the same time as a scaling operation was happening. + * Fixed an issue where the agent pod would panic when running as a non-root user. + +## MongoDBCommunity Resource +* Changes + * Added `spec.security.authentication.ignoreUnknownUsers` field. This value defaults to `true`. When enabled, + any MongoDB users added through external sources will not be removed. + + +## Miscellaneous +* Changes + * Internal code refactorings to allow libraries to be imported into other projects. + + + ## Updated Image Tags + * mongodb-kubernetes-operator:0.5.2 + * mongodb-agent:10.27.0.6772-1 + * mongodb-kubernetes-readinessprobe:1.0.1 [new image] diff --git a/scripts/dev/create_github_release.sh b/scripts/dev/create_github_release.sh new file mode 100755 index 000000000..ce00b555e --- /dev/null +++ b/scripts/dev/create_github_release.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +version="$(jq -r '."mongodb-kubernetes-operator"' < ./release.json)" +gh release create v"${version}" --title "MongoDB Kubernetes Operator ${version}" --draft --notes-file ./dev_notes/RELEASE_NOTES.md + +# move the release notes +cp ./dev_notes/RELEASE_NOTES.md "./dev_notes/past_release_notes/v${version}.md" From 837aaec363490107c6d463384fc890f9a850a122 Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Fri, 23 Apr 2021 17:39:53 +0100 Subject: [PATCH 233/790] CLOUDP-83092: Add context images for agents. (#435) --- .dockerignore | 1 - inventory.yaml | 64 +++++++++++++++---- pipeline.py | 5 +- .../dev/templates/agent/Dockerfile.builder | 11 ++++ .../dev/templates/agent/Dockerfile.template | 39 +++++------ 5 files changed, 86 insertions(+), 34 deletions(-) create mode 100644 scripts/dev/templates/agent/Dockerfile.builder diff --git a/.dockerignore b/.dockerignore index 5ba6cada8..9ff6084c6 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,5 @@ .github .idea -agent zz_* vendor/ scripts/ diff --git a/inventory.yaml b/inventory.yaml index 38c626401..c0bac0416 100644 --- a/inventory.yaml +++ b/inventory.yaml @@ -7,26 +7,47 @@ images: context: . template_context: scripts/dev/templates/agent + inputs: + - agent_version + - tools_version + stages: + - name: agent-ubuntu-context + task_type: docker_build + dockerfile: scripts/dev/templates/agent/Dockerfile.builder + tags: ["ubuntu"] + buildargs: + agent_version: $(inputs.params.agent_version) + tools_version: $(inputs.params.tools_version) + agent_distro: linux_x86_64 + tools_distro: ubuntu1604-x86_64 + + output: + - registry: $(inputs.params.registry)/agent-ubuntu-context-dev + tag: $(inputs.params.version_id) + - name: agent-template-ubuntu task_type: dockerfile_template - distro: ubuntu tags: ["ubuntu"] + distro: ubuntu + # TODO: remove - this is only here as sonar raises a key error on version_id + # if no other inputs are specified. inputs: - - agent_version - - tools_version - - tools_distro - - agent_distro + - noop output: - dockerfile: scripts/dev/templates/agent/Dockerfile.ubuntu-$(inputs.params.version_id) - name: agent-ubuntu-build task_type: docker_build - dockerfile: scripts/dev/templates/agent/Dockerfile.ubuntu-$(inputs.params.version_id) tags: ["ubuntu"] + dockerfile: scripts/dev/templates/agent/Dockerfile.ubuntu-$(inputs.params.version_id) + + buildargs: + imagebase: $(inputs.params.registry)/agent-ubuntu-context-dev:$(inputs.params.version_id) + labels: quay.expires-after: 48h @@ -40,17 +61,35 @@ images: context: . template_context: scripts/dev/templates/agent + inputs: + - agent_version + - tools_version + stages: + - name: agent-ubi-context + task_type: docker_build + dockerfile: scripts/dev/templates/agent/Dockerfile.builder + tags: ["ubi"] + buildargs: + agent_version: $(inputs.params.agent_version) + tools_version: $(inputs.params.tools_version) + agent_distro: rhel7_x86_64 + tools_distro: rhel70-x86_64 + + output: + - registry: $(inputs.params.registry)/agent-ubi-context-dev + tag: $(inputs.params.version_id) + + - name: agent-template-ubi task_type: dockerfile_template distro: ubi tags: ["ubi"] + # TODO: remove - this is only here as sonar raises a key error on version_id + # if no other inputs are specified. inputs: - - agent_version - - tools_version - - tools_distro - - agent_distro + - noop output: - dockerfile: scripts/dev/templates/agent/Dockerfile.ubi-$(inputs.params.version_id) @@ -58,8 +97,11 @@ images: - name: agent-ubi-build task_type: docker_build - dockerfile: scripts/dev/templates/agent/Dockerfile.ubi-$(inputs.params.version_id) tags: ["ubi"] + dockerfile: scripts/dev/templates/agent/Dockerfile.ubi-$(inputs.params.version_id) + + buildargs: + imagebase: $(inputs.params.registry)/agent-ubi-context-dev:$(inputs.params.version_id) labels: quay.expires-after: 48h diff --git a/pipeline.py b/pipeline.py index 4ed1d919b..7706bc763 100644 --- a/pipeline.py +++ b/pipeline.py @@ -24,11 +24,10 @@ def build_agent_image_ubi(config: DevConfig) -> None: image_name = "agent-ubi" with open("release.json") as f: release = json.loads(f.read()) + args = { "agent_version": release["agent"]["version"], "tools_version": release["agent"]["tools_version"], - "tools_distro": "rhel70-x86_64", - "agent_distro": "rhel7_x86_64", "registry": config.repo_url, } @@ -48,8 +47,6 @@ def build_agent_image_ubuntu(config: DevConfig) -> None: args = { "agent_version": release["agent"]["version"], "tools_version": release["agent"]["tools_version"], - "tools_distro": "ubuntu1604-x86_64", - "agent_distro": "linux_x86_64", "registry": config.repo_url, } diff --git a/scripts/dev/templates/agent/Dockerfile.builder b/scripts/dev/templates/agent/Dockerfile.builder new file mode 100644 index 000000000..d8d7f3ec3 --- /dev/null +++ b/scripts/dev/templates/agent/Dockerfile.builder @@ -0,0 +1,11 @@ +FROM scratch + +ARG agent_version +ARG agent_distro +ARG tools_distro +ARG tools_version + +ADD https://mciuploads.s3.amazonaws.com/mms-automation/mongodb-mms-build-agent/builds/automation-agent/prod/mongodb-mms-automation-agent-${agent_version}.${agent_distro}.tar.gz /data/mongodb-agent.tar.gz +ADD https://downloads.mongodb.org/tools/db/mongodb-database-tools-${tools_distro}-${tools_version}.tgz /data/mongodb-tools.tgz + +ADD agent/LICENSE /data/licenses diff --git a/scripts/dev/templates/agent/Dockerfile.template b/scripts/dev/templates/agent/Dockerfile.template index 260dfe9bb..a6c46e63e 100644 --- a/scripts/dev/templates/agent/Dockerfile.template +++ b/scripts/dev/templates/agent/Dockerfile.template @@ -1,29 +1,32 @@ +ARG imagebase +FROM ${imagebase} as base + FROM {{base_image}} {% block packages -%} {% endblock -%} -RUN mkdir -p agent \ - && curl --fail --retry 3 --silent https://mciuploads.s3.amazonaws.com/mms-automation/mongodb-mms-build-agent/builds/automation-agent/prod/mongodb-mms-automation-agent-{{agent_version}}.{{agent_distro}}.tar.gz -o agent/mongodb-agent.tar.gz \ - && tar xfz agent/mongodb-agent.tar.gz \ - && mv mongodb-mms-automation-agent-*/mongodb-mms-automation-agent agent/mongodb-agent \ - && chmod +x agent/mongodb-agent \ +RUN mkdir -p /agent \ + && mkdir -p /var/lib/mongodb-mms-automation \ + && mkdir -p /var/log/mongodb-mms-automation/ \ + && chmod -R +wr /var/log/mongodb-mms-automation/ \ + # ensure that the agent user can write the logs in OpenShift + && touch /var/log/mongodb-mms-automation/readiness.log \ + && chmod ugo+rw /var/log/mongodb-mms-automation/readiness.log + + +COPY --from=base /data/mongodb-agent.tar.gz /agent +COPY --from=base /data/mongodb-tools.tgz /agent + +RUN tar xfz /agent/mongodb-agent.tar.gz \ + && mv mongodb-mms-automation-agent-*/mongodb-mms-automation-agent /agent/mongodb-agent \ + && chmod +x /agent/mongodb-agent \ && mkdir -p /var/lib/automation/config \ && chmod -R +r /var/lib/automation/config \ - && rm agent/mongodb-agent.tar.gz \ + && rm /agent/mongodb-agent.tar.gz \ && rm -r mongodb-mms-automation-agent-* -RUN mkdir -p /var/lib/mongodb-mms-automation \ - && mkdir -p /var/log/mongodb-mms-automation/ \ - && chmod -R +wr /var/log/mongodb-mms-automation/ \ - # ensure that the agent user can write the logs in OpenShift - && touch /var/log/mongodb-mms-automation/readiness.log \ - && chmod ugo+rw /var/log/mongodb-mms-automation/readiness.log - -# Install MongoDB tools. The agent will automatically search the folder and find the binaries. -RUN curl --fail --retry 3 --silent https://downloads.mongodb.org/tools/db/mongodb-database-tools-{{tools_distro}}-{{tools_version}}.tgz -o mongodb-tools.tgz \ - && tar xfz mongodb-tools.tgz --directory /var/lib/mongodb-mms-automation/ \ - && rm mongodb-tools.tgz +RUN tar xfz /agent/mongodb-tools.tgz --directory /var/lib/mongodb-mms-automation/ && rm /agent/mongodb-tools.tgz USER 2000 -CMD ["agent/mongodb-agent", "-cluster=/var/lib/automation/config/automation-config.json"] +CMD ["/agent/mongodb-agent", "-cluster=/var/lib/automation/config/automation-config.json"] From 2b50bc922be8030d8f290f95c01042c525f0d0ce Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Fri, 23 Apr 2021 18:20:20 +0100 Subject: [PATCH 234/790] Fix incorrect arg parse field (#442) --- pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipeline.py b/pipeline.py index 7706bc763..895835553 100644 --- a/pipeline.py +++ b/pipeline.py @@ -110,7 +110,7 @@ def sonar_build_image( def _parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser() parser.add_argument("--image-name", type=str) - parser.add_argument("--release", type=bool) + parser.add_argument("--release", type=lambda x: x.lower() == "true") return parser.parse_args() From 36d1c6957ff43bac9401e7765768bfb4699c1ab7 Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Mon, 26 Apr 2021 11:08:57 +0100 Subject: [PATCH 235/790] CLOUDP-85272: remove persistence method from interface (#444) --- api/v1/mongodbcommunity_types.go | 4 --- controllers/construct/mongodbstatefulset.go | 32 +++++++++------------ 2 files changed, 14 insertions(+), 22 deletions(-) diff --git a/api/v1/mongodbcommunity_types.go b/api/v1/mongodbcommunity_types.go index 2fe0cc5cc..fa6dd9f1a 100644 --- a/api/v1/mongodbcommunity_types.go +++ b/api/v1/mongodbcommunity_types.go @@ -502,10 +502,6 @@ func (m MongoDBCommunity) GetMongoDBVersionForAnnotation() string { return m.GetMongoDBVersion() } -func (m MongoDBCommunity) Persistent() bool { - return true -} - func (m *MongoDBCommunity) StatefulSetReplicasThisReconciliation() int { return scale.ReplicasThisReconciliation(m) } diff --git a/controllers/construct/mongodbstatefulset.go b/controllers/construct/mongodbstatefulset.go index 4adb1e6ca..38f00c106 100644 --- a/controllers/construct/mongodbstatefulset.go +++ b/controllers/construct/mongodbstatefulset.go @@ -76,8 +76,6 @@ type MongoDBStatefulSetOwner interface { AutomationConfigSecretName() string // GetUpdateStrategyType returns the UpdateStrategyType of the statefulset. GetUpdateStrategyType() appsv1.StatefulSetUpdateStrategyType - // Persistent returns whether or not the statefulset should have persistent volumes added. - Persistent() bool // HasSeparateDataAndLogsVolumes returns whether or not the volumes for data and logs would need to be different. HasSeparateDataAndLogsVolumes() bool // GetAgentScramKeyfileSecretNamespacedName returns the NamespacedName of the secret which stores the keyfile for the agent. @@ -120,23 +118,21 @@ func BuildMongoDBReplicaSetStatefulSetModificationFunction(mdb MongoDBStatefulSe dataVolumeClaim := statefulset.NOOP() logVolumeClaim := statefulset.NOOP() singleModeVolumeClaim := func(s *appsv1.StatefulSet) {} - if mdb.Persistent() { - if mdb.HasSeparateDataAndLogsVolumes() { - logVolumeMount := statefulset.CreateVolumeMount(logVolumeName, automationconfig.DefaultAgentLogPath) - dataVolumeMount := statefulset.CreateVolumeMount(dataVolumeName, "/data") - dataVolumeClaim = statefulset.WithVolumeClaim(dataVolumeName, dataPvc()) - logVolumeClaim = statefulset.WithVolumeClaim(logVolumeName, logsPvc()) - mongodbAgentVolumeMounts = append(mongodbAgentVolumeMounts, dataVolumeMount, logVolumeMount) - mongodVolumeMounts = append(mongodVolumeMounts, dataVolumeMount, logVolumeMount) - } else { - mounts := []corev1.VolumeMount{ - statefulset.CreateVolumeMount(dataVolumeName, "/data", statefulset.WithSubPath("data")), - statefulset.CreateVolumeMount(dataVolumeName, automationconfig.DefaultAgentLogPath, statefulset.WithSubPath("logs")), - } - mongodbAgentVolumeMounts = append(mongodbAgentVolumeMounts, mounts...) - mongodVolumeMounts = append(mongodVolumeMounts, mounts...) - singleModeVolumeClaim = statefulset.WithVolumeClaim(dataVolumeName, dataPvc()) + if mdb.HasSeparateDataAndLogsVolumes() { + logVolumeMount := statefulset.CreateVolumeMount(logVolumeName, automationconfig.DefaultAgentLogPath) + dataVolumeMount := statefulset.CreateVolumeMount(dataVolumeName, "/data") + dataVolumeClaim = statefulset.WithVolumeClaim(dataVolumeName, dataPvc()) + logVolumeClaim = statefulset.WithVolumeClaim(logVolumeName, logsPvc()) + mongodbAgentVolumeMounts = append(mongodbAgentVolumeMounts, dataVolumeMount, logVolumeMount) + mongodVolumeMounts = append(mongodVolumeMounts, dataVolumeMount, logVolumeMount) + } else { + mounts := []corev1.VolumeMount{ + statefulset.CreateVolumeMount(dataVolumeName, "/data", statefulset.WithSubPath("data")), + statefulset.CreateVolumeMount(dataVolumeName, automationconfig.DefaultAgentLogPath, statefulset.WithSubPath("logs")), } + mongodbAgentVolumeMounts = append(mongodbAgentVolumeMounts, mounts...) + mongodVolumeMounts = append(mongodVolumeMounts, mounts...) + singleModeVolumeClaim = statefulset.WithVolumeClaim(dataVolumeName, dataPvc()) } podSecurityContext := podtemplatespec.NOOP() From 1bf5f80d6b3fd71d7279806257739bc479ae2cd7 Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Mon, 26 Apr 2021 11:46:29 +0100 Subject: [PATCH 236/790] Cloudp 83092 upload dockerfiles to s3 (#436) --- .evergreen.yml | 2 ++ inventory.yaml | 58 ++++++++++++++++++++++++++++------ pipeline.py | 26 +++++++-------- scripts/ci/setup_virtualenv.sh | 2 +- 4 files changed, 63 insertions(+), 25 deletions(-) diff --git a/.evergreen.yml b/.evergreen.yml index ba9eff144..c45038b32 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -115,6 +115,8 @@ functions: params: env: MONGODB_COMMUNITY_CONFIG: ${workdir}/mongodb-kubernetes-operator/scripts/ci/config.json + AWS_ACCESS_KEY_ID: ${community_aws_access_key_id} + AWS_SECRET_ACCESS_KEY: ${community_aws_secret_access_key} include_expansions_in_env: - version_id - quay_user_name diff --git a/inventory.yaml b/inventory.yaml index c0bac0416..561f87432 100644 --- a/inventory.yaml +++ b/inventory.yaml @@ -1,5 +1,7 @@ vars: registry: + s3_bucket_http: https://enterprise-operator-dockerfiles.s3.amazonaws.com/dockerfiles/mongodb-agent + s3_bucket: s3://enterprise-operator-dockerfiles/dockerfiles/mongodb-agent images: - name: agent-ubuntu @@ -31,11 +33,6 @@ images: tags: ["ubuntu"] distro: ubuntu - # TODO: remove - this is only here as sonar raises a key error on version_id - # if no other inputs are specified. - inputs: - - noop - output: - dockerfile: scripts/dev/templates/agent/Dockerfile.ubuntu-$(inputs.params.version_id) @@ -55,6 +52,29 @@ images: - registry: $(inputs.params.registry)/mongodb-agent-ubuntu-dev tag: $(inputs.params.version_id) + - name: agent-template-ubuntu-s3 + task_type: dockerfile_template + tags: ["ubuntu", "release"] + distro: ubuntu + + inputs: + - release_version + + output: + - dockerfile: $(inputs.params.s3_bucket)/Dockerfile.ubuntu-$(inputs.params.release_version) + + - name: agent-context-ubuntu-release + task_type: tag_image + tags: ["ubuntu", "release"] + distro: ubuntu + + source: + registry: $(inputs.params.registry)/agent-ubuntu-context-dev + tag: $(inputs.params.version_id) + + destination: + - registry: $(inputs.params.registry)/mongodb-agent-ubuntu-context + tag: $(inputs.params.release_version)-context - name: agent-ubi vars: @@ -86,11 +106,6 @@ images: distro: ubi tags: ["ubi"] - # TODO: remove - this is only here as sonar raises a key error on version_id - # if no other inputs are specified. - inputs: - - noop - output: - dockerfile: scripts/dev/templates/agent/Dockerfile.ubi-$(inputs.params.version_id) @@ -110,6 +125,29 @@ images: - registry: $(inputs.params.registry)/mongodb-agent-ubi-dev tag: $(inputs.params.version_id) + - name: agent-template-ubi-s3 + task_type: dockerfile_template + tags: ["ubi", "release"] + distro: ubi + + inputs: + - release_version + + output: + - dockerfile: $(inputs.params.s3_bucket)/Dockerfile.ubi-$(inputs.params.release_version) + + - name: agent-context-ubi-release + task_type: tag_image + tags: ["ubi", "release"] + distro: ubi + + source: + registry: $(inputs.params.registry)/agent-ubi-context-dev + tag: $(inputs.params.version_id) + + destination: + - registry: $(inputs.params.registry)/mongodb-agent-ubi-context + tag: $(inputs.params.release_version)-context - name: readiness-probe-init vars: diff --git a/pipeline.py b/pipeline.py index 895835553..4ede753d0 100644 --- a/pipeline.py +++ b/pipeline.py @@ -20,17 +20,24 @@ DEFAULT_NAMESPACE = "default" -def build_agent_image_ubi(config: DevConfig) -> None: - image_name = "agent-ubi" +def _load_release() -> Dict: with open("release.json") as f: release = json.loads(f.read()) + return release - args = { + +def _build_agent_args(config: DevConfig) -> Dict[str, str]: + release = _load_release() + return { "agent_version": release["agent"]["version"], "tools_version": release["agent"]["tools_version"], "registry": config.repo_url, } + +def build_agent_image_ubi(config: DevConfig) -> None: + image_name = "agent-ubi" + args = _build_agent_args(config) config.ensure_tag_is_run("ubi") sonar_build_image( @@ -42,14 +49,7 @@ def build_agent_image_ubi(config: DevConfig) -> None: def build_agent_image_ubuntu(config: DevConfig) -> None: image_name = "agent-ubuntu" - with open("release.json") as f: - release = json.loads(f.read()) - args = { - "agent_version": release["agent"]["version"], - "tools_version": release["agent"]["tools_version"], - "registry": config.repo_url, - } - + args = _build_agent_args(config) config.ensure_tag_is_run("ubuntu") sonar_build_image( @@ -60,9 +60,7 @@ def build_agent_image_ubuntu(config: DevConfig) -> None: def build_readiness_probe_image(config: DevConfig) -> None: - with open("release.json") as f: - release = json.loads(f.read()) - + release = _load_release() config.ensure_tag_is_run("readiness-probe") sonar_build_image( diff --git a/scripts/ci/setup_virtualenv.sh b/scripts/ci/setup_virtualenv.sh index af8cf9b6a..89375a374 100755 --- a/scripts/ci/setup_virtualenv.sh +++ b/scripts/ci/setup_virtualenv.sh @@ -7,4 +7,4 @@ virtualenv --python /opt/python/3.7/bin/python3 ./venv pip3 install -r ./requirements.txt # shellcheck disable=SC2154 -pip3 install "git+https://${sonar_github_token}@github.com/10gen/sonar.git@0.0.9" +pip3 install "git+https://${sonar_github_token}@github.com/10gen/sonar.git@0.0.10" From 9e35d2cebb0d896a305326453201bbde0b919184 Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Mon, 26 Apr 2021 11:47:04 +0100 Subject: [PATCH 237/790] Release Operator v0.6.0 (#441) --- config/manager/manager.yaml | 4 ++-- deploy/openshift/operator_openshift.yaml | 8 +++++-- dev_notes/RELEASE_NOTES.md | 3 ++- dev_notes/past_release_notes/v0.6.0.md | 28 ++++++++++++++++++++++++ release.json | 2 +- 5 files changed, 39 insertions(+), 6 deletions(-) create mode 100644 dev_notes/past_release_notes/v0.6.0.md diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 1077cb544..a952985e6 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -16,7 +16,7 @@ spec: serviceAccountName: mongodb-kubernetes-operator containers: - name: mongodb-kubernetes-operator - image: quay.io/mongodb/mongodb-kubernetes-operator:0.5.2 + image: quay.io/mongodb/mongodb-kubernetes-operator:0.6.0 command: - /usr/local/bin/entrypoint imagePullPolicy: Always @@ -36,7 +36,7 @@ spec: - name: VERSION_UPGRADE_HOOK_IMAGE value: quay.io/mongodb/mongodb-kubernetes-operator-version-upgrade-post-start-hook:1.0.2 - name: READINESS_PROBE_IMAGE - value: quay.io/mongodb/mongodb-kubernetes-readinessprobe:1.0.1 + value: quay.io/mongodb/mongodb-kubernetes-readinessprobe:1.0.3 - name: MONGODB_IMAGE value: "library/mongo" - name: MONGODB_REPO_URL diff --git a/deploy/openshift/operator_openshift.yaml b/deploy/openshift/operator_openshift.yaml index 7ca027261..89503d558 100644 --- a/deploy/openshift/operator_openshift.yaml +++ b/deploy/openshift/operator_openshift.yaml @@ -15,7 +15,7 @@ spec: serviceAccountName: mongodb-kubernetes-operator containers: - name: mongodb-kubernetes-operator - image: quay.io/mongodb/mongodb-kubernetes-operator:0.5.2 + image: quay.io/mongodb/mongodb-kubernetes-operator:0.6.0 command: - mongodb-kubernetes-operator imagePullPolicy: Always @@ -35,6 +35,10 @@ spec: - name: AGENT_IMAGE # The MongoDB Agent the operator will deploy to manage MongoDB deployments value: quay.io/mongodb/mongodb-agent:10.29.0.6830-1 - name: READINESS_PROBE_IMAGE - value: quay.io/mongodb/mongodb-kubernetes-readinessprobe:1.0.1 + value: quay.io/mongodb/mongodb-kubernetes-readinessprobe:1.0.3 - name: VERSION_UPGRADE_HOOK_IMAGE value: quay.io/mongodb/mongodb-kubernetes-operator-version-upgrade-post-start-hook:1.0.2 + - name: MONGODB_IMAGE + value: "library/mongo" + - name: MONGODB_REPO_URL + value: "registry.hub.docker.com" diff --git a/dev_notes/RELEASE_NOTES.md b/dev_notes/RELEASE_NOTES.md index a903802e2..603818df9 100644 --- a/dev_notes/RELEASE_NOTES.md +++ b/dev_notes/RELEASE_NOTES.md @@ -12,7 +12,7 @@ Remove the current operator deployment - `kubectl delete deployment ` Delete the existing StatefulSet for the MongoDBCommunity resource - Note: to ensure existing data is not lost, ensure that the retain policy of your Persistent Volumes is configured correctly. + Note: to ensure existing data is not lost, ensure that the retain policy of your Persistent Volumes is configured correctly. Please reference the [official docs](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#reclaiming) for these configuration options. - `kubectl delete statefulset ` Install the new operator - follow the regular [installation instruction](https://github.com/mongodb/mongodb-kubernetes-operator/blob/master/docs/install-upgrade.md) @@ -25,3 +25,4 @@ ## Updated Image Tags * mongodb-kubernetes-operator:0.6.0 * mongodb-agent:0.29.0.6830-1 + * mongodb-kubernetes-readinessprobe:1.0.3 diff --git a/dev_notes/past_release_notes/v0.6.0.md b/dev_notes/past_release_notes/v0.6.0.md new file mode 100644 index 000000000..33adf0f8a --- /dev/null +++ b/dev_notes/past_release_notes/v0.6.0.md @@ -0,0 +1,28 @@ +# MongoDB Kubernetes Operator 0.6.0 +## Kubernetes Operator + +* Breaking Changes + * A new VolumeClaimTemplate has been added `logs-volume`. When you deploy the operator, if there is an existing StatefulSet the operator will attempt to perform an invalid update. The existing StatefulSet must be deleted before upgrading the operator. + + * The user of the mongod and mongodb-agent containers has changed. This means that there will be permissions + issues when upgrading from an earlier version of the operator. In order to update the permissions in the volume, you can use an init container. + +* Upgrade instructions + + Remove the current operator deployment + - `kubectl delete deployment ` + Delete the existing StatefulSet for the MongoDBCommunity resource + Note: to ensure existing data is not lost, ensure that the retain policy of your Persistent Volumes is configured correctly. + - `kubectl delete statefulset ` + Install the new operator + - follow the regular [installation instruction](https://github.com/mongodb/mongodb-kubernetes-operator/blob/master/docs/install-upgrade.md) + Patch the StatefulSet once it has been created. This will add an init container that will update the permissions of the existing volume. + - `kubectl patch statefulset --type='json' --patch '[ {"op":"add","path":"/spec/template/spec/initContainers/-", "value": { "name": "change-data-dir-permissions", "image": "busybox", "command": [ "chown", "-R", "2000", "/data" ], "securityContext": { "runAsNonRoot": false, "runAsUser": 0, "runAsGroup":0 }, "volumeMounts": [ { "mountPath": "/data", "name" : "data-volume" } ] } } ]'` + +* Bug fixes + * Fixes an issue that prevented the agents from reaching goal state when upgrading minor version of MongoDB. + + ## Updated Image Tags + * mongodb-kubernetes-operator:0.6.0 + * mongodb-agent:0.29.0.6830-1 + * mongodb-kubernetes-readinessprobe:1.0.3 diff --git a/release.json b/release.json index 26a022111..17071c634 100644 --- a/release.json +++ b/release.json @@ -1,5 +1,5 @@ { - "mongodb-kubernetes-operator": "0.5.2", + "mongodb-kubernetes-operator": "0.6.0", "version-upgrade-hook": "1.0.2", "readiness-probe": "1.0.3", "agent": { From 5491b3c781b0393b8825c438f69cd817752adb61 Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Mon, 26 Apr 2021 13:10:00 +0100 Subject: [PATCH 238/790] Fix warning message in Stale action (#445) --- .github/workflows/close-stale-issues.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/close-stale-issues.yml b/.github/workflows/close-stale-issues.yml index e0592e2e5..8a806294a 100644 --- a/.github/workflows/close-stale-issues.yml +++ b/.github/workflows/close-stale-issues.yml @@ -22,3 +22,4 @@ jobs: days-before-pr-close: -1 # never close PRs exempt-issue-labels: 'bug,feature-request' + ascending: true From b93bef7c55729af272217ccbb4e38a9eaa3a5c72 Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Mon, 26 Apr 2021 16:21:01 +0100 Subject: [PATCH 239/790] CLOUDP-83092: Add agent release tasks (#437) --- .evergreen.yml | 19 +++++++++++++++++++ inventory.yaml | 29 ++++++++++++++++++++++++++++- pipeline.py | 5 ++--- scripts/ci/config.json | 2 +- 4 files changed, 50 insertions(+), 5 deletions(-) diff --git a/.evergreen.yml b/.evergreen.yml index c45038b32..145399cc9 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -368,6 +368,23 @@ tasks: image_type: readiness-probe-init release: true + - name: release_agent_ubuntu + commands: + - func: clone + - func: setup_virtualenv + - func: build_and_push_image_sonar + vars: + image_type: agent-ubuntu + release: true + + - name: release_agent_ubi + commands: + - func: clone + - func: setup_virtualenv + - func: build_and_push_image_sonar + vars: + image_type: agent-ubi + release: true buildvariants: - name: e2e_tests_ubuntu @@ -448,3 +465,5 @@ buildvariants: - name: release_operator - name: release_version_upgrade_post_start_hook - name: release_readiness_probe + - name: release_agent_ubuntu + - name: release_agent_ubi diff --git a/inventory.yaml b/inventory.yaml index 561f87432..32947617d 100644 --- a/inventory.yaml +++ b/inventory.yaml @@ -49,7 +49,7 @@ images: quay.expires-after: 48h output: - - registry: $(inputs.params.registry)/mongodb-agent-ubuntu-dev + - registry: $(inputs.params.registry)/mongodb-agent-dev tag: $(inputs.params.version_id) - name: agent-template-ubuntu-s3 @@ -76,6 +76,19 @@ images: - registry: $(inputs.params.registry)/mongodb-agent-ubuntu-context tag: $(inputs.params.release_version)-context + - name: agent-ubuntu-release + task_type: tag_image + tags: ["ubuntu", "release"] + distro: ubuntu + + source: + registry: $(inputs.params.registry)/mongodb-agent-dev + tag: $(inputs.params.version_id) + + destination: + - registry: $(inputs.params.registry)/mongodb-agent + tag: $(inputs.params.release_version) + - name: agent-ubi vars: context: . @@ -149,6 +162,20 @@ images: - registry: $(inputs.params.registry)/mongodb-agent-ubi-context tag: $(inputs.params.release_version)-context + - name: agent-ubi-release + task_type: tag_image + tags: ["ubi", "release"] + distro: ubi + + source: + registry: $(inputs.params.registry)/mongodb-agent-ubi-dev + tag: $(inputs.params.version_id) + + destination: + - registry: $(inputs.params.registry)/mongodb-agent-ubi + tag: $(inputs.params.release_version) + + - name: readiness-probe-init vars: context: . diff --git a/pipeline.py b/pipeline.py index 4ede753d0..9f853600c 100644 --- a/pipeline.py +++ b/pipeline.py @@ -30,6 +30,7 @@ def _build_agent_args(config: DevConfig) -> Dict[str, str]: release = _load_release() return { "agent_version": release["agent"]["version"], + "release_version": release["agent"]["version"], "tools_version": release["agent"]["tools_version"], "registry": config.repo_url, } @@ -74,9 +75,7 @@ def build_readiness_probe_image(config: DevConfig) -> None: def build_version_post_start_hook_image(config: DevConfig) -> None: - with open("release.json") as f: - release = json.loads(f.read()) - + release = _load_release() config.ensure_tag_is_run("post-start-hook") sonar_build_image( diff --git a/scripts/ci/config.json b/scripts/ci/config.json index 565aa58d2..de612ceba 100644 --- a/scripts/ci/config.json +++ b/scripts/ci/config.json @@ -5,6 +5,6 @@ "e2e_image": "community-operator-e2e", "version_upgrade_hook_image": "community-operator-version-upgrade-post-start-hook", "testrunner_image": "community-operator-testrunner", - "agent_image_ubuntu": "mongodb-agent-ubuntu-dev", + "agent_image_ubuntu": "mongodb-agent-dev", "agent_image_ubi": "mongodb-agent-ubi-dev" } From e28758d8f8397d9d4eaff75546ffe64cb7612d03 Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Mon, 26 Apr 2021 18:07:57 +0100 Subject: [PATCH 240/790] Move context images into same repo as the built image (#446) --- inventory.yaml | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/inventory.yaml b/inventory.yaml index 32947617d..adbfe514a 100644 --- a/inventory.yaml +++ b/inventory.yaml @@ -24,9 +24,12 @@ images: agent_distro: linux_x86_64 tools_distro: ubuntu1604-x86_64 + labels: + quay.expires-after: 48h + output: - - registry: $(inputs.params.registry)/agent-ubuntu-context-dev - tag: $(inputs.params.version_id) + - registry: $(inputs.params.registry)/mongodb-agent-dev + tag: $(inputs.params.version_id)-context - name: agent-template-ubuntu task_type: dockerfile_template @@ -43,7 +46,7 @@ images: dockerfile: scripts/dev/templates/agent/Dockerfile.ubuntu-$(inputs.params.version_id) buildargs: - imagebase: $(inputs.params.registry)/agent-ubuntu-context-dev:$(inputs.params.version_id) + imagebase: $(inputs.params.registry)/mongodb-agent-dev:$(inputs.params.version_id)-context labels: quay.expires-after: 48h @@ -69,11 +72,11 @@ images: distro: ubuntu source: - registry: $(inputs.params.registry)/agent-ubuntu-context-dev - tag: $(inputs.params.version_id) + registry: $(inputs.params.registry)/mongodb-agent-dev + tag: $(inputs.params.version_id)-context destination: - - registry: $(inputs.params.registry)/mongodb-agent-ubuntu-context + - registry: $(inputs.params.registry)/mongodb-agent tag: $(inputs.params.release_version)-context - name: agent-ubuntu-release @@ -110,8 +113,8 @@ images: tools_distro: rhel70-x86_64 output: - - registry: $(inputs.params.registry)/agent-ubi-context-dev - tag: $(inputs.params.version_id) + - registry: $(inputs.params.registry)/mongodb-agent-ubi-dev + tag: $(inputs.params.version_id)-context - name: agent-template-ubi @@ -129,7 +132,7 @@ images: dockerfile: scripts/dev/templates/agent/Dockerfile.ubi-$(inputs.params.version_id) buildargs: - imagebase: $(inputs.params.registry)/agent-ubi-context-dev:$(inputs.params.version_id) + imagebase: $(inputs.params.registry)/mongodb-agent-ubi-dev:$(inputs.params.version_id)-context labels: quay.expires-after: 48h @@ -155,11 +158,11 @@ images: distro: ubi source: - registry: $(inputs.params.registry)/agent-ubi-context-dev - tag: $(inputs.params.version_id) + registry: $(inputs.params.registry)/mongodb-agent-ubi-dev + tag: $(inputs.params.version_id)-context destination: - - registry: $(inputs.params.registry)/mongodb-agent-ubi-context + - registry: $(inputs.params.registry)/mongodb-agent-ubi tag: $(inputs.params.release_version)-context - name: agent-ubi-release From 64022dfaaa35a76acb4c5329e35cf5e0d5be61cc Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Mon, 26 Apr 2021 21:34:56 +0100 Subject: [PATCH 241/790] Fix Makefile (#447) --- Makefile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 09387eca1..341a2a12d 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,9 @@ BUNDLE_METADATA_OPTS ?= $(BUNDLE_CHANNELS) $(BUNDLE_DEFAULT_CHANNEL) BUNDLE_IMG ?= controller-bundle:$(VERSION) # Image URL to use all building/pushing image targets -IMG ?= /mongodb-kubernetes-operator +REPO_URL := $(shell jq -r .repo_url < ~/.community-operator-dev/config.json) +OPERATOR_IMAGE := $(shell jq -r .operator_image < ~/.community-operator-dev/config.json) +IMG := $(REPO_URL)/$(OPERATOR_IMAGE) DOCKERFILE ?= operator # Produce CRDs that work back to Kubernetes 1.11 (no version conversion) CRD_OPTIONS ?= "crd:trivialVersions=true,preserveUnknownFields=true,crdVersions=v1beta1" @@ -71,7 +73,7 @@ uninstall: manifests kustomize # Deploy controller in the configured Kubernetes cluster in ~/.kube/config deploy: manifests kustomize - cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} + cd config/manager && $(KUSTOMIZE) edit set image quay.io/mongodb/mongodb-kubernetes-operator=$(IMG):latest $(KUSTOMIZE) build config/default | kubectl apply -f - # UnDeploy controller from the configured Kubernetes cluster in ~/.kube/config From ed50d39a32a3222b945369be12a22074ccc2339f Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Tue, 27 Apr 2021 12:40:37 +0100 Subject: [PATCH 242/790] Make docker file S3 bucket configurable (#448) --- inventory.yaml | 7 +++---- pipeline.py | 1 + scripts/ci/config.json | 17 +++++++++-------- scripts/dev/dev_config.py | 4 ++++ 4 files changed, 17 insertions(+), 12 deletions(-) diff --git a/inventory.yaml b/inventory.yaml index adbfe514a..8f68d1abe 100644 --- a/inventory.yaml +++ b/inventory.yaml @@ -1,7 +1,5 @@ vars: registry: - s3_bucket_http: https://enterprise-operator-dockerfiles.s3.amazonaws.com/dockerfiles/mongodb-agent - s3_bucket: s3://enterprise-operator-dockerfiles/dockerfiles/mongodb-agent images: - name: agent-ubuntu @@ -62,9 +60,10 @@ images: inputs: - release_version + - s3_bucket output: - - dockerfile: $(inputs.params.s3_bucket)/Dockerfile.ubuntu-$(inputs.params.release_version) + - dockerfile: $(inputs.params.s3_bucket)/mongodb-agent/Dockerfile.ubuntu-$(inputs.params.release_version) - name: agent-context-ubuntu-release task_type: tag_image @@ -150,7 +149,7 @@ images: - release_version output: - - dockerfile: $(inputs.params.s3_bucket)/Dockerfile.ubi-$(inputs.params.release_version) + - dockerfile: $(inputs.params.s3_bucket)/mongodb-agent/Dockerfile.ubi-$(inputs.params.release_version) - name: agent-context-ubi-release task_type: tag_image diff --git a/pipeline.py b/pipeline.py index 9f853600c..b0c324746 100644 --- a/pipeline.py +++ b/pipeline.py @@ -33,6 +33,7 @@ def _build_agent_args(config: DevConfig) -> Dict[str, str]: "release_version": release["agent"]["version"], "tools_version": release["agent"]["tools_version"], "registry": config.repo_url, + "s3_bucket": config.s3_bucket, } diff --git a/scripts/ci/config.json b/scripts/ci/config.json index de612ceba..f499b125e 100644 --- a/scripts/ci/config.json +++ b/scripts/ci/config.json @@ -1,10 +1,11 @@ { - "namespace": "default", - "repo_url": "quay.io/mongodb", - "operator_image": "community-operator-dev", - "e2e_image": "community-operator-e2e", - "version_upgrade_hook_image": "community-operator-version-upgrade-post-start-hook", - "testrunner_image": "community-operator-testrunner", - "agent_image_ubuntu": "mongodb-agent-dev", - "agent_image_ubi": "mongodb-agent-ubi-dev" + "namespace": "default", + "repo_url": "quay.io/mongodb", + "operator_image": "community-operator-dev", + "e2e_image": "community-operator-e2e", + "version_upgrade_hook_image": "community-operator-version-upgrade-post-start-hook", + "testrunner_image": "community-operator-testrunner", + "agent_image_ubuntu": "mongodb-agent-dev", + "agent_image_ubi": "mongodb-agent-ubi-dev", + "s3_bucket": "s3://enterprise-operator-dockerfiles/dockerfiles" } diff --git a/scripts/dev/dev_config.py b/scripts/dev/dev_config.py index 1bcaa0cb3..1a5f0ddeb 100644 --- a/scripts/dev/dev_config.py +++ b/scripts/dev/dev_config.py @@ -50,6 +50,10 @@ def namespace(self) -> str: def repo_url(self) -> str: return self._config["repo_url"] + @property + def s3_bucket(self) -> str: + return self._config["s3_bucket"] + @property def expire_after(self) -> str: return self._config.get("expire_after", "never") From b86c703a1bb229080e65ccb210ae84705af0fdca Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Wed, 28 Apr 2021 12:15:43 +0100 Subject: [PATCH 243/790] CLOUDP-83092: Create context images for init containers (#440) --- inventory.yaml | 105 ++++++++++++++---- .../readiness/Dockerfile.builder} | 0 .../templates/readiness/Dockerfile.readiness | 6 + .../versionhook/Dockerfile.builder} | 0 .../versionhook/Dockerfile.versionhook | 6 + 5 files changed, 96 insertions(+), 21 deletions(-) rename scripts/dev/{dockerfiles/Dockerfile.readiness => templates/readiness/Dockerfile.builder} (100%) create mode 100644 scripts/dev/templates/readiness/Dockerfile.readiness rename scripts/dev/{dockerfiles/Dockerfile.versionhook => templates/versionhook/Dockerfile.builder} (100%) create mode 100644 scripts/dev/templates/versionhook/Dockerfile.versionhook diff --git a/inventory.yaml b/inventory.yaml index 8f68d1abe..67a20f77f 100644 --- a/inventory.yaml +++ b/inventory.yaml @@ -183,32 +183,64 @@ images: context: . stages: - - name: readiness-probe-init-build - task_type: docker_build - dockerfile: scripts/dev/dockerfiles/Dockerfile.readiness - tags: ["readiness-probe"] + - name: readiness-init-context-build + task_type: docker_build + dockerfile: scripts/dev/templates/readiness/Dockerfile.builder + tags: ["readiness-probe"] - labels: - quay.expires-after: 48h + labels: + quay.expires-after: 48h - output: - - registry: $(inputs.params.registry)/mongodb-kubernetes-readinessprobe-dev - tag: $(inputs.params.version_id) + output: + - registry: $(inputs.params.registry)/mongodb-kubernetes-readinessprobe-dev + tag: $(inputs.params.version_id)-context + + - name: readiness-init-build + task_type: docker_build + dockerfile: scripts/dev/templates/readiness/Dockerfile.readiness + + labels: + quay.expires-after: 48h + + tags: ["readiness-probe"] + buildargs: + imagebase: $(inputs.params.registry)/mongodb-kubernetes-readinessprobe-dev:$(inputs.params.version_id)-context + + output: + - registry: $(inputs.params.registry)/mongodb-kubernetes-readinessprobe-dev + tag: $(inputs.params.version_id) - - name: readiness-probe-init-release - task_type: tag_image - tags: ["release", "readiness-probe"] - inputs: - - release_version + - name: readiness-init-context-release + task_type: tag_image + tags: ["release", "readiness-probe"] + + inputs: + - release_version + + source: + registry: $(inputs.params.registry)/mongodb-kubernetes-readinessprobe-dev + tag: $(inputs.params.version_id)-context + + destination: + - registry: $(inputs.params.registry)/mongodb-kubernetes-readinessprobe + tag: $(inputs.params.release_version)-context + + + - name: readiness-init-build-release + task_type: tag_image + tags: ["release", "readiness-probe"] + + inputs: + - release_version - source: - registry: $(inputs.params.registry)/mongodb-kubernetes-readinessprobe-dev - tag: $(inputs.params.version_id) + source: + registry: $(inputs.params.registry)/mongodb-kubernetes-readinessprobe-dev + tag: $(inputs.params.version_id) - destination: - - registry: $(inputs.params.registry)/mongodb-kubernetes-readinessprobe - tag: $(inputs.params.release_version) + destination: + - registry: $(inputs.params.registry)/mongodb-kubernetes-readinessprobe + tag: $(inputs.params.release_version) - name: version-post-start-hook-init @@ -216,10 +248,24 @@ images: context: . stages: + - name: version-post-start-hook-init-context-build + task_type: docker_build + dockerfile: scripts/dev/templates/versionhook/Dockerfile.builder + tags: ["post-start-hook"] + + labels: + quay.expires-after: 48h + + output: + - registry: $(inputs.params.registry)/mongodb-kubernetes-operator-version-upgrade-post-start-hook-dev + tag: $(inputs.params.version_id)-context + - name: version-post-start-hook-init-build task_type: docker_build - dockerfile: scripts/dev/dockerfiles/Dockerfile.versionhook + dockerfile: scripts/dev/templates/versionhook/Dockerfile.versionhook tags: ["post-start-hook"] + buildargs: + imagebase: $(inputs.params.registry)/mongodb-kubernetes-operator-version-upgrade-post-start-hook-dev:$(inputs.params.version_id)-context labels: quay.expires-after: 48h @@ -228,6 +274,23 @@ images: - registry: $(inputs.params.registry)/mongodb-kubernetes-operator-version-upgrade-post-start-hook-dev tag: $(inputs.params.version_id) + + - name: version-post-start-hook-init-context-release + task_type: tag_image + tags: ["release", "post-start-hook"] + + inputs: + - release_version + + source: + registry: $(inputs.params.registry)/mongodb-kubernetes-operator-version-upgrade-post-start-hook-dev + tag: $(inputs.params.version_id)-context + + destination: + - registry: $(inputs.params.registry)/mongodb-kubernetes-operator-version-upgrade-post-start-hook-context + tag: $(inputs.params.release_version)-context + + - name: version-post-start-hook-init-build-release task_type: tag_image tags: ["release", "post-start-hook"] diff --git a/scripts/dev/dockerfiles/Dockerfile.readiness b/scripts/dev/templates/readiness/Dockerfile.builder similarity index 100% rename from scripts/dev/dockerfiles/Dockerfile.readiness rename to scripts/dev/templates/readiness/Dockerfile.builder diff --git a/scripts/dev/templates/readiness/Dockerfile.readiness b/scripts/dev/templates/readiness/Dockerfile.readiness new file mode 100644 index 000000000..afca0e011 --- /dev/null +++ b/scripts/dev/templates/readiness/Dockerfile.readiness @@ -0,0 +1,6 @@ +ARG imagebase +FROM ${imagebase} as base + +FROM busybox + +COPY --from=base /probes/readinessprobe /probes/readinessprobe diff --git a/scripts/dev/dockerfiles/Dockerfile.versionhook b/scripts/dev/templates/versionhook/Dockerfile.builder similarity index 100% rename from scripts/dev/dockerfiles/Dockerfile.versionhook rename to scripts/dev/templates/versionhook/Dockerfile.builder diff --git a/scripts/dev/templates/versionhook/Dockerfile.versionhook b/scripts/dev/templates/versionhook/Dockerfile.versionhook new file mode 100644 index 000000000..5d2cc0127 --- /dev/null +++ b/scripts/dev/templates/versionhook/Dockerfile.versionhook @@ -0,0 +1,6 @@ +ARG imagebase +FROM ${imagebase} as base + +FROM busybox + +COPY --from=base /version-upgrade-hook /version-upgrade-hook From f8aea36771b939f0141e1130c419cd136b9625e2 Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Wed, 28 Apr 2021 12:54:01 +0100 Subject: [PATCH 244/790] Build images with no expiry (#450) --- inventory.yaml | 131 ++++++++++++++++++++++++++++--------------------- 1 file changed, 76 insertions(+), 55 deletions(-) diff --git a/inventory.yaml b/inventory.yaml index 67a20f77f..3fe06de7b 100644 --- a/inventory.yaml +++ b/inventory.yaml @@ -66,30 +66,38 @@ images: - dockerfile: $(inputs.params.s3_bucket)/mongodb-agent/Dockerfile.ubuntu-$(inputs.params.release_version) - name: agent-context-ubuntu-release - task_type: tag_image + task_type: docker_build + dockerfile: scripts/dev/templates/agent/Dockerfile.builder tags: ["ubuntu", "release"] - distro: ubuntu + buildargs: + agent_version: $(inputs.params.agent_version) + tools_version: $(inputs.params.tools_version) + agent_distro: linux_x86_64 + tools_distro: ubuntu1604-x86_64 - source: - registry: $(inputs.params.registry)/mongodb-agent-dev - tag: $(inputs.params.version_id)-context + labels: + quay.expires-after: Never - destination: + output: - registry: $(inputs.params.registry)/mongodb-agent - tag: $(inputs.params.release_version)-context + tag: $(inputs.params.version_id)-context - name: agent-ubuntu-release - task_type: tag_image + task_type: docker_build tags: ["ubuntu", "release"] distro: ubuntu - source: - registry: $(inputs.params.registry)/mongodb-agent-dev - tag: $(inputs.params.version_id) + dockerfile: scripts/dev/templates/agent/Dockerfile.ubuntu-$(inputs.params.version_id) - destination: + buildargs: + imagebase: $(inputs.params.registry)/mongodb-agent:$(inputs.params.version_id)-context + + labels: + quay.expires-after: Never + + output: - registry: $(inputs.params.registry)/mongodb-agent - tag: $(inputs.params.release_version) + tag: $(inputs.params.version_id) - name: agent-ubi vars: @@ -111,6 +119,9 @@ images: agent_distro: rhel7_x86_64 tools_distro: rhel70-x86_64 + labels: + quay.expires-after: 48h + output: - registry: $(inputs.params.registry)/mongodb-agent-ubi-dev tag: $(inputs.params.version_id)-context @@ -152,31 +163,36 @@ images: - dockerfile: $(inputs.params.s3_bucket)/mongodb-agent/Dockerfile.ubi-$(inputs.params.release_version) - name: agent-context-ubi-release - task_type: tag_image + task_type: docker_build + dockerfile: scripts/dev/templates/agent/Dockerfile.builder tags: ["ubi", "release"] - distro: ubi + buildargs: + agent_version: $(inputs.params.agent_version) + tools_version: $(inputs.params.tools_version) + agent_distro: rhel7_x86_64 + tools_distro: rhel70-x86_64 - source: - registry: $(inputs.params.registry)/mongodb-agent-ubi-dev - tag: $(inputs.params.version_id)-context + labels: + quay.expires-after: Never - destination: + output: - registry: $(inputs.params.registry)/mongodb-agent-ubi - tag: $(inputs.params.release_version)-context + tag: $(inputs.params.version_id)-context - name: agent-ubi-release - task_type: tag_image + task_type: docker_build tags: ["ubi", "release"] - distro: ubi + dockerfile: scripts/dev/templates/agent/Dockerfile.ubi-$(inputs.params.version_id) - source: - registry: $(inputs.params.registry)/mongodb-agent-ubi-dev - tag: $(inputs.params.version_id) + buildargs: + imagebase: $(inputs.params.registry)/mongodb-agent-ubi:$(inputs.params.version_id)-context - destination: - - registry: $(inputs.params.registry)/mongodb-agent-ubi - tag: $(inputs.params.release_version) + labels: + quay.expires-after: Never + output: + - registry: $(inputs.params.registry)/mongodb-agent-ubi + tag: $(inputs.params.version_id) - name: readiness-probe-init vars: @@ -212,33 +228,36 @@ images: - name: readiness-init-context-release - task_type: tag_image - tags: ["release", "readiness-probe"] + task_type: docker_build + dockerfile: scripts/dev/templates/readiness/Dockerfile.builder + tags: ["readiness-probe", "release"] + + labels: + quay.expires-after: Never inputs: - release_version - source: - registry: $(inputs.params.registry)/mongodb-kubernetes-readinessprobe-dev - tag: $(inputs.params.version_id)-context - - destination: + output: - registry: $(inputs.params.registry)/mongodb-kubernetes-readinessprobe tag: $(inputs.params.release_version)-context - name: readiness-init-build-release - task_type: tag_image - tags: ["release", "readiness-probe"] + task_type: docker_build + dockerfile: scripts/dev/templates/readiness/Dockerfile.readiness + tags: ["readiness-probe", "release"] + + buildargs: + imagebase: $(inputs.params.registry)/mongodb-kubernetes-readinessprobe:$(inputs.params.release_version)-context + + labels: + quay.expires-after: Never inputs: - release_version - source: - registry: $(inputs.params.registry)/mongodb-kubernetes-readinessprobe-dev - tag: $(inputs.params.version_id) - - destination: + output: - registry: $(inputs.params.registry)/mongodb-kubernetes-readinessprobe tag: $(inputs.params.release_version) @@ -276,32 +295,34 @@ images: - name: version-post-start-hook-init-context-release - task_type: tag_image + task_type: docker_build + dockerfile: scripts/dev/templates/versionhook/Dockerfile.builder tags: ["release", "post-start-hook"] + labels: + quay.expires-after: Never + inputs: - release_version - source: - registry: $(inputs.params.registry)/mongodb-kubernetes-operator-version-upgrade-post-start-hook-dev - tag: $(inputs.params.version_id)-context - - destination: - - registry: $(inputs.params.registry)/mongodb-kubernetes-operator-version-upgrade-post-start-hook-context + output: + - registry: $(inputs.params.registry)/mongodb-kubernetes-operator-version-upgrade-post-start-hook tag: $(inputs.params.release_version)-context - name: version-post-start-hook-init-build-release - task_type: tag_image + task_type: docker_build + dockerfile: scripts/dev/templates/versionhook/Dockerfile.versionhook tags: ["release", "post-start-hook"] + buildargs: + imagebase: $(inputs.params.registry)/mongodb-kubernetes-operator-version-upgrade-post-start-hook:$(inputs.params.release_version)-context + + labels: + quay.expires-after: Never inputs: - release_version - source: - registry: $(inputs.params.registry)/mongodb-kubernetes-operator-version-upgrade-post-start-hook-dev - tag: $(inputs.params.version_id) - - destination: + output: - registry: $(inputs.params.registry)/mongodb-kubernetes-operator-version-upgrade-post-start-hook tag: $(inputs.params.release_version) From fa2e9401cbb1fb7cae909b292fde3fcadbc88daa Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 28 Apr 2021 12:51:30 +0000 Subject: [PATCH 245/790] Bump k8s.io/apimachinery from 0.20.5 to 0.21.0 (#424) --- go.mod | 2 +- go.sum | 53 +++++++++++++++++++++++++++++++++++------------------ 2 files changed, 36 insertions(+), 19 deletions(-) diff --git a/go.mod b/go.mod index d8a3c04a9..702c42f7d 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( go.mongodb.org/mongo-driver v1.5.1 go.uber.org/zap v1.16.0 k8s.io/api v0.19.2 - k8s.io/apimachinery v0.20.5 + k8s.io/apimachinery v0.21.0 k8s.io/client-go v0.19.2 sigs.k8s.io/controller-runtime v0.7.0 sigs.k8s.io/yaml v1.2.0 diff --git a/go.sum b/go.sum index b6acd0ea5..46a487da0 100644 --- a/go.sum +++ b/go.sum @@ -72,6 +72,7 @@ github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfc github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -183,8 +184,9 @@ github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/V github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -238,6 +240,7 @@ github.com/googleapis/gnostic v0.5.1 h1:A8Yhf6EtqTv9RMsU6MQTyrtV1TjWlR6xU9BsZIwu github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= @@ -277,6 +280,7 @@ github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaR github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.9.8 h1:VMAMUUOh+gaxKTMk+zqbjsSjsIcUcL/LF4o63i82QyA= @@ -286,12 +290,12 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxv github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -309,6 +313,7 @@ github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182aff github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= @@ -321,6 +326,8 @@ github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8m github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= @@ -430,6 +437,7 @@ github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1: github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= @@ -520,8 +528,9 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 h1:OgUuv8lsRpBibGNbSizVwKWlysjaNzmC9gYMhPVfqFM= +golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -533,8 +542,9 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -567,8 +577,10 @@ golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201112073958-5cba982894dd h1:5CtCZbICpIOFdgO940moixOPjc0178IU44m4EjOO5IY= -golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073 h1:8qxJSnu+7dRq6upnbntrmriWByIakBuct5OM/MdQC1M= +golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -614,8 +626,10 @@ golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200616133436-c1934b75d054 h1:HHeAlu5H9b71C+Fx0K+1dGgVFN1DM1/wz4aoGOA5qS8= golang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a h1:CB3a9Nez8M13wwlr/E2YtwoU+qYHKfC+JrDa45RXXoQ= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -666,8 +680,9 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= @@ -684,8 +699,9 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -702,8 +718,8 @@ k8s.io/api v0.19.2/go.mod h1:IQpK0zFQ1xc5iNIQPqzgoOwuFugaYHK4iCknlAQP9nI= k8s.io/apiextensions-apiserver v0.19.2 h1:oG84UwiDsVDu7dlsGQs5GySmQHCzMhknfhFExJMz9tA= k8s.io/apiextensions-apiserver v0.19.2/go.mod h1:EYNjpqIAvNZe+svXVx9j4uBaVhTB4C94HkY3w058qcg= k8s.io/apimachinery v0.19.2/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= -k8s.io/apimachinery v0.20.5 h1:wO/FxMVRn223rAKxnBbwCyuN96bS9MFTIvP0e/V7cps= -k8s.io/apimachinery v0.20.5/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= +k8s.io/apimachinery v0.21.0 h1:3Fx+41if+IRavNcKOz09FwEXDBG6ORh6iMsTSelhkMA= +k8s.io/apimachinery v0.21.0/go.mod h1:jbreFvJo3ov9rj7eWT7+sYiRx+qZuCYXwWT1bcDswPY= k8s.io/apiserver v0.19.2/go.mod h1:FreAq0bJ2vtZFj9Ago/X0oNGC51GfubKK/ViOKfVAOA= k8s.io/client-go v0.19.2 h1:gMJuU3xJZs86L1oQ99R4EViAADUPMHHtS9jFshasHSc= k8s.io/client-go v0.19.2/go.mod h1:S5wPhCqyDNAlzM9CnEdgTGV4OqhsW3jGO1UM1epwfJA= @@ -714,11 +730,11 @@ k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8 k8s.io/gengo v0.0.0-20200428234225-8167cfdcfc14/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.4.0 h1:7+X0fUguPyrKEC4WjH8iGDg3laWgMo5tMnRTIGTTxGQ= -k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/klog/v2 v2.8.0 h1:Q3gmuM9hKEjefWFFYF0Mat+YyFJvsUyYuwyNNJ5C9Ts= +k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= -k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd h1:sOHNzJIkytDF6qadMNKhhDRpc6ODik8lVC6nOur7B2c= -k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= +k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 h1:vEx13qjvaZ4yfObSSXW7BrMc/KQBBT/Jyee8XtLf4x0= +k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE= k8s.io/utils v0.0.0-20200729134348-d5654de09c73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20200912215256-4140de9c8800 h1:9ZNvfPvVIEsp/T1ez4GQuzCcCTEQWhovSofhqR73A6g= k8s.io/utils v0.0.0-20200912215256-4140de9c8800/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= @@ -727,8 +743,9 @@ sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.9/go.mod h1:dzAXnQb sigs.k8s.io/controller-runtime v0.7.0 h1:bU20IBBEPccWz5+zXpLnpVsgBYxqclaHu1pVDl/gEt8= sigs.k8s.io/controller-runtime v0.7.0/go.mod h1:pJ3YBrJiAqMAZKi6UVGuE98ZrroV1p+pIhoHsMm9wdU= sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.0.2 h1:YHQV7Dajm86OuqnIR6zAelnDWBRjo+YhYV9PmGrh1s8= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.1.0 h1:C4r9BgJ98vrKnnVCjwCSXcWjWe0NKcUQkmzDXZXGwH8= +sigs.k8s.io/structured-merge-diff/v4 v4.1.0/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= From ed5135e0ec88952b383057499057bbaf0d4b74da Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Wed, 28 Apr 2021 14:06:56 +0100 Subject: [PATCH 246/790] Add supported agent version on release (#449) --- .evergreen.yml | 36 ++++++++++- pipeline.py | 6 +- release.json | 2 +- requirements.txt | 2 + scripts/ci/add_supported_release.py | 92 +++++++++++++++++++++++++++++ 5 files changed, 132 insertions(+), 6 deletions(-) create mode 100755 scripts/ci/add_supported_release.py diff --git a/.evergreen.yml b/.evergreen.yml index 145399cc9..1ab481794 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -94,6 +94,18 @@ functions: - distro binary: scripts/ci/run_test.sh + python_venv: &python_venv + command: subprocess.exec + type: setup + params: + command: virtualenv --python /opt/python/3.7/bin/python3 ./venv + + python_requirements: &python_requirements + command: subprocess.exec + type: setup + params: + working_dir: mongodb-kubernetes-operator + command: ${workdir}/venv/bin/python -m pip install -r requirements.txt --quiet --no-warn-script-location build_and_push_image: - command: subprocess.exec @@ -140,6 +152,20 @@ functions: - image_type command: scripts/ci/run_image_release.sh + add_supported_release: + - *python_venv + - *python_requirements + - command: subprocess.exec + type: setup + params: + working_dir: mongodb-kubernetes-operator + include_expansions_in_env: + - atlas_connection_string + - atlas_password + - atlas_database + - image_name + command: "${workdir}/venv/bin/python scripts/ci/add_supported_release.py --c ${image_name}" + task_groups: - name: e2e_test_group max_hosts: 8 @@ -374,8 +400,11 @@ tasks: - func: setup_virtualenv - func: build_and_push_image_sonar vars: - image_type: agent-ubuntu + image_name: agent-ubuntu release: true + - func: add_supported_release + vars: + image_name: mongodb-agent - name: release_agent_ubi commands: @@ -383,8 +412,11 @@ tasks: - func: setup_virtualenv - func: build_and_push_image_sonar vars: - image_type: agent-ubi + image_name: agent-ubi release: true + - func: add_supported_release + vars: + image_name: mongodb-agent buildvariants: - name: e2e_tests_ubuntu diff --git a/pipeline.py b/pipeline.py index b0c324746..352a404c5 100644 --- a/pipeline.py +++ b/pipeline.py @@ -29,9 +29,9 @@ def _load_release() -> Dict: def _build_agent_args(config: DevConfig) -> Dict[str, str]: release = _load_release() return { - "agent_version": release["agent"]["version"], - "release_version": release["agent"]["version"], - "tools_version": release["agent"]["tools_version"], + "agent_version": release["mongodb-agent"]["version"], + "release_version": release["mongodb-agent"]["version"], + "tools_version": release["mongodb-agent"]["tools_version"], "registry": config.repo_url, "s3_bucket": config.s3_bucket, } diff --git a/release.json b/release.json index 17071c634..8a4b8ac38 100644 --- a/release.json +++ b/release.json @@ -2,7 +2,7 @@ "mongodb-kubernetes-operator": "0.6.0", "version-upgrade-hook": "1.0.2", "readiness-probe": "1.0.3", - "agent": { + "mongodb-agent": { "version": "10.29.0.6830-1", "tools_version": "100.2.0" } diff --git a/requirements.txt b/requirements.txt index 574ca74b9..b1d6da5fe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,3 +6,5 @@ black==20.8b1 mypy==0.782 tqdm==v4.49.0 boto3==1.16.21 +pymongo==3.11.2 +dnspython==2.0.0 diff --git a/scripts/ci/add_supported_release.py b/scripts/ci/add_supported_release.py new file mode 100755 index 000000000..094f20483 --- /dev/null +++ b/scripts/ci/add_supported_release.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 + +import argparse + +import datetime +import json +import logging +import os +import subprocess +import sys +from typing import Dict, Any + +import pymongo + +LOGLEVEL = os.environ.get("LOGLEVEL", "INFO").upper() +logging.basicConfig(level=LOGLEVEL) + +VALID_IMAGES = frozenset(["mongodb-agent"]) + + +def get_repo_root() -> str: + output = subprocess.check_output("git rev-parse --show-toplevel".split()) + + return output.decode("utf-8").strip() + + +def get_release() -> Dict[str, Any]: + release_file = os.path.join(get_repo_root(), "release.json") + return json.load(open(release_file)) + + +def get_atlas_connection_string() -> str: + password = os.environ["atlas_password"] + cnx_str = os.environ["atlas_connection_string"] + + return cnx_str.format(password=password) + + +def mongo_client() -> pymongo.MongoClient: + cnx_str = get_atlas_connection_string() + return pymongo.MongoClient(cnx_str) + + +def add_release_version(image: str, version: str) -> None: + client = mongo_client() + + database = os.environ["atlas_database"] + collection = client[database][image] + + year_from_now = datetime.datetime.now() + datetime.timedelta(days=365) + + existing_entry = collection.find_one({}, {"version": version}) + if existing_entry is not None: + logging.info("Entry for version {} already present".format(version)) + return + + result = collection.insert_one( + { + "released_on": datetime.datetime.now(), + "version": version, + "supported": True, + "eol": year_from_now, + "variants": ["ubi", "ubuntu"], + } + ) + + logging.info( + "Added new supported version: {} (id: {})".format(version, result.inserted_id) + ) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument( + "--image", help="image to add a new supported version", type=str + ) + args = parser.parse_args() + + if args.image not in VALID_IMAGES: + raise ValueError("Image {} not supported".format(args.image)) + + # for now, there is just one version to add as a supported release. + version = get_release()[args.image]["version"] + logging.info("Adding new release: {} {}".format(args.image, version)) + + add_release_version(args.image, version) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From d944fe52eb20870fe82f5309dc863d191c108faf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 28 Apr 2021 14:46:50 +0100 Subject: [PATCH 247/790] Bump github.com/xdg/stringprep from 1.0.0 to 1.0.3 (#419) Bumps [github.com/xdg/stringprep](https://github.com/xdg/stringprep) from 1.0.0 to 1.0.3. - [Release notes](https://github.com/xdg/stringprep/releases) - [Commits](https://github.com/xdg/stringprep/compare/v1.0.0...v1.0.3) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 702c42f7d..ac4ab7409 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/spf13/cast v1.3.1 github.com/stretchr/objx v0.3.0 github.com/stretchr/testify v1.7.0 - github.com/xdg/stringprep v1.0.0 + github.com/xdg/stringprep v1.0.3 go.mongodb.org/mongo-driver v1.5.1 go.uber.org/zap v1.16.0 k8s.io/api v0.19.2 diff --git a/go.sum b/go.sum index 46a487da0..62bf66ff5 100644 --- a/go.sum +++ b/go.sum @@ -430,8 +430,8 @@ github.com/xdg-go/scram v1.0.2 h1:akYIkZ28e6A96dkWNJQu3nmCzH3YfwMPQExUYDaRv7w= github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= github.com/xdg-go/stringprep v1.0.2 h1:6iq84/ryjjeRmMJwxutI51F2GIPlP5BfTvXHeYjyhBc= github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= -github.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0= -github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= +github.com/xdg/stringprep v1.0.3 h1:cmL5Enob4W83ti/ZHuZLuKD/xqJfus4fVPwE+/BDm+4= +github.com/xdg/stringprep v1.0.3/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= From 1c13809912a49085e53043451660aae9abe28d90 Mon Sep 17 00:00:00 2001 From: Rodrigo Valin Date: Wed, 28 Apr 2021 15:34:22 +0100 Subject: [PATCH 248/790] Updates Golang to 1.15 (#454) --- .github/workflows/go.yml | 2 +- go.mod | 2 +- scripts/dev/dockerfile_generator.py | 2 +- testdata/tls/ca.crt | 45 +++++++++++------- testdata/tls/ca.key | 74 +++++++++++++++++++---------- testdata/tls/server.crt | 39 +++++++++------ testdata/tls/server.key | 50 +++++++++---------- testdata/tls/server_rotated.crt | 39 +++++++++------ testdata/tls/server_rotated.key | 50 +++++++++---------- 9 files changed, 178 insertions(+), 125 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 8fdab9c06..e3f417ea8 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -14,7 +14,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v2 with: - go-version: 1.14 + go-version: 1.15 - name: Test api run: go test -v ./api/... diff --git a/go.mod b/go.mod index ac4ab7409..3e8bab16a 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/mongodb/mongodb-kubernetes-operator -go 1.14 +go 1.15 require ( github.com/blang/semver v3.5.0+incompatible diff --git a/scripts/dev/dockerfile_generator.py b/scripts/dev/dockerfile_generator.py index 6bf824378..111b7c508 100755 --- a/scripts/dev/dockerfile_generator.py +++ b/scripts/dev/dockerfile_generator.py @@ -8,7 +8,7 @@ DockerParameters = Dict[str, Union[bool, str, List[str]]] -GOLANG_TAG = "1.14" +GOLANG_TAG = "1.15" def operator_params(files_to_add: List[str]) -> DockerParameters: diff --git a/testdata/tls/ca.crt b/testdata/tls/ca.crt index 8d0e090d5..4e7bae6d0 100644 --- a/testdata/tls/ca.crt +++ b/testdata/tls/ca.crt @@ -1,19 +1,30 @@ -----BEGIN CERTIFICATE----- -MIIDDjCCAfagAwIBAgIJAIhrBCmDk2GrMA0GCSqGSIb3DQEBCwUAMA0xCzAJBgNV -BAMMAkNBMB4XDTIwMDYxNjIyMzYyNFoXDTMwMDYxNDIyMzYyNFowDTELMAkGA1UE -AwwCQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQD1p9rOsE7LYUhc -XPKmlI9GGRnSdjUee3MQPUiYCa8YwPfbgTtfwrhGqiffh2avX63MeMqTOxUCJjkO -umY2TBdMyRqiWYRN8K1yPT/AMHsSDhdP38qeUvWigLGH56nwee/GsM637KCDW195 -5J2IiC9S+jlRAmQ2APCzNMi8PHrsI5vtlD3d746CuoP7xHytvVMY6jyuVOsPXZZK -uPMsVMvoWLJn1wirrOjHRJK75r0hWrtmwTPPx05SMrokiWLSf+PvO6lCKnURkDk8 -TrTh8WY6AVhdpADQei7Bqin1XmzdXbJWavOfRKxkIHAT2YpIXHkw5voEwAnEtC1f -1XHE92K1AgMBAAGjcTBvMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFDH9Ney+ -SiEiMvmeBp1LKcOotvRlMD0GA1UdIwQ2MDSAFDH9Ney+SiEiMvmeBp1LKcOotvRl -oRGkDzANMQswCQYDVQQDDAJDQYIJAIhrBCmDk2GrMA0GCSqGSIb3DQEBCwUAA4IB -AQDIrAMb/STtBQ2am45TkwsiNOWccDXZwZ78MnhQOMyhgI/EvbIQi4Wz4/KPa3mv -VnJYRfOVtAvi+cX9NVgPmZKTxR42CexnJ423gBSCfFLFlGoDwRxuRGnuvps5AJDO -8vVHkxRAX0JapGivRZIN1WfkfUM80OKxFquVowJ11a3DCqRMIFaJcBZ8Mw2+wrYN -1aylCuIV7N7xM3MVpTMFk3mE5fEqU8MUi2acZXldiy+vTtdO9YQUR0eGV2cfbNcM -jkcAzdF0hJ+YbOZSpUBFA8cDJQCQS1izJlgcrVh03duD3t7pY8Lt4Cbmz+fjQMDB -mfWO09sY+f4gdHh4EyigPRPN +MIIFGDCCAwCgAwIBAgIUKYge8FLQT5AJgyJuurGLeeu/qEMwDQYJKoZIhvcNAQEL +BQAwJDEQMA4GA1UECgwHTW9uZ29EQjEQMA4GA1UEAwwHUm9vdCBDQTAeFw0yMTA0 +MjgxMjA4NTlaFw0zMTA0MjYxMjA4NTlaMCQxEDAOBgNVBAoMB01vbmdvREIxEDAO +BgNVBAMMB1Jvb3QgQ0EwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCu +Y0D6TPSlpR+M7s4QRwFxOcZ/X0qmELmOw579ZbFBK+sEoepnyTiJaHpmHOHKr12g +0KFa/cFo3hZNx4wV6QimdygIzSJzf/h3IJzn1JjhRTQLOz0WdVajg6ITJzxi8Y6V +BpzizoGeQeTKxABLRDslsZ2TtpmNjkJIyqsbhQKoNx0JU64nzGcpuPt5duRSqbxy +iJpYH19OBgrET/clDDwvk04Wi0X3wCESiZG08Zy3oW/Fpn3CQWZeANpsFGDtqLDU +m7YZsejHL8uzhY4Q5bHmLWwMOEwR5j3+7gDojzdL4wjGd1wzgwcFHLTF6WyV1w3q +mSIAOZ2RmkrBDMDRURJs9eG5yonWS+XS2m/H8EyG65if9GXV5mnKHIt+bZgTD98s +xJM4Wa0v46w17rcynQK/OrGF1NG7NLlORasDq0VCtqAbQnGmbgNzmYGFX3US2H/o +u1lOmHRsDZTpQ9gLUqSfQh4mdVPT9CE/C+DGqsqsi5iNXMP6z6JJzibzInnx8URd +ug6iI3YjeP3GYsuLnwp9RoOAq5/FdsKAb63AzY+mRZ7tkANNFenJAzbgwXeO86YZ +JK89gZ5rEp9RlTH9yZ6et7zan0FVaP/fwqsJp6o+ZmAfSmtHZvbCK45q+0KYvyKG +/h5bweER/cRCmQl7UA8Cb5ZEXaa8mdQu1WVzl6VBSwIDAQABo0IwQDAdBgNVHQ4E +FgQU0qBFL0HfYJevdR0b0vAaYxSLpZswDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8B +Af8EBAMCAuQwDQYJKoZIhvcNAQELBQADggIBAFFQzP/UDDxRX1C4OBG5Ak4PcBMb +i2BEObBTGJpGzHsK4na8wO+9p2DFNTipE4BYLmLaUzcKP+Q3pef32Ks5i88bn9sr +KmqOzrcgzcLpIOR6CZoJ4VfVIelGzKqbdDB6At6/tj42TO7UMt49fFOVgvM/uVX8 +/G5l5ZJGUVmD3NgyazLEsnp8AJAR3Yq0S/ODHWswWlgw7oucuZJ6bMaafqSzCA7i +pqdBLfFVRv0DshifnM2sy3fq6X1iDWhiGqj43FdTHqYjAB0zi/4fBVYuQ0zlKDNp +JRMNgIWzjMw6mhl5aMGskazVmQQC5s8ET1mXZ3RyGcFD872EQ/JLjKOARXIO442z +f+Sr50i5KU5mmgSKFYDyKavVs4XlTiagg/2hw+uOdas8IiQb3wkFITco7kFWeg7r +qQbUllAJlvaNV/wX27NMyUp6wt5lhjN6HYS0l7FvhzR9UgGpRC/KOjrmuCk9AMoS +Gfz5rmkiQbLhQUnmxTXUy5ddkCJ5uSP3NMoMLOte3tUZJXqvleb77xkltL9oXSqr +P/EzTaTuVgXBXEU5ODvaFjOtsXfgBuYIKT4/xDCFFK6jwkRZ/JyIS81j9dINAZez +SLeM3BYydpb+ZRcJR4A5710VAs6U05eAlcW7Zzb6sO6Ex/iGS37FS2N1l3YnPHr0 +RjPbH3tDZRVxi4Vt -----END CERTIFICATE----- diff --git a/testdata/tls/ca.key b/testdata/tls/ca.key index e89d2e86d..fb14ad0e5 100644 --- a/testdata/tls/ca.key +++ b/testdata/tls/ca.key @@ -1,27 +1,51 @@ -----BEGIN RSA PRIVATE KEY----- -MIIEpAIBAAKCAQEA9afazrBOy2FIXFzyppSPRhkZ0nY1HntzED1ImAmvGMD324E7 -X8K4Rqon34dmr1+tzHjKkzsVAiY5DrpmNkwXTMkaolmETfCtcj0/wDB7Eg4XT9/K -nlL1ooCxh+ep8HnvxrDOt+ygg1tfeeSdiIgvUvo5UQJkNgDwszTIvDx67COb7ZQ9 -3e+OgrqD+8R8rb1TGOo8rlTrD12WSrjzLFTL6FiyZ9cIq6zox0SSu+a9IVq7ZsEz -z8dOUjK6JIli0n/j7zupQip1EZA5PE604fFmOgFYXaQA0Houwaop9V5s3V2yVmrz -n0SsZCBwE9mKSFx5MOb6BMAJxLQtX9VxxPditQIDAQABAoIBAQDan2EO8z+cpt50 -PNBCyhHcS7h8YnERyMH1YOT7Uhr/h7TOKGwD5FKFqSdFOsZ7K0aMo699amZe/E9p -OSCAECXoU02d0Nb7WkeyVKooy7yrq9i5YCx9zM0Iq0nT7quB9bMb+/xtGW0cmjQV -Ug2VFcYwLbrWyq8OlLWid0Zyp8yp26Srkp+Ww32/HSE4SrJsHCDDbm1cv6J+5cQN -AJZxy3DRG8Da5SMziBXF6Ak9DCPJ4WNY/h97nR1hGRpRL/43EF5KxXLE5PQ5K2vS -f4u+XQ9xr+wNdpPhCRlGjumrN160NfwnRueZpP28qsNYp+eKMgytgKVza+9C8xpc -nbN9hCUdAoGBAPtOCRK/GOrlAk406hUfnSdULNAAx6HNxQahRqVS3LBhYbBPjUQF -dfrsxAatR/3kregExMiX2RDWQ2tP1VhgkD8f8jTuNUHqg9yyI4GulUk5nwOOpR5i -c+S6EbDfyihF0I5VRLZsslyIZ2dwokCMVf2G/v82WzJjxZuobbLBUqxPAoGBAPo+ -zM3GD+laeSPksPej5wdPM2TXqpC/4pyQEdqYowbdZhuUgcPfs/nd5nRyfqHJEBYg -rDRCEknnlQiROy2VsT2xk9yzm8K2wbAmYPL18U/P3UK9ro+HPrhAssAH3f1AdlQQ -p/m7r/+yCm+Nu/ABclpBTSakDvNTPqqT/RDzaeu7AoGBAOi86axB7PrGIu95E/2C -LNJao7Hztu8EGd+9BydfaWG0nntiAYuonw5HrngcjRr3aZQCrpi0AOUV+pRs+EIE -FDNSC8e3gP1OCtce58FXhYyuX7U0q3lLZtfC91VevOO1XKQlFiD44+rMrS9gYfPz -xEOrfYewuJ8fRciWIRo27rEDAoGAJYP+riawQfqXSKNDSIzg2L6t5MNURckOjwwm -MRBK+H9+jEnyPkwuLhpvLyvvAhAjgj25s0fod/3o4NTs+0OCw834bdDbXBCL7v2B -8Uqft6hg8l1/avMKmxLP+Wv0gsAEPUwV8qGJnBqz589mym9dFlzN1SopN59WOLpg -6Qak7XsCgYBWzrfNDHrx9NB8ifsqabeyj/V9Cx6LqiLfkiu6jkV8VBthLtSlKZDi -lU3dzpYYFdgHA10uEQ73DaErHiM4cADiypRzOu4AVMMyui5zkHuYXNQCuKDpHvAE -PlrsiHdaqVq2t4X2+C6e3UQuhJQPsUHYpbzaFRE6BmPLIDbUQOl9gA== +MIIJJwIBAAKCAgEArmNA+kz0paUfjO7OEEcBcTnGf19KphC5jsOe/WWxQSvrBKHq +Z8k4iWh6Zhzhyq9doNChWv3BaN4WTceMFekIpncoCM0ic3/4dyCc59SY4UU0Czs9 +FnVWo4OiEyc8YvGOlQac4s6BnkHkysQAS0Q7JbGdk7aZjY5CSMqrG4UCqDcdCVOu +J8xnKbj7eXbkUqm8coiaWB9fTgYKxE/3JQw8L5NOFotF98AhEomRtPGct6FvxaZ9 +wkFmXgDabBRg7aiw1Ju2GbHoxy/Ls4WOEOWx5i1sDDhMEeY9/u4A6I83S+MIxndc +M4MHBRy0xelsldcN6pkiADmdkZpKwQzA0VESbPXhucqJ1kvl0tpvx/BMhuuYn/Rl +1eZpyhyLfm2YEw/fLMSTOFmtL+OsNe63Mp0CvzqxhdTRuzS5TkWrA6tFQragG0Jx +pm4Dc5mBhV91Eth/6LtZTph0bA2U6UPYC1Kkn0IeJnVT0/QhPwvgxqrKrIuYjVzD ++s+iSc4m8yJ58fFEXboOoiN2I3j9xmLLi58KfUaDgKufxXbCgG+twM2PpkWe7ZAD +TRXpyQM24MF3jvOmGSSvPYGeaxKfUZUx/cmenre82p9BVWj/38KrCaeqPmZgH0pr +R2b2wiuOavtCmL8ihv4eW8HhEf3EQpkJe1APAm+WRF2mvJnULtVlc5elQUsCAwEA +AQKCAgBBfCAQXgmYklMwtxRGZIOUIx/5AK2lgq5LgAYaHa/cS0Orr6m4Y4WJg+RC +qCHZ3NSJ6Q5Ofu+8E3nIp2Bhceq/qAsukumW+b0x7ts860algTkz5oDgCBwKtwmL +q7YvaYojSCJtwSJHbXMe+U8q9GpJk1Ma/vzWfU9CymhKoz4GMPwEXpoNc+Jhdodo +a07+A6MyVz2uTcmaIQa2BVlHXjrTmrs+F/qkOE+zCFng2sIA6uxwCj87TRfFwQhE +gbqMREZy8C4HpBlHgxk5RrO8gKS3TONAC3v1VMJ/Epzgt7cKFi1bacMDzPY74zW3 +BBq0gddF+08bSody/7+GkuVtNMZ36YV2Cr7tBSCrji2phXEZp2yfJB6H8ER0xSmI +CMIy8fIaQ38nKjzoqREM2z3+5n7L/rLbHmI17cFR2IvR6E32x7PnWVau49aGx+gp +SMFAN6bWnmXfolOaBKreDtzjU2csPuWqQRuBiACOPxxDcTkyoc97tdpNPLF/OJQf +r9/all91jn+9icC7gY1GJ5HZU0aexHgelQsMINobp4zZ/85o42Rcf0icRnBv/uKh +FZ2Infu1nblelmOrfqaZN4AeWkTnQH9GCECyN81lFe3xk4R0AWYws2NUAJlC0VX8 +wkREOYjAOhGk4QEhfnAxH6JskC9JG7f822RYeIp40Yum5ksw0QKCAQEA0/0l2t7f +tzRCQTcRU7rSFqXKUZfZ5MuQDWQ20JNqDlhl4V6+hlcWYEj65OaDHMIvFEO3YDxJ +KEPIUQykbkbbPfFlTeQYN0HIGBEMcYjCZfOsSh6V+C7YBAoUUNNeVvK0A9/8ox0C +KGA9hjXWveoqohjE+WBmMYzj7bKV+0roJM3zAiVcQYU4QRQpPy9s7rBGc7hO7C8x +E2tP2J8ppX+Cm1BtrmSSyfSHzl5H/z2teVh1WMY29qJk4kSz9ar8Wgql97jr1De6 +trkLC2ELHJZRvj/AH2aOPrfXHZ9hgp+Zm6Y0QYjU9sXtNgqrcXXkpIhABA5qnrvv +oEDtlGTgXzxqpwKCAQEA0perAZyxl6atTMsziAymXgGO6eBarfV4ZMLSqv8Wb7ef +sidnW4BYJ+8PDfJIjhHPZrTFzwHgJY6jdTG51z1zlUgXfrHgvXjY+thLpi72QBX1 +nFoiv3yPpQwP6YWELrPbfk0MJVSwLk3e+CEJ9QaU0B3MXmqkix99Mqj/IycGEoLR +LmaEF+0T42rK1oIVhe63clWokkGR6dGZOD7R8X0LPibOPCZgw4kBDV+Ts11+6SrW +hh8dOO27j0NrhlcIJtUNpOF5lc/YkF6J85HmZRFO2xMjlpPivwt5fzoZTQULGnJB +myG38qJzh6Z0bwDUyTyj46phUI6t94bMtOZGA+XcvQKCAQAmyweXYvu7kfOh7Yrd +MK3reRFqFwjHxryoxrMFPkTFNUYHlQf2m487tX88TjamF5737WBsWvvkQ2sv8clR +aOQMFNW+CESqL/6G8O+/AxDYCVx4/9nf3eqn6pRHKjb0YFuy7dVUoCVZ8CqyGb4f +aO++VBwPqqVo4eiAAhNSNiX/PjT/KokUcGWX+zGFH4+mqllqKcs/i29Gp3eoI5BC +efATrgc0R7FZBceoazZvrgDF2Ps4cKV0QsmFYp8wEMc3TwWKLKvzXPNtJrWvsmWP +KK7yysXEuUbEkW3rPNRiTASXKDYd3AVk11mObytqivF5bnmQhHbcb3XtdJRFeKRa +qTq3AoIBAHETy1jkQF/Duc5AVf05faOAhrKCK265HdpM2j3DFtMJ8BADJHZ3Zd6b +eKPucpMGcS0e6BANW8QO4SoudzFh58xCl4sKDDjwyOF4frZPBR7IK8e2hm33flIL +NggYDy/cl4Er8dehr7BNU+o8I1KUJBfTyEbyUEP7togbQlQnwhTydUvK9Ca8wC8e +yqd/1frCgXI9GHTgDG9WgkU3Rhe7IuxOffLCULbtu8johzCKhXjB7cq6rkBBNrLo +arYJyXDAecEM6PJOVmS+vqgwsVuXfZYYosVZxyj6ClUMZ0V57cpXi263HkSEgjBW +arKn6Zncm8vFUtekpTkCAq0n8HRrWqkCggEASI96vzDeUdt52bThenMdQMC1jCwr +eN289YFjP6llfX3MSxg3IW/moebMqJ5MULj7BrNErqYYCE9NsDp14fnXHgpAkxyi +EduxU+JFgHtjTnVLyX0NNIPGHfI7iQvqEJYwpjjyF8uQZgQAknNGjJFu80mwzuZi +LVzl3ssM8erk6KGpkMa35CKH7MvcfSRZ6//T0NxPceste42Af5ZdT8OZyF/VBgci +EXrk42dHzUlJkzAWzLAV0IUzcXvnP517cP7KOIuNo4gYNpZd0Esj62HZfWMIK5A3 +zKhW/vj04bmjb5wkoly/2ydKXBPbCOnl0/+6UKCdQ+9j3dyH9iRuFFj4sA== -----END RSA PRIVATE KEY----- diff --git a/testdata/tls/server.crt b/testdata/tls/server.crt index 7aaaca225..2772fe138 100644 --- a/testdata/tls/server.crt +++ b/testdata/tls/server.crt @@ -1,17 +1,26 @@ -----BEGIN CERTIFICATE----- -MIICszCCAZsCAQEwDQYJKoZIhvcNAQELBQAwDTELMAkGA1UEAwwCQ0EwHhcNMjAw -NzIyMDgxMDQzWhcNMzAwNzIwMDgxMDQzWjAyMTAwLgYDVQQDDCcqLm1kYi10bHMt -c3ZjLmRlZmF1bHQuc3ZjLmNsdXN0ZXIubG9jYWwwggEiMA0GCSqGSIb3DQEBAQUA -A4IBDwAwggEKAoIBAQDsJftQQU+5IyP7SrJw5lydbNjtdfDH6LP5nd58rUh0KxLJ -iQmsQBoePQpTh2uF9bzKzCuglrqHjTRszUJiziO5/eTMucH26AlsT9bR6md8w42I -mnLQ+UNnt89k+tWsr0lXrV31IHkeOSAmme5mK10T9r0YtMQojOept5s5vt4zsNKs -pdS5cv8eSPQu7+GzwEA0QrC75MIXRyVbuSf0HKZJET1Ic36mL/ucwK2LjEQbUT/Q -Daf/WbzVvPMochvMPxwEiFMIDQyzJJO/WZbUX1jGLn09JJJQWNYbrv0wRJst0H11 -5zVlN1mYgYpVtaCSPOEaZRT0Qe4sjn2aG8NtIgLhAgMBAAEwDQYJKoZIhvcNAQEL -BQADggEBAOdhoqeMzFn5c4m71V9RhJlGGQ75mlmWqZZFBZFEHwZl0eZz8YS/SXDZ -xG1rurkvpM/dRHbJRt4x1rc300dRLaeJ3jbIXnYpEt3vlWmfpJmoReyHvH0e5jGc -AFBn+bRo/Gl59NlKUh3h2dXfqZ7bI63sYUYp4dUAH2b/jhKp+onMkdT5KiglOKIP -1ZwmttTVZo/QIUgQzS2VZSisAHQnt8tHhh0byQJiKK29Lm5ghzsAs2BIvajsCKva -2LkaDUONaVkFn1S3xD7JMJfM5YgYOmANsQYa5CSUzrEz6BT1idJAwekWRDRaEuHj -M0D03eMd66Soh8UvwQnLT3U4vd3aGNk= +MIIEWjCCAkKgAwIBAgIUKXYivNfzneHnf77o/hmGJPPZmiMwDQYJKoZIhvcNAQEL +BQAwJDEQMA4GA1UECgwHTW9uZ29EQjEQMA4GA1UEAwwHUm9vdCBDQTAeFw0yMTA0 +MjgxMjI0MjRaFw0zNTAxMDUxMjI0MjRaMDkxCzAJBgNVBAYTAlVTMQswCQYDVQQI +DAJOWTELMAkGA1UEBwwCTlkxEDAOBgNVBAoMB01vbmdvREIwggEiMA0GCSqGSIb3 +DQEBAQUAA4IBDwAwggEKAoIBAQDMgVLGC5blAlvcmbggfnmFZ0wHAstOxbjOPija +53TzvKi9L2Smrwf5/RtRQSZ6cgNfTLzDbz+jKHn5v0jWqSW5TzWSL1VcDiYSoito ++RwJcRmrLBuceqP8anUjCgqmDH5xFL2w+QNh9knGdOvbUkGr+gaUxeQxNclup8jV +v9qyRva2an8MB7VbSG8ZVDVkcBkH2xlO+S1ITl/SPBXKsDbOB/hWEAqkOoEom5lQ +6a4IjUYU8HUhebzERH71Jhgc53hcs1RourMLQmAZQoqy7E8On6B/jZxMqq4HsqiQ +PYV/FLPlT12hgKMBZ6POwIFxEueTGVuGzHU37aoxPwNT7cp/AgMBAAGjbzBtMB8G +A1UdIwQYMBaAFNKgRS9B32CXr3UdG9LwGmMUi6WbMAkGA1UdEwQCMAAwCwYDVR0P +BAQDAgTwMDIGA1UdEQQrMCmCJyoubWRiLXRscy1zdmMuZGVmYXVsdC5zdmMuY2x1 +c3Rlci5sb2NhbDANBgkqhkiG9w0BAQsFAAOCAgEAeeVcqSuI3UjmThAufNN5I+Z5 +jIUyU/kTcOHUr5hDA83+W8IuEHo/g+ZsvtCVqTqiXNd5Ehn5rdO+YB8fqXC2jgUr +VLbel87qdxqTwdZ6pO3X0StO1AuSN/ZydnfZqRyI7fJn28A0fzTHP5AZdOAYtGBR +nld9omH85p2EsZkhtdsZpRPr11mQoFnJ9lGcz2z/6GRbrlEYrM9nU4Ij8cBAlhrM +hkqNpQT56XM1QxJ6MdEwYQv4Fbkr5Aa75NGyb0m6uQNYDPyXgvvkSZ+lZTXBhVl1 +5GouRqRMe+hlGPYL4VKy23PAwag7dNlQ1GQLur+pWkXfHLdKIaoPLDwFx4m8PWGr +rErXOXKKYZIw+xsQYKOXNePMM3/bRlEZTt52wrBEDB3LNhNkuKB8J1+/dfE557l1 +5/Gyt+MuRAq/gi+ffR7KxuzYDipGSUmmWzFF/5LyOCAS9lKi8xyKzsYpdDDkcx8k +aC86zOjYseMKytk2hgOmNPjva9iG4mlQQ/S7FgOn01jJadpu9X0zVgmq7uKIemUM +6ts8qEK3zIGir10FfT0zxwaXQOMLMHrLvELGJEhHJPTQDMjPopKVEXtk9Upeveas +PK3QLsn3xE2XytH2HJnAHL3GR1nLT3HgdyrOlJlV37ZPXr3di7nfQQM4UJoghyWH +JZ6umbgvvVMWeLFX/IE= -----END CERTIFICATE----- diff --git a/testdata/tls/server.key b/testdata/tls/server.key index e653a77b1..a7289b9af 100644 --- a/testdata/tls/server.key +++ b/testdata/tls/server.key @@ -1,27 +1,27 @@ -----BEGIN RSA PRIVATE KEY----- -MIIEpAIBAAKCAQEA7CX7UEFPuSMj+0qycOZcnWzY7XXwx+iz+Z3efK1IdCsSyYkJ -rEAaHj0KU4drhfW8yswroJa6h400bM1CYs4juf3kzLnB9ugJbE/W0epnfMONiJpy -0PlDZ7fPZPrVrK9JV61d9SB5HjkgJpnuZitdE/a9GLTEKIznqbebOb7eM7DSrKXU -uXL/Hkj0Lu/hs8BANEKwu+TCF0clW7kn9BymSRE9SHN+pi/7nMCti4xEG1E/0A2n -/1m81bzzKHIbzD8cBIhTCA0MsySTv1mW1F9Yxi59PSSSUFjWG679MESbLdB9dec1 -ZTdZmIGKVbWgkjzhGmUU9EHuLI59mhvDbSIC4QIDAQABAoIBAEnqBYR0PODk9+Ey -2zFtWTXJGQkSbmAUHSkXWclKb7A0vzenlgh9M++dCXtlmqkeZo5PY6RrKU0+TFd1 -076baSFRL+lIh0aiEDj/sGyZ4vRxPP6x4Rg5vPhc1yRzQqg/YUR5NjyAgoiMNtz+ -N1lxXzvdcgimo/NRTz2XA5YKgQBKUVLw92wXKwCi6rrEuTuaxr0OdHEdoQzLWv/f -gRVeXLf9eHkhYwh3Wu9Si2Yl4lUKcx7OdFvZr5TzY3y9IgxE+gfT+U4z3zGfwC2y -euml+A/7KYnt2/oA+BBrWbb0W7enOLJ4T5jbsHQgGLk03fe4sALIo4T1z8a2t2HR -wZusUAECgYEA/rkvJxWoIzU2LO4j66NLOAz+N62zKPWdQlYL25LSLz/5IJgAFpNA -r9k1p3GKRYfX7p9tyHS4/hhyo2AR+IXwJePOJvG97/U3lu3EkfY7Lb9Vuu3n6VK2 -gPgax30n5LkYJjTxIPd4sYSuIfeDew7xs1hB1x6sCNhP6qIMGI82m2ECgYEA7VT3 -IRcvxS/fbnZt/NMEvvkwdh2Un8DG9H29XEsiLcYv58q/aDNaR/LyvDZFbgW+FE9R -X2TpwnAgosW8Ctiyf08qM9jxdU402WhbHgAptncWilxsWNJXkJA5RGJ6imbbDjZ6 -YNdImvnSxenWp37+Dm2Np16JJQhJIYHzJUOoF4ECgYEAhXBFf1gdCLSreMYYEy0s -DmTgGBLqtB5XD5U8CP7VFOOSgryd7zWcwYIsVVdpdBtBx0PFoylib9om0+dUArlH -oNHCASzKr5XqVSqhu0Ueo0yEgLR4tQYbjVxryu2JpIxCVmGNoBOEKpqzDiA3xJOD -ksw9UZBD1y5aTzQs2gDMPoECgYA/oW8ctR2+rYYnFKOKjH0SQrdGg8nMRyBQfsHd -U9uXEDLZ35cP9ey3q6B+68ITrIB464cyn1i5I9zsJz2yXsUEsxHqkriyLcSnoX4E -fiCw5h7p+7uk2MhXXwOrnQejwc3rcpm/CxlRS7fCDl7Zy4eMEL1Q6Vy1zBHnZPLu -w8P2gQKBgQCtBUlZzsy2MNK78gWtflLZZl01FZDjMd8ucyezvTkEx7WWvi1p8l0v -mpUGF43d1zcKAbVBPY0gbaD1aPYK4aMyRkS0lN6k44tW3qKjmIhA7SGdqHSX66Tr -8/sqbf/7LPoWdaB/l1T4jBHLNur6Iy+lLWx8psbTqNhocaTkNI6M6Q== +MIIEpAIBAAKCAQEAzIFSxguW5QJb3Jm4IH55hWdMBwLLTsW4zj4o2ud087yovS9k +pq8H+f0bUUEmenIDX0y8w28/oyh5+b9I1qkluU81ki9VXA4mEqIraPkcCXEZqywb +nHqj/Gp1IwoKpgx+cRS9sPkDYfZJxnTr21JBq/oGlMXkMTXJbqfI1b/askb2tmp/ +DAe1W0hvGVQ1ZHAZB9sZTvktSE5f0jwVyrA2zgf4VhAKpDqBKJuZUOmuCI1GFPB1 +IXm8xER+9SYYHOd4XLNUaLqzC0JgGUKKsuxPDp+gf42cTKquB7KokD2FfxSz5U9d +oYCjAWejzsCBcRLnkxlbhsx1N+2qMT8DU+3KfwIDAQABAoIBAQC7HjVbim0l25Or +9Gb6LF8KhiqVW6Qkzls7Mrr1GMT045FNkRi6PvrAbSvanA8WCE43m6I3/AmxQy7g +Knr+FsSymtw8htzGnxeNAx9PLGfP59GBwpj9A2YaZloJln2J03K6Cy1JyX6j2tNE +J+VKxyfZsKrm427Y7AsEGbd0hNgZN5s9l70q0FSCkFcb37b497k0gYcE+63wEaq3 +FHGoYvbjUVKqp1YpVQyALlHk2toDMOOVBt4MQzP6RsVQJ3LY7K0ZYlNu83EWutsJ +oIMjDwMoCpDtFqrUDzCgbYoDPAaREOBFJZcUrqQ3oTMCo8qEZgiinOVQks6vqnpd +vke/qfsRAoGBAPcm/4AkVeRCmEmR16U9K8pk2KyOJxbXvSvLPO55ytAHSeHEQYaE +FevTOYj+Whd5B/OWOcGXrvby0OpzfEizpE/cLyCGPQqONh5RyJUeG9mzmSCGfJKw +dru99Sg2njU+ZYmHtf6FtY6RGZ4OrwiifVzk/slGE9r0LJt0uVJ/Db83AoGBANPT +fWAetG/JJVG8RoQnddHzZhpmJAnqQt6QbKiYZ/WsH5mchsuJg2oybY9uf9TL1OMy +yxhCie1vFBBRD1s6j06btqF38i9D2H6R55i2PtP5AKFD6S9wucpFRiR0A5r5r69V +KwnYA1fu0uA6tYw457f0vS8NfIaiEDmERfiy4qL5AoGBANYsXUzWL/hWHVHjqFPw +5nnFWl5t8UHCQpQo0ux1bmNHbabPQ1kmLTjnGfy1La0ZnOJhVDuHDn/Be3kwCouV +4NWzoMM2kL8M7ajohkFyjf/hutiMsncLpFidDE2ExySspaDAkd22UNbytphZcSSy +aqCNcJ1KtPoQjndIdzAeGfORAoGABbbm4vjxFTLv9syFens2CnvufTfUMRBIzYhH +5iR2aYJDN/mpCUSkbvD9U6k/eZYmIBr2r6jb37PnbqlBKMzjoNNCkgiSWAQUixWU +keIYv88v3Snf2I/J81L7GXCnyD6EJs69Yn6ZWH3w4muzCh1e4u+PSv2qJleo6GRR +Hux0gMECgYBN0BaeyUPqRLgq3JsrPTK0VQ+8J5+3la7wZWU5vCr2cvLftzb7Devj +m5K901mFCPdtO5LJ8OdeOi1PHnG/+WCfuwDitN8OufPJ+tdSteG+F9XIu5sTMGLB +QJeIyHolsPZhW4OA3C7p6uZHAeDIqIpkv8j7974cLBWhGlQJx9403A== -----END RSA PRIVATE KEY----- diff --git a/testdata/tls/server_rotated.crt b/testdata/tls/server_rotated.crt index d6079114e..22c9abff6 100644 --- a/testdata/tls/server_rotated.crt +++ b/testdata/tls/server_rotated.crt @@ -1,17 +1,26 @@ -----BEGIN CERTIFICATE----- -MIICszCCAZsCAQIwDQYJKoZIhvcNAQELBQAwDTELMAkGA1UEAwwCQ0EwHhcNMjAw -NzIyMTYwOTE4WhcNMzAwNzIwMTYwOTE4WjAyMTAwLgYDVQQDDCcqLm1kYi10bHMt -c3ZjLmRlZmF1bHQuc3ZjLmNsdXN0ZXIubG9jYWwwggEiMA0GCSqGSIb3DQEBAQUA -A4IBDwAwggEKAoIBAQDS9+ePhIjG9lDMo/UYov1MYo/bVUZ5dourMwUKAFnhjZYH -Z0HWqboaD+THsqfA6eoRcPadavucV+J1fg+ZUo4srbcGEQTNjBfogusLYwVdPm+/ -C0kyEc6nU10AVvhNrB4egALBSD6/nEyzd5Ua4bscQT1Hj3q8834cCJRcEr7MxLyY -Pr8vnBzILyOf7JOoJ148JwJjs1WuSh0fVmVSNLJ76GLbd3X0s1zMPPkl2bv+IZqb -Far3oLvCEyyKxeum0PmzTGzBAefyQs2dPwaGgws3Tfjxvlb8vZyCa4N/pW87QhC6 -AbSsheGk05WxVyIb8EWQ2LpBJnmFyLzDdlRNcra1AgMBAAEwDQYJKoZIhvcNAQEL -BQADggEBAL3oBfO7malbfuLvApwnzFFwWHHqe0GtUGsy6GK2v0wlA0hVmMvkB0RK -jfUQyEI911bF/ll3V32l52fkBM9p6OEEuJvS09FnqQVFWDSkZtLiyog5L6IUo5I8 -4Yq/MHKnw/ziDLjrfRfkEFeLTjDthONVebERYOWEFPpRQrW6H9U2ToNvc91/wZ7H -XhtgdukSYDCG39X+SnTgXCCJXuPcaqc1na6FzigNGmJ4IrI2VGU6MRnLt6m969VN -3Wkc/Gx+qyWobpmI8hfTARlP8jPjG3DBIjNHPeF7R2hy0ppSrn+4LVdjf/DMD7EH -9OH2mOsVXeSLUu9TzjvavZwrB6QMKyU= +MIIEWjCCAkKgAwIBAgIUKXYivNfzneHnf77o/hmGJPPZmiQwDQYJKoZIhvcNAQEL +BQAwJDEQMA4GA1UECgwHTW9uZ29EQjEQMA4GA1UEAwwHUm9vdCBDQTAeFw0yMTA0 +MjgxMjI2MDFaFw0zNTAxMDUxMjI2MDFaMDkxCzAJBgNVBAYTAlVTMQswCQYDVQQI +DAJOWTELMAkGA1UEBwwCTlkxEDAOBgNVBAoMB01vbmdvREIwggEiMA0GCSqGSIb3 +DQEBAQUAA4IBDwAwggEKAoIBAQCzFl/uTaH7b3+VGaa6iyYvYYxmp4zmV9O0SSoF +ihwqU7v4Ar7F+cKMduU3ueLlZh7BXdziqWeiIVOxhPTPPpcLzKndc6xzGuJyng7G +paDjPgDBjCMGZd9z+u56rYxqBRbIILyB6t1p6BK4DUMQ+tbZyaikAMLvNlw1c6se +oWO6+fr/wGyaw84VDW7qGmn0eWx2hvatY44BDthypoa3UFW54OIx8uzn9/Xye43J +V/+15Qwc9YADgXducDkSBussC891hwVJJ//r3ljkHavmCFlwnquf4eLQgyzma85w +EEMNrVDsNUO8kmLrm3se9WNiPMW41p/o/vBiBZuid4rr02YvAgMBAAGjbzBtMB8G +A1UdIwQYMBaAFNKgRS9B32CXr3UdG9LwGmMUi6WbMAkGA1UdEwQCMAAwCwYDVR0P +BAQDAgTwMDIGA1UdEQQrMCmCJyoubWRiLXRscy1zdmMuZGVmYXVsdC5zdmMuY2x1 +c3Rlci5sb2NhbDANBgkqhkiG9w0BAQsFAAOCAgEAi0snRlGNjVn/tfA2wGz+LRT5 +cMnYdESgdNdvqj7VA3etNEPxiZ4zBt6yRTN1Q0R1q1gHVzWdlWBUS3TCd3nyorPE +tS5BkKApICa7m1rVj9ZkDT+aRZsI1Gqve4p8/Ofdn3uEJ7xHEBtX6dlodHSr88UW +ig0xVhKEjc6Z9WviuCBTog3jsonW64iG01D3W3h/9dh0uCbZAoVbAO281qWv6Mov +fqg+FJ6iZ9FMhTUrSIyaJB1VaJvuprchcBjAPQkw7/AYyloOTbajejTprFdqmwHk +JTydwMleW4PnRLxqk/+/csJJfZt0acGC4XxJ414pXVMWoQBYQsaNVgGEgQ0SXKtK +FnenswMyYorZaOd3SxJXbSA6eF5q+3zvS4zxW9Qj6aVklUYcHAx1+WPlAz/7w80x +SsVP/qYYlLm8u5yd1pi/xDGiG+RrPedSltKb3hQk1NAci45WwAbYONiEhvJ6+/UH +5NJxgVqfCBDoi8+TtZPC7xXl07SZwsmhVfBNyow6miK0lkko4n1i7mU42XTYNZZD +51OkDAV2oKZhrSfV19zKn8AO0RFYDcmSRBFIC366qWUIQeLxPKLZ4Tu5wH0F8/y/ +epaut2X2S36LRGBEg9Qsrd3a4oLwvqKADO4oGN2pSCHWJznhxxoak3DtinSu9YBW +JV2fmZcawmu1X6FHRI4= -----END CERTIFICATE----- diff --git a/testdata/tls/server_rotated.key b/testdata/tls/server_rotated.key index ca6db81d7..2ea518aa0 100644 --- a/testdata/tls/server_rotated.key +++ b/testdata/tls/server_rotated.key @@ -1,27 +1,27 @@ -----BEGIN RSA PRIVATE KEY----- -MIIEpAIBAAKCAQEA0vfnj4SIxvZQzKP1GKL9TGKP21VGeXaLqzMFCgBZ4Y2WB2dB -1qm6Gg/kx7KnwOnqEXD2nWr7nFfidX4PmVKOLK23BhEEzYwX6ILrC2MFXT5vvwtJ -MhHOp1NdAFb4TaweHoACwUg+v5xMs3eVGuG7HEE9R496vPN+HAiUXBK+zMS8mD6/ -L5wcyC8jn+yTqCdePCcCY7NVrkodH1ZlUjSye+hi23d19LNczDz5Jdm7/iGamxWq -96C7whMsisXrptD5s0xswQHn8kLNnT8GhoMLN0348b5W/L2cgmuDf6VvO0IQugG0 -rIXhpNOVsVciG/BFkNi6QSZ5hci8w3ZUTXK2tQIDAQABAoIBAFOEAA84kEzzRZk9 -btGJ9GxAuGJDp9p9q1zinaQP74Ty3+meXtnz5tBaal8DwpUEBL1S0s4Og/yobeXF -ObegjtfxDPtB0XztcKmC6jlfwcff56zhcuB3XC+xOKfhxfo5oNLCKlUJnI2N+m8x -b71sjv3odbHWgug6HDpMyy6H3TCCGSu7UoWtjA8hmRW/xMOactdAxck6NdfYWvHA -REXs+SQEPkk+T73It8QDHGVqKAXddmF2CDhQOe0BMLkuwqAzeekO3eAulE/WKQMj -y12I79tTTb2Ms5tl89drOeYzs1ux8/lxWtCtN3P12Gis8wa4rswbY4jLPZwgIMRG -U5tcQiECgYEA91KjFC2WVeg3Y2FJv31eE+Z52dIsKbq2/hkl2WeVbL+Inc8zZBds -7aOFO2U/REWJr45EZ3jNkSu5AzAjYXKIjNIk0+SVvDa851+2wAZbQXdxS5+ruiRu -zsDlw48OT3ynMEIIuwiEsPKfq2FWae+UliWxwLuKw2/K5KuDn5/TXqcCgYEA2l6+ -0C2s38YFFu0pjaQfqk7aGqfJ9qWt7tm3FBrDRyWgPYi1LhWN5f6xlHZBw+WxS3IN -joEH4CzHmADi1nhEsb+tQHVc5CZ2GmgIfMw13XE1FBdRW6PgdEPIC7UYBDNh9wMC -zPDkI6jV7zipsRyIoEs9PTXkRa7C6DuPAbTOp0MCgYEAwFW9iPWi0hAS8vA3v/ko -7mTwIdr2iUUxBg5chuOtKrMQ9ViraI1nIq9l7zjfqKJDXwlOXQFvLBRKfxYyjZfa -ZVkPVtGPOJ2A7pZasp6+3PycWOlFTS8EFTmh9SENSfdwtXDFBV6sgkdMsKSz5RJy -BQovX+j5Et+fc5GGfN54LEUCgYBV4RkN6kiooMnzoEXNTJSfd+9SuFY3SCVFYB4e -LABMhMGmMZN/kj6CC05vYqqujjDRyQMH3jrosPO2FfMgAaCSfx110jI8D9w2ul9M -JUux0Qnc4ua+MY7eaqHL6OaPEF4gtPBvBPXUCFxKfnBOFTiuQajN39nshbRlfLbb -Ju523QKBgQCO1uWOdZFrMKejK8lS+OLJBZ0IQl4SKATIce4xppLGyV/p837uAhet -n0LuuzjwC+8DG8ZPeC+bhAgO0Sow7NHAfWnxL3kVExVFm3qVX0uiw3+rRMu1bRuU -OUIoHZf7nUmYEXqCAwMecQDZpGtPyMKGuOx8EFlYQGo7I0JNF1cl3w== +MIIEpAIBAAKCAQEAsxZf7k2h+29/lRmmuosmL2GMZqeM5lfTtEkqBYocKlO7+AK+ +xfnCjHblN7ni5WYewV3c4qlnoiFTsYT0zz6XC8yp3XOscxricp4OxqWg4z4AwYwj +BmXfc/rueq2MagUWyCC8gerdaegSuA1DEPrW2cmopADC7zZcNXOrHqFjuvn6/8Bs +msPOFQ1u6hpp9Hlsdob2rWOOAQ7YcqaGt1BVueDiMfLs5/f18nuNyVf/teUMHPWA +A4F3bnA5EgbrLAvPdYcFSSf/695Y5B2r5ghZcJ6rn+Hi0IMs5mvOcBBDDa1Q7DVD +vJJi65t7HvVjYjzFuNaf6P7wYgWboneK69NmLwIDAQABAoIBAGnVjTe9dT6sM8+f +ayLO3PAfS+PWnLP7r0bZ/hVr+x0ggvMcXDWPVmPAV9HI7sf2w7IukDz7NB1iaJ1+ +H1bifE0i1DflBkK33p8xvTWz6BKjL7sx3/kF9zoJTyn8qgB1pXL7tatpaxQNbBKM +89dzBcmLHTheotTPYUrNYpEle1ShLXUYTr2oJ+ggCCuaPsUCFIbrvdj1KwlRFMls +B9MRe1jgsrsAJp2VTNWlh2OVq2NYcVRJ7xZgrsUNjWgK+F34YsZGLRob13RRPGSB +OGVtZJ8iWqRtQ6IvR6WxU8eVpn08W9sVWU/CGY0q95vXckz+2IxFo1lcN87rQRFp +bpZWCXECgYEA3Y8q1ZKWrhJKm8Zdd6C5AEkFWPWd4C8uQ0Xk8Rzw8sLcIoC+FMxL +UnNzzcE9rY2DsPLul3LCBKlE8yRIVc77Y8FmtWUROeUbLApZLvfi/CB8rpUQvYdq +4FCp3uG/nGpERdOXN7AWHPNKqsUYgbJlAB3Ktnz2Hkqu+8LjUEuC/h0CgYEAzu0Q +FcUKYvC+Mib2AB7JrfB0sgFpnenN4Ek2fczdAsn96rKuY40xVePhk/MhHAQozDOC +qZWnZODFCr8kdxWKWsPJCiDrfrs7yYu4iAs0TW7BpQHGehf9pji174zFqIJ5cRpv +CPzZs6HoeolT8AA81vgPLfkQu0eVVsbvJB1UM7sCgYEAz/xgB5HOpaZCJ621fGGC +igQCYxpflF52HWz1mGrEvf+yyyj0R23on4QGB/cJwWyBXZEP7VgrTljggydiSDs/ +vsuFcW9pFmI+eb9VLURC7tBIe1MwHduLtvvCG5rsVxdi8/HHmN1SROcXBuxzLv2a +1tsguuLf4FvXnDu9TFk2pPkCgYEAkRxQJmQax426Y6NM6oKsvk0dGOPCtFoM5VeS +XiA3cUhDx73sa6XNTkVToRRkSKhmD+WjoNdxH1488C9hHB2/+6zHJFI3s5UL9WgF +TWpGEHM2W0fmbyK530Jv3ADlcCmnjniSG0RuSvmk9aGuemVARSSKSJwcbqlhggxa +EgJ7VRcCgYA6YY42K6B0zqt/dhq2vcEYgz+lRVROZ5R0VYUWsF1pYx2JRjPoTYq/ +cl47VQOqS+m06Mz9OSI/QrH/kaO2PhQBi2FP2Y39GQPVxs+6nVoWzirszstSfy9A +TRlkCp4NMixsn/kTIeUbnYEoLGmQ7XvMIDRl0ZjcuFms0Q6kFEqWmw== -----END RSA PRIVATE KEY----- From 492a3191ac347b11833f899f3b81825dec433465 Mon Sep 17 00:00:00 2001 From: Rodrigo Valin Date: Thu, 29 Apr 2021 08:52:54 +0100 Subject: [PATCH 249/790] CLOUDP-86193: Updated to controller-runtime 0.8.3 (#456) --- go.mod | 10 +-- go.sum | 195 +++++++++++++++++++++++++++++++++++++++++++++ test/e2e/client.go | 2 +- 3 files changed, 201 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 3e8bab16a..63dbea953 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/mongodb/mongodb-kubernetes-operator go 1.15 require ( - github.com/blang/semver v3.5.0+incompatible + github.com/blang/semver v3.5.1+incompatible github.com/go-logr/logr v0.4.0 github.com/hashicorp/go-multierror v1.1.1 github.com/imdario/mergo v0.3.12 @@ -15,10 +15,10 @@ require ( github.com/xdg/stringprep v1.0.3 go.mongodb.org/mongo-driver v1.5.1 go.uber.org/zap v1.16.0 - k8s.io/api v0.19.2 - k8s.io/apimachinery v0.21.0 - k8s.io/client-go v0.19.2 - sigs.k8s.io/controller-runtime v0.7.0 + k8s.io/api v0.20.4 + k8s.io/apimachinery v0.20.4 + k8s.io/client-go v0.20.4 + sigs.k8s.io/controller-runtime v0.8.3 sigs.k8s.io/yaml v1.2.0 ) diff --git a/go.sum b/go.sum index 62bf66ff5..ecae3201b 100644 --- a/go.sum +++ b/go.sum @@ -5,24 +5,47 @@ cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6A cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= cloud.google.com/go v0.51.0/go.mod h1:hWtGJ6gnXH+KgDv+V0zFGDvpi07n3z8ZNj3T1RW0Gcw= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +github.com/Azure/go-autorest v13.3.2+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= github.com/Azure/go-autorest/autorest v0.9.6/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630= +github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= +github.com/Azure/go-autorest/autorest v0.11.12/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= github.com/Azure/go-autorest/autorest/adal v0.8.2/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= +github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= +github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g= +github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= +github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= +github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= +github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= @@ -39,7 +62,10 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.34.28 h1:sscPpn/Ns3i0F4HPEWAVcwdIRaZZCuL7llJ2/60yPIk= @@ -49,8 +75,11 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/blang/semver v3.5.0+incompatible h1:CGxCgetQ64DKk7rdZ++Vfnb1+ogGNnB17OJKJXD2Cfs= github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= +github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= @@ -63,6 +92,7 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -93,6 +123,7 @@ github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLi github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -100,7 +131,9 @@ github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2H github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= @@ -184,6 +217,7 @@ github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/V github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= @@ -197,10 +231,13 @@ github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4er github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -228,6 +265,8 @@ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXi github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -238,6 +277,7 @@ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5m github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= github.com/googleapis/gnostic v0.5.1 h1:A8Yhf6EtqTv9RMsU6MQTyrtV1TjWlR6xU9BsZIwuTCM= github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= @@ -247,15 +287,31 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= @@ -275,6 +331,7 @@ github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= @@ -297,6 +354,7 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -306,12 +364,20 @@ github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7 github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= @@ -344,6 +410,7 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.2 h1:aY/nuoWlKJud2J6U0E3NWsjlg+0GtwXxgEqthRdzlcs= github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= @@ -354,6 +421,7 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= @@ -375,12 +443,16 @@ github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7z github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.2.0 h1:wH4vA7pcjKuZzjF7lM8awk4fnuJO6idemZXoKnULUx4= +github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -388,6 +460,8 @@ github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= @@ -397,6 +471,7 @@ github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= +github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= @@ -404,6 +479,7 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -417,6 +493,7 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -442,6 +519,7 @@ go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/etcd v0.5.0-alpha.5.0.20200819165624-17cef6e3e9d5/go.mod h1:skWido08r9w6Lq/w70DO5XYIKMu4QFu1+4VsqLQuJy8= +go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= @@ -450,6 +528,7 @@ go.mongodb.org/mongo-driver v1.5.1/go.mod h1:gRXCHX4Jo7J0IJ1oDQyUxF7jfy19UfxniMS go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= @@ -467,6 +546,7 @@ go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.16.0 h1:uFRZXykJGK9lLY4HtgSw44DnIcAM+kRBP7x5m+NpAOM= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= @@ -479,11 +559,20 @@ golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0 h1:hb9wdF1z5waM+dSIICn1l0DkLVDT3hqhhQsDNUmHPRE= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g= +golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -495,11 +584,14 @@ golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f h1:J5lckAjkw6qYlOZNj90mLYNTEKDvWeuc1yieZ8qUzUE= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -507,7 +599,9 @@ golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -520,15 +614,21 @@ golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 h1:OgUuv8lsRpBibGNbSizVwKWlysjaNzmC9gYMhPVfqFM= golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -536,18 +636,23 @@ golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6 h1:pE8b58s1HRDMi8RDc79m0HISf9D4TzseP40cEA6IGfs= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -563,24 +668,37 @@ golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201112073958-5cba982894dd h1:5CtCZbICpIOFdgO940moixOPjc0178IU44m4EjOO5IY= +golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073 h1:8qxJSnu+7dRq6upnbntrmriWByIakBuct5OM/MdQC1M= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE= +golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -594,6 +712,8 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e h1:EHBhcS0mlXEAVwNyO2dLfjToGsyY4j24pTs2ScHnX7s= golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -604,6 +724,7 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -623,9 +744,23 @@ golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a h1:CB3a9Nez8M13wwlr/E2YtwoU+qYHKfC+JrDa45RXXoQ= @@ -641,7 +776,12 @@ google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEt google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -657,8 +797,18 @@ google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -666,6 +816,7 @@ google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -688,6 +839,7 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= @@ -699,6 +851,7 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= @@ -713,36 +866,78 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/api v0.19.2 h1:q+/krnHWKsL7OBZg/rxnycsl9569Pud76UJ77MvKXms= k8s.io/api v0.19.2/go.mod h1:IQpK0zFQ1xc5iNIQPqzgoOwuFugaYHK4iCknlAQP9nI= +k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo= +k8s.io/api v0.20.2 h1:y/HR22XDZY3pniu9hIFDLpUCPq2w5eQ6aV/VFQ7uJMw= +k8s.io/api v0.20.2/go.mod h1:d7n6Ehyzx+S+cE3VhTGfVNNqtGc/oL9DCdYYahlurV8= +k8s.io/api v0.20.4 h1:xZjKidCirayzX6tHONRQyTNDVIR55TYVqgATqo6ZULY= +k8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ= +k8s.io/api v0.21.0 h1:gu5iGF4V6tfVCQ/R+8Hc0h7H1JuEhzyEi9S4R5LM8+Y= +k8s.io/api v0.21.0/go.mod h1:+YbrhBBGgsxbF6o6Kj4KJPJnBmAKuXDeS3E18bgHNVU= k8s.io/apiextensions-apiserver v0.19.2 h1:oG84UwiDsVDu7dlsGQs5GySmQHCzMhknfhFExJMz9tA= k8s.io/apiextensions-apiserver v0.19.2/go.mod h1:EYNjpqIAvNZe+svXVx9j4uBaVhTB4C94HkY3w058qcg= +k8s.io/apiextensions-apiserver v0.20.1 h1:ZrXQeslal+6zKM/HjDXLzThlz/vPSxrfK3OqL8txgVQ= +k8s.io/apiextensions-apiserver v0.20.1/go.mod h1:ntnrZV+6a3dB504qwC5PN/Yg9PBiDNt1EVqbW2kORVk= k8s.io/apimachinery v0.19.2/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= +k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= +k8s.io/apimachinery v0.20.2 h1:hFx6Sbt1oG0n6DZ+g4bFt5f6BoMkOjKWsQFu077M3Vg= +k8s.io/apimachinery v0.20.2/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= +k8s.io/apimachinery v0.20.4 h1:vhxQ0PPUUU2Ns1b9r4/UFp13UPs8cw2iOoTjnY9faa0= +k8s.io/apimachinery v0.20.4/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apimachinery v0.21.0 h1:3Fx+41if+IRavNcKOz09FwEXDBG6ORh6iMsTSelhkMA= k8s.io/apimachinery v0.21.0/go.mod h1:jbreFvJo3ov9rj7eWT7+sYiRx+qZuCYXwWT1bcDswPY= k8s.io/apiserver v0.19.2/go.mod h1:FreAq0bJ2vtZFj9Ago/X0oNGC51GfubKK/ViOKfVAOA= +k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU= k8s.io/client-go v0.19.2 h1:gMJuU3xJZs86L1oQ99R4EViAADUPMHHtS9jFshasHSc= k8s.io/client-go v0.19.2/go.mod h1:S5wPhCqyDNAlzM9CnEdgTGV4OqhsW3jGO1UM1epwfJA= +k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y= +k8s.io/client-go v0.20.2 h1:uuf+iIAbfnCSw8IGAv/Rg0giM+2bOzHLOsbbrwrdhNQ= +k8s.io/client-go v0.20.2/go.mod h1:kH5brqWqp7HDxUFKoEgiI4v8G1xzbe9giaCenUWJzgE= +k8s.io/client-go v0.20.4 h1:85crgh1IotNkLpKYKZHVNI1JT86nr/iDCvq2iWKsql4= +k8s.io/client-go v0.20.4/go.mod h1:LiMv25ND1gLUdBeYxBIwKpkSC5IsozMMmOOeSJboP+k= +k8s.io/client-go v0.21.0 h1:n0zzzJsAQmJngpC0IhgFcApZyoGXPrDIAD601HD09ag= +k8s.io/client-go v0.21.0/go.mod h1:nNBytTF9qPFDEhoqgEPaarobC8QPae13bElIVHzIglA= +k8s.io/client-go v1.5.2 h1:JOxmv4FxrCIOS54kAABbN8/hA9jqGpns+Zc6soNgd8U= k8s.io/code-generator v0.19.2/go.mod h1:moqLn7w0t9cMs4+5CQyxnfA/HV8MF6aAVENF+WZZhgk= +k8s.io/code-generator v0.20.1/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= k8s.io/component-base v0.19.2 h1:jW5Y9RcZTb79liEhW3XDVTW7MuvEGP0tQZnfSX6/+gs= k8s.io/component-base v0.19.2/go.mod h1:g5LrsiTiabMLZ40AR6Hl45f088DevyGY+cCE2agEIVo= +k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk= +k8s.io/component-base v0.20.2 h1:LMmu5I0pLtwjpp5009KLuMGFqSc2S2isGw8t1hpYKLE= +k8s.io/component-base v0.20.2/go.mod h1:pzFtCiwe/ASD0iV7ySMu8SYVJjCapNM9bjvk7ptpKh0= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20200428234225-8167cfdcfc14/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/klog/v2 v2.4.0 h1:7+X0fUguPyrKEC4WjH8iGDg3laWgMo5tMnRTIGTTxGQ= +k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.8.0 h1:Q3gmuM9hKEjefWFFYF0Mat+YyFJvsUyYuwyNNJ5C9Ts= k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= +k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd h1:sOHNzJIkytDF6qadMNKhhDRpc6ODik8lVC6nOur7B2c= +k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 h1:vEx13qjvaZ4yfObSSXW7BrMc/KQBBT/Jyee8XtLf4x0= k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE= k8s.io/utils v0.0.0-20200729134348-d5654de09c73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20200912215256-4140de9c8800 h1:9ZNvfPvVIEsp/T1ez4GQuzCcCTEQWhovSofhqR73A6g= k8s.io/utils v0.0.0-20200912215256-4140de9c8800/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20210111153108-fddb29f9d009 h1:0T5IaWHO3sJTEmCP6mUlBvMukxPKUQWqiI/YuiBNMiQ= +k8s.io/utils v0.0.0-20210111153108-fddb29f9d009/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.9/go.mod h1:dzAXnQbTRyDlZPJX2SUPEqvnB+j7AJjtlox7PEwigU0= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/controller-runtime v0.7.0 h1:bU20IBBEPccWz5+zXpLnpVsgBYxqclaHu1pVDl/gEt8= sigs.k8s.io/controller-runtime v0.7.0/go.mod h1:pJ3YBrJiAqMAZKi6UVGuE98ZrroV1p+pIhoHsMm9wdU= +sigs.k8s.io/controller-runtime v0.8.3 h1:GMHvzjTmaWHQB8HadW+dIvBoJuLvZObYJ5YoZruPRao= +sigs.k8s.io/controller-runtime v0.8.3/go.mod h1:U/l+DUopBc1ecfRZ5aviA9JDmGFQKvLf5YkZNx2e0sU= sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.0.2 h1:YHQV7Dajm86OuqnIR6zAelnDWBRjo+YhYV9PmGrh1s8= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.1.0 h1:C4r9BgJ98vrKnnVCjwCSXcWjWe0NKcUQkmzDXZXGwH8= sigs.k8s.io/structured-merge-diff/v4 v4.1.0/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= diff --git a/test/e2e/client.go b/test/e2e/client.go index 21e4b6ddb..8a63ff9df 100644 --- a/test/e2e/client.go +++ b/test/e2e/client.go @@ -8,7 +8,7 @@ import ( "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/deprecated/scheme" + "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" From 72891443c8fe1354e5ff42562bc033531cbff2b0 Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Thu, 29 Apr 2021 16:59:16 +0100 Subject: [PATCH 250/790] Update podspec_template.go (#459) --- pkg/kube/podtemplatespec/podspec_template.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/kube/podtemplatespec/podspec_template.go b/pkg/kube/podtemplatespec/podspec_template.go index 4074638fd..6ce66b936 100644 --- a/pkg/kube/podtemplatespec/podspec_template.go +++ b/pkg/kube/podtemplatespec/podspec_template.go @@ -222,7 +222,7 @@ func WithAnnotations(annotations map[string]string) Modification { // WithVolumeMounts will add volume mounts to a container or init container by name func WithVolumeMounts(containerName string, volumeMounts ...corev1.VolumeMount) Modification { return func(podTemplateSpec *corev1.PodTemplateSpec) { - c := findContainerByName(containerName, podTemplateSpec) + c := FindContainerByName(containerName, podTemplateSpec) if c == nil { return } @@ -230,8 +230,8 @@ func WithVolumeMounts(containerName string, volumeMounts ...corev1.VolumeMount) } } -// findContainerByName will find either a container or init container by name in a pod template spec -func findContainerByName(name string, podTemplateSpec *corev1.PodTemplateSpec) *corev1.Container { +// FindContainerByName will find either a container or init container by name in a pod template spec +func FindContainerByName(name string, podTemplateSpec *corev1.PodTemplateSpec) *corev1.Container { containerIdx := findIndexByName(name, podTemplateSpec.Spec.Containers) if containerIdx != notFound { return &podTemplateSpec.Spec.Containers[containerIdx] From a1b823b02ed8e0823dfaf34d32376e4607aab884 Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Fri, 7 May 2021 16:53:04 +0100 Subject: [PATCH 251/790] Support for AppDB upgrade path (#469) --- api/v1/mongodbcommunity_types.go | 8 ++++++ controllers/construct/mongodbstatefulset.go | 26 ++++++++++--------- .../automation_config_builder.go | 22 ++++++++++++++-- .../automation_config_test.go | 6 ++--- 4 files changed, 45 insertions(+), 17 deletions(-) diff --git a/api/v1/mongodbcommunity_types.go b/api/v1/mongodbcommunity_types.go index fa6dd9f1a..8c4232c2a 100644 --- a/api/v1/mongodbcommunity_types.go +++ b/api/v1/mongodbcommunity_types.go @@ -534,6 +534,14 @@ func (m MongoDBCommunity) GetAnnotations() map[string]string { return m.Annotations } +func (m MongoDBCommunity) DataVolumeName() string { + return "data-volume" +} + +func (m MongoDBCommunity) LogsVolumeName() string { + return "logs-volume" +} + type automationConfigReplicasScaler struct { current, desired int } diff --git a/controllers/construct/mongodbstatefulset.go b/controllers/construct/mongodbstatefulset.go index 38f00c106..15ebd00f8 100644 --- a/controllers/construct/mongodbstatefulset.go +++ b/controllers/construct/mongodbstatefulset.go @@ -26,8 +26,6 @@ const ( versionUpgradeHookName = "mongod-posthook" ReadinessProbeContainerName = "mongodb-agent-readinessprobe" - dataVolumeName = "data-volume" - logVolumeName = "logs-volume" readinessProbePath = "/opt/scripts/readinessprobe" agentHealthStatusFilePathEnv = "AGENT_STATUS_FILEPATH" clusterFilePath = "/var/lib/automation/config/cluster-config.json" @@ -80,6 +78,10 @@ type MongoDBStatefulSetOwner interface { HasSeparateDataAndLogsVolumes() bool // GetAgentScramKeyfileSecretNamespacedName returns the NamespacedName of the secret which stores the keyfile for the agent. GetAgentKeyfileSecretNamespacedName() types.NamespacedName + // DataVolumeName returns the name that the data volume should have + DataVolumeName() string + // LogsVolumeName returns the name that the data volume should have + LogsVolumeName() string } // BuildMongoDBReplicaSetStatefulSetModificationFunction builds the parts of the replica set that are common between every resource that implements @@ -119,20 +121,20 @@ func BuildMongoDBReplicaSetStatefulSetModificationFunction(mdb MongoDBStatefulSe logVolumeClaim := statefulset.NOOP() singleModeVolumeClaim := func(s *appsv1.StatefulSet) {} if mdb.HasSeparateDataAndLogsVolumes() { - logVolumeMount := statefulset.CreateVolumeMount(logVolumeName, automationconfig.DefaultAgentLogPath) - dataVolumeMount := statefulset.CreateVolumeMount(dataVolumeName, "/data") - dataVolumeClaim = statefulset.WithVolumeClaim(dataVolumeName, dataPvc()) - logVolumeClaim = statefulset.WithVolumeClaim(logVolumeName, logsPvc()) + logVolumeMount := statefulset.CreateVolumeMount(mdb.LogsVolumeName(), automationconfig.DefaultAgentLogPath) + dataVolumeMount := statefulset.CreateVolumeMount(mdb.DataVolumeName(), "/data") + dataVolumeClaim = statefulset.WithVolumeClaim(mdb.DataVolumeName(), dataPvc(mdb.DataVolumeName())) + logVolumeClaim = statefulset.WithVolumeClaim(mdb.LogsVolumeName(), logsPvc(mdb.LogsVolumeName())) mongodbAgentVolumeMounts = append(mongodbAgentVolumeMounts, dataVolumeMount, logVolumeMount) mongodVolumeMounts = append(mongodVolumeMounts, dataVolumeMount, logVolumeMount) } else { mounts := []corev1.VolumeMount{ - statefulset.CreateVolumeMount(dataVolumeName, "/data", statefulset.WithSubPath("data")), - statefulset.CreateVolumeMount(dataVolumeName, automationconfig.DefaultAgentLogPath, statefulset.WithSubPath("logs")), + statefulset.CreateVolumeMount(mdb.DataVolumeName(), "/data", statefulset.WithSubPath("data")), + statefulset.CreateVolumeMount(mdb.DataVolumeName(), automationconfig.DefaultAgentLogPath, statefulset.WithSubPath("logs")), } mongodbAgentVolumeMounts = append(mongodbAgentVolumeMounts, mounts...) mongodVolumeMounts = append(mongodVolumeMounts, mounts...) - singleModeVolumeClaim = statefulset.WithVolumeClaim(dataVolumeName, dataPvc()) + singleModeVolumeClaim = statefulset.WithVolumeClaim(mdb.DataVolumeName(), dataPvc(mdb.DataVolumeName())) } podSecurityContext := podtemplatespec.NOOP() @@ -237,7 +239,7 @@ func DefaultReadiness() probes.Modification { ) } -func dataPvc() persistentvolumeclaim.Modification { +func dataPvc(dataVolumeName string) persistentvolumeclaim.Modification { return persistentvolumeclaim.Apply( persistentvolumeclaim.WithName(dataVolumeName), persistentvolumeclaim.WithAccessModes(corev1.ReadWriteOnce), @@ -245,9 +247,9 @@ func dataPvc() persistentvolumeclaim.Modification { ) } -func logsPvc() persistentvolumeclaim.Modification { +func logsPvc(logsVolumeName string) persistentvolumeclaim.Modification { return persistentvolumeclaim.Apply( - persistentvolumeclaim.WithName(logVolumeName), + persistentvolumeclaim.WithName(logsVolumeName), persistentvolumeclaim.WithAccessModes(corev1.ReadWriteOnce), persistentvolumeclaim.WithResourceRequests(resourcerequirements.BuildStorageRequirements("2G")), ) diff --git a/pkg/automationconfig/automation_config_builder.go b/pkg/automationconfig/automation_config_builder.go index bdc8413bc..c373ce44b 100644 --- a/pkg/automationconfig/automation_config_builder.go +++ b/pkg/automationconfig/automation_config_builder.go @@ -3,6 +3,7 @@ package automationconfig import ( "fmt" "path" + "reflect" "github.com/blang/semver" "github.com/pkg/errors" @@ -111,6 +112,13 @@ func (b *Builder) SetCAFilePath(caFilePath string) *Builder { return b } +func (b *Builder) AddVersions(versions []MongoDbVersionConfig) *Builder { + for _, v := range versions { + b.AddVersion(v) + } + return b +} + func (b *Builder) AddVersion(version MongoDbVersionConfig) *Builder { for idx := range version.Builds { if version.Builds[idx].Modules == nil { @@ -238,8 +246,9 @@ func (b *Builder) Build() (AutomationConfig, error) { b.auth = &disabled } - if len(b.versions) == 0 { - b.versions = append(b.versions, buildDummyMongoDbVersionConfig(b.mongodbVersion)) + dummyConfig := buildDummyMongoDbVersionConfig(b.mongodbVersion) + if !versionsContain(b.versions, dummyConfig) { + b.versions = append(b.versions, dummyConfig) } currentAc := AutomationConfig{ @@ -297,6 +306,15 @@ func toProcessName(name string, index int) string { return fmt.Sprintf("%s-%d", name, index) } +func versionsContain(versions []MongoDbVersionConfig, version MongoDbVersionConfig) bool { + for _, v := range versions { + if reflect.DeepEqual(v, version) { + return true + } + } + return false +} + // buildDummyMongoDbVersionConfig create a MongoDbVersionConfig which // will be valid for any version of MongoDB. This is used as a default if no // versions are manually specified. diff --git a/pkg/automationconfig/automation_config_test.go b/pkg/automationconfig/automation_config_test.go index 00ca5457e..43a444409 100644 --- a/pkg/automationconfig/automation_config_test.go +++ b/pkg/automationconfig/automation_config_test.go @@ -95,7 +95,7 @@ func TestMongoDbVersions(t *testing.T) { assert.NoError(t, err) assert.Len(t, ac.Processes, 3) - assert.Len(t, ac.Versions, 1) + assert.Len(t, ac.Versions, 2) assert.Len(t, ac.Versions[0].Builds, 1) // TODO: be able to pass amount of builds @@ -123,7 +123,7 @@ func TestMongoDbVersions(t *testing.T) { assert.NoError(t, err) assert.Len(t, ac.Processes, 3) - assert.Len(t, ac.Versions, 2) + assert.Len(t, ac.Versions, 3) assert.Len(t, ac.Versions[0].Builds, 1) assert.Len(t, ac.Versions[1].Builds, 2) } @@ -218,7 +218,7 @@ func TestMongoDBVersionsConfig(t *testing.T) { assert.NoError(t, err) versions := ac.Versions - assert.Len(t, versions, 1) + assert.Len(t, versions, 2) v := versions[0] dummyConfig := buildDummyMongoDbVersionConfig("4.4.2") assert.NotEqual(t, v, dummyConfig) From 2c77ad7f25661a0a35253b9860d2491a636606bb Mon Sep 17 00:00:00 2001 From: Rajdeep Das Date: Mon, 10 May 2021 10:54:51 +0200 Subject: [PATCH 252/790] Remove "persistent" field from testdata samples (#476) --- controllers/testdata/change_data_volume.yaml | 1 - .../testdata/custom_storage_class.yaml | 1 - controllers/testdata/tolerations_example.yaml | 1 - .../testdata/volume_claim_templates_mdb.yaml | 1 - go.sum | 174 +----------------- 5 files changed, 6 insertions(+), 172 deletions(-) diff --git a/controllers/testdata/change_data_volume.yaml b/controllers/testdata/change_data_volume.yaml index 58cdabb9f..0ab77019c 100644 --- a/controllers/testdata/change_data_volume.yaml +++ b/controllers/testdata/change_data_volume.yaml @@ -6,7 +6,6 @@ spec: members: 3 type: ReplicaSet version: "4.2.6" - persistent: true security: authentication: modes: ["SCRAM"] diff --git a/controllers/testdata/custom_storage_class.yaml b/controllers/testdata/custom_storage_class.yaml index 4579c29ba..d3c5c59cc 100644 --- a/controllers/testdata/custom_storage_class.yaml +++ b/controllers/testdata/custom_storage_class.yaml @@ -6,7 +6,6 @@ spec: members: 3 type: ReplicaSet version: "4.2.6" - persistent: true statefulSet: spec: volumeClaimTemplates: diff --git a/controllers/testdata/tolerations_example.yaml b/controllers/testdata/tolerations_example.yaml index 9d02db09c..88e808ec3 100644 --- a/controllers/testdata/tolerations_example.yaml +++ b/controllers/testdata/tolerations_example.yaml @@ -6,7 +6,6 @@ spec: members: 3 type: ReplicaSet version: "4.2.6" - persistent: true statefulSet: spec: template: diff --git a/controllers/testdata/volume_claim_templates_mdb.yaml b/controllers/testdata/volume_claim_templates_mdb.yaml index 0403e6525..797cc4b8b 100644 --- a/controllers/testdata/volume_claim_templates_mdb.yaml +++ b/controllers/testdata/volume_claim_templates_mdb.yaml @@ -6,7 +6,6 @@ spec: members: 3 type: ReplicaSet version: "4.2.6" - persistent: true statefulSet: spec: volumeClaimTemplates: diff --git a/go.sum b/go.sum index ecae3201b..4757f164a 100644 --- a/go.sum +++ b/go.sum @@ -6,7 +6,6 @@ cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxK cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.51.0/go.mod h1:hWtGJ6gnXH+KgDv+V0zFGDvpi07n3z8ZNj3T1RW0Gcw= cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= @@ -25,48 +24,28 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-autorest v13.3.2+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= -github.com/Azure/go-autorest/autorest v0.9.6/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630= github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= -github.com/Azure/go-autorest/autorest v0.11.12/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= -github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= -github.com/Azure/go-autorest/autorest/adal v0.8.2/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= -github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= -github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g= github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= -github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= -github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= -github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= -github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= -github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.34.28 h1:sscPpn/Ns3i0F4HPEWAVcwdIRaZZCuL7llJ2/60yPIk= github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= @@ -76,8 +55,6 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= -github.com/blang/semver v3.5.0+incompatible h1:CGxCgetQ64DKk7rdZ++Vfnb1+ogGNnB17OJKJXD2Cfs= -github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -91,7 +68,6 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -102,14 +78,11 @@ github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfc github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -129,8 +102,6 @@ github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWo github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= -github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -145,49 +116,13 @@ github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/zapr v0.2.0 h1:v6Ji8yBW77pva6NkJKQdHLAJKrIJKRHz0RXwPqCHSR4= github.com/go-logr/zapr v0.2.0/go.mod h1:qhKdvif7YF5GI9NWEpyxTSSBdGmzkNguibrdCNVPunU= -github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= -github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= -github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= -github.com/go-openapi/analysis v0.19.2/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk= -github.com/go-openapi/analysis v0.19.5/go.mod h1:hkEAkxagaIvIP7VTn8ygJNkd4kAYON2rCu0v0ObL0AU= -github.com/go-openapi/errors v0.17.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= -github.com/go-openapi/errors v0.18.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= -github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= -github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= -github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= -github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= -github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= -github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= -github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= -github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= -github.com/go-openapi/loads v0.19.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= -github.com/go-openapi/loads v0.19.2/go.mod h1:QAskZPMX5V0C2gvfkGZzJlINuP7Hx/4+ix5jWFxsNPs= -github.com/go-openapi/loads v0.19.4/go.mod h1:zZVHonKd8DXyxyw4yfnVjPzBjIQcLt0CCsn0N0ZrQsk= -github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9/go.mod h1:6v9a6LTXWQCdL8k1AO3cvqx5OtZY/Y9wKTgaoP6YRfA= -github.com/go-openapi/runtime v0.19.0/go.mod h1:OwNfisksmmaZse4+gpV3Ne9AyMOlP1lt4sK4FXt0O64= -github.com/go-openapi/runtime v0.19.4/go.mod h1:X277bwSUBxVlCYR3r7xgZZGKVvBd/29gLDlFGtJ8NL4= -github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= -github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= -github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= -github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY= github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= -github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= -github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= -github.com/go-openapi/strfmt v0.19.0/go.mod h1:+uW+93UVvGGq2qGaZxdDeJqSAqBqBdl+ZPMF/cC8nDY= -github.com/go-openapi/strfmt v0.19.3/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU= -github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= -github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= -github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= -github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= -github.com/go-openapi/validate v0.19.5/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -219,8 +154,6 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -279,7 +212,6 @@ github.com/googleapis/gnostic v0.5.1 h1:A8Yhf6EtqTv9RMsU6MQTyrtV1TjWlR6xU9BsZIwu github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= @@ -337,7 +269,6 @@ github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaR github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.9.8 h1:VMAMUUOh+gaxKTMk+zqbjsSjsIcUcL/LF4o63i82QyA= @@ -347,17 +278,13 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxv github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= @@ -379,7 +306,6 @@ github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS4 github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= @@ -392,8 +318,6 @@ github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8m github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= @@ -411,7 +335,6 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y github.com/onsi/gomega v1.10.2 h1:aY/nuoWlKJud2J6U0E3NWsjlg+0GtwXxgEqthRdzlcs= github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= @@ -441,7 +364,6 @@ github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB8 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.2.0 h1:wH4vA7pcjKuZzjF7lM8awk4fnuJO6idemZXoKnULUx4= github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= @@ -453,7 +375,6 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -470,7 +391,6 @@ github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkU github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= @@ -478,7 +398,6 @@ github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -498,9 +417,7 @@ github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.0.2 h1:akYIkZ28e6A96dkWNJQu3nmCzH3YfwMPQExUYDaRv7w= @@ -510,19 +427,13 @@ github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6 github.com/xdg/stringprep v1.0.3 h1:cmL5Enob4W83ti/ZHuZLuKD/xqJfus4fVPwE+/BDm+4= github.com/xdg/stringprep v1.0.3/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -go.etcd.io/etcd v0.5.0-alpha.5.0.20200819165624-17cef6e3e9d5/go.mod h1:skWido08r9w6Lq/w70DO5XYIKMu4QFu1+4VsqLQuJy8= go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= -go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= -go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= -go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.5.1 h1:9nOVLGDfOaZ9R0tBumx/BcuqkbFpyTCU2r/Po7A2azI= go.mongodb.org/mongo-driver v1.5.1/go.mod h1:gRXCHX4Jo7J0IJ1oDQyUxF7jfy19UfxniMS4xxMmUqw= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= @@ -548,21 +459,15 @@ go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0 h1:hb9wdF1z5waM+dSIICn1l0DkLVDT3hqhhQsDNUmHPRE= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g= -golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -582,9 +487,9 @@ golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f h1:J5lckAjkw6qYlOZNj90mLYNTEKDvWeuc1yieZ8qUzUE= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= @@ -598,7 +503,6 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -606,11 +510,9 @@ golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -625,16 +527,11 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 h1:OgUuv8lsRpBibGNbSizVwKWlysjaNzmC9gYMhPVfqFM= -golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6 h1:pE8b58s1HRDMi8RDc79m0HISf9D4TzseP40cEA6IGfs= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -646,8 +543,6 @@ golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -657,7 +552,6 @@ golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -673,7 +567,6 @@ golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -687,17 +580,9 @@ golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201112073958-5cba982894dd h1:5CtCZbICpIOFdgO940moixOPjc0178IU44m4EjOO5IY= golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073 h1:8qxJSnu+7dRq6upnbntrmriWByIakBuct5OM/MdQC1M= -golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE= -golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -712,14 +597,10 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e h1:EHBhcS0mlXEAVwNyO2dLfjToGsyY4j24pTs2ScHnX7s= golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE= -golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -734,7 +615,6 @@ golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= @@ -761,10 +641,8 @@ golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200616133436-c1934b75d054 h1:HHeAlu5H9b71C+Fx0K+1dGgVFN1DM1/wz4aoGOA5qS8= golang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a h1:CB3a9Nez8M13wwlr/E2YtwoU+qYHKfC+JrDa45RXXoQ= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -811,7 +689,6 @@ google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEY google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= @@ -831,9 +708,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= @@ -853,8 +729,6 @@ gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -864,83 +738,47 @@ honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3 h1:sXmLre5bzIR6ypkjXCDI3jHPssRhc8KD/Ome589sc3U= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.19.2 h1:q+/krnHWKsL7OBZg/rxnycsl9569Pud76UJ77MvKXms= -k8s.io/api v0.19.2/go.mod h1:IQpK0zFQ1xc5iNIQPqzgoOwuFugaYHK4iCknlAQP9nI= k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo= -k8s.io/api v0.20.2 h1:y/HR22XDZY3pniu9hIFDLpUCPq2w5eQ6aV/VFQ7uJMw= k8s.io/api v0.20.2/go.mod h1:d7n6Ehyzx+S+cE3VhTGfVNNqtGc/oL9DCdYYahlurV8= k8s.io/api v0.20.4 h1:xZjKidCirayzX6tHONRQyTNDVIR55TYVqgATqo6ZULY= k8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ= -k8s.io/api v0.21.0 h1:gu5iGF4V6tfVCQ/R+8Hc0h7H1JuEhzyEi9S4R5LM8+Y= -k8s.io/api v0.21.0/go.mod h1:+YbrhBBGgsxbF6o6Kj4KJPJnBmAKuXDeS3E18bgHNVU= -k8s.io/apiextensions-apiserver v0.19.2 h1:oG84UwiDsVDu7dlsGQs5GySmQHCzMhknfhFExJMz9tA= -k8s.io/apiextensions-apiserver v0.19.2/go.mod h1:EYNjpqIAvNZe+svXVx9j4uBaVhTB4C94HkY3w058qcg= k8s.io/apiextensions-apiserver v0.20.1 h1:ZrXQeslal+6zKM/HjDXLzThlz/vPSxrfK3OqL8txgVQ= k8s.io/apiextensions-apiserver v0.20.1/go.mod h1:ntnrZV+6a3dB504qwC5PN/Yg9PBiDNt1EVqbW2kORVk= -k8s.io/apimachinery v0.19.2/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= -k8s.io/apimachinery v0.20.2 h1:hFx6Sbt1oG0n6DZ+g4bFt5f6BoMkOjKWsQFu077M3Vg= k8s.io/apimachinery v0.20.2/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apimachinery v0.20.4 h1:vhxQ0PPUUU2Ns1b9r4/UFp13UPs8cw2iOoTjnY9faa0= k8s.io/apimachinery v0.20.4/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= -k8s.io/apimachinery v0.21.0 h1:3Fx+41if+IRavNcKOz09FwEXDBG6ORh6iMsTSelhkMA= -k8s.io/apimachinery v0.21.0/go.mod h1:jbreFvJo3ov9rj7eWT7+sYiRx+qZuCYXwWT1bcDswPY= -k8s.io/apiserver v0.19.2/go.mod h1:FreAq0bJ2vtZFj9Ago/X0oNGC51GfubKK/ViOKfVAOA= k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU= -k8s.io/client-go v0.19.2 h1:gMJuU3xJZs86L1oQ99R4EViAADUPMHHtS9jFshasHSc= -k8s.io/client-go v0.19.2/go.mod h1:S5wPhCqyDNAlzM9CnEdgTGV4OqhsW3jGO1UM1epwfJA= k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y= -k8s.io/client-go v0.20.2 h1:uuf+iIAbfnCSw8IGAv/Rg0giM+2bOzHLOsbbrwrdhNQ= k8s.io/client-go v0.20.2/go.mod h1:kH5brqWqp7HDxUFKoEgiI4v8G1xzbe9giaCenUWJzgE= k8s.io/client-go v0.20.4 h1:85crgh1IotNkLpKYKZHVNI1JT86nr/iDCvq2iWKsql4= k8s.io/client-go v0.20.4/go.mod h1:LiMv25ND1gLUdBeYxBIwKpkSC5IsozMMmOOeSJboP+k= -k8s.io/client-go v0.21.0 h1:n0zzzJsAQmJngpC0IhgFcApZyoGXPrDIAD601HD09ag= -k8s.io/client-go v0.21.0/go.mod h1:nNBytTF9qPFDEhoqgEPaarobC8QPae13bElIVHzIglA= -k8s.io/client-go v1.5.2 h1:JOxmv4FxrCIOS54kAABbN8/hA9jqGpns+Zc6soNgd8U= -k8s.io/code-generator v0.19.2/go.mod h1:moqLn7w0t9cMs4+5CQyxnfA/HV8MF6aAVENF+WZZhgk= k8s.io/code-generator v0.20.1/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= -k8s.io/component-base v0.19.2 h1:jW5Y9RcZTb79liEhW3XDVTW7MuvEGP0tQZnfSX6/+gs= -k8s.io/component-base v0.19.2/go.mod h1:g5LrsiTiabMLZ40AR6Hl45f088DevyGY+cCE2agEIVo= k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk= k8s.io/component-base v0.20.2 h1:LMmu5I0pLtwjpp5009KLuMGFqSc2S2isGw8t1hpYKLE= k8s.io/component-base v0.20.2/go.mod h1:pzFtCiwe/ASD0iV7ySMu8SYVJjCapNM9bjvk7ptpKh0= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20200428234225-8167cfdcfc14/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.4.0 h1:7+X0fUguPyrKEC4WjH8iGDg3laWgMo5tMnRTIGTTxGQ= k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.8.0 h1:Q3gmuM9hKEjefWFFYF0Mat+YyFJvsUyYuwyNNJ5C9Ts= -k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= -k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd h1:sOHNzJIkytDF6qadMNKhhDRpc6ODik8lVC6nOur7B2c= k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= -k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 h1:vEx13qjvaZ4yfObSSXW7BrMc/KQBBT/Jyee8XtLf4x0= -k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE= -k8s.io/utils v0.0.0-20200729134348-d5654de09c73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20200912215256-4140de9c8800 h1:9ZNvfPvVIEsp/T1ez4GQuzCcCTEQWhovSofhqR73A6g= -k8s.io/utils v0.0.0-20200912215256-4140de9c8800/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210111153108-fddb29f9d009 h1:0T5IaWHO3sJTEmCP6mUlBvMukxPKUQWqiI/YuiBNMiQ= k8s.io/utils v0.0.0-20210111153108-fddb29f9d009/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.9/go.mod h1:dzAXnQbTRyDlZPJX2SUPEqvnB+j7AJjtlox7PEwigU0= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= -sigs.k8s.io/controller-runtime v0.7.0 h1:bU20IBBEPccWz5+zXpLnpVsgBYxqclaHu1pVDl/gEt8= -sigs.k8s.io/controller-runtime v0.7.0/go.mod h1:pJ3YBrJiAqMAZKi6UVGuE98ZrroV1p+pIhoHsMm9wdU= sigs.k8s.io/controller-runtime v0.8.3 h1:GMHvzjTmaWHQB8HadW+dIvBoJuLvZObYJ5YoZruPRao= sigs.k8s.io/controller-runtime v0.8.3/go.mod h1:U/l+DUopBc1ecfRZ5aviA9JDmGFQKvLf5YkZNx2e0sU= -sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.0.2 h1:YHQV7Dajm86OuqnIR6zAelnDWBRjo+YhYV9PmGrh1s8= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.1.0 h1:C4r9BgJ98vrKnnVCjwCSXcWjWe0NKcUQkmzDXZXGwH8= -sigs.k8s.io/structured-merge-diff/v4 v4.1.0/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= From d2f01724ccfffd71d9ffd62743462d89a26d3059 Mon Sep 17 00:00:00 2001 From: Rajdeep Das Date: Mon, 10 May 2021 14:08:18 +0200 Subject: [PATCH 253/790] Add notes regarding enterprise and community operator docs. (#477) --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 5473481a3..0911a9044 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,8 @@ See the [documentation](/docs) to learn how to: 1. [Create a database user](/docs/users.md) with SCRAM authentication. 1. [Secure MongoDB resource connections](/docs/secure.md) using TLS. +*NOTE: [MongoDB Enterprise Kubernetes Operator](https://docs.mongodb.com/kubernetes-operator/master/) docs are for the enterprise operator use case and NOT for the community operator. In addition to the docs mentioned above, you can refer to this [blog post](https://www.mongodb.com/blog/post/run-secure-containerized-mongodb-deployments-using-the-mongo-db-community-kubernetes-oper) as well to learn more about community operator deployment* + ## Supported Features The MongoDB Community Kubernetes Operator supports the following features: From 333cd4328ec4ef6a973685dd9eadac1af4288db8 Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Tue, 11 May 2021 13:38:35 +0100 Subject: [PATCH 254/790] add_code_owners_file (#466) --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 000000000..7935763f8 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @bznein @chatton @irajdeep @rodrigovalin @tibulca @priyolahiri From 4940fef8d4b9527e56d613d81fb08d9770c76956 Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Tue, 11 May 2021 17:08:08 +0100 Subject: [PATCH 255/790] CLOUDP-89359: Ensure Mongod logs are sent to stdout upon restart (#480) --- .../construct/build_statefulset_test.go | 21 +++++++++++++++++++ controllers/construct/mongodbstatefulset.go | 4 ++++ pkg/automationconfig/automation_config.go | 4 +++- .../automation_config_builder.go | 1 + 4 files changed, 29 insertions(+), 1 deletion(-) diff --git a/controllers/construct/build_statefulset_test.go b/controllers/construct/build_statefulset_test.go index 72f438bd3..f7ca3d7b4 100644 --- a/controllers/construct/build_statefulset_test.go +++ b/controllers/construct/build_statefulset_test.go @@ -5,6 +5,9 @@ import ( "reflect" "testing" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/container" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/resourcerequirements" + corev1 "k8s.io/api/core/v1" "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/probes" @@ -57,6 +60,24 @@ func TestMultipleCalls_DoNotCauseSideEffects(t *testing.T) { }) } +func TestMongod_Container(t *testing.T) { + c := container.New(mongodbContainer("4.2", []corev1.VolumeMount{})) + + t.Run("Has correct Env vars", func(t *testing.T) { + assert.Len(t, c.Env, 1) + assert.Equal(t, agentHealthStatusFilePathEnv, c.Env[0].Name) + assert.Equal(t, "/healthstatus/agent-health-status.json", c.Env[0].Value) + }) + + t.Run("Image is correct", func(t *testing.T) { + assert.Equal(t, getMongoDBImage("4.2"), c.Image) + }) + + t.Run("Resource requirements are correct", func(t *testing.T) { + assert.Equal(t, resourcerequirements.Defaults(), c.Resources) + }) +} + func assertStatefulSetIsBuiltCorrectly(t *testing.T, mdb mdbv1.MongoDBCommunity, sts *appsv1.StatefulSet) { assert.Len(t, sts.Spec.Template.Spec.Containers, 2) assert.Len(t, sts.Spec.Template.Spec.InitContainers, 2) diff --git a/controllers/construct/mongodbstatefulset.go b/controllers/construct/mongodbstatefulset.go index 15ebd00f8..08320612a 100644 --- a/controllers/construct/mongodbstatefulset.go +++ b/controllers/construct/mongodbstatefulset.go @@ -284,9 +284,13 @@ func mongodbContainer(version string, volumeMounts []corev1.VolumeMount) contain # wait for config and keyfile to be created by the agent while ! [ -f %s -a -f %s ]; do sleep 3 ; done ; sleep 2 ; +# with mongod configured to append logs, we need to provide them to stdout as +# mongod does not write to stdout and a log file +tail -F /var/log/mongodb-mms-automation/mongodb.log > /dev/stdout & # start mongod with this configuration exec mongod -f %s; + `, automationconfFilePath, keyfileFilePath, automationconfFilePath) containerCommand := []string{ diff --git a/pkg/automationconfig/automation_config.go b/pkg/automationconfig/automation_config.go index 304925aa5..311d96f51 100644 --- a/pkg/automationconfig/automation_config.go +++ b/pkg/automationconfig/automation_config.go @@ -67,7 +67,8 @@ func (p *Process) SetReplicaSetName(replSetName string) *Process { func (p *Process) SetSystemLog(systemLog SystemLog) *Process { return p.SetArgs26Field("systemLog.path", systemLog.Path). - SetArgs26Field("systemLog.destination", systemLog.Destination) + SetArgs26Field("systemLog.destination", systemLog.Destination). + SetArgs26Field("systemLog.logAppend", systemLog.LogAppend) } func (p *Process) SetWiredTigerCache(cacheSizeGb *float32) *Process { @@ -105,6 +106,7 @@ type ProcessType string type SystemLog struct { Destination string `json:"destination"` Path string `json:"path"` + LogAppend bool `json:"logAppend"` } type WiredTiger struct { diff --git a/pkg/automationconfig/automation_config_builder.go b/pkg/automationconfig/automation_config_builder.go index c373ce44b..f1dfb584e 100644 --- a/pkg/automationconfig/automation_config_builder.go +++ b/pkg/automationconfig/automation_config_builder.go @@ -216,6 +216,7 @@ func (b *Builder) Build() (AutomationConfig, error) { process.SetSystemLog(SystemLog{ Destination: "file", Path: path.Join(DefaultAgentLogPath, "/mongodb.log"), + LogAppend: true, }) if b.fcv != "" { From ad71d9494e388327f8575dd7f284cf13f08fa8d5 Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Tue, 11 May 2021 17:08:23 +0100 Subject: [PATCH 256/790] Ensure Agent Images pass RedHat certification (#479) --- .dockerignore | 2 ++ inventory.yaml | 4 ++++ scripts/dev/templates/agent/Dockerfile.builder | 3 +-- scripts/dev/templates/agent/Dockerfile.template | 11 +++++++++++ {agent => scripts/dev/templates/agent}/LICENSE | 0 {agent => scripts/dev/templates/agent}/README.md | 0 6 files changed, 18 insertions(+), 2 deletions(-) rename {agent => scripts/dev/templates/agent}/LICENSE (100%) rename {agent => scripts/dev/templates/agent}/README.md (100%) diff --git a/.dockerignore b/.dockerignore index 9ff6084c6..f052dfccf 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,3 +4,5 @@ zz_* vendor/ scripts/ .git/ +# allow agent LICENSE +!scripts/dev/templates/agent/LICENSE diff --git a/inventory.yaml b/inventory.yaml index 3fe06de7b..1b7e2373e 100644 --- a/inventory.yaml +++ b/inventory.yaml @@ -45,6 +45,7 @@ images: buildargs: imagebase: $(inputs.params.registry)/mongodb-agent-dev:$(inputs.params.version_id)-context + agent_version: $(inputs.params.agent_version) labels: quay.expires-after: 48h @@ -91,6 +92,7 @@ images: buildargs: imagebase: $(inputs.params.registry)/mongodb-agent:$(inputs.params.version_id)-context + agent_version: $(inputs.params.agent_version) labels: quay.expires-after: Never @@ -143,6 +145,7 @@ images: buildargs: imagebase: $(inputs.params.registry)/mongodb-agent-ubi-dev:$(inputs.params.version_id)-context + agent_version: $(inputs.params.agent_version) labels: quay.expires-after: 48h @@ -186,6 +189,7 @@ images: buildargs: imagebase: $(inputs.params.registry)/mongodb-agent-ubi:$(inputs.params.version_id)-context + agent_version: $(inputs.params.agent_version) labels: quay.expires-after: Never diff --git a/scripts/dev/templates/agent/Dockerfile.builder b/scripts/dev/templates/agent/Dockerfile.builder index d8d7f3ec3..6e1cd8a71 100644 --- a/scripts/dev/templates/agent/Dockerfile.builder +++ b/scripts/dev/templates/agent/Dockerfile.builder @@ -7,5 +7,4 @@ ARG tools_version ADD https://mciuploads.s3.amazonaws.com/mms-automation/mongodb-mms-build-agent/builds/automation-agent/prod/mongodb-mms-automation-agent-${agent_version}.${agent_distro}.tar.gz /data/mongodb-agent.tar.gz ADD https://downloads.mongodb.org/tools/db/mongodb-database-tools-${tools_distro}-${tools_version}.tgz /data/mongodb-tools.tgz - -ADD agent/LICENSE /data/licenses +ADD scripts/dev/templates/agent/LICENSE /data/LICENSE diff --git a/scripts/dev/templates/agent/Dockerfile.template b/scripts/dev/templates/agent/Dockerfile.template index a6c46e63e..f6549db05 100644 --- a/scripts/dev/templates/agent/Dockerfile.template +++ b/scripts/dev/templates/agent/Dockerfile.template @@ -3,6 +3,16 @@ FROM ${imagebase} as base FROM {{base_image}} +ARG agent_version + +LABEL name="MongoDB Agent" \ + version="${agent_version}" \ + summary="MongoDB Agent" \ + description="MongoDB Agent" \ + vendor="MongoDB" \ + release="1" \ + maintainer="support@mongodb.com" + {% block packages -%} {% endblock -%} @@ -17,6 +27,7 @@ RUN mkdir -p /agent \ COPY --from=base /data/mongodb-agent.tar.gz /agent COPY --from=base /data/mongodb-tools.tgz /agent +COPY --from=base /data/LICENSE /licenses/LICENSE RUN tar xfz /agent/mongodb-agent.tar.gz \ && mv mongodb-mms-automation-agent-*/mongodb-mms-automation-agent /agent/mongodb-agent \ diff --git a/agent/LICENSE b/scripts/dev/templates/agent/LICENSE similarity index 100% rename from agent/LICENSE rename to scripts/dev/templates/agent/LICENSE diff --git a/agent/README.md b/scripts/dev/templates/agent/README.md similarity index 100% rename from agent/README.md rename to scripts/dev/templates/agent/README.md From e02efcc75fa707cd5bb89e39cd8bade46ddee16b Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Wed, 12 May 2021 17:11:06 +0100 Subject: [PATCH 257/790] Update dev quick start (#465) --- Makefile | 5 ++ dev_notes/config-options.md | 12 ++++ dev_notes/dev-quick-start.md | 56 +++++++++++++++---- .../dev_quick_start_rs.yaml | 26 +++++++++ docs/contributing.md | 8 ++- 5 files changed, 94 insertions(+), 13 deletions(-) create mode 100644 dev_notes/config-options.md create mode 100644 dev_notes/dev_quick_start_resources/dev_quick_start_rs.yaml diff --git a/Makefile b/Makefile index 341a2a12d..c0c690009 100644 --- a/Makefile +++ b/Makefile @@ -76,6 +76,11 @@ deploy: manifests kustomize cd config/manager && $(KUSTOMIZE) edit set image quay.io/mongodb/mongodb-kubernetes-operator=$(IMG):latest $(KUSTOMIZE) build config/default | kubectl apply -f - +# Deploy a simple ReplicaSet, this is intended for first time use only as part of the quick start guide. +deploy-dev-quick-start-rs: manifests kustomize + kubectl create secret generic quick-start-rs --from-literal=password=dev-quick-start-password --from-literal=username=admin || true + kubectl apply -f dev_notes/dev_quick_start_resources/dev_quick_start_rs.yaml + # UnDeploy controller from the configured Kubernetes cluster in ~/.kube/config undeploy: $(KUSTOMIZE) build config/default | kubectl delete -f - diff --git a/dev_notes/config-options.md b/dev_notes/config-options.md new file mode 100644 index 000000000..6e28df8e3 --- /dev/null +++ b/dev_notes/config-options.md @@ -0,0 +1,12 @@ + +#### Config Options + +1. `namespace` is the namespace that will be used by scripts/tooling. All of the resources will be deployed here. +2. `operator_name` will be used as the name of the operator deployment, and the name of the operator image when build. +3. `image_type` this can be either `ubi` or `ubuntu` and determines the distro of the images built. (currently only the agent image has multiple distros) +4. `repo_url` the repository that should be used to push/pull all images. +5. `e2e_image` the name of e2e test image that will be built. +6. `version_upgrade_hook_image` the name of the version upgrade post start hook image. +7. `agent_image_ubuntu` the name of the ubuntu agent image. +8. `agent_image_ubi` the name of the ubi agent image. +9. `s3_bucket` the S3 bucket that Dockerfiles will be pushed to as part of the release process. Note: this is only required when running the release tasks locally. diff --git a/dev_notes/dev-quick-start.md b/dev_notes/dev-quick-start.md index 0cfcf7324..884204fa3 100644 --- a/dev_notes/dev-quick-start.md +++ b/dev_notes/dev-quick-start.md @@ -1,15 +1,24 @@ #### Prerequisites -* install [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) -* create a python virtual enironment +* Install [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) +* Install [jq](https://stedolan.github.io/jq/download/) +* Optionally install [kind](https://kind.sigs.k8s.io/docs/user/quick-start/#installation) if you want to use a local cluster. + +* create a python virtual environment ```bash python3 -m venv /path/to/new/virtual/environment source path/to/new/virtual/environment/bin/activate ``` -* install python dependencies ```pip install -r requirements.txt``` +* install python dependencies +``` +pip install -r requirements.txt + +# Note: sonar requires access to the 10gen repo and is used for the release pipeline +pip install git+ssh://git@github.com/10gen/sonar.git@0.0.10 +``` #### Create a Kind cluster and a local registry ```bash @@ -21,7 +30,7 @@ source path/to/new/virtual/environment/bin/activate export KUBECONFIG=~/.kube/kind ``` -#### get kind credentials +#### Get kind credentials ```bash kind export kubeconfig @@ -29,7 +38,8 @@ kind export kubeconfig kubectl cluster-info --context kind-kind --kubeconfig $KUBECONFIG ``` -#### create the namespace to work in + +#### (Optional) Create a non-default namespace to work in ```bash kubectl create namespace mongodb @@ -41,20 +51,20 @@ kubectl config set-context --current --namespace=mongodb ```bash cat > ~/.community-operator-dev/config.json << EOL { - "namespace": "mongodb", + "namespace": "default", "repo_url": "localhost:5000", "operator_image": "mongodb-kubernetes-operator", "e2e_image": "e2e", - "prestop_hook_image": "prehook", - "testrunner_image": "test-runner", "version_upgrade_hook_image": "community-operator-version-upgrade-post-start-hook" } EOL ``` +More details about the config options can be found [here](./config-options.md) + #### build and deploy the operator to the cluster ```bash -python scripts/dev/build_and_deploy_operator.py +make docker-build docker-push deploy ``` @@ -65,5 +75,31 @@ kubectl get pods #### Deploy a Replica Set ```bash -kubectl apply -f deploy/crds/mongodb.com_v1_mongodbcommunity_cr.yaml +make deploy-dev-quick-start-rs ``` + +#### See the deployed replica set +```bash +kubectl get pods + +NAME READY STATUS RESTARTS AGE +mongodb-kubernetes-operator-5568d769b8-smt4h 1/1 Running 0 4m12s +quick-start-rs-0 2/2 Running 0 2m49s +quick-start-rs-1 2/2 Running 0 2m5s +quick-start-rs-2 2/2 Running 0 87s + + +kubectl get statefulset quick-start-rs + +NAME READY AGE +quick-start-rs 3/3 3m10s +``` + +### Clean up all resources +```bash +make undeploy +``` + +### Running Tests + +Follow the tests in [contributing.md](../docs/contributing.md) to run e2e tests. \ No newline at end of file diff --git a/dev_notes/dev_quick_start_resources/dev_quick_start_rs.yaml b/dev_notes/dev_quick_start_resources/dev_quick_start_rs.yaml new file mode 100644 index 000000000..688385f5b --- /dev/null +++ b/dev_notes/dev_quick_start_resources/dev_quick_start_rs.yaml @@ -0,0 +1,26 @@ +--- +apiVersion: mongodbcommunity.mongodb.com/v1 +kind: MongoDBCommunity +metadata: + name: quick-start-rs +spec: + members: 3 + type: ReplicaSet + version: "4.2.6" + security: + authentication: + modes: ["SCRAM"] + users: + - name: my-user + db: admin + passwordSecretRef: # a reference to the secret that will be used to generate the user's password + name: my-user-password + roles: + - name: clusterAdmin + db: admin + - name: userAdminAnyDatabase + db: admin + scramCredentialsSecretName: my-scram + additionalMongodConfig: + storage.wiredTiger.engineConfig.journalCompressor: zlib + diff --git a/docs/contributing.md b/docs/contributing.md index 5a3117287..3ab1e173a 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -51,8 +51,7 @@ to be able to run properly. Create a json file with the following content: "repo_url": "localhost:5000", "operator_image": "mongodb-kubernetes-operator", "e2e_image": "e2e", - "version_upgrade_hook_image": "version_upgrade_hook", - "testrunner_image": "test-runner" + "version_upgrade_hook_image": "community-operator-version-upgrade-post-start-hook" } ``` @@ -92,7 +91,7 @@ python -m pip install -r requirements.txt Unit tests should be run from the root of the project with: ```sh -go test ./pkg/... +make test ``` # Running E2E Tests @@ -122,6 +121,9 @@ The tests should run individually using the runner like this: # python scripts/dev/e2e.py --test # for example python scripts/dev/e2e.py --test replica_set + +# or if you hasve not yet built the e2e test image +python scripts/dev/e2e.py --test replica_set --build-images ``` This will run the `replica_set` E2E test which is a simple test that installs a From f8f0716237c36f361ee4aebd1f92ae09a1213af7 Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Wed, 12 May 2021 17:58:25 +0100 Subject: [PATCH 258/790] Allow readiness probe to be configurable in test runs (#485) --- dev_notes/dev-quick-start.md | 3 +++ scripts/ci/config.json | 3 ++- scripts/dev/dev_config.py | 4 ++++ scripts/dev/e2e.py | 7 +++++-- test/e2e/setup/setup.go | 27 ++------------------------- test/e2e/setup/test_config.go | 2 ++ 6 files changed, 18 insertions(+), 28 deletions(-) diff --git a/dev_notes/dev-quick-start.md b/dev_notes/dev-quick-start.md index 884204fa3..6df366caa 100644 --- a/dev_notes/dev-quick-start.md +++ b/dev_notes/dev-quick-start.md @@ -55,6 +55,9 @@ cat > ~/.community-operator-dev/config.json << EOL "repo_url": "localhost:5000", "operator_image": "mongodb-kubernetes-operator", "e2e_image": "e2e", + "testrunner_image": "test-runner", + "version_upgrade_hook_image": "community-operator-version-upgrade-post-start-hook", + "readiness_probe_image": "mongodb-kubernetes-readinessprobe" "version_upgrade_hook_image": "community-operator-version-upgrade-post-start-hook" } EOL diff --git a/scripts/ci/config.json b/scripts/ci/config.json index f499b125e..92c0e18e4 100644 --- a/scripts/ci/config.json +++ b/scripts/ci/config.json @@ -3,9 +3,10 @@ "repo_url": "quay.io/mongodb", "operator_image": "community-operator-dev", "e2e_image": "community-operator-e2e", - "version_upgrade_hook_image": "community-operator-version-upgrade-post-start-hook", + "version_upgrade_hook_image": "mongodb-kubernetes-operator-version-upgrade-post-start-hook-dev", "testrunner_image": "community-operator-testrunner", "agent_image_ubuntu": "mongodb-agent-dev", "agent_image_ubi": "mongodb-agent-ubi-dev", + "readiness_probe_image": "mongodb-kubernetes-readinessprobe-dev", "s3_bucket": "s3://enterprise-operator-dockerfiles/dockerfiles" } diff --git a/scripts/dev/dev_config.py b/scripts/dev/dev_config.py index 1a5f0ddeb..6389337b1 100644 --- a/scripts/dev/dev_config.py +++ b/scripts/dev/dev_config.py @@ -70,6 +70,10 @@ def e2e_image(self) -> str: def version_upgrade_hook_image(self) -> str: return self._config["version_upgrade_hook_image"] + @property + def readiness_probe_image(self) -> str: + return self._config["readiness_probe_image"] + @property def agent_image(self) -> str: if self._distro == Distro.UBI: diff --git a/scripts/dev/e2e.py b/scripts/dev/e2e.py index 11cc8779a..10aaa2b18 100644 --- a/scripts/dev/e2e.py +++ b/scripts/dev/e2e.py @@ -156,8 +156,11 @@ def create_test_pod(args: argparse.Namespace, dev_config: DevConfig) -> None: }, { "name": "VERSION_UPGRADE_HOOK_IMAGE", - # TODO: this needs to come from somewhere else - "value": "quay.io/mongodb/mongodb-kubernetes-operator-version-upgrade-post-start-hook:1.0.2", + "value": f"{dev_config.repo_url}/{dev_config.version_upgrade_hook_image}:{args.tag}", + }, + { + "name": "READINESS_PROBE_IMAGE", + "value": f"{dev_config.repo_url}/{dev_config.readiness_probe_image}:{args.tag}", }, { "name": "PERFORM_CLEANUP", diff --git a/test/e2e/setup/setup.go b/test/e2e/setup/setup.go index 18e0ded43..d7de61d25 100644 --- a/test/e2e/setup/setup.go +++ b/test/e2e/setup/setup.go @@ -144,9 +144,10 @@ func deployOperator() error { &appsv1.Deployment{}, withNamespace(testConfig.namespace), withOperatorImage(testConfig.operatorImage), - withVersionUpgradeHookImage(testConfig.versionUpgradeHookImage), withEnvVar("WATCH_NAMESPACE", watchNamespace), withEnvVar(construct.AgentImageEnv, testConfig.agentImage), + withEnvVar(construct.ReadinessProbeImageEnv, testConfig.readinessProbeImage), + withEnvVar(construct.VersionUpgradeHookImageEnv, testConfig.versionUpgradeHookImage), ); err != nil { return errors.Errorf("error building operator deployment: %s", err) } @@ -234,30 +235,6 @@ func updateEnvVarList(envVarList []corev1.EnvVar, key, val string) []corev1.EnvV return append(envVarList, corev1.EnvVar{Name: key, Value: val}) } -// withVersionUpgradeHookImage sets the value of the VERSION_UPGRADE_HOOK_IMAGE -// EnvVar from first container to `image`. The EnvVar is updated -// if it exists. Or appended if there is no EnvVar with this `Name`. -func withVersionUpgradeHookImage(image string) func(runtime.Object) { - return func(obj runtime.Object) { - if dep, ok := obj.(*appsv1.Deployment); ok { - versionUpgradeHookEnv := corev1.EnvVar{ - Name: construct.VersionUpgradeHookImageEnv, - Value: image, - } - found := false - for idx := range dep.Spec.Template.Spec.Containers[0].Env { - if dep.Spec.Template.Spec.Containers[0].Env[idx].Name == versionUpgradeHookEnv.Name { - dep.Spec.Template.Spec.Containers[0].Env[idx].Value = versionUpgradeHookEnv.Value - found = true - } - } - if !found { - dep.Spec.Template.Spec.Containers[0].Env = append(dep.Spec.Template.Spec.Containers[0].Env, versionUpgradeHookEnv) - } - } - } -} - // withOperatorImage assumes that the underlying type is an appsv1.Deployment // which has the operator container as the first container. There will be // no effect when used with a non-deployment type diff --git a/test/e2e/setup/test_config.go b/test/e2e/setup/test_config.go index e4fdfb500..0dae114e7 100644 --- a/test/e2e/setup/test_config.go +++ b/test/e2e/setup/test_config.go @@ -19,6 +19,7 @@ type testConfig struct { clusterWide bool performCleanup bool agentImage string + readinessProbeImage string } func loadTestConfigFromEnv() testConfig { @@ -29,5 +30,6 @@ func loadTestConfigFromEnv() testConfig { agentImage: envvar.GetEnvOrDefault(construct.AgentImageEnv, "quay.io/mongodb/mongodb-agent:10.29.0.6830-1"), // TODO: better way to decide default agent image. clusterWide: envvar.ReadBool(clusterWideEnvName), performCleanup: envvar.ReadBool(performCleanupEnvName), + readinessProbeImage: envvar.GetEnvOrDefault(construct.ReadinessProbeImageEnv, "quay.io/mongodb/mongodb-kubernetes-readinessprobe:1.0.3"), } } From 05b67abc9a175ed405cad05445839fbfbb55f029 Mon Sep 17 00:00:00 2001 From: Rajdeep Das Date: Wed, 12 May 2021 19:16:26 +0200 Subject: [PATCH 259/790] Use namespace specified in config.json for operator deployment (#486) --- Makefile | 3 ++- config/default/kustomization.yaml | 3 --- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index c0c690009..223a16236 100644 --- a/Makefile +++ b/Makefile @@ -33,6 +33,7 @@ BUNDLE_IMG ?= controller-bundle:$(VERSION) # Image URL to use all building/pushing image targets REPO_URL := $(shell jq -r .repo_url < ~/.community-operator-dev/config.json) OPERATOR_IMAGE := $(shell jq -r .operator_image < ~/.community-operator-dev/config.json) +NAMESPACE := $(shell jq -r .namespace < ~/.community-operator-dev/config.json) IMG := $(REPO_URL)/$(OPERATOR_IMAGE) DOCKERFILE ?= operator # Produce CRDs that work back to Kubernetes 1.11 (no version conversion) @@ -74,7 +75,7 @@ uninstall: manifests kustomize # Deploy controller in the configured Kubernetes cluster in ~/.kube/config deploy: manifests kustomize cd config/manager && $(KUSTOMIZE) edit set image quay.io/mongodb/mongodb-kubernetes-operator=$(IMG):latest - $(KUSTOMIZE) build config/default | kubectl apply -f - + $(KUSTOMIZE) build config/default | kubectl apply -n $(NAMESPACE) -f - # Deploy a simple ReplicaSet, this is intended for first time use only as part of the quick start guide. deploy-dev-quick-start-rs: manifests kustomize diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index f7385fb48..8410b870a 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -1,6 +1,3 @@ -# Adds namespace to all resources. -namespace: default - # Value of this field is prepended to the # names of all resources, e.g. a deployment named # "wordpress" becomes "alices-wordpress". From 19a911b69030657e479bc6a55bbe19b07e2a0015 Mon Sep 17 00:00:00 2001 From: Caio Ferreira Date: Fri, 14 May 2021 04:56:27 -0300 Subject: [PATCH 260/790] Fix pod annotations overwrite (#484) --- cmd/readiness/testdata/k8sobjects.go | 7 +++++ pkg/readiness/pod/podannotation.go | 39 ++++++++++++++++--------- pkg/readiness/pod/podannotation_test.go | 9 +++++- 3 files changed, 40 insertions(+), 15 deletions(-) diff --git a/cmd/readiness/testdata/k8sobjects.go b/cmd/readiness/testdata/k8sobjects.go index 145fa0f63..e4e6d4d64 100644 --- a/cmd/readiness/testdata/k8sobjects.go +++ b/cmd/readiness/testdata/k8sobjects.go @@ -7,6 +7,10 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +// Currently seems like the appending functionality on the library used by the fake +// implementation to simulate JSONPatch is broken: https://github.com/evanphx/json-patch/issues/138 +// The short term workaround is to have the annotation empty. + // These are just k8s objects used for testing. Note, that these are defined in a non "_test.go" file as they are reused // by other modules func TestSecret(namespace, name string, version int) *corev1.Secret { @@ -21,6 +25,9 @@ func TestPod(namespace, name string) *corev1.Pod { ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: namespace, + Annotations: map[string]string{ + "agent.mongodb.com/version": "", + }, }, } } diff --git a/pkg/readiness/pod/podannotation.go b/pkg/readiness/pod/podannotation.go index 3fa4f22c6..3f4da8e90 100644 --- a/pkg/readiness/pod/podannotation.go +++ b/pkg/readiness/pod/podannotation.go @@ -1,31 +1,42 @@ package pod import ( + "context" + "go.uber.org/zap" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "strconv" + "strings" - "go.uber.org/zap" "k8s.io/client-go/kubernetes" ) -type mongodbAgentVersion struct { - Version string `json:"agent.mongodb.com/version"` -} +const mongodbAgentVersionAnnotation = "agent.mongodb.com/version" func PatchPodAnnotation(podNamespace string, lastVersionAchieved int64, memberName string, clientSet kubernetes.Interface) error { - patcher := NewKubernetesPodPatcher(clientSet) - mdbAgentVersion := mongodbAgentVersion{Version: strconv.FormatInt(lastVersionAchieved, 10)} - return patchPod(patcher, podNamespace, mdbAgentVersion, memberName) -} + pod, err := clientSet.CoreV1().Pods(podNamespace).Get(context.Background(), memberName, metav1.GetOptions{}) + if err != nil { + return err + } + + var payload []patchValue -func patchPod(patcher Patcher, podNamespace string, mdbAgentVersion mongodbAgentVersion, memberName string) error { - payload := []patchValue{{ + if len(pod.Annotations) == 0 { + payload = append(payload, patchValue{ + Op: "add", + Path: "/metadata/annotations", + Value: make(map[string]string), + }) + } + mdbAgentVersion := strconv.FormatInt(lastVersionAchieved, 10) + payload = append(payload, patchValue{ Op: "add", - Path: "/metadata/annotations", + Path: "/metadata/annotations/" + strings.Replace(mongodbAgentVersionAnnotation, "/", "~1", -1), Value: mdbAgentVersion, - }} + }) - pod, err := patcher.patchPod(podNamespace, memberName, payload) - if pod != nil { + patcher := NewKubernetesPodPatcher(clientSet) + updatedPod, err := patcher.patchPod(podNamespace, memberName, payload) + if updatedPod != nil { zap.S().Debugf("Updated Pod annotation: %v (%s)", pod.Annotations, memberName) } return err diff --git a/pkg/readiness/pod/podannotation_test.go b/pkg/readiness/pod/podannotation_test.go index ef311254c..d9ab9b0fd 100644 --- a/pkg/readiness/pod/podannotation_test.go +++ b/pkg/readiness/pod/podannotation_test.go @@ -11,17 +11,24 @@ import ( "k8s.io/client-go/kubernetes/fake" ) +// Currently seems like the appending functionality on the library used by the fake +// implementation to simulate JSONPatch is broken: https://github.com/evanphx/json-patch/issues/138 +// The short term workaround is to have the annotation empty. + // TestPatchPodAnnotation verifies that patching of the pod works correctly func TestPatchPodAnnotation(t *testing.T) { clientset := fake.NewSimpleClientset(&v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "my-replica-set-0", Namespace: "test-ns", + Annotations: map[string]string{ + mongodbAgentVersionAnnotation: "", + }, }, }) pod, _ := clientset.CoreV1().Pods("test-ns").Get(context.TODO(), "my-replica-set-0", metav1.GetOptions{}) - assert.Empty(t, pod.Annotations) + assert.Empty(t, pod.Annotations[mongodbAgentVersionAnnotation]) // adding the annotations assert.NoError(t, PatchPodAnnotation("test-ns", 1, "my-replica-set-0", clientset)) From 50d52a28f9c41b5fb95060bae82382818232ba3a Mon Sep 17 00:00:00 2001 From: Rajdeep Das Date: Fri, 14 May 2021 10:08:07 +0200 Subject: [PATCH 261/790] Logging improvement in manager main method (#490) i. Use dedicated logging instead of os.exit ii. Use pre-existing logger --- cmd/manager/main.go | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/cmd/manager/main.go b/cmd/manager/main.go index dba576fc0..4a1eb6d93 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -11,15 +11,13 @@ import ( "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" - ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client/config" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/manager/signals" ) var ( - scheme = runtime.NewScheme() - setupLog = ctrl.Log.WithName("setup") + scheme = runtime.NewScheme() ) const ( @@ -54,7 +52,7 @@ func hasRequiredVariables(logger *zap.Logger, envVariables ...string) bool { func main() { log, err := configureLogger() if err != nil { - os.Exit(1) + log.Sugar().Fatalf("Failed to configure logger: %v", err) } if !hasRequiredVariables(log, construct.AgentImageEnv, construct.VersionUpgradeHookImageEnv, construct.ReadinessProbeImageEnv) { @@ -64,7 +62,7 @@ func main() { // Get watch namespace from environment variable. namespace, nsSpecified := os.LookupEnv(WatchNamespaceEnv) if !nsSpecified { - os.Exit(1) + log.Sugar().Fatal("No namespace specified to watch") } // If namespace is a wildcard use the empty string to represent all namespaces @@ -79,8 +77,7 @@ func main() { // Get a config to talk to the apiserver cfg, err := config.GetConfig() if err != nil { - setupLog.Error(err, "Unable to get config") - os.Exit(1) + log.Sugar().Fatalf("Unable to get config: %v", err) } // Create a new Cmd to provide shared dependencies and start components @@ -88,22 +85,19 @@ func main() { Namespace: watchNamespace, }) if err != nil { - setupLog.Error(err, "Unable to create manager") - os.Exit(1) + log.Sugar().Fatalf("Unable to create manager: %v", err) } log.Info("Registering Components.") // Setup Scheme for all resources if err := mdbv1.AddToScheme(mgr.GetScheme()); err != nil { - setupLog.Error(err, "Unable to add mdbv1 to scheme") - os.Exit(1) + log.Sugar().Fatalf("Unable to add mdbv1 to scheme: %v", err) } // Setup Controller. if err = controllers.NewReconciler(mgr).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "Unable to create controller") - os.Exit(1) + log.Sugar().Fatalf("Unable to create controller: %v", err) } // +kubebuilder:scaffold:builder @@ -111,7 +105,6 @@ func main() { // Start the Cmd if err := mgr.Start(signals.SetupSignalHandler()); err != nil { - setupLog.Error(err, "Unable to start manager") - os.Exit(1) + log.Sugar().Fatalf("Unable to start manager: %v", err) } } From 8334b1408d1152c3c0b3d6708ba0c1bd4f0dd4bf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 14 May 2021 11:53:57 +0100 Subject: [PATCH 262/790] Bump k8s.io/apimachinery from 0.20.4 to 0.21.0 (#463) Bumps [k8s.io/apimachinery](https://github.com/kubernetes/apimachinery) from 0.20.4 to 0.21.0. - [Release notes](https://github.com/kubernetes/apimachinery/releases) - [Commits](https://github.com/kubernetes/apimachinery/compare/v0.20.4...v0.21.0) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 47 ++++++++++++++++++++++++++++++++++------------- 2 files changed, 35 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index 63dbea953..782bcbaa0 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( go.mongodb.org/mongo-driver v1.5.1 go.uber.org/zap v1.16.0 k8s.io/api v0.20.4 - k8s.io/apimachinery v0.20.4 + k8s.io/apimachinery v0.21.0 k8s.io/client-go v0.20.4 sigs.k8s.io/controller-runtime v0.8.3 sigs.k8s.io/yaml v1.2.0 diff --git a/go.sum b/go.sum index 4757f164a..300e188c1 100644 --- a/go.sum +++ b/go.sum @@ -78,6 +78,7 @@ github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfc github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -152,8 +153,9 @@ github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/V github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -269,6 +271,7 @@ github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaR github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.9.8 h1:VMAMUUOh+gaxKTMk+zqbjsSjsIcUcL/LF4o63i82QyA= @@ -278,12 +281,12 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxv github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -306,6 +309,7 @@ github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS4 github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= @@ -318,6 +322,8 @@ github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8m github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= @@ -430,6 +436,7 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= @@ -527,8 +534,10 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 h1:OgUuv8lsRpBibGNbSizVwKWlysjaNzmC9gYMhPVfqFM= +golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -541,8 +550,9 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -581,8 +591,11 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201112073958-5cba982894dd h1:5CtCZbICpIOFdgO940moixOPjc0178IU44m4EjOO5IY= golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073 h1:8qxJSnu+7dRq6upnbntrmriWByIakBuct5OM/MdQC1M= +golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -641,8 +654,10 @@ golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200616133436-c1934b75d054 h1:HHeAlu5H9b71C+Fx0K+1dGgVFN1DM1/wz4aoGOA5qS8= golang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a h1:CB3a9Nez8M13wwlr/E2YtwoU+qYHKfC+JrDa45RXXoQ= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -708,8 +723,9 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= @@ -727,8 +743,9 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -749,8 +766,9 @@ k8s.io/apiextensions-apiserver v0.20.1 h1:ZrXQeslal+6zKM/HjDXLzThlz/vPSxrfK3OqL8 k8s.io/apiextensions-apiserver v0.20.1/go.mod h1:ntnrZV+6a3dB504qwC5PN/Yg9PBiDNt1EVqbW2kORVk= k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apimachinery v0.20.2/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= -k8s.io/apimachinery v0.20.4 h1:vhxQ0PPUUU2Ns1b9r4/UFp13UPs8cw2iOoTjnY9faa0= k8s.io/apimachinery v0.20.4/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= +k8s.io/apimachinery v0.21.0 h1:3Fx+41if+IRavNcKOz09FwEXDBG6ORh6iMsTSelhkMA= +k8s.io/apimachinery v0.21.0/go.mod h1:jbreFvJo3ov9rj7eWT7+sYiRx+qZuCYXwWT1bcDswPY= k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU= k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y= k8s.io/client-go v0.20.2/go.mod h1:kH5brqWqp7HDxUFKoEgiI4v8G1xzbe9giaCenUWJzgE= @@ -764,10 +782,12 @@ k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8 k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.4.0 h1:7+X0fUguPyrKEC4WjH8iGDg3laWgMo5tMnRTIGTTxGQ= k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd h1:sOHNzJIkytDF6qadMNKhhDRpc6ODik8lVC6nOur7B2c= +k8s.io/klog/v2 v2.8.0 h1:Q3gmuM9hKEjefWFFYF0Mat+YyFJvsUyYuwyNNJ5C9Ts= +k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= +k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 h1:vEx13qjvaZ4yfObSSXW7BrMc/KQBBT/Jyee8XtLf4x0= +k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210111153108-fddb29f9d009 h1:0T5IaWHO3sJTEmCP6mUlBvMukxPKUQWqiI/YuiBNMiQ= k8s.io/utils v0.0.0-20210111153108-fddb29f9d009/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= @@ -777,8 +797,9 @@ rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/controller-runtime v0.8.3 h1:GMHvzjTmaWHQB8HadW+dIvBoJuLvZObYJ5YoZruPRao= sigs.k8s.io/controller-runtime v0.8.3/go.mod h1:U/l+DUopBc1ecfRZ5aviA9JDmGFQKvLf5YkZNx2e0sU= -sigs.k8s.io/structured-merge-diff/v4 v4.0.2 h1:YHQV7Dajm86OuqnIR6zAelnDWBRjo+YhYV9PmGrh1s8= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.1.0 h1:C4r9BgJ98vrKnnVCjwCSXcWjWe0NKcUQkmzDXZXGwH8= +sigs.k8s.io/structured-merge-diff/v4 v4.1.0/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= From 36b75840930ce663c10b8d472852536bbc461306 Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Wed, 19 May 2021 10:08:42 +0100 Subject: [PATCH 263/790] Enterprise only (#491) --- pkg/automationconfig/automation_config_builder.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/pkg/automationconfig/automation_config_builder.go b/pkg/automationconfig/automation_config_builder.go index f1dfb584e..d50ea5d89 100644 --- a/pkg/automationconfig/automation_config_builder.go +++ b/pkg/automationconfig/automation_config_builder.go @@ -4,6 +4,7 @@ import ( "fmt" "path" "reflect" + "strings" "github.com/blang/semver" "github.com/pkg/errors" @@ -320,7 +321,7 @@ func versionsContain(versions []MongoDbVersionConfig, version MongoDbVersionConf // will be valid for any version of MongoDB. This is used as a default if no // versions are manually specified. func buildDummyMongoDbVersionConfig(version string) MongoDbVersionConfig { - return MongoDbVersionConfig{ + versionConfig := MongoDbVersionConfig{ Name: version, Builds: []BuildConfig{ { @@ -337,4 +338,12 @@ func buildDummyMongoDbVersionConfig(version string) MongoDbVersionConfig { }, }, } + + // if we are using an enterprise version of MongoDB, we need to add the enterprise string to the modules array. + if strings.HasSuffix(version, "-ent") { + for i := range versionConfig.Builds { + versionConfig.Builds[i].Modules = append(versionConfig.Builds[i].Modules, "enterprise") + } + } + return versionConfig } From 05b1f5659f8c6489e944de5e5ce0cad5770f1c15 Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Wed, 19 May 2021 13:44:07 +0100 Subject: [PATCH 264/790] Add operator to pipeline (#451) --- .evergreen.yml | 27 +---- Makefile | 36 ++++-- dev_notes/dev-quick-start.md | 10 +- docs/build_operator_locally.md | 21 +--- docs/contributing.md | 27 ++--- inventories/e2e-inventory.yaml | 35 ++++++ inventories/operator-inventory.yaml | 35 ++++++ inventory.yaml | 8 ++ pipeline.py | 32 +++++ pkg/readiness/pod/podannotation.go | 9 +- scripts/dev/build_and_deploy_operator.py | 137 ---------------------- scripts/dev/dockerfile_generator.py | 20 +--- scripts/dev/dockerutil.py | 2 +- scripts/dev/e2e.py | 61 +--------- scripts/dev/templates/Dockerfile.template | 4 +- 15 files changed, 177 insertions(+), 287 deletions(-) create mode 100644 inventories/e2e-inventory.yaml create mode 100644 inventories/operator-inventory.yaml delete mode 100644 scripts/dev/build_and_deploy_operator.py diff --git a/.evergreen.yml b/.evergreen.yml index 1ab481794..21acc2ed5 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -107,20 +107,6 @@ functions: working_dir: mongodb-kubernetes-operator command: ${workdir}/venv/bin/python -m pip install -r requirements.txt --quiet --no-warn-script-location - build_and_push_image: - - command: subprocess.exec - type: setup - params: - include_expansions_in_env: - - version_id - - quay_user_name - - quay_password - - image - - image_type - - expire_after - working_dir: mongodb-kubernetes-operator - binary: scripts/ci/build_and_push_image.sh - build_and_push_image_sonar: - command: subprocess.exec type: setup @@ -202,11 +188,9 @@ tasks: commands: - func: clone - func: setup_virtualenv - - func: build_and_push_image + - func: build_and_push_image_sonar vars: - image_type: operator - image: quay.io/mongodb/community-operator-dev:${version_id} - expire_after: 48h + image_name: operator-ubi - name: build_e2e_image priority: 60 @@ -214,11 +198,10 @@ tasks: commands: - func: clone - func: setup_virtualenv - - func: build_and_push_image + - func: build_and_push_image_sonar vars: - image: quay.io/mongodb/community-operator-e2e:${version_id} - image_type: e2e - expire_after: 48h + image_name: e2e + - name: build_agent_image_ubuntu priority: 60 diff --git a/Makefile b/Makefile index 223a16236..8fc406d42 100644 --- a/Makefile +++ b/Makefile @@ -98,17 +98,36 @@ fmt: vet: go vet ./... +e2e: install + python scripts/dev/e2e.py --perform-cleanup --test $(test) + # Generate code generate: controller-gen $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." -# Build the docker image -docker-build: dockerfile - docker build -t ${IMG} . +# Build and push the operator image +operator-image: + python pipeline.py --image-name operator-ubi + +# Build and push e2e test image +e2e-image: + python pipeline.py --image-name e2e + +# Build and push agent image +agent-image: + python pipeline.py --image-name agent-ubuntu + +# Build and push readiness probe image +readiness-probe-image: + python pipeline.py --image-name readiness-probe-init + +# Build and push version upgrade post start hook image +version-upgrade-post-start-hook-image: + python pipeline.py --image-name version-post-start-hook-init + +# create all required images +all-images: operator-image e2e-image agent-image readiness-probe-image version-upgrade-post-start-hook-image -# Push the docker image -docker-push: - docker push ${IMG} # Download controller-gen locally if necessary CONTROLLER_GEN = $(shell pwd)/bin/controller-gen @@ -146,8 +165,3 @@ bundle: manifests kustomize .PHONY: bundle-build bundle-build: docker build -f bundle.Dockerfile -t $(BUNDLE_IMG) . - -# Generate Dockerfile -.PHONY: dockerfile -dockerfile: - python scripts/dev/dockerfile_generator.py ${DOCKERFILE} > Dockerfile diff --git a/dev_notes/dev-quick-start.md b/dev_notes/dev-quick-start.md index 6df366caa..5659a423f 100644 --- a/dev_notes/dev-quick-start.md +++ b/dev_notes/dev-quick-start.md @@ -55,22 +55,18 @@ cat > ~/.community-operator-dev/config.json << EOL "repo_url": "localhost:5000", "operator_image": "mongodb-kubernetes-operator", "e2e_image": "e2e", - "testrunner_image": "test-runner", + "prestop_hook_image": "prehook", "version_upgrade_hook_image": "community-operator-version-upgrade-post-start-hook", "readiness_probe_image": "mongodb-kubernetes-readinessprobe" - "version_upgrade_hook_image": "community-operator-version-upgrade-post-start-hook" } EOL ``` -More details about the config options can be found [here](./config-options.md) - -#### build and deploy the operator to the cluster +#### build all required images and deploy operator to cluster ```bash -make docker-build docker-push deploy +make all-images deploy ``` - #### See the operator deployment ```bash kubectl get pods diff --git a/docs/build_operator_locally.md b/docs/build_operator_locally.md index 728bbcd74..4fe827bec 100644 --- a/docs/build_operator_locally.md +++ b/docs/build_operator_locally.md @@ -35,7 +35,8 @@ kubectl cluster-info --context kind-kind --kubeconfig $KUBECONFIG 4. Build and deploy the operator: ```sh -python ./scripts/dev/build_and_deploy_operator.py +# builds all required images and then deploys the operator +make all-images deploy ``` @@ -44,21 +45,3 @@ Note: this will build and push the operator at `repo_url/mongodb-kubernetes-oper 5. Change the [manager yaml file](../config/manager/manager.yaml) `image` field to have the image you just built 6. You can now deploy your resources following the [docs](../docs/README.md) - - -## Troubleshooting -If you run into an issue in step 1, you can try the following steps as workaround: -1. Manually build the operator Dockerfile -```sh -python ./scripts/dev/dockerfile_generator.py > Dockerfile -``` - -2. Build the image -```sh -docker build . -t localhost:5000/mongodb-kubernetes-operator -``` - -3. Push the image -```sh -docker push localhost:5000/mongodb-kubernetes-operator -``` diff --git a/docs/contributing.md b/docs/contributing.md index 3ab1e173a..d9b8efc95 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -50,8 +50,18 @@ to be able to run properly. Create a json file with the following content: "namespace": "default", "repo_url": "localhost:5000", "operator_image": "mongodb-kubernetes-operator", +<<<<<<< HEAD + "e2e_image": "community-e2e", + "version_upgrade_hook_image": "version_upgrade_hook", + "prestop_hook_image": "prehook", + "testrunner_image": "test-runner", + "agent_image_ubuntu": "mongodb-agent-dev", + "agent_image_ubi": "mongodb-agent-ubi-dev", + "readiness_probe_image": "mongodb-kubernetes-readiness" +======= "e2e_image": "e2e", "version_upgrade_hook_image": "community-operator-version-upgrade-post-start-hook" +>>>>>>> master } ``` @@ -118,29 +128,18 @@ replica_set_scale The tests should run individually using the runner like this: ```sh -# python scripts/dev/e2e.py --test -# for example -python scripts/dev/e2e.py --test replica_set - -# or if you hasve not yet built the e2e test image -python scripts/dev/e2e.py --test replica_set --build-images +make e2e test= ``` This will run the `replica_set` E2E test which is a simple test that installs a MongoDB Replica Set and asserts that the deployed server can be connected to. -The python script has several flags to control its behaviour, please run - -```sh -python scripts/dev/e2e.py --help -``` - to get a list. ## Troubleshooting When you run a test locally, if the `e2e-test` pod is present, you will have to -first manually delete it; failing to do so will cause the `test-runner` pod to fail. +first manually delete it; failing to do so will cause the `e2e-test` pod to fail. # Writing new E2E tests @@ -151,7 +150,7 @@ Adding a new test is as easy as to create a new directory in `test/e2e` with the new E2E test, and to run them: ```sh -python scripts/dev/e2e.py --test +make e2e test= ``` # Before Committing your code diff --git a/inventories/e2e-inventory.yaml b/inventories/e2e-inventory.yaml new file mode 100644 index 000000000..f444c2117 --- /dev/null +++ b/inventories/e2e-inventory.yaml @@ -0,0 +1,35 @@ +vars: + registry: + +images: + - name: e2e + vars: + context: . + template_context: scripts/dev/templates + + stages: + - name: e2e-template + task_type: dockerfile_template + distro: e2e + + inputs: + - builder + - base_image + + output: + - dockerfile: scripts/dev/templates/Dockerfile.ubi-$(inputs.params.version_id) + + - name: e2e-build + task_type: docker_build + + dockerfile: scripts/dev/templates/Dockerfile.ubi-$(inputs.params.version_id) + + labels: + quay.expires-after: 48h + + output: + - registry: $(inputs.params.registry)/community-operator-e2e + tag: $(inputs.params.version_id) + - registry: $(inputs.params.registry)/community-operator-e2e + tag: latest + diff --git a/inventories/operator-inventory.yaml b/inventories/operator-inventory.yaml new file mode 100644 index 000000000..680c593c5 --- /dev/null +++ b/inventories/operator-inventory.yaml @@ -0,0 +1,35 @@ +vars: + registry: + +images: + - name: operator-ubi + vars: + context: . + template_context: scripts/dev/templates + + stages: + - name: operator-template-ubi + task_type: dockerfile_template + tags: ["ubi"] + distro: operator + + inputs: + - builder + - builder_image + - base_image + + output: + - dockerfile: scripts/dev/templates/Dockerfile.ubi-$(inputs.params.version_id) + + - name: operator-build-ubi + task_type: docker_build + tags: ["ubi"] + + dockerfile: scripts/dev/templates/Dockerfile.ubi-$(inputs.params.version_id) + + output: + - registry: $(inputs.params.registry)/community-operator-dev + tag: $(inputs.params.version_id) + - registry: $(inputs.params.registry)/community-operator-dev + tag: latest + diff --git a/inventory.yaml b/inventory.yaml index 1b7e2373e..c8d842a73 100644 --- a/inventory.yaml +++ b/inventory.yaml @@ -53,6 +53,8 @@ images: output: - registry: $(inputs.params.registry)/mongodb-agent-dev tag: $(inputs.params.version_id) + - registry: $(inputs.params.registry)/mongodb-agent-dev + tag: latest - name: agent-template-ubuntu-s3 task_type: dockerfile_template @@ -153,6 +155,8 @@ images: output: - registry: $(inputs.params.registry)/mongodb-agent-ubi-dev tag: $(inputs.params.version_id) + - registry: $(inputs.params.registry)/mongodb-agent-ubi-dev + tag: latest - name: agent-template-ubi-s3 task_type: dockerfile_template @@ -229,6 +233,8 @@ images: output: - registry: $(inputs.params.registry)/mongodb-kubernetes-readinessprobe-dev tag: $(inputs.params.version_id) + - registry: $(inputs.params.registry)/mongodb-kubernetes-readinessprobe-dev + tag: latest - name: readiness-init-context-release @@ -296,6 +302,8 @@ images: output: - registry: $(inputs.params.registry)/mongodb-kubernetes-operator-version-upgrade-post-start-hook-dev tag: $(inputs.params.version_id) + - registry: $(inputs.params.registry)/mongodb-kubernetes-operator-version-upgrade-post-start-hook-dev + tag: latest - name: version-post-start-hook-init-context-release diff --git a/pipeline.py b/pipeline.py index 352a404c5..d152e5ab6 100644 --- a/pipeline.py +++ b/pipeline.py @@ -13,9 +13,12 @@ "agent-ubuntu", "readiness-probe-init", "version-post-start-hook-init", + "operator-ubi", + "e2e", ] ) +GOLANG_TAG = "1.15" DEFAULT_IMAGE_TYPE = "ubuntu" DEFAULT_NAMESPACE = "default" @@ -89,6 +92,33 @@ def build_version_post_start_hook_image(config: DevConfig) -> None: ) +def build_operator_ubi_image(config: DevConfig) -> None: + config.ensure_tag_is_run("ubi") + sonar_build_image( + "operator-ubi", + config, + args={ + "registry": config.repo_url, + "builder": "true", + "builder_image": f"golang:{GOLANG_TAG}", + "base_image": "registry.access.redhat.com/ubi8/ubi-minimal:latest", + }, + inventory="inventories/operator-inventory.yaml", + ) + + +def build_e2e_image(config: DevConfig) -> None: + sonar_build_image( + "e2e", + config, + args={ + "registry": config.repo_url, + "base_image": f"golang:{GOLANG_TAG}", + }, + inventory="inventories/e2e-inventory.yaml", + ) + + def sonar_build_image( image_name: str, config: DevConfig, @@ -137,6 +167,8 @@ def main() -> int: "agent-ubuntu": build_agent_image_ubuntu, "readiness-probe-init": build_readiness_probe_image, "version-post-start-hook-init": build_version_post_start_hook_image, + "operator-ubi": build_operator_ubi_image, + "e2e": build_e2e_image, }[image_name] image_build_function(config) diff --git a/pkg/readiness/pod/podannotation.go b/pkg/readiness/pod/podannotation.go index 3f4da8e90..211991822 100644 --- a/pkg/readiness/pod/podannotation.go +++ b/pkg/readiness/pod/podannotation.go @@ -2,11 +2,12 @@ package pod import ( "context" - "go.uber.org/zap" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "strconv" "strings" + "go.uber.org/zap" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" ) @@ -22,8 +23,8 @@ func PatchPodAnnotation(podNamespace string, lastVersionAchieved int64, memberNa if len(pod.Annotations) == 0 { payload = append(payload, patchValue{ - Op: "add", - Path: "/metadata/annotations", + Op: "add", + Path: "/metadata/annotations", Value: make(map[string]string), }) } diff --git a/scripts/dev/build_and_deploy_operator.py b/scripts/dev/build_and_deploy_operator.py deleted file mode 100644 index 93926c756..000000000 --- a/scripts/dev/build_and_deploy_operator.py +++ /dev/null @@ -1,137 +0,0 @@ -import io -import os -from typing import Dict - -import sys - -import yaml -from kubernetes import client, config - -from dev_config import DevConfig, load_config -from dockerfile_generator import render -from dockerutil import build_and_push_image - -import k8s_conditions - - -def _load_operator_service_account() -> Dict: - return load_yaml_from_file("deploy/operator/service_account.yaml") - - -def _load_operator_role() -> Dict: - return load_yaml_from_file("deploy/operator/role.yaml") - - -def _load_operator_role_binding() -> Dict: - return load_yaml_from_file("deploy/operator/role_binding.yaml") - - -def _load_operator_deployment(operator_image: str) -> Dict: - operator = load_yaml_from_file("deploy/operator/operator.yaml") - operator["spec"]["template"]["spec"]["containers"][0]["image"] = operator_image - return operator - - -def _load_mongodb_crd() -> Dict: - return load_yaml_from_file( - "config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml" - ) - - -def load_yaml_from_file(path: str) -> Dict: - with open(path, "r") as f: - return yaml.full_load(f.read()) - - -def _ensure_crds() -> None: - """ - ensure_crds makes sure that all the required CRDs have been created - """ - crdv1 = client.ApiextensionsV1beta1Api() - crd = _load_mongodb_crd() - - k8s_conditions.ignore_if_doesnt_exist( - lambda: crdv1.delete_custom_resource_definition("mongodbcommunity.mongodb.com") - ) - - # Make sure that the CRD has being deleted before trying to create it again - if not k8s_conditions.wait( - lambda: crdv1.list_custom_resource_definition( - field_selector="metadata.name==mongodbcommunity.mongodb.com" - ), - lambda crd_list: len(crd_list.items) == 0, - timeout=5, - sleep_time=0.5, - ): - raise Exception("Execution timed out while waiting for the CRD to be deleted") - - # TODO: fix this, when calling create_custom_resource_definition, we get the error - # ValueError("Invalid value for `conditions`, must not be `None`") - # but the crd is still successfully created - try: - crdv1.create_custom_resource_definition(body=crd) - except ValueError as e: - pass - - print("Ensured CRDs") - - -def build_and_push_operator(repo_url: str, tag: str, path: str) -> None: - """ - build_and_push_operator creates the Dockerfile for the operator - and pushes it to the target repo - """ - build_and_push_image(repo_url, tag, path, "operator") - - -def deploy_operator() -> None: - """ - deploy_operator ensures the CRDs are created, and als creates all the required ServiceAccounts, Roles - and RoleBindings for the operator, and then creates the operator deployment. - """ - appsv1 = client.AppsV1Api() - corev1 = client.CoreV1Api() - rbacv1 = client.RbacAuthorizationV1Api() - - dev_config = load_config() - _ensure_crds() - - k8s_conditions.ignore_if_already_exists( - lambda: rbacv1.create_namespaced_role( - dev_config.namespace, _load_operator_role() - ) - ) - k8s_conditions.ignore_if_already_exists( - lambda: rbacv1.create_namespaced_role_binding( - dev_config.namespace, _load_operator_role_binding() - ) - ) - k8s_conditions.ignore_if_already_exists( - lambda: corev1.create_namespaced_service_account( - dev_config.namespace, _load_operator_service_account() - ) - ) - k8s_conditions.ignore_if_already_exists( - lambda: appsv1.create_namespaced_deployment( - dev_config.namespace, - _load_operator_deployment( - f"{dev_config.repo_url}/mongodb-kubernetes-operator" - ), - ) - ) - - -def main() -> int: - config.load_kube_config() - dev_config = load_config() - build_and_push_operator( - dev_config.repo_url, - f"{dev_config.repo_url}/mongodb-kubernetes-operator", - ".", - ) - deploy_operator() - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/dev/dockerfile_generator.py b/scripts/dev/dockerfile_generator.py index 111b7c508..234135a86 100755 --- a/scripts/dev/dockerfile_generator.py +++ b/scripts/dev/dockerfile_generator.py @@ -11,27 +11,25 @@ GOLANG_TAG = "1.15" -def operator_params(files_to_add: List[str]) -> DockerParameters: +def operator_params() -> DockerParameters: return { "builder": True, "builder_image": f"golang:{GOLANG_TAG}", "base_image": "registry.access.redhat.com/ubi8/ubi-minimal:latest", - "files_to_add": files_to_add, } -def e2e_params(files_to_add: List[str]) -> DockerParameters: +def e2e_params() -> DockerParameters: return { "base_image": f"golang:{GOLANG_TAG}", # TODO: make this image smaller, error: 'exec: "gcc": executable file not found in $PATH' with golang:alpine - "files_to_add": files_to_add, } -def render(image_name: str, files_to_add: List[str]) -> str: +def render(image_name: str) -> str: param_dict = { - "e2e": e2e_params(files_to_add), - "operator": operator_params(files_to_add), + "e2e": e2e_params(), + "operator": operator_params(), } render_values = param_dict.get(image_name, dict()) @@ -46,19 +44,13 @@ def render(image_name: str, files_to_add: List[str]) -> str: def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser() parser.add_argument("image", help="Type of image for the Dockerfile") - parser.add_argument( - "--files_to_add", - help='Paths to use in the ADD command (defaults to ".")', - type=str, - default=".", - ) return parser.parse_args() def main() -> int: args = parse_args() - print(render(args.image, args.files_to_add.split(os.linesep))) + print(render(args.image)) return 0 diff --git a/scripts/dev/dockerutil.py b/scripts/dev/dockerutil.py index 1368b793b..976779d8f 100644 --- a/scripts/dev/dockerutil.py +++ b/scripts/dev/dockerutil.py @@ -97,7 +97,7 @@ def build_and_push_image(repo_url: str, tag: str, path: str, image_type: str) -> build_and_push_operator creates the Dockerfile for the operator and pushes it to the target repo """ - dockerfile_text = render(image_type, ["."]) + dockerfile_text = render(image_type) with open(f"{path}/Dockerfile", "w") as f: f.write(dockerfile_text) diff --git a/scripts/dev/e2e.py b/scripts/dev/e2e.py index 10aaa2b18..ede500e2f 100644 --- a/scripts/dev/e2e.py +++ b/scripts/dev/e2e.py @@ -1,14 +1,9 @@ #!/usr/bin/env python from kubernetes.client.rest import ApiException -from build_and_deploy_operator import ( - build_and_push_operator, - deploy_operator, - load_yaml_from_file, -) + import k8s_conditions import dump_diagnostic -from dockerutil import build_and_push_image from typing import Dict from dev_config import load_config, DevConfig, Distro from kubernetes import client, config @@ -20,6 +15,11 @@ TEST_POD_NAME = "e2e-test" +def load_yaml_from_file(path: str) -> Dict: + with open(path, "r") as f: + return yaml.full_load(f.read()) + + def _load_test_service_account() -> Dict: return load_yaml_from_file("deploy/e2e/service_account.yaml") @@ -92,20 +92,6 @@ def create_kube_config(config_file: str) -> None: ) -def build_and_push_e2e(repo_url: str, tag: str, path: str) -> None: - """ - build_and_push_e2e builds and pushes the e2e image. - """ - build_and_push_image(repo_url, tag, path, "e2e") - - -def build_and_push_version_upgrade_hook(repo_url: str, tag: str, path: str) -> None: - """ - build_and_push_version_upgrade_hook builds and pushes the version upgrade hook image. - """ - build_and_push_image(repo_url, tag, path, "versionhook") - - def _delete_test_pod(config_file: str) -> None: """ _delete_testrunner_pod deletes the test runner pod @@ -227,16 +213,6 @@ def wait_for_pod_to_be_running( def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser() parser.add_argument("--test", help="Name of the test to run") - parser.add_argument( - "--install-operator", - help="Install the operator instead of assuming one already exists", - action="store_true", - ) - parser.add_argument( - "--build-images", - help="Build e2e and version upgrade hook images", - action="store_true", - ) parser.add_argument( "--tag", help="Tag for the images, it will be the same for all images", @@ -268,30 +244,6 @@ def parse_args() -> argparse.Namespace: return parser.parse_args() -def build_and_push_images(args: argparse.Namespace, dev_config: DevConfig) -> None: - if args.install_operator: - build_and_push_operator( - dev_config.repo_url, - f"{dev_config.repo_url}/{dev_config.operator_image}:{args.tag}", - ".", - ) - deploy_operator() - - if args.build_images: - build_and_push_e2e( - dev_config.repo_url, - "{}/{}:{}".format(dev_config.repo_url, dev_config.e2e_image, args.tag), - ".", - ) - build_and_push_version_upgrade_hook( - dev_config.repo_url, - "{}/{}:{}".format( - dev_config.repo_url, dev_config.version_upgrade_hook_image, args.tag - ), - ".", - ) - - def prepare_and_run_test(args: argparse.Namespace, dev_config: DevConfig) -> None: _prepare_test_environment(args.config_file) create_test_pod(args, dev_config) @@ -318,7 +270,6 @@ def main() -> int: create_kube_config(args.config_file) try: - build_and_push_images(args, dev_config) prepare_and_run_test(args, dev_config) finally: if not args.skip_dump_diagnostic: diff --git a/scripts/dev/templates/Dockerfile.template b/scripts/dev/templates/Dockerfile.template index d8a494360..2f41a5596 100644 --- a/scripts/dev/templates/Dockerfile.template +++ b/scripts/dev/templates/Dockerfile.template @@ -15,9 +15,7 @@ WORKDIR /workspace COPY go.mod go.sum ./ RUN go mod download -{% for file in files_to_add %} -ADD {{file}} {{file}} -{% endfor %} +ADD . . # build the binary {% block build_binary -%} From ec6293d5592301b82e4c637b72e247f69f63a208 Mon Sep 17 00:00:00 2001 From: Ciprian Tibulca Date: Fri, 21 May 2021 13:26:45 +0100 Subject: [PATCH 265/790] CLOUDP-73359: resource cleanup (#492) * set owner reference for the service and agent secrets * update e2e tests to check service and agent secrets owner references * resources cleanup - move getOwnerReference inside MongoDBCommunity * update release notes --- Makefile | 3 +- api/v1/mongodbcommunity_types.go | 10 ++++ config/default/kustomization.yaml | 11 +--- config/local_run/kustomization.yaml | 6 ++ controllers/mongodb_tls.go | 4 +- controllers/replica_set_controller.go | 15 +---- dev_notes/RELEASE_NOTES.md | 44 ++++++-------- dev_notes/release-notes-template.md | 7 +++ docs/contributing.md | 25 ++++---- pkg/authentication/scram/mock_types_test.go | 5 ++ pkg/authentication/scram/scram.go | 8 ++- pkg/kube/secret/secret.go | 4 +- release.json | 2 +- test/e2e/mongodbtests/mongodbtests.go | 63 ++++++++++++++++----- 14 files changed, 124 insertions(+), 83 deletions(-) create mode 100644 config/local_run/kustomization.yaml diff --git a/Makefile b/Makefile index 8fc406d42..33dfd6e63 100644 --- a/Makefile +++ b/Makefile @@ -61,7 +61,8 @@ manager: generate fmt vet go build -o bin/manager ./cmd/manager/main.go # Run against the configured Kubernetes cluster in ~/.kube/config -run: generate fmt vet manifests +run: install + $(KUSTOMIZE) build config/local_run | kubectl apply -n $(NAMESPACE) -f - go run ./cmd/manager/main.go # Install CRDs into a cluster diff --git a/api/v1/mongodbcommunity_types.go b/api/v1/mongodbcommunity_types.go index 8c4232c2a..395630f07 100644 --- a/api/v1/mongodbcommunity_types.go +++ b/api/v1/mongodbcommunity_types.go @@ -14,6 +14,7 @@ import ( "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/scale" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" @@ -378,6 +379,15 @@ func (m MongoDBCommunity) GetAgentKeyfileSecretNamespacedName() types.Namespaced return types.NamespacedName{Name: m.Name + "-keyfile", Namespace: m.Namespace} } +func (m MongoDBCommunity) GetOwnerReferences() []metav1.OwnerReference { + ownerReference := *metav1.NewControllerRef(&m, schema.GroupVersionKind{ + Group: GroupVersion.Group, + Version: GroupVersion.Version, + Kind: m.Kind, + }) + return []metav1.OwnerReference{ownerReference} +} + // GetScramOptions returns a set of Options that are used to configure scram // authentication. func (m MongoDBCommunity) GetScramOptions() scram.Options { diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index 8410b870a..464479cdd 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -1,11 +1,6 @@ -# Value of this field is prepended to the -# names of all resources, e.g. a deployment named -# "wordpress" becomes "alices-wordpress". -# Note that it should also match with the prefix (text before '-') of the namespace -# field above. namePrefix: "" bases: -- ../crd -- ../rbac -- ../manager + - ../crd + - ../rbac + - ../manager diff --git a/config/local_run/kustomization.yaml b/config/local_run/kustomization.yaml new file mode 100644 index 000000000..1f251d15e --- /dev/null +++ b/config/local_run/kustomization.yaml @@ -0,0 +1,6 @@ +# used to run the operator locally +namePrefix: "" + +bases: + - ../crd + - ../rbac diff --git a/controllers/mongodb_tls.go b/controllers/mongodb_tls.go index da45631a3..eda541f46 100644 --- a/controllers/mongodb_tls.go +++ b/controllers/mongodb_tls.go @@ -7,8 +7,6 @@ import ( "github.com/pkg/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/mongodb/mongodb-kubernetes-operator/controllers/construct" "github.com/mongodb/mongodb-kubernetes-operator/pkg/automationconfig" @@ -135,7 +133,7 @@ func ensureTLSSecret(getUpdateCreator secret.GetUpdateCreator, mdb mdbv1.MongoDB SetName(mdb.TLSOperatorSecretNamespacedName().Name). SetNamespace(mdb.TLSOperatorSecretNamespacedName().Namespace). SetField(fileName, certKey). - SetOwnerReferences([]metav1.OwnerReference{getOwnerReference(mdb)}). + SetOwnerReferences(mdb.GetOwnerReferences()). Build() return secret.CreateOrUpdate(getUpdateCreator, operatorSecret) diff --git a/controllers/replica_set_controller.go b/controllers/replica_set_controller.go index c97ba8217..d62969a64 100644 --- a/controllers/replica_set_controller.go +++ b/controllers/replica_set_controller.go @@ -40,9 +40,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apiErrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" k8sClient "sigs.k8s.io/controller-runtime/pkg/client" @@ -410,7 +408,7 @@ func (r ReplicaSetReconciler) ensureAutomationConfig(mdb mdbv1.MongoDBCommunity) return automationconfig.EnsureSecret( r.client, types.NamespacedName{Name: mdb.AutomationConfigSecretName(), Namespace: mdb.Namespace}, - []metav1.OwnerReference{getOwnerReference(mdb)}, + mdb.GetOwnerReferences(), ac, ) @@ -451,6 +449,7 @@ func buildService(mdb mdbv1.MongoDBCommunity) corev1.Service { SetClusterIP("None"). SetPort(27017). SetPublishNotReadyAddresses(true). + SetOwnerReferences(mdb.GetOwnerReferences()). Build() } @@ -538,7 +537,7 @@ func buildStatefulSetModificationFunction(mdb mdbv1.MongoDBCommunity) statefulse commonModification := construct.BuildMongoDBReplicaSetStatefulSetModificationFunction(&mdb, mdb) return statefulset.Apply( commonModification, - statefulset.WithOwnerReference([]metav1.OwnerReference{getOwnerReference(mdb)}), + statefulset.WithOwnerReference(mdb.GetOwnerReferences()), statefulset.WithPodSpecTemplate( podtemplatespec.Apply( buildTLSPodSpecModification(mdb), @@ -549,14 +548,6 @@ func buildStatefulSetModificationFunction(mdb mdbv1.MongoDBCommunity) statefulse ) } -func getOwnerReference(mdb mdbv1.MongoDBCommunity) metav1.OwnerReference { - return *metav1.NewControllerRef(&mdb, schema.GroupVersionKind{ - Group: mdbv1.GroupVersion.Group, - Version: mdbv1.GroupVersion.Version, - Kind: mdb.Kind, - }) -} - func getDomain(service, namespace, clusterName string) string { if clusterName == "" { clusterName = "cluster.local" diff --git a/dev_notes/RELEASE_NOTES.md b/dev_notes/RELEASE_NOTES.md index 603818df9..d4329700d 100644 --- a/dev_notes/RELEASE_NOTES.md +++ b/dev_notes/RELEASE_NOTES.md @@ -1,28 +1,20 @@ -# MongoDB Kubernetes Operator 0.6.0 +# MongoDB Kubernetes Operator 0.6.1 + ## Kubernetes Operator -* Breaking Changes - * A new VolumeClaimTemplate has been added `logs-volume`. When you deploy the operator, if there is an existing StatefulSet the operator will attempt to perform an invalid update. The existing StatefulSet must be deleted before upgrading the operator. - - * The user of the mongod and mongodb-agent containers has changed. This means that there will be permissions - issues when upgrading from an earlier version of the operator. In order to update the permissions in the volume, you can use an init container. - -* Upgrade instructions - - Remove the current operator deployment - - `kubectl delete deployment ` - Delete the existing StatefulSet for the MongoDBCommunity resource - Note: to ensure existing data is not lost, ensure that the retain policy of your Persistent Volumes is configured correctly. Please reference the [official docs](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#reclaiming) for these configuration options. - - `kubectl delete statefulset ` - Install the new operator - - follow the regular [installation instruction](https://github.com/mongodb/mongodb-kubernetes-operator/blob/master/docs/install-upgrade.md) - Patch the StatefulSet once it has been created. This will add an init container that will update the permissions of the existing volume. - - `kubectl patch statefulset --type='json' --patch '[ {"op":"add","path":"/spec/template/spec/initContainers/-", "value": { "name": "change-data-dir-permissions", "image": "busybox", "command": [ "chown", "-R", "2000", "/data" ], "securityContext": { "runAsNonRoot": false, "runAsUser": 0, "runAsGroup":0 }, "volumeMounts": [ { "mountPath": "/data", "name" : "data-volume" } ] } } ]'` - -* Bug fixes - * Fixes an issue that prevented the agents from reaching goal state when upgrading minor version of MongoDB. - - ## Updated Image Tags - * mongodb-kubernetes-operator:0.6.0 - * mongodb-agent:0.29.0.6830-1 - * mongodb-kubernetes-readinessprobe:1.0.3 +- Bug fixes + - when deleting MongoDB Resource cleanup related resources (k8s services, secrets). + +## MongoDB Agent ReadinessProbe + +- Changes + - Readiness probe now patches pod annotations rather than overwriting them. + +## Updated Image Tags + +- mongodb-kubernetes-operator:0.6.1 +- mongodb-kubernetes-readinessprobe:1.0.4 + +_All the images can be found in:_ + +https://quay.io/mongodb diff --git a/dev_notes/release-notes-template.md b/dev_notes/release-notes-template.md index 58a9e7986..d3c87ed0f 100644 --- a/dev_notes/release-notes-template.md +++ b/dev_notes/release-notes-template.md @@ -20,6 +20,13 @@ * Bug fixes (*CVE issues go first*) * Fixes an issue ... +## MongoDB Agent ReadinessProbe +* Breaking Changes + * Breaking Change 1 +* Changes + * Change 1 +* Bug fixes (*CVE issues go first*) + * Fixes an issue ... ## Known Issues * Issue 1 diff --git a/docs/contributing.md b/docs/contributing.md index d9b8efc95..a432b7fe5 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -47,21 +47,16 @@ to be able to run properly. Create a json file with the following content: ```json { - "namespace": "default", - "repo_url": "localhost:5000", - "operator_image": "mongodb-kubernetes-operator", -<<<<<<< HEAD - "e2e_image": "community-e2e", - "version_upgrade_hook_image": "version_upgrade_hook", - "prestop_hook_image": "prehook", - "testrunner_image": "test-runner", - "agent_image_ubuntu": "mongodb-agent-dev", - "agent_image_ubi": "mongodb-agent-ubi-dev", - "readiness_probe_image": "mongodb-kubernetes-readiness" -======= - "e2e_image": "e2e", - "version_upgrade_hook_image": "community-operator-version-upgrade-post-start-hook" ->>>>>>> master + "namespace": "default", + "repo_url": "localhost:5000", + "operator_image": "mongodb-kubernetes-operator", + "e2e_image": "community-e2e", + "version_upgrade_hook_image": "community-operator-version-upgrade-post-start-hook", + "prestop_hook_image": "prehook", + "testrunner_image": "test-runner", + "agent_image_ubuntu": "mongodb-agent-dev", + "agent_image_ubi": "mongodb-agent-ubi-dev", + "readiness_probe_image": "mongodb-kubernetes-readiness" } ``` diff --git a/pkg/authentication/scram/mock_types_test.go b/pkg/authentication/scram/mock_types_test.go index 97ca0204f..fbbd662fc 100644 --- a/pkg/authentication/scram/mock_types_test.go +++ b/pkg/authentication/scram/mock_types_test.go @@ -2,6 +2,7 @@ package scram import ( corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -57,3 +58,7 @@ func (m mockConfigurable) GetScramUsers() []User { func (m mockConfigurable) NamespacedName() types.NamespacedName { return m.nsName } + +func (m mockConfigurable) GetOwnerReferences() []metav1.OwnerReference { + return nil +} diff --git a/pkg/authentication/scram/scram.go b/pkg/authentication/scram/scram.go index c8855eec9..79ea36ff9 100644 --- a/pkg/authentication/scram/scram.go +++ b/pkg/authentication/scram/scram.go @@ -13,6 +13,7 @@ import ( "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/secret" "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/generate" apiErrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" ) @@ -43,6 +44,9 @@ type Configurable interface { // NamespacedName returns the NamespacedName for the resource that is being configured. NamespacedName() types.NamespacedName + + // GetOwnerReferences returns the OwnerReferences pointing to the current resource. + GetOwnerReferences() []metav1.OwnerReference } // Role is a struct which will map to automationconfig.Role. @@ -116,13 +120,13 @@ func Enable(auth *automationconfig.Auth, secretGetUpdateCreateDeleter secret.Get } // ensure that the agent password secret exists or read existing password. - agentPassword, err := secret.EnsureSecretWithKey(secretGetUpdateCreateDeleter, mdb.GetAgentPasswordSecretNamespacedName(), AgentPasswordKey, generatedPassword) + agentPassword, err := secret.EnsureSecretWithKey(secretGetUpdateCreateDeleter, mdb.GetAgentPasswordSecretNamespacedName(), mdb.GetOwnerReferences(), AgentPasswordKey, generatedPassword) if err != nil { return err } // ensure that the agent keyfile secret exists or read existing keyfile. - agentKeyFile, err := secret.EnsureSecretWithKey(secretGetUpdateCreateDeleter, mdb.GetAgentKeyfileSecretNamespacedName(), AgentKeyfileKey, generatedContents) + agentKeyFile, err := secret.EnsureSecretWithKey(secretGetUpdateCreateDeleter, mdb.GetAgentKeyfileSecretNamespacedName(), mdb.GetOwnerReferences(), AgentKeyfileKey, generatedContents) if err != nil { return err } diff --git a/pkg/kube/secret/secret.go b/pkg/kube/secret/secret.go index 02e551165..52bf253b9 100644 --- a/pkg/kube/secret/secret.go +++ b/pkg/kube/secret/secret.go @@ -5,6 +5,7 @@ import ( corev1 "k8s.io/api/core/v1" apiErrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -112,7 +113,7 @@ func HasAllKeys(secret corev1.Secret, keys ...string) bool { // EnsureSecretWithKey makes sure the Secret with the given name has a key with the given value if the key is not already present. // if the key is present, it will return the existing value associated with this key. -func EnsureSecretWithKey(secretGetUpdateCreateDeleter GetUpdateCreateDeleter, nsName types.NamespacedName, key, value string) (string, error) { +func EnsureSecretWithKey(secretGetUpdateCreateDeleter GetUpdateCreateDeleter, nsName types.NamespacedName, ownerReferences []metav1.OwnerReference, key, value string) (string, error) { existingSecret, err0 := secretGetUpdateCreateDeleter.GetSecret(nsName) if err0 != nil { if apiErrors.IsNotFound(err0) { @@ -120,6 +121,7 @@ func EnsureSecretWithKey(secretGetUpdateCreateDeleter GetUpdateCreateDeleter, ns SetNamespace(nsName.Namespace). SetName(nsName.Name). SetField(key, value). + SetOwnerReferences(ownerReferences). Build() if err1 := secretGetUpdateCreateDeleter.CreateSecret(s); err1 != nil { diff --git a/release.json b/release.json index 8a4b8ac38..b7893458c 100644 --- a/release.json +++ b/release.json @@ -1,7 +1,7 @@ { "mongodb-kubernetes-operator": "0.6.0", "version-upgrade-hook": "1.0.2", - "readiness-probe": "1.0.3", + "readiness-probe": "1.0.4", "mongodb-agent": { "version": "10.29.0.6830-1", "tools_version": "100.2.0" diff --git a/test/e2e/mongodbtests/mongodbtests.go b/test/e2e/mongodbtests/mongodbtests.go index 1d750923a..56e31e039 100644 --- a/test/e2e/mongodbtests/mongodbtests.go +++ b/test/e2e/mongodbtests/mongodbtests.go @@ -74,21 +74,41 @@ func statefulSetIsNotReady(mdb *mdbv1.MongoDBCommunity, interval time.Duration, func StatefulSetHasOwnerReference(mdb *mdbv1.MongoDBCommunity, expectedOwnerReference metav1.OwnerReference) func(t *testing.T) { return func(t *testing.T) { + stsNamespacedName := types.NamespacedName{Name: mdb.Name, Namespace: mdb.Namespace} sts := appsv1.StatefulSet{} - err := e2eutil.TestClient.Get(context.TODO(), types.NamespacedName{Name: mdb.Name, Namespace: mdb.Namespace}, &sts) + err := e2eutil.TestClient.Get(context.TODO(), stsNamespacedName, &sts) + + if err != nil { + t.Fatal(err) + } + assertEqualOwnerReference(t, "StatefulSet", stsNamespacedName, sts.GetOwnerReferences(), expectedOwnerReference) + } +} + +func ServiceHasOwnerReference(mdb *mdbv1.MongoDBCommunity, expectedOwnerReference metav1.OwnerReference) func(t *testing.T) { + return func(t *testing.T) { + serviceNamespacedName := types.NamespacedName{Name: mdb.ServiceName(), Namespace: mdb.Namespace} + srv := corev1.Service{} + err := e2eutil.TestClient.Get(context.TODO(), serviceNamespacedName, &srv) if err != nil { t.Fatal(err) } - ownerReferences := sts.GetOwnerReferences() + assertEqualOwnerReference(t, "Service", serviceNamespacedName, srv.GetOwnerReferences(), expectedOwnerReference) + } +} - assert.Len(t, ownerReferences, 1, "StatefulSet doesn't have OwnerReferences") +func AgentSecretsHaveOwnerReference(mdb *mdbv1.MongoDBCommunity, expectedOwnerReference metav1.OwnerReference) func(t *testing.T) { + checkSecret := func(t *testing.T, resourceNamespacedName types.NamespacedName) { + secret := corev1.Secret{} + err := e2eutil.TestClient.Get(context.TODO(), resourceNamespacedName, &secret) - assert.Equal(t, expectedOwnerReference.APIVersion, ownerReferences[0].APIVersion) - assert.Equal(t, "MongoDBCommunity", ownerReferences[0].Kind) - assert.Equal(t, expectedOwnerReference.Name, ownerReferences[0].Name) - assert.Equal(t, expectedOwnerReference.UID, ownerReferences[0].UID) + assert.NoError(t, err) + assertEqualOwnerReference(t, "Secret", resourceNamespacedName, secret.GetOwnerReferences(), expectedOwnerReference) + } - t.Logf("StatefulSet %s/%s has the correct OwnerReference!", mdb.Namespace, mdb.Name) + return func(t *testing.T) { + checkSecret(t, mdb.GetAgentPasswordSecretNamespacedName()) + checkSecret(t, mdb.GetAgentKeyfileSecretNamespacedName()) } } @@ -174,17 +194,23 @@ func CreateMongoDBResource(mdb *mdbv1.MongoDBCommunity, ctx *e2eutil.Context) fu } } +func getOwnerReference(mdb *mdbv1.MongoDBCommunity) metav1.OwnerReference { + return *metav1.NewControllerRef(mdb, schema.GroupVersionKind{ + Group: mdbv1.GroupVersion.Group, + Version: mdbv1.GroupVersion.Version, + Kind: mdb.Kind, + }) +} + func BasicFunctionality(mdb *mdbv1.MongoDBCommunity) func(*testing.T) { return func(t *testing.T) { + mdbOwnerReference := getOwnerReference(mdb) t.Run("Secret Was Correctly Created", AutomationConfigSecretExists(mdb)) t.Run("Stateful Set Reaches Ready State", StatefulSetBecomesReady(mdb)) t.Run("MongoDB Reaches Running Phase", MongoDBReachesRunningPhase(mdb)) - t.Run("Stateful Set has OwnerReference", StatefulSetHasOwnerReference(mdb, - *metav1.NewControllerRef(mdb, schema.GroupVersionKind{ - Group: mdbv1.GroupVersion.Group, - Version: mdbv1.GroupVersion.Version, - Kind: mdb.Kind, - }))) + t.Run("Stateful Set Has OwnerReference", StatefulSetHasOwnerReference(mdb, mdbOwnerReference)) + t.Run("Service Set Has OwnerReference", ServiceHasOwnerReference(mdb, mdbOwnerReference)) + t.Run("Agent Secrets Have OwnerReference", AgentSecretsHaveOwnerReference(mdb, mdbOwnerReference)) t.Run("Test Status Was Updated", Status(mdb, mdbv1.MongoDBCommunityStatus{ MongoURI: mdb.MongoURI(), @@ -333,3 +359,12 @@ func findContainerByName(name string, containers []corev1.Container) *corev1.Con return nil } + +func assertEqualOwnerReference(t *testing.T, resourceType string, resourceNamespacedName types.NamespacedName, ownerReferences []metav1.OwnerReference, expectedOwnerReference metav1.OwnerReference) { + assert.Len(t, ownerReferences, 1, fmt.Sprintf("%s %s/%s doesn't have OwnerReferences", resourceType, resourceNamespacedName.Name, resourceNamespacedName.Namespace)) + + assert.Equal(t, expectedOwnerReference.APIVersion, ownerReferences[0].APIVersion) + assert.Equal(t, "MongoDBCommunity", ownerReferences[0].Kind) + assert.Equal(t, expectedOwnerReference.Name, ownerReferences[0].Name) + assert.Equal(t, expectedOwnerReference.UID, ownerReferences[0].UID) +} From ebbab2c974f769fca128c6a64815e30e4b001e3c Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Fri, 21 May 2021 18:03:11 +0100 Subject: [PATCH 266/790] Add predicates to controller (#494) --- controllers/replica_set_controller.go | 5 ++++- dev_notes/RELEASE_NOTES.md | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/controllers/replica_set_controller.go b/controllers/replica_set_controller.go index d62969a64..bd1c765ea 100644 --- a/controllers/replica_set_controller.go +++ b/controllers/replica_set_controller.go @@ -6,6 +6,9 @@ import ( "fmt" "os" + "github.com/mongodb/mongodb-kubernetes-operator/controllers/predicates" + "sigs.k8s.io/controller-runtime/pkg/builder" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/container" "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/functions" @@ -77,7 +80,7 @@ func NewReconciler(mgr manager.Manager) *ReplicaSetReconciler { // SetupWithManager sets up the controller with the Manager and configures the necessary watches. func (r *ReplicaSetReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). - For(&mdbv1.MongoDBCommunity{}). + For(&mdbv1.MongoDBCommunity{}, builder.WithPredicates(predicates.OnlyOnSpecChange())). Complete(r) } diff --git a/dev_notes/RELEASE_NOTES.md b/dev_notes/RELEASE_NOTES.md index d4329700d..1120ed52b 100644 --- a/dev_notes/RELEASE_NOTES.md +++ b/dev_notes/RELEASE_NOTES.md @@ -4,6 +4,9 @@ - Bug fixes - when deleting MongoDB Resource cleanup related resources (k8s services, secrets). + +- Changes + - fixed an issue where the operator would reconcile based on events emitted by itself in certain situations. ## MongoDB Agent ReadinessProbe From 0ff354c936d980153f3372f98e431e64eeaee3aa Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Mon, 24 May 2021 11:01:50 +0100 Subject: [PATCH 267/790] Allows both TLS and SSL to be specified (#497) --- pkg/automationconfig/automation_config_builder.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pkg/automationconfig/automation_config_builder.go b/pkg/automationconfig/automation_config_builder.go index d50ea5d89..efe821d97 100644 --- a/pkg/automationconfig/automation_config_builder.go +++ b/pkg/automationconfig/automation_config_builder.go @@ -274,17 +274,11 @@ func (b *Builder) Build() (AutomationConfig, error) { }, } - if b.tlsConfig != nil && b.sslConfig != nil { - return AutomationConfig{}, errors.Errorf("must specify only one of tlsConfig and sslConfig") - } - if b.tlsConfig != nil { - currentAc.SSLConfig = nil currentAc.TLSConfig = b.tlsConfig } if b.sslConfig != nil { - currentAc.TLSConfig = nil currentAc.SSLConfig = b.sslConfig } From 790711c411deda60fdb6131e9b5e93fea4999a6d Mon Sep 17 00:00:00 2001 From: Dmitry Kropachev <40304587+dkropachev@users.noreply.github.com> Date: Mon, 24 May 2021 14:31:53 +0300 Subject: [PATCH 268/790] Fix .evergreen.yaml (#498) --- .evergreen.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen.yml b/.evergreen.yml index 21acc2ed5..ae1a3ce76 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -254,7 +254,7 @@ tasks: commands: - func: run_e2e_test vars: - test: feature_compatibility_version + test: feature_compatibility_version_upgrade - name: e2e_test_replica_set commands: From ff0e711884bf077166c895bc57c228d62177b76f Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Mon, 24 May 2021 13:42:56 +0100 Subject: [PATCH 269/790] Fix FCV Upgrade Test (#499) --- .../feature_compatibility_version_upgrade_test.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/test/e2e/feature_compatibility_version_upgrade/feature_compatibility_version_upgrade_test.go b/test/e2e/feature_compatibility_version_upgrade/feature_compatibility_version_upgrade_test.go index 0db80f72d..fb3b011a6 100644 --- a/test/e2e/feature_compatibility_version_upgrade/feature_compatibility_version_upgrade_test.go +++ b/test/e2e/feature_compatibility_version_upgrade/feature_compatibility_version_upgrade_test.go @@ -40,11 +40,6 @@ func TestFeatureCompatibilityVersionUpgrade(t *testing.T) { t.Fatal(err) } - _, err = setup.GeneratePasswordForUser(user, ctx, "") - if err != nil { - t.Fatal(err) - } - tester, err := mongotester.FromResource(t, mdb) if err != nil { t.Fatal(err) From a1f9aecb86b2d6c7d75427ff214c925d9f7394c6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 May 2021 17:50:25 +0100 Subject: [PATCH 270/790] Bump go.mongodb.org/mongo-driver from 1.5.1 to 1.5.2 (#478) Bumps [go.mongodb.org/mongo-driver](https://github.com/mongodb/mongo-go-driver) from 1.5.1 to 1.5.2. - [Release notes](https://github.com/mongodb/mongo-go-driver/releases) - [Commits](https://github.com/mongodb/mongo-go-driver/compare/v1.5.1...v1.5.2) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 782bcbaa0..70a811939 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/stretchr/objx v0.3.0 github.com/stretchr/testify v1.7.0 github.com/xdg/stringprep v1.0.3 - go.mongodb.org/mongo-driver v1.5.1 + go.mongodb.org/mongo-driver v1.5.2 go.uber.org/zap v1.16.0 k8s.io/api v0.20.4 k8s.io/apimachinery v0.21.0 diff --git a/go.sum b/go.sum index 300e188c1..0937f0234 100644 --- a/go.sum +++ b/go.sum @@ -441,8 +441,8 @@ go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= -go.mongodb.org/mongo-driver v1.5.1 h1:9nOVLGDfOaZ9R0tBumx/BcuqkbFpyTCU2r/Po7A2azI= -go.mongodb.org/mongo-driver v1.5.1/go.mod h1:gRXCHX4Jo7J0IJ1oDQyUxF7jfy19UfxniMS4xxMmUqw= +go.mongodb.org/mongo-driver v1.5.2 h1:AsxOLoJTgP6YNM0fXWw4OjdluYmWzQYp+lFJL7xu9fU= +go.mongodb.org/mongo-driver v1.5.2/go.mod h1:gRXCHX4Jo7J0IJ1oDQyUxF7jfy19UfxniMS4xxMmUqw= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= From 65283d44dd22a7fb232a566cca4c56ad6d618177 Mon Sep 17 00:00:00 2001 From: Anton Date: Tue, 25 May 2021 08:52:57 +0100 Subject: [PATCH 271/790] CLOUDP-89260: improve readiness probe (#493) --- cmd/readiness/main.go | 86 +++++++++++------------ cmd/readiness/readiness_test.go | 120 ++++++++++++++++++++++++++------ pkg/readiness/config/config.go | 11 ++- 3 files changed, 150 insertions(+), 67 deletions(-) diff --git a/cmd/readiness/main.go b/cmd/readiness/main.go index a01eddcb6..c15dd52de 100644 --- a/cmd/readiness/main.go +++ b/cmd/readiness/main.go @@ -3,6 +3,7 @@ package main import ( "encoding/json" "fmt" + "io" "io/ioutil" "os" "time" @@ -19,7 +20,8 @@ import ( ) const ( - headlessAgent = "HEADLESS_AGENT" + headlessAgent = "HEADLESS_AGENT" + mongodNotReadyIntervalMinutes = time.Minute * 1 ) var riskySteps []string @@ -44,58 +46,38 @@ func init() { // - if AppDB: the 'mmsStatus[0].lastGoalVersionAchieved' field is compared with the one from mounted automation config // Additionally if the previous check hasn't returned 'true' the "deadlock" case is checked to make sure the Agent is // not waiting for the other members. -func isPodReady(conf config.Config) bool { - fd, err := os.Open(conf.HealthStatusFilePath) +func isPodReady(conf config.Config) (bool, error) { + healthStatus, err := parseHealthStatus(conf.HealthStatusReader) if err != nil { - logger.Warn("No health status file exists, assuming the Automation agent is old") - return true - } - defer fd.Close() - - health, err := readAgentHealthStatus(fd) - if err != nil { - logger.Errorf("Failed to read agent health status file: %s", err) - // panicking allows to see the problem in the events for the pod (kubectl describe pod ..) - panic("Failed to read agent health status file: %s") + logger.Errorf("There was problem parsing health status file: %s", err) + return false, err } // The 'statuses' file can be empty only for OM Agents - if len(health.Healthiness) == 0 && !isHeadlessMode() { + if len(healthStatus.Healthiness) == 0 && !isHeadlessMode() { logger.Info("'statuses' is empty. We assume there is no automation config for the agent yet.") - return true + return true, nil } - // If the agent has reached the goal state - returning true - inGoalState, err := isInGoalState(health, conf) + // If the agent has reached the goal state + inGoalState, err := isInGoalState(healthStatus, conf) if err != nil { logger.Errorf("There was problem checking the health status: %s", err) - panic(err) + return false, err } - inReadyState := isInReadyState(health) + inReadyState := isInReadyState(healthStatus) if inGoalState && inReadyState { logger.Info("Agent has reached goal state") - return true + return true, nil } // Failback logic: the agent is not in goal state and got stuck in some steps - if hasDeadlockedSteps(health) { - return true + if !inGoalState && hasDeadlockedSteps(healthStatus) { + return true, nil } - return false -} - -func readAgentHealthStatus(file *os.File) (health.Status, error) { - var health health.Status - - data, err := ioutil.ReadAll(file) - if err != nil { - return health, err - } - - err = json.Unmarshal(data, &health) - return health, err + return false, nil } // hasDeadlockedSteps returns true if the agent is stuck on waiting for the other agents @@ -199,6 +181,17 @@ func kubernetesClientset() (kubernetes.Interface, error) { return clientset, nil } +func parseHealthStatus(reader io.Reader) (health.Status, error) { + var health health.Status + data, err := ioutil.ReadAll(reader) + if err != nil { + return health, err + } + + err = json.Unmarshal(data, &health) + return health, err +} + func main() { clientSet, err := kubernetesClientset() if err != nil { @@ -219,12 +212,17 @@ func main() { panic(err) } logger = log.Sugar() - if !isPodReady(config) { + + ready, err := isPodReady(config) + if err != nil { + panic(err) + } + if !ready { os.Exit(1) } } -// isInReadyState checks the MongoDB Server state. It returns true if the state +// isInReadyState checks the MongoDB Server state. It returns true if the mongod process is up and its state // is PRIMARY or SECONDARY. func isInReadyState(health health.Status) bool { if len(health.Healthiness) == 0 { @@ -233,14 +231,16 @@ func isInReadyState(health health.Status) bool { for _, processHealth := range health.Healthiness { // We know this loop should run only once, in Kubernetes there's // only 1 server managed per host. + if !processHealth.ExpectedToBeUp { + // Process may be down intentionally (if the process is marked as disabled in the automation config) + return true + } - // Every time the process health is created by the agent, - // it checks if the MongoDB process is up and populates this field - // (https://github.com/10gen/mms-automation/blob/bb72f74a22d98cfa635c1317e623386b089dc69f/go_planner/src/com.tengen/cm/healthcheck/status.go#L43) - // So it's enough to check that this value is not the zero-value for int64 - + timeMongoUp := time.Unix(processHealth.LastMongoUpTime, 0) + mongoUpThreshold := time.Now().Add(-mongodNotReadyIntervalMinutes) + mongoIsHealthy := timeMongoUp.After(mongoUpThreshold) // The case in which the agent is too old to publish replication status is handled inside "IsReadyState" - return processHealth.LastMongoUpTime != 0 && processHealth.IsReadyState() + return mongoIsHealthy && processHealth.IsReadyState() } return false } diff --git a/cmd/readiness/readiness_test.go b/cmd/readiness/readiness_test.go index 988acb571..6dc54c6d7 100644 --- a/cmd/readiness/readiness_test.go +++ b/cmd/readiness/readiness_test.go @@ -1,7 +1,10 @@ package main import ( + "bytes" "context" + "encoding/json" + "io" "os" "testing" "time" @@ -19,19 +22,24 @@ import ( // TestDeadlockDetection verifies that if the agent is stuck in "WaitAllRsMembersUp" phase (started > 15 seconds ago) // then the function returns "ready" func TestDeadlockDetection(t *testing.T) { - assert.True(t, isPodReady(testConfig("testdata/health-status-deadlocked.json"))) + ready, err := isPodReady(testConfig("testdata/health-status-deadlocked.json")) + assert.True(t, ready) + assert.NoError(t, err) } // TestNoDeadlock verifies that if the agent has started (but not finished) "WaitRsInit" and then there is another // started phase ("WaitFeatureCompatibilityVersionCorrect") then no deadlock is found as the latter is considered to // be the "current" step func TestNoDeadlock(t *testing.T) { - health := readHealthinessFile("testdata/health-status-no-deadlock.json") + health, err := parseHealthStatus(testConfig("testdata/health-status-no-deadlock.json").HealthStatusReader) + assert.NoError(t, err) stepStatus := findCurrentStep(health.ProcessPlans) assert.Equal(t, "WaitFeatureCompatibilityVersionCorrect", stepStatus.Step) - assert.False(t, isPodReady(testConfig("testdata/health-status-no-deadlock.json"))) + ready, err := isPodReady(testConfig("testdata/health-status-no-deadlock.json")) + assert.False(t, ready) + assert.NoError(t, err) } // TestDeadlockDetection verifies that if the agent is in "WaitAllRsMembersUp" phase but started < 15 seconds ago @@ -39,25 +47,64 @@ func TestNoDeadlock(t *testing.T) { // Note, that the status file is artificial: it has two plans (the first one is complete and has no moves) to make sure // the readiness logic takes only the last plan for consideration func TestNotReadyWaitingForRsReady(t *testing.T) { - assert.False(t, isPodReady(testConfig("testdata/health-status-pending.json"))) + ready, err := isPodReady(testConfig("testdata/health-status-pending.json")) + assert.False(t, ready) + assert.NoError(t, err) } // TestNotReadyHealthFileHasNoPlans verifies that the readiness script doesn't panic if the health file has unexpected // data (there are no plans at all) func TestNotReadyHealthFileHasNoPlans(t *testing.T) { - assert.False(t, isPodReady(testConfig("testdata/health-status-no-plans.json"))) + ready, err := isPodReady(testConfig("testdata/health-status-no-plans.json")) + assert.False(t, ready) + assert.NoError(t, err) } // TestNotReadyHealthFileHasNoProcesses verifies that the readiness script doesn't panic if the health file has unexpected // data (there are no processes at all) func TestNotReadyHealthFileHasNoProcesses(t *testing.T) { - assert.False(t, isPodReady(testConfig("testdata/health-status-no-processes.json"))) + ready, err := isPodReady(testConfig("testdata/health-status-no-processes.json")) + assert.False(t, ready) + assert.NoError(t, err) +} + +func TestNotReadyMongodIsDown(t *testing.T) { + t.Run("Mongod is down for 90 seconds", func(t *testing.T) { + ready, err := isPodReady(testConfigWithMongoUp("testdata/health-status-ok.json", time.Second*90)) + assert.False(t, ready) + assert.NoError(t, err) + }) + t.Run("Mongod is down for 1 hour", func(t *testing.T) { + ready, err := isPodReady(testConfigWithMongoUp("testdata/health-status-ok.json", time.Hour*1)) + assert.False(t, ready) + assert.NoError(t, err) + }) + t.Run("Mongod is down for 2 days", func(t *testing.T) { + ready, err := isPodReady(testConfigWithMongoUp("testdata/health-status-ok.json", time.Hour*48)) + assert.False(t, ready) + assert.NoError(t, err) + }) +} + +func TestReadyMongodIsUp(t *testing.T) { + t.Run("Mongod is down for 30 seconds", func(t *testing.T) { + ready, err := isPodReady(testConfigWithMongoUp("testdata/health-status-ok.json", time.Second*30)) + assert.True(t, ready) + assert.NoError(t, err) + }) + t.Run("Mongod is down for 1 second", func(t *testing.T) { + ready, err := isPodReady(testConfigWithMongoUp("testdata/health-status-ok.json", time.Second*1)) + assert.True(t, ready) + assert.NoError(t, err) + }) } // TestReady verifies that the probe reports "ready" despite "WaitRsInit" stage reporting as not reached // (this is some bug in Automation Agent which we can work with) func TestReady(t *testing.T) { - assert.True(t, isPodReady(testConfig("testdata/health-status-ok.json"))) + ready, err := isPodReady(testConfig("testdata/health-status-ok.json")) + assert.True(t, ready) + assert.NoError(t, err) } // TestNoDeadlockForDownloadProcess verifies that the steps not listed as "riskySteps" (like "download") are not @@ -97,7 +144,9 @@ func TestHeadlessAgentHasntReachedGoal(t *testing.T) { _ = os.Setenv(headlessAgent, "true") c := testConfig("testdata/health-status-ok.json") c.ClientSet = fake.NewSimpleClientset(testdata.TestPod(c.Namespace, c.Hostname), testdata.TestSecret(c.Namespace, c.AutomationConfigSecretName, 6)) - assert.False(t, isPodReady(c)) + ready, err := isPodReady(c) + assert.False(t, ready) + assert.NoError(t, err) thePod, _ := c.ClientSet.CoreV1().Pods(c.Namespace).Get(context.TODO(), c.Hostname, metav1.GetOptions{}) assert.Equal(t, map[string]string{"agent.mongodb.com/version": "5"}, thePod.Annotations) @@ -110,7 +159,9 @@ func TestHeadlessAgentReachedGoal(t *testing.T) { _ = os.Setenv(headlessAgent, "true") c := testConfig("testdata/health-status-ok.json") c.ClientSet = fake.NewSimpleClientset(testdata.TestPod(c.Namespace, c.Hostname), testdata.TestSecret(c.Namespace, c.AutomationConfigSecretName, 5)) - assert.True(t, isPodReady(c)) + ready, err := isPodReady(c) + assert.True(t, ready) + assert.NoError(t, err) thePod, _ := c.ClientSet.CoreV1().Pods(c.Namespace).Get(context.TODO(), c.Hostname, metav1.GetOptions{}) assert.Equal(t, map[string]string{"agent.mongodb.com/version": "5"}, thePod.Annotations) @@ -119,33 +170,58 @@ func TestHeadlessAgentReachedGoal(t *testing.T) { func TestPodReadiness(t *testing.T) { t.Run("Pod readiness is correctly checked when no ReplicationStatus is present on the file ", func(t *testing.T) { - assert.True(t, isPodReady(testConfig("testdata/health-status-no-replication.json"))) + ready, err := isPodReady(testConfig("testdata/health-status-no-replication.json")) + assert.True(t, ready) + assert.NoError(t, err) }) t.Run("MongoDB replication state is reported by agents", func(t *testing.T) { - assert.True(t, isPodReady(testConfig("testdata/health-status-ok-no-replica-status.json"))) + ready, err := isPodReady(testConfig("testdata/health-status-ok-no-replica-status.json")) + assert.True(t, ready) + assert.NoError(t, err) }) t.Run("If replication state is not PRIMARY or SECONDARY, Pod is not ready", func(t *testing.T) { - assert.False(t, isPodReady(testConfig("testdata/health-status-not-readable-state.json"))) - }) - - t.Run("If replication state is readable", func(t *testing.T) { - assert.True(t, isPodReady(testConfig("testdata/health-status-readable-state.json"))) + ready, err := isPodReady(testConfig("testdata/health-status-not-readable-state.json")) + assert.False(t, ready) + assert.NoError(t, err) }) } -func readHealthinessFile(path string) health.Status { - fd, _ := os.Open(path) - health, _ := readAgentHealthStatus(fd) - return health +func testConfig(healthFilePath string) config.Config { + return testConfigWithMongoUp(healthFilePath, 15*time.Second) } -func testConfig(healthFilePath string) config.Config { +func testConfigWithMongoUp(healthFilePath string, timeSinceMongoLastUp time.Duration) config.Config { + file, err := os.Open(healthFilePath) + if err != nil { + panic(err) + } + defer file.Close() + + status, err := parseHealthStatus(file) + if err != nil { + panic(err) + } + + for key, processHealth := range status.Healthiness { + processHealth.LastMongoUpTime = time.Now().Add(-timeSinceMongoLastUp).Unix() + // Need to reassign the object back to map as 'processHealth' is a copy of the struct + status.Healthiness[key] = processHealth + } + return config.Config{ - HealthStatusFilePath: healthFilePath, + HealthStatusReader: NewTestHealthStatusReader(status), Namespace: "test-ns", AutomationConfigSecretName: "test-mongodb-automation-config", Hostname: "test-mongodb-0", } } + +func NewTestHealthStatusReader(status health.Status) io.Reader { + data, err := json.Marshal(status) + if err != nil { + panic(err) + } + return bytes.NewReader(data) +} diff --git a/pkg/readiness/config/config.go b/pkg/readiness/config/config.go index e7082adac..91216dbfb 100644 --- a/pkg/readiness/config/config.go +++ b/pkg/readiness/config/config.go @@ -2,6 +2,7 @@ package config import ( "fmt" + "io" "os" "strings" @@ -23,7 +24,7 @@ type Config struct { Namespace string Hostname string AutomationConfigSecretName string - HealthStatusFilePath string + HealthStatusReader io.Reader LogFilePath string } @@ -47,12 +48,18 @@ func BuildFromEnvVariables(clientSet kubernetes.Interface, isHeadless bool) (Con return Config{}, fmt.Errorf("the '%s' environment variable must be set", hostNameEnv) } } + // Note, that we shouldn't close the file here - it will be closed very soon by the 'ioutil.ReadAll' + // in main.go + file, err := os.Open(healthStatusFilePath) + if err != nil { + return Config{}, err + } return Config{ ClientSet: clientSet, Namespace: namespace, AutomationConfigSecretName: automationConfigName, Hostname: hostname, - HealthStatusFilePath: healthStatusFilePath, + HealthStatusReader: file, LogFilePath: logFilePath, }, nil } From 8a806e2d06617af2b02a1995e4bce1e16c39655b Mon Sep 17 00:00:00 2001 From: Anton Date: Tue, 25 May 2021 10:51:32 +0100 Subject: [PATCH 272/790] Some fixes in dev-start scripts/docs (#500) --- Makefile | 2 +- dev_notes/dev-quick-start.md | 6 ++++-- scripts/dev/setup_kind_cluster.sh | 14 ++++++++++++++ 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 33dfd6e63..aa30e35c2 100644 --- a/Makefile +++ b/Makefile @@ -80,7 +80,7 @@ deploy: manifests kustomize # Deploy a simple ReplicaSet, this is intended for first time use only as part of the quick start guide. deploy-dev-quick-start-rs: manifests kustomize - kubectl create secret generic quick-start-rs --from-literal=password=dev-quick-start-password --from-literal=username=admin || true + kubectl create secret generic my-user-password --from-literal=password=dev-quick-start-password --from-literal=username=admin || true kubectl apply -f dev_notes/dev_quick_start_resources/dev_quick_start_rs.yaml # UnDeploy controller from the configured Kubernetes cluster in ~/.kube/config diff --git a/dev_notes/dev-quick-start.md b/dev_notes/dev-quick-start.md index 5659a423f..019f17819 100644 --- a/dev_notes/dev-quick-start.md +++ b/dev_notes/dev-quick-start.md @@ -49,15 +49,17 @@ kubectl config set-context --current --namespace=mongodb #### create a config file for the dev environment ```bash +mkdir -p ~/.community-operator-dev cat > ~/.community-operator-dev/config.json << EOL { "namespace": "default", "repo_url": "localhost:5000", - "operator_image": "mongodb-kubernetes-operator", + "operator_image": "community-operator-dev", "e2e_image": "e2e", "prestop_hook_image": "prehook", "version_upgrade_hook_image": "community-operator-version-upgrade-post-start-hook", - "readiness_probe_image": "mongodb-kubernetes-readinessprobe" + "readiness_probe_image": "mongodb-kubernetes-readinessprobe", + "s3_bucket": "" } EOL ``` diff --git a/scripts/dev/setup_kind_cluster.sh b/scripts/dev/setup_kind_cluster.sh index 40a38c9d6..fcb7d440a 100755 --- a/scripts/dev/setup_kind_cluster.sh +++ b/scripts/dev/setup_kind_cluster.sh @@ -29,3 +29,17 @@ containerdConfigPatches: [plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:${reg_port}"] endpoint = ["http://${ip}:${reg_port}"] EOF + +# Document the local registry (from https://kind.sigs.k8s.io/docs/user/local-registry/) +# https://github.com/kubernetes/enhancements/tree/master/keps/sig-cluster-lifecycle/generic/1755-communicating-a-local-registry +cat < Date: Wed, 26 May 2021 11:23:06 +0100 Subject: [PATCH 273/790] Ensure Unique Execution ID With Each Test (Unique Password) (#501) --- test/e2e/client.go | 32 +++++++++++++++---- test/e2e/e2eutil.go | 4 +-- .../feature_compatibility_version_test.go | 11 +++---- ...ture_compatibility_version_upgrade_test.go | 11 +++---- test/e2e/replica_set/replica_set_test.go | 10 +++--- .../replica_set_test.go | 11 +++---- ...replica_set_cross_namespace_deploy_test.go | 11 +++---- .../replica_set_custom_role_test.go | 11 +++---- .../replica_set_mongod_config_test.go | 10 +++--- .../replica_set_multiple_test.go | 15 ++++----- .../replica_set_readiness_probe_test.go | 11 +++---- .../replica_set_scaling_test.go | 11 +++---- .../replica_set_scale_down_test.go | 10 +++--- .../replica_set_tls/replica_set_tls_test.go | 10 +++--- .../replica_set_tls_rotate_test.go | 10 +++--- .../replica_set_tls_upgrade_test.go | 10 +++--- test/e2e/setup/setup.go | 14 ++++---- .../statefulset_arbitrary_config_test.go | 10 +++--- ...tatefulset_arbitrary_config_update_test.go | 10 +++--- 19 files changed, 102 insertions(+), 120 deletions(-) diff --git a/test/e2e/client.go b/test/e2e/client.go index 8a63ff9df..fc13acc01 100644 --- a/test/e2e/client.go +++ b/test/e2e/client.go @@ -5,6 +5,8 @@ import ( "fmt" "testing" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/generate" + "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -32,17 +34,35 @@ func (*CleanupOptions) ApplyToCreate(*client.CreateOptions) {} // Context tracks cleanup functions to be called at the end of a test. type Context struct { - cleanupFuncs [](func() error) - t *testing.T + // shouldPerformCleanup indicates whether or not cleanup should happen after this test + shouldPerformCleanup bool + + // ExecutionId is a unique identifier for this test run. + ExecutionId string + + // cleanupFuncs is a list of functions which will clean up resources + // after the test ends. + cleanupFuncs []func() error + + // t is the testing.T which will be used for the duration of the test. + t *testing.T } // NewContext creates a context. -func NewContext(t *testing.T) *Context { - return &Context{t: t} +func NewContext(t *testing.T, performCleanup bool) (*Context, error) { + testId, err := generate.RandomValidDNS1123Label(10) + if err != nil { + return nil, err + } + + return &Context{t: t, ExecutionId: testId, shouldPerformCleanup: performCleanup}, nil } -// Cleanup is called at the end of a test. -func (ctx *Context) Cleanup() { +// Teardown is called at the end of a test. +func (ctx *Context) Teardown() { + if !ctx.shouldPerformCleanup { + return + } for _, fn := range ctx.cleanupFuncs { err := fn() if err != nil { diff --git a/test/e2e/e2eutil.go b/test/e2e/e2eutil.go index bdac62c47..e1f8302cc 100644 --- a/test/e2e/e2eutil.go +++ b/test/e2e/e2eutil.go @@ -142,7 +142,7 @@ func waitForRuntimeObjectToExist(name string, retryInterval, timeout time.Durati }) } -func NewTestMongoDB(name string, namespace string) (mdbv1.MongoDBCommunity, mdbv1.MongoDBUser) { +func NewTestMongoDB(ctx *Context, name string, namespace string) (mdbv1.MongoDBCommunity, mdbv1.MongoDBUser) { mongodbNamespace := namespace if mongodbNamespace == "" { mongodbNamespace = OperatorNamespace @@ -167,7 +167,7 @@ func NewTestMongoDB(name string, namespace string) (mdbv1.MongoDBCommunity, mdbv DB: "admin", PasswordSecretRef: mdbv1.SecretKeyReference{ Key: fmt.Sprintf("%s-password", name), - Name: fmt.Sprintf("%s-password-secret", name), + Name: fmt.Sprintf("%s-%s-password-secret", name, ctx.ExecutionId), }, Roles: []mdbv1.Role{ // roles on testing db for general connectivity diff --git a/test/e2e/feature_compatibility_version/feature_compatibility_version_test.go b/test/e2e/feature_compatibility_version/feature_compatibility_version_test.go index 3ad948464..853e15f19 100644 --- a/test/e2e/feature_compatibility_version/feature_compatibility_version_test.go +++ b/test/e2e/feature_compatibility_version/feature_compatibility_version_test.go @@ -23,17 +23,14 @@ func TestMain(m *testing.M) { func TestFeatureCompatibilityVersion(t *testing.T) { - ctx, shouldCleanup := setup.InitTest(t) + ctx := setup.Setup(t) + defer ctx.Teardown() - if shouldCleanup { - defer ctx.Cleanup() - } - - mdb, user := e2eutil.NewTestMongoDB("mdb0", "") + mdb, user := e2eutil.NewTestMongoDB(ctx, "mdb0", "") mdb.Spec.Version = "4.0.6" mdb.Spec.FeatureCompatibilityVersion = "4.0" - _, err := setup.GeneratePasswordForUser(user, ctx, "") + _, err := setup.GeneratePasswordForUser(ctx, user, "") if err != nil { t.Fatal(err) } diff --git a/test/e2e/feature_compatibility_version_upgrade/feature_compatibility_version_upgrade_test.go b/test/e2e/feature_compatibility_version_upgrade/feature_compatibility_version_upgrade_test.go index fb3b011a6..1442c9be8 100644 --- a/test/e2e/feature_compatibility_version_upgrade/feature_compatibility_version_upgrade_test.go +++ b/test/e2e/feature_compatibility_version_upgrade/feature_compatibility_version_upgrade_test.go @@ -25,17 +25,14 @@ func TestMain(m *testing.M) { func TestFeatureCompatibilityVersionUpgrade(t *testing.T) { - ctx, shouldCleanup := setup.InitTest(t) + ctx := setup.Setup(t) + defer ctx.Teardown() - if shouldCleanup { - defer ctx.Cleanup() - } - - mdb, user := e2eutil.NewTestMongoDB("mdb0", "") + mdb, user := e2eutil.NewTestMongoDB(ctx, "mdb0", "") mdb.Spec.Version = "4.0.6" mdb.Spec.FeatureCompatibilityVersion = "4.0" - _, err := setup.GeneratePasswordForUser(user, ctx, "") + _, err := setup.GeneratePasswordForUser(ctx, user, "") if err != nil { t.Fatal(err) } diff --git a/test/e2e/replica_set/replica_set_test.go b/test/e2e/replica_set/replica_set_test.go index 19c33b41a..33f0301a6 100644 --- a/test/e2e/replica_set/replica_set_test.go +++ b/test/e2e/replica_set/replica_set_test.go @@ -21,14 +21,12 @@ func TestMain(m *testing.M) { } func TestReplicaSet(t *testing.T) { - ctx, shouldCleanup := setup.InitTest(t) + ctx := setup.Setup(t) + defer ctx.Teardown() - if shouldCleanup { - defer ctx.Cleanup() - } - mdb, user := e2eutil.NewTestMongoDB("mdb0", "") + mdb, user := e2eutil.NewTestMongoDB(ctx, "mdb0", "") - _, err := setup.GeneratePasswordForUser(user, ctx, "") + _, err := setup.GeneratePasswordForUser(ctx, user, "") if err != nil { t.Fatal(err) } diff --git a/test/e2e/replica_set_change_version/replica_set_test.go b/test/e2e/replica_set_change_version/replica_set_test.go index 1aa6f88d2..d94b4ec83 100644 --- a/test/e2e/replica_set_change_version/replica_set_test.go +++ b/test/e2e/replica_set_change_version/replica_set_test.go @@ -25,16 +25,13 @@ func TestMain(m *testing.M) { func TestReplicaSetUpgradeVersion(t *testing.T) { - ctx, shouldCleanup := setup.InitTest(t) + ctx := setup.Setup(t) + defer ctx.Teardown() - if shouldCleanup { - defer ctx.Cleanup() - } - - mdb, user := e2eutil.NewTestMongoDB("mdb0", "") + mdb, user := e2eutil.NewTestMongoDB(ctx, "mdb0", "") mdb.Spec.Version = "4.2.0" - _, err := setup.GeneratePasswordForUser(user, ctx, "") + _, err := setup.GeneratePasswordForUser(ctx, user, "") if err != nil { t.Fatal(err) } diff --git a/test/e2e/replica_set_cross_namespace_deploy/replica_set_cross_namespace_deploy_test.go b/test/e2e/replica_set_cross_namespace_deploy/replica_set_cross_namespace_deploy_test.go index 7f9c0e2f0..118cb83fd 100644 --- a/test/e2e/replica_set_cross_namespace_deploy/replica_set_cross_namespace_deploy_test.go +++ b/test/e2e/replica_set_cross_namespace_deploy/replica_set_cross_namespace_deploy_test.go @@ -24,11 +24,8 @@ func TestMain(m *testing.M) { } func TestCrossNamespaceDeploy(t *testing.T) { - ctx, shouldCleanup := setup.InitTest(t) - - if shouldCleanup { - defer ctx.Cleanup() - } + ctx := setup.Setup(t) + defer ctx.Teardown() postfix, err := generate.RandomValidDNS1123Label(5) if err != nil { @@ -149,9 +146,9 @@ func TestCrossNamespaceDeploy(t *testing.T) { t.Fatal(err) } - mdb, user := e2eutil.NewTestMongoDB("mdb0", namespace) + mdb, user := e2eutil.NewTestMongoDB(ctx, "mdb0", namespace) - _, err = setup.GeneratePasswordForUser(user, ctx, namespace) + _, err = setup.GeneratePasswordForUser(ctx, user, namespace) if err != nil { t.Fatal(err) } diff --git a/test/e2e/replica_set_custom_role/replica_set_custom_role_test.go b/test/e2e/replica_set_custom_role/replica_set_custom_role_test.go index af580400b..db120ba4c 100644 --- a/test/e2e/replica_set_custom_role/replica_set_custom_role_test.go +++ b/test/e2e/replica_set_custom_role/replica_set_custom_role_test.go @@ -22,18 +22,15 @@ func TestMain(m *testing.M) { } func TestReplicaSetCustomRole(t *testing.T) { - ctx, shouldCleanup := setup.InitTest(t) - - if shouldCleanup { - defer ctx.Cleanup() - } + ctx := setup.Setup(t) + defer ctx.Teardown() someDB := "test" someCollection := "foo" anyDB := "" anyCollection := "" - mdb, user := e2eutil.NewTestMongoDB("mdb0", "") + mdb, user := e2eutil.NewTestMongoDB(ctx, "mdb0", "") mdb.Spec.Security.Roles = []mdbv1.CustomRole{ { Role: "testRole", @@ -74,7 +71,7 @@ func TestReplicaSetCustomRole(t *testing.T) { }, } - _, err := setup.GeneratePasswordForUser(user, ctx, "") + _, err := setup.GeneratePasswordForUser(ctx, user, "") if err != nil { t.Fatal(err) } diff --git a/test/e2e/replica_set_mongod_config/replica_set_mongod_config_test.go b/test/e2e/replica_set_mongod_config/replica_set_mongod_config_test.go index da8855d73..7e7beeab8 100644 --- a/test/e2e/replica_set_mongod_config/replica_set_mongod_config_test.go +++ b/test/e2e/replica_set_mongod_config/replica_set_mongod_config_test.go @@ -22,14 +22,12 @@ func TestMain(m *testing.M) { } func TestReplicaSet(t *testing.T) { - ctx, shouldCleanup := setup.InitTest(t) + ctx := setup.Setup(t) + defer ctx.Teardown() - if shouldCleanup { - defer ctx.Cleanup() - } - mdb, user := e2eutil.NewTestMongoDB("mdb0", "") + mdb, user := e2eutil.NewTestMongoDB(ctx, "mdb0", "") - _, err := setup.GeneratePasswordForUser(user, ctx, "") + _, err := setup.GeneratePasswordForUser(ctx, user, "") if err != nil { t.Fatal(err) } diff --git a/test/e2e/replica_set_multiple/replica_set_multiple_test.go b/test/e2e/replica_set_multiple/replica_set_multiple_test.go index 4cade2f99..d3396eb07 100644 --- a/test/e2e/replica_set_multiple/replica_set_multiple_test.go +++ b/test/e2e/replica_set_multiple/replica_set_multiple_test.go @@ -26,21 +26,18 @@ func TestMain(m *testing.M) { // same time. One of them is scaled to 5 and then back to 3 func TestReplicaSetMultiple(t *testing.T) { - ctx, shouldCleanup := setup.InitTest(t) + ctx := setup.Setup(t) + defer ctx.Teardown() - if shouldCleanup { - defer ctx.Cleanup() - } - - mdb0, user0 := e2eutil.NewTestMongoDB("mdb0", "") - mdb1, user1 := e2eutil.NewTestMongoDB("mdb1", "") + mdb0, user0 := e2eutil.NewTestMongoDB(ctx, "mdb0", "") + mdb1, user1 := e2eutil.NewTestMongoDB(ctx, "mdb1", "") - _, err := setup.GeneratePasswordForUser(user0, ctx, "") + _, err := setup.GeneratePasswordForUser(ctx, user0, "") if err != nil { t.Fatal(err) } - _, err = setup.GeneratePasswordForUser(user1, ctx, "") + _, err = setup.GeneratePasswordForUser(ctx, user1, "") if err != nil { t.Fatal(err) } diff --git a/test/e2e/replica_set_readiness_probe/replica_set_readiness_probe_test.go b/test/e2e/replica_set_readiness_probe/replica_set_readiness_probe_test.go index 9e6f6d9c5..2568679e6 100644 --- a/test/e2e/replica_set_readiness_probe/replica_set_readiness_probe_test.go +++ b/test/e2e/replica_set_readiness_probe/replica_set_readiness_probe_test.go @@ -25,14 +25,11 @@ func TestMain(m *testing.M) { } func TestReplicaSetReadinessProbeScaling(t *testing.T) { - ctx, shouldCleanup := setup.InitTest(t) + ctx := setup.Setup(t) + defer ctx.Teardown() - if shouldCleanup { - defer ctx.Cleanup() - } - - mdb, user := e2eutil.NewTestMongoDB("mdb0", "") - _, err := setup.GeneratePasswordForUser(user, ctx, "") + mdb, user := e2eutil.NewTestMongoDB(ctx, "mdb0", "") + _, err := setup.GeneratePasswordForUser(ctx, user, "") if err != nil { t.Fatal(err) } diff --git a/test/e2e/replica_set_scale/replica_set_scaling_test.go b/test/e2e/replica_set_scale/replica_set_scaling_test.go index 4ff0907f7..19d5d535d 100644 --- a/test/e2e/replica_set_scale/replica_set_scaling_test.go +++ b/test/e2e/replica_set_scale/replica_set_scaling_test.go @@ -23,15 +23,12 @@ func TestMain(m *testing.M) { func TestReplicaSetScaleUp(t *testing.T) { - ctx, shouldCleanup := setup.InitTest(t) + ctx := setup.Setup(t) + defer ctx.Teardown() - if shouldCleanup { - defer ctx.Cleanup() - } - - mdb, user := e2eutil.NewTestMongoDB("mdb0", "") + mdb, user := e2eutil.NewTestMongoDB(ctx, "mdb0", "") - _, err := setup.GeneratePasswordForUser(user, ctx, "") + _, err := setup.GeneratePasswordForUser(ctx, user, "") if err != nil { t.Fatal(err) } diff --git a/test/e2e/replica_set_scale_down/replica_set_scale_down_test.go b/test/e2e/replica_set_scale_down/replica_set_scale_down_test.go index 306364e64..59cdbdfd4 100644 --- a/test/e2e/replica_set_scale_down/replica_set_scale_down_test.go +++ b/test/e2e/replica_set_scale_down/replica_set_scale_down_test.go @@ -24,14 +24,12 @@ func TestMain(m *testing.M) { } func TestReplicaSetScaleDown(t *testing.T) { - ctx, shouldCleanup := setup.InitTest(t) + ctx := setup.Setup(t) + defer ctx.Teardown() - if shouldCleanup { - defer ctx.Cleanup() - } - mdb, user := e2eutil.NewTestMongoDB("replica-set-scale-down", "") + mdb, user := e2eutil.NewTestMongoDB(ctx, "replica-set-scale-down", "") - _, err := setup.GeneratePasswordForUser(user, ctx, "") + _, err := setup.GeneratePasswordForUser(ctx, user, "") if err != nil { t.Fatal(err) } diff --git a/test/e2e/replica_set_tls/replica_set_tls_test.go b/test/e2e/replica_set_tls/replica_set_tls_test.go index 919cff5b8..289705758 100644 --- a/test/e2e/replica_set_tls/replica_set_tls_test.go +++ b/test/e2e/replica_set_tls/replica_set_tls_test.go @@ -21,15 +21,13 @@ func TestMain(m *testing.M) { } func TestReplicaSetTLS(t *testing.T) { - ctx, shouldCleanup := setup.InitTest(t) - if shouldCleanup { - defer ctx.Cleanup() - } + ctx := setup.Setup(t) + defer ctx.Teardown() - mdb, user := e2eutil.NewTestMongoDB("mdb-tls", "") + mdb, user := e2eutil.NewTestMongoDB(ctx, "mdb-tls", "") mdb.Spec.Security.TLS = e2eutil.NewTestTLSConfig(false) - _, err := setup.GeneratePasswordForUser(user, ctx, "") + _, err := setup.GeneratePasswordForUser(ctx, user, "") if err != nil { t.Fatal(err) } diff --git a/test/e2e/replica_set_tls_rotate/replica_set_tls_rotate_test.go b/test/e2e/replica_set_tls_rotate/replica_set_tls_rotate_test.go index 26a75547c..07f496cc8 100644 --- a/test/e2e/replica_set_tls_rotate/replica_set_tls_rotate_test.go +++ b/test/e2e/replica_set_tls_rotate/replica_set_tls_rotate_test.go @@ -23,15 +23,13 @@ func TestMain(m *testing.M) { } func TestReplicaSetTLSRotate(t *testing.T) { - ctx, shouldCleanup := setup.InitTest(t) - if shouldCleanup { - defer ctx.Cleanup() - } + ctx := setup.Setup(t) + defer ctx.Teardown() - mdb, user := e2eutil.NewTestMongoDB("mdb-tls", "") + mdb, user := e2eutil.NewTestMongoDB(ctx, "mdb-tls", "") mdb.Spec.Security.TLS = e2eutil.NewTestTLSConfig(false) - _, err := setup.GeneratePasswordForUser(user, ctx, "") + _, err := setup.GeneratePasswordForUser(ctx, user, "") if err != nil { t.Fatal(err) } diff --git a/test/e2e/replica_set_tls_upgrade/replica_set_tls_upgrade_test.go b/test/e2e/replica_set_tls_upgrade/replica_set_tls_upgrade_test.go index 6942d28c0..8ed5ba7ee 100644 --- a/test/e2e/replica_set_tls_upgrade/replica_set_tls_upgrade_test.go +++ b/test/e2e/replica_set_tls_upgrade/replica_set_tls_upgrade_test.go @@ -24,13 +24,11 @@ func TestMain(m *testing.M) { } func TestReplicaSetTLSUpgrade(t *testing.T) { - ctx, shouldCleanup := setup.InitTest(t) - if shouldCleanup { - defer ctx.Cleanup() - } + ctx := setup.Setup(t) + defer ctx.Teardown() - mdb, user := e2eutil.NewTestMongoDB("mdb-tls", "") - _, err := setup.GeneratePasswordForUser(user, ctx, "") + mdb, user := e2eutil.NewTestMongoDB(ctx, "mdb-tls", "") + _, err := setup.GeneratePasswordForUser(ctx, user, "") if err != nil { t.Fatal(err) } diff --git a/test/e2e/setup/setup.go b/test/e2e/setup/setup.go index d7de61d25..631ae6de5 100644 --- a/test/e2e/setup/setup.go +++ b/test/e2e/setup/setup.go @@ -35,16 +35,18 @@ const ( roleDir = "/workspace/config/rbac" ) -func InitTest(t *testing.T) (*e2eutil.Context, bool) { - ctx := e2eutil.NewContext(t) +func Setup(t *testing.T) *e2eutil.Context { + ctx, err := e2eutil.NewContext(t, os.Getenv(performCleanup) == "True") - if err := deployOperator(); err != nil { + if err != nil { t.Fatal(err) } - clean := os.Getenv(performCleanup) + if err := deployOperator(); err != nil { + t.Fatal(err) + } - return ctx, clean == "True" + return ctx } // CreateTLSResources will setup the CA ConfigMap and cert-key Secret necessary for TLS @@ -90,7 +92,7 @@ func CreateTLSResources(namespace string, ctx *e2eutil.Context) error { //nolint } // GeneratePasswordForUser will create a secret with a password for the given user -func GeneratePasswordForUser(mdbu mdbv1.MongoDBUser, ctx *e2eutil.Context, namespace string) (string, error) { +func GeneratePasswordForUser(ctx *e2eutil.Context, mdbu mdbv1.MongoDBUser, namespace string) (string, error) { passwordKey := mdbu.PasswordSecretRef.Key if passwordKey == "" { passwordKey = "password" diff --git a/test/e2e/statefulset_arbitrary_config/statefulset_arbitrary_config_test.go b/test/e2e/statefulset_arbitrary_config/statefulset_arbitrary_config_test.go index dad4990cb..8372c23c9 100644 --- a/test/e2e/statefulset_arbitrary_config/statefulset_arbitrary_config_test.go +++ b/test/e2e/statefulset_arbitrary_config/statefulset_arbitrary_config_test.go @@ -23,14 +23,12 @@ func TestMain(m *testing.M) { } func TestStatefulSetArbitraryConfig(t *testing.T) { - ctx, shouldCleanup := setup.InitTest(t) + ctx := setup.Setup(t) + defer ctx.Teardown() - if shouldCleanup { - defer ctx.Cleanup() - } - mdb, user := e2eutil.NewTestMongoDB("mdb0", "") + mdb, user := e2eutil.NewTestMongoDB(ctx, "mdb0", "") - _, err := setup.GeneratePasswordForUser(user, ctx, "") + _, err := setup.GeneratePasswordForUser(ctx, user, "") if err != nil { t.Fatal(err) } diff --git a/test/e2e/statefulset_arbitrary_config_update/statefulset_arbitrary_config_update_test.go b/test/e2e/statefulset_arbitrary_config_update/statefulset_arbitrary_config_update_test.go index 5263ce1ce..89711e9e1 100644 --- a/test/e2e/statefulset_arbitrary_config_update/statefulset_arbitrary_config_update_test.go +++ b/test/e2e/statefulset_arbitrary_config_update/statefulset_arbitrary_config_update_test.go @@ -24,14 +24,12 @@ func TestMain(m *testing.M) { } func TestStatefulSetArbitraryConfig(t *testing.T) { - ctx, shouldCleanup := setup.InitTest(t) + ctx := setup.Setup(t) + defer ctx.Teardown() - if shouldCleanup { - defer ctx.Cleanup() - } - mdb, user := e2eutil.NewTestMongoDB("mdb0", "") + mdb, user := e2eutil.NewTestMongoDB(ctx, "mdb0", "") - _, err := setup.GeneratePasswordForUser(user, ctx, "") + _, err := setup.GeneratePasswordForUser(ctx, user, "") if err != nil { t.Fatal(err) } From b40661e04983abe2bf34cb590e658d7f9167cfe7 Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Wed, 26 May 2021 12:33:05 +0100 Subject: [PATCH 274/790] Run e2e locally (#506) --- Makefile | 9 ++++- docs/contributing.md | 1 - scripts/dev/dev_config.py | 19 +++++++++++ scripts/dev/get_e2e_env_vars.py | 42 ++++++++++++++++++++++++ test/e2e/e2eutil.go | 12 ++++++- test/e2e/setup/setup.go | 35 +++++++++++++------- test/e2e/tlstests/tlstests.go | 5 +-- test/e2e/util/mongotester/mongotester.go | 3 +- 8 files changed, 108 insertions(+), 18 deletions(-) create mode 100755 scripts/dev/get_e2e_env_vars.py diff --git a/Makefile b/Makefile index aa30e35c2..ea93b737f 100644 --- a/Makefile +++ b/Makefile @@ -99,9 +99,16 @@ fmt: vet: go vet ./... -e2e: install +# Run e2e test by deploying test image in kubernetes. +e2e-k8s: install e2e-image python scripts/dev/e2e.py --perform-cleanup --test $(test) +# Run e2e test locally. +# e.g. make e2e test=replica_set cleanup=true +e2e: install + eval $$(scripts/dev/get_e2e_env_vars.py $(cleanup)); \ + go test -v -timeout=30m -failfast ./test/e2e/$(test) + # Generate code generate: controller-gen $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." diff --git a/docs/contributing.md b/docs/contributing.md index a432b7fe5..2519e1644 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -53,7 +53,6 @@ to be able to run properly. Create a json file with the following content: "e2e_image": "community-e2e", "version_upgrade_hook_image": "community-operator-version-upgrade-post-start-hook", "prestop_hook_image": "prehook", - "testrunner_image": "test-runner", "agent_image_ubuntu": "mongodb-agent-dev", "agent_image_ubi": "mongodb-agent-ubi-dev", "readiness_probe_image": "mongodb-kubernetes-readiness" diff --git a/scripts/dev/dev_config.py b/scripts/dev/dev_config.py index 6389337b1..f1ad46cff 100644 --- a/scripts/dev/dev_config.py +++ b/scripts/dev/dev_config.py @@ -74,6 +74,25 @@ def version_upgrade_hook_image(self) -> str: def readiness_probe_image(self) -> str: return self._config["readiness_probe_image"] + # these directories are used from within the E2E tests when running locally. + @property + def role_dir(self) -> str: + if "role_dir" in self._config: + return self._config["role_dir"] + return os.path.join(os.getcwd(), "config", "rbac") + + @property + def deploy_dir(self) -> str: + if "deploy_dir" in self._config: + return self._config["deploy_dir"] + return os.path.join(os.getcwd(), "config", "manager") + + @property + def test_data_dir(self) -> str: + if "test_data_dir" in self._config: + return self._config["test_data_dir"] + return os.path.join(os.getcwd(), "testdata") + @property def agent_image(self) -> str: if self._distro == Distro.UBI: diff --git a/scripts/dev/get_e2e_env_vars.py b/scripts/dev/get_e2e_env_vars.py new file mode 100755 index 000000000..ca2db6588 --- /dev/null +++ b/scripts/dev/get_e2e_env_vars.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python +import sys +from typing import Dict + +from dev_config import load_config, DevConfig + + +def _get_e2e_test_envs(dev_config: DevConfig) -> Dict[str, str]: + """ + _get_e2e_test_envs returns a dictionary of all the required environment variables + that need to be set in order to run a local e2e test. + + :param dev_config: The local dev config + :return: A diction of env vars to be set + """ + cleanup = False + if len(sys.argv) > 1: + cleanup = sys.argv[1] == "true" + return { + "ROLE_DIR": dev_config.role_dir, + "DEPLOY_DIR": dev_config.deploy_dir, + "OPERATOR_IMAGE": f"{dev_config.repo_url}/{dev_config.operator_image}", + "VERSION_UPGRADE_HOOK_IMAGE": f"{dev_config.repo_url}/{dev_config.version_upgrade_hook_image}", + "AGENT_IMAGE": f"{dev_config.repo_url}/{dev_config.agent_image}", + "TEST_DATA_DIR": dev_config.test_data_dir, + "TEST_NAMESPACE": dev_config.namespace, + "READINESS_PROBE_IMAGE": f"{dev_config.repo_url}/{dev_config.readiness_probe_image}", + "PERFORM_CLEANUP": "true" if cleanup else "false", + } + + +# convert all values in config.json to env vars. +# this can be used to provide configuration for e2e tests. +def main() -> int: + dev_config = load_config() + for k, v in _get_e2e_test_envs(dev_config).items(): + print(f"export {k.upper()}={v}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/e2e/e2eutil.go b/test/e2e/e2eutil.go index e1f8302cc..dce1679ef 100644 --- a/test/e2e/e2eutil.go +++ b/test/e2e/e2eutil.go @@ -7,6 +7,8 @@ import ( "testing" "time" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/envvar" + "github.com/pkg/errors" mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" @@ -22,7 +24,15 @@ import ( k8sClient "sigs.k8s.io/controller-runtime/pkg/client" ) -const TestdataDir = "/workspace/testdata/tls" +const testDataDirEnv = "TEST_DATA_DIR" + +func TestDataDir() string { + return envvar.GetEnvOrDefault(testDataDirEnv, "/workspace/testdata") +} + +func TlsTestDataDir() string { + return fmt.Sprintf("%s/tls", TestDataDir()) +} // UpdateMongoDBResource applies the provided function to the most recent version of the MongoDB resource // and retries when there are conflicts diff --git a/test/e2e/setup/setup.go b/test/e2e/setup/setup.go index 631ae6de5..96e08f373 100644 --- a/test/e2e/setup/setup.go +++ b/test/e2e/setup/setup.go @@ -5,10 +5,11 @@ import ( "encoding/json" "fmt" "io/ioutil" - "os" "path" "testing" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/envvar" + "github.com/mongodb/mongodb-kubernetes-operator/controllers/construct" "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/generate" "github.com/pkg/errors" @@ -30,13 +31,13 @@ import ( ) const ( - performCleanup = "PERFORM_CLEANUP" - deployDir = "/workspace/config/manager" - roleDir = "/workspace/config/rbac" + performCleanupEnv = "PERFORM_CLEANUP" + deployDirEnv = "DEPLOY_DIR" + roleDirEnv = "ROLE_DIR" ) func Setup(t *testing.T) *e2eutil.Context { - ctx, err := e2eutil.NewContext(t, os.Getenv(performCleanup) == "True") + ctx, err := e2eutil.NewContext(t, envvar.ReadBool(performCleanupEnv)) if err != nil { t.Fatal(err) @@ -55,7 +56,8 @@ func CreateTLSResources(namespace string, ctx *e2eutil.Context) error { //nolint tlsConfig := e2eutil.NewTestTLSConfig(false) // Create CA ConfigMap - ca, err := ioutil.ReadFile(path.Join(e2eutil.TestdataDir, "ca.crt")) + testDataDir := e2eutil.TlsTestDataDir() + ca, err := ioutil.ReadFile(path.Join(testDataDir, "ca.crt")) if err != nil { return nil } @@ -72,11 +74,11 @@ func CreateTLSResources(namespace string, ctx *e2eutil.Context) error { //nolint } // Create server key and certificate secret - cert, err := ioutil.ReadFile(path.Join(e2eutil.TestdataDir, "server.crt")) + cert, err := ioutil.ReadFile(path.Join(testDataDir, "server.crt")) if err != nil { return err } - key, err := ioutil.ReadFile(path.Join(e2eutil.TestdataDir, "server.key")) + key, err := ioutil.ReadFile(path.Join(testDataDir, "server.key")) if err != nil { return err } @@ -117,6 +119,14 @@ func GeneratePasswordForUser(ctx *e2eutil.Context, mdbu mdbv1.MongoDBUser, names return password, e2eutil.TestClient.Create(context.TODO(), &passwordSecret, &e2eutil.CleanupOptions{TestContext: ctx}) } +func roleDir() string { + return envvar.GetEnvOrDefault(roleDirEnv, "/workspace/config/rbac") +} + +func deployDir() string { + return envvar.GetEnvOrDefault(deployDirEnv, "/workspace/config/manager") +} + func deployOperator() error { testConfig := loadTestConfigFromEnv() @@ -128,21 +138,22 @@ func deployOperator() error { } fmt.Printf("Setting namespace to watch to %s\n", watchNamespace) - if err := buildKubernetesResourceFromYamlFile(path.Join(roleDir, "role.yaml"), &rbacv1.Role{}, withNamespace(testConfig.namespace)); err != nil { + if err := buildKubernetesResourceFromYamlFile(path.Join(roleDir(), "role.yaml"), &rbacv1.Role{}, withNamespace(testConfig.namespace)); err != nil { return errors.Errorf("error building operator role: %s", err) } fmt.Println("Successfully created the operator Role") - if err := buildKubernetesResourceFromYamlFile(path.Join(roleDir, "service_account.yaml"), &corev1.ServiceAccount{}, withNamespace(testConfig.namespace)); err != nil { + if err := buildKubernetesResourceFromYamlFile(path.Join(roleDir(), "service_account.yaml"), &corev1.ServiceAccount{}, withNamespace(testConfig.namespace)); err != nil { return errors.Errorf("error building operator service account: %s", err) } fmt.Println("Successfully created the operator Service Account") - if err := buildKubernetesResourceFromYamlFile(path.Join(roleDir, "role_binding.yaml"), &rbacv1.RoleBinding{}, withNamespace(testConfig.namespace)); err != nil { + if err := buildKubernetesResourceFromYamlFile(path.Join(roleDir(), "role_binding.yaml"), &rbacv1.RoleBinding{}, withNamespace(testConfig.namespace)); err != nil { return errors.Errorf("error building operator role binding: %s", err) } + fmt.Println("Successfully created the operator Role Binding") - if err := buildKubernetesResourceFromYamlFile(path.Join(deployDir, "manager.yaml"), + if err := buildKubernetesResourceFromYamlFile(path.Join(deployDir(), "manager.yaml"), &appsv1.Deployment{}, withNamespace(testConfig.namespace), withOperatorImage(testConfig.operatorImage), diff --git a/test/e2e/tlstests/tlstests.go b/test/e2e/tlstests/tlstests.go index cd61da167..ee0f26839 100644 --- a/test/e2e/tlstests/tlstests.go +++ b/test/e2e/tlstests/tlstests.go @@ -28,9 +28,10 @@ func EnableTLS(mdb *v1.MongoDBCommunity, optional bool) func(*testing.T) { func RotateCertificate(mdb *v1.MongoDBCommunity) func(*testing.T) { return func(t *testing.T) { // Load new certificate and key - cert, err := ioutil.ReadFile(path.Join(e2eutil.TestdataDir, "server_rotated.crt")) + testDataDir := e2eutil.TlsTestDataDir() + cert, err := ioutil.ReadFile(path.Join(testDataDir, "server_rotated.crt")) assert.NoError(t, err) - key, err := ioutil.ReadFile(path.Join(e2eutil.TestdataDir, "server_rotated.key")) + key, err := ioutil.ReadFile(path.Join(testDataDir, "server_rotated.key")) assert.NoError(t, err) certKeySecret := secret.Builder(). diff --git a/test/e2e/util/mongotester/mongotester.go b/test/e2e/util/mongotester/mongotester.go index b54152fe7..202954745 100644 --- a/test/e2e/util/mongotester/mongotester.go +++ b/test/e2e/util/mongotester/mongotester.go @@ -432,7 +432,8 @@ func WithoutTls() OptionApplier { // getClientTLSConfig reads in the tls fixtures func getClientTLSConfig() (*tls.Config, error) { // Read the CA certificate from test data - caPEM, err := ioutil.ReadFile(path.Join(e2eutil.TestdataDir, "ca.crt")) + testDataDir := e2eutil.TlsTestDataDir() + caPEM, err := ioutil.ReadFile(path.Join(testDataDir, "ca.crt")) if err != nil { return nil, err } From 252559ee1b05fa1b2fe20bee954db4a345d690fe Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Wed, 26 May 2021 19:05:05 +0100 Subject: [PATCH 275/790] Ensure Inventory files use images specified in config.json (#504) --- inventories/e2e-inventory.yaml | 7 +-- inventories/operator-inventory.yaml | 7 ++- inventory.yaml | 70 +++++++++++++++++------------ pipeline.py | 11 +++++ scripts/ci/config.json | 16 ++++--- scripts/dev/dev_config.py | 39 +++++++++++++++- scripts/dev/e2e.py | 6 +-- 7 files changed, 111 insertions(+), 45 deletions(-) diff --git a/inventories/e2e-inventory.yaml b/inventories/e2e-inventory.yaml index f444c2117..c8a77b8e9 100644 --- a/inventories/e2e-inventory.yaml +++ b/inventories/e2e-inventory.yaml @@ -6,7 +6,8 @@ images: vars: context: . template_context: scripts/dev/templates - + inputs: + - e2e_image stages: - name: e2e-template task_type: dockerfile_template @@ -28,8 +29,8 @@ images: quay.expires-after: 48h output: - - registry: $(inputs.params.registry)/community-operator-e2e + - registry: $(inputs.params.registry)/$(inputs.params.e2e_image) tag: $(inputs.params.version_id) - - registry: $(inputs.params.registry)/community-operator-e2e + - registry: $(inputs.params.registry)/$(inputs.params.e2e_image) tag: latest diff --git a/inventories/operator-inventory.yaml b/inventories/operator-inventory.yaml index 680c593c5..142e69629 100644 --- a/inventories/operator-inventory.yaml +++ b/inventories/operator-inventory.yaml @@ -7,6 +7,9 @@ images: context: . template_context: scripts/dev/templates + inputs: + - operator_image_dev + stages: - name: operator-template-ubi task_type: dockerfile_template @@ -28,8 +31,8 @@ images: dockerfile: scripts/dev/templates/Dockerfile.ubi-$(inputs.params.version_id) output: - - registry: $(inputs.params.registry)/community-operator-dev + - registry: $(inputs.params.registry)/$(inputs.params.operator_image_dev) tag: $(inputs.params.version_id) - - registry: $(inputs.params.registry)/community-operator-dev + - registry: $(inputs.params.registry)/$(inputs.params.operator_image_dev) tag: latest diff --git a/inventory.yaml b/inventory.yaml index c8d842a73..713f1e10b 100644 --- a/inventory.yaml +++ b/inventory.yaml @@ -10,6 +10,8 @@ images: inputs: - agent_version - tools_version + - agent_image + - agent_image_dev stages: - name: agent-ubuntu-context @@ -26,7 +28,7 @@ images: quay.expires-after: 48h output: - - registry: $(inputs.params.registry)/mongodb-agent-dev + - registry: $(inputs.params.registry)/$(inputs.params.agent_image_dev) tag: $(inputs.params.version_id)-context - name: agent-template-ubuntu @@ -44,16 +46,16 @@ images: dockerfile: scripts/dev/templates/agent/Dockerfile.ubuntu-$(inputs.params.version_id) buildargs: - imagebase: $(inputs.params.registry)/mongodb-agent-dev:$(inputs.params.version_id)-context + imagebase: $(inputs.params.registry)/$(inputs.params.agent_image_dev):$(inputs.params.version_id)-context agent_version: $(inputs.params.agent_version) labels: quay.expires-after: 48h output: - - registry: $(inputs.params.registry)/mongodb-agent-dev + - registry: $(inputs.params.registry)/$(inputs.params.agent_image_dev) tag: $(inputs.params.version_id) - - registry: $(inputs.params.registry)/mongodb-agent-dev + - registry: $(inputs.params.registry)/$(inputs.params.agent_image_dev) tag: latest - name: agent-template-ubuntu-s3 @@ -82,7 +84,7 @@ images: quay.expires-after: Never output: - - registry: $(inputs.params.registry)/mongodb-agent + - registry: $(inputs.params.registry)/$(inputs.params.agent_image) tag: $(inputs.params.version_id)-context - name: agent-ubuntu-release @@ -93,14 +95,14 @@ images: dockerfile: scripts/dev/templates/agent/Dockerfile.ubuntu-$(inputs.params.version_id) buildargs: - imagebase: $(inputs.params.registry)/mongodb-agent:$(inputs.params.version_id)-context + imagebase: $(inputs.params.registry)/$(inputs.params.agent_image):$(inputs.params.version_id)-context agent_version: $(inputs.params.agent_version) labels: quay.expires-after: Never output: - - registry: $(inputs.params.registry)/mongodb-agent + - registry: $(inputs.params.registry)/$(inputs.params.agent_image) tag: $(inputs.params.version_id) - name: agent-ubi @@ -111,6 +113,8 @@ images: inputs: - agent_version - tools_version + - agent_image + - agent_image_dev stages: - name: agent-ubi-context @@ -127,7 +131,7 @@ images: quay.expires-after: 48h output: - - registry: $(inputs.params.registry)/mongodb-agent-ubi-dev + - registry: $(inputs.params.registry)/$(inputs.params.agent_image_dev) tag: $(inputs.params.version_id)-context @@ -143,19 +147,20 @@ images: - name: agent-ubi-build task_type: docker_build tags: ["ubi"] + dockerfile: scripts/dev/templates/agent/Dockerfile.ubi-$(inputs.params.version_id) buildargs: - imagebase: $(inputs.params.registry)/mongodb-agent-ubi-dev:$(inputs.params.version_id)-context + imagebase: $(inputs.params.registry)/$(inputs.params.agent_image_dev):$(inputs.params.version_id)-context agent_version: $(inputs.params.agent_version) labels: quay.expires-after: 48h output: - - registry: $(inputs.params.registry)/mongodb-agent-ubi-dev + - registry: $(inputs.params.registry)/$(inputs.params.agent_image_dev) tag: $(inputs.params.version_id) - - registry: $(inputs.params.registry)/mongodb-agent-ubi-dev + - registry: $(inputs.params.registry)/$(inputs.params.agent_image_dev) tag: latest - name: agent-template-ubi-s3 @@ -183,7 +188,7 @@ images: quay.expires-after: Never output: - - registry: $(inputs.params.registry)/mongodb-agent-ubi + - registry: $(inputs.params.registry)/$(inputs.params.agent_image) tag: $(inputs.params.version_id)-context - name: agent-ubi-release @@ -192,31 +197,34 @@ images: dockerfile: scripts/dev/templates/agent/Dockerfile.ubi-$(inputs.params.version_id) buildargs: - imagebase: $(inputs.params.registry)/mongodb-agent-ubi:$(inputs.params.version_id)-context + imagebase: $(inputs.params.registry)/$(inputs.params.agent_image):$(inputs.params.version_id)-context agent_version: $(inputs.params.agent_version) labels: quay.expires-after: Never output: - - registry: $(inputs.params.registry)/mongodb-agent-ubi + - registry: $(inputs.params.registry)/$(inputs.params.agent_image) tag: $(inputs.params.version_id) - name: readiness-probe-init vars: context: . + inputs: + - readiness_probe_image + - readiness_probe_image_dev + stages: - name: readiness-init-context-build task_type: docker_build dockerfile: scripts/dev/templates/readiness/Dockerfile.builder tags: ["readiness-probe"] - labels: quay.expires-after: 48h output: - - registry: $(inputs.params.registry)/mongodb-kubernetes-readinessprobe-dev + - registry: $(inputs.params.registry)/$(inputs.params.readiness_probe_image_dev) tag: $(inputs.params.version_id)-context - name: readiness-init-build @@ -228,12 +236,12 @@ images: tags: ["readiness-probe"] buildargs: - imagebase: $(inputs.params.registry)/mongodb-kubernetes-readinessprobe-dev:$(inputs.params.version_id)-context + imagebase: $(inputs.params.registry)/$(inputs.params.readiness_probe_image_dev):$(inputs.params.version_id)-context output: - - registry: $(inputs.params.registry)/mongodb-kubernetes-readinessprobe-dev + - registry: $(inputs.params.registry)/$(inputs.params.readiness_probe_image_dev) tag: $(inputs.params.version_id) - - registry: $(inputs.params.registry)/mongodb-kubernetes-readinessprobe-dev + - registry: $(inputs.params.registry)/$(inputs.params.readiness_probe_image_dev) tag: latest @@ -249,7 +257,7 @@ images: - release_version output: - - registry: $(inputs.params.registry)/mongodb-kubernetes-readinessprobe + - registry: $(inputs.params.registry)/(inputs.params.readiness_probe_image) tag: $(inputs.params.release_version)-context @@ -259,7 +267,7 @@ images: tags: ["readiness-probe", "release"] buildargs: - imagebase: $(inputs.params.registry)/mongodb-kubernetes-readinessprobe:$(inputs.params.release_version)-context + imagebase: $(inputs.params.registry)/$(inputs.params.readiness_probe_image):$(inputs.params.release_version)-context labels: quay.expires-after: Never @@ -268,7 +276,7 @@ images: - release_version output: - - registry: $(inputs.params.registry)/mongodb-kubernetes-readinessprobe + - registry: $(inputs.params.registry)/$(inputs.params.readiness_probe_image) tag: $(inputs.params.release_version) @@ -276,6 +284,10 @@ images: vars: context: . + inputs: + - version_post_start_hook_image + - version_post_start_hook_image_dev + stages: - name: version-post-start-hook-init-context-build task_type: docker_build @@ -286,7 +298,7 @@ images: quay.expires-after: 48h output: - - registry: $(inputs.params.registry)/mongodb-kubernetes-operator-version-upgrade-post-start-hook-dev + - registry: $(inputs.params.registry)/$(inputs.params.version_post_start_hook_image_dev) tag: $(inputs.params.version_id)-context - name: version-post-start-hook-init-build @@ -294,15 +306,15 @@ images: dockerfile: scripts/dev/templates/versionhook/Dockerfile.versionhook tags: ["post-start-hook"] buildargs: - imagebase: $(inputs.params.registry)/mongodb-kubernetes-operator-version-upgrade-post-start-hook-dev:$(inputs.params.version_id)-context + imagebase: $(inputs.params.registry)/$(inputs.params.version_post_start_hook_image_dev):$(inputs.params.version_id)-context labels: quay.expires-after: 48h output: - - registry: $(inputs.params.registry)/mongodb-kubernetes-operator-version-upgrade-post-start-hook-dev + - registry: $(inputs.params.registry)/$(inputs.params.version_post_start_hook_image_dev) tag: $(inputs.params.version_id) - - registry: $(inputs.params.registry)/mongodb-kubernetes-operator-version-upgrade-post-start-hook-dev + - registry: $(inputs.params.registry)/$(inputs.params.version_post_start_hook_image_dev) tag: latest @@ -318,7 +330,7 @@ images: - release_version output: - - registry: $(inputs.params.registry)/mongodb-kubernetes-operator-version-upgrade-post-start-hook + - registry: $(inputs.params.registry)/$(inputs.params.version_post_start_hook_image) tag: $(inputs.params.release_version)-context @@ -327,7 +339,7 @@ images: dockerfile: scripts/dev/templates/versionhook/Dockerfile.versionhook tags: ["release", "post-start-hook"] buildargs: - imagebase: $(inputs.params.registry)/mongodb-kubernetes-operator-version-upgrade-post-start-hook:$(inputs.params.release_version)-context + imagebase: $(inputs.params.registry)/$(inputs.params.version_post_start_hook_image):$(inputs.params.release_version)-context labels: quay.expires-after: Never @@ -336,5 +348,5 @@ images: - release_version output: - - registry: $(inputs.params.registry)/mongodb-kubernetes-operator-version-upgrade-post-start-hook + - registry: $(inputs.params.registry)/$(inputs.params.version_post_start_hook_image) tag: $(inputs.params.release_version) diff --git a/pipeline.py b/pipeline.py index d152e5ab6..18f1e1056 100644 --- a/pipeline.py +++ b/pipeline.py @@ -43,6 +43,8 @@ def _build_agent_args(config: DevConfig) -> Dict[str, str]: def build_agent_image_ubi(config: DevConfig) -> None: image_name = "agent-ubi" args = _build_agent_args(config) + args["agent_image"] = config.agent_image_ubi + args["agent_image_dev"] = config.agent_dev_image_ubi config.ensure_tag_is_run("ubi") sonar_build_image( @@ -55,6 +57,8 @@ def build_agent_image_ubi(config: DevConfig) -> None: def build_agent_image_ubuntu(config: DevConfig) -> None: image_name = "agent-ubuntu" args = _build_agent_args(config) + args["agent_image"] = config.agent_image_ubuntu + args["agent_image_dev"] = config.agent_dev_image_ubuntu config.ensure_tag_is_run("ubuntu") sonar_build_image( @@ -74,6 +78,8 @@ def build_readiness_probe_image(config: DevConfig) -> None: args={ "registry": config.repo_url, "release_version": release["readiness-probe"], + "readiness_probe_image": config.readiness_probe_image, + "readiness_probe_image_dev": config.readiness_probe_image_dev, }, ) @@ -88,6 +94,8 @@ def build_version_post_start_hook_image(config: DevConfig) -> None: args={ "registry": config.repo_url, "release_version": release["version-upgrade-hook"], + "version_post_start_hook_image": config.version_upgrade_hook_image, + "version_post_start_hook_image_dev": config.version_upgrade_hook_image_dev, }, ) @@ -102,6 +110,8 @@ def build_operator_ubi_image(config: DevConfig) -> None: "builder": "true", "builder_image": f"golang:{GOLANG_TAG}", "base_image": "registry.access.redhat.com/ubi8/ubi-minimal:latest", + "operator_image": config.operator_image, + "operator_image_dev": config.operator_image_dev, }, inventory="inventories/operator-inventory.yaml", ) @@ -114,6 +124,7 @@ def build_e2e_image(config: DevConfig) -> None: args={ "registry": config.repo_url, "base_image": f"golang:{GOLANG_TAG}", + "e2e_image": config.e2e_image, }, inventory="inventories/e2e-inventory.yaml", ) diff --git a/scripts/ci/config.json b/scripts/ci/config.json index 92c0e18e4..9ca65ed1c 100644 --- a/scripts/ci/config.json +++ b/scripts/ci/config.json @@ -1,12 +1,16 @@ { "namespace": "default", "repo_url": "quay.io/mongodb", - "operator_image": "community-operator-dev", + "operator_image": "mongodb-kubernetes-operator", + "operator_image_dev": "community-operator-dev", "e2e_image": "community-operator-e2e", - "version_upgrade_hook_image": "mongodb-kubernetes-operator-version-upgrade-post-start-hook-dev", - "testrunner_image": "community-operator-testrunner", - "agent_image_ubuntu": "mongodb-agent-dev", - "agent_image_ubi": "mongodb-agent-ubi-dev", - "readiness_probe_image": "mongodb-kubernetes-readinessprobe-dev", + "version_upgrade_hook_image": "mongodb-kubernetes-operator-version-upgrade-post-start-hook", + "version_upgrade_hook_image_dev": "mongodb-kubernetes-operator-version-upgrade-post-start-hook-dev", + "agent_image_ubuntu": "mongodb-agent", + "agent_image_ubuntu_dev": "mongodb-agent-dev", + "agent_image_ubi": "mongodb-agent-ubi", + "agent_image_ubi_dev": "mongodb-agent-ubi-dev", + "readiness_probe_image": "mongodb-kubernetes-readinessprobe", + "readiness_probe_image_dev": "mongodb-kubernetes-readinessprobe-dev", "s3_bucket": "s3://enterprise-operator-dockerfiles/dockerfiles" } diff --git a/scripts/dev/dev_config.py b/scripts/dev/dev_config.py index f1ad46cff..2d011ed1b 100644 --- a/scripts/dev/dev_config.py +++ b/scripts/dev/dev_config.py @@ -62,6 +62,10 @@ def expire_after(self) -> str: def operator_image(self) -> str: return self._config["operator_image"] + @property + def operator_image_dev(self) -> str: + return self._get_dev_image("operator_image_dev", "operator_image") + @property def e2e_image(self) -> str: return self._config["e2e_image"] @@ -70,6 +74,12 @@ def e2e_image(self) -> str: def version_upgrade_hook_image(self) -> str: return self._config["version_upgrade_hook_image"] + @property + def version_upgrade_hook_image_dev(self) -> str: + return self._get_dev_image( + "version_upgrade_hook_image_dev", "version_upgrade_hook_image" + ) + @property def readiness_probe_image(self) -> str: return self._config["readiness_probe_image"] @@ -93,16 +103,41 @@ def test_data_dir(self) -> str: return self._config["test_data_dir"] return os.path.join(os.getcwd(), "testdata") + @property + def readiness_probe_image_dev(self) -> str: + return self._get_dev_image("readiness_probe_image_dev", "readiness_probe_image") + + @property + def agent_dev_image_ubi(self) -> str: + return self._get_dev_image("agent_image_ubi_dev", "agent_image_ubi") + + @property + def agent_dev_image_ubuntu(self) -> str: + return self._get_dev_image("agent_image_ubuntu_dev", "agent_image_ubuntu") + + @property + def agent_image_ubuntu(self) -> str: + return self._config["agent_image_ubuntu"] + + @property + def agent_image_ubi(self) -> str: + return self._config["agent_image_ubi"] + @property def agent_image(self) -> str: if self._distro == Distro.UBI: - return self._config["agent_image_ubi"] - return self._config["agent_image_ubuntu"] + return self.agent_dev_image_ubi + return self.agent_dev_image_ubuntu def ensure_skip_tag(self, tag: str) -> None: if tag not in self.skip_tags: self.skip_tags.append(tag) + def _get_dev_image(self, dev_image: str, image: str) -> str: + if dev_image in self._config: + return self._config[dev_image] + return self._config[image] + def load_config( config_file_path: Optional[str] = None, distro: Distro = Distro.UBUNTU diff --git a/scripts/dev/e2e.py b/scripts/dev/e2e.py index ede500e2f..3b6b26307 100644 --- a/scripts/dev/e2e.py +++ b/scripts/dev/e2e.py @@ -130,7 +130,7 @@ def create_test_pod(args: argparse.Namespace, dev_config: DevConfig) -> None: }, { "name": "OPERATOR_IMAGE", - "value": f"{dev_config.repo_url}/{dev_config.operator_image}:{args.tag}", + "value": f"{dev_config.repo_url}/{dev_config.operator_image_dev}:{args.tag}", }, { "name": "AGENT_IMAGE", @@ -142,11 +142,11 @@ def create_test_pod(args: argparse.Namespace, dev_config: DevConfig) -> None: }, { "name": "VERSION_UPGRADE_HOOK_IMAGE", - "value": f"{dev_config.repo_url}/{dev_config.version_upgrade_hook_image}:{args.tag}", + "value": f"{dev_config.repo_url}/{dev_config.version_upgrade_hook_image_dev}:{args.tag}", }, { "name": "READINESS_PROBE_IMAGE", - "value": f"{dev_config.repo_url}/{dev_config.readiness_probe_image}:{args.tag}", + "value": f"{dev_config.repo_url}/{dev_config.readiness_probe_image_dev}:{args.tag}", }, { "name": "PERFORM_CLEANUP", From 68196a2c248a4976f154cf6e7eef9fd62d1979ca Mon Sep 17 00:00:00 2001 From: Rajdeep Das Date: Thu, 27 May 2021 10:59:16 +0200 Subject: [PATCH 276/790] Skip mongodb connectivity tests if running locally (#508) --- Makefile | 2 +- test/e2e/mongodbtests/mongodbtests.go | 9 +++++++++ test/e2e/replica_set_tls/replica_set_tls_test.go | 10 ++++++---- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index ea93b737f..a0e646a03 100644 --- a/Makefile +++ b/Makefile @@ -107,7 +107,7 @@ e2e-k8s: install e2e-image # e.g. make e2e test=replica_set cleanup=true e2e: install eval $$(scripts/dev/get_e2e_env_vars.py $(cleanup)); \ - go test -v -timeout=30m -failfast ./test/e2e/$(test) + go test -v -short -timeout=30m -failfast ./test/e2e/$(test) # Generate code generate: controller-gen diff --git a/test/e2e/mongodbtests/mongodbtests.go b/test/e2e/mongodbtests/mongodbtests.go index 56e31e039..af876dd0d 100644 --- a/test/e2e/mongodbtests/mongodbtests.go +++ b/test/e2e/mongodbtests/mongodbtests.go @@ -24,6 +24,15 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" ) +// SkipTestIfLocal skips tests locally which tests connectivity to mongodb pods +func SkipTestIfLocal(t *testing.T, msg string, f func(t *testing.T)) { + if testing.Short() { + t.Log("Skipping [" + msg + "]") + return + } + t.Run(msg, f) +} + // StatefulSetBecomesReady ensures that the underlying stateful set // reaches the running state. func StatefulSetBecomesReady(mdb *mdbv1.MongoDBCommunity) func(t *testing.T) { diff --git a/test/e2e/replica_set_tls/replica_set_tls_test.go b/test/e2e/replica_set_tls/replica_set_tls_test.go index 289705758..2d94e5120 100644 --- a/test/e2e/replica_set_tls/replica_set_tls_test.go +++ b/test/e2e/replica_set_tls/replica_set_tls_test.go @@ -43,10 +43,12 @@ func TestReplicaSetTLS(t *testing.T) { t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb)) - t.Run("Wait for TLS to be enabled", tester.HasTlsMode("requireSSL", 60, WithTls())) - t.Run("Test Basic TLS Connectivity", tester.ConnectivitySucceeds(WithTls())) - t.Run("Test TLS required", tester.ConnectivityFails(WithoutTls())) - t.Run("Ensure Authentication", tester.EnsureAuthenticationIsConfigured(3, WithTls())) + mongodbtests.SkipTestIfLocal(t, "Ensure MongoDB TLS Configuration", func(t *testing.T) { + tester.HasTlsMode("requireSSL", 60, WithTls()) + tester.ConnectivitySucceeds(WithTls()) + tester.ConnectivityFails(WithoutTls()) + tester.EnsureAuthenticationIsConfigured(3, WithTls()) + }) t.Run("TLS is disabled", mongodbtests.DisableTLS(&mdb)) t.Run("MongoDB Reaches Failed Phase", mongodbtests.MongoDBReachesFailedPhase(&mdb)) t.Run("TLS is enabled", mongodbtests.EnableTLS(&mdb)) From 1195bb1060a8623de1fc56d4dea87b4e00869710 Mon Sep 17 00:00:00 2001 From: Snyk bot Date: Thu, 27 May 2021 12:53:05 +0300 Subject: [PATCH 277/790] fix: requirements.txt to reduce vulnerabilities (#509) The following vulnerabilities are fixed by pinning transitive dependencies: - https://snyk.io/vuln/SNYK-PYTHON-RSA-1038401 --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index b1d6da5fe..55834085b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,3 +8,4 @@ tqdm==v4.49.0 boto3==1.16.21 pymongo==3.11.2 dnspython==2.0.0 +rsa>=4.7 # not directly required, pinned by Snyk to avoid a vulnerability From 17261ca0f0b6fa411ad3b2a81db288acd146f5c9 Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Thu, 27 May 2021 14:22:06 +0100 Subject: [PATCH 278/790] Create mongodb.com_v1_custom_storage_size_cr.yaml (#511) --- .../mongodb.com_v1_custom_volume_cr.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/config/samples/arbitrary_statefulset_configuration/mongodb.com_v1_custom_volume_cr.yaml b/config/samples/arbitrary_statefulset_configuration/mongodb.com_v1_custom_volume_cr.yaml index 900691a7c..dbd7cc3a9 100644 --- a/config/samples/arbitrary_statefulset_configuration/mongodb.com_v1_custom_volume_cr.yaml +++ b/config/samples/arbitrary_statefulset_configuration/mongodb.com_v1_custom_volume_cr.yaml @@ -25,7 +25,18 @@ spec: spec: serviceName: example-openshift-mongodb-svc selector: {} + # Specifies a size for the data volume different from the default 10Gi + volumeClaimTemplates: + - metadata: + name: data-volume + spec: + accessModes: [ "ReadWriteOnce", "ReadWriteMany" ] + resources: + requests: + storage: 50Gi + template: + # Adds a custom volume to the pods spec: volumes: - name: custom-volume From 5899d69c12c3d5ca8b2b2ed0ec8ab6ad95b3f624 Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Fri, 28 May 2021 16:37:24 +0100 Subject: [PATCH 279/790] Fix wrong header in README.md (#512) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0911a9044..549b41bc6 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ -###v0.6.0 has introduced breaking changes. If you are upgrading from a previous version, follow the upgrade instructions outlined [in the release notes](https://github.com/mongodb/mongodb-kubernetes-operator/releases/tag/v0.6.0) +### v0.6.0 has introduced breaking changes. If you are upgrading from a previous version, follow the upgrade instructions outlined [in the release notes](https://github.com/mongodb/mongodb-kubernetes-operator/releases/tag/v0.6.0) This is a [Kubernetes Operator](https://coreos.com/operators/) which deploys MongoDB Community into Kubernetes clusters. From 1cc035d6cee6251798526b43eb020d91496006f0 Mon Sep 17 00:00:00 2001 From: Anton Date: Mon, 31 May 2021 12:29:28 +0100 Subject: [PATCH 280/790] CLOUDP-89260: e2e test for readiness probe (#507) --- .evergreen.yml | 13 +++-- cmd/readiness/main.go | 4 ++ controllers/construct/mongodbstatefulset.go | 2 +- deploy/e2e/role.yaml | 6 +++ go.sum | 1 + test/e2e/client.go | 50 ++++++++++++++++++- test/e2e/e2eutil.go | 16 ++++++ test/e2e/mongodbtests/mongodbtests.go | 41 ++++++++++++--- .../replica_set_mongod_readiness_test.go | 50 +++++++++++++++++++ .../replica_set_recovery_test.go} | 4 +- 10 files changed, 173 insertions(+), 14 deletions(-) create mode 100644 test/e2e/replica_set_mongod_readiness/replica_set_mongod_readiness_test.go rename test/e2e/{replica_set_readiness_probe/replica_set_readiness_probe_test.go => replica_set_recovery/replica_set_recovery_test.go} (95%) diff --git a/.evergreen.yml b/.evergreen.yml index ae1a3ce76..ffd55bd94 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -163,7 +163,8 @@ task_groups: - func: create_kind_cluster tasks: - e2e_test_replica_set - - e2e_test_replica_set_readiness_probe + - e2e_test_replica_set_recovery + - e2e_test_replica_set_mongod_readiness - e2e_test_replica_set_scale - e2e_test_replica_set_scale_down - e2e_test_replica_set_change_version @@ -262,11 +263,17 @@ tasks: vars: test: replica_set - - name: e2e_test_replica_set_readiness_probe + - name: e2e_test_replica_set_recovery commands: - func: run_e2e_test vars: - test: replica_set_readiness_probe + test: replica_set_recovery + + - name: e2e_test_replica_set_mongod_readiness + commands: + - func: run_e2e_test + vars: + test: replica_set_mongod_readiness - name: e2e_test_replica_set_scale commands: diff --git a/cmd/readiness/main.go b/cmd/readiness/main.go index c15dd52de..9e629822d 100644 --- a/cmd/readiness/main.go +++ b/cmd/readiness/main.go @@ -67,6 +67,10 @@ func isPodReady(conf config.Config) (bool, error) { } inReadyState := isInReadyState(healthStatus) + if !inReadyState { + logger.Info("Mongod is not ready") + } + if inGoalState && inReadyState { logger.Info("Agent has reached goal state") return true, nil diff --git a/controllers/construct/mongodbstatefulset.go b/controllers/construct/mongodbstatefulset.go index 08320612a..af04a7471 100644 --- a/controllers/construct/mongodbstatefulset.go +++ b/controllers/construct/mongodbstatefulset.go @@ -234,7 +234,7 @@ func versionUpgradeHookInit(volumeMount []corev1.VolumeMount) container.Modifica func DefaultReadiness() probes.Modification { return probes.Apply( probes.WithExecCommand([]string{readinessProbePath}), - probes.WithFailureThreshold(60), // TODO: this value needs further consideration + probes.WithFailureThreshold(40), probes.WithInitialDelaySeconds(5), ) } diff --git a/deploy/e2e/role.yaml b/deploy/e2e/role.yaml index 0f2178e42..f420945c7 100644 --- a/deploy/e2e/role.yaml +++ b/deploy/e2e/role.yaml @@ -61,6 +61,12 @@ rules: - pods verbs: - get +- apiGroups: + - "" + resources: + - pods/exec + verbs: + - create - apiGroups: - apps resources: diff --git a/go.sum b/go.sum index 0937f0234..b59fb3e50 100644 --- a/go.sum +++ b/go.sum @@ -309,6 +309,7 @@ github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS4 github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= diff --git a/test/e2e/client.go b/test/e2e/client.go index fc13acc01..42b3efdf2 100644 --- a/test/e2e/client.go +++ b/test/e2e/client.go @@ -1,17 +1,21 @@ package e2eutil import ( + "bytes" "context" "fmt" "testing" "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/generate" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/scheme" + corev1client "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/rest" + "k8s.io/client-go/tools/remotecommand" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" @@ -79,6 +83,10 @@ func (ctx *Context) AddCleanupFunc(fn func() error) { // E2ETestClient is a wrapper on client.Client that provides cleanup functionality. type E2ETestClient struct { Client client.Client + // We need the core API client for some operations that the controller-runtime client doesn't support + // (e.g. exec into the container) + CoreV1Client corev1client.CoreV1Client + restConfig *rest.Config } // NewE2ETestClient creates a new E2ETestClient. @@ -87,7 +95,11 @@ func newE2ETestClient(config *rest.Config, scheme *runtime.Scheme) (*E2ETestClie if err != nil { return nil, err } - return &E2ETestClient{Client: cli}, err + coreClient, err := corev1client.NewForConfig(config) + if err != nil { + return nil, err + } + return &E2ETestClient{Client: cli, CoreV1Client: *coreClient, restConfig: config}, err } // Create wraps client.Create to provide post-test cleanup functionality. @@ -127,6 +139,42 @@ func (c *E2ETestClient) Get(ctx context.Context, key types.NamespacedName, obj c return c.Client.Get(ctx, key, obj) } +func (c *E2ETestClient) Execute(pod corev1.Pod, containerName, command string) (string, error) { + req := c.CoreV1Client.RESTClient(). + Post(). + Namespace(pod.Namespace). + Resource("pods"). + Name(pod.Name). + SubResource("exec"). + VersionedParams(&corev1.PodExecOptions{ + Container: containerName, + Command: []string{"/bin/sh", "-c", command}, + Stdin: false, + Stdout: true, + Stderr: true, + TTY: true, + }, scheme.ParameterCodec) + + buf := &bytes.Buffer{} + errBuf := &bytes.Buffer{} + exec, err := remotecommand.NewSPDYExecutor(c.restConfig, "POST", req.URL()) + if err != nil { + return "", err + } + err = exec.Stream(remotecommand.StreamOptions{ + Stdout: buf, + Stderr: errBuf, + }) + if err != nil { + return "", fmt.Errorf(`failed executing command "%s" on %v/%v: %s ("%s")`, command, pod.Namespace, pod.Name, err, errBuf.String()) + } + + if errBuf.String() != "" { + return buf.String(), fmt.Errorf("remote command %s on %v/%v raised an error: %s", command, pod.Namespace, pod.Name, errBuf.String()) + } + return buf.String(), nil +} + // RunTest is the main entry point function for an e2e test. func RunTest(m *testing.M) (int, error) { var cfg *rest.Config diff --git a/test/e2e/e2eutil.go b/test/e2e/e2eutil.go index dce1679ef..88be05e76 100644 --- a/test/e2e/e2eutil.go +++ b/test/e2e/e2eutil.go @@ -140,6 +140,22 @@ func waitForStatefulSetCondition(t *testing.T, mdb *mdbv1.MongoDBCommunity, retr }) } +func WaitForPodReadiness(t *testing.T, isReady bool, containerName string, timeout time.Duration, pod corev1.Pod) error { + return wait.Poll(time.Second*3, timeout, func() (done bool, err error) { + err = TestClient.Get(context.TODO(), types.NamespacedName{Name: pod.Name, Namespace: pod.Namespace}, &pod) + if err != nil { + return false, err + } + for _, status := range pod.Status.ContainerStatuses { + t.Logf("%s (%s), ready: %v\n", pod.Name, status.Name, status.Ready) + if status.Name == containerName && status.Ready == isReady { + return true, nil + } + } + return false, nil + }) +} + // waitForRuntimeObjectToExist waits until a runtime.Object of the given name exists // using the provided retryInterval and timeout provided. func waitForRuntimeObjectToExist(name string, retryInterval, timeout time.Duration, obj client.Object, namespace string) error { diff --git a/test/e2e/mongodbtests/mongodbtests.go b/test/e2e/mongodbtests/mongodbtests.go index af876dd0d..2d81df379 100644 --- a/test/e2e/mongodbtests/mongodbtests.go +++ b/test/e2e/mongodbtests/mongodbtests.go @@ -8,7 +8,6 @@ import ( "time" "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/wait" mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" @@ -233,12 +232,7 @@ func BasicFunctionality(mdb *mdbv1.MongoDBCommunity) func(*testing.T) { // DeletePod will delete a pod that belongs to this MongoDB resource's StatefulSet func DeletePod(mdb *mdbv1.MongoDBCommunity, podNum int) func(*testing.T) { return func(t *testing.T) { - pod := corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%d", mdb.Name, podNum), - Namespace: mdb.Namespace, - }, - } + pod := podFromMongoDBCommunity(mdb, podNum) if err := e2eutil.TestClient.Delete(context.TODO(), &pod); err != nil { t.Fatal(err) } @@ -359,6 +353,30 @@ func StatefulSetContainerConditionIsTrue(mdb *mdbv1.MongoDBCommunity, containerN } } +// PodContainerBecomesReady waits until the container with 'containerName' in the pod #podNum becomes not ready. +func PodContainerBecomesNotReady(mdb *mdbv1.MongoDBCommunity, podNum int, containerName string) func(*testing.T) { + return func(t *testing.T) { + pod := podFromMongoDBCommunity(mdb, podNum) + assert.NoError(t, e2eutil.WaitForPodReadiness(t, false, containerName, time.Minute*10, pod)) + } +} + +// PodContainerBecomesReady waits until the container with 'containerName' in the pod #podNum becomes ready. +func PodContainerBecomesReady(mdb *mdbv1.MongoDBCommunity, podNum int, containerName string) func(*testing.T) { + return func(t *testing.T) { + pod := podFromMongoDBCommunity(mdb, podNum) + assert.NoError(t, e2eutil.WaitForPodReadiness(t, true, containerName, time.Minute*3, pod)) + } +} + +func ExecInContainer(mdb *mdbv1.MongoDBCommunity, podNum int, containerName, command string) func(*testing.T) { + return func(t *testing.T) { + pod := podFromMongoDBCommunity(mdb, podNum) + _, err := e2eutil.TestClient.Execute(pod, containerName, command) + assert.NoError(t, err) + } +} + func findContainerByName(name string, containers []corev1.Container) *corev1.Container { for _, c := range containers { if c.Name == name { @@ -369,6 +387,15 @@ func findContainerByName(name string, containers []corev1.Container) *corev1.Con return nil } +func podFromMongoDBCommunity(mdb *mdbv1.MongoDBCommunity, podNum int) corev1.Pod { + return corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%d", mdb.Name, podNum), + Namespace: mdb.Namespace, + }, + } +} + func assertEqualOwnerReference(t *testing.T, resourceType string, resourceNamespacedName types.NamespacedName, ownerReferences []metav1.OwnerReference, expectedOwnerReference metav1.OwnerReference) { assert.Len(t, ownerReferences, 1, fmt.Sprintf("%s %s/%s doesn't have OwnerReferences", resourceType, resourceNamespacedName.Name, resourceNamespacedName.Namespace)) diff --git a/test/e2e/replica_set_mongod_readiness/replica_set_mongod_readiness_test.go b/test/e2e/replica_set_mongod_readiness/replica_set_mongod_readiness_test.go new file mode 100644 index 000000000..be8652690 --- /dev/null +++ b/test/e2e/replica_set_mongod_readiness/replica_set_mongod_readiness_test.go @@ -0,0 +1,50 @@ +package replica_set_mongod_readiness + +import ( + "fmt" + "os" + "testing" + + e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" + "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/mongodbtests" + setup "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" +) + +func TestMain(m *testing.M) { + code, err := e2eutil.RunTest(m) + if err != nil { + fmt.Println(err) + } + os.Exit(code) +} + +func TestReplicaSet(t *testing.T) { + ctx := setup.Setup(t) + defer ctx.Teardown() + + mdb, user := e2eutil.NewTestMongoDB(ctx, "mdb0", "") + + _, err := setup.GeneratePasswordForUser(ctx, user, "") + if err != nil { + t.Fatal(err) + } + + t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) + t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb)) + t.Run("Ensure Agent container is marked as non-ready", func(t *testing.T) { + t.Run("Break mongod data files", mongodbtests.ExecInContainer(&mdb, 0, "mongod", "mkdir /data/tmp; mv /data/WiredTiger.wt /data/tmp")) + // Just moving the file doesn't fail the mongod until any data is written - the easiest way is to kill the mongod + // and in this case it won't restart + t.Run("Kill mongod process", mongodbtests.ExecInContainer(&mdb, 0, "mongod", "kill 1")) + // CLOUDP-89260: mongod uptime 1 minute and readiness probe failureThreshold 40 (40 * 5 -> 200 seconds) + // note, that this may take much longer on evergreen than locally + t.Run("Pod agent container becomes not-ready", mongodbtests.PodContainerBecomesNotReady(&mdb, 0, "mongodb-agent")) + }) + t.Run("Ensure Agent container gets fixed", func(t *testing.T) { + // Note, that we call this command on the 'mongodb-agent' container as the 'mongod' container is down and we cannot + // execute shell there. But both containers share the same /data directory so we can do it from any of them. + t.Run("Fix mongod data files", mongodbtests.ExecInContainer(&mdb, 0, "mongodb-agent", "mv /data/tmp/WiredTiger.wt /data/")) + // Eventually the agent will start mongod again + t.Run("Pod agent container becomes ready", mongodbtests.PodContainerBecomesReady(&mdb, 0, "mongodb-agent")) + }) +} diff --git a/test/e2e/replica_set_readiness_probe/replica_set_readiness_probe_test.go b/test/e2e/replica_set_recovery/replica_set_recovery_test.go similarity index 95% rename from test/e2e/replica_set_readiness_probe/replica_set_readiness_probe_test.go rename to test/e2e/replica_set_recovery/replica_set_recovery_test.go index 2568679e6..9f83460b4 100644 --- a/test/e2e/replica_set_readiness_probe/replica_set_readiness_probe_test.go +++ b/test/e2e/replica_set_recovery/replica_set_recovery_test.go @@ -1,4 +1,4 @@ -package replica_set_readiness_probe +package replica_set_recovery import ( "crypto/rand" @@ -24,7 +24,7 @@ func TestMain(m *testing.M) { os.Exit(code) } -func TestReplicaSetReadinessProbeScaling(t *testing.T) { +func TestReplicaSetRecovery(t *testing.T) { ctx := setup.Setup(t) defer ctx.Teardown() From eef857ee667ac63488dcb3fe3be98d9e356178ad Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Tue, 1 Jun 2021 11:40:47 +0100 Subject: [PATCH 281/790] Fixed image_name field (#520) --- .evergreen.yml | 4 ++-- inventory.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.evergreen.yml b/.evergreen.yml index ffd55bd94..14c59a5eb 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -371,7 +371,7 @@ tasks: - func: setup_virtualenv - func: build_and_push_image_sonar vars: - image_type: version-post-start-hook-init + image_name: version-post-start-hook-init release: true @@ -381,7 +381,7 @@ tasks: - func: setup_virtualenv - func: build_and_push_image_sonar vars: - image_type: readiness-probe-init + image_name: readiness-probe-init release: true - name: release_agent_ubuntu diff --git a/inventory.yaml b/inventory.yaml index 713f1e10b..dbd295447 100644 --- a/inventory.yaml +++ b/inventory.yaml @@ -257,7 +257,7 @@ images: - release_version output: - - registry: $(inputs.params.registry)/(inputs.params.readiness_probe_image) + - registry: $(inputs.params.registry)/$(inputs.params.readiness_probe_image) tag: $(inputs.params.release_version)-context From 9bafcd21cc56a6d7e3659d09007618c737e0f209 Mon Sep 17 00:00:00 2001 From: Rajdeep Das Date: Tue, 1 Jun 2021 13:33:14 +0200 Subject: [PATCH 282/790] Create role/rolebinging in namespace configured in config.json and create namespace if it doesn't exist. (#521) --- scripts/dev/e2e.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/scripts/dev/e2e.py b/scripts/dev/e2e.py index 3b6b26307..aaf475297 100644 --- a/scripts/dev/e2e.py +++ b/scripts/dev/e2e.py @@ -49,8 +49,12 @@ def _prepare_test_environment(config_file: str) -> None: ) print("Creating Role Binding") + role_binding = _load_test_role_binding() + # set namespace specified in config.json + role_binding["subjects"][0]["namespace"] = dev_config.namespace + k8s_conditions.ignore_if_already_exists( - lambda: rbacv1.create_cluster_role_binding(_load_test_role_binding()) + lambda: rbacv1.create_cluster_role_binding(role_binding) ) print("Creating ServiceAccount") @@ -87,6 +91,13 @@ def create_kube_config(config_file: str) -> None: metadata=client.V1ObjectMeta(name="kube-config"), data=data ) + print("Creating Namespace") + k8s_conditions.ignore_if_already_exists( + lambda: corev1.create_namespace( + client.V1Namespace(metadata=dict(name=dev_config.namespace)) + ) + ) + k8s_conditions.ignore_if_already_exists( lambda: corev1.create_namespaced_config_map(dev_config.namespace, config_map) ) From 12097e5932a9162465686b007a9ff15a32f75cc3 Mon Sep 17 00:00:00 2001 From: Priyadarshi Lahiri <1254077+priyolahiri@users.noreply.github.com> Date: Tue, 1 Jun 2021 16:24:48 +0200 Subject: [PATCH 283/790] Cloudp 76562: support srv records (#517) * CLOUDP-76562: Port naming and SRV URI Added port naming to replica set controller by default and added SRV Connection function to URI * CLOUDP-76562: Added e2e tests E2E Tests added for SRV connection test * CLOUDP-76562: uri fix in fixture fixed missed namespace var * CLOUDP-76562: Added WithReplicaSet Method added WithReplicaSet method for explicitly setting the replicaset name in the conn * CLOUDP-76562: Suggested review changes Fixed docstring. Used mdb.Name instead of "mdb0" * CLOUDP-76562: Fixed unit test and removed unneeded method Fixed unit test by providing port name and removed unused ConnectSRV function. --- api/v1/mongodbcommunity_types.go | 6 ++++++ controllers/replica_set_controller.go | 1 + controllers/replicaset_controller_test.go | 2 +- test/e2e/replica_set/replica_set_test.go | 2 ++ test/e2e/util/mongotester/mongotester.go | 16 ++++++++++++++++ 5 files changed, 26 insertions(+), 1 deletion(-) diff --git a/api/v1/mongodbcommunity_types.go b/api/v1/mongodbcommunity_types.go index 395630f07..2f0e8a01b 100644 --- a/api/v1/mongodbcommunity_types.go +++ b/api/v1/mongodbcommunity_types.go @@ -449,6 +449,12 @@ func (m MongoDBCommunity) MongoURI() string { return fmt.Sprintf("mongodb://%s", strings.Join(members, ",")) } +// MongoSRVURI returns a mongo srv uri which can be used to connect to this deployment +func (m MongoDBCommunity) MongoSRVURI() string { + clusterDomain := "svc.cluster.local" // TODO: make this configurable + return fmt.Sprintf("mongodb+srv://%s.%s.%s", m.ServiceName(), m.Namespace, clusterDomain) +} + func (m MongoDBCommunity) Hosts() []string { hosts := make([]string, m.Spec.Members) clusterDomain := "svc.cluster.local" // TODO: make this configurable diff --git a/controllers/replica_set_controller.go b/controllers/replica_set_controller.go index bd1c765ea..3cd1cce45 100644 --- a/controllers/replica_set_controller.go +++ b/controllers/replica_set_controller.go @@ -451,6 +451,7 @@ func buildService(mdb mdbv1.MongoDBCommunity) corev1.Service { SetServiceType(corev1.ServiceTypeClusterIP). SetClusterIP("None"). SetPort(27017). + SetPortName("mongodb"). SetPublishNotReadyAddresses(true). SetOwnerReferences(mdb.GetOwnerReferences()). Build() diff --git a/controllers/replicaset_controller_test.go b/controllers/replicaset_controller_test.go index 9b612043b..7ca49ed96 100644 --- a/controllers/replicaset_controller_test.go +++ b/controllers/replicaset_controller_test.go @@ -256,7 +256,7 @@ func TestService_isCorrectlyCreatedAndUpdated(t *testing.T) { assert.Equal(t, svc.Spec.Type, corev1.ServiceTypeClusterIP) assert.Equal(t, svc.Spec.Selector["app"], mdb.ServiceName()) assert.Len(t, svc.Spec.Ports, 1) - assert.Equal(t, svc.Spec.Ports[0], corev1.ServicePort{Port: 27017}) + assert.Equal(t, svc.Spec.Ports[0], corev1.ServicePort{Port: 27017, Name: "mongodb"}) res, err = r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) diff --git a/test/e2e/replica_set/replica_set_test.go b/test/e2e/replica_set/replica_set_test.go index 33f0301a6..a213e3113 100644 --- a/test/e2e/replica_set/replica_set_test.go +++ b/test/e2e/replica_set/replica_set_test.go @@ -5,6 +5,7 @@ import ( "os" "testing" + "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/util/mongotester" . "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/util/mongotester" e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" @@ -40,6 +41,7 @@ func TestReplicaSet(t *testing.T) { t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb)) t.Run("Keyfile authentication is configured", tester.HasKeyfileAuth(3)) t.Run("Test Basic Connectivity", tester.ConnectivitySucceeds()) + t.Run("Test SRV Connectivity", tester.ConnectivitySucceeds(mongotester.WithSRV(mdb.MongoSRVURI()), WithoutTls(), WithReplicaSet((mdb.Name)))) t.Run("Ensure Authentication", tester.EnsureAuthenticationIsConfigured(3)) t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 1)) } diff --git a/test/e2e/util/mongotester/mongotester.go b/test/e2e/util/mongotester/mongotester.go index 202954745..f8d53a257 100644 --- a/test/e2e/util/mongotester/mongotester.go +++ b/test/e2e/util/mongotester/mongotester.go @@ -429,6 +429,22 @@ func WithoutTls() OptionApplier { } } +// WithSRV will add SRV connection string +func WithSRV(uri string) OptionApplier { + opt := &options.ClientOptions{} + opt.ApplyURI(uri) + return clientOptionAdder{option: opt} +} + +// WithReplicaSet will explicitly add a replicaset name +func WithReplicaSet(rsname string) OptionApplier { + return clientOptionAdder{ + option: &options.ClientOptions{ + ReplicaSet: &rsname, + }, + } +} + // getClientTLSConfig reads in the tls fixtures func getClientTLSConfig() (*tls.Config, error) { // Read the CA certificate from test data From e718f4012ab3180b5449bee55b70a0e49b570183 Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Tue, 1 Jun 2021 19:47:11 +0100 Subject: [PATCH 284/790] Add operator release to pipeline (#505) --- .evergreen.yml | 21 +----- inventories/operator-inventory.yaml | 19 +++++ pipeline.py | 4 ++ scripts/ci/run_image_release.sh | 5 -- scripts/dev/dev_config.py | 6 ++ scripts/dev/dockerutil.py | 106 ---------------------------- scripts/dev/e2e.py | 2 +- scripts/dev/release_image.py | 83 ---------------------- 8 files changed, 33 insertions(+), 213 deletions(-) delete mode 100755 scripts/ci/run_image_release.sh delete mode 100644 scripts/dev/dockerutil.py delete mode 100644 scripts/dev/release_image.py diff --git a/.evergreen.yml b/.evergreen.yml index 14c59a5eb..dc9ccf7ca 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -124,19 +124,6 @@ functions: working_dir: mongodb-kubernetes-operator binary: scripts/ci/build_and_push_image_sonar.sh - release_docker_image: - - command: subprocess.exec - type: system - params: - working_dir: mongodb-kubernetes-operator - include_expansions_in_env: - - version_id - - quay_user_name - - quay_password - - old_image - - new_image - - image_type - command: scripts/ci/run_image_release.sh add_supported_release: - *python_venv @@ -358,12 +345,10 @@ tasks: commands: - func: clone - func: setup_virtualenv - - func: release_docker_image + - func: build_and_push_image_sonar vars: - old_image: quay.io/mongodb/community-operator-dev - new_image: quay.io/mongodb/mongodb-kubernetes-operator - image_type: mongodb-kubernetes-operator - + image_name: operator-ubi + release: true - name: release_version_upgrade_post_start_hook commands: diff --git a/inventories/operator-inventory.yaml b/inventories/operator-inventory.yaml index 142e69629..f57d63e94 100644 --- a/inventories/operator-inventory.yaml +++ b/inventories/operator-inventory.yaml @@ -8,6 +8,7 @@ images: template_context: scripts/dev/templates inputs: + - operator_image - operator_image_dev stages: @@ -30,9 +31,27 @@ images: dockerfile: scripts/dev/templates/Dockerfile.ubi-$(inputs.params.version_id) + labels: + quay.expires-after: 48h + output: - registry: $(inputs.params.registry)/$(inputs.params.operator_image_dev) tag: $(inputs.params.version_id) - registry: $(inputs.params.registry)/$(inputs.params.operator_image_dev) tag: latest + - name: operator-release-ubi + task_type: docker_build + tags: ["release"] + + inputs: + - release_version + + dockerfile: scripts/dev/templates/Dockerfile.ubi-$(inputs.params.version_id) + + labels: + quay.expires-after: Never + + output: + - registry: $(inputs.params.registry)/$(inputs.params.operator_image) + tag: $(inputs.params.release_version) diff --git a/pipeline.py b/pipeline.py index 18f1e1056..4b08a5635 100644 --- a/pipeline.py +++ b/pipeline.py @@ -35,6 +35,8 @@ def _build_agent_args(config: DevConfig) -> Dict[str, str]: "agent_version": release["mongodb-agent"]["version"], "release_version": release["mongodb-agent"]["version"], "tools_version": release["mongodb-agent"]["tools_version"], + "agent_image": config.agent_image, + "agent_image_dev": config.agent_dev_image, "registry": config.repo_url, "s3_bucket": config.s3_bucket, } @@ -101,6 +103,7 @@ def build_version_post_start_hook_image(config: DevConfig) -> None: def build_operator_ubi_image(config: DevConfig) -> None: + release = _load_release() config.ensure_tag_is_run("ubi") sonar_build_image( "operator-ubi", @@ -112,6 +115,7 @@ def build_operator_ubi_image(config: DevConfig) -> None: "base_image": "registry.access.redhat.com/ubi8/ubi-minimal:latest", "operator_image": config.operator_image, "operator_image_dev": config.operator_image_dev, + "release_version": release["mongodb-kubernetes-operator"], }, inventory="inventories/operator-inventory.yaml", ) diff --git a/scripts/ci/run_image_release.sh b/scripts/ci/run_image_release.sh deleted file mode 100755 index 3dd7ae1ae..000000000 --- a/scripts/ci/run_image_release.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash - -# shellcheck disable=SC1091 -. venv/bin/activate -python3 scripts/dev/release_image.py --old_repo_url "${old_image:-}" --new_repo_url "${new_image:-}" --old_tag "${version_id:-}" --labels "{\"quay.expires-after\":\"never\"}" --username "${quay_user_name:-}" --password "${quay_password:-}" --registry https://quay.io/organization/mongodb --release_file "./release.json" --image_type "${image_type:-}" diff --git a/scripts/dev/dev_config.py b/scripts/dev/dev_config.py index 2d011ed1b..b8172f2b1 100644 --- a/scripts/dev/dev_config.py +++ b/scripts/dev/dev_config.py @@ -123,6 +123,12 @@ def agent_image_ubuntu(self) -> str: def agent_image_ubi(self) -> str: return self._config["agent_image_ubi"] + @property + def agent_dev_image(self) -> str: + if self._distro == Distro.UBI: + return self._get_dev_image("agent_image_ubi_dev", "agent_image_ubi") + return self._get_dev_image("agent_image_ubuntu_dev", "agent_image_ubuntu") + @property def agent_image(self) -> str: if self._distro == Distro.UBI: diff --git a/scripts/dev/dockerutil.py b/scripts/dev/dockerutil.py deleted file mode 100644 index 976779d8f..000000000 --- a/scripts/dev/dockerutil.py +++ /dev/null @@ -1,106 +0,0 @@ -import docker -from dockerfile_generator import render -import os -import json -from tqdm import tqdm - -from typing import Union, Any, Optional - - -def build_image(repo_url: str, tag: str, path: str) -> None: - """ - build_image builds the image with the given tag - """ - client = docker.from_env() - print(f"Building image: {tag}") - client.images.build(tag=tag, path=path) - print("Successfully built image!") - - -def push_image(tag: str) -> None: - """ - push_image pushes the given tag. It uses - the current docker environment - """ - client = docker.from_env() - print(f"Pushing image: {tag}") - with tqdm(total=100, ascii=False) as progress_bar: - last_percent = 0.0 - for line in client.images.push(tag, stream=True): - percent = get_completion_percentage(line) - if percent: - progress_bar.update(percent - last_percent) - last_percent = percent - - -def retag_image( - old_repo_url: str, - new_repo_url: str, - old_tag: str, - new_tag: str, - path: str, - labels: Optional[dict] = None, - username: Optional[str] = None, - password: Optional[str] = None, - registry: Optional[str] = None, -) -> None: - with open(f"{path}/Dockerfile", "w") as f: - f.write(f"FROM {old_repo_url}:{old_tag}") - client = docker.from_env() - if all(value is not None for value in [username, password, registry]): - client.login(username=username, password=password, registry=registry) - - image, _ = client.images.build(path=f"{path}", labels=labels, tag=new_tag) - image.tag(new_repo_url, new_tag) - os.remove(f"{path}/Dockerfile") - - # We do not want to republish an image that has not changed, so we check if the new - # pair repo:tag already exists. - try: - image = client.images.pull(new_repo_url, new_tag) - return - # We also need to catch APIError as if the image has been recently deleted (uncommon, but might happen?) - # we will get this kind of error: - # docker.errors.APIError: 500 Server Error: Internal Server Error - # ("unknown: Tag was deleted or has expired. To pull, revive via time machine" - except (docker.errors.ImageNotFound, docker.errors.APIError) as e: - pass - print(f"Pushing to {new_repo_url}:{new_tag}") - client.images.push(new_repo_url, new_tag) - - -def get_completion_percentage(line: Any) -> float: - try: - line = json.loads(line.strip().decode("utf-8")) - except ValueError: - return 0 - - to_skip = ("Preparing", "Waiting", "Layer already exists") - if "status" in line: - if line["status"] in to_skip: - return 0 - if line["status"] == "Pushing": - try: - current = float(line["progressDetail"]["current"]) - total = float(line["progressDetail"]["total"]) - except KeyError: - return 0 - result = (current / total) * 100 - if result > 100.0: - return 100.0 - return result - return 0 - - -def build_and_push_image(repo_url: str, tag: str, path: str, image_type: str) -> None: - """ - build_and_push_operator creates the Dockerfile for the operator - and pushes it to the target repo - """ - dockerfile_text = render(image_type) - with open(f"{path}/Dockerfile", "w") as f: - f.write(dockerfile_text) - - build_image(repo_url, tag, path) - os.remove(f"{path}/Dockerfile") - push_image(tag) diff --git a/scripts/dev/e2e.py b/scripts/dev/e2e.py index aaf475297..bcc6e0eb4 100644 --- a/scripts/dev/e2e.py +++ b/scripts/dev/e2e.py @@ -145,7 +145,7 @@ def create_test_pod(args: argparse.Namespace, dev_config: DevConfig) -> None: }, { "name": "AGENT_IMAGE", - "value": f"{dev_config.repo_url}/{dev_config.agent_image}:{args.tag}", + "value": f"{dev_config.repo_url}/{dev_config.agent_dev_image}:{args.tag}", }, { "name": "TEST_NAMESPACE", diff --git a/scripts/dev/release_image.py b/scripts/dev/release_image.py deleted file mode 100644 index db8261fa8..000000000 --- a/scripts/dev/release_image.py +++ /dev/null @@ -1,83 +0,0 @@ -import dockerutil -import json -import sys -import argparse - -ALLOWED_RELEASE_OPTIONS = frozenset( - ["mongodb-kubernetes-operator", "version-upgrade-hook"] -) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser() - parser.add_argument("--old_repo_url", help="Url of the image to retag", type=str) - parser.add_argument( - "--new_repo_url", help="Url to where the new image should be pushed", type=str - ) - parser.add_argument( - "--path", - help="path to use for the temporarily generated Dockerfile", - type=str, - default=".", - ) - parser.add_argument("--release_file", help="Path to the release file", type=str) - parser.add_argument( - "--old_tag", - help="the old tag of the image to retag", - type=str, - ) - parser.add_argument( - "--username", - help="username for the registry", - type=str, - ) - parser.add_argument( - "--password", - help="password for the registry", - type=str, - ) - parser.add_argument( - "--registry", - help="The docker registry", - type=str, - ) - parser.add_argument( - "--labels", - help="Labels for the new image", - type=json.loads, - ) - parser.add_argument( - "--image_type", - help="Type of image to be released", - choices=ALLOWED_RELEASE_OPTIONS, - ) - args = parser.parse_args() - - return args - - -def main() -> int: - args = parse_args() - with open(args.release_file) as f: - release = json.load(f) - - if args.image_type not in ALLOWED_RELEASE_OPTIONS: - print(f"Image type {args.image_type} is not supported by the release script!") - return 1 - - dockerutil.retag_image( - args.old_repo_url, - args.new_repo_url, - args.old_tag, - release[args.image_type], - args.path, - args.labels, - args.username, - args.password, - args.registry, - ) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) From 947a95effd33f22ef483b6e93b48b3b6de1a614f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 2 Jun 2021 11:09:51 +0000 Subject: [PATCH 285/790] Bump go.uber.org/zap from 1.16.0 to 1.17.0 (#516) --- go.mod | 2 +- go.sum | 17 +++++++++-------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 70a811939..cf8d0d9be 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/stretchr/testify v1.7.0 github.com/xdg/stringprep v1.0.3 go.mongodb.org/mongo-driver v1.5.2 - go.uber.org/zap v1.16.0 + go.uber.org/zap v1.17.0 k8s.io/api v0.20.4 k8s.io/apimachinery v0.21.0 k8s.io/client-go v0.20.4 diff --git a/go.sum b/go.sum index b59fb3e50..e38c79720 100644 --- a/go.sum +++ b/go.sum @@ -32,7 +32,6 @@ github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935 github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= @@ -88,6 +87,7 @@ github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZ github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= @@ -450,20 +450,21 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= -go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= +go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.8.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= -go.uber.org/zap v1.16.0 h1:uFRZXykJGK9lLY4HtgSw44DnIcAM+kRBP7x5m+NpAOM= -go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= +go.uber.org/zap v1.17.0 h1:MTjgFu6ZLKvY6Pvaqk97GlxNBuMpV4Hy/3P6tRGlI2U= +go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -748,8 +749,9 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -757,7 +759,6 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3 h1:sXmLre5bzIR6ypkjXCDI3jHPssRhc8KD/Ome589sc3U= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo= k8s.io/api v0.20.2/go.mod h1:d7n6Ehyzx+S+cE3VhTGfVNNqtGc/oL9DCdYYahlurV8= From 09e436f523fd993c8cefa064f6e59566f23664da Mon Sep 17 00:00:00 2001 From: Rajdeep Das Date: Wed, 2 Jun 2021 13:39:51 +0200 Subject: [PATCH 286/790] Remove unused bundle target from Makefile (#524) --- Makefile | 42 ------------------------------------------ 1 file changed, 42 deletions(-) diff --git a/Makefile b/Makefile index a0e646a03..0280984bc 100644 --- a/Makefile +++ b/Makefile @@ -1,35 +1,5 @@ SHELL := /bin/bash -# VERSION defines the project version for the bundle. -# Update this value when you upgrade the version of your project. -# To re-generate a bundle for another specific version without changing the standard setup, you can: -# - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2) -# - use environment variables to overwrite this value (e.g export VERSION=0.0.2) -VERSION ?= 0.0.1 - -# CHANNELS define the bundle channels used in the bundle. -# Add a new line here if you would like to change its default config. (E.g CHANNELS = "preview,fast,stable") -# To re-generate a bundle for other specific channels without changing the standard setup, you can: -# - use the CHANNELS as arg of the bundle target (e.g make bundle CHANNELS=preview,fast,stable) -# - use environment variables to overwrite this value (e.g export CHANNELS="preview,fast,stable") -ifneq ($(origin CHANNELS), undefined) -BUNDLE_CHANNELS := --channels=$(CHANNELS) -endif - -# DEFAULT_CHANNEL defines the default channel used in the bundle. -# Add a new line here if you would like to change its default config. (E.g DEFAULT_CHANNEL = "stable") -# To re-generate a bundle for any other default channel without changing the default setup, you can: -# - use the DEFAULT_CHANNEL as arg of the bundle target (e.g make bundle DEFAULT_CHANNEL=stable) -# - use environment variables to overwrite this value (e.g export DEFAULT_CHANNEL="stable") -ifneq ($(origin DEFAULT_CHANNEL), undefined) -BUNDLE_DEFAULT_CHANNEL := --default-channel=$(DEFAULT_CHANNEL) -endif -BUNDLE_METADATA_OPTS ?= $(BUNDLE_CHANNELS) $(BUNDLE_DEFAULT_CHANNEL) - -# BUNDLE_IMG defines the image:tag used for the bundle. -# You can use it as an arg. (E.g make bundle-build BUNDLE_IMG=/:) -BUNDLE_IMG ?= controller-bundle:$(VERSION) - # Image URL to use all building/pushing image targets REPO_URL := $(shell jq -r .repo_url < ~/.community-operator-dev/config.json) OPERATOR_IMAGE := $(shell jq -r .operator_image < ~/.community-operator-dev/config.json) @@ -161,15 +131,3 @@ rm -rf $$TMP_DIR ;\ } endef -# Generate bundle manifests and metadata, then validate generated files. -.PHONY: bundle -bundle: manifests kustomize - operator-sdk generate kustomize manifests -q - cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG) - $(KUSTOMIZE) build config/manifests | operator-sdk generate bundle -q --overwrite --version $(VERSION) $(BUNDLE_METADATA_OPTS) - operator-sdk bundle validate ./bundle - -# Build the bundle image. -.PHONY: bundle-build -bundle-build: - docker build -f bundle.Dockerfile -t $(BUNDLE_IMG) . From 205fd5746d4c3c67a635a905130cc1c1842962b0 Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Thu, 3 Jun 2021 10:17:05 +0100 Subject: [PATCH 287/790] Upgrade to sonar 0.0.11 (#526) --- dev_notes/dev-quick-start.md | 3 --- requirements.txt | 1 + scripts/ci/setup_virtualenv.sh | 2 -- 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/dev_notes/dev-quick-start.md b/dev_notes/dev-quick-start.md index 019f17819..b0512430c 100644 --- a/dev_notes/dev-quick-start.md +++ b/dev_notes/dev-quick-start.md @@ -15,9 +15,6 @@ source path/to/new/virtual/environment/bin/activate * install python dependencies ``` pip install -r requirements.txt - -# Note: sonar requires access to the 10gen repo and is used for the release pipeline -pip install git+ssh://git@github.com/10gen/sonar.git@0.0.10 ``` #### Create a Kind cluster and a local registry diff --git a/requirements.txt b/requirements.txt index 55834085b..82eef4f61 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +git+https://github.com/mongodb/sonar@0.0.11 docker==4.3.1 kubernetes==11.0.0 jinja2==2.11.3 diff --git a/scripts/ci/setup_virtualenv.sh b/scripts/ci/setup_virtualenv.sh index 89375a374..8edf35157 100755 --- a/scripts/ci/setup_virtualenv.sh +++ b/scripts/ci/setup_virtualenv.sh @@ -6,5 +6,3 @@ virtualenv --python /opt/python/3.7/bin/python3 ./venv pip3 install -r ./requirements.txt -# shellcheck disable=SC2154 -pip3 install "git+https://${sonar_github_token}@github.com/10gen/sonar.git@0.0.10" From 5957c87ab65477bbf026ca5fd9b02dc2c9f6b82a Mon Sep 17 00:00:00 2001 From: fireba11 Date: Thu, 3 Jun 2021 11:52:45 +0200 Subject: [PATCH 288/790] add detailed upgrade steps (#518) --- docs/install-upgrade.md | 51 ++++++++++++++++++++++++++++++++++------- 1 file changed, 43 insertions(+), 8 deletions(-) diff --git a/docs/install-upgrade.md b/docs/install-upgrade.md index d11ab48cc..6a665368e 100644 --- a/docs/install-upgrade.md +++ b/docs/install-upgrade.md @@ -155,12 +155,47 @@ To install the MongoDB Community Kubernetes Operator: ## Upgrade the Operator -To upgrade the MongoDB Community Kubernetes Operator to a specific version: +The release v0.6.0 had some breaking changes (see https://github.com/mongodb/mongodb-kubernetes-operator/releases/tag/v0.6.0) requiring special steps to upgrade from a pre-0.6.0 Version. +As always, have backups. +Make sure you run commands in the correct namespace. -1. Change the version in the Operator [resource definition](../config/manager/manager.yaml) -2. Change to the directory in which you cloned the repository. -3. Checkout the specific tag matching the operator version (e.g. `v0.5.1`) -4. Invoke the following `kubectl` command to upgrade the [Custom Resource Definitions](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/). - ``` - kubectl apply -f config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml - ``` +1. Prepare for the upgrade. + + a. Migrate your cr by updating apiVersion and kind to + ``` + apiVersion: mongodbcommunity.mongodb.com/v1 + kind: MongoDBCommunity + ``` + If you upgrade from pre-0.3.0 you need to also add the field spec.users[n].scramCredentialsSecretName for each resource. This will be used to determine the name of the generated secret which stores MongoDB user credentials. This field must comply with DNS-1123 rules (see https://kubernetes.io/docs/concepts/overview/working-with-objects/names/). + b. Plan a downtime. +2. Out with the old + + a. Delete the old operator. + ``` + kubectl delete deployments.apps mongodb-kubernetes-operator + ``` + b. Delete the old statefulset. + ``` + kubectl delete statefulsets.apps mongodb + ``` + c. Delete the old customResourceDefinition. Not strictly needed but no need to keep it around anymore (unless you got more installations of operator in your cluster) + ``` + kubectl delete crd mongodb.mongodb.com + ``` +3. In with the new + + Follow the normal installation procedure above. +4. Start up your Replica Set again + a. Re-create your cr using the new Version from Step 1.a + b. Patch your statefulset to have it update the permissions + ``` + kubectl patch statefulset --type='json' --patch '[ {"op":"add","path":"/spec/template/spec/initContainers/-", "value": { "name": "change-data-dir-permissions", "image": "busybox", "command": [ "chown", "-R", "2000", "/data" ], "securityContext": { "runAsNonRoot": false, "runAsUser": 0, "runAsGroup":0 }, "volumeMounts": [ { "mountPath": "/data", "name" : "data-volume" } ] } } ]' + ``` + c. Delete your pod manually + Since you added your cr in step a. kubernetes will immediately try to get your cluster up and running. + You will now have one pod that isn't working since it got created before you patched your statefulset with the additional migration container. + Delete that pod. + ``` + kubectl delete pod -0 + ``` + d. You're done. Now Kubernetes will create the pod fresh, causing the migration to run and then the pod to start up. Then kubernetes will proceed creating the next pod until it reaches the number specified in your cr. \ No newline at end of file From 82cc7c77eee45f070c2a4b9072540c57b8bec35b Mon Sep 17 00:00:00 2001 From: Priyadarshi Lahiri <1254077+priyolahiri@users.noreply.github.com> Date: Thu, 3 Jun 2021 12:56:58 +0200 Subject: [PATCH 289/790] Switch to sonar 0.0.11 (#528) * Updated to Sonar 0.0.11 * removed reverence to 10gen sonar * Pipenv ready Added support for Pipenv with Pipfile and Pipfile.lock. Changes sonar url to include egg fragment for Pipenv compatibility. * updated docs for pipenv support added steps for pipenv users Co-authored-by: chatton --- Pipfile | 23 ++ Pipfile.lock | 532 +++++++++++++++++++++++++++++++++++ dev_notes/dev-quick-start.md | 16 +- requirements.txt | 2 +- 4 files changed, 569 insertions(+), 4 deletions(-) create mode 100644 Pipfile create mode 100644 Pipfile.lock diff --git a/Pipfile b/Pipfile new file mode 100644 index 000000000..1b5657a03 --- /dev/null +++ b/Pipfile @@ -0,0 +1,23 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +sonar = {ref = "0.0.11", git = "https://github.com/mongodb/sonar"} +docker = "==4.3.1" +kubernetes = "==11.0.0" +black = "==20.8b1" +mypy = "==0.782" +tqdm = "==v4.49.0" +boto3 = "==1.16.21" +pymongo = "==3.11.2" +dnspython = "==2.0.0" +rsa = ">=4.7" +Jinja2 = "==2.11.3" +PyYAML = "==5.4.1" + +[dev-packages] + +[requires] +python_version = "3.9" diff --git a/Pipfile.lock b/Pipfile.lock new file mode 100644 index 000000000..0a187740a --- /dev/null +++ b/Pipfile.lock @@ -0,0 +1,532 @@ +{ + "_meta": { + "hash": { + "sha256": "3aec4830a4917fa2e79c444ea34161d3c138fae7b2f5c406342ec14517fecc6b" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.9" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "appdirs": { + "hashes": [ + "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41", + "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128" + ], + "version": "==1.4.4" + }, + "black": { + "hashes": [ + "sha256:1c02557aa099101b9d21496f8a914e9ed2222ef70336404eeeac8edba836fbea" + ], + "index": "pypi", + "version": "==20.8b1" + }, + "boto3": { + "hashes": [ + "sha256:7a8063418d11690a8ee87bc1c3e32aa4830aee20e3767fe780a99a7fa2410b13", + "sha256:dd63506b8b5e46be0b55d18ab9a62dea8b3b46aa46ce6c3b1fc4e41788c375b8" + ], + "index": "pypi", + "version": "==1.16.21" + }, + "botocore": { + "hashes": [ + "sha256:ad4adfcc195b5401d84b0c65d3a89e507c1d54c201879c8761ff10ef5c361e21", + "sha256:d3694f6ef918def8082513e5ef309cd6cd83b612e9984e3a66e8adc98c650a92" + ], + "version": "==1.19.63" + }, + "cachetools": { + "hashes": [ + "sha256:2cc0b89715337ab6dbba85b5b50effe2b0c74e035d83ee8ed637cf52f12ae001", + "sha256:61b5ed1e22a0924aed1d23b478f37e8d52549ff8a961de2909c69bf950020cff" + ], + "markers": "python_version ~= '3.5'", + "version": "==4.2.2" + }, + "certifi": { + "hashes": [ + "sha256:2bbf76fd432960138b3ef6dda3dde0544f27cbf8546c458e60baf371917ba9ee", + "sha256:50b1e4f8446b06f41be7dd6338db18e0990601dce795c2b1686458aa7e8fa7d8" + ], + "version": "==2021.5.30" + }, + "chardet": { + "hashes": [ + "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa", + "sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==4.0.0" + }, + "click": { + "hashes": [ + "sha256:8c04c11192119b1ef78ea049e0a6f0463e4c48ef00a30160c704337586f3ad7a", + "sha256:fba402a4a47334742d782209a7c79bc448911afe1149d07bdabdf480b3e2f4b6" + ], + "markers": "python_version >= '3.6'", + "version": "==8.0.1" + }, + "dnspython": { + "hashes": [ + "sha256:044af09374469c3a39eeea1a146e8cac27daec951f1f1f157b1962fc7cb9d1b7", + "sha256:40bb3c24b9d4ec12500f0124288a65df232a3aa749bb0c39734b782873a2544d" + ], + "index": "pypi", + "version": "==2.0.0" + }, + "docker": { + "hashes": [ + "sha256:13966471e8bc23b36bfb3a6fb4ab75043a5ef1dac86516274777576bed3b9828", + "sha256:bad94b8dd001a8a4af19ce4becc17f41b09f228173ffe6a4e0355389eef142f2" + ], + "index": "pypi", + "version": "==4.3.1" + }, + "google-auth": { + "hashes": [ + "sha256:044d81b1e58012f8ebc71cc134e191c1fa312f543f1fbc99973afe28c25e3228", + "sha256:b3ca7a8ff9ab3bdefee3ad5aefb11fc6485423767eee016f5942d8e606ca23fb" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", + "version": "==1.30.1" + }, + "idna": { + "hashes": [ + "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6", + "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.10" + }, + "jinja2": { + "hashes": [ + "sha256:03e47ad063331dd6a3f04a43eddca8a966a26ba0c5b7207a9a9e4e08f1b29419", + "sha256:a6d58433de0ae800347cab1fa3043cebbabe8baa9d29e668f1c768cb87a333c6" + ], + "index": "pypi", + "version": "==2.11.3" + }, + "jmespath": { + "hashes": [ + "sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9", + "sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.10.0" + }, + "kubernetes": { + "hashes": [ + "sha256:1a2472f8b01bc6aa87e3a34781f859bded5a5c8ff791a53d889a8bd6cc550430", + "sha256:4af81201520977139a143f96123fb789fa351879df37f122916b9b6ed050bbaf" + ], + "index": "pypi", + "version": "==11.0.0" + }, + "markupsafe": { + "hashes": [ + "sha256:01a9b8ea66f1658938f65b93a85ebe8bc016e6769611be228d797c9d998dd298", + "sha256:023cb26ec21ece8dc3907c0e8320058b2e0cb3c55cf9564da612bc325bed5e64", + "sha256:0446679737af14f45767963a1a9ef7620189912317d095f2d9ffa183a4d25d2b", + "sha256:0717a7390a68be14b8c793ba258e075c6f4ca819f15edfc2a3a027c823718567", + "sha256:0955295dd5eec6cb6cc2fe1698f4c6d84af2e92de33fbcac4111913cd100a6ff", + "sha256:10f82115e21dc0dfec9ab5c0223652f7197feb168c940f3ef61563fc2d6beb74", + "sha256:1d609f577dc6e1aa17d746f8bd3c31aa4d258f4070d61b2aa5c4166c1539de35", + "sha256:2ef54abee730b502252bcdf31b10dacb0a416229b72c18b19e24a4509f273d26", + "sha256:3c112550557578c26af18a1ccc9e090bfe03832ae994343cfdacd287db6a6ae7", + "sha256:47ab1e7b91c098ab893b828deafa1203de86d0bc6ab587b160f78fe6c4011f75", + "sha256:49e3ceeabbfb9d66c3aef5af3a60cc43b85c33df25ce03d0031a608b0a8b2e3f", + "sha256:4efca8f86c54b22348a5467704e3fec767b2db12fc39c6d963168ab1d3fc9135", + "sha256:53edb4da6925ad13c07b6d26c2a852bd81e364f95301c66e930ab2aef5b5ddd8", + "sha256:594c67807fb16238b30c44bdf74f36c02cdf22d1c8cda91ef8a0ed8dabf5620a", + "sha256:611d1ad9a4288cf3e3c16014564df047fe08410e628f89805e475368bd304914", + "sha256:6557b31b5e2c9ddf0de32a691f2312a32f77cd7681d8af66c2692efdbef84c18", + "sha256:693ce3f9e70a6cf7d2fb9e6c9d8b204b6b39897a2c4a1aa65728d5ac97dcc1d8", + "sha256:6a7fae0dd14cf60ad5ff42baa2e95727c3d81ded453457771d02b7d2b3f9c0c2", + "sha256:6c4ca60fa24e85fe25b912b01e62cb969d69a23a5d5867682dd3e80b5b02581d", + "sha256:7d91275b0245b1da4d4cfa07e0faedd5b0812efc15b702576d103293e252af1b", + "sha256:905fec760bd2fa1388bb5b489ee8ee5f7291d692638ea5f67982d968366bef9f", + "sha256:97383d78eb34da7e1fa37dd273c20ad4320929af65d156e35a5e2d89566d9dfb", + "sha256:984d76483eb32f1bcb536dc27e4ad56bba4baa70be32fa87152832cdd9db0833", + "sha256:a30e67a65b53ea0a5e62fe23682cfe22712e01f453b95233b25502f7c61cb415", + "sha256:ab3ef638ace319fa26553db0624c4699e31a28bb2a835c5faca8f8acf6a5a902", + "sha256:b2f4bf27480f5e5e8ce285a8c8fd176c0b03e93dcc6646477d4630e83440c6a9", + "sha256:b7f2d075102dc8c794cbde1947378051c4e5180d52d276987b8d28a3bd58c17d", + "sha256:be98f628055368795d818ebf93da628541e10b75b41c559fdf36d104c5787066", + "sha256:d7f9850398e85aba693bb640262d3611788b1f29a79f0c93c565694658f4071f", + "sha256:f5653a225f31e113b152e56f154ccbe59eeb1c7487b39b9d9f9cdb58e6c79dc5", + "sha256:f826e31d18b516f653fe296d967d700fddad5901ae07c622bb3705955e1faa94", + "sha256:f8ba0e8349a38d3001fae7eadded3f6606f0da5d748ee53cc1dab1d6527b9509", + "sha256:f9081981fe268bd86831e5c75f7de206ef275defcb82bc70740ae6dc507aee51", + "sha256:fa130dd50c57d53368c9d59395cb5526eda596d3ffe36666cd81a44d56e48872" + ], + "markers": "python_version >= '3.6'", + "version": "==2.0.1" + }, + "mypy": { + "hashes": [ + "sha256:2c6cde8aa3426c1682d35190b59b71f661237d74b053822ea3d748e2c9578a7c", + "sha256:3fdda71c067d3ddfb21da4b80e2686b71e9e5c72cca65fa216d207a358827f86", + "sha256:5dd13ff1f2a97f94540fd37a49e5d255950ebcdf446fb597463a40d0df3fac8b", + "sha256:6731603dfe0ce4352c555c6284c6db0dc935b685e9ce2e4cf220abe1e14386fd", + "sha256:6bb93479caa6619d21d6e7160c552c1193f6952f0668cdda2f851156e85186fc", + "sha256:81c7908b94239c4010e16642c9102bfc958ab14e36048fa77d0be3289dda76ea", + "sha256:9c7a9a7ceb2871ba4bac1cf7217a7dd9ccd44c27c2950edbc6dc08530f32ad4e", + "sha256:a4a2cbcfc4cbf45cd126f531dedda8485671545b43107ded25ce952aac6fb308", + "sha256:b7fbfabdbcc78c4f6fc4712544b9b0d6bf171069c6e0e3cb82440dd10ced3406", + "sha256:c05b9e4fb1d8a41d41dec8786c94f3b95d3c5f528298d769eb8e73d293abc48d", + "sha256:d7df6eddb6054d21ca4d3c6249cae5578cb4602951fd2b6ee2f5510ffb098707", + "sha256:e0b61738ab504e656d1fe4ff0c0601387a5489ca122d55390ade31f9ca0e252d", + "sha256:eff7d4a85e9eea55afa34888dfeaccde99e7520b51f867ac28a48492c0b1130c", + "sha256:f05644db6779387ccdb468cc47a44b4356fc2ffa9287135d05b70a98dc83b89a" + ], + "index": "pypi", + "version": "==0.782" + }, + "mypy-extensions": { + "hashes": [ + "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d", + "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8" + ], + "version": "==0.4.3" + }, + "oauthlib": { + "hashes": [ + "sha256:42bf6354c2ed8c6acb54d971fce6f88193d97297e18602a3a886603f9d7730cc", + "sha256:8f0215fcc533dd8dd1bee6f4c412d4f0cd7297307d43ac61666389e3bc3198a3" + ], + "markers": "python_version >= '3.6'", + "version": "==3.1.1" + }, + "pathspec": { + "hashes": [ + "sha256:86379d6b86d75816baba717e64b1a3a3469deb93bb76d613c9ce79edc5cb68fd", + "sha256:aa0cb481c4041bf52ffa7b0d8fa6cd3e88a2ca4879c533c9153882ee2556790d" + ], + "version": "==0.8.1" + }, + "pyasn1": { + "hashes": [ + "sha256:014c0e9976956a08139dc0712ae195324a75e142284d5f87f1a87ee1b068a359", + "sha256:03840c999ba71680a131cfaee6fab142e1ed9bbd9c693e285cc6aca0d555e576", + "sha256:0458773cfe65b153891ac249bcf1b5f8f320b7c2ce462151f8fa74de8934becf", + "sha256:08c3c53b75eaa48d71cf8c710312316392ed40899cb34710d092e96745a358b7", + "sha256:39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d", + "sha256:5c9414dcfede6e441f7e8f81b43b34e834731003427e5b09e4e00e3172a10f00", + "sha256:6e7545f1a61025a4e58bb336952c5061697da694db1cae97b116e9c46abcf7c8", + "sha256:78fa6da68ed2727915c4767bb386ab32cdba863caa7dbe473eaae45f9959da86", + "sha256:7ab8a544af125fb704feadb008c99a88805126fb525280b2270bb25cc1d78a12", + "sha256:99fcc3c8d804d1bc6d9a099921e39d827026409a58f2a720dcdb89374ea0c776", + "sha256:aef77c9fb94a3ac588e87841208bdec464471d9871bd5050a287cc9a475cd0ba", + "sha256:e89bf84b5437b532b0803ba5c9a5e054d21fec423a89952a74f87fa2c9b7bce2", + "sha256:fec3e9d8e36808a28efb59b489e4528c10ad0f480e57dcc32b4de5c9d8c9fdf3" + ], + "version": "==0.4.8" + }, + "pyasn1-modules": { + "hashes": [ + "sha256:0845a5582f6a02bb3e1bde9ecfc4bfcae6ec3210dd270522fee602365430c3f8", + "sha256:0fe1b68d1e486a1ed5473f1302bd991c1611d319bba158e98b106ff86e1d7199", + "sha256:15b7c67fabc7fc240d87fb9aabf999cf82311a6d6fb2c70d00d3d0604878c811", + "sha256:426edb7a5e8879f1ec54a1864f16b882c2837bfd06eee62f2c982315ee2473ed", + "sha256:65cebbaffc913f4fe9e4808735c95ea22d7a7775646ab690518c056784bc21b4", + "sha256:905f84c712230b2c592c19470d3ca8d552de726050d1d1716282a1f6146be65e", + "sha256:a50b808ffeb97cb3601dd25981f6b016cbb3d31fbf57a8b8a87428e6158d0c74", + "sha256:a99324196732f53093a84c4369c996713eb8c89d360a496b599fb1a9c47fc3eb", + "sha256:b80486a6c77252ea3a3e9b1e360bc9cf28eaac41263d173c032581ad2f20fe45", + "sha256:c29a5e5cc7a3f05926aff34e097e84f8589cd790ce0ed41b67aed6857b26aafd", + "sha256:cbac4bc38d117f2a49aeedec4407d23e8866ea4ac27ff2cf7fb3e5b570df19e0", + "sha256:f39edd8c4ecaa4556e989147ebf219227e2cd2e8a43c7e7fcb1f1c18c5fd6a3d", + "sha256:fe0644d9ab041506b62782e92b06b8c68cca799e1a9636ec398675459e031405" + ], + "version": "==0.2.8" + }, + "pymongo": { + "hashes": [ + "sha256:019ddf7ced8e42cc6c8c608927c799be8097237596c94ffe551f6ef70e55237e", + "sha256:047c325c4a96e7be7d11acf58639bcf71a81ca212d9c6590e3369bc28678647a", + "sha256:047cc2007b280672ddfdf2e7b862aad8d898f481f65bbc9067bfa4e420a019a9", + "sha256:061d59f525831c4051af0b6dbafa62b0b8b168d4ef5b6e3c46d0811b8499d100", + "sha256:082832a59da18efab4d9148cca396451bac99da9757f31767f706e828b5b8500", + "sha256:0a53a751d977ad02f1bd22ddb6288bb4816c4758f44a50225462aeeae9cbf6a0", + "sha256:1222025db539641071a1b67f6950f65a6342a39db5b454bf306abd6954f1ad8a", + "sha256:1580fad512c678b720784e5c9018621b1b3bd37fb5b1633e874738862d6435c7", + "sha256:202ea1d4edc8a5439fc179802d807b49e7e563207fea5610779e56674ac770c6", + "sha256:21d7b48567a1c80f9266e0ab61c1218a31279d911da345679188733e354f81cc", + "sha256:264843ce2af0640994a4331148ef5312989bc004678c457460758766c9b4decc", + "sha256:270a1f6a331eac3a393090af06df68297cb31a8b2df0bdcbd97dc613c5758e78", + "sha256:29a6840c2ac778010547cad5870f3db2e080ad7fad01197b07fff993c08692c8", + "sha256:3646c2286d889618d43e01d9810ac1fc17709d2b4dec61366df5edc8ba228b3e", + "sha256:36b9b98a39565a8f33803c81569442b35e749a72fb1aa7d0bcdb1a33052f8bcc", + "sha256:3ec8f8e106a1476659d8c020228b45614daabdbdb6c6454a843a1d4f77d13339", + "sha256:422069f2cebf58c9dd9e8040b4768f7be4f228c95bc4505e8fa8e7b4f7191ad8", + "sha256:44376a657717de8847d5d71a9305f3595c7e78c91ac77edbb87058d12ede87a6", + "sha256:45728e6aae3023afb5b2829586d1d2bfd9f0d71cfd7d3c924b71a5e9aef617a8", + "sha256:46792b71ab802d9caf1fc9d52e83399ef8e1a36e91eef4d827c06e36b8df2230", + "sha256:4942a5659ae927bb764a123a6409870ca5dd572d83b3bfb71412c9a191bbf792", + "sha256:4be4fe9d18523da98deeb0b554ac76e1dc1562ee879d62572b34dda8593efcc1", + "sha256:523804bd8fcb5255508052b50073a27c701b90a73ea46e29be46dad5fe01bde6", + "sha256:540dafd6f4a0590fc966465c726b80fa7c0804490c39786ef29236fe68c94401", + "sha256:5980509801cbd2942df31714d055d89863684b4de26829c349362e610a48694e", + "sha256:5ad7b96c27acd7e256b33f47cf3d23bd7dd902f9c033ae43f32ffcbc37bebafd", + "sha256:6122470dfa61d4909b75c98012c1577404ba4ab860d0095e0c6980560cb3711f", + "sha256:6175fd105da74a09adb38f93be96e1f64873294c906e5e722cbbc5bd10c44e3b", + "sha256:646d4d30c5aa7c0ddbfe9b990f0f77a88621024a21ad0b792bd9d58caa9611f0", + "sha256:6700e251c6396cc05d7460dc05ef8e19e60a7b53b62c007725b48e123aaa2b1c", + "sha256:6aac7e0e8de92f11a410eb68c24a2decbac6f094e82fd95d22546d0168e7a18b", + "sha256:6e7a6057481a644970e43475292e1c0af095ca39a20fe83781196bd6e6690a38", + "sha256:76579fcf77052b39796fe4a11818d1289dd48cffe15951b3403288fa163c29f6", + "sha256:7e69fa025a1db189443428f345fea5555d16413df6addc056e17bb8c9794b006", + "sha256:7f0c507e1f108790840d6c4b594019ebf595025c324c9f7e9c9b2b15b41f884e", + "sha256:813db97e9955b6b1b50b5cebd18cb148580603bb9b067ea4c5cc656b333bc906", + "sha256:82d5ded5834b6c92380847860eb28dcaf20b847a27cee5811c4aaceef87fd280", + "sha256:82f6e42ba40440a7e0a20bfe12465a3b62d65966a4c7ad1a21b36ffff88de6fe", + "sha256:8d669c720891781e7c82d412cad39f9730ef277e3957b48a3344dae47d3caa03", + "sha256:944ed467feb949e103555863fa934fb84216a096b0004ca364d3ddf9d18e2b9e", + "sha256:96c6aef7ffb0d37206c0342abb82d874fa8cdc344267277ec63f562b94335c22", + "sha256:9be785bd4e1ba0148fb00ca84e4dbfbd1c74df3af3a648559adc60b0782f34de", + "sha256:9d19843568df9d263dc92ae4cc2279879add8a26996473f9155590cac635b321", + "sha256:a118a1df7280ffab7fe0f3eab325868339ff1c4d5b8e0750db0f0a796da8f849", + "sha256:b4294ddf76452459433ecfa6a93258608b5e462c76ef15e4695ed5e2762f009f", + "sha256:b50af6701b4a5288b77fb4db44a363aa9485caf2c3e7a40c0373fd45e34440af", + "sha256:b875bb4b438931dce550e170bfb558597189b8d0160f4ac60f14a21955161699", + "sha256:b95d2c2829b5956bf54d9a22ffec911dea75abf0f0f7e0a8a57423434bfbde91", + "sha256:c046e09e886f4539f8626afba17fa8f2e6552731f9384e2827154e3e3b7fda4e", + "sha256:c1d1992bbdf363b22b5a9543ab7d7c6f27a1498826d50d91319b803ddcf1142e", + "sha256:c2b67881392a9e85aa108e75f62cdbe372d5a3f17ea5f8d3436dcf4662052f14", + "sha256:c6cf288c9e03195d8e12b72a6388b32f18a5e9c2545622417a963e428e1fe496", + "sha256:c812b6e53344e92f10f12235219fb769c491a4a87a02c9c3f93fe632e493bda8", + "sha256:cc421babc687dc52ce0fc19787b2404518ca749d9db59576100946ff886f38ed", + "sha256:ce53c00be204ec4428d3c1f3c478ae89d388efec575544c27f57b61e9fa4a7f2", + "sha256:ce9964c117cbe5cf6269f30a2b334d28675956e988b7dbd0b4f7370924afda2e", + "sha256:d6f82e86896a8db70e8ae8fa4b7556a0f188f1d8a6c53b2ba229889d55a59308", + "sha256:d9d3ae537f61011191b2fd6f8527b9f9f8a848b37d4c85a0f7bb28004c42b546", + "sha256:e565d1e4388765c135052717f15f9e0314f9d172062444c6b3fc0002e93ed04b", + "sha256:ed98683d8f01f1c46ef2d02469e04e9a8fe9a73a9741a4e6e66677a73b59bec8", + "sha256:ef18aa15b1aa18c42933deed5233b3284186e9ed85c25d2704ceff5099a3964c", + "sha256:fa741e9c805567239f845c7e9a016aff797f9bb02ff9bc8ccd2fbd9eafefedd4", + "sha256:fc4946acb6cdada08f60aca103b61334995523da65be5fe816ea8571c9967d46", + "sha256:fcc66d17a3363b7bd6d2655de8706e25a3cd1be2bd1b8e8d8a5c504a6ef893ae" + ], + "index": "pypi", + "version": "==3.11.2" + }, + "python-dateutil": { + "hashes": [ + "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c", + "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.8.1" + }, + "pyyaml": { + "hashes": [ + "sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf", + "sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696", + "sha256:129def1b7c1bf22faffd67b8f3724645203b79d8f4cc81f674654d9902cb4393", + "sha256:294db365efa064d00b8d1ef65d8ea2c3426ac366c0c4368d930bf1c5fb497f77", + "sha256:3b2b1824fe7112845700f815ff6a489360226a5609b96ec2190a45e62a9fc922", + "sha256:3bd0e463264cf257d1ffd2e40223b197271046d09dadf73a0fe82b9c1fc385a5", + "sha256:4465124ef1b18d9ace298060f4eccc64b0850899ac4ac53294547536533800c8", + "sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10", + "sha256:4e0583d24c881e14342eaf4ec5fbc97f934b999a6828693a99157fde912540cc", + "sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018", + "sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e", + "sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253", + "sha256:72a01f726a9c7851ca9bfad6fd09ca4e090a023c00945ea05ba1638c09dc3347", + "sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183", + "sha256:895f61ef02e8fed38159bb70f7e100e00f471eae2bc838cd0f4ebb21e28f8541", + "sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb", + "sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185", + "sha256:bfb51918d4ff3d77c1c856a9699f8492c612cde32fd3bcd344af9be34999bfdc", + "sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db", + "sha256:cb333c16912324fd5f769fff6bc5de372e9e7a202247b48870bc251ed40239aa", + "sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46", + "sha256:d483ad4e639292c90170eb6f7783ad19490e7a8defb3e46f97dfe4bacae89122", + "sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b", + "sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63", + "sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df", + "sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc", + "sha256:fd7f6999a8070df521b6384004ef42833b9bd62cfee11a09bda1079b4b704247", + "sha256:fdc842473cd33f45ff6bce46aea678a54e3d21f1b61a7750ce3c498eedfe25d6", + "sha256:fe69978f3f768926cfa37b867e3843918e012cf83f680806599ddce33c2c68b0" + ], + "index": "pypi", + "version": "==5.4.1" + }, + "regex": { + "hashes": [ + "sha256:01afaf2ec48e196ba91b37451aa353cb7eda77efe518e481707e0515025f0cd5", + "sha256:11d773d75fa650cd36f68d7ca936e3c7afaae41b863b8c387a22aaa78d3c5c79", + "sha256:18c071c3eb09c30a264879f0d310d37fe5d3a3111662438889ae2eb6fc570c31", + "sha256:1e1c20e29358165242928c2de1482fb2cf4ea54a6a6dea2bd7a0e0d8ee321500", + "sha256:281d2fd05555079448537fe108d79eb031b403dac622621c78944c235f3fcf11", + "sha256:314d66636c494ed9c148a42731b3834496cc9a2c4251b1661e40936814542b14", + "sha256:32e65442138b7b76dd8173ffa2cf67356b7bc1768851dded39a7a13bf9223da3", + "sha256:339456e7d8c06dd36a22e451d58ef72cef293112b559010db3d054d5560ef439", + "sha256:3916d08be28a1149fb97f7728fca1f7c15d309a9f9682d89d79db75d5e52091c", + "sha256:3a9cd17e6e5c7eb328517969e0cb0c3d31fd329298dd0c04af99ebf42e904f82", + "sha256:47bf5bf60cf04d72bf6055ae5927a0bd9016096bf3d742fa50d9bf9f45aa0711", + "sha256:4c46e22a0933dd783467cf32b3516299fb98cfebd895817d685130cc50cd1093", + "sha256:4c557a7b470908b1712fe27fb1ef20772b78079808c87d20a90d051660b1d69a", + "sha256:52ba3d3f9b942c49d7e4bc105bb28551c44065f139a65062ab7912bef10c9afb", + "sha256:563085e55b0d4fb8f746f6a335893bda5c2cef43b2f0258fe1020ab1dd874df8", + "sha256:598585c9f0af8374c28edd609eb291b5726d7cbce16be6a8b95aa074d252ee17", + "sha256:619d71c59a78b84d7f18891fe914446d07edd48dc8328c8e149cbe0929b4e000", + "sha256:67bdb9702427ceddc6ef3dc382455e90f785af4c13d495f9626861763ee13f9d", + "sha256:6d1b01031dedf2503631d0903cb563743f397ccaf6607a5e3b19a3d76fc10480", + "sha256:741a9647fcf2e45f3a1cf0e24f5e17febf3efe8d4ba1281dcc3aa0459ef424dc", + "sha256:7c2a1af393fcc09e898beba5dd59196edaa3116191cc7257f9224beaed3e1aa0", + "sha256:7d9884d86dd4dd489e981d94a65cd30d6f07203d90e98f6f657f05170f6324c9", + "sha256:90f11ff637fe8798933fb29f5ae1148c978cccb0452005bf4c69e13db951e765", + "sha256:919859aa909429fb5aa9cf8807f6045592c85ef56fdd30a9a3747e513db2536e", + "sha256:96fcd1888ab4d03adfc9303a7b3c0bd78c5412b2bfbe76db5b56d9eae004907a", + "sha256:97f29f57d5b84e73fbaf99ab3e26134e6687348e95ef6b48cfd2c06807005a07", + "sha256:980d7be47c84979d9136328d882f67ec5e50008681d94ecc8afa8a65ed1f4a6f", + "sha256:a91aa8619b23b79bcbeb37abe286f2f408d2f2d6f29a17237afda55bb54e7aac", + "sha256:ade17eb5d643b7fead300a1641e9f45401c98eee23763e9ed66a43f92f20b4a7", + "sha256:b9c3db21af35e3b3c05764461b262d6f05bbca08a71a7849fd79d47ba7bc33ed", + "sha256:bd28bc2e3a772acbb07787c6308e00d9626ff89e3bfcdebe87fa5afbfdedf968", + "sha256:bf5824bfac591ddb2c1f0a5f4ab72da28994548c708d2191e3b87dd207eb3ad7", + "sha256:c0502c0fadef0d23b128605d69b58edb2c681c25d44574fc673b0e52dce71ee2", + "sha256:c38c71df845e2aabb7fb0b920d11a1b5ac8526005e533a8920aea97efb8ec6a4", + "sha256:ce15b6d103daff8e9fee13cf7f0add05245a05d866e73926c358e871221eae87", + "sha256:d3029c340cfbb3ac0a71798100ccc13b97dddf373a4ae56b6a72cf70dfd53bc8", + "sha256:e512d8ef5ad7b898cdb2d8ee1cb09a8339e4f8be706d27eaa180c2f177248a10", + "sha256:e8e5b509d5c2ff12f8418006d5a90e9436766133b564db0abaec92fd27fcee29", + "sha256:ee54ff27bf0afaf4c3b3a62bcd016c12c3fdb4ec4f413391a90bd38bc3624605", + "sha256:fa4537fb4a98fe8fde99626e4681cc644bdcf2a795038533f9f711513a862ae6", + "sha256:fd45ff9293d9274c5008a2054ecef86a9bfe819a67c7be1afb65e69b405b3042" + ], + "version": "==2021.4.4" + }, + "requests": { + "hashes": [ + "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804", + "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==2.25.1" + }, + "requests-oauthlib": { + "hashes": [ + "sha256:7f71572defaecd16372f9006f33c2ec8c077c3cfa6f5911a9a90202beb513f3d", + "sha256:b4261601a71fd721a8bd6d7aa1cc1d6a8a93b4a9f5e96626f8e4d91e8beeaa6a", + "sha256:fa6c47b933f01060936d87ae9327fead68768b69c6c9ea2109c48be30f2d4dbc" + ], + "version": "==1.3.0" + }, + "rsa": { + "hashes": [ + "sha256:78f9a9bf4e7be0c5ded4583326e7461e3a3c5aae24073648b4bdfa797d78c9d2", + "sha256:9d689e6ca1b3038bc82bf8d23e944b6b6037bc02301a574935b2dd946e0353b9" + ], + "index": "pypi", + "version": "==4.7.2" + }, + "s3transfer": { + "hashes": [ + "sha256:35627b86af8ff97e7ac27975fe0a98a312814b46c6333d8a6b889627bcd80994", + "sha256:efa5bd92a897b6a8d5c1383828dca3d52d0790e0756d49740563a3fb6ed03246" + ], + "version": "==0.3.7" + }, + "six": { + "hashes": [ + "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", + "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.16.0" + }, + "sonar": { + "git": "https://github.com/mongodb/sonar", + "ref": "37edec8af37d3d17ca194239a3a35c62567788e0" + }, + "toml": { + "hashes": [ + "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", + "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.10.2" + }, + "tqdm": { + "hashes": [ + "sha256:8f3c5815e3b5e20bc40463fa6b42a352178859692a68ffaa469706e6d38342a5", + "sha256:faf9c671bd3fad5ebaeee366949d969dca2b2be32c872a7092a1e1a9048d105b" + ], + "index": "pypi", + "version": "==v4.49.0" + }, + "typed-ast": { + "hashes": [ + "sha256:01ae5f73431d21eead5015997ab41afa53aa1fbe252f9da060be5dad2c730ace", + "sha256:067a74454df670dcaa4e59349a2e5c81e567d8d65458d480a5b3dfecec08c5ff", + "sha256:0fb71b8c643187d7492c1f8352f2c15b4c4af3f6338f21681d3681b3dc31a266", + "sha256:1b3ead4a96c9101bef08f9f7d1217c096f31667617b58de957f690c92378b528", + "sha256:2068531575a125b87a41802130fa7e29f26c09a2833fea68d9a40cf33902eba6", + "sha256:209596a4ec71d990d71d5e0d312ac935d86930e6eecff6ccc7007fe54d703808", + "sha256:2c726c276d09fc5c414693a2de063f521052d9ea7c240ce553316f70656c84d4", + "sha256:398e44cd480f4d2b7ee8d98385ca104e35c81525dd98c519acff1b79bdaac363", + "sha256:52b1eb8c83f178ab787f3a4283f68258525f8d70f778a2f6dd54d3b5e5fb4341", + "sha256:5feca99c17af94057417d744607b82dd0a664fd5e4ca98061480fd8b14b18d04", + "sha256:7538e495704e2ccda9b234b82423a4038f324f3a10c43bc088a1636180f11a41", + "sha256:760ad187b1041a154f0e4d0f6aae3e40fdb51d6de16e5c99aedadd9246450e9e", + "sha256:777a26c84bea6cd934422ac2e3b78863a37017618b6e5c08f92ef69853e765d3", + "sha256:95431a26309a21874005845c21118c83991c63ea800dd44843e42a916aec5899", + "sha256:9ad2c92ec681e02baf81fdfa056fe0d818645efa9af1f1cd5fd6f1bd2bdfd805", + "sha256:9c6d1a54552b5330bc657b7ef0eae25d00ba7ffe85d9ea8ae6540d2197a3788c", + "sha256:aee0c1256be6c07bd3e1263ff920c325b59849dc95392a05f258bb9b259cf39c", + "sha256:af3d4a73793725138d6b334d9d247ce7e5f084d96284ed23f22ee626a7b88e39", + "sha256:b36b4f3920103a25e1d5d024d155c504080959582b928e91cb608a65c3a49e1a", + "sha256:b9574c6f03f685070d859e75c7f9eeca02d6933273b5e69572e5ff9d5e3931c3", + "sha256:bff6ad71c81b3bba8fa35f0f1921fb24ff4476235a6e94a26ada2e54370e6da7", + "sha256:c190f0899e9f9f8b6b7863debfb739abcb21a5c054f911ca3596d12b8a4c4c7f", + "sha256:c907f561b1e83e93fad565bac5ba9c22d96a54e7ea0267c708bffe863cbe4075", + "sha256:cae53c389825d3b46fb37538441f75d6aecc4174f615d048321b716df2757fb0", + "sha256:dd4a21253f42b8d2b48410cb31fe501d32f8b9fbeb1f55063ad102fe9c425e40", + "sha256:dde816ca9dac1d9c01dd504ea5967821606f02e510438120091b84e852367428", + "sha256:f2362f3cb0f3172c42938946dbc5b7843c2a28aec307c49100c8b38764eb6927", + "sha256:f328adcfebed9f11301eaedfa48e15bdece9b519fb27e6a8c01aa52a17ec31b3", + "sha256:f8afcf15cc511ada719a88e013cec87c11aff7b91f019295eb4530f96fe5ef2f", + "sha256:fb1bbeac803adea29cedd70781399c99138358c26d05fcbd23c13016b7f5ec65" + ], + "version": "==1.4.3" + }, + "typing-extensions": { + "hashes": [ + "sha256:0ac0f89795dd19de6b97debb0c6af1c70987fd80a2d62d1958f7e56fcc31b497", + "sha256:50b6f157849174217d0656f99dc82fe932884fb250826c18350e159ec6cdf342", + "sha256:779383f6086d90c99ae41cf0ff39aac8a7937a9283ce0a414e5dd782f4c94a84" + ], + "version": "==3.10.0.0" + }, + "urllib3": { + "hashes": [ + "sha256:753a0374df26658f99d826cfe40394a686d05985786d946fbe4165b5148f5a7c", + "sha256:a7acd0977125325f516bda9735fa7142b909a8d01e8b2e4c8108d0984e6e0098" + ], + "markers": "python_version != '3.4'", + "version": "==1.26.5" + }, + "websocket-client": { + "hashes": [ + "sha256:3e2bf58191d4619b161389a95bdce84ce9e0b24eb8107e7e590db682c2d0ca81", + "sha256:abf306dc6351dcef07f4d40453037e51cc5d9da2ef60d0fc5d0fe3bcda255372" + ], + "markers": "python_version >= '3.6'", + "version": "==1.0.1" + } + }, + "develop": {} +} diff --git a/dev_notes/dev-quick-start.md b/dev_notes/dev-quick-start.md index b0512430c..92cae2114 100644 --- a/dev_notes/dev-quick-start.md +++ b/dev_notes/dev-quick-start.md @@ -5,18 +5,28 @@ * Install [jq](https://stedolan.github.io/jq/download/) * Optionally install [kind](https://kind.sigs.k8s.io/docs/user/quick-start/#installation) if you want to use a local cluster. -* create a python virtual environment +* create a python virtual environment and activate it (skip if you use pipenv) ```bash python3 -m venv /path/to/new/virtual/environment source path/to/new/virtual/environment/bin/activate ``` -* install python dependencies -``` +* install python dependencies (skip if you use pipenv) +```bash pip install -r requirements.txt ``` +* create a python environment and install dependencies (for pipenv users) +```bash +pipenv install +``` + +* activate the python environment (for pipenv users) +```bash +pipenv shell +``` + #### Create a Kind cluster and a local registry ```bash ./scripts/dev/setup_kind_cluster.sh diff --git a/requirements.txt b/requirements.txt index 82eef4f61..14e01968f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -git+https://github.com/mongodb/sonar@0.0.11 +git+https://github.com/mongodb/sonar@0.0.11#egg=sonar docker==4.3.1 kubernetes==11.0.0 jinja2==2.11.3 From 48376c2dbc936924c7385cb1a7b4ceaae317dfe1 Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Thu, 3 Jun 2021 12:56:35 +0100 Subject: [PATCH 290/790] Merge contributing.md and dev_quick start docs (#529) --- .github/PULL_REQUEST_TEMPLATE.md | 1 + Makefile | 5 - dev_notes/config-options.md | 12 -- .../dev_quick_start_rs.yaml | 26 ---- docs/contributing.md | 121 ++++++++++++++---- 5 files changed, 94 insertions(+), 71 deletions(-) delete mode 100644 dev_notes/config-options.md delete mode 100644 dev_notes/dev_quick_start_resources/dev_quick_start_rs.yaml diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index f77733901..7d52039b2 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -2,5 +2,6 @@ * [ ] Have you opened an Issue before filing this PR? * [ ] Have you signed our [CLA](https://www.mongodb.com/legal/contributor-agreement)? +* [ ] Have you signed all of your commits? * [ ] Have you checked to ensure there aren't other open [Pull Requests](../../../pulls) for the same update/change? * [ ] Put `closes #XXXX` in your comment to auto-close the issue that your PR fixes (if such). diff --git a/Makefile b/Makefile index 0280984bc..06784c51f 100644 --- a/Makefile +++ b/Makefile @@ -48,11 +48,6 @@ deploy: manifests kustomize cd config/manager && $(KUSTOMIZE) edit set image quay.io/mongodb/mongodb-kubernetes-operator=$(IMG):latest $(KUSTOMIZE) build config/default | kubectl apply -n $(NAMESPACE) -f - -# Deploy a simple ReplicaSet, this is intended for first time use only as part of the quick start guide. -deploy-dev-quick-start-rs: manifests kustomize - kubectl create secret generic my-user-password --from-literal=password=dev-quick-start-password --from-literal=username=admin || true - kubectl apply -f dev_notes/dev_quick_start_resources/dev_quick_start_rs.yaml - # UnDeploy controller from the configured Kubernetes cluster in ~/.kube/config undeploy: $(KUSTOMIZE) build config/default | kubectl delete -f - diff --git a/dev_notes/config-options.md b/dev_notes/config-options.md deleted file mode 100644 index 6e28df8e3..000000000 --- a/dev_notes/config-options.md +++ /dev/null @@ -1,12 +0,0 @@ - -#### Config Options - -1. `namespace` is the namespace that will be used by scripts/tooling. All of the resources will be deployed here. -2. `operator_name` will be used as the name of the operator deployment, and the name of the operator image when build. -3. `image_type` this can be either `ubi` or `ubuntu` and determines the distro of the images built. (currently only the agent image has multiple distros) -4. `repo_url` the repository that should be used to push/pull all images. -5. `e2e_image` the name of e2e test image that will be built. -6. `version_upgrade_hook_image` the name of the version upgrade post start hook image. -7. `agent_image_ubuntu` the name of the ubuntu agent image. -8. `agent_image_ubi` the name of the ubi agent image. -9. `s3_bucket` the S3 bucket that Dockerfiles will be pushed to as part of the release process. Note: this is only required when running the release tasks locally. diff --git a/dev_notes/dev_quick_start_resources/dev_quick_start_rs.yaml b/dev_notes/dev_quick_start_resources/dev_quick_start_rs.yaml deleted file mode 100644 index 688385f5b..000000000 --- a/dev_notes/dev_quick_start_resources/dev_quick_start_rs.yaml +++ /dev/null @@ -1,26 +0,0 @@ ---- -apiVersion: mongodbcommunity.mongodb.com/v1 -kind: MongoDBCommunity -metadata: - name: quick-start-rs -spec: - members: 3 - type: ReplicaSet - version: "4.2.6" - security: - authentication: - modes: ["SCRAM"] - users: - - name: my-user - db: admin - passwordSecretRef: # a reference to the secret that will be used to generate the user's password - name: my-user-password - roles: - - name: clusterAdmin - db: admin - - name: userAdminAnyDatabase - db: admin - scramCredentialsSecretName: my-scram - additionalMongodConfig: - storage.wiredTiger.engineConfig.journalCompressor: zlib - diff --git a/docs/contributing.md b/docs/contributing.md index 2519e1644..5acb649f1 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -39,7 +39,14 @@ by the operator and mounted in the Agent's Pod. Each Pod holds a member of a Replica Set, and each Pod has different components, each one of them in charge of some part of the lifecycle of the MongoDB database. -# Developing locally +# Getting Started + +## PR Prerequisites +* Please ensure you have signed our Contributor Agreement. You can find it [here](https://www.mongodb.com/legal/contributor-agreement). + +* Please ensure that all commits are signed. + +## Developer Configuration The operator is built using `golang`. We use a simple json file that describe some local options that you need to set for the testing environment @@ -52,25 +59,35 @@ to be able to run properly. Create a json file with the following content: "operator_image": "mongodb-kubernetes-operator", "e2e_image": "community-e2e", "version_upgrade_hook_image": "community-operator-version-upgrade-post-start-hook", - "prestop_hook_image": "prehook", "agent_image_ubuntu": "mongodb-agent-dev", "agent_image_ubi": "mongodb-agent-ubi-dev", - "readiness_probe_image": "mongodb-kubernetes-readiness" + "readiness_probe_image": "mongodb-kubernetes-readiness", + "s3_bucket": "" } ``` -The `namespace` attribute sets the Kubernetes namespace to be used for running -your tests. The `repo_url` sets the Docker registry. In my case I have a -`local-config.json` file in the root of this repo. For the e2e tests to pick -this file, set the `MONGODB_COMMUNITY_CONFIG` env variable to the absolute path -of this file. +#### Config Options + +1. `namespace` is the namespace that will be used by scripts/tooling. All the resources will be deployed here. +2. `repo_url` the repository that should be used to push/pull all images. +3. `operator_image` will be used as the name of the operator deployment, and the name of the operator image when build. +4. `e2e_image` the name of e2e test image that will be built. +5. `version_upgrade_hook_image` the name of the version upgrade post start hook image. +6. `image_type` this can be either `ubi` or `ubuntu` and determines the distro of the images built. (currently only the agent image has multiple distros) +7. `agent_image_ubuntu` the name of the ubuntu agent image. +8. `agent_image_ubi` the name of the ubi agent image. +9. `s3_bucket` the S3 bucket that Dockerfiles will be pushed to as part of the release process. Note: this is only required when running the release tasks locally. + + +You can set the `MONGODB_COMMUNITY_CONFIG` environment variable to be the absolute path of this file. +It will default to `~/.community-operator-dev/config.json` Please see [here](./build_operator_locally.md) to see how to build and deploy the operator locally. ## Configure Docker registry -The build process consist in multiple Docker images being built, you need to specify -where you want the locally build images to be pushed. The Docker registry needs to be +The build process consists of multiple Docker images being built. You need to specify +where you want the locally built images to be pushed. The Docker registry needs to be accessible from your Kubernetes cluster. ## Test Namespace @@ -81,16 +98,59 @@ instance, you can leave this as `default`. ## Python Environment The test runner is a Python script, in order to use it a virtualenv needs to be -created. The dependencies of the Python environment are described, as usual, in -a `requirements.txt` file: - +created. + +### Pip ```sh python -m venv venv source venv/bin/activate python -m pip install -r requirements.txt ``` -# Running Unit tests +### Pipenv + +* create a python environment and install dependencies. +```bash +pipenv install +``` + +* activate the python environment. +```bash +pipenv shell +``` + + +# Deploying the Operator + +In order to deploy the Operator from source, you can run the following command. + +```sh +make operator-image deploy +``` + +This will build and deploy the operator to namespace specified in your configuration file. + + +#### See the operator deployment +```sh +kubectl get pods +``` + +#### (Optional) Create a MongoDBCommunity Resource + +Follow the steps outlined [here](./deploy-configure.md) to deploy some resource. + +#### Cleanup +To remove the operator and any created resources you can run + +```sh +make undeploy +``` + + +# Running Tests + +### Unit tests Unit tests should be run from the root of the project with: @@ -98,9 +158,22 @@ Unit tests should be run from the root of the project with: make test ``` -# Running E2E Tests +### E2E Tests + +If this is the first time running E2E tests, you will need to ensure that you have built and pushed +all images required by the E2E tests. You can do this by running. + +```sh +make all-images +``` + +For subsequent tests you can use + +```sh +make e2e-k8s test= +``` -## Running an E2E test +This will only re-build the e2e test image. We have built a simple mechanism to run E2E tests on your cluster using a runner that deploys a series of Kubernetes objects, runs them, and awaits for their @@ -122,25 +195,22 @@ replica_set_scale The tests should run individually using the runner like this: ```sh -make e2e test= +make e2e-k8s test= ``` -This will run the `replica_set` E2E test which is a simple test that installs a +This will run the `replica_set` E2E test which is a simple test which installs a MongoDB Replica Set and asserts that the deployed server can be connected to. - -to get a list. - ## Troubleshooting When you run a test locally, if the `e2e-test` pod is present, you will have to first manually delete it; failing to do so will cause the `e2e-test` pod to fail. # Writing new E2E tests -You can start with `replica_set` test as an starting point to write a new test. +You can start with the `replica_set` test as a starting point to write a new test. The tests are written using `envtest` and they are run using `go test`. -Adding a new test is as easy as to create a new directory in `test/e2e` with the +Adding a new test is as easy as creating a new directory in `test/e2e` with the new E2E test, and to run them: ```sh @@ -159,8 +229,3 @@ To set up the pre-commit hooks, please create symbolic links from the provided [ * Create a symlink for every file in the `scripts/git-hooks` directory: `ln -s -f ../../scripts/git-hooks/* .` - - -## Please make sure you sign our Contributor Agreement -You can find it [here](https://www.mongodb.com/legal/contributor-agreement). This will be -required when creating a PR against this repo! From 11623b79d89dc344ff981a493cd3dd1d6435859d Mon Sep 17 00:00:00 2001 From: Rajdeep Das Date: Thu, 3 Jun 2021 15:55:58 +0200 Subject: [PATCH 291/790] Consolidate documentations in one docs folder and removed uneeded files (#530) --- dev_notes/dev-quick-start.md | 113 ------------------ dev_notes/past_release_notes/v0.5.2.md | 27 ----- dev_notes/past_release_notes/v0.6.0.md | 28 ----- {dev_notes => docs}/RELEASE_NOTES.md | 0 {dev_notes => docs}/how-to-release.md | 0 {dev_notes => docs}/release-notes-template.md | 0 6 files changed, 168 deletions(-) delete mode 100644 dev_notes/dev-quick-start.md delete mode 100644 dev_notes/past_release_notes/v0.5.2.md delete mode 100644 dev_notes/past_release_notes/v0.6.0.md rename {dev_notes => docs}/RELEASE_NOTES.md (100%) rename {dev_notes => docs}/how-to-release.md (100%) rename {dev_notes => docs}/release-notes-template.md (100%) diff --git a/dev_notes/dev-quick-start.md b/dev_notes/dev-quick-start.md deleted file mode 100644 index 92cae2114..000000000 --- a/dev_notes/dev-quick-start.md +++ /dev/null @@ -1,113 +0,0 @@ - -#### Prerequisites - -* Install [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) -* Install [jq](https://stedolan.github.io/jq/download/) -* Optionally install [kind](https://kind.sigs.k8s.io/docs/user/quick-start/#installation) if you want to use a local cluster. - -* create a python virtual environment and activate it (skip if you use pipenv) - -```bash -python3 -m venv /path/to/new/virtual/environment -source path/to/new/virtual/environment/bin/activate -``` - -* install python dependencies (skip if you use pipenv) -```bash -pip install -r requirements.txt -``` - -* create a python environment and install dependencies (for pipenv users) -```bash -pipenv install -``` - -* activate the python environment (for pipenv users) -```bash -pipenv shell -``` - -#### Create a Kind cluster and a local registry -```bash -./scripts/dev/setup_kind_cluster.sh -``` - -#### set the kind kubernetes context -```bash -export KUBECONFIG=~/.kube/kind -``` - -#### Get kind credentials -```bash -kind export kubeconfig - -# check it worked by running: -kubectl cluster-info --context kind-kind --kubeconfig $KUBECONFIG -``` - - -#### (Optional) Create a non-default namespace to work in -```bash -kubectl create namespace mongodb - -# optionally set it as the default -kubectl config set-context --current --namespace=mongodb -``` - -#### create a config file for the dev environment -```bash -mkdir -p ~/.community-operator-dev -cat > ~/.community-operator-dev/config.json << EOL -{ - "namespace": "default", - "repo_url": "localhost:5000", - "operator_image": "community-operator-dev", - "e2e_image": "e2e", - "prestop_hook_image": "prehook", - "version_upgrade_hook_image": "community-operator-version-upgrade-post-start-hook", - "readiness_probe_image": "mongodb-kubernetes-readinessprobe", - "s3_bucket": "" -} -EOL -``` - -#### build all required images and deploy operator to cluster -```bash -make all-images deploy -``` - -#### See the operator deployment -```bash -kubectl get pods -``` - -#### Deploy a Replica Set -```bash -make deploy-dev-quick-start-rs -``` - -#### See the deployed replica set -```bash -kubectl get pods - -NAME READY STATUS RESTARTS AGE -mongodb-kubernetes-operator-5568d769b8-smt4h 1/1 Running 0 4m12s -quick-start-rs-0 2/2 Running 0 2m49s -quick-start-rs-1 2/2 Running 0 2m5s -quick-start-rs-2 2/2 Running 0 87s - - -kubectl get statefulset quick-start-rs - -NAME READY AGE -quick-start-rs 3/3 3m10s -``` - -### Clean up all resources -```bash -make undeploy -``` - -### Running Tests - -Follow the tests in [contributing.md](../docs/contributing.md) to run e2e tests. \ No newline at end of file diff --git a/dev_notes/past_release_notes/v0.5.2.md b/dev_notes/past_release_notes/v0.5.2.md deleted file mode 100644 index f333ce5fa..000000000 --- a/dev_notes/past_release_notes/v0.5.2.md +++ /dev/null @@ -1,27 +0,0 @@ -# MongoDB Kubernetes Operator 0.5.2 -## Kubernetes Operator -* Changes - * Readiness probe has been moved into an init container from the Agent image. - * Security context is now added when the `MANAGED_SECURITY_CONTEXT` environment variable is not set. -* Bug fixes - * Removed unnecessary environment variable configuration in the openshift samples. - * Fixed an issue where the operator would perform unnecessary reconcilliations when Secrets were modified. - * Fixed an issue where a race condition could cause the deployment to get into a bad state when TLS - settings when being changed at the same time as a scaling operation was happening. - * Fixed an issue where the agent pod would panic when running as a non-root user. - -## MongoDBCommunity Resource -* Changes - * Added `spec.security.authentication.ignoreUnknownUsers` field. This value defaults to `true`. When enabled, - any MongoDB users added through external sources will not be removed. - - -## Miscellaneous -* Changes - * Internal code refactorings to allow libraries to be imported into other projects. - - - ## Updated Image Tags - * mongodb-kubernetes-operator:0.5.2 - * mongodb-agent:10.27.0.6772-1 - * mongodb-kubernetes-readinessprobe:1.0.1 [new image] diff --git a/dev_notes/past_release_notes/v0.6.0.md b/dev_notes/past_release_notes/v0.6.0.md deleted file mode 100644 index 33adf0f8a..000000000 --- a/dev_notes/past_release_notes/v0.6.0.md +++ /dev/null @@ -1,28 +0,0 @@ -# MongoDB Kubernetes Operator 0.6.0 -## Kubernetes Operator - -* Breaking Changes - * A new VolumeClaimTemplate has been added `logs-volume`. When you deploy the operator, if there is an existing StatefulSet the operator will attempt to perform an invalid update. The existing StatefulSet must be deleted before upgrading the operator. - - * The user of the mongod and mongodb-agent containers has changed. This means that there will be permissions - issues when upgrading from an earlier version of the operator. In order to update the permissions in the volume, you can use an init container. - -* Upgrade instructions - - Remove the current operator deployment - - `kubectl delete deployment ` - Delete the existing StatefulSet for the MongoDBCommunity resource - Note: to ensure existing data is not lost, ensure that the retain policy of your Persistent Volumes is configured correctly. - - `kubectl delete statefulset ` - Install the new operator - - follow the regular [installation instruction](https://github.com/mongodb/mongodb-kubernetes-operator/blob/master/docs/install-upgrade.md) - Patch the StatefulSet once it has been created. This will add an init container that will update the permissions of the existing volume. - - `kubectl patch statefulset --type='json' --patch '[ {"op":"add","path":"/spec/template/spec/initContainers/-", "value": { "name": "change-data-dir-permissions", "image": "busybox", "command": [ "chown", "-R", "2000", "/data" ], "securityContext": { "runAsNonRoot": false, "runAsUser": 0, "runAsGroup":0 }, "volumeMounts": [ { "mountPath": "/data", "name" : "data-volume" } ] } } ]'` - -* Bug fixes - * Fixes an issue that prevented the agents from reaching goal state when upgrading minor version of MongoDB. - - ## Updated Image Tags - * mongodb-kubernetes-operator:0.6.0 - * mongodb-agent:0.29.0.6830-1 - * mongodb-kubernetes-readinessprobe:1.0.3 diff --git a/dev_notes/RELEASE_NOTES.md b/docs/RELEASE_NOTES.md similarity index 100% rename from dev_notes/RELEASE_NOTES.md rename to docs/RELEASE_NOTES.md diff --git a/dev_notes/how-to-release.md b/docs/how-to-release.md similarity index 100% rename from dev_notes/how-to-release.md rename to docs/how-to-release.md diff --git a/dev_notes/release-notes-template.md b/docs/release-notes-template.md similarity index 100% rename from dev_notes/release-notes-template.md rename to docs/release-notes-template.md From f923235f9a4673f6aeb3e826e549d58814e4f39b Mon Sep 17 00:00:00 2001 From: Ciprian Tibulca Date: Fri, 4 Jun 2021 09:20:14 +0100 Subject: [PATCH 292/790] CLOUDP-89492 Expose connection string for deployments for easy of use (#523) * publish connection strings - scram * expose connection strings - e2e test * expose connection strings - validate users config --- api/v1/mongodbcommunity_types.go | 33 +++++++++-- controllers/mongodb_users.go | 56 +++++++++++++++++++ controllers/replica_set_controller.go | 31 +++++++--- controllers/validation/validation.go | 48 +++++++++++++++- controllers/watch/watch_test.go | 2 +- pkg/authentication/scram/scram.go | 6 ++ test/e2e/mongodbtests/mongodbtests.go | 35 ++++++++++++ test/e2e/replica_set/replica_set_test.go | 8 ++- .../replica_set_tls/replica_set_tls_test.go | 14 +++-- test/e2e/util/mongotester/mongotester.go | 4 +- 10 files changed, 212 insertions(+), 25 deletions(-) create mode 100644 controllers/mongodb_users.go diff --git a/api/v1/mongodbcommunity_types.go b/api/v1/mongodbcommunity_types.go index 2f0e8a01b..5a353e12a 100644 --- a/api/v1/mongodbcommunity_types.go +++ b/api/v1/mongodbcommunity_types.go @@ -3,6 +3,7 @@ package v1 import ( "encoding/json" "fmt" + "net/url" "strings" "github.com/mongodb/mongodb-kubernetes-operator/pkg/authentication/scram" @@ -441,12 +442,7 @@ func (m MongoDBCommunity) AutomationConfigMembersThisReconciliation() int { // MongoURI returns a mongo uri which can be used to connect to this deployment func (m MongoDBCommunity) MongoURI() string { - members := make([]string, m.Spec.Members) - clusterDomain := "svc.cluster.local" // TODO: make this configurable - for i := 0; i < m.Spec.Members; i++ { - members[i] = fmt.Sprintf("%s-%d.%s.%s.%s:%d", m.Name, i, m.ServiceName(), m.Namespace, clusterDomain, 27017) - } - return fmt.Sprintf("mongodb://%s", strings.Join(members, ",")) + return fmt.Sprintf("mongodb://%s", strings.Join(m.Hosts(), ",")) } // MongoSRVURI returns a mongo srv uri which can be used to connect to this deployment @@ -455,6 +451,31 @@ func (m MongoDBCommunity) MongoSRVURI() string { return fmt.Sprintf("mongodb+srv://%s.%s.%s", m.ServiceName(), m.Namespace, clusterDomain) } +// MongoAuthUserURI returns a mongo uri which can be used to connect to this deployment +// and includes the authentication data for the user +func (m MongoDBCommunity) MongoAuthUserURI(user scram.User, password string) string { + return fmt.Sprintf("mongodb://%s:%s@%s/%s?ssl=%t", + url.QueryEscape(user.Username), + url.QueryEscape(password), + strings.Join(m.Hosts(), ","), + user.Database, + m.Spec.Security.TLS.Enabled) +} + +// MongoAuthUserSRVURI returns a mongo srv uri which can be used to connect to this deployment +// and includes the authentication data for the user +func (m MongoDBCommunity) MongoAuthUserSRVURI(user scram.User, password string) string { + clusterDomain := "svc.cluster.local" // TODO: make this configurable + return fmt.Sprintf("mongodb+srv://%s:%s@%s.%s.%s/%s?ssl=%t", + url.QueryEscape(user.Username), + url.QueryEscape(password), + m.ServiceName(), + m.Namespace, + clusterDomain, + user.Database, + m.Spec.Security.TLS.Enabled) +} + func (m MongoDBCommunity) Hosts() []string { hosts := make([]string, m.Spec.Members) clusterDomain := "svc.cluster.local" // TODO: make this configurable diff --git a/controllers/mongodb_users.go b/controllers/mongodb_users.go new file mode 100644 index 000000000..480332da7 --- /dev/null +++ b/controllers/mongodb_users.go @@ -0,0 +1,56 @@ +package controllers + +import ( + "fmt" + + mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/secret" + apiErrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" +) + +// ensureUserResources will check that the configured user password secrets can be found +// and will start monitor them so that the reconcile process is triggered every time these secrets are updated +func (r ReplicaSetReconciler) ensureUserResources(mdb mdbv1.MongoDBCommunity) error { + for _, user := range mdb.GetScramUsers() { + secretNamespacedName := types.NamespacedName{Name: user.PasswordSecretName, Namespace: mdb.Namespace} + if _, err := secret.ReadKey(r.client, user.PasswordSecretKey, secretNamespacedName); err != nil { + if apiErrors.IsNotFound(err) { + return fmt.Errorf(`user password secret "%s" not found: %s`, secretNamespacedName, err) + } + return err + } + + r.secretWatcher.Watch(secretNamespacedName, mdb.NamespacedName()) + } + + return nil +} + +// updateConnectionStringSecrets updates secrets where user specific connection strings are stored. +// The client applications can mount these secrets and connect to the mongodb cluster +func (r ReplicaSetReconciler) updateConnectionStringSecrets(mdb mdbv1.MongoDBCommunity) error { + for _, user := range mdb.GetScramUsers() { + secretNamespacedName := types.NamespacedName{Name: user.PasswordSecretName, Namespace: mdb.Namespace} + pwd, err := secret.ReadKey(r.client, user.PasswordSecretKey, secretNamespacedName) + if err != nil { + return err + } + + connectionStringSecret := secret.Builder(). + SetName(user.GetConnectionStringSecretName(mdb)). + SetNamespace(mdb.Namespace). + SetField("connectionString.standard", mdb.MongoAuthUserURI(user, pwd)). + SetField("connectionString.standardSrv", mdb.MongoAuthUserSRVURI(user, pwd)). + SetField("username", user.Username). + SetField("password", pwd). + SetOwnerReferences(mdb.GetOwnerReferences()). + Build() + + if err := secret.CreateOrUpdate(r.client, connectionStringSecret); err != nil { + return err + } + } + + return nil +} diff --git a/controllers/replica_set_controller.go b/controllers/replica_set_controller.go index 3cd1cce45..8e24f22da 100644 --- a/controllers/replica_set_controller.go +++ b/controllers/replica_set_controller.go @@ -8,6 +8,7 @@ import ( "github.com/mongodb/mongodb-kubernetes-operator/controllers/predicates" "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/source" "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/container" @@ -81,6 +82,7 @@ func NewReconciler(mgr manager.Manager) *ReplicaSetReconciler { func (r *ReplicaSetReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&mdbv1.MongoDBCommunity{}, builder.WithPredicates(predicates.OnlyOnSpecChange())). + Watches(&source.Kind{Type: &corev1.Secret{}}, r.secretWatcher). Complete(r) } @@ -127,7 +129,7 @@ func (r ReplicaSetReconciler) Reconcile(ctx context.Context, request reconcile.R r.log.Infow("Reconciling MongoDB", "MongoDB.Spec", mdb.Spec, "MongoDB.Status", mdb.Status) r.log.Debug("Validating MongoDB.Spec") - if err := r.validateUpdate(mdb); err != nil { + if err := r.validateSpec(mdb); err != nil { return status.Update(r.client.Status(), &mdb, statusOptions(). withMessage(Error, fmt.Sprintf("error validating new Spec: %s", err)). @@ -169,6 +171,14 @@ func (r ReplicaSetReconciler) Reconcile(ctx context.Context, request reconcile.R ) } + if err := r.ensureUserResources(mdb); err != nil { + return status.Update(r.client.Status(), &mdb, + statusOptions(). + withMessage(Error, fmt.Sprintf("Error ensuring User config: %s", err)). + withFailedPhase(), + ) + } + ready, err := r.deployMongoDBReplicaSet(mdb) if err != nil { return status.Update(r.client.Status(), &mdb, @@ -218,6 +228,10 @@ func (r ReplicaSetReconciler) Reconcile(ctx context.Context, request reconcile.R return res, err } + if err := r.updateConnectionStringSecrets(mdb); err != nil { + r.log.Errorf("Could not update connection string secrets: %s", err) + } + // the last version will be duplicated in two annotations. // This is needed to reuse the update strategy logic in enterprise if err := annotations.UpdateLastAppliedMongoDBVersion(&mdb, r.client); err != nil { @@ -457,14 +471,15 @@ func buildService(mdb mdbv1.MongoDBCommunity) corev1.Service { Build() } -// validateUpdate validates that the new Spec, corresponding to the existing one -// is still valid. If there is no a previous Spec, then the function assumes this is -// the first version of the MongoDB resource and skips. -func (r ReplicaSetReconciler) validateUpdate(mdb mdbv1.MongoDBCommunity) error { +// validateSpec checks if MongoDB resource Spec is valid. +// If there is no a previous Spec, then the function assumes this is the first version +// and runs the initial spec validations otherwise it validates that the new Spec, +// corresponding to the existing one is still valid +func (r ReplicaSetReconciler) validateSpec(mdb mdbv1.MongoDBCommunity) error { lastSuccessfulConfigurationSaved, ok := mdb.Annotations[lastSuccessfulConfiguration] if !ok { - // First version of Spec, no need to validate - return nil + // First version of Spec + return validation.ValidateInitalSpec(mdb) } prevSpec := mdbv1.MongoDBCommunitySpec{} @@ -473,7 +488,7 @@ func (r ReplicaSetReconciler) validateUpdate(mdb mdbv1.MongoDBCommunity) error { return err } - return validation.Validate(prevSpec, mdb.Spec) + return validation.ValidateUpdate(mdb, prevSpec) } func getCustomRolesModification(mdb mdbv1.MongoDBCommunity) (automationconfig.Modification, error) { diff --git a/controllers/validation/validation.go b/controllers/validation/validation.go index bd67fbcb9..7c68d2b60 100644 --- a/controllers/validation/validation.go +++ b/controllers/validation/validation.go @@ -1,14 +1,58 @@ package validation import ( + "fmt" + "strings" + mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/authentication/scram" "github.com/pkg/errors" ) -func Validate(oldSpec, newSpec mdbv1.MongoDBCommunitySpec) error { - if oldSpec.Security.TLS.Enabled && !newSpec.Security.TLS.Enabled { +// ValidateInitalSpec checks if the resource initial Spec is valid +func ValidateInitalSpec(mdb mdbv1.MongoDBCommunity) error { + if err := validateUsers(mdb); err != nil { + return err + } + + return nil +} + +// ValidateUpdate validates that the new Spec, corresponding to the existing one is still valid +func ValidateUpdate(mdb mdbv1.MongoDBCommunity, oldSpec mdbv1.MongoDBCommunitySpec) error { + if oldSpec.Security.TLS.Enabled && !mdb.Spec.Security.TLS.Enabled { return errors.New("TLS can't be set to disabled after it has been enabled") } + if err := validateUsers(mdb); err != nil { + return err + } + + return nil +} + +// validateUsers checks if the users configuration is valid +func validateUsers(mdb mdbv1.MongoDBCommunity) error { + connectionStringSecretNameMap := map[string]scram.User{} + nameCollisions := []string{} + for _, user := range mdb.GetScramUsers() { + secretName := user.GetConnectionStringSecretName(mdb) + if previousUser, exists := connectionStringSecretNameMap[secretName]; exists { + nameCollisions = append(nameCollisions, + fmt.Sprintf(`[secret name: "%s" for user: "%s", db: "%s" and user: "%s", db: "%s"]`, + secretName, + previousUser.Username, + previousUser.Database, + user.Username, + user.Database)) + } else { + connectionStringSecretNameMap[secretName] = user + } + } + if len(nameCollisions) > 0 { + return errors.Errorf("connection string secret names collision, update at least one of the users so that the resulted secret names (--) are unique: %s", + strings.Join(nameCollisions, ", ")) + } + return nil } diff --git a/controllers/watch/watch_test.go b/controllers/watch/watch_test.go index ad4202445..027c1b78d 100644 --- a/controllers/watch/watch_test.go +++ b/controllers/watch/watch_test.go @@ -53,7 +53,7 @@ func TestWatcher(t *testing.T) { assert.Equal(t, 0, queue.Len()) }) - t.Run("Multiple objects to reconile", func(t *testing.T) { + t.Run("Multiple objects to reconcile", func(t *testing.T) { watcher := New() queue := controllertest.Queue{Interface: workqueue.New()} watcher.Watch(objNsName, mdb1.NamespacedName()) diff --git a/pkg/authentication/scram/scram.go b/pkg/authentication/scram/scram.go index 79ea36ff9..2887f7d11 100644 --- a/pkg/authentication/scram/scram.go +++ b/pkg/authentication/scram/scram.go @@ -2,6 +2,7 @@ package scram import ( "encoding/base64" + "fmt" "github.com/pkg/errors" @@ -336,3 +337,8 @@ func convertMongoDBUserToAutomationConfigUser(secretGetUpdateCreateDeleter secre acUser.ScramSha256Creds = &sha256Creds return acUser, nil } + +// GetConnectionStringSecretName returns the name of the secret where the operator stores the connection string for current user +func (u User) GetConnectionStringSecretName(mdb Configurable) string { + return fmt.Sprintf("%s-%s-%s", mdb.NamespacedName().Name, u.Database, u.Username) +} diff --git a/test/e2e/mongodbtests/mongodbtests.go b/test/e2e/mongodbtests/mongodbtests.go index 2d81df379..29239f209 100644 --- a/test/e2e/mongodbtests/mongodbtests.go +++ b/test/e2e/mongodbtests/mongodbtests.go @@ -11,6 +11,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/authentication/scram" "github.com/mongodb/mongodb-kubernetes-operator/pkg/automationconfig" e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" "github.com/stretchr/testify/assert" @@ -120,6 +121,21 @@ func AgentSecretsHaveOwnerReference(mdb *mdbv1.MongoDBCommunity, expectedOwnerRe } } +// ConnectionStringSecretsAreConfigured verifies that secrets storing the connection string were generated for all scram users +// and that they have the expected owner reference +func ConnectionStringSecretsAreConfigured(mdb *mdbv1.MongoDBCommunity, expectedOwnerReference metav1.OwnerReference) func(t *testing.T) { + return func(t *testing.T) { + for _, user := range mdb.GetScramUsers() { + secret := corev1.Secret{} + secretNamespacedName := types.NamespacedName{Name: user.GetConnectionStringSecretName(mdb), Namespace: mdb.Namespace} + err := e2eutil.TestClient.Get(context.TODO(), secretNamespacedName, &secret) + + assert.NoError(t, err) + assertEqualOwnerReference(t, "Secret", secretNamespacedName, secret.GetOwnerReferences(), expectedOwnerReference) + } + } +} + // StatefulSetHasUpdateStrategy verifies that the StatefulSet holding this MongoDB // resource has the correct Update Strategy func StatefulSetHasUpdateStrategy(mdb *mdbv1.MongoDBCommunity, strategy appsv1.StatefulSetUpdateStrategyType) func(t *testing.T) { @@ -202,6 +218,24 @@ func CreateMongoDBResource(mdb *mdbv1.MongoDBCommunity, ctx *e2eutil.Context) fu } } +// GetConnectionStringSecret returnes the secret generated by the operator that is storing the connection string for a specific user +func GetConnectionStringSecret(mdb mdbv1.MongoDBCommunity, user scram.User) corev1.Secret { + secret := corev1.Secret{} + secretNamespacedName := types.NamespacedName{Name: user.GetConnectionStringSecretName(mdb), Namespace: mdb.Namespace} + e2eutil.TestClient.Get(context.TODO(), secretNamespacedName, &secret) + return secret +} + +// GetConnectionStringForUser returns the mongodb standard connection string for a user +func GetConnectionStringForUser(mdb mdbv1.MongoDBCommunity, user scram.User) string { + return string(GetConnectionStringSecret(mdb, user).Data["connectionString.standard"]) +} + +// GetConnectionStringForUser returns the mongodb service connection string for a user +func GetSrvConnectionStringForUser(mdb mdbv1.MongoDBCommunity, user scram.User) string { + return string(GetConnectionStringSecret(mdb, user).Data["connectionString.standardSrv"]) +} + func getOwnerReference(mdb *mdbv1.MongoDBCommunity) metav1.OwnerReference { return *metav1.NewControllerRef(mdb, schema.GroupVersionKind{ Group: mdbv1.GroupVersion.Group, @@ -219,6 +253,7 @@ func BasicFunctionality(mdb *mdbv1.MongoDBCommunity) func(*testing.T) { t.Run("Stateful Set Has OwnerReference", StatefulSetHasOwnerReference(mdb, mdbOwnerReference)) t.Run("Service Set Has OwnerReference", ServiceHasOwnerReference(mdb, mdbOwnerReference)) t.Run("Agent Secrets Have OwnerReference", AgentSecretsHaveOwnerReference(mdb, mdbOwnerReference)) + t.Run("Connection string secrets are configured", ConnectionStringSecretsAreConfigured(mdb, mdbOwnerReference)) t.Run("Test Status Was Updated", Status(mdb, mdbv1.MongoDBCommunityStatus{ MongoURI: mdb.MongoURI(), diff --git a/test/e2e/replica_set/replica_set_test.go b/test/e2e/replica_set/replica_set_test.go index a213e3113..74249b524 100644 --- a/test/e2e/replica_set/replica_set_test.go +++ b/test/e2e/replica_set/replica_set_test.go @@ -5,7 +5,6 @@ import ( "os" "testing" - "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/util/mongotester" . "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/util/mongotester" e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" @@ -26,6 +25,7 @@ func TestReplicaSet(t *testing.T) { defer ctx.Teardown() mdb, user := e2eutil.NewTestMongoDB(ctx, "mdb0", "") + scramUser := mdb.GetScramUsers()[0] _, err := setup.GeneratePasswordForUser(ctx, user, "") if err != nil { @@ -41,7 +41,11 @@ func TestReplicaSet(t *testing.T) { t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb)) t.Run("Keyfile authentication is configured", tester.HasKeyfileAuth(3)) t.Run("Test Basic Connectivity", tester.ConnectivitySucceeds()) - t.Run("Test SRV Connectivity", tester.ConnectivitySucceeds(mongotester.WithSRV(mdb.MongoSRVURI()), WithoutTls(), WithReplicaSet((mdb.Name)))) + t.Run("Test SRV Connectivity", tester.ConnectivitySucceeds(WithURI(mdb.MongoSRVURI()), WithoutTls(), WithReplicaSet((mdb.Name)))) + t.Run("Test Basic Connectivity with generated connection string secret", + tester.ConnectivitySucceeds(WithURI(mongodbtests.GetConnectionStringForUser(mdb, scramUser)))) + t.Run("Test SRV Connectivity with generated connection string secret", + tester.ConnectivitySucceeds(WithURI(mongodbtests.GetSrvConnectionStringForUser(mdb, scramUser)))) t.Run("Ensure Authentication", tester.EnsureAuthenticationIsConfigured(3)) t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 1)) } diff --git a/test/e2e/replica_set_tls/replica_set_tls_test.go b/test/e2e/replica_set_tls/replica_set_tls_test.go index 2d94e5120..67241f779 100644 --- a/test/e2e/replica_set_tls/replica_set_tls_test.go +++ b/test/e2e/replica_set_tls/replica_set_tls_test.go @@ -25,6 +25,7 @@ func TestReplicaSetTLS(t *testing.T) { defer ctx.Teardown() mdb, user := e2eutil.NewTestMongoDB(ctx, "mdb-tls", "") + scramUser := mdb.GetScramUsers()[0] mdb.Spec.Security.TLS = e2eutil.NewTestTLSConfig(false) _, err := setup.GeneratePasswordForUser(ctx, user, "") @@ -44,10 +45,15 @@ func TestReplicaSetTLS(t *testing.T) { t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb)) mongodbtests.SkipTestIfLocal(t, "Ensure MongoDB TLS Configuration", func(t *testing.T) { - tester.HasTlsMode("requireSSL", 60, WithTls()) - tester.ConnectivitySucceeds(WithTls()) - tester.ConnectivityFails(WithoutTls()) - tester.EnsureAuthenticationIsConfigured(3, WithTls()) + t.Run("Has TLS Mode", tester.HasTlsMode("requireSSL", 60, WithTls())) + t.Run("Basic Connectivity Succeeds", tester.ConnectivitySucceeds(WithTls())) + t.Run("SRV Connectivity Succeeds", tester.ConnectivitySucceeds(WithURI(mdb.MongoSRVURI()), WithTls())) + t.Run("Basic Connectivity With Generated Connection String Secret Succeeds", + tester.ConnectivitySucceeds(WithURI(mongodbtests.GetConnectionStringForUser(mdb, scramUser)), WithTls())) + t.Run("SRV Connectivity With Generated Connection String Secret Succeeds", + tester.ConnectivitySucceeds(WithURI(mongodbtests.GetSrvConnectionStringForUser(mdb, scramUser)), WithTls())) + t.Run("Connectivity Fails", tester.ConnectivityFails(WithoutTls())) + t.Run("Ensure authentication is configured", tester.EnsureAuthenticationIsConfigured(3, WithTls())) }) t.Run("TLS is disabled", mongodbtests.DisableTLS(&mdb)) t.Run("MongoDB Reaches Failed Phase", mongodbtests.MongoDBReachesFailedPhase(&mdb)) diff --git a/test/e2e/util/mongotester/mongotester.go b/test/e2e/util/mongotester/mongotester.go index f8d53a257..592088c16 100644 --- a/test/e2e/util/mongotester/mongotester.go +++ b/test/e2e/util/mongotester/mongotester.go @@ -429,8 +429,8 @@ func WithoutTls() OptionApplier { } } -// WithSRV will add SRV connection string -func WithSRV(uri string) OptionApplier { +// WithURI will add URI connection string +func WithURI(uri string) OptionApplier { opt := &options.ClientOptions{} opt.ApplyURI(uri) return clientOptionAdder{option: opt} From 160afa5ffa80c49e6a55f8e87bde0d780fec193f Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Sat, 5 Jun 2021 17:19:06 +0100 Subject: [PATCH 293/790] Removed dead code (#531) --- test/e2e/mongodbtests/mongodbtests.go | 61 +++------------------------ 1 file changed, 6 insertions(+), 55 deletions(-) diff --git a/test/e2e/mongodbtests/mongodbtests.go b/test/e2e/mongodbtests/mongodbtests.go index 29239f209..6dce668c3 100644 --- a/test/e2e/mongodbtests/mongodbtests.go +++ b/test/e2e/mongodbtests/mongodbtests.go @@ -4,24 +4,20 @@ import ( "context" "encoding/json" "fmt" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/container" "testing" "time" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/wait" - mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" "github.com/mongodb/mongodb-kubernetes-operator/pkg/authentication/scram" "github.com/mongodb/mongodb-kubernetes-operator/pkg/automationconfig" e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" "github.com/stretchr/testify/assert" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" ) // SkipTestIfLocal skips tests locally which tests connectivity to mongodb pods @@ -276,20 +272,6 @@ func DeletePod(mdb *mdbv1.MongoDBCommunity, podNum int) func(*testing.T) { } } -// Connectivity returns a test function which performs -// a basic MongoDB connectivity test -func Connectivity(mdb *mdbv1.MongoDBCommunity, username, password string) func(t *testing.T) { - return func(t *testing.T) { - if err := Connect(mdb, options.Client().SetAuth(options.Credential{ - AuthMechanism: "SCRAM-SHA-256", - Username: username, - Password: password, - })); err != nil { - t.Fatalf("Error connecting to MongoDB deployment: %s", err) - } - } -} - // Status compares the given status to the actual status of the MongoDB resource func Status(mdb *mdbv1.MongoDBCommunity, expectedStatus mdbv1.MongoDBCommunityStatus) func(t *testing.T) { return func(t *testing.T) { @@ -348,28 +330,7 @@ func ChangeVersion(mdb *mdbv1.MongoDBCommunity, newVersion string) func(*testing } } -// Connect performs a connectivity check by initializing a mongo client -// and inserting a document into the MongoDB resource. Custom client -// options can be passed, for example to configure TLS. -func Connect(mdb *mdbv1.MongoDBCommunity, opts *options.ClientOptions) error { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) - defer cancel() - mongoClient, err := mongo.Connect(ctx, opts.ApplyURI(mdb.MongoURI())) - if err != nil { - return err - } - - return wait.Poll(time.Second*1, time.Second*30, func() (done bool, err error) { - collection := mongoClient.Database("testing").Collection("numbers") - _, err = collection.InsertOne(ctx, bson.M{"name": "pi", "value": 3.14159}) - if err != nil { - return false, nil - } - return true, nil - }) -} - -func StatefulSetContainerConditionIsTrue(mdb *mdbv1.MongoDBCommunity, containerName string, condition func(container corev1.Container) bool) func(*testing.T) { +func StatefulSetContainerConditionIsTrue(mdb *mdbv1.MongoDBCommunity, containerName string, condition func(c corev1.Container) bool) func(*testing.T) { return func(t *testing.T) { sts := appsv1.StatefulSet{} err := e2eutil.TestClient.Get(context.TODO(), types.NamespacedName{Name: mdb.Name, Namespace: mdb.Namespace}, &sts) @@ -377,12 +338,12 @@ func StatefulSetContainerConditionIsTrue(mdb *mdbv1.MongoDBCommunity, containerN t.Fatal(err) } - container := findContainerByName(containerName, sts.Spec.Template.Spec.Containers) - if container == nil { + existingContainer := container.GetByName(containerName, sts.Spec.Template.Spec.Containers) + if existingContainer == nil { t.Fatalf(`No container found with name "%s" in StatefulSet pod template`, containerName) } - if !condition(*container) { + if !condition(*existingContainer) { t.Fatalf(`Container "%s" does not satisfy condition`, containerName) } } @@ -412,16 +373,6 @@ func ExecInContainer(mdb *mdbv1.MongoDBCommunity, podNum int, containerName, com } } -func findContainerByName(name string, containers []corev1.Container) *corev1.Container { - for _, c := range containers { - if c.Name == name { - return &c - } - } - - return nil -} - func podFromMongoDBCommunity(mdb *mdbv1.MongoDBCommunity, podNum int) corev1.Pod { return corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ From 22e5bf1bf805b625cd6ed0341357ac1488a5fd0a Mon Sep 17 00:00:00 2001 From: Rajdeep Das Date: Mon, 7 Jun 2021 13:31:57 +0200 Subject: [PATCH 294/790] Fix apiVersion in docs files (#535) --- docs/deploy-configure.md | 10 +++++----- docs/secure.md | 2 +- docs/users.md | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/deploy-configure.md b/docs/deploy-configure.md index 9d0d55f6d..f3e166cf6 100644 --- a/docs/deploy-configure.md +++ b/docs/deploy-configure.md @@ -39,7 +39,7 @@ members in a replica set. Consider the following example MongoDB resource definition: ```yaml -apiVersion: mongodb.com/v1 +apiVersion: mongodbcommunity.mongodb.com/v1 kind: MongoDBCommunity metadata: name: example-mongodb @@ -56,7 +56,7 @@ To scale a replica set: Update `members` to the number of members that you want the replica set to have. ```yaml - apiVersion: mongodb.com/v1 + apiVersion: mongodbcommunity.mongodb.com/v1 kind: MongoDBCommunity metadata: name: example-mongodb @@ -90,7 +90,7 @@ If you update `spec.version` to a later version, consider setting `spec.featureC Consider the following example MongoDB resource definition: ```yaml -apiVersion: mongodb.com/v1 +apiVersion: mongodbcommunity.mongodb.com/v1 kind: MongoDBCommunity metadata: name: example-mongodb @@ -108,7 +108,7 @@ To upgrade this resource from `4.0.6` to `4.2.7`: b. Update `spec.featureCompatibilityVersion` to `4.0`. ```yaml - apiVersion: mongodb.com/v1 + apiVersion: mongodbcommunity.mongodb.com/v1 kind: MongoDBCommunity metadata: name: example-mongodb @@ -165,7 +165,7 @@ To define a custom role: ```yaml --- - apiVersion: mongodb.com/v1 + apiVersion: mongodbcommunity.mongodb.com/v1 kind: MongoDBCommunity metadata: name: custom-role-mongodb diff --git a/docs/secure.md b/docs/secure.md index 560dec06d..33bdb9da3 100644 --- a/docs/secure.md +++ b/docs/secure.md @@ -76,7 +76,7 @@ To secure connections to MongoDB resources using TLS: - `spec.security.tls.caConfigMapRef.name`: Name of the Kubernetes ConfigMap that contains the Certificate Authority certificate used to sign the server certificate that you created in the [prerequisites](#prerequisites-1). ```yaml - apiVersion: mongodb.com/v1 + apiVersion: mongodbcommunity.mongodb.com/v1 kind: MongoDBCommunity metadata: name: example-mongodb diff --git a/docs/users.md b/docs/users.md index 47b5c78ba..aca71c6d2 100644 --- a/docs/users.md +++ b/docs/users.md @@ -45,7 +45,7 @@ You cannot disable SCRAM authentication. ```yaml --- - apiVersion: mongodb.com/v1 + apiVersion: mongodbcommunity.mongodb.com/v1 kind: MongoDBCommunity metadata: name: example-scram-mongodb From cf15a17994af37c3f0fd0a60dfdacbb1248386a2 Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Tue, 8 Jun 2021 15:40:45 +0100 Subject: [PATCH 295/790] Add GitHub action to create release pr (#522) --- .github/workflows/create_release_pr.yml | 33 ++++++++++++ config/manager/manager.yaml | 55 ++++++++++---------- deploy/openshift/operator_openshift.yaml | 58 ++++++++++----------- requirements.txt | 1 + scripts/ci/update_release.py | 65 ++++++++++++++++++++++++ 5 files changed, 155 insertions(+), 57 deletions(-) create mode 100644 .github/workflows/create_release_pr.yml create mode 100755 scripts/ci/update_release.py diff --git a/.github/workflows/create_release_pr.yml b/.github/workflows/create_release_pr.yml new file mode 100644 index 000000000..f3ba2d564 --- /dev/null +++ b/.github/workflows/create_release_pr.yml @@ -0,0 +1,33 @@ +name: Create Release PR +on: + workflow_dispatch + +jobs: + create-release-pr: + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@v2 + - name: Determine Release Version + id: release_version + run: | + OUTPUT=$(jq -r '."mongodb-kubernetes-operator"' < $GITHUB_WORKSPACE/release.json) + echo "::set-output name=OUTPUT::$OUTPUT" + DATE=$(date '+%Y-%m-%d-%H-%M-%S') + echo "::set-output name=DATE::$DATE" + - uses: actions/setup-python@v2 + with: + python-version: '3.6' + architecture: 'x64' + - run: pip install -r requirements.txt + - name: Create Release PR + uses: technote-space/create-pr-action@v2 + with: + EXECUTE_COMMANDS: | + python scripts/ci/update_release.py + COMMIT_MESSAGE: 'Release Operator v${{ steps.release_version.outputs.OUTPUT }}' + COMMIT_NAME: 'GitHub Actions' + COMMIT_EMAIL: 'support@mongodb.com' + # include date in the pr branch name to ensure duplicate branches to not get created. + PR_BRANCH_NAME: 'release-v${{steps.release_version.outputs.OUTPUT}}-${{ steps.release_version.outputs.DATE }}' + PR_TITLE: 'Release MongoDB Kubernetes Operator v${{ steps.release_version.outputs.OUTPUT }}' diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index a952985e6..b4a5c3479 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -1,4 +1,3 @@ ---- apiVersion: apps/v1 kind: Deployment metadata: @@ -13,31 +12,31 @@ spec: labels: name: mongodb-kubernetes-operator spec: - serviceAccountName: mongodb-kubernetes-operator containers: - - name: mongodb-kubernetes-operator - image: quay.io/mongodb/mongodb-kubernetes-operator:0.6.0 - command: - - /usr/local/bin/entrypoint - imagePullPolicy: Always - env: - - name: WATCH_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: OPERATOR_NAME - value: "mongodb-kubernetes-operator" - - name: AGENT_IMAGE # The MongoDB Agent the operator will deploy to manage MongoDB deployments - value: quay.io/mongodb/mongodb-agent:10.29.0.6830-1 - - name: VERSION_UPGRADE_HOOK_IMAGE - value: quay.io/mongodb/mongodb-kubernetes-operator-version-upgrade-post-start-hook:1.0.2 - - name: READINESS_PROBE_IMAGE - value: quay.io/mongodb/mongodb-kubernetes-readinessprobe:1.0.3 - - name: MONGODB_IMAGE - value: "library/mongo" - - name: MONGODB_REPO_URL - value: "registry.hub.docker.com" + - command: + - /usr/local/bin/entrypoint + env: + - name: WATCH_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: OPERATOR_NAME + value: mongodb-kubernetes-operator + - name: AGENT_IMAGE + value: quay.io/mongodb/mongodb-agent:10.29.0.6830-1 + - name: VERSION_UPGRADE_HOOK_IMAGE + value: quay.io/mongodb/mongodb-kubernetes-operator-version-upgrade-post-start-hook:1.0.2 + - name: READINESS_PROBE_IMAGE + value: quay.io/mongodb/mongodb-kubernetes-readinessprobe:1.0.4 + - name: MONGODB_IMAGE + value: library/mongo + - name: MONGODB_REPO_URL + value: registry.hub.docker.com + image: quay.io/mongodb/mongodb-kubernetes-operator:0.6.0 + imagePullPolicy: Always + name: mongodb-kubernetes-operator + serviceAccountName: mongodb-kubernetes-operator diff --git a/deploy/openshift/operator_openshift.yaml b/deploy/openshift/operator_openshift.yaml index 89503d558..19d254cdd 100644 --- a/deploy/openshift/operator_openshift.yaml +++ b/deploy/openshift/operator_openshift.yaml @@ -12,33 +12,33 @@ spec: labels: name: mongodb-kubernetes-operator spec: - serviceAccountName: mongodb-kubernetes-operator containers: - - name: mongodb-kubernetes-operator - image: quay.io/mongodb/mongodb-kubernetes-operator:0.6.0 - command: - - mongodb-kubernetes-operator - imagePullPolicy: Always - env: - - name: WATCH_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: MANAGED_SECURITY_CONTEXT - value: 'true' - - name: OPERATOR_NAME - value: "mongodb-kubernetes-operator" - - name: AGENT_IMAGE # The MongoDB Agent the operator will deploy to manage MongoDB deployments - value: quay.io/mongodb/mongodb-agent:10.29.0.6830-1 - - name: READINESS_PROBE_IMAGE - value: quay.io/mongodb/mongodb-kubernetes-readinessprobe:1.0.3 - - name: VERSION_UPGRADE_HOOK_IMAGE - value: quay.io/mongodb/mongodb-kubernetes-operator-version-upgrade-post-start-hook:1.0.2 - - name: MONGODB_IMAGE - value: "library/mongo" - - name: MONGODB_REPO_URL - value: "registry.hub.docker.com" + - command: + - mongodb-kubernetes-operator + env: + - name: WATCH_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: MANAGED_SECURITY_CONTEXT + value: 'true' + - name: OPERATOR_NAME + value: mongodb-kubernetes-operator + - name: AGENT_IMAGE + value: quay.io/mongodb/mongodb-agent:10.29.0.6830-1 + - name: READINESS_PROBE_IMAGE + value: quay.io/mongodb/mongodb-kubernetes-readinessprobe:1.0.4 + - name: VERSION_UPGRADE_HOOK_IMAGE + value: quay.io/mongodb/mongodb-kubernetes-operator-version-upgrade-post-start-hook:1.0.2 + - name: MONGODB_IMAGE + value: library/mongo + - name: MONGODB_REPO_URL + value: registry.hub.docker.com + image: quay.io/mongodb/mongodb-kubernetes-operator:0.6.0 + imagePullPolicy: Always + name: mongodb-kubernetes-operator + serviceAccountName: mongodb-kubernetes-operator diff --git a/requirements.txt b/requirements.txt index 14e01968f..888a1b16a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,4 +9,5 @@ tqdm==v4.49.0 boto3==1.16.21 pymongo==3.11.2 dnspython==2.0.0 +pyyaml==5.4.1 rsa>=4.7 # not directly required, pinned by Snyk to avoid a vulnerability diff --git a/scripts/ci/update_release.py b/scripts/ci/update_release.py new file mode 100755 index 000000000..511f9d220 --- /dev/null +++ b/scripts/ci/update_release.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 + +import json +import sys +import yaml +from typing import Dict + +RELATIVE_PATH_TO_MANAGER_YAML = "config/manager/manager.yaml" +RELATIVE_PATH_TO_OPENSHIFT_MANAGER_YAML = "deploy/openshift/operator_openshift.yaml" + + +def _load_yaml_file(path: str) -> Dict: + with open(path, "r") as f: + return yaml.safe_load(f.read()) + + +def _dump_yaml(operator: Dict, path: str) -> None: + with open(path, "w+") as f: + yaml.dump(operator, f) + + +def update_and_write_file(path: str) -> None: + release = _load_release() + yaml_file = _load_yaml_file(path) + _update_operator_deployment(yaml_file, release) + _dump_yaml(yaml_file, path) + + +def _load_release() -> Dict: + with open("release.json", "r") as f: + return json.loads(f.read()) + + +def _replace_tag(image: str, new_tag: str) -> str: + split_image = image.split(":") + return split_image[0] + ":" + new_tag + + +def _update_operator_deployment(operator_deployment: Dict, release: Dict) -> None: + operator_container = operator_deployment["spec"]["template"]["spec"]["containers"][ + 0 + ] + operator_container["image"] = _replace_tag( + operator_container["image"], release["mongodb-kubernetes-operator"] + ) + operator_envs = operator_container["env"] + for env in operator_envs: + if env["name"] == "VERSION_UPGRADE_HOOK_IMAGE": + env["value"] = _replace_tag(env["value"], release["version-upgrade-hook"]) + if env["name"] == "READINESS_PROBE_IMAGE": + env["value"] = _replace_tag(env["value"], release["readiness-probe"]) + if env["name"] == "AGENT_IMAGE": + env["value"] = _replace_tag( + env["value"], release["mongodb-agent"]["version"] + ) + + +def main() -> int: + update_and_write_file(RELATIVE_PATH_TO_MANAGER_YAML) + update_and_write_file(RELATIVE_PATH_TO_OPENSHIFT_MANAGER_YAML) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 0aa1ed41d546010be724838fef5c555fdf56c9ea Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Wed, 9 Jun 2021 09:21:36 +0100 Subject: [PATCH 296/790] Run E2E Tests with Telepresence (#515) --- Makefile | 11 +++++++++++ docs/contributing.md | 13 +++++++++++++ scripts/dev/install_prerequisites.sh | 9 +++++++++ 3 files changed, 33 insertions(+) create mode 100755 scripts/dev/install_prerequisites.sh diff --git a/Makefile b/Makefile index 06784c51f..af82b7fe9 100644 --- a/Makefile +++ b/Makefile @@ -64,6 +64,15 @@ fmt: vet: go vet ./... +# Run e2e tests locally using go build while also setting up a proxy in the shell to allow +# the test to run as if it were inside the cluster. This enables mongodb connectivity while running locally. +e2e-telepresence: install + telepresence connect; \ + telepresence status; \ + eval $$(scripts/dev/get_e2e_env_vars.py $(cleanup)); \ + go test -v -timeout=30m -failfast ./test/e2e/$(test); \ + telepresence quit + # Run e2e test by deploying test image in kubernetes. e2e-k8s: install e2e-image python scripts/dev/e2e.py --perform-cleanup --test $(test) @@ -126,3 +135,5 @@ rm -rf $$TMP_DIR ;\ } endef +install-prerequisites-macos: + scripts/dev/install_prerequisites.sh diff --git a/docs/contributing.md b/docs/contributing.md index 5acb649f1..e5e2a821a 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -201,6 +201,19 @@ make e2e-k8s test= This will run the `replica_set` E2E test which is a simple test which installs a MongoDB Replica Set and asserts that the deployed server can be connected to. + +### Run the test locally with go test & Telepresence +```sh +make e2e-telepresence test= +``` + +This method uses telepresence to allow connectivity as if your local machine is in the kubernetes cluster, +there will be full MongoDB connectivity using `go test` locally. + +Note: you must install [telepresence](https://www.getambassador.io/docs/telepresence/latest/quick-start/) before using this method. + +If on MacOS, you can run `make install-prerequisites-macos` which will perform the installation. + ## Troubleshooting When you run a test locally, if the `e2e-test` pod is present, you will have to first manually delete it; failing to do so will cause the `e2e-test` pod to fail. diff --git a/scripts/dev/install_prerequisites.sh b/scripts/dev/install_prerequisites.sh new file mode 100755 index 000000000..4f95bfb90 --- /dev/null +++ b/scripts/dev/install_prerequisites.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +if ! command -v telepresence &> /dev/null; then \ + echo "Telepresence not found, installing now" + sudo curl -fL https://app.getambassador.io/download/tel2/darwin/amd64/latest/telepresence -o /usr/local/bin/telepresence + sudo chmod a+x /usr/local/bin/telepresence +else + echo "Telepresence already installed." +fi From 8847bfda61a168cbb989468eb257dd3ac08687c5 Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Wed, 9 Jun 2021 09:57:15 +0100 Subject: [PATCH 297/790] Remove Dockerfile Generator (#544) --- scripts/dev/dockerfile_generator.py | 58 ----------------------------- 1 file changed, 58 deletions(-) delete mode 100755 scripts/dev/dockerfile_generator.py diff --git a/scripts/dev/dockerfile_generator.py b/scripts/dev/dockerfile_generator.py deleted file mode 100755 index 234135a86..000000000 --- a/scripts/dev/dockerfile_generator.py +++ /dev/null @@ -1,58 +0,0 @@ -import json - -import jinja2 -import argparse -import os -import sys -from typing import List, Dict, Union - -DockerParameters = Dict[str, Union[bool, str, List[str]]] - -GOLANG_TAG = "1.15" - - -def operator_params() -> DockerParameters: - return { - "builder": True, - "builder_image": f"golang:{GOLANG_TAG}", - "base_image": "registry.access.redhat.com/ubi8/ubi-minimal:latest", - } - - -def e2e_params() -> DockerParameters: - return { - "base_image": f"golang:{GOLANG_TAG}", - # TODO: make this image smaller, error: 'exec: "gcc": executable file not found in $PATH' with golang:alpine - } - - -def render(image_name: str) -> str: - param_dict = { - "e2e": e2e_params(), - "operator": operator_params(), - } - - render_values = param_dict.get(image_name, dict()) - - search_path = str(render_values.get("template_path", "scripts/dev/templates")) - - env = jinja2.Environment() - env.loader = jinja2.FileSystemLoader(searchpath=search_path) - return env.get_template(f"Dockerfile.{image_name}").render(render_values) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser() - parser.add_argument("image", help="Type of image for the Dockerfile") - - return parser.parse_args() - - -def main() -> int: - args = parse_args() - print(render(args.image)) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) From c0fbf418e92b75327c026363b891ca5508a1e744 Mon Sep 17 00:00:00 2001 From: Ciprian Tibulca Date: Wed, 9 Jun 2021 13:02:55 +0300 Subject: [PATCH 298/790] CLOUDP-88995: update k8s controller runtime (#541) * update k8s controller runtime * upgrade golang to 1.16 --- .github/workflows/go.yml | 2 +- go.mod | 12 +- go.sum | 182 ++++++++++-------- pipeline.py | 2 +- pkg/kube/client/mocked_manager.go | 9 + .../templates/readiness/Dockerfile.builder | 3 +- 6 files changed, 117 insertions(+), 93 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index e3f417ea8..229710374 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -14,7 +14,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v2 with: - go-version: 1.15 + go-version: 1.16 - name: Test api run: go test -v ./api/... diff --git a/go.mod b/go.mod index cf8d0d9be..bfaa64c8a 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/mongodb/mongodb-kubernetes-operator -go 1.15 +go 1.16 require ( github.com/blang/semver v3.5.1+incompatible @@ -13,12 +13,12 @@ require ( github.com/stretchr/objx v0.3.0 github.com/stretchr/testify v1.7.0 github.com/xdg/stringprep v1.0.3 - go.mongodb.org/mongo-driver v1.5.2 + go.mongodb.org/mongo-driver v1.5.3 go.uber.org/zap v1.17.0 - k8s.io/api v0.20.4 - k8s.io/apimachinery v0.21.0 - k8s.io/client-go v0.20.4 - sigs.k8s.io/controller-runtime v0.8.3 + k8s.io/api v0.21.1 + k8s.io/apimachinery v0.21.1 + k8s.io/client-go v0.21.1 + sigs.k8s.io/controller-runtime v0.9.0 sigs.k8s.io/yaml v1.2.0 ) diff --git a/go.sum b/go.sum index e38c79720..a0798fcb7 100644 --- a/go.sum +++ b/go.sum @@ -24,17 +24,16 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-autorest v13.3.2+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= -github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= +github.com/Azure/go-autorest/autorest v0.11.12/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= -github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= @@ -42,6 +41,7 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= @@ -78,12 +78,12 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfc github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -93,40 +93,43 @@ github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses= +github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v4.11.0+incompatible h1:glyUF9yIYtMHzn8xaKw5rMhdWcwsYV8dZHIq5567/xs= +github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/logr v0.3.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/zapr v0.2.0 h1:v6Ji8yBW77pva6NkJKQdHLAJKrIJKRHz0RXwPqCHSR4= -github.com/go-logr/zapr v0.2.0/go.mod h1:qhKdvif7YF5GI9NWEpyxTSSBdGmzkNguibrdCNVPunU= +github.com/go-logr/zapr v0.4.0 h1:uc1uML3hRYL9/ZZPdgHS/n8Nzo+eaYL/Efxkkamf7OM= +github.com/go-logr/zapr v0.4.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= +github.com/go-openapi/spec v0.19.5/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk= github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg= @@ -153,7 +156,6 @@ github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/V github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -180,8 +182,10 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -191,8 +195,10 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -210,8 +216,8 @@ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= -github.com/googleapis/gnostic v0.5.1 h1:A8Yhf6EtqTv9RMsU6MQTyrtV1TjWlR6xU9BsZIwuTCM= -github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= +github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw= +github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= @@ -249,28 +255,30 @@ github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/J github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= @@ -311,7 +319,7 @@ github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:F github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= -github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= +github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -322,25 +330,28 @@ github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJ github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.14.1 h1:jMU0WaQrP0a/YAEq8eJmJKjBoMs+pClEr1vDMlM/Do4= -github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E= +github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.10.2 h1:aY/nuoWlKJud2J6U0E3NWsjlg+0GtwXxgEqthRdzlcs= -github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.13.0 h1:7lLHu94wT9Ij0o6EWWclhu0aOh32VxhkwEJvzuWPeak= +github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= @@ -356,8 +367,9 @@ github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prY github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.7.1 h1:NTGy1Ja9pByO+xAeH/qiWnLrKtr3hJPNjaVUwnjpdpA= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ= +github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -366,14 +378,16 @@ github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6T github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= +github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.2.0 h1:wH4vA7pcjKuZzjF7lM8awk4fnuJO6idemZXoKnULUx4= github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -388,6 +402,7 @@ github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= @@ -442,27 +457,22 @@ go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= -go.mongodb.org/mongo-driver v1.5.2 h1:AsxOLoJTgP6YNM0fXWw4OjdluYmWzQYp+lFJL7xu9fU= -go.mongodb.org/mongo-driver v1.5.2/go.mod h1:gRXCHX4Jo7J0IJ1oDQyUxF7jfy19UfxniMS4xxMmUqw= +go.mongodb.org/mongo-driver v1.5.3 h1:wWbFB6zaGHpzguF3f7tW94sVE8sFl3lHx8OZx/4OuFI= +go.mongodb.org/mongo-driver v1.5.3/go.mod h1:gRXCHX4Jo7J0IJ1oDQyUxF7jfy19UfxniMS4xxMmUqw= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= -go.uber.org/zap v1.8.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.17.0 h1:MTjgFu6ZLKvY6Pvaqk97GlxNBuMpV4Hy/3P6tRGlI2U= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -475,8 +485,9 @@ golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0 h1:hb9wdF1z5waM+dSIICn1l0DkLVDT3hqhhQsDNUmHPRE= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g= +golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -507,8 +518,8 @@ golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -536,10 +547,11 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 h1:OgUuv8lsRpBibGNbSizVwKWlysjaNzmC9gYMhPVfqFM= golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 h1:DzZ89McO9/gWPsQXS/FVKAlG02ZjaQ6AlZRBimEYOd0= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -553,8 +565,9 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -579,6 +592,7 @@ golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -590,31 +604,39 @@ golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073 h1:8qxJSnu+7dRq6upnbntrmriWByIakBuct5OM/MdQC1M= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40 h1:JWgyZ1qgdTaF3N3oxC+MdTV7qvEEgHo3otj+HB5CM7Q= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE= +golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e h1:EHBhcS0mlXEAVwNyO2dLfjToGsyY4j24pTs2ScHnX7s= -golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -636,8 +658,6 @@ golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -656,17 +676,18 @@ golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a h1:CB3a9Nez8M13wwlr/E2YtwoU+qYHKfC+JrDa45RXXoQ= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gomodules.xyz/jsonpatch/v2 v2.1.0 h1:Phva6wqu+xR//Njw6iorylFFgn/z547tw5Ne3HZPQ+k= -gomodules.xyz/jsonpatch/v2 v2.1.0/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU= +gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY= +gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -682,8 +703,8 @@ google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -703,6 +724,7 @@ google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -720,8 +742,10 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -752,53 +776,43 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= +gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo= -k8s.io/api v0.20.2/go.mod h1:d7n6Ehyzx+S+cE3VhTGfVNNqtGc/oL9DCdYYahlurV8= -k8s.io/api v0.20.4 h1:xZjKidCirayzX6tHONRQyTNDVIR55TYVqgATqo6ZULY= -k8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ= -k8s.io/apiextensions-apiserver v0.20.1 h1:ZrXQeslal+6zKM/HjDXLzThlz/vPSxrfK3OqL8txgVQ= -k8s.io/apiextensions-apiserver v0.20.1/go.mod h1:ntnrZV+6a3dB504qwC5PN/Yg9PBiDNt1EVqbW2kORVk= -k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= -k8s.io/apimachinery v0.20.2/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= -k8s.io/apimachinery v0.20.4/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= -k8s.io/apimachinery v0.21.0 h1:3Fx+41if+IRavNcKOz09FwEXDBG6ORh6iMsTSelhkMA= -k8s.io/apimachinery v0.21.0/go.mod h1:jbreFvJo3ov9rj7eWT7+sYiRx+qZuCYXwWT1bcDswPY= -k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU= -k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y= -k8s.io/client-go v0.20.2/go.mod h1:kH5brqWqp7HDxUFKoEgiI4v8G1xzbe9giaCenUWJzgE= -k8s.io/client-go v0.20.4 h1:85crgh1IotNkLpKYKZHVNI1JT86nr/iDCvq2iWKsql4= -k8s.io/client-go v0.20.4/go.mod h1:LiMv25ND1gLUdBeYxBIwKpkSC5IsozMMmOOeSJboP+k= -k8s.io/code-generator v0.20.1/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= -k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk= -k8s.io/component-base v0.20.2 h1:LMmu5I0pLtwjpp5009KLuMGFqSc2S2isGw8t1hpYKLE= -k8s.io/component-base v0.20.2/go.mod h1:pzFtCiwe/ASD0iV7ySMu8SYVJjCapNM9bjvk7ptpKh0= +k8s.io/api v0.21.1 h1:94bbZ5NTjdINJEdzOkpS4vdPhkb1VFpTYC9zh43f75c= +k8s.io/api v0.21.1/go.mod h1:FstGROTmsSHBarKc8bylzXih8BLNYTiS3TZcsoEDg2s= +k8s.io/apiextensions-apiserver v0.21.1 h1:AA+cnsb6w7SZ1vD32Z+zdgfXdXY8X9uGX5bN6EoPEIo= +k8s.io/apiextensions-apiserver v0.21.1/go.mod h1:KESQFCGjqVcVsZ9g0xX5bacMjyX5emuWcS2arzdEouA= +k8s.io/apimachinery v0.21.1 h1:Q6XuHGlj2xc+hlMCvqyYfbv3H7SRGn2c8NycxJquDVs= +k8s.io/apimachinery v0.21.1/go.mod h1:jbreFvJo3ov9rj7eWT7+sYiRx+qZuCYXwWT1bcDswPY= +k8s.io/apiserver v0.21.1/go.mod h1:nLLYZvMWn35glJ4/FZRhzLG/3MPxAaZTgV4FJZdr+tY= +k8s.io/client-go v0.21.1 h1:bhblWYLZKUu+pm50plvQF8WpY6TXdRRtcS/K9WauOj4= +k8s.io/client-go v0.21.1/go.mod h1:/kEw4RgW+3xnBGzvp9IWxKSNA+lXn3A7AuH3gdOAzLs= +k8s.io/code-generator v0.21.1/go.mod h1:hUlps5+9QaTrKx+jiM4rmq7YmH8wPOIko64uZCHDh6Q= +k8s.io/component-base v0.21.1 h1:iLpj2btXbR326s/xNQWmPNGu0gaYSjzn7IN/5i28nQw= +k8s.io/component-base v0.21.1/go.mod h1:NgzFZ2qu4m1juby4TnrmpR8adRk6ka62YdH5DkIIyKA= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.8.0 h1:Q3gmuM9hKEjefWFFYF0Mat+YyFJvsUyYuwyNNJ5C9Ts= k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= -k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 h1:vEx13qjvaZ4yfObSSXW7BrMc/KQBBT/Jyee8XtLf4x0= k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20210111153108-fddb29f9d009 h1:0T5IaWHO3sJTEmCP6mUlBvMukxPKUQWqiI/YuiBNMiQ= -k8s.io/utils v0.0.0-20210111153108-fddb29f9d009/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20210527160623-6fdb442a123b h1:MSqsVQ3pZvPGTqCjptfimO2WjG7A9un2zcpiHkA6M/s= +k8s.io/utils v0.0.0-20210527160623-6fdb442a123b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= -sigs.k8s.io/controller-runtime v0.8.3 h1:GMHvzjTmaWHQB8HadW+dIvBoJuLvZObYJ5YoZruPRao= -sigs.k8s.io/controller-runtime v0.8.3/go.mod h1:U/l+DUopBc1ecfRZ5aviA9JDmGFQKvLf5YkZNx2e0sU= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= +sigs.k8s.io/controller-runtime v0.9.0 h1:ZIZ/dtpboPSbZYY7uUz2OzrkaBTOThx2yekLtpGB+zY= +sigs.k8s.io/controller-runtime v0.9.0/go.mod h1:TgkfvrhhEw3PlI0BRL/5xM+89y3/yc0ZDfdbTl84si8= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.1.0 h1:C4r9BgJ98vrKnnVCjwCSXcWjWe0NKcUQkmzDXZXGwH8= sigs.k8s.io/structured-merge-diff/v4 v4.1.0/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= diff --git a/pipeline.py b/pipeline.py index 4b08a5635..389e22c58 100644 --- a/pipeline.py +++ b/pipeline.py @@ -18,7 +18,7 @@ ] ) -GOLANG_TAG = "1.15" +GOLANG_TAG = "1.16" DEFAULT_IMAGE_TYPE = "ubuntu" DEFAULT_NAMESPACE = "default" diff --git a/pkg/kube/client/mocked_manager.go b/pkg/kube/client/mocked_manager.go index 2d9913117..194e7b5e5 100644 --- a/pkg/kube/client/mocked_manager.go +++ b/pkg/kube/client/mocked_manager.go @@ -3,6 +3,7 @@ package client import ( "context" "net/http" + "time" "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/api/meta" @@ -11,6 +12,7 @@ import ( "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/cache" k8sClient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/webhook" @@ -126,3 +128,10 @@ func (m *MockedManager) AddReadyzCheck(name string, check healthz.Checker) error func (m *MockedManager) GetLogger() logr.Logger { return nil } + +func (m *MockedManager) GetControllerOptions() v1alpha1.ControllerConfigurationSpec { + var duration = time.Duration(0) + return v1alpha1.ControllerConfigurationSpec{ + CacheSyncTimeout: &duration, + } +} diff --git a/scripts/dev/templates/readiness/Dockerfile.builder b/scripts/dev/templates/readiness/Dockerfile.builder index 81a0de8aa..e9e694e4e 100644 --- a/scripts/dev/templates/readiness/Dockerfile.builder +++ b/scripts/dev/templates/readiness/Dockerfile.builder @@ -1,9 +1,10 @@ -FROM golang:1.15-alpine as builder +FROM golang:1.16-alpine as builder COPY ./cmd/readiness /build/ COPY ./pkg /build/pkg COPY ./api /build/api COPY ./go.mod /build/ +COPY ./go.sum /build/ WORKDIR /build RUN CGO_ENABLED=0 GOOS=linux GOARCH=386 go build -a -i -o readinessprobe . From 7ebc64cab51883db9b0462f7591de72ef2e956ee Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Wed, 9 Jun 2021 12:01:42 +0100 Subject: [PATCH 299/790] CLOUDP-57889: Update agent to Ubuntu 20.04 (#545) --- docs/RELEASE_NOTES.md | 3 +++ scripts/dev/templates/agent/Dockerfile.ubuntu | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/RELEASE_NOTES.md b/docs/RELEASE_NOTES.md index 1120ed52b..74fe2a72b 100644 --- a/docs/RELEASE_NOTES.md +++ b/docs/RELEASE_NOTES.md @@ -12,6 +12,9 @@ - Changes - Readiness probe now patches pod annotations rather than overwriting them. + +## Miscellaneous +Ubuntu-based agent images are now based on Ubuntu 20.04 instead of Ubuntu 16.06 ## Updated Image Tags diff --git a/scripts/dev/templates/agent/Dockerfile.ubuntu b/scripts/dev/templates/agent/Dockerfile.ubuntu index c0cca8bb7..d6c0a8c19 100644 --- a/scripts/dev/templates/agent/Dockerfile.ubuntu +++ b/scripts/dev/templates/agent/Dockerfile.ubuntu @@ -1,6 +1,6 @@ {% extends "Dockerfile.template" %} -{% set base_image = "ubuntu:16.04" %} +{% set base_image = "ubuntu:20.04" %} {% block packages -%} RUN apt-get -qq update \ From ebc073373fc83f539656c27b2b55a3b5115939b3 Mon Sep 17 00:00:00 2001 From: Ciprian Tibulca Date: Wed, 9 Jun 2021 15:42:47 +0300 Subject: [PATCH 300/790] expose connection strings - update release notes (#546) update release notes: * expose connection strings * support connection strings using SRV --- docs/RELEASE_NOTES.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/RELEASE_NOTES.md b/docs/RELEASE_NOTES.md index 74fe2a72b..43ad724e2 100644 --- a/docs/RELEASE_NOTES.md +++ b/docs/RELEASE_NOTES.md @@ -7,6 +7,8 @@ - Changes - fixed an issue where the operator would reconcile based on events emitted by itself in certain situations. + - support connection strings using SRV. + - expose connection strings (including auth/tls values) for deployments as secrets for easy of use. Secrets name template: _\-\-\_ ## MongoDB Agent ReadinessProbe From 66cd12b8a86740b2a15ff0a26bd6aebd741d19c5 Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Wed, 9 Jun 2021 18:48:15 +0100 Subject: [PATCH 301/790] CLOUDP-79509: Added deleter interface (#547) --- pkg/kube/client/client.go | 13 ++++++++++++- pkg/kube/service/service.go | 20 ++++++++++++++++++-- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/pkg/kube/client/client.go b/pkg/kube/client/client.go index 070446b2c..70c9fc387 100644 --- a/pkg/kube/client/client.go +++ b/pkg/kube/client/client.go @@ -28,7 +28,7 @@ type Client interface { // TODO: remove this function, add mongodb package which has GetAndUpdate function GetAndUpdate(nsName types.NamespacedName, obj k8sClient.Object, updateFunc func()) error configmap.GetUpdateCreateDeleter - service.GetUpdateCreator + service.GetUpdateCreateDeleter secret.GetUpdateCreateDeleter statefulset.GetUpdateCreateDeleter pod.Getter @@ -139,6 +139,17 @@ func (c client) CreateService(service corev1.Service) error { return c.Create(context.TODO(), &service) } +// DeleteService provides a thin wrapper around client.Client to delete corev1.Service types +func (c client) DeleteService(objectKey k8sClient.ObjectKey) error { + svc := corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: objectKey.Name, + Namespace: objectKey.Namespace, + }, + } + return c.Delete(context.TODO(), &svc) +} + // GetStatefulSet provides a thin wrapper and client.Client to access appsv1.StatefulSet types func (c client) GetStatefulSet(objectKey k8sClient.ObjectKey) (appsv1.StatefulSet, error) { sts := appsv1.StatefulSet{} diff --git a/pkg/kube/service/service.go b/pkg/kube/service/service.go index 9f38234a7..875b9c620 100644 --- a/pkg/kube/service/service.go +++ b/pkg/kube/service/service.go @@ -10,11 +10,20 @@ type Getter interface { } type Updater interface { - UpdateService(secret corev1.Service) error + UpdateService(service corev1.Service) error } type Creator interface { - CreateService(secret corev1.Service) error + CreateService(service corev1.Service) error +} + +type Deleter interface { + DeleteService(objectKey client.ObjectKey) error +} + +type GetDeleter interface { + Getter + Deleter } type GetUpdater interface { @@ -28,6 +37,13 @@ type GetUpdateCreator interface { Creator } +type GetUpdateCreateDeleter interface { + Getter + Updater + Creator + Deleter +} + // Merge merges `source` into `dest`. Both arguments will remain unchanged // a new service will be created and returned. // The "merging" process is arbitrary and it only handle specific attributes From 4ce5f88250178dbf8575ca592055493f2b433ed8 Mon Sep 17 00:00:00 2001 From: Rodrigo Valin Date: Thu, 10 Jun 2021 09:03:25 +0100 Subject: [PATCH 302/790] Updates image release process. (#548) --- .evergreen.yml | 12 ++++++------ requirements.txt | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.evergreen.yml b/.evergreen.yml index dc9ccf7ca..073a79e0e 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -98,7 +98,7 @@ functions: command: subprocess.exec type: setup params: - command: virtualenv --python /opt/python/3.7/bin/python3 ./venv + command: virtualenv --python /opt/python/3.9/bin/python3 ./venv python_requirements: &python_requirements command: subprocess.exec @@ -137,7 +137,7 @@ functions: - atlas_password - atlas_database - image_name - command: "${workdir}/venv/bin/python scripts/ci/add_supported_release.py --c ${image_name}" + command: "${workdir}/venv/bin/python scripts/ci/add_supported_release.py --image ${image_name}" task_groups: - name: e2e_test_group @@ -399,7 +399,7 @@ buildvariants: expansions: distro: ubuntu run_on: - - ubuntu1604-build + - ubuntu1804-large depends_on: - name: build_operator_image variant: init_test_run @@ -419,7 +419,7 @@ buildvariants: expansions: distro: ubi run_on: - - ubuntu1604-build + - ubuntu1804-large depends_on: - name: build_operator_image variant: init_test_run @@ -437,7 +437,7 @@ buildvariants: - name: init_test_run display_name: init_test_run run_on: - - ubuntu1604-build + - ubuntu1804-small tasks: - name: build_operator_image - name: build_e2e_image @@ -467,7 +467,7 @@ buildvariants: - name: build_agent_image_ubi variant: init_test_run run_on: - - ubuntu1604-test + - ubuntu1804-small tasks: - name: release_operator - name: release_version_upgrade_post_start_hook diff --git a/requirements.txt b/requirements.txt index 888a1b16a..6bbd2ec52 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ black==20.8b1 mypy==0.782 tqdm==v4.49.0 boto3==1.16.21 -pymongo==3.11.2 +pymongo==3.11.4 dnspython==2.0.0 pyyaml==5.4.1 rsa>=4.7 # not directly required, pinned by Snyk to avoid a vulnerability From 46a824062de9f6b5f14eb71279797a6e677ba735 Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Thu, 10 Jun 2021 09:27:16 +0100 Subject: [PATCH 303/790] Moved DeleteServiceIfItExists to Community (#549) --- pkg/kube/service/service.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/pkg/kube/service/service.go b/pkg/kube/service/service.go index 875b9c620..72da6fa1c 100644 --- a/pkg/kube/service/service.go +++ b/pkg/kube/service/service.go @@ -1,7 +1,11 @@ package service import ( + "fmt" + corev1 "k8s.io/api/core/v1" + apiErrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -44,6 +48,19 @@ type GetUpdateCreateDeleter interface { Deleter } +func DeleteServiceIfItExists(getterDeleter GetDeleter, serviceName types.NamespacedName) error { + _, err := getterDeleter.GetService(serviceName) + if err != nil { + // If it is not found return + if apiErrors.IsNotFound(err) { + return nil + } + // Otherwise we got an error when trying to get it + return fmt.Errorf("can't get service %s: %s", serviceName, err) + } + return getterDeleter.DeleteService(serviceName) +} + // Merge merges `source` into `dest`. Both arguments will remain unchanged // a new service will be created and returned. // The "merging" process is arbitrary and it only handle specific attributes From acc4b4fa86d8d542ade30e325fe738290db74af8 Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Thu, 10 Jun 2021 11:14:11 +0100 Subject: [PATCH 304/790] Move Wait functions into separate package (#551) --- test/e2e/e2eutil.go | 129 ------------------------ test/e2e/mongodbtests/mongodbtests.go | 24 +++-- test/e2e/util/wait/wait.go | 138 ++++++++++++++++++++++++++ 3 files changed, 151 insertions(+), 140 deletions(-) create mode 100644 test/e2e/util/wait/wait.go diff --git a/test/e2e/e2eutil.go b/test/e2e/e2eutil.go index 88be05e76..dd4ca3e6a 100644 --- a/test/e2e/e2eutil.go +++ b/test/e2e/e2eutil.go @@ -4,23 +4,15 @@ import ( "context" "fmt" "reflect" - "testing" - "time" "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/envvar" - "github.com/pkg/errors" - mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" - "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/statefulset" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" apiErrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/wait" - "sigs.k8s.io/controller-runtime/pkg/client" k8sClient "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -47,127 +39,6 @@ func UpdateMongoDBResource(original *mdbv1.MongoDBCommunity, updateFunc func(*md return TestClient.Update(context.TODO(), original) } -// WaitForConfigMapToExist waits until a ConfigMap of the given name exists -// using the provided retryInterval and timeout -func WaitForConfigMapToExist(cmName string, retryInterval, timeout time.Duration) (corev1.ConfigMap, error) { - cm := corev1.ConfigMap{} - return cm, waitForRuntimeObjectToExist(cmName, retryInterval, timeout, &cm, OperatorNamespace) -} - -// WaitForSecretToExist waits until a Secret of the given name exists -// using the provided retryInterval and timeout -func WaitForSecretToExist(cmName string, retryInterval, timeout time.Duration, namespace string) (corev1.Secret, error) { - s := corev1.Secret{} - return s, waitForRuntimeObjectToExist(cmName, retryInterval, timeout, &s, namespace) -} - -// WaitForMongoDBToReachPhase waits until the given MongoDB resource reaches the expected phase -func WaitForMongoDBToReachPhase(t *testing.T, mdb *mdbv1.MongoDBCommunity, phase mdbv1.Phase, retryInterval, timeout time.Duration) error { - return waitForMongoDBCondition(mdb, retryInterval, timeout, func(db mdbv1.MongoDBCommunity) bool { - t.Logf("current phase: %s, waiting for phase: %s", db.Status.Phase, phase) - return db.Status.Phase == phase - }) -} - -// waitForMongoDBCondition polls and waits for a given condition to be true -func waitForMongoDBCondition(mdb *mdbv1.MongoDBCommunity, retryInterval, timeout time.Duration, condition func(mdbv1.MongoDBCommunity) bool) error { - mdbNew := mdbv1.MongoDBCommunity{} - return wait.Poll(retryInterval, timeout, func() (done bool, err error) { - err = TestClient.Get(context.TODO(), mdb.NamespacedName(), &mdbNew) - if err != nil { - return false, err - } - ready := condition(mdbNew) - return ready, nil - }) -} - -// WaitForStatefulSetToExist waits until a StatefulSet of the given name exists -// using the provided retryInterval and timeout -func WaitForStatefulSetToExist(stsName string, retryInterval, timeout time.Duration, namespace string) (appsv1.StatefulSet, error) { - sts := appsv1.StatefulSet{} - return sts, waitForRuntimeObjectToExist(stsName, retryInterval, timeout, &sts, namespace) -} - -// WaitForStatefulSetToHaveUpdateStrategy waits until all replicas of the StatefulSet with the given name -// have reached the ready status -func WaitForStatefulSetToHaveUpdateStrategy(t *testing.T, mdb *mdbv1.MongoDBCommunity, strategy appsv1.StatefulSetUpdateStrategyType, retryInterval, timeout time.Duration) error { - return waitForStatefulSetCondition(t, mdb, retryInterval, timeout, func(sts appsv1.StatefulSet) bool { - return sts.Spec.UpdateStrategy.Type == strategy - }) -} - -// WaitForStatefulSetToBeReady waits until all replicas of the StatefulSet with the given name -// have reached the ready status -func WaitForStatefulSetToBeReady(t *testing.T, mdb *mdbv1.MongoDBCommunity, retryInterval, timeout time.Duration) error { - return waitForStatefulSetCondition(t, mdb, retryInterval, timeout, func(sts appsv1.StatefulSet) bool { - return statefulset.IsReady(sts, mdb.Spec.Members) - }) -} - -// waitForStatefulSetCondition waits until all replicas of the StatefulSet with the given name -// is not ready. -func WaitForStatefulSetToBeUnready(t *testing.T, mdb *mdbv1.MongoDBCommunity, retryInterval, timeout time.Duration) error { - return waitForStatefulSetCondition(t, mdb, retryInterval, timeout, func(sts appsv1.StatefulSet) bool { - return !statefulset.IsReady(sts, mdb.Spec.Members) - }) -} - -// WaitForStatefulSetToBeReadyAfterScaleDown waits for just the ready replicas to be correct -// and does not account for the updated replicas -func WaitForStatefulSetToBeReadyAfterScaleDown(t *testing.T, mdb *mdbv1.MongoDBCommunity, retryInterval, timeout time.Duration) error { - return waitForStatefulSetCondition(t, mdb, retryInterval, timeout, func(sts appsv1.StatefulSet) bool { - return int32(mdb.Spec.Members) == sts.Status.ReadyReplicas - }) -} - -func waitForStatefulSetCondition(t *testing.T, mdb *mdbv1.MongoDBCommunity, retryInterval, timeout time.Duration, condition func(set appsv1.StatefulSet) bool) error { - _, err := WaitForStatefulSetToExist(mdb.Name, retryInterval, timeout, mdb.Namespace) - if err != nil { - return errors.Errorf("error waiting for stateful set to be created: %s", err) - } - - sts := appsv1.StatefulSet{} - return wait.Poll(retryInterval, timeout, func() (done bool, err error) { - err = TestClient.Get(context.TODO(), mdb.NamespacedName(), &sts) - if err != nil { - return false, err - } - t.Logf("Waiting for %s to have %d replicas. Current ready replicas: %d, Current updated replicas: %d, Current generation: %d, Observed Generation: %d\n", - mdb.Name, mdb.Spec.Members, sts.Status.ReadyReplicas, sts.Status.UpdatedReplicas, sts.Generation, sts.Status.ObservedGeneration) - ready := condition(sts) - return ready, nil - }) -} - -func WaitForPodReadiness(t *testing.T, isReady bool, containerName string, timeout time.Duration, pod corev1.Pod) error { - return wait.Poll(time.Second*3, timeout, func() (done bool, err error) { - err = TestClient.Get(context.TODO(), types.NamespacedName{Name: pod.Name, Namespace: pod.Namespace}, &pod) - if err != nil { - return false, err - } - for _, status := range pod.Status.ContainerStatuses { - t.Logf("%s (%s), ready: %v\n", pod.Name, status.Name, status.Ready) - if status.Name == containerName && status.Ready == isReady { - return true, nil - } - } - return false, nil - }) -} - -// waitForRuntimeObjectToExist waits until a runtime.Object of the given name exists -// using the provided retryInterval and timeout provided. -func waitForRuntimeObjectToExist(name string, retryInterval, timeout time.Duration, obj client.Object, namespace string) error { - return wait.Poll(retryInterval, timeout, func() (done bool, err error) { - err = TestClient.Get(context.TODO(), types.NamespacedName{Name: name, Namespace: namespace}, obj) - if err != nil { - return false, client.IgnoreNotFound(err) - } - return true, nil - }) -} - func NewTestMongoDB(ctx *Context, name string, namespace string) (mdbv1.MongoDBCommunity, mdbv1.MongoDBUser) { mongodbNamespace := namespace if mongodbNamespace == "" { diff --git a/test/e2e/mongodbtests/mongodbtests.go b/test/e2e/mongodbtests/mongodbtests.go index 6dce668c3..94acc2eee 100644 --- a/test/e2e/mongodbtests/mongodbtests.go +++ b/test/e2e/mongodbtests/mongodbtests.go @@ -4,10 +4,12 @@ import ( "context" "encoding/json" "fmt" - "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/container" "testing" "time" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/container" + "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/util/wait" + mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" "github.com/mongodb/mongodb-kubernetes-operator/pkg/authentication/scram" "github.com/mongodb/mongodb-kubernetes-operator/pkg/automationconfig" @@ -46,7 +48,7 @@ func StatefulSetBecomesUnready(mdb *mdbv1.MongoDBCommunity) func(t *testing.T) { // failure threshold being high func StatefulSetIsReadyAfterScaleDown(mdb *mdbv1.MongoDBCommunity) func(t *testing.T) { return func(t *testing.T) { - err := e2eutil.WaitForStatefulSetToBeReadyAfterScaleDown(t, mdb, time.Second*60, time.Minute*45) + err := wait.ForStatefulSetToBeReadyAfterScaleDown(t, mdb, time.Second*60, time.Minute*45) if err != nil { t.Fatal(err) } @@ -58,7 +60,7 @@ func StatefulSetIsReadyAfterScaleDown(mdb *mdbv1.MongoDBCommunity) func(t *testi // reaches the running state func statefulSetIsReady(mdb *mdbv1.MongoDBCommunity, interval time.Duration, timeout time.Duration) func(t *testing.T) { return func(t *testing.T) { - err := e2eutil.WaitForStatefulSetToBeReady(t, mdb, interval, timeout) + err := wait.ForStatefulSetToBeReady(t, mdb, interval, timeout) if err != nil { t.Fatal(err) } @@ -69,7 +71,7 @@ func statefulSetIsReady(mdb *mdbv1.MongoDBCommunity, interval time.Duration, tim // statefulSetIsNotReady ensures that the underlying stateful set reaches the unready state. func statefulSetIsNotReady(mdb *mdbv1.MongoDBCommunity, interval time.Duration, timeout time.Duration) func(t *testing.T) { return func(t *testing.T) { - err := e2eutil.WaitForStatefulSetToBeUnready(t, mdb, interval, timeout) + err := wait.ForStatefulSetToBeUnready(t, mdb, interval, timeout) if err != nil { t.Fatal(err) } @@ -136,7 +138,7 @@ func ConnectionStringSecretsAreConfigured(mdb *mdbv1.MongoDBCommunity, expectedO // resource has the correct Update Strategy func StatefulSetHasUpdateStrategy(mdb *mdbv1.MongoDBCommunity, strategy appsv1.StatefulSetUpdateStrategyType) func(t *testing.T) { return func(t *testing.T) { - err := e2eutil.WaitForStatefulSetToHaveUpdateStrategy(t, mdb, strategy, time.Second*15, time.Minute*8) + err := wait.ForStatefulSetToHaveUpdateStrategy(t, mdb, strategy, time.Second*15, time.Minute*8) if err != nil { t.Fatal(err) } @@ -147,7 +149,7 @@ func StatefulSetHasUpdateStrategy(mdb *mdbv1.MongoDBCommunity, strategy appsv1.S // MongoDBReachesRunningPhase ensure the MongoDB resource reaches the Running phase func MongoDBReachesRunningPhase(mdb *mdbv1.MongoDBCommunity) func(t *testing.T) { return func(t *testing.T) { - err := e2eutil.WaitForMongoDBToReachPhase(t, mdb, mdbv1.Running, time.Second*15, time.Minute*12) + err := wait.ForMongoDBToReachPhase(t, mdb, mdbv1.Running, time.Second*15, time.Minute*12) if err != nil { t.Fatal(err) } @@ -158,7 +160,7 @@ func MongoDBReachesRunningPhase(mdb *mdbv1.MongoDBCommunity) func(t *testing.T) // MongoDBReachesFailed ensure the MongoDB resource reaches the Failed phase. func MongoDBReachesFailedPhase(mdb *mdbv1.MongoDBCommunity) func(t *testing.T) { return func(t *testing.T) { - err := e2eutil.WaitForMongoDBToReachPhase(t, mdb, mdbv1.Failed, time.Second*15, time.Minute*5) + err := wait.ForMongoDBToReachPhase(t, mdb, mdbv1.Failed, time.Second*15, time.Minute*5) if err != nil { t.Fatal(err) } @@ -168,7 +170,7 @@ func MongoDBReachesFailedPhase(mdb *mdbv1.MongoDBCommunity) func(t *testing.T) { func AutomationConfigSecretExists(mdb *mdbv1.MongoDBCommunity) func(t *testing.T) { return func(t *testing.T) { - s, err := e2eutil.WaitForSecretToExist(mdb.AutomationConfigSecretName(), time.Second*5, time.Minute*1, mdb.Namespace) + s, err := wait.ForSecretToExist(mdb.AutomationConfigSecretName(), time.Second*5, time.Minute*1, mdb.Namespace) assert.NoError(t, err) t.Logf("Secret %s/%s was successfully created", mdb.AutomationConfigSecretName(), mdb.Namespace) @@ -218,7 +220,7 @@ func CreateMongoDBResource(mdb *mdbv1.MongoDBCommunity, ctx *e2eutil.Context) fu func GetConnectionStringSecret(mdb mdbv1.MongoDBCommunity, user scram.User) corev1.Secret { secret := corev1.Secret{} secretNamespacedName := types.NamespacedName{Name: user.GetConnectionStringSecretName(mdb), Namespace: mdb.Namespace} - e2eutil.TestClient.Get(context.TODO(), secretNamespacedName, &secret) + _ = e2eutil.TestClient.Get(context.TODO(), secretNamespacedName, &secret) return secret } @@ -353,7 +355,7 @@ func StatefulSetContainerConditionIsTrue(mdb *mdbv1.MongoDBCommunity, containerN func PodContainerBecomesNotReady(mdb *mdbv1.MongoDBCommunity, podNum int, containerName string) func(*testing.T) { return func(t *testing.T) { pod := podFromMongoDBCommunity(mdb, podNum) - assert.NoError(t, e2eutil.WaitForPodReadiness(t, false, containerName, time.Minute*10, pod)) + assert.NoError(t, wait.ForPodReadiness(t, false, containerName, time.Minute*10, pod)) } } @@ -361,7 +363,7 @@ func PodContainerBecomesNotReady(mdb *mdbv1.MongoDBCommunity, podNum int, contai func PodContainerBecomesReady(mdb *mdbv1.MongoDBCommunity, podNum int, containerName string) func(*testing.T) { return func(t *testing.T) { pod := podFromMongoDBCommunity(mdb, podNum) - assert.NoError(t, e2eutil.WaitForPodReadiness(t, true, containerName, time.Minute*3, pod)) + assert.NoError(t, wait.ForPodReadiness(t, true, containerName, time.Minute*3, pod)) } } diff --git a/test/e2e/util/wait/wait.go b/test/e2e/util/wait/wait.go new file mode 100644 index 000000000..37d3807d3 --- /dev/null +++ b/test/e2e/util/wait/wait.go @@ -0,0 +1,138 @@ +package wait + +import ( + "context" + "testing" + "time" + + mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/statefulset" + e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" + "github.com/pkg/errors" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// ForConfigMapToExist waits until a ConfigMap of the given name exists +// using the provided retryInterval and timeout +func ForConfigMapToExist(cmName string, retryInterval, timeout time.Duration) (corev1.ConfigMap, error) { + cm := corev1.ConfigMap{} + return cm, waitForRuntimeObjectToExist(cmName, retryInterval, timeout, &cm, e2eutil.OperatorNamespace) +} + +// ForSecretToExist waits until a Secret of the given name exists +// using the provided retryInterval and timeout +func ForSecretToExist(cmName string, retryInterval, timeout time.Duration, namespace string) (corev1.Secret, error) { + s := corev1.Secret{} + return s, waitForRuntimeObjectToExist(cmName, retryInterval, timeout, &s, namespace) +} + +// ForMongoDBToReachPhase waits until the given MongoDB resource reaches the expected phase +func ForMongoDBToReachPhase(t *testing.T, mdb *mdbv1.MongoDBCommunity, phase mdbv1.Phase, retryInterval, timeout time.Duration) error { + return waitForMongoDBCondition(mdb, retryInterval, timeout, func(db mdbv1.MongoDBCommunity) bool { + t.Logf("current phase: %s, waiting for phase: %s", db.Status.Phase, phase) + return db.Status.Phase == phase + }) +} + +// waitForMongoDBCondition polls and waits for a given condition to be true +func waitForMongoDBCondition(mdb *mdbv1.MongoDBCommunity, retryInterval, timeout time.Duration, condition func(mdbv1.MongoDBCommunity) bool) error { + mdbNew := mdbv1.MongoDBCommunity{} + return wait.Poll(retryInterval, timeout, func() (done bool, err error) { + err = e2eutil.TestClient.Get(context.TODO(), mdb.NamespacedName(), &mdbNew) + if err != nil { + return false, err + } + ready := condition(mdbNew) + return ready, nil + }) +} + +// ForStatefulSetToExist waits until a StatefulSet of the given name exists +// using the provided retryInterval and timeout +func ForStatefulSetToExist(stsName string, retryInterval, timeout time.Duration, namespace string) (appsv1.StatefulSet, error) { + sts := appsv1.StatefulSet{} + return sts, waitForRuntimeObjectToExist(stsName, retryInterval, timeout, &sts, namespace) +} + +// ForStatefulSetToHaveUpdateStrategy waits until all replicas of the StatefulSet with the given name +// have reached the ready status +func ForStatefulSetToHaveUpdateStrategy(t *testing.T, mdb *mdbv1.MongoDBCommunity, strategy appsv1.StatefulSetUpdateStrategyType, retryInterval, timeout time.Duration) error { + return waitForStatefulSetCondition(t, mdb, retryInterval, timeout, func(sts appsv1.StatefulSet) bool { + return sts.Spec.UpdateStrategy.Type == strategy + }) +} + +// ForStatefulSetToBeReady waits until all replicas of the StatefulSet with the given name +// have reached the ready status +func ForStatefulSetToBeReady(t *testing.T, mdb *mdbv1.MongoDBCommunity, retryInterval, timeout time.Duration) error { + return waitForStatefulSetCondition(t, mdb, retryInterval, timeout, func(sts appsv1.StatefulSet) bool { + return statefulset.IsReady(sts, mdb.Spec.Members) + }) +} + +// ForStatefulSetToBeUnready waits until all replicas of the StatefulSet with the given name +// is not ready. +func ForStatefulSetToBeUnready(t *testing.T, mdb *mdbv1.MongoDBCommunity, retryInterval, timeout time.Duration) error { + return waitForStatefulSetCondition(t, mdb, retryInterval, timeout, func(sts appsv1.StatefulSet) bool { + return !statefulset.IsReady(sts, mdb.Spec.Members) + }) +} + +// ForStatefulSetToBeReadyAfterScaleDown waits for just the ready replicas to be correct +// and does not account for the updated replicas +func ForStatefulSetToBeReadyAfterScaleDown(t *testing.T, mdb *mdbv1.MongoDBCommunity, retryInterval, timeout time.Duration) error { + return waitForStatefulSetCondition(t, mdb, retryInterval, timeout, func(sts appsv1.StatefulSet) bool { + return int32(mdb.Spec.Members) == sts.Status.ReadyReplicas + }) +} + +func waitForStatefulSetCondition(t *testing.T, mdb *mdbv1.MongoDBCommunity, retryInterval, timeout time.Duration, condition func(set appsv1.StatefulSet) bool) error { + _, err := ForStatefulSetToExist(mdb.Name, retryInterval, timeout, mdb.Namespace) + if err != nil { + return errors.Errorf("error waiting for stateful set to be created: %s", err) + } + + sts := appsv1.StatefulSet{} + return wait.Poll(retryInterval, timeout, func() (done bool, err error) { + err = e2eutil.TestClient.Get(context.TODO(), mdb.NamespacedName(), &sts) + if err != nil { + return false, err + } + t.Logf("Waiting for %s to have %d replicas. Current ready replicas: %d, Current updated replicas: %d, Current generation: %d, Observed Generation: %d\n", + mdb.Name, mdb.Spec.Members, sts.Status.ReadyReplicas, sts.Status.UpdatedReplicas, sts.Generation, sts.Status.ObservedGeneration) + ready := condition(sts) + return ready, nil + }) +} + +func ForPodReadiness(t *testing.T, isReady bool, containerName string, timeout time.Duration, pod corev1.Pod) error { + return wait.Poll(time.Second*3, timeout, func() (done bool, err error) { + err = e2eutil.TestClient.Get(context.TODO(), types.NamespacedName{Name: pod.Name, Namespace: pod.Namespace}, &pod) + if err != nil { + return false, err + } + for _, status := range pod.Status.ContainerStatuses { + t.Logf("%s (%s), ready: %v\n", pod.Name, status.Name, status.Ready) + if status.Name == containerName && status.Ready == isReady { + return true, nil + } + } + return false, nil + }) +} + +// waitForRuntimeObjectToExist waits until a runtime.Object of the given name exists +// using the provided retryInterval and timeout provided. +func waitForRuntimeObjectToExist(name string, retryInterval, timeout time.Duration, obj client.Object, namespace string) error { + return wait.Poll(retryInterval, timeout, func() (done bool, err error) { + err = e2eutil.TestClient.Get(context.TODO(), types.NamespacedName{Name: name, Namespace: namespace}, obj) + if err != nil { + return false, client.IgnoreNotFound(err) + } + return true, nil + }) +} From cb8f25073511f16a9cc230ea08ebaa066a4f9d33 Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Thu, 10 Jun 2021 13:49:24 +0100 Subject: [PATCH 305/790] Increase Stability of FCV tests (#550) --- api/v1/mongodbcommunity_types.go | 2 +- .../feature_compatibility_version_test.go | 6 ++- ...ture_compatibility_version_upgrade_test.go | 6 ++- test/e2e/mongodbtests/mongodbtests.go | 30 +++++++++----- test/e2e/util/wait/wait.go | 26 +++++++----- test/e2e/util/wait/wait_options.go | 41 +++++++++++++++++++ 6 files changed, 85 insertions(+), 26 deletions(-) create mode 100644 test/e2e/util/wait/wait_options.go diff --git a/api/v1/mongodbcommunity_types.go b/api/v1/mongodbcommunity_types.go index 5a353e12a..8131e9a1c 100644 --- a/api/v1/mongodbcommunity_types.go +++ b/api/v1/mongodbcommunity_types.go @@ -339,7 +339,7 @@ type Authentication struct { // +optional // +kubebuilder:default:=true // +nullable - IgnoreUnknownUsers *bool `json:"ignoreUnknownUsers"` + IgnoreUnknownUsers *bool `json:"ignoreUnknownUsers,omitempty"` } // +kubebuilder:validation:Enum=SCRAM diff --git a/test/e2e/feature_compatibility_version/feature_compatibility_version_test.go b/test/e2e/feature_compatibility_version/feature_compatibility_version_test.go index 853e15f19..9938192c6 100644 --- a/test/e2e/feature_compatibility_version/feature_compatibility_version_test.go +++ b/test/e2e/feature_compatibility_version/feature_compatibility_version_test.go @@ -6,6 +6,8 @@ import ( "testing" "time" + "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/util/wait" + . "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/util/mongotester" e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" @@ -49,7 +51,7 @@ func TestFeatureCompatibilityVersion(t *testing.T) { t.Run("MongoDB is reachable while version is upgraded", func(t *testing.T) { defer tester.StartBackgroundConnectivityTest(t, time.Second*10)() t.Run("Test Version can be upgraded", mongodbtests.ChangeVersion(&mdb, "4.2.6")) - t.Run("Stateful Set Reaches Ready State, after Upgrading", mongodbtests.StatefulSetBecomesReady(&mdb)) + t.Run("Stateful Set Reaches Ready State, after Upgrading", mongodbtests.StatefulSetBecomesReady(&mdb, wait.Timeout(20*time.Minute))) }) t.Run("Test Basic Connectivity after upgrade has completed", tester.ConnectivitySucceeds()) @@ -59,7 +61,7 @@ func TestFeatureCompatibilityVersion(t *testing.T) { t.Run("MongoDB is reachable while version is downgraded", func(t *testing.T) { defer tester.StartBackgroundConnectivityTest(t, time.Second*10)() t.Run("Test Version can be downgraded", mongodbtests.ChangeVersion(&mdb, "4.0.6")) - t.Run("Stateful Set Reaches Ready State, after Upgrading", mongodbtests.StatefulSetBecomesReady(&mdb)) + t.Run("Stateful Set Reaches Ready State, after Upgrading", mongodbtests.StatefulSetBecomesReady(&mdb, wait.Timeout(20*time.Minute))) }) t.Run("Test FeatureCompatibilityVersion, after downgrade, is 4.0", tester.HasFCV("4.0", 3)) diff --git a/test/e2e/feature_compatibility_version_upgrade/feature_compatibility_version_upgrade_test.go b/test/e2e/feature_compatibility_version_upgrade/feature_compatibility_version_upgrade_test.go index 1442c9be8..4d7707855 100644 --- a/test/e2e/feature_compatibility_version_upgrade/feature_compatibility_version_upgrade_test.go +++ b/test/e2e/feature_compatibility_version_upgrade/feature_compatibility_version_upgrade_test.go @@ -6,6 +6,8 @@ import ( "testing" "time" + "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/util/wait" + "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/util/mongotester" mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" @@ -51,7 +53,7 @@ func TestFeatureCompatibilityVersionUpgrade(t *testing.T) { t.Run("MongoDB is reachable", func(t *testing.T) { defer tester.StartBackgroundConnectivityTest(t, time.Second*10)() t.Run("Test Version can be upgraded", mongodbtests.ChangeVersion(&mdb, "4.2.6")) - t.Run("Stateful Set Reaches Ready State, after Upgrading", mongodbtests.StatefulSetBecomesReady(&mdb)) + t.Run("Stateful Set Reaches Ready State, after Upgrading", mongodbtests.StatefulSetBecomesReady(&mdb, wait.Timeout(20*time.Minute))) t.Run("Test Basic Connectivity after upgrade has completed", tester.ConnectivitySucceeds()) }) @@ -64,7 +66,7 @@ func TestFeatureCompatibilityVersionUpgrade(t *testing.T) { }) assert.NoError(t, err) }) - t.Run("Stateful Set Reaches Ready State", mongodbtests.StatefulSetBecomesReady(&mdb)) + t.Run("Stateful Set Reaches Ready State", mongodbtests.StatefulSetBecomesReady(&mdb, wait.Timeout(20*time.Minute))) t.Run("MongoDB Reaches Running Phase", mongodbtests.MongoDBReachesRunningPhase(&mdb)) }) t.Run("Test FeatureCompatibilityVersion, after upgrade, is 4.2", tester.HasFCV("4.2", 3)) diff --git a/test/e2e/mongodbtests/mongodbtests.go b/test/e2e/mongodbtests/mongodbtests.go index 94acc2eee..399c08d68 100644 --- a/test/e2e/mongodbtests/mongodbtests.go +++ b/test/e2e/mongodbtests/mongodbtests.go @@ -33,14 +33,24 @@ func SkipTestIfLocal(t *testing.T, msg string, f func(t *testing.T)) { // StatefulSetBecomesReady ensures that the underlying stateful set // reaches the running state. -func StatefulSetBecomesReady(mdb *mdbv1.MongoDBCommunity) func(t *testing.T) { - return statefulSetIsReady(mdb, time.Second*15, time.Minute*12) +func StatefulSetBecomesReady(mdb *mdbv1.MongoDBCommunity, opts ...wait.Configuration) func(t *testing.T) { + defaultOpts := []wait.Configuration{ + wait.RetryInterval(time.Second * 15), + wait.Timeout(time.Minute * 15), + } + defaultOpts = append(defaultOpts, opts...) + return statefulSetIsReady(mdb, defaultOpts...) } // StatefulSetBecomesUnready ensures the underlying stateful set reaches // the unready state. -func StatefulSetBecomesUnready(mdb *mdbv1.MongoDBCommunity) func(t *testing.T) { - return statefulSetIsNotReady(mdb, time.Second*15, time.Minute*12) +func StatefulSetBecomesUnready(mdb *mdbv1.MongoDBCommunity, opts ...wait.Configuration) func(t *testing.T) { + defaultOpts := []wait.Configuration{ + wait.RetryInterval(time.Second * 15), + wait.Timeout(time.Minute * 15), + } + defaultOpts = append(defaultOpts, opts...) + return statefulSetIsNotReady(mdb, defaultOpts...) } // StatefulSetIsReadyAfterScaleDown ensures that a replica set is scaled down correctly @@ -48,7 +58,7 @@ func StatefulSetBecomesUnready(mdb *mdbv1.MongoDBCommunity) func(t *testing.T) { // failure threshold being high func StatefulSetIsReadyAfterScaleDown(mdb *mdbv1.MongoDBCommunity) func(t *testing.T) { return func(t *testing.T) { - err := wait.ForStatefulSetToBeReadyAfterScaleDown(t, mdb, time.Second*60, time.Minute*45) + err := wait.ForStatefulSetToBeReadyAfterScaleDown(t, mdb, wait.RetryInterval(time.Second*60), wait.Timeout(time.Minute*45)) if err != nil { t.Fatal(err) } @@ -58,9 +68,9 @@ func StatefulSetIsReadyAfterScaleDown(mdb *mdbv1.MongoDBCommunity) func(t *testi // StatefulSetIsReady ensures that the underlying stateful set // reaches the running state -func statefulSetIsReady(mdb *mdbv1.MongoDBCommunity, interval time.Duration, timeout time.Duration) func(t *testing.T) { +func statefulSetIsReady(mdb *mdbv1.MongoDBCommunity, opts ...wait.Configuration) func(t *testing.T) { return func(t *testing.T) { - err := wait.ForStatefulSetToBeReady(t, mdb, interval, timeout) + err := wait.ForStatefulSetToBeReady(t, mdb, opts...) if err != nil { t.Fatal(err) } @@ -69,9 +79,9 @@ func statefulSetIsReady(mdb *mdbv1.MongoDBCommunity, interval time.Duration, tim } // statefulSetIsNotReady ensures that the underlying stateful set reaches the unready state. -func statefulSetIsNotReady(mdb *mdbv1.MongoDBCommunity, interval time.Duration, timeout time.Duration) func(t *testing.T) { +func statefulSetIsNotReady(mdb *mdbv1.MongoDBCommunity, opts ...wait.Configuration) func(t *testing.T) { return func(t *testing.T) { - err := wait.ForStatefulSetToBeUnready(t, mdb, interval, timeout) + err := wait.ForStatefulSetToBeUnready(t, mdb, opts...) if err != nil { t.Fatal(err) } @@ -138,7 +148,7 @@ func ConnectionStringSecretsAreConfigured(mdb *mdbv1.MongoDBCommunity, expectedO // resource has the correct Update Strategy func StatefulSetHasUpdateStrategy(mdb *mdbv1.MongoDBCommunity, strategy appsv1.StatefulSetUpdateStrategyType) func(t *testing.T) { return func(t *testing.T) { - err := wait.ForStatefulSetToHaveUpdateStrategy(t, mdb, strategy, time.Second*15, time.Minute*8) + err := wait.ForStatefulSetToHaveUpdateStrategy(t, mdb, strategy, wait.RetryInterval(time.Second*15), wait.Timeout(time.Minute*8)) if err != nil { t.Fatal(err) } diff --git a/test/e2e/util/wait/wait.go b/test/e2e/util/wait/wait.go index 37d3807d3..907a3925b 100644 --- a/test/e2e/util/wait/wait.go +++ b/test/e2e/util/wait/wait.go @@ -60,44 +60,48 @@ func ForStatefulSetToExist(stsName string, retryInterval, timeout time.Duration, // ForStatefulSetToHaveUpdateStrategy waits until all replicas of the StatefulSet with the given name // have reached the ready status -func ForStatefulSetToHaveUpdateStrategy(t *testing.T, mdb *mdbv1.MongoDBCommunity, strategy appsv1.StatefulSetUpdateStrategyType, retryInterval, timeout time.Duration) error { - return waitForStatefulSetCondition(t, mdb, retryInterval, timeout, func(sts appsv1.StatefulSet) bool { +func ForStatefulSetToHaveUpdateStrategy(t *testing.T, mdb *mdbv1.MongoDBCommunity, strategy appsv1.StatefulSetUpdateStrategyType, opts ...Configuration) error { + options := newOptions(opts...) + return waitForStatefulSetCondition(t, mdb, options, func(sts appsv1.StatefulSet) bool { return sts.Spec.UpdateStrategy.Type == strategy }) } // ForStatefulSetToBeReady waits until all replicas of the StatefulSet with the given name // have reached the ready status -func ForStatefulSetToBeReady(t *testing.T, mdb *mdbv1.MongoDBCommunity, retryInterval, timeout time.Duration) error { - return waitForStatefulSetCondition(t, mdb, retryInterval, timeout, func(sts appsv1.StatefulSet) bool { +func ForStatefulSetToBeReady(t *testing.T, mdb *mdbv1.MongoDBCommunity, opts ...Configuration) error { + options := newOptions(opts...) + return waitForStatefulSetCondition(t, mdb, options, func(sts appsv1.StatefulSet) bool { return statefulset.IsReady(sts, mdb.Spec.Members) }) } // ForStatefulSetToBeUnready waits until all replicas of the StatefulSet with the given name // is not ready. -func ForStatefulSetToBeUnready(t *testing.T, mdb *mdbv1.MongoDBCommunity, retryInterval, timeout time.Duration) error { - return waitForStatefulSetCondition(t, mdb, retryInterval, timeout, func(sts appsv1.StatefulSet) bool { +func ForStatefulSetToBeUnready(t *testing.T, mdb *mdbv1.MongoDBCommunity, opts ...Configuration) error { + options := newOptions(opts...) + return waitForStatefulSetCondition(t, mdb, options, func(sts appsv1.StatefulSet) bool { return !statefulset.IsReady(sts, mdb.Spec.Members) }) } // ForStatefulSetToBeReadyAfterScaleDown waits for just the ready replicas to be correct // and does not account for the updated replicas -func ForStatefulSetToBeReadyAfterScaleDown(t *testing.T, mdb *mdbv1.MongoDBCommunity, retryInterval, timeout time.Duration) error { - return waitForStatefulSetCondition(t, mdb, retryInterval, timeout, func(sts appsv1.StatefulSet) bool { +func ForStatefulSetToBeReadyAfterScaleDown(t *testing.T, mdb *mdbv1.MongoDBCommunity, opts ...Configuration) error { + options := newOptions(opts...) + return waitForStatefulSetCondition(t, mdb, options, func(sts appsv1.StatefulSet) bool { return int32(mdb.Spec.Members) == sts.Status.ReadyReplicas }) } -func waitForStatefulSetCondition(t *testing.T, mdb *mdbv1.MongoDBCommunity, retryInterval, timeout time.Duration, condition func(set appsv1.StatefulSet) bool) error { - _, err := ForStatefulSetToExist(mdb.Name, retryInterval, timeout, mdb.Namespace) +func waitForStatefulSetCondition(t *testing.T, mdb *mdbv1.MongoDBCommunity, waitOpts Options, condition func(set appsv1.StatefulSet) bool) error { + _, err := ForStatefulSetToExist(mdb.Name, waitOpts.RetryInterval, waitOpts.Timeout, mdb.Namespace) if err != nil { return errors.Errorf("error waiting for stateful set to be created: %s", err) } sts := appsv1.StatefulSet{} - return wait.Poll(retryInterval, timeout, func() (done bool, err error) { + return wait.Poll(waitOpts.RetryInterval, waitOpts.Timeout, func() (done bool, err error) { err = e2eutil.TestClient.Get(context.TODO(), mdb.NamespacedName(), &sts) if err != nil { return false, err diff --git a/test/e2e/util/wait/wait_options.go b/test/e2e/util/wait/wait_options.go new file mode 100644 index 000000000..01aac62dd --- /dev/null +++ b/test/e2e/util/wait/wait_options.go @@ -0,0 +1,41 @@ +package wait + +import "time" + +type Configuration func(*Options) + +// Options holds values which can be configured when waiting for specific confitions. +type Options struct { + RetryInterval time.Duration + Timeout time.Duration +} + +// RetryInterval specifies the RetryInterval +func RetryInterval(retryInterval time.Duration) Configuration { + return func(options *Options) { + options.RetryInterval = retryInterval + } +} + +// Timeout specifies the Timeout +func Timeout(timeout time.Duration) Configuration { + return func(options *Options) { + options.Timeout = timeout + } +} + +// newOptions returns an Options that has been configured with default values. +func newOptions(fns ...Configuration) Options { + defaults := defaultStatefulSetReadinessOptions() + for _, fn := range fns { + fn(&defaults) + } + return defaults +} + +func defaultStatefulSetReadinessOptions() Options { + return Options{ + RetryInterval: time.Second * 15, + Timeout: time.Minute * 12, + } +} From 948706d04d3eda6a026605930dc65f2e51a98991 Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Thu, 10 Jun 2021 14:59:37 +0100 Subject: [PATCH 306/790] Add GitHub Action to Release All Images (#514) --- .evergreen.yml | 108 ------------------ .../{code_health.yml => code-health.yml} | 0 ...e_release_pr.yml => create-release-pr.yml} | 0 .github/workflows/release-images.yml | 92 +++++++++++++++ docs/how-to-release.md | 22 ++-- release.json | 2 +- requirements.txt | 1 + scripts/ci/add_supported_release.py | 24 ++-- scripts/ci/determine_required_releases.py | 62 ++++++++++ scripts/ci/release_blocker | 10 -- scripts/dev/create_github_release.sh | 7 -- 11 files changed, 180 insertions(+), 148 deletions(-) rename .github/workflows/{code_health.yml => code-health.yml} (100%) rename .github/workflows/{create_release_pr.yml => create-release-pr.yml} (100%) create mode 100644 .github/workflows/release-images.yml create mode 100755 scripts/ci/determine_required_releases.py delete mode 100755 scripts/ci/release_blocker delete mode 100755 scripts/dev/create_github_release.sh diff --git a/.evergreen.yml b/.evergreen.yml index 073a79e0e..574b2c6c9 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -39,15 +39,6 @@ functions: permissions: public-read content_type: text/plain - # This is a blocker for the release process. It will *always* fail and needs to be overriden - # if the release needs to proceed. - release_blocker: - - command: subprocess.exec - type: setup - params: - working_dir: mongodb-kubernetes-operator - binary: scripts/ci/release_blocker - setup_kubernetes_environment: - command: subprocess.exec type: setup @@ -125,20 +116,6 @@ functions: binary: scripts/ci/build_and_push_image_sonar.sh - add_supported_release: - - *python_venv - - *python_requirements - - command: subprocess.exec - type: setup - params: - working_dir: mongodb-kubernetes-operator - include_expansions_in_env: - - atlas_connection_string - - atlas_password - - atlas_database - - image_name - command: "${workdir}/venv/bin/python scripts/ci/add_supported_release.py --image ${image_name}" - task_groups: - name: e2e_test_group max_hosts: 8 @@ -336,62 +313,6 @@ tasks: vars: test: replica_set_custom_role - - name: release_blocker - commands: - - func: clone - - func: release_blocker - - - name: release_operator - commands: - - func: clone - - func: setup_virtualenv - - func: build_and_push_image_sonar - vars: - image_name: operator-ubi - release: true - - - name: release_version_upgrade_post_start_hook - commands: - - func: clone - - func: setup_virtualenv - - func: build_and_push_image_sonar - vars: - image_name: version-post-start-hook-init - release: true - - - - name: release_readiness_probe - commands: - - func: clone - - func: setup_virtualenv - - func: build_and_push_image_sonar - vars: - image_name: readiness-probe-init - release: true - - - name: release_agent_ubuntu - commands: - - func: clone - - func: setup_virtualenv - - func: build_and_push_image_sonar - vars: - image_name: agent-ubuntu - release: true - - func: add_supported_release - vars: - image_name: mongodb-agent - - - name: release_agent_ubi - commands: - - func: clone - - func: setup_virtualenv - - func: build_and_push_image_sonar - vars: - image_name: agent-ubi - release: true - - func: add_supported_release - vars: - image_name: mongodb-agent buildvariants: - name: e2e_tests_ubuntu @@ -445,32 +366,3 @@ buildvariants: - name: build_agent_image_ubi - name: build_agent_image_ubuntu - name: build_readiness_probe_image - - - name: release_blocker - display_name: release_blocker - run_on: - - ubuntu1604-packer # Note: cheapest machine I found - tasks: - - name: release_blocker - - - name: release_images_quay - display_name: release_images_quay - depends_on: - - name: release_blocker - variant: release_blocker - - name: build_operator_image - variant: init_test_run - - name: build_prehook_image - variant: init_test_run - - name: build_agent_image_ubuntu - variant: init_test_run - - name: build_agent_image_ubi - variant: init_test_run - run_on: - - ubuntu1804-small - tasks: - - name: release_operator - - name: release_version_upgrade_post_start_hook - - name: release_readiness_probe - - name: release_agent_ubuntu - - name: release_agent_ubi diff --git a/.github/workflows/code_health.yml b/.github/workflows/code-health.yml similarity index 100% rename from .github/workflows/code_health.yml rename to .github/workflows/code-health.yml diff --git a/.github/workflows/create_release_pr.yml b/.github/workflows/create-release-pr.yml similarity index 100% rename from .github/workflows/create_release_pr.yml rename to .github/workflows/create-release-pr.yml diff --git a/.github/workflows/release-images.yml b/.github/workflows/release-images.yml new file mode 100644 index 000000000..5b60881a9 --- /dev/null +++ b/.github/workflows/release-images.yml @@ -0,0 +1,92 @@ +name: Release Images + +on: + pull_request_review: + types: [submitted] + +jobs: + release-images: + runs-on: ubuntu-latest + if: startsWith(github.event.pull_request.title, 'Release MongoDB Kubernetes Operator') && github.event.review.state == 'approved' + strategy: + matrix: + include: + - pipeline-argument: operator-ubi + release-key: mongodb-kubernetes-operator + - pipeline-argument: version-post-start-hook-init + release-key: version-upgrade-hook + - pipeline-argument: readiness-probe-init + release-key: readiness-probe + - pipeline-argument: agent-ubi + release-key: mongodb-agent + - pipeline-argument: agent-ubuntu + release-key: mongodb-agent + steps: + - name: Checkout Code + uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: '3.8' + architecture: 'x64' + + - uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ hashFiles('requirements.txt') }} + + - name: Install Python Dependencies + run: pip install -r requirements.txt + - name: Determine if release is needed + id: release_status + run: | + OUTPUT=$(scripts/ci/determine_required_releases.py ${{ matrix.release-key }}) + echo "::set-output name=OUTPUT::$OUTPUT" + + - name: Login to Quay.io + uses: docker/login-action@v1 + with: + registry: quay.io + username: ${{ secrets.QUAY_USERNAME }} + password: ${{ secrets.QUAY_ROBOT_TOKEN }} + + - name: Publish Image To Quay + if: steps.release_status.outputs.OUTPUT == 'unreleased' + run: python pipeline.py --image-name ${{ matrix.pipeline-argument }} --release true + env: + MONGODB_COMMUNITY_CONFIG: "${{ github.workspace }}/scripts/ci/config.json" + + - name: Add Supported Release + if: steps.release_status.outputs.OUTPUT == 'unreleased' + run: python scripts/ci/add_supported_release.py --image-name ${{ matrix.release-key }} + env: + ATLAS_DATABASE: "${{ secrets.ATLAS_DATABASE }}" + ATLAS_CONNECTION_STRING: "${{ secrets.ATLAS_CONNECTION_STRING }}" + + merge-release-pr-and-draft-github-release: + runs-on: ubuntu-latest + needs: [release-images] + steps: + - name: Checkout Code + uses: actions/checkout@v2 + - name: Determine Release Tag + id: release_tag + run: | + OUTPUT=$(jq -r '."mongodb-kubernetes-operator"' < $GITHUB_WORKSPACE/release.json) + echo "::set-output name=OUTPUT::$OUTPUT" + - name: Merge Release PR + uses: pascalgn/automerge-action@v0.14.2 + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + MERGE_LABELS: "" + MERGE_DELETE_BRANCH: true + MERGE_METHOD: squash + MERGE_COMMIT_MESSAGE: "Release MongoDB Kubernetes Operator v${{ steps.release_tag.outputs.OUTPUT }}" + - name: Create Github Release + uses: ncipollo/release-action@v1 + with: + tag: ${{ steps.release_tag.outputs.OUTPUT }} + name: MongoDB Kubernetes Operator + bodyFile: "${{ github.workspace }}/docs/RELEASE_NOTES.md" + draft: true + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/docs/how-to-release.md b/docs/how-to-release.md index 19205d2a0..98b71a964 100644 --- a/docs/how-to-release.md +++ b/docs/how-to-release.md @@ -1,17 +1,17 @@ ## How to Release -* Update any finished tickets in [kube-community-next](https://jira.mongodb.org/issues?jql=project%20%3D%20CLOUDP%20AND%20component%20%3D%20%22Kubernetes%20Community%22%20%20AND%20status%20in%20(Resolved%2C%20Closed)%20and%20fixVersion%3D%20kube-community-next%20%20ORDER%20BY%20resolved) to have the version of the release you're doing (kube-community-x.y) +* Ensure the versions specified in [release.json](../release.json) are correct. + * Prepare a PR to bump and versions as required. -* Prepare the release PR - 1. Increment any image version changes. - 2. Create a github draft release `./scripts/dev/create_github_release.sh`. - 3. Commit changes. - -* Create release PR - 1. Reconfigure the Evergreen run to add the relevant release task(s). +* Ensure that [the release notes](./RELEASE_NOTES.md) are up to date for this release. + * Review the [tickets for this release](https://jira.mongodb.org/issues?jql=project%20%3D%20CLOUDP%20AND%20component%20%20%3D%20"Kubernetes%20Community"%20%20AND%20status%20in%20(Resolved%2C%20Closed)%20AND%20fixVersion%20%3D%20kube-community-0.6.0%20) (ensure relevant fix version is in the jql query) +* Run the `Create Release PR` GitHub Action + * In the GitHub UI: + * `Actions` > `Create Release PR` > `Run Workflow` (on master) + +* Review and Approve the release PR that is created by this action. + * Upon approval, all new images for this release will be built and released, and a Github release draft will be created. -* Unblock release task once everything is green - -Once the images are released, merge release PR & publish github release \ No newline at end of file +* Review and publish the new GitHub release draft. diff --git a/release.json b/release.json index b7893458c..33d47baa7 100644 --- a/release.json +++ b/release.json @@ -1,5 +1,5 @@ { - "mongodb-kubernetes-operator": "0.6.0", + "mongodb-kubernetes-operator": "0.6.1", "version-upgrade-hook": "1.0.2", "readiness-probe": "1.0.4", "mongodb-agent": { diff --git a/requirements.txt b/requirements.txt index 6bbd2ec52..7ae27f692 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,5 +9,6 @@ tqdm==v4.49.0 boto3==1.16.21 pymongo==3.11.4 dnspython==2.0.0 +requests==2.24.0 pyyaml==5.4.1 rsa>=4.7 # not directly required, pinned by Snyk to avoid a vulnerability diff --git a/scripts/ci/add_supported_release.py b/scripts/ci/add_supported_release.py index 094f20483..d2780e499 100755 --- a/scripts/ci/add_supported_release.py +++ b/scripts/ci/add_supported_release.py @@ -30,10 +30,7 @@ def get_release() -> Dict[str, Any]: def get_atlas_connection_string() -> str: - password = os.environ["atlas_password"] - cnx_str = os.environ["atlas_connection_string"] - - return cnx_str.format(password=password) + return os.environ["ATLAS_CONNECTION_STRING"] def mongo_client() -> pymongo.MongoClient: @@ -44,7 +41,7 @@ def mongo_client() -> pymongo.MongoClient: def add_release_version(image: str, version: str) -> None: client = mongo_client() - database = os.environ["atlas_database"] + database = os.environ["ATLAS_DATABASE"] collection = client[database][image] year_from_now = datetime.datetime.now() + datetime.timedelta(days=365) @@ -72,18 +69,23 @@ def add_release_version(image: str, version: str) -> None: def main() -> int: parser = argparse.ArgumentParser() parser.add_argument( - "--image", help="image to add a new supported version", type=str + "--image-name", help="image to add a new supported version", type=str ) args = parser.parse_args() - if args.image not in VALID_IMAGES: - raise ValueError("Image {} not supported".format(args.image)) + if args.image_name not in VALID_IMAGES: + print( + "Image {} not supported. Not adding release version.".format( + args.image_name + ) + ) + return 0 # for now, there is just one version to add as a supported release. - version = get_release()[args.image]["version"] - logging.info("Adding new release: {} {}".format(args.image, version)) + version = get_release()[args.image_name]["version"] + logging.info("Adding new release: {} {}".format(args.image_name, version)) - add_release_version(args.image, version) + add_release_version(args.image_name, version) return 0 diff --git a/scripts/ci/determine_required_releases.py b/scripts/ci/determine_required_releases.py new file mode 100755 index 000000000..87a49a65d --- /dev/null +++ b/scripts/ci/determine_required_releases.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 + +# This script accepts a key from the "release.json" file. +# If the corresponding image of the specified version has been released, + +import json +import sys +from typing import List, Dict + +import requests + +# contains a map of the quay urls to fetch data about the corresponding images. +QUAY_URL_MAP = { + "mongodb-agent": "https://quay.io/api/v1/repository/mongodb/mongodb-agent-ubi", + "readiness-probe": "https://quay.io/api/v1/repository/mongodb/mongodb-kubernetes-readinessprobe", + "version-upgrade-hook": "https://quay.io/api/v1/repository/mongodb/mongodb-kubernetes-operator-version-upgrade-post-start-hook", + "mongodb-kubernetes-operator": "https://quay.io/api/v1/repository/mongodb/mongodb-kubernetes-operator", +} + + +def _get_all_released_tags(image_type: str) -> List[str]: + url = QUAY_URL_MAP[image_type] + resp = requests.get(url).json() + tags = resp["tags"] + return list(tags.keys()) + + +def _load_image_name_to_version_map() -> Dict: + with open("release.json") as f: + release = json.loads(f.read()) + + # agent section is a sub object, we change the mapping so the key corresponds to the version directly. + release["mongodb-agent"] = release["mongodb-agent"]["version"] + return release + + +def main() -> int: + if len(sys.argv) != 2: + raise ValueError("usage: determine_required_releases.py [image-name]") + image_name = sys.argv[1] + image_name_map = _load_image_name_to_version_map() + + if image_name not in image_name_map: + raise ValueError( + "Unknown image type [{}], valid values are [{}]".format( + image_name, ",".join(image_name_map.keys()) + ) + ) + + if image_name not in QUAY_URL_MAP: + raise ValueError("No associated image url with key [{}]".format(image_name)) + + tags = _get_all_released_tags(image_name) + if image_name_map[image_name] in tags: + print("released") + else: + print("unreleased") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/ci/release_blocker b/scripts/ci/release_blocker deleted file mode 100755 index 4e2d3f351..000000000 --- a/scripts/ci/release_blocker +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env sh - -echo "🚀" -echo "This is the Release Blocker Task. This task is set to always fail in order to block release operations." -echo "To release/publish the images you have to manually override dependencies on the release_* build variants." -echo "This can only be done on commits that have been tagged!" -echo "🚢" - -# This command will always fail! -exit 1 diff --git a/scripts/dev/create_github_release.sh b/scripts/dev/create_github_release.sh deleted file mode 100755 index ce00b555e..000000000 --- a/scripts/dev/create_github_release.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash - -version="$(jq -r '."mongodb-kubernetes-operator"' < ./release.json)" -gh release create v"${version}" --title "MongoDB Kubernetes Operator ${version}" --draft --notes-file ./dev_notes/RELEASE_NOTES.md - -# move the release notes -cp ./dev_notes/RELEASE_NOTES.md "./dev_notes/past_release_notes/v${version}.md" From 3a70cdd5aa5a061b15f0ca5fb0b962f5e85df620 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 10 Jun 2021 16:35:43 +0100 Subject: [PATCH 307/790] Release Operator v0.6.1 (#553) --- config/manager/manager.yaml | 2 +- deploy/openshift/operator_openshift.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index b4a5c3479..e04ffe83b 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -36,7 +36,7 @@ spec: value: library/mongo - name: MONGODB_REPO_URL value: registry.hub.docker.com - image: quay.io/mongodb/mongodb-kubernetes-operator:0.6.0 + image: quay.io/mongodb/mongodb-kubernetes-operator:0.6.1 imagePullPolicy: Always name: mongodb-kubernetes-operator serviceAccountName: mongodb-kubernetes-operator diff --git a/deploy/openshift/operator_openshift.yaml b/deploy/openshift/operator_openshift.yaml index 19d254cdd..66d5d7fdd 100644 --- a/deploy/openshift/operator_openshift.yaml +++ b/deploy/openshift/operator_openshift.yaml @@ -38,7 +38,7 @@ spec: value: library/mongo - name: MONGODB_REPO_URL value: registry.hub.docker.com - image: quay.io/mongodb/mongodb-kubernetes-operator:0.6.0 + image: quay.io/mongodb/mongodb-kubernetes-operator:0.6.1 imagePullPolicy: Always name: mongodb-kubernetes-operator serviceAccountName: mongodb-kubernetes-operator From 0423fef55d8e92372d1f074771abe08525daf917 Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Fri, 11 Jun 2021 14:29:40 +0100 Subject: [PATCH 308/790] Include MongoDBCommunity resource spec in diagnostics (#552) --- scripts/dev/dump_diagnostic.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/scripts/dev/dump_diagnostic.py b/scripts/dev/dump_diagnostic.py index 1f9cc46e4..3abb5617a 100644 --- a/scripts/dev/dump_diagnostic.py +++ b/scripts/dev/dump_diagnostic.py @@ -1,13 +1,13 @@ import os import shutil import yaml -from typing import Dict, TextIO, List +from typing import Dict, TextIO, List, Union from base64 import b64decode import json import k8s_request_data -def clean_nones(value: Dict) -> Dict: +def clean_nones(value: Dict) -> Union[List, Dict]: """ Recursively remove all None values from dictionaries and lists, and returns the result as a new dictionary or list. @@ -41,10 +41,17 @@ def dump_persistent_volume(diagnostic_file: TextIO) -> None: def dump_stateful_sets_namespaced(diagnostic_file: TextIO, namespace: str) -> None: - sst = k8s_request_data.get_stateful_sets_namespaced(namespace) - if sst is not None: + sts = k8s_request_data.get_stateful_sets_namespaced(namespace) + if sts is not None: diagnostic_file.write(header("Stateful Sets")) - diagnostic_file.write(yaml.dump(clean_nones(sst))) + diagnostic_file.write(yaml.dump(clean_nones(sts))) + + +def dump_mongodbcommunity_namespaced(diagnostic_file: TextIO, namespace: str) -> None: + mdb = k8s_request_data.get_all_mongodb_namespaced(namespace) + if mdb is not None: + diagnostic_file.write(header("MongoDBCommunity")) + diagnostic_file.write(yaml.dump(clean_nones(mdb))) def dump_pod_log_namespaced(namespace: str, name: str, containers: list) -> None: @@ -115,6 +122,7 @@ def dump_all(namespace: str) -> None: with open( "logs/e2e/diagnostics.txt", mode="w", encoding="utf-8" ) as diagnostic_file: + dump_mongodbcommunity_namespaced(diagnostic_file, namespace) dump_persistent_volume(diagnostic_file) dump_stateful_sets_namespaced(diagnostic_file, namespace) dump_pods_and_logs_namespaced(diagnostic_file, namespace) From 70f7499db362ff82bb15e36e0b3d21e6da3d1675 Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Fri, 11 Jun 2021 15:42:56 +0100 Subject: [PATCH 309/790] Added Single Image Github Action (#556) --- .github/workflows/release-single-image.yml | 54 ++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 .github/workflows/release-single-image.yml diff --git a/.github/workflows/release-single-image.yml b/.github/workflows/release-single-image.yml new file mode 100644 index 000000000..3d89f4f62 --- /dev/null +++ b/.github/workflows/release-single-image.yml @@ -0,0 +1,54 @@ +name: Release Single Image +on: + workflow_dispatch: + inputs: + pipeline-argument: + description: 'Argument to pass to pipeline' + required: true + release-key: + description: 'Corresponding release.json key' + required: true +jobs: + release-single-image: + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: '3.8' + architecture: 'x64' + + - uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ hashFiles('requirements.txt') }} + + - name: Install Python Dependencies + run: pip install -r requirements.txt + - name: Determine if release is needed + id: release_status + run: | + OUTPUT=$(scripts/ci/determine_required_releases.py ${{ github.event.inputs.release-key }}) + echo "::set-output name=OUTPUT::$OUTPUT" + + - name: Login to Quay.io + uses: docker/login-action@v1 + with: + registry: quay.io + username: ${{ secrets.QUAY_USERNAME }} + password: ${{ secrets.QUAY_ROBOT_TOKEN }} + + - name: Publish Image To Quay + if: steps.release_status.outputs.OUTPUT == 'unreleased' + run: python pipeline.py --image-name ${{ github.event.inputs.pipeline-argument }} --release true + env: + MONGODB_COMMUNITY_CONFIG: "${{ github.workspace }}/scripts/ci/config.json" + + - name: Add Supported Release + if: steps.release_status.outputs.OUTPUT == 'unreleased' + run: python scripts/ci/add_supported_release.py --image-name ${{ github.event.inputs.release-key }} + env: + ATLAS_DATABASE: "${{ secrets.ATLAS_DATABASE }}" + ATLAS_CONNECTION_STRING: "${{ secrets.ATLAS_CONNECTION_STRING }}" From c76460aaca45ec9e443c1fedc3fa4cd019693397 Mon Sep 17 00:00:00 2001 From: Rodrigo Valin Date: Fri, 11 Jun 2021 16:13:28 +0100 Subject: [PATCH 310/790] Fixes query to find existing releases. (#555) --- scripts/ci/add_supported_release.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ci/add_supported_release.py b/scripts/ci/add_supported_release.py index d2780e499..70da9cb76 100755 --- a/scripts/ci/add_supported_release.py +++ b/scripts/ci/add_supported_release.py @@ -46,7 +46,7 @@ def add_release_version(image: str, version: str) -> None: year_from_now = datetime.datetime.now() + datetime.timedelta(days=365) - existing_entry = collection.find_one({}, {"version": version}) + existing_entry = collection.find_one({"version": version}) if existing_entry is not None: logging.info("Entry for version {} already present".format(version)) return From 7aa51fed68612251860258dfb742a5bd10c21843 Mon Sep 17 00:00:00 2001 From: killian2k Date: Mon, 14 Jun 2021 18:03:31 +0200 Subject: [PATCH 311/790] Fixing the small issue with * (#561) --- docs/install-upgrade.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/install-upgrade.md b/docs/install-upgrade.md index 6a665368e..805124851 100644 --- a/docs/install-upgrade.md +++ b/docs/install-upgrade.md @@ -59,7 +59,7 @@ To configure the Operator to watch resources in other namespaces: imagePullPolicy: Always env: - name: WATCH_NAMESPACE - value: * + value: "*" ``` 2. Run the following command to create cluster-wide roles and role-bindings in the default namespace: From 1f185f38514db89cb9a7fb59ee4510df9bc367d9 Mon Sep 17 00:00:00 2001 From: killian2k Date: Mon, 14 Jun 2021 18:04:01 +0200 Subject: [PATCH 312/790] Fixing the Enterprise MongoDB Kubernetes (#562) --- docs/contributing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/contributing.md b/docs/contributing.md index e5e2a821a..fb1fbbad5 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -4,7 +4,7 @@ First you need to get familiar with the [Architecture guide](architecture.md), w from a high perspective how everything works together. After our experience building the [Enterprise MongoDB Kubernetes -Operator](https://github.com/mongodb/mongodb-enterprise-operator), we have +Operator](https://github.com/mongodb/mongodb-enterprise-kubernetes), we have realized that is is very important to have a clean environment to work, and as such we have adopted a strategy that makes it easier for everyone to contribute. From 5745a5fea047450db5299b4b2d164c8ae5a6650e Mon Sep 17 00:00:00 2001 From: corryroot <72401712+corryroot@users.noreply.github.com> Date: Mon, 14 Jun 2021 12:55:38 -0400 Subject: [PATCH 313/790] (DOCSP-16456): Added new field. (#557) --- docs/deploy-configure.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/deploy-configure.md b/docs/deploy-configure.md index f3e166cf6..a15f22919 100644 --- a/docs/deploy-configure.md +++ b/docs/deploy-configure.md @@ -148,6 +148,7 @@ To define a custom role: | Key | Type | Description | Required? | |----|----|----|----| + | `spec.security.authentication.ignoreUnknownUsers` | boolean | Flag that indicates whether you can add users that don't exist in the `MongoDBCommunity` resource. If omitted, defaults to `true`. | No | | `spec.security.roles` | array | Array that defines [custom roles](https://docs.mongodb.com/manual/core/security-user-defined-roles/) roles that give you fine-grained access control over your MongoDB deployment. | Yes | | `spec.security.roles.role` | string | Name of the custom role. | Yes | | `spec.security.roles.db` | string | Database in which you want to store the user-defined role. | Yes | From 51ee64a16163ee40fabd6d749b5c88036c20ff62 Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Mon, 14 Jun 2021 19:11:11 +0100 Subject: [PATCH 314/790] Run E2E tests with GitHub actions (#560) --- .evergreen.yml | 368 ------------------ .github/workflows/e2e.yml | 199 ++++++++++ .gitignore | 2 + scripts/ci/build_and_push_image.sh | 13 - scripts/ci/build_and_push_image_sonar.sh | 10 - scripts/ci/download.go | 72 ---- scripts/ci/dump_diagnostics.sh | 58 +++ scripts/ci/run_test.sh | 9 - scripts/ci/setup_virtualenv.sh | 8 - scripts/dev/dump_diagnostic.py | 133 ------- scripts/dev/e2e.py | 67 +--- test/e2e/e2eutil.go | 40 ++ .../statefulset_arbitrary_config_test.go | 7 +- ...tatefulset_arbitrary_config_update_test.go | 5 +- 14 files changed, 308 insertions(+), 683 deletions(-) delete mode 100644 .evergreen.yml create mode 100644 .github/workflows/e2e.yml delete mode 100755 scripts/ci/build_and_push_image.sh delete mode 100755 scripts/ci/build_and_push_image_sonar.sh delete mode 100644 scripts/ci/download.go create mode 100755 scripts/ci/dump_diagnostics.sh delete mode 100755 scripts/ci/run_test.sh delete mode 100755 scripts/ci/setup_virtualenv.sh delete mode 100644 scripts/dev/dump_diagnostic.py diff --git a/.evergreen.yml b/.evergreen.yml deleted file mode 100644 index 574b2c6c9..000000000 --- a/.evergreen.yml +++ /dev/null @@ -1,368 +0,0 @@ -ignore: - - "*.md" - -functions: - setup_virtualenv: - - command: subprocess.exec - type: setup - params: - working_dir: mongodb-kubernetes-operator - binary: scripts/ci/setup_virtualenv.sh - include_expansions_in_env: - - sonar_github_token - - clone: - - command: subprocess.exec - type: setup - params: - command: "mkdir -p mongodb-kubernetes-operator" - - command: git.get_project - type: setup - params: - directory: mongodb-kubernetes-operator - - # upload_e2e_logs has the responsibility of dumping as much information as - # possible into the S3 bucket - upload_e2e_logs: - - command: s3.put - params: - aws_key: ${community_aws_access_key_id} - aws_secret: ${community_aws_secret_access_key} - local_files_include_filter_prefix: mongodb-kubernetes-operator/logs/ - local_files_include_filter: - - e2e/*.txt - - e2e/*.log - - e2e/*.json - region: us-east-1 - remote_file: logs/${task_id}/${execution}/ - bucket: community-operator-e2e-logs - permissions: public-read - content_type: text/plain - - setup_kubernetes_environment: - - command: subprocess.exec - type: setup - params: - working_dir: mongodb-kubernetes-operator/scripts/ci - command: go run download.go - env: - URL: https://storage.googleapis.com/kubernetes-release/release/v1.15.4/bin/linux/amd64/kubectl - FILENAME: kubectl - DIR: ${workdir}/bin - - - command: subprocess.exec - type: setup - params: - working_dir: mongodb-kubernetes-operator/scripts/ci - command: go run download.go - env: - URL: https://github.com/kubernetes-sigs/kind/releases/download/v0.7.0/kind-linux-amd64 - FILENAME: kind - DIR: ${workdir}/bin - - create_kind_cluster: - - command: subprocess.exec - type: setup - params: - add_to_path: - - ${workdir}/bin - working_dir: mongodb-kubernetes-operator - binary: scripts/ci/create_kind_cluster.sh - env: - KUBECONFIG: ${workdir}/kube_config - - run_e2e_test: - - command: subprocess.exec - type: test - params: - working_dir: mongodb-kubernetes-operator - env: - KUBECONFIG: ${workdir}/kube_config - include_expansions_in_env: - - version_id - - test - - clusterwide - - distro - binary: scripts/ci/run_test.sh - - python_venv: &python_venv - command: subprocess.exec - type: setup - params: - command: virtualenv --python /opt/python/3.9/bin/python3 ./venv - - python_requirements: &python_requirements - command: subprocess.exec - type: setup - params: - working_dir: mongodb-kubernetes-operator - command: ${workdir}/venv/bin/python -m pip install -r requirements.txt --quiet --no-warn-script-location - - build_and_push_image_sonar: - - command: subprocess.exec - type: setup - params: - env: - MONGODB_COMMUNITY_CONFIG: ${workdir}/mongodb-kubernetes-operator/scripts/ci/config.json - AWS_ACCESS_KEY_ID: ${community_aws_access_key_id} - AWS_SECRET_ACCESS_KEY: ${community_aws_secret_access_key} - include_expansions_in_env: - - version_id - - quay_user_name - - quay_password - - image_name - - release - working_dir: mongodb-kubernetes-operator - binary: scripts/ci/build_and_push_image_sonar.sh - - -task_groups: -- name: e2e_test_group - max_hosts: 8 - setup_group: - - func: clone - - func: setup_virtualenv - - func: setup_kubernetes_environment - setup_task: - - func: create_kind_cluster - tasks: - - e2e_test_replica_set - - e2e_test_replica_set_recovery - - e2e_test_replica_set_mongod_readiness - - e2e_test_replica_set_scale - - e2e_test_replica_set_scale_down - - e2e_test_replica_set_change_version - - e2e_test_feature_compatibility_version - - e2e_test_feature_compatibility_version_upgrade - - e2e_test_replica_set_multiple - - e2e_test_replica_set_tls - - e2e_test_replica_set_tls_upgrade - - e2e_test_replica_set_tls_rotate - - e2e_test_statefulset_arbitrary_config - - e2e_test_statefulset_arbitrary_config_update - - e2e_test_replica_set_mongod_config - - e2e_test_replica_set_cross_namespace_deploy - - e2e_test_replica_set_custom_role - teardown_task: - - func: upload_e2e_logs - -tasks: - - name: build_operator_image - priority: 60 - exec_timeout_secs: 600 - commands: - - func: clone - - func: setup_virtualenv - - func: build_and_push_image_sonar - vars: - image_name: operator-ubi - - - name: build_e2e_image - priority: 60 - exec_timeout_secs: 600 - commands: - - func: clone - - func: setup_virtualenv - - func: build_and_push_image_sonar - vars: - image_name: e2e - - - - name: build_agent_image_ubuntu - priority: 60 - exec_timeout_secs: 600 - commands: - - func: clone - - func: setup_virtualenv - - func: build_and_push_image_sonar - vars: - image_name: agent-ubuntu - - - name: build_agent_image_ubi - priority: 60 - exec_timeout_secs: 600 - commands: - - func: clone - - func: setup_virtualenv - - func: build_and_push_image_sonar - vars: - image_name: agent-ubi - - - name: build_prehook_image - priority: 60 - exec_timeout_secs: 600 - commands: - - func: clone - - func: setup_virtualenv - - func: build_and_push_image_sonar - vars: - image_name: version-post-start-hook-init - - - name: build_readiness_probe_image - priority: 60 - exec_timeout_secs: 600 - commands: - - func: clone - - func: setup_virtualenv - - func: build_and_push_image_sonar - vars: - image_name: readiness-probe-init - - - - name: e2e_test_feature_compatibility_version - commands: - - func: run_e2e_test - vars: - test: feature_compatibility_version - - - name: e2e_test_feature_compatibility_version_upgrade - commands: - - func: run_e2e_test - vars: - test: feature_compatibility_version_upgrade - - - name: e2e_test_replica_set - commands: - - func: run_e2e_test - vars: - test: replica_set - - - name: e2e_test_replica_set_recovery - commands: - - func: run_e2e_test - vars: - test: replica_set_recovery - - - name: e2e_test_replica_set_mongod_readiness - commands: - - func: run_e2e_test - vars: - test: replica_set_mongod_readiness - - - name: e2e_test_replica_set_scale - commands: - - func: run_e2e_test - vars: - test: replica_set_scale - - - name: e2e_test_replica_set_scale_down - exec_timeout_secs: 3600 - commands: - - func: run_e2e_test - vars: - test: replica_set_scale_down - - - name: e2e_test_replica_set_change_version - commands: - - func: run_e2e_test - vars: - test: replica_set_change_version - - - name: e2e_test_replica_set_multiple - commands: - - func: run_e2e_test - vars: - test: replica_set_multiple - - - name: e2e_test_replica_set_tls - commands: - - func: run_e2e_test - vars: - test: replica_set_tls - - - name: e2e_test_replica_set_tls_upgrade - commands: - - func: run_e2e_test - vars: - test: replica_set_tls_upgrade - - - name: e2e_test_replica_set_tls_rotate - commands: - - func: run_e2e_test - vars: - test: replica_set_tls_rotate - - - name: e2e_test_statefulset_arbitrary_config - commands: - - func: run_e2e_test - vars: - test: statefulset_arbitrary_config - - - name: e2e_test_statefulset_arbitrary_config_update - commands: - - func: run_e2e_test - vars: - test: statefulset_arbitrary_config_update - - - name: e2e_test_replica_set_mongod_config - commands: - - func: run_e2e_test - vars: - test: replica_set_mongod_config - - - name: e2e_test_replica_set_cross_namespace_deploy - commands: - - func: run_e2e_test - vars: - test: replica_set_cross_namespace_deploy - clusterwide: true - - - name: e2e_test_replica_set_custom_role - commands: - - func: run_e2e_test - vars: - test: replica_set_custom_role - - -buildvariants: - - name: e2e_tests_ubuntu - display_name: e2e_tests_ubuntu - expansions: - distro: ubuntu - run_on: - - ubuntu1804-large - depends_on: - - name: build_operator_image - variant: init_test_run - - name: build_e2e_image - variant: init_test_run - - name: build_prehook_image - variant: init_test_run - - name: build_readiness_probe_image - variant: init_test_run - - name: build_agent_image_ubuntu - variant: init_test_run - tasks: - - name: e2e_test_group - - - name: e2e_tests_ubi - display_name: e2e_tests_ubi - expansions: - distro: ubi - run_on: - - ubuntu1804-large - depends_on: - - name: build_operator_image - variant: init_test_run - - name: build_e2e_image - variant: init_test_run - - name: build_prehook_image - variant: init_test_run - - name: build_readiness_probe_image - variant: init_test_run - - name: build_agent_image_ubi - variant: init_test_run - tasks: - - name: e2e_test_group - - - name: init_test_run - display_name: init_test_run - run_on: - - ubuntu1804-small - tasks: - - name: build_operator_image - - name: build_e2e_image - - name: build_prehook_image - - name: build_agent_image_ubi - - name: build_agent_image_ubuntu - - name: build_readiness_probe_image diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 000000000..278466177 --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,199 @@ +name: Run E2E +on: + pull_request: + branches: + - master + push: + branches: + - master +jobs: + setup: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - pipeline-argument: operator-ubi + - pipeline-argument: version-post-start-hook-init + - pipeline-argument: readiness-probe-init + - pipeline-argument: agent-ubi + - pipeline-argument: agent-ubuntu + - pipeline-argument: e2e + steps: + - name: Checkout Code + uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: '3.8' + + - uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ hashFiles('requirements.txt') }} + + - name: Install Python Dependencies + run: pip install -r requirements.txt + + - name: Login to Quay.io + uses: docker/login-action@v1 + with: + registry: quay.io + username: ${{ secrets.QUAY_USERNAME }} + password: ${{ secrets.QUAY_ROBOT_TOKEN }} + + - name: Build and Push Images + run: python pipeline.py --image-name ${{ matrix.pipeline-argument }} --release false + env: + MONGODB_COMMUNITY_CONFIG: "${{ github.workspace }}/scripts/ci/config.json" + version_id: "${{ github.run_id }}" + + tests: + runs-on: ubuntu-latest + needs: [setup] + strategy: + fail-fast: false + matrix: + include: + - test-name: replica_set + distro: ubuntu + - test-name: replica_set_recovery + distro: ubuntu + - test-name: replica_set_recovery + distro: ubuntu + - test-name: replica_set_mongod_readiness + distro: ubuntu + - test-name: replica_set_scale + distro: ubuntu + - test-name: replica_set_scale_down + distro: ubuntu + - test-name: replica_set_change_version + distro: ubuntu + - test-name: feature_compatibility_version + distro: ubuntu + - test-name: feature_compatibility_version_upgrade + distro: ubuntu + - test-name: replica_set_tls + distro: ubuntu + - test-name: replica_set_tls_upgrade + distro: ubuntu + - test-name: statefulset_arbitrary_config + distro: ubuntu + - test-name: statefulset_arbitrary_config_update + distro: ubuntu + - test-name: replica_set_mongod_config + distro: ubuntu + - test-name: replica_set_cross_namespace_deploy + distro: ubuntu + cluster-wide: true + - test-name: replica_set_custom_role + distro: ubuntu + + - test-name: replica_set + distro: ubi + - test-name: replica_set_recovery + distro: ubi + - test-name: replica_set_recovery + distro: ubi + - test-name: replica_set_mongod_readiness + distro: ubi + - test-name: replica_set_scale + distro: ubi + - test-name: replica_set_scale_down + distro: ubi + - test-name: replica_set_change_version + distro: ubi + - test-name: feature_compatibility_version + distro: ubi + - test-name: feature_compatibility_version_upgrade + distro: ubi + - test-name: replica_set_tls + distro: ubi + - test-name: replica_set_tls_upgrade + distro: ubi + - test-name: statefulset_arbitrary_config + distro: ubi + - test-name: statefulset_arbitrary_config_update + distro: ubi + - test-name: replica_set_mongod_config + distro: ubi + - test-name: replica_set_cross_namespace_deploy + distro: ubi + cluster-wide: true + - test-name: replica_set_custom_role + distro: ubi + + + steps: + - name: Checkout Code + uses: actions/checkout@v2 + + - name: Set default run status + run: echo "::set-output name=last_run_status::pending" > last_run_status + + # Tracking of the state of the previous test run is a workaround to the fact that it is not + # possible to re-run a single failed job, only re-running the entire workflow is currently possible. + # This workaround skips jobs if they have already passed. + # see https://github.com/actions/runner/issues/432 + - name: Restore last run status + id: last_run + uses: actions/cache@v2 + with: + path: last_run_status + key: ${{ github.run_id }}-${{ matrix.test-name }}-${{ matrix.distro }} + + - name: Set last run status + id: last_run_status + run: cat last_run_status + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: '3.8' + + - name: Cache Dependencies + uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ hashFiles('requirements.txt') }} + + - name: Install Python Dependencies + run: pip install -r requirements.txt + + - name: Setup Kind Cluster + if: steps.last_run_status.outputs.last_run_status != 'success' + uses: engineerd/setup-kind@v0.5.0 + with: + version: "v0.8.1" + + - name: Install CRD + if: steps.last_run_status.outputs.last_run_status != 'success' + run: kubectl apply -f config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml + + - name: Run Test + id: e2e_test + if: steps.last_run_status.outputs.last_run_status != 'success' + run: | + cluster_wide=${{ matrix.cluster-wide }} + if [ -z "$cluster_wide" ]; then + cluster_wide="false" + fi + python3 ./scripts/dev/e2e.py --test ${{ matrix.test-name }} --tag ${{ github.run_id }} --config_file ./scripts/ci/config.json --distro ${{ matrix.distro }} --cluster-wide ${cluster_wide} + + - name: Save run status + if: always() + run: echo "::set-output name=last_run_status::${{ steps.e2e_test.outcome }}" > last_run_status + + - name: Dump Diagnostics + id: dump_diagnostics + if: always() && steps.e2e_test.outcome == 'failure' + continue-on-error: true + run: scripts/ci/dump_diagnostics.sh default # default since kind is running in the default namespace + + - name: Upload Diagnostics + if: always() && steps.dump_diagnostics.outcome == 'success' + uses: actions/upload-artifact@v2 + continue-on-error: true + with: + name: "${{ matrix.test-name }}-${{ matrix.distro }}-diagnostics" + path: "${{ github.workspace }}/diagnostics" diff --git a/.gitignore b/.gitignore index d1e3f2356..332b5af7e 100644 --- a/.gitignore +++ b/.gitignore @@ -92,3 +92,5 @@ testbin/bin # ignore files generated by sonar Dockerfile.ubi-* Dockerfile.ubuntu-* + +diagnostics diff --git a/scripts/ci/build_and_push_image.sh b/scripts/ci/build_and_push_image.sh deleted file mode 100755 index 30006795b..000000000 --- a/scripts/ci/build_and_push_image.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash - -set +x - -# shellcheck disable=SC1091 -. venv/bin/activate -echo "${quay_password:?}" | docker login "-u=${quay_user_name:?}" quay.io --password-stdin - -python3 scripts/dev/dockerfile_generator.py "${image_type:?}" > Dockerfile - -# Providing the quay.expires-after configures quay to delete this image after the provided amount of time -docker build . --label "quay.expires-after=${expire_after:-never}" -f Dockerfile -t "${image:?}" -docker push "${image:?}" diff --git a/scripts/ci/build_and_push_image_sonar.sh b/scripts/ci/build_and_push_image_sonar.sh deleted file mode 100755 index a9c760a23..000000000 --- a/scripts/ci/build_and_push_image_sonar.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash - -set +x - -# shellcheck disable=SC1091 -. venv/bin/activate -echo "${quay_password:?}" | docker login "-u=${quay_user_name:?}" quay.io --password-stdin - -# shellcheck disable=SC2154 -python3 pipeline.py --image-name "${image_name}" --release "${release:-false}" diff --git a/scripts/ci/download.go b/scripts/ci/download.go deleted file mode 100644 index 761344d31..000000000 --- a/scripts/ci/download.go +++ /dev/null @@ -1,72 +0,0 @@ -package main - -import ( - "fmt" - "io" - "net/http" - "os" - "path" -) - -// download.go uses the following environment variables: -// URL: The url of the file to download -// DIR: The directory which the newly downloaded file will be placed -// FILENAME: The name the file should have after being downloaded - -func main() { - if err := downloadFile(makeOptions()); err != nil { - os.Exit(1) - } -} - -type downloadOptions struct { - url, fileName, dir string - perms os.FileMode -} - -func makeOptions() downloadOptions { - return downloadOptions{ - url: os.Getenv("URL"), - fileName: os.Getenv("FILENAME"), - perms: os.FileMode(755), - dir: os.Getenv("DIR"), - } -} - -func downloadFile(opts downloadOptions) error { - fmt.Printf("Using download options: %+v\n", opts) - fullPath := path.Join(opts.dir, opts.fileName) - fmt.Printf("full path to directory: %s\n", fullPath) - if err := os.MkdirAll(opts.dir, opts.perms); err != nil { - return fmt.Errorf("error making directory %s with permissions %d: %s", opts.dir, opts.perms, err) - } - if err := fetchFile(fullPath, opts.url); err != nil { - return fmt.Errorf("error fetching file: %s", err) - } - fmt.Printf("successfully downloaded file from %s to %s\n", opts.url, fullPath) - if err := os.Chmod(fullPath, opts.perms); err != nil { - return fmt.Errorf("error changing file permissions: %s", err) - } - return nil -} - -func fetchFile(filePath, url string) error { - resp, err := http.Get(url) //nolint - if err != nil { - return fmt.Errorf("error getting url: %s", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("bad status: %s", resp.Status) - } - - out, err := os.Create(filePath) - if err != nil { - return fmt.Errorf("error creating file: %s", err) - } - defer out.Close() - - _, err = io.Copy(out, resp.Body) - return err -} diff --git a/scripts/ci/dump_diagnostics.sh b/scripts/ci/dump_diagnostics.sh new file mode 100755 index 000000000..5f63fac73 --- /dev/null +++ b/scripts/ci/dump_diagnostics.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash + +mkdir -p diagnostics/secrets + +namespace="$1" + +echo "Dumping CRD" +kubectl get crd mongodbcommunity.mongodbcommunity.mongodb.com -o yaml > diagnostics/crd.yaml + +# dump operator deployment information. +for deployment_name in $(kubectl get deployment -n "${namespace}" --output=jsonpath={.items..metadata.name}); do + echo "Writing Deployment describe for deployment ${deployment_name}" + kubectl describe deploy "${deployment_name}" > "diagnostics/${deployment_name}.txt" + + echo "Writing Deployment yaml for deployment ${deployment_name}" + kubectl get deploy "${deployment_name}" -o yaml > "diagnostics/${deployment_name}.yaml" +done + +# dump logs for every container in every pod in the given namespace +for pod_name in $(kubectl get pod -n "${namespace}" --output=jsonpath={.items..metadata.name}); do + echo "Writing Pod describe for pod ${pod_name}" + kubectl describe pod "${pod_name}" > "diagnostics/${pod_name}.txt" + + echo "Writing Pod yaml for pod ${pod_name}" + kubectl get pod "${pod_name}" -o yaml > "diagnostics/${pod_name}.yaml" + + for container_name in $(kubectl get pods -n "${namespace}" "${pod_name}" -o jsonpath='{.spec.containers[*].name}'); do + echo "Writing log file for pod ${pod_name} - container ${container_name} to diagnostics/${pod_name}-${container_name}.log" + kubectl logs -n "${namespace}" "${pod_name}" -c "${container_name}" > "diagnostics/${pod_name}-${container_name}.log"; + done +done + +# dump information about MongoDBCommunity resources and statefulsets. +for mdbc_name in $(kubectl get mongodbcommunity -n "${namespace}" --output=jsonpath={.items..metadata.name}); do + echo "Writing MongoDBCommunity describe" + kubectl describe mongodbcommunity "${mdbc_name}" -n "${namespace}" > "diagnostics/${mdbc_name}-mongodbcommunity.txt" + echo "Writing yaml output for MongoDBCommunity ${mdbc_name}" + kubectl get mongodbcommunity "${mdbc_name}" -n "${namespace}" -o yaml > "diagnostics/${mdbc_name}-mongodbcommunity.yaml" + echo "Writing describe output for StatefulSet ${mdbc_name}" + kubectl describe sts "${mdbc_name}" -n "${namespace}" > "diagnostics/${mdbc_name}-statefulset.txt" + echo "Writing yaml output for StatefulSet ${mdbc_name}" + kubectl get sts "${mdbc_name}" -n "${namespace}" -o yaml > "diagnostics/${mdbc_name}-statefulset.yaml" + + echo "Writing Automation Config Secret" + kubectl get secret "${mdbc_name}-config" -o jsonpath='{ .data.cluster-config\.json}' | base64 -d | jq > "diagnostics/secrets/${mdbc_name}-config.json" +done + + +# dump information about relevant secrets. +# Skip service account tokens, and also skip the Automation Config as this is handled as a special case above. +for secret in $(kubectl get secret -n "${namespace}" --output=jsonpath={.items..metadata.name}); do + if ! echo "${secret}" | grep -qE "token|-config"; then + echo "Writing secret ${secret}" + kubectl get secret "${secret}" -o json | jq -r '.data | with_entries(.value |= @base64d)' > "diagnostics/secrets/${secret}.json" + else + echo "Skipping secret ${secret}" + fi +done diff --git a/scripts/ci/run_test.sh b/scripts/ci/run_test.sh deleted file mode 100755 index c8cd77937..000000000 --- a/scripts/ci/run_test.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash - -# shellcheck disable=SC1091 -. venv/bin/activate -if [ -z "${clusterwide:-}" ]; then - python3 ./scripts/dev/e2e.py --test "${test:?}" --tag "${version_id:?}" --config_file ./scripts/ci/config.json --distro "${distro:?}" -else - python3 ./scripts/dev/e2e.py --test "${test:?}" --tag "${version_id:?}" --config_file ./scripts/ci/config.json --cluster-wide --distro "${distro:?}" -fi diff --git a/scripts/ci/setup_virtualenv.sh b/scripts/ci/setup_virtualenv.sh deleted file mode 100755 index 8edf35157..000000000 --- a/scripts/ci/setup_virtualenv.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash - -# shellcheck disable=SC1091 -virtualenv --python /opt/python/3.7/bin/python3 ./venv -. venv/bin/activate - -pip3 install -r ./requirements.txt - diff --git a/scripts/dev/dump_diagnostic.py b/scripts/dev/dump_diagnostic.py deleted file mode 100644 index 3abb5617a..000000000 --- a/scripts/dev/dump_diagnostic.py +++ /dev/null @@ -1,133 +0,0 @@ -import os -import shutil -import yaml -from typing import Dict, TextIO, List, Union -from base64 import b64decode -import json -import k8s_request_data - - -def clean_nones(value: Dict) -> Union[List, Dict]: - """ - Recursively remove all None values from dictionaries and lists, and returns - the result as a new dictionary or list. - """ - if isinstance(value, list): - return [clean_nones(x) for x in value if x is not None] - if isinstance(value, dict): - return {key: clean_nones(val) for key, val in value.items() if val is not None} - return value - - -def header(msg: str) -> str: - dashes = ( - "----------------------------------------------------------------------------" - ) - return f"\n{dashes}\n{msg}\n{dashes}\n" - - -def dump_crd(crd_log: TextIO) -> None: - crd = k8s_request_data.get_crds() - if crd is not None: - crd_log.write(header("CRD")) - crd_log.write(yaml.dump(clean_nones(crd))) - - -def dump_persistent_volume(diagnostic_file: TextIO) -> None: - pv = k8s_request_data.get_persistent_volumes() - if pv is not None: - diagnostic_file.write(header("Persistent Volumes")) - diagnostic_file.write(yaml.dump(clean_nones(pv))) - - -def dump_stateful_sets_namespaced(diagnostic_file: TextIO, namespace: str) -> None: - sts = k8s_request_data.get_stateful_sets_namespaced(namespace) - if sts is not None: - diagnostic_file.write(header("Stateful Sets")) - diagnostic_file.write(yaml.dump(clean_nones(sts))) - - -def dump_mongodbcommunity_namespaced(diagnostic_file: TextIO, namespace: str) -> None: - mdb = k8s_request_data.get_all_mongodb_namespaced(namespace) - if mdb is not None: - diagnostic_file.write(header("MongoDBCommunity")) - diagnostic_file.write(yaml.dump(clean_nones(mdb))) - - -def dump_pod_log_namespaced(namespace: str, name: str, containers: list) -> None: - for container in containers: - with open( - f"logs/e2e/{name}-{container.name}.log", - mode="w", - encoding="utf-8", - ) as log_file: - log = k8s_request_data.get_pod_log_namespaced( - namespace, name, container.name - ) - if log is not None: - log_file.write(log) - - -def dump_pods_and_logs_namespaced(diagnostic_file: TextIO, namespace: str) -> None: - pods = k8s_request_data.get_pods_namespaced(namespace) - if pods is not None: - for pod in pods: - name = pod.metadata.name - diagnostic_file.write(header(f"Pod {name}")) - diagnostic_file.write(yaml.dump(clean_nones(pod.to_dict()))) - dump_pod_log_namespaced(namespace, name, pod.spec.containers) - - -def dump_secret_keys_namespaced( - namespace: str, keys: List[str], secret_name: str -) -> None: - secret = k8s_request_data.get_secret_namespaced(namespace, secret_name) - if secret is not None: - for key in keys: - with open( - f"logs/e2e/{secret_name}-{key}", - mode="w", - encoding="utf-8", - ) as log_file: - if key in secret["data"]: - decoded_data = _decode_secret(secret["data"]) - log_file.write(json.dumps(json.loads(decoded_data[key]), indent=4)) - - -def _decode_secret(data: Dict[str, str]) -> Dict[str, str]: - return {k: b64decode(v).decode("utf-8") for (k, v) in data.items()} - - -def dump_automation_configs(namespace: str) -> None: - mongodb_resources = k8s_request_data.get_all_mongodb_namespaced(namespace) - if mongodb_resources is None: - print("No MongoDB resources found, not dumping any automation configs") - return - for mdb in mongodb_resources: - name = mdb["metadata"]["name"] - dump_secret_keys_namespaced( - namespace, ["cluster-config.json"], f"{name}-config" - ) - - -def dump_all(namespace: str) -> None: - if os.path.exists("logs"): - shutil.rmtree("logs") - - os.makedirs("logs") - - if not os.path.exists("logs/e2e"): - os.makedirs("logs/e2e") - - with open( - "logs/e2e/diagnostics.txt", mode="w", encoding="utf-8" - ) as diagnostic_file: - dump_mongodbcommunity_namespaced(diagnostic_file, namespace) - dump_persistent_volume(diagnostic_file) - dump_stateful_sets_namespaced(diagnostic_file, namespace) - dump_pods_and_logs_namespaced(diagnostic_file, namespace) - - with open("logs/e2e/crd.log", mode="w", encoding="utf-8") as crd_log: - dump_crd(crd_log) - - dump_automation_configs(namespace) diff --git a/scripts/dev/e2e.py b/scripts/dev/e2e.py index bcc6e0eb4..80b1e2b0b 100644 --- a/scripts/dev/e2e.py +++ b/scripts/dev/e2e.py @@ -3,12 +3,10 @@ from kubernetes.client.rest import ApiException import k8s_conditions -import dump_diagnostic from typing import Dict from dev_config import load_config, DevConfig, Distro from kubernetes import client, config import argparse -import os import sys import yaml @@ -65,44 +63,6 @@ def _prepare_test_environment(config_file: str) -> None: ) -def create_kube_config(config_file: str) -> None: - """Replicates the local kubeconfig file (pointed at by KUBECONFIG), - as a ConfigMap.""" - corev1 = client.CoreV1Api() - print("Creating kube-config ConfigMap") - dev_config = load_config(config_file) - - svc = corev1.read_namespaced_service("kubernetes", "default") - - kube_config_path = os.getenv("KUBECONFIG") - if kube_config_path is None: - raise ValueError("kube_config_path must not be None") - - with open(kube_config_path) as fd: - kube_config = yaml.safe_load(fd.read()) - - if kube_config is None: - raise ValueError("kube_config_path must not be None") - - kube_config["clusters"][0]["cluster"]["server"] = "https://" + svc.spec.cluster_ip - kube_config = yaml.safe_dump(kube_config) - data = {"kubeconfig": kube_config} - config_map = client.V1ConfigMap( - metadata=client.V1ObjectMeta(name="kube-config"), data=data - ) - - print("Creating Namespace") - k8s_conditions.ignore_if_already_exists( - lambda: corev1.create_namespace( - client.V1Namespace(metadata=dict(name=dev_config.namespace)) - ) - ) - - k8s_conditions.ignore_if_already_exists( - lambda: corev1.create_namespaced_config_map(dev_config.namespace, config_map) - ) - - def _delete_test_pod(config_file: str) -> None: """ _delete_testrunner_pod deletes the test runner pod @@ -131,9 +91,6 @@ def create_test_pod(args: argparse.Namespace, dev_config: DevConfig) -> None: "name": TEST_POD_NAME, "image": f"{dev_config.repo_url}/{dev_config.e2e_image}:{args.tag}", "imagePullPolicy": "Always", - "volumeMounts": [ - {"mountPath": "/etc/config", "name": "kube-config-volume"} - ], "env": [ { "name": "CLUSTER_WIDE", @@ -174,14 +131,6 @@ def create_test_pod(args: argparse.Namespace, dev_config: DevConfig) -> None: ], } ], - "volumes": [ - { - "name": "kube-config-volume", - "configMap": { - "name": "kube-config", - }, - } - ], }, } if not k8s_conditions.wait( @@ -214,10 +163,12 @@ def wait_for_pod_to_be_running( lambda: corev1.read_namespaced_pod(name, namespace), lambda pod: pod.status.phase == "Running", sleep_time=5, - timeout=180, + timeout=240, exceptions_to_ignore=ApiException, ): - raise Exception("Pod never got into Running state!") + + pod = corev1.read_namespaced_pod(name, namespace) + raise Exception("Pod never got into Running state: {}".format(pod)) print("Pod is running") @@ -243,7 +194,7 @@ def parse_args() -> argparse.Namespace: parser.add_argument( "--cluster-wide", help="Watch all namespaces", - action="store_true", + type=lambda x: x.lower() == "true", ) parser.add_argument( "--distro", @@ -278,13 +229,7 @@ def main() -> int: config.load_kube_config() dev_config = load_config(args.config_file, Distro.from_string(args.distro)) - create_kube_config(args.config_file) - - try: - prepare_and_run_test(args, dev_config) - finally: - if not args.skip_dump_diagnostic: - dump_diagnostic.dump_all(dev_config.namespace) + prepare_and_run_test(args, dev_config) corev1 = client.CoreV1Api() if not k8s_conditions.wait( diff --git a/test/e2e/e2eutil.go b/test/e2e/e2eutil.go index dd4ca3e6a..c9fc1d0c3 100644 --- a/test/e2e/e2eutil.go +++ b/test/e2e/e2eutil.go @@ -5,6 +5,9 @@ import ( "fmt" "reflect" + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/api/resource" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/envvar" mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" @@ -93,6 +96,43 @@ func NewTestMongoDB(ctx *Context, name string, namespace string) (mdbv1.MongoDBC ScramCredentialsSecretName: fmt.Sprintf("%s-my-scram", name), }, }, + StatefulSetConfiguration: mdbv1.StatefulSetConfiguration{ + SpecWrapper: mdbv1.StatefulSetSpecWrapper{ + Spec: appsv1.StatefulSetSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "mongod", + Resources: corev1.ResourceRequirements{ + Limits: map[corev1.ResourceName]resource.Quantity{ + "cpu": resource.MustParse("0.1"), + "memory": resource.MustParse("200M"), + }, + Requests: map[corev1.ResourceName]resource.Quantity{ + "cpu": resource.MustParse("0.1"), + "memory": resource.MustParse("200M"), + }, + }, + }, + { + Name: "mongodb-agent", + Resources: corev1.ResourceRequirements{ + Limits: map[corev1.ResourceName]resource.Quantity{ + "cpu": resource.MustParse("0.1"), + "memory": resource.MustParse("200M"), + }, + Requests: map[corev1.ResourceName]resource.Quantity{ + "cpu": resource.MustParse("0.1"), + "memory": resource.MustParse("200M"), + }, + }, + }, + }, + }, + }, + }}, + }, }, } return mdb, mdb.Spec.Users[0] diff --git a/test/e2e/statefulset_arbitrary_config/statefulset_arbitrary_config_test.go b/test/e2e/statefulset_arbitrary_config/statefulset_arbitrary_config_test.go index 8372c23c9..527fdcc09 100644 --- a/test/e2e/statefulset_arbitrary_config/statefulset_arbitrary_config_test.go +++ b/test/e2e/statefulset_arbitrary_config/statefulset_arbitrary_config_test.go @@ -5,7 +5,6 @@ import ( "os" "testing" - v1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/util/mongotester" e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" @@ -33,11 +32,7 @@ func TestStatefulSetArbitraryConfig(t *testing.T) { t.Fatal(err) } - overrideSpec := v1.StatefulSetConfiguration{} - overrideSpec.SpecWrapper.Spec.Template.Spec.Containers = []corev1.Container{ - {Name: "mongodb-agent", ReadinessProbe: &corev1.Probe{TimeoutSeconds: 100}}} - - mdb.Spec.StatefulSetConfiguration = overrideSpec + mdb.Spec.StatefulSetConfiguration.SpecWrapper.Spec.Template.Spec.Containers[1].ReadinessProbe = &corev1.Probe{TimeoutSeconds: 100} tester, err := mongotester.FromResource(t, mdb) if err != nil { diff --git a/test/e2e/statefulset_arbitrary_config_update/statefulset_arbitrary_config_update_test.go b/test/e2e/statefulset_arbitrary_config_update/statefulset_arbitrary_config_update_test.go index 89711e9e1..143c71b77 100644 --- a/test/e2e/statefulset_arbitrary_config_update/statefulset_arbitrary_config_update_test.go +++ b/test/e2e/statefulset_arbitrary_config_update/statefulset_arbitrary_config_update_test.go @@ -44,9 +44,8 @@ func TestStatefulSetArbitraryConfig(t *testing.T) { t.Run("Test basic connectivity", tester.ConnectivitySucceeds()) t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 1)) - overrideSpec := v1.StatefulSetConfiguration{} - overrideSpec.SpecWrapper.Spec.Template.Spec.Containers = []corev1.Container{ - {Name: "mongodb-agent", ReadinessProbe: &corev1.Probe{TimeoutSeconds: 100}}} + overrideSpec := mdb.Spec.StatefulSetConfiguration + overrideSpec.SpecWrapper.Spec.Template.Spec.Containers[1].ReadinessProbe = &corev1.Probe{TimeoutSeconds: 100} err = e2eutil.UpdateMongoDBResource(&mdb, func(mdb *v1.MongoDBCommunity) { mdb.Spec.StatefulSetConfiguration = overrideSpec }) From 629744ded416d6e045d6c11ec2fe17850a0a3832 Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Tue, 15 Jun 2021 09:14:56 +0100 Subject: [PATCH 315/790] Add Manually Triggerable E2E Workflow (#564) --- .github/workflows/e2e-dispatch.yml | 103 +++++++++++++++++++++++++++++ .github/workflows/e2e.yml | 10 +++ Makefile | 4 ++ scripts/dev/run_e2e_gh.sh | 11 +++ 4 files changed, 128 insertions(+) create mode 100644 .github/workflows/e2e-dispatch.yml create mode 100755 scripts/dev/run_e2e_gh.sh diff --git a/.github/workflows/e2e-dispatch.yml b/.github/workflows/e2e-dispatch.yml new file mode 100644 index 000000000..93a3aaacf --- /dev/null +++ b/.github/workflows/e2e-dispatch.yml @@ -0,0 +1,103 @@ +name: Run Single E2E +on: + workflow_dispatch: + inputs: + distro: + description: 'Distro to run test' + required: true + default: "ubuntu" + test-name: + description: 'Name of test to run' + required: true + cluster-wide: + description: 'Whether or not the test is cluster wide' + required: true + default: "false" + +jobs: + setup: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - pipeline-argument: operator-ubi + - pipeline-argument: version-post-start-hook-init + - pipeline-argument: readiness-probe-init + - pipeline-argument: agent-ubi + - pipeline-argument: agent-ubuntu + - pipeline-argument: e2e + steps: + - name: Checkout Code + uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: '3.8' + + - uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ hashFiles('requirements.txt') }} + + - name: Install Python Dependencies + run: pip install -r requirements.txt + + - name: Login to Quay.io + uses: docker/login-action@v1 + with: + registry: quay.io + username: ${{ secrets.QUAY_USERNAME }} + password: ${{ secrets.QUAY_ROBOT_TOKEN }} + + - name: Build and Push Images + run: python pipeline.py --image-name ${{ matrix.pipeline-argument }} --release false + env: + MONGODB_COMMUNITY_CONFIG: "${{ github.workspace }}/scripts/ci/config.json" + version_id: "${{ github.run_id }}" + + tests: + runs-on: ubuntu-latest + needs: [setup] + + steps: + - name: Checkout Code + uses: actions/checkout@v2 + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: '3.8' + + - name: Cache Dependencies + uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ hashFiles('requirements.txt') }} + + - name: Install Python Dependencies + run: pip install -r requirements.txt + + - name: Setup Kind Cluster + uses: engineerd/setup-kind@v0.5.0 + with: + version: "v0.8.1" + + - name: Install CRD + run: kubectl apply -f config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml + + - name: Run Test + run: python3 ./scripts/dev/e2e.py --test ${{ github.event.inputs.test-name }} --tag ${{ github.run_id }} --config_file ./scripts/ci/config.json --distro ${{ github.event.inputs.distro }} --cluster-wide ${{ github.event.inputs.cluster-wide }} + + - name: Dump Diagnostics + if: always() + continue-on-error: true + run: scripts/ci/dump_diagnostics.sh default # default since kind is running in the default namespace + + - name: Upload Diagnostics + if: always() + uses: actions/upload-artifact@v2 + continue-on-error: true + with: + name: "${{ github.event.inputs.test-name }}-${{ github.event.inputs.distro }}-diagnostics" + path: "${{ github.workspace }}/diagnostics" diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 278466177..30a0e0e08 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -20,6 +20,11 @@ jobs: - pipeline-argument: agent-ubuntu - pipeline-argument: e2e steps: + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.9.0 + with: + access_token: ${{ github.token }} + - name: Checkout Code uses: actions/checkout@v2 - name: Setup Python @@ -125,6 +130,11 @@ jobs: steps: + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.9.0 + with: + access_token: ${{ github.token }} + - name: Checkout Code uses: actions/checkout@v2 diff --git a/Makefile b/Makefile index af82b7fe9..c172a7c27 100644 --- a/Makefile +++ b/Makefile @@ -83,6 +83,10 @@ e2e: install eval $$(scripts/dev/get_e2e_env_vars.py $(cleanup)); \ go test -v -short -timeout=30m -failfast ./test/e2e/$(test) +# Trigger a Github Action of the given test +e2e-gh: + scripts/dev/run_e2e_gh.sh $(test) + # Generate code generate: controller-gen $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." diff --git a/scripts/dev/run_e2e_gh.sh b/scripts/dev/run_e2e_gh.sh new file mode 100755 index 000000000..28daaa3f0 --- /dev/null +++ b/scripts/dev/run_e2e_gh.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -Eeou pipefail + +test_name="${1}" +current_branch="$(git branch --show-current)" + +gh workflow run e2e-dispatch.yml -f "test-name=${test_name}" --ref "${current_branch}" + +run_id="$(gh run list --workflow=e2e-dispatch.yml | grep workflow_dispatch | grep -Eo "[0-9]+" | head -n 1)" + +gh run view "${run_id}" --web From b4e97f72f77c7381bd8f0c77b07af745728b5e09 Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Tue, 15 Jun 2021 11:15:40 +0100 Subject: [PATCH 316/790] Fix script to launch github action (#565) --- scripts/dev/run_e2e_gh.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/dev/run_e2e_gh.sh b/scripts/dev/run_e2e_gh.sh index 28daaa3f0..52297c97e 100755 --- a/scripts/dev/run_e2e_gh.sh +++ b/scripts/dev/run_e2e_gh.sh @@ -6,6 +6,9 @@ current_branch="$(git branch --show-current)" gh workflow run e2e-dispatch.yml -f "test-name=${test_name}" --ref "${current_branch}" -run_id="$(gh run list --workflow=e2e-dispatch.yml | grep workflow_dispatch | grep -Eo "[0-9]+" | head -n 1)" +echo "Waiting for task to start..." +sleep 2 + +run_id="$(gh run list --workflow=e2e-dispatch.yml | grep workflow_dispatch | grep -Eo "[0-9]{9,11}" | head -n 1)" gh run view "${run_id}" --web From 1c12c9f330db0f710f5dd0353da174b9a1642719 Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Tue, 15 Jun 2021 13:36:38 +0100 Subject: [PATCH 317/790] CLOUDP-92134: Fix Flakey Version Changing (#566) --- controllers/replica_set_controller.go | 9 ++++----- controllers/replicaset_controller_test.go | 15 ++++++++++++++ docs/RELEASE_NOTES.md | 20 +++---------------- ....go => replica_set_change_version_test.go} | 0 4 files changed, 22 insertions(+), 22 deletions(-) rename test/e2e/replica_set_change_version/{replica_set_test.go => replica_set_change_version_test.go} (100%) diff --git a/controllers/replica_set_controller.go b/controllers/replica_set_controller.go index 8e24f22da..ee0bfbef7 100644 --- a/controllers/replica_set_controller.go +++ b/controllers/replica_set_controller.go @@ -56,6 +56,7 @@ const ( clusterDNSName = "CLUSTER_DNS_NAME" lastSuccessfulConfiguration = "mongodb.com/v1.lastSuccessfulConfiguration" + lastAppliedMongoDBVersion = "mongodb.com/v1.lastAppliedMongoDBVersion" ) func init() { @@ -232,11 +233,6 @@ func (r ReplicaSetReconciler) Reconcile(ctx context.Context, request reconcile.R r.log.Errorf("Could not update connection string secrets: %s", err) } - // the last version will be duplicated in two annotations. - // This is needed to reuse the update strategy logic in enterprise - if err := annotations.UpdateLastAppliedMongoDBVersion(&mdb, r.client); err != nil { - r.log.Errorf("Could not save current version as an annotation: %s", err) - } if err := r.updateLastSuccessfulConfiguration(mdb); err != nil { r.log.Errorf("Could not save current spec as an annotation: %s", err) } @@ -259,6 +255,9 @@ func (r *ReplicaSetReconciler) updateLastSuccessfulConfiguration(mdb mdbv1.Mongo specAnnotations := map[string]string{ lastSuccessfulConfiguration: string(currentSpec), + // the last version will be duplicated in two annotations. + // This is needed to reuse the update strategy logic in enterprise + lastAppliedMongoDBVersion: mdb.Spec.Version, } return annotations.SetAnnotations(&mdb, specAnnotations, r.client) } diff --git a/controllers/replicaset_controller_test.go b/controllers/replicaset_controller_test.go index 7ca49ed96..5ed2c55f0 100644 --- a/controllers/replicaset_controller_test.go +++ b/controllers/replicaset_controller_test.go @@ -523,7 +523,22 @@ func TestIgnoreUnknownUsers(t *testing.T) { mdb.Spec.Security.Authentication.IgnoreUnknownUsers = &ignoreUnknownUsers assertAuthoritativeSet(t, mdb, true) }) +} + +func TestAnnotationsAreAppliedToResource(t *testing.T) { + mdb := newTestReplicaSet() + + mgr := client.NewManager(&mdb) + r := NewReconciler(mgr) + res, err := r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) + assertReconciliationSuccessful(t, res, err) + + err = mgr.GetClient().Get(context.TODO(), mdb.NamespacedName(), &mdb) + assert.NoError(t, err) + assert.NotNil(t, mdb.Annotations) + assert.NotEmpty(t, mdb.Annotations[lastSuccessfulConfiguration], "last successful spec should have been saved as annotation but was not") + assert.Equal(t, mdb.Annotations[lastAppliedMongoDBVersion], mdb.Spec.Version, "last version should have been saved as an annotation but was not") } // assertAuthoritativeSet asserts that a reconciliation of the given MongoDBCommunity resource diff --git a/docs/RELEASE_NOTES.md b/docs/RELEASE_NOTES.md index 43ad724e2..aa90d9b8b 100644 --- a/docs/RELEASE_NOTES.md +++ b/docs/RELEASE_NOTES.md @@ -1,27 +1,13 @@ -# MongoDB Kubernetes Operator 0.6.1 +# MongoDB Kubernetes Operator 0.6.2 ## Kubernetes Operator - -- Bug fixes - - when deleting MongoDB Resource cleanup related resources (k8s services, secrets). - Changes - - fixed an issue where the operator would reconcile based on events emitted by itself in certain situations. - - support connection strings using SRV. - - expose connection strings (including auth/tls values) for deployments as secrets for easy of use. Secrets name template: _\-\-\_ - -## MongoDB Agent ReadinessProbe - -- Changes - - Readiness probe now patches pod annotations rather than overwriting them. - -## Miscellaneous -Ubuntu-based agent images are now based on Ubuntu 20.04 instead of Ubuntu 16.06 + - stability improvements when changing version of MongoDB. ## Updated Image Tags -- mongodb-kubernetes-operator:0.6.1 -- mongodb-kubernetes-readinessprobe:1.0.4 +- mongodb-kubernetes-operator:0.6.2 _All the images can be found in:_ diff --git a/test/e2e/replica_set_change_version/replica_set_test.go b/test/e2e/replica_set_change_version/replica_set_change_version_test.go similarity index 100% rename from test/e2e/replica_set_change_version/replica_set_test.go rename to test/e2e/replica_set_change_version/replica_set_change_version_test.go From 8f1e30a4510bcdd8f3c655e7568eaef28613fa15 Mon Sep 17 00:00:00 2001 From: Rajdeep Das Date: Tue, 15 Jun 2021 17:15:07 +0200 Subject: [PATCH 318/790] Test increasing number of parallel reconciles (#567) --- controllers/replica_set_controller.go | 2 ++ docs/RELEASE_NOTES.md | 1 + 2 files changed, 3 insertions(+) diff --git a/controllers/replica_set_controller.go b/controllers/replica_set_controller.go index ee0bfbef7..cfbc6f60c 100644 --- a/controllers/replica_set_controller.go +++ b/controllers/replica_set_controller.go @@ -8,6 +8,7 @@ import ( "github.com/mongodb/mongodb-kubernetes-operator/controllers/predicates" "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/source" "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/container" @@ -82,6 +83,7 @@ func NewReconciler(mgr manager.Manager) *ReplicaSetReconciler { // SetupWithManager sets up the controller with the Manager and configures the necessary watches. func (r *ReplicaSetReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). + WithOptions(controller.Options{MaxConcurrentReconciles: 3}). For(&mdbv1.MongoDBCommunity{}, builder.WithPredicates(predicates.OnlyOnSpecChange())). Watches(&source.Kind{Type: &corev1.Secret{}}, r.secretWatcher). Complete(r) diff --git a/docs/RELEASE_NOTES.md b/docs/RELEASE_NOTES.md index aa90d9b8b..513517d5e 100644 --- a/docs/RELEASE_NOTES.md +++ b/docs/RELEASE_NOTES.md @@ -4,6 +4,7 @@ - Changes - stability improvements when changing version of MongoDB. + - increased number of concurrent resources the operator can act on. ## Updated Image Tags From aaed582fcc73f3e3af2e2836b3c09a8eebcda0df Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Tue, 15 Jun 2021 16:51:39 +0100 Subject: [PATCH 319/790] Added job e2e-success (#568) --- .github/workflows/e2e.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 30a0e0e08..8a25a33fb 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -207,3 +207,12 @@ jobs: with: name: "${{ matrix.test-name }}-${{ matrix.distro }}-diagnostics" path: "${{ github.workspace }}/diagnostics" + + e2e-success: + if: always() + needs: [tests] + runs-on: ubuntu-latest + steps: + - name: Check E2E Result + if: needs.tests.result != 'success' + run: exit 1 From 254d071698b6825180ba5fc620944d05020a751b Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Wed, 16 Jun 2021 11:04:56 +0100 Subject: [PATCH 320/790] Configure Env Vars with make run (#569) --- Makefile | 1 + scripts/dev/get_e2e_env_vars.py | 1 + 2 files changed, 2 insertions(+) diff --git a/Makefile b/Makefile index c172a7c27..fee8be3e4 100644 --- a/Makefile +++ b/Makefile @@ -33,6 +33,7 @@ manager: generate fmt vet # Run against the configured Kubernetes cluster in ~/.kube/config run: install $(KUSTOMIZE) build config/local_run | kubectl apply -n $(NAMESPACE) -f - + eval $$(scripts/dev/get_e2e_env_vars.py $(cleanup)); \ go run ./cmd/manager/main.go # Install CRDs into a cluster diff --git a/scripts/dev/get_e2e_env_vars.py b/scripts/dev/get_e2e_env_vars.py index ca2db6588..3dbd55c34 100755 --- a/scripts/dev/get_e2e_env_vars.py +++ b/scripts/dev/get_e2e_env_vars.py @@ -26,6 +26,7 @@ def _get_e2e_test_envs(dev_config: DevConfig) -> Dict[str, str]: "TEST_NAMESPACE": dev_config.namespace, "READINESS_PROBE_IMAGE": f"{dev_config.repo_url}/{dev_config.readiness_probe_image}", "PERFORM_CLEANUP": "true" if cleanup else "false", + "WATCH_NAMESPACE": dev_config.namespace, } From f646ed19e778e811e21c880af7fb0953f5ed1518 Mon Sep 17 00:00:00 2001 From: John Williams <55147273+jwilliams-mongo@users.noreply.github.com> Date: Wed, 16 Jun 2021 09:24:58 -0500 Subject: [PATCH 321/790] (DOCSP-16408): Expose connection string for deployments for easy of use (#559) --- docs/deploy-configure.md | 54 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 50 insertions(+), 4 deletions(-) diff --git a/docs/deploy-configure.md b/docs/deploy-configure.md index a15f22919..427e72619 100644 --- a/docs/deploy-configure.md +++ b/docs/deploy-configure.md @@ -24,12 +24,58 @@ To deploy your first replica set: ``` kubectl get mongodbcommunity --namespace ``` -4. Connect clients to the MongoDB replica set: + +4. The Community Kubernetes Operator creates secrets that contains users' connection strings and credentials. + + The secrets follow this naming convention: `--`, where: + + | Variable | Description | Value in Sample | + |----|----|----| + | `` | Name of the MongoDB database resource. | `example-mongodb` | + | `` | [Authentication database](https://docs.mongodb.com/manual/core/security-users/#std-label-user-authentication-database) where you defined the database user. | `admin` | + | `` | Username of the database user. | `my-user` | + + Update the variables in the following command, then run it to retrieve a user's connection strings to the replica set from the secret: + + **NOTE**: The following command requires [jq](https://stedolan.github.io/jq/) version 1.6 or higher.

+ + ```sh + kubectl get secret -- -n mongodb -o json | \ + jq -r '.data | with_entries(.value |= @base64d)' + ``` + + The command returns the replica set's standard and DNS seed list [connection strings](https://docs.mongodb.com/manual/reference/connection-string/#connection-string-formats) in addition to the user's name and password: + + ```json + { + "connectionString.standard": "mongodb://:@example-mongodb-0.example-mongodb-svc.mongodb.svc.cluster.local:27017,example-mongodb-1.example-mongodb-svc.mongodb.svc.cluster.local:27017,example-mongodb-2.example-mongodb-svc.mongodb.svc.cluster.local:27017/admin?ssl=true", + "connectionString.standardSrv": "mongodb+srv://:@example-mongodb-svc.mongodb.svc.cluster.local/admin?ssl=true", + "password": "", + "username": "" + } + ``` + + **NOTE**: The Community Kubernetes Operator sets the [`ssl` connection option](https://docs.mongodb.com/manual/reference/connection-string/#connection-options) to `true` if you [Secure MongoDB Resource Connections using TLS](secure.md#secure-mongodb-resource-connections-using-tls).

+ + You can use the connection strings in this secret in your application: + + ```yaml + containers: + - name: test-app + env: + - name: "CONNECTION_STRING" + valueFrom: + secretKeyRef: + name: -- + key: connectionString.standardSrv + +5. Use one of the connection strings returned in the previous step to connect to the replica set: ``` - mongo "mongodb://..svc.cluster.local:27017/?replicaSet=" + mongo "mongodb+srv://:@example-mongodb-svc.mongodb.svc.cluster.local/admin?ssl=true" ``` -**NOTE**: You can access each `mongod` process in the replica set only from within a pod -running in the cluster. + + **NOTE**: You can access each `mongod` process in the replica set only from within a pod running in the cluster. + ## Scale a Replica Set From 5dc63f9c80a9ca09e829a61c0eed9e1f4ea6950a Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Mon, 21 Jun 2021 15:05:33 +0100 Subject: [PATCH 322/790] added action to comment on release pr (#577) --- .github/workflows/create-release-pr.yml | 3 +++ .github/workflows/release-images.yml | 22 ++++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/.github/workflows/create-release-pr.yml b/.github/workflows/create-release-pr.yml index f3ba2d564..05d35a8cd 100644 --- a/.github/workflows/create-release-pr.yml +++ b/.github/workflows/create-release-pr.yml @@ -31,3 +31,6 @@ jobs: # include date in the pr branch name to ensure duplicate branches to not get created. PR_BRANCH_NAME: 'release-v${{steps.release_version.outputs.OUTPUT}}-${{ steps.release_version.outputs.DATE }}' PR_TITLE: 'Release MongoDB Kubernetes Operator v${{ steps.release_version.outputs.OUTPUT }}' + + # Note: sonar github is a repo scoped token, which allows this action to trigger other actions. + GITHUB_TOKEN: ${{ secrets.SONAR_GITHUB_TOKEN }} diff --git a/.github/workflows/release-images.yml b/.github/workflows/release-images.yml index 5b60881a9..c58be708e 100644 --- a/.github/workflows/release-images.yml +++ b/.github/workflows/release-images.yml @@ -90,3 +90,25 @@ jobs: bodyFile: "${{ github.workspace }}/docs/RELEASE_NOTES.md" draft: true token: ${{ secrets.GITHUB_TOKEN }} + + comment: + runs-on: ubuntu-latest + needs: [merge-release-pr-and-draft-github-release] + steps: + - name: Checkout Code + uses: actions/checkout@v2 + - name: Determine Release Version + id: release_version + run: | + VERSION=$(jq -r '."mongodb-kubernetes-operator"' < $GITHUB_WORKSPACE/release.json) + echo "::set-output name=version::${VERSION}" + - uses: actions/github-script@v3 + with: + github-token: ${{secrets.GITHUB_TOKEN}} + script: | + github.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: 'Review and publish the release here: https://github.com/mongodb/mongodb-kubernetes-operator/releases/tag/v${{ steps.release_version.outputs.version }}' + }) From 9ab2dc23ce9bcd6d2fad118806b7bdca6ccd0543 Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Tue, 22 Jun 2021 09:34:40 +0100 Subject: [PATCH 323/790] CLOUDP-92631: mongodb log to stdout (#584) --- docs/RELEASE_NOTES.md | 1 + pkg/automationconfig/automation_config_builder.go | 6 ------ .../automation_config_secret_test.go | 12 ++++++------ 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/docs/RELEASE_NOTES.md b/docs/RELEASE_NOTES.md index 513517d5e..60ef8d89a 100644 --- a/docs/RELEASE_NOTES.md +++ b/docs/RELEASE_NOTES.md @@ -5,6 +5,7 @@ - Changes - stability improvements when changing version of MongoDB. - increased number of concurrent resources the operator can act on. + - mongodb will now send its log to stdout by default. ## Updated Image Tags diff --git a/pkg/automationconfig/automation_config_builder.go b/pkg/automationconfig/automation_config_builder.go index efe821d97..51059cdee 100644 --- a/pkg/automationconfig/automation_config_builder.go +++ b/pkg/automationconfig/automation_config_builder.go @@ -2,7 +2,6 @@ package automationconfig import ( "fmt" - "path" "reflect" "strings" @@ -214,11 +213,6 @@ func (b *Builder) Build() (AutomationConfig, error) { Version: b.mongodbVersion, AuthSchemaVersion: 5, } - process.SetSystemLog(SystemLog{ - Destination: "file", - Path: path.Join(DefaultAgentLogPath, "/mongodb.log"), - LogAppend: true, - }) if b.fcv != "" { process.FeatureCompatibilityVersion = b.fcv diff --git a/pkg/automationconfig/automation_config_secret_test.go b/pkg/automationconfig/automation_config_secret_test.go index 0cc5484c1..0450fc954 100644 --- a/pkg/automationconfig/automation_config_secret_test.go +++ b/pkg/automationconfig/automation_config_secret_test.go @@ -25,7 +25,7 @@ func TestEnsureSecret(t *testing.T) { SetNamespace(secretNsName.Namespace). Build() - secretGetUpdateCreator := mockSecretGetUpdateCreator{secret: &s} + secretGetUpdateCreator := &mockSecretGetUpdateCreator{secret: &s} ac, err := EnsureSecret(secretGetUpdateCreator, secretNsName, []metav1.OwnerReference{}, desiredAutomationConfig) assert.NoError(t, err) @@ -45,7 +45,7 @@ func TestEnsureSecret(t *testing.T) { existingSecret, err := newAutomationConfigSecret(oldAc, secretNsName) assert.NoError(t, err) - secretGetUpdateCreator := mockSecretGetUpdateCreator{secret: &existingSecret} + secretGetUpdateCreator := &mockSecretGetUpdateCreator{secret: &existingSecret} newAc, err := newAutomationConfigBuilder().SetDomain("different-domain").Build() assert.NoError(t, err) @@ -83,7 +83,7 @@ type mockSecretGetUpdateCreator struct { secret *corev1.Secret } -func (m mockSecretGetUpdateCreator) GetSecret(objectKey client.ObjectKey) (corev1.Secret, error) { +func (m *mockSecretGetUpdateCreator) GetSecret(objectKey client.ObjectKey) (corev1.Secret, error) { if m.secret != nil { if objectKey.Name == m.secret.Name && objectKey.Namespace == m.secret.Namespace { return *m.secret, nil @@ -92,13 +92,13 @@ func (m mockSecretGetUpdateCreator) GetSecret(objectKey client.ObjectKey) (corev return corev1.Secret{}, notFoundError() } -func (m mockSecretGetUpdateCreator) UpdateSecret(secret corev1.Secret) error { +func (m *mockSecretGetUpdateCreator) UpdateSecret(secret corev1.Secret) error { m.secret = &secret return nil } -func (m mockSecretGetUpdateCreator) CreateSecret(secret corev1.Secret) error { - if m.secret != nil { +func (m *mockSecretGetUpdateCreator) CreateSecret(secret corev1.Secret) error { + if m.secret == nil { m.secret = &secret return nil } From dffc4f68e75d76b867e265391cb9d8c1351e8a10 Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Tue, 22 Jun 2021 13:27:06 +0100 Subject: [PATCH 324/790] Changed repo url and image name (#587) --- config/manager/manager.yaml | 4 ++-- deploy/openshift/operator_openshift.yaml | 4 ++-- docs/RELEASE_NOTES.md | 1 + docs/install-upgrade.md | 6 +++--- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index e04ffe83b..e49335f0f 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -33,9 +33,9 @@ spec: - name: READINESS_PROBE_IMAGE value: quay.io/mongodb/mongodb-kubernetes-readinessprobe:1.0.4 - name: MONGODB_IMAGE - value: library/mongo + value: mongo - name: MONGODB_REPO_URL - value: registry.hub.docker.com + value: docker.io image: quay.io/mongodb/mongodb-kubernetes-operator:0.6.1 imagePullPolicy: Always name: mongodb-kubernetes-operator diff --git a/deploy/openshift/operator_openshift.yaml b/deploy/openshift/operator_openshift.yaml index 66d5d7fdd..fd75b46aa 100644 --- a/deploy/openshift/operator_openshift.yaml +++ b/deploy/openshift/operator_openshift.yaml @@ -35,9 +35,9 @@ spec: - name: VERSION_UPGRADE_HOOK_IMAGE value: quay.io/mongodb/mongodb-kubernetes-operator-version-upgrade-post-start-hook:1.0.2 - name: MONGODB_IMAGE - value: library/mongo + value: mongo - name: MONGODB_REPO_URL - value: registry.hub.docker.com + value: docker.io image: quay.io/mongodb/mongodb-kubernetes-operator:0.6.1 imagePullPolicy: Always name: mongodb-kubernetes-operator diff --git a/docs/RELEASE_NOTES.md b/docs/RELEASE_NOTES.md index 60ef8d89a..cc29916a9 100644 --- a/docs/RELEASE_NOTES.md +++ b/docs/RELEASE_NOTES.md @@ -6,6 +6,7 @@ - stability improvements when changing version of MongoDB. - increased number of concurrent resources the operator can act on. - mongodb will now send its log to stdout by default. + - changed the default values for `MONGODB_REPO_URL` and `MONGODB_IMAGE` in the operator deployment ## Updated Image Tags diff --git a/docs/install-upgrade.md b/docs/install-upgrade.md index 805124851..e21909035 100644 --- a/docs/install-upgrade.md +++ b/docs/install-upgrade.md @@ -87,8 +87,8 @@ for MongoDB Docker images: | Environment Variable | Description | Default | |----|----|----| - | `MONGODB_IMAGE` | From the `MONGODB_REPO_URL`, absolute path to the MongoDB Docker image that you want to deploy. | `"library/mongo"` | - | `MONGODB_REPO_URL` | URL of the container registry that contains the MongoDB Docker image that you want to deploy. | `"registry.hub.docker.com"` | + | `MONGODB_IMAGE` | From the `MONGODB_REPO_URL`, absolute path to the MongoDB Docker image that you want to deploy. | `"mongo"` | + | `MONGODB_REPO_URL` | URL of the container registry that contains the MongoDB Docker image that you want to deploy. | `"docker.io"` | ```yaml spec: @@ -198,4 +198,4 @@ Make sure you run commands in the correct namespace. ``` kubectl delete pod -0 ``` - d. You're done. Now Kubernetes will create the pod fresh, causing the migration to run and then the pod to start up. Then kubernetes will proceed creating the next pod until it reaches the number specified in your cr. \ No newline at end of file + d. You're done. Now Kubernetes will create the pod fresh, causing the migration to run and then the pod to start up. Then kubernetes will proceed creating the next pod until it reaches the number specified in your cr. From 2ce509d520d98b5fa59fae71616fcb512b7f4af1 Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Tue, 22 Jun 2021 13:36:08 +0100 Subject: [PATCH 325/790] bump release.json (#588) --- release.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release.json b/release.json index 33d47baa7..b30731b28 100644 --- a/release.json +++ b/release.json @@ -1,5 +1,5 @@ { - "mongodb-kubernetes-operator": "0.6.1", + "mongodb-kubernetes-operator": "0.6.2", "version-upgrade-hook": "1.0.2", "readiness-probe": "1.0.4", "mongodb-agent": { From 8616b647a874ea88034870e7c631482c38d82e8f Mon Sep 17 00:00:00 2001 From: Rajdeep Das Date: Tue, 22 Jun 2021 15:44:48 +0200 Subject: [PATCH 326/790] Create Namespace for e2e if it doesn't exist (#586) --- scripts/dev/e2e.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/dev/e2e.py b/scripts/dev/e2e.py index 80b1e2b0b..8917a05d1 100644 --- a/scripts/dev/e2e.py +++ b/scripts/dev/e2e.py @@ -39,6 +39,12 @@ def _prepare_test_environment(config_file: str) -> None: corev1 = client.CoreV1Api() dev_config = load_config(config_file) + print("Creating Namespace") + k8s_conditions.ignore_if_already_exists( + lambda: corev1.create_namespace( + client.V1Namespace(metadata=dict(name=dev_config.namespace)) + ) + ) _delete_test_pod(config_file) print("Creating Role") From e8c4d350c6738dcc6452234707aa2f498b0ee339 Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Tue, 22 Jun 2021 14:57:21 +0100 Subject: [PATCH 327/790] Release MongoDB Kubernetes Operator v0.6.2 (#589) Co-authored-by: GitHub Actions --- config/manager/manager.yaml | 2 +- deploy/openshift/operator_openshift.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index e49335f0f..3dbdc9ad3 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -36,7 +36,7 @@ spec: value: mongo - name: MONGODB_REPO_URL value: docker.io - image: quay.io/mongodb/mongodb-kubernetes-operator:0.6.1 + image: quay.io/mongodb/mongodb-kubernetes-operator:0.6.2 imagePullPolicy: Always name: mongodb-kubernetes-operator serviceAccountName: mongodb-kubernetes-operator diff --git a/deploy/openshift/operator_openshift.yaml b/deploy/openshift/operator_openshift.yaml index fd75b46aa..0f5bf582e 100644 --- a/deploy/openshift/operator_openshift.yaml +++ b/deploy/openshift/operator_openshift.yaml @@ -38,7 +38,7 @@ spec: value: mongo - name: MONGODB_REPO_URL value: docker.io - image: quay.io/mongodb/mongodb-kubernetes-operator:0.6.1 + image: quay.io/mongodb/mongodb-kubernetes-operator:0.6.2 imagePullPolicy: Always name: mongodb-kubernetes-operator serviceAccountName: mongodb-kubernetes-operator From dbfd2dc4ab7e4a61b2b9b9065ea74b34cb417d81 Mon Sep 17 00:00:00 2001 From: killian2k Date: Wed, 23 Jun 2021 08:20:40 +0200 Subject: [PATCH 328/790] Cloudp 78267 add arbiters to replicasets (#580) * First version, a few unit tests pass * Adding e2e tests (not working yet) * Add the e2e tests to the arbiter feature * Modification related to comments on PR * Modifying the code based on the PR review * modified the release note and added the tests to the e2e.yml * Error fixed * Update api/v1/mongodbcommunity_types.go Co-authored-by: Nikolas De Giorgis * Update controllers/validation/validation.go Co-authored-by: Nikolas De Giorgis Co-authored-by: Nikolas De Giorgis --- .github/workflows/e2e.yml | 4 + api/v1/mongodbcommunity_types.go | 4 + ...ommunity.mongodb.com_mongodbcommunity.yaml | 4 + controllers/replica_set_controller.go | 1 + controllers/validation/validation.go | 20 ++++ docs/RELEASE_NOTES.md | 3 +- pkg/automationconfig/automation_config.go | 8 +- .../automation_config_builder.go | 13 ++- .../automation_config_test.go | 103 ++++++++++++++++++ test/e2e/e2eutil.go | 7 +- test/e2e/mongodbtests/mongodbtests.go | 28 +++++ .../replica_set_arbiter_test.go | 69 ++++++++++++ test/e2e/util/wait/wait.go | 8 ++ 13 files changed, 263 insertions(+), 9 deletions(-) create mode 100644 test/e2e/replica_set_arbiter/replica_set_arbiter_test.go diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 8a25a33fb..ee0b23095 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -93,6 +93,8 @@ jobs: cluster-wide: true - test-name: replica_set_custom_role distro: ubuntu + - test-name: replica_set_arbiter + distro: ubuntu - test-name: replica_set distro: ubi @@ -127,6 +129,8 @@ jobs: cluster-wide: true - test-name: replica_set_custom_role distro: ubi + - test-name: replica_set_arbiter + distro: ubi steps: diff --git a/api/v1/mongodbcommunity_types.go b/api/v1/mongodbcommunity_types.go index 8131e9a1c..8c9250115 100644 --- a/api/v1/mongodbcommunity_types.go +++ b/api/v1/mongodbcommunity_types.go @@ -51,6 +51,10 @@ type MongoDBCommunitySpec struct { // Version defines which version of MongoDB will be used Version string `json:"version"` + // Arbiters is the number of arbiters (each counted as a member) in the replica set + // +optional + Arbiters int `json:"arbiters"` + // FeatureCompatibilityVersion configures the feature compatibility version that will // be set for the deployment // +optional diff --git a/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml b/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml index 31d86b059..e1abda907 100644 --- a/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml +++ b/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml @@ -53,6 +53,10 @@ spec: structure as the mongod configuration file: https://docs.mongodb.com/manual/reference/configuration-options/' nullable: true type: object + arbiters: + description: Arbiters is the number arbiters (each counted as a member) + in the replica set + type: integer featureCompatibilityVersion: description: FeatureCompatibilityVersion configures the feature compatibility version that will be set for the deployment diff --git a/controllers/replica_set_controller.go b/controllers/replica_set_controller.go index cfbc6f60c..0998babd1 100644 --- a/controllers/replica_set_controller.go +++ b/controllers/replica_set_controller.go @@ -441,6 +441,7 @@ func buildAutomationConfig(mdb mdbv1.MongoDBCommunity, auth automationconfig.Aut SetName(mdb.Name). SetDomain(domain). SetMembers(mdb.AutomationConfigMembersThisReconciliation()). + SetArbiters(mdb.Spec.Arbiters). SetReplicaSetHorizons(mdb.Spec.ReplicaSetHorizons). SetPreviousAutomationConfig(currentAc). SetMongoDBVersion(mdb.Spec.Version). diff --git a/controllers/validation/validation.go b/controllers/validation/validation.go index 7c68d2b60..7be3707aa 100644 --- a/controllers/validation/validation.go +++ b/controllers/validation/validation.go @@ -15,6 +15,10 @@ func ValidateInitalSpec(mdb mdbv1.MongoDBCommunity) error { return err } + if err := validateArbiterSpec(mdb); err != nil { + return err + } + return nil } @@ -28,6 +32,10 @@ func ValidateUpdate(mdb mdbv1.MongoDBCommunity, oldSpec mdbv1.MongoDBCommunitySp return err } + if err := validateArbiterSpec(mdb); err != nil { + return err + } + return nil } @@ -56,3 +64,15 @@ func validateUsers(mdb mdbv1.MongoDBCommunity) error { return nil } + +// validateArbiterSpec checks if the initial Member spec is valid. +func validateArbiterSpec(mdb mdbv1.MongoDBCommunity) error { + if mdb.Spec.Arbiters < 0 { + return fmt.Errorf("number of arbiters must be greater or equal than 0") + } + if mdb.Spec.Arbiters >= mdb.Spec.Members { + return fmt.Errorf("number of arbiters specified (%v) is greater or equal than the number of members in the replicaset (%v). At least one member must not be an arbiter", mdb.Spec.Arbiters, mdb.Spec.Members) + } + + return nil +} diff --git a/docs/RELEASE_NOTES.md b/docs/RELEASE_NOTES.md index cc29916a9..f34d1360b 100644 --- a/docs/RELEASE_NOTES.md +++ b/docs/RELEASE_NOTES.md @@ -1,12 +1,13 @@ # MongoDB Kubernetes Operator 0.6.2 ## Kubernetes Operator - + - Changes - stability improvements when changing version of MongoDB. - increased number of concurrent resources the operator can act on. - mongodb will now send its log to stdout by default. - changed the default values for `MONGODB_REPO_URL` and `MONGODB_IMAGE` in the operator deployment + - added the support of arbiters for the replica sets. ## Updated Image Tags diff --git a/pkg/automationconfig/automation_config.go b/pkg/automationconfig/automation_config.go index 311d96f51..b0c3ba79b 100644 --- a/pkg/automationconfig/automation_config.go +++ b/pkg/automationconfig/automation_config.go @@ -121,6 +121,7 @@ type ReplicaSet struct { Id string `json:"_id"` Members []ReplicaSetMember `json:"members"` ProtocolVersion string `json:"protocolVersion"` + NumberArbiters int `json:"numberArbiters"` } type ReplicaSetMember struct { @@ -134,11 +135,14 @@ type ReplicaSetMember struct { type ReplicaSetHorizons map[string]string -func newReplicaSetMember(p Process, id int, horizons ReplicaSetHorizons, totalVotesSoFar int) ReplicaSetMember { +func newReplicaSetMember(p Process, id int, horizons ReplicaSetHorizons, totalVotesSoFar int, numberArbiters int) ReplicaSetMember { // ensure that the number of voting members in the replica set is not more than 7 // as this is the maximum number of voting members. votes := 1 priority := 1 + + isArbiter := totalVotesSoFar < numberArbiters + if totalVotesSoFar > maxVotingMembers { votes = 0 priority = 0 @@ -148,7 +152,7 @@ func newReplicaSetMember(p Process, id int, horizons ReplicaSetHorizons, totalVo Id: id, Host: p.Name, Priority: priority, - ArbiterOnly: false, + ArbiterOnly: isArbiter, Votes: votes, Horizons: horizons, } diff --git a/pkg/automationconfig/automation_config_builder.go b/pkg/automationconfig/automation_config_builder.go index 51059cdee..6bd466fcd 100644 --- a/pkg/automationconfig/automation_config_builder.go +++ b/pkg/automationconfig/automation_config_builder.go @@ -29,6 +29,7 @@ type Builder struct { replicaSets []ReplicaSet replicaSetHorizons []ReplicaSetHorizons members int + arbiters int domain string name string fcv string @@ -92,6 +93,11 @@ func (b *Builder) SetMembers(members int) *Builder { return b } +func (b *Builder) SetArbiters(arbiters int) *Builder { + b.arbiters = arbiters + return b +} + func (b *Builder) SetDomain(domain string) *Builder { b.domain = domain return b @@ -203,6 +209,7 @@ func (b *Builder) Build() (AutomationConfig, error) { if err := b.setFeatureCompatibilityVersionIfUpgradeIsHappening(); err != nil { return AutomationConfig{}, errors.Errorf("can't build the automation config: %s", err) } + totalVotes := 0 for i, h := range hostnames { process := &Process{ @@ -228,13 +235,13 @@ func (b *Builder) Build() (AutomationConfig, error) { processes[i] = *process - totalVotes := 0 if b.replicaSetHorizons != nil { - members[i] = newReplicaSetMember(*process, i, b.replicaSetHorizons[i], totalVotes) + members[i] = newReplicaSetMember(*process, i, b.replicaSetHorizons[i], totalVotes, b.arbiters) } else { - members[i] = newReplicaSetMember(*process, i, nil, totalVotes) + members[i] = newReplicaSetMember(*process, i, nil, totalVotes, b.arbiters) } totalVotes += members[i].Votes + } if b.auth == nil { diff --git a/pkg/automationconfig/automation_config_test.go b/pkg/automationconfig/automation_config_test.go index 43a444409..85bc013fc 100644 --- a/pkg/automationconfig/automation_config_test.go +++ b/pkg/automationconfig/automation_config_test.go @@ -62,6 +62,109 @@ func TestBuildAutomationConfig(t *testing.T) { } } +func TestBuildAutomationConfigArbiters(t *testing.T) { + // Test no arbiter (field specified) + noArbiters := 0 + ac, err := NewBuilder(). + SetMembers(4). + SetArbiters(noArbiters). + Build() + + assert.NoError(t, err) + + rs := ac.ReplicaSets[0] + for _, member := range rs.Members { + assert.False(t, member.ArbiterOnly, "No member should be an arbiter") + } + + // Test no arbiter (field NOT specified) + ac, err = NewBuilder(). + SetMembers(4). + Build() + + assert.NoError(t, err) + + rs = ac.ReplicaSets[0] + for _, member := range rs.Members { + assert.False(t, member.ArbiterOnly, "No member should be an arbiter") + } + + // Test only one arbiter + noArbiters = 1 + ac, err = NewBuilder(). + SetMembers(4). + SetArbiters(noArbiters). + Build() + + assert.NoError(t, err) + + rs = ac.ReplicaSets[0] + for i, member := range rs.Members { + if i < noArbiters { + assert.True(t, member.ArbiterOnly, "The first member should be an arbiter") + } else { + assert.False(t, member.ArbiterOnly, "These members should not be arbiters") + } + } + + // Test with multiple arbiters + noArbiters = 2 + ac, err = NewBuilder(). + SetMembers(4). + SetArbiters(noArbiters). + Build() + + assert.NoError(t, err) + + rs = ac.ReplicaSets[0] + for i, member := range rs.Members { + if i < noArbiters { + assert.True(t, member.ArbiterOnly, "The first two members should be arbiters") + } else { + assert.False(t, member.ArbiterOnly, "These members should not be arbiters") + } + } + + //Test With only Arbiters + // The error will be generated when the reconcile loop is called (tested in the e2e tests). + noArbiters = 4 + ac, err = NewBuilder(). + SetMembers(noArbiters). + SetArbiters(noArbiters). + Build() + + assert.NoError(t, err) + + rs = ac.ReplicaSets[0] + for i, member := range rs.Members { + if i < noArbiters { + assert.True(t, member.ArbiterOnly, "The first two members should be arbiters") + } else { + assert.False(t, member.ArbiterOnly, "These members should not be arbiters") + } + } + + //Test With more Arbiters than members + // The error will be generated when the reconcile loop is called (tested in the e2e tests). + noMembers := 3 + noArbiters = noMembers + 1 + ac, err = NewBuilder(). + SetMembers(noMembers). + SetArbiters(noArbiters). + Build() + + assert.NoError(t, err) + + rs = ac.ReplicaSets[0] + for i, member := range rs.Members { + if i < noArbiters { + assert.True(t, member.ArbiterOnly, "The first two members should be arbiters") + } else { + assert.False(t, member.ArbiterOnly, "These members should not be arbiters") + } + } +} + func TestReplicaSetHorizons(t *testing.T) { ac, err := NewBuilder(). SetName("my-rs"). diff --git a/test/e2e/e2eutil.go b/test/e2e/e2eutil.go index c9fc1d0c3..dcac0399b 100644 --- a/test/e2e/e2eutil.go +++ b/test/e2e/e2eutil.go @@ -53,9 +53,10 @@ func NewTestMongoDB(ctx *Context, name string, namespace string) (mdbv1.MongoDBC Namespace: mongodbNamespace, }, Spec: mdbv1.MongoDBCommunitySpec{ - Members: 3, - Type: "ReplicaSet", - Version: "4.4.0", + Members: 3, + Type: "ReplicaSet", + Version: "4.4.0", + Arbiters: 0, Security: mdbv1.Security{ Authentication: mdbv1.Authentication{ Modes: []mdbv1.AuthMode{"SCRAM"}, diff --git a/test/e2e/mongodbtests/mongodbtests.go b/test/e2e/mongodbtests/mongodbtests.go index 399c08d68..953c619f9 100644 --- a/test/e2e/mongodbtests/mongodbtests.go +++ b/test/e2e/mongodbtests/mongodbtests.go @@ -208,6 +208,23 @@ func AutomationConfigVersionHasTheExpectedVersion(mdb *mdbv1.MongoDBCommunity, e } } +// AutomationConfigVersionHasTheExpectedVersion verifies that the automation config has the expected version. +func AutomationConfigReplicaSetsHaveExpectedArbiters(mdb *mdbv1.MongoDBCommunity, expectedArbiters int) func(t *testing.T) { + return func(t *testing.T) { + currentAc := getAutomationConfig(t, mdb) + lsRs := currentAc.ReplicaSets + for _, rs := range lsRs { + arbiters := 0 + for _, rsMember := range rs.Members { + if rsMember.ArbiterOnly { + arbiters += 1 + } + } + assert.Equal(t, expectedArbiters, arbiters) + } + } +} + // AutomationConfigHasTheExpectedCustomRoles verifies that the automation config has the expected custom roles. func AutomationConfigHasTheExpectedCustomRoles(mdb *mdbv1.MongoDBCommunity, roles []automationconfig.CustomRole) func(t *testing.T) { return func(t *testing.T) { @@ -385,6 +402,17 @@ func ExecInContainer(mdb *mdbv1.MongoDBCommunity, podNum int, containerName, com } } +// StatefulSetMessageIsReceived waits (up to 5 minutes) to get desiredMessageStatus as a mongodb message status or returns a fatal error. +func StatefulSetMessageIsReceived(mdb *mdbv1.MongoDBCommunity, ctx *e2eutil.Context, desiredMessageStatus string) func(t *testing.T) { + return func(t *testing.T) { + err := wait.ForMongoDBMessageStatus(t, mdb, time.Second*15, time.Minute*5, desiredMessageStatus) + if err != nil { + t.Fatal(err) + } + + } +} + func podFromMongoDBCommunity(mdb *mdbv1.MongoDBCommunity, podNum int) corev1.Pod { return corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ diff --git a/test/e2e/replica_set_arbiter/replica_set_arbiter_test.go b/test/e2e/replica_set_arbiter/replica_set_arbiter_test.go new file mode 100644 index 000000000..04a357a16 --- /dev/null +++ b/test/e2e/replica_set_arbiter/replica_set_arbiter_test.go @@ -0,0 +1,69 @@ +package replica_set + +import ( + "fmt" + "os" + "testing" + "time" + + e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" + "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/mongodbtests" + setup "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" + "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/util/wait" +) + +func TestMain(m *testing.M) { + code, err := e2eutil.RunTest(m) + if err != nil { + fmt.Println(err) + } + os.Exit(code) +} + +func TestReplicaSetArbiter(t *testing.T) { + ctx := setup.Setup(t) + defer ctx.Teardown() + + // Invalid case 1 + numberArbiters := 3 + numberMembers := 3 + desiredStatus := fmt.Sprintf("error validating new Spec: number of arbiters specified (%v) is greater or equal than the number of members in the replicaset (%v). At least one member must not be an arbiter", numberArbiters, numberMembers) + mdb, user := e2eutil.NewTestMongoDB(ctx, "mdb0", "") + mdb.Spec.Arbiters = numberArbiters + mdb.Spec.Members = numberMembers + _, err := setup.GeneratePasswordForUser(ctx, user, "") + if err != nil { + t.Fatal(err) + } + t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) + t.Run("Check status for case 1", mongodbtests.StatefulSetMessageIsReceived(&mdb, ctx, desiredStatus)) + + // Invalid case 2 + numberArbiters = -1 + numberMembers = 3 + desiredStatus = "error validating new Spec: number of arbiters must be greater or equal than 0" + mdb, user = e2eutil.NewTestMongoDB(ctx, "mdb1", "") + mdb.Spec.Arbiters = numberArbiters + mdb.Spec.Members = numberMembers + _, err = setup.GeneratePasswordForUser(ctx, user, "") + if err != nil { + t.Fatal(err) + } + t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) + t.Run("Check status for case 2", mongodbtests.StatefulSetMessageIsReceived(&mdb, ctx, desiredStatus)) + + // Valid case 1 + numberArbiters = 1 + numberMembers = 3 + mdb, user = e2eutil.NewTestMongoDB(ctx, "mdb2", "") + mdb.Spec.Arbiters = numberArbiters + mdb.Spec.Members = numberMembers + _, err = setup.GeneratePasswordForUser(ctx, user, "") + if err != nil { + t.Fatal(err) + } + t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) + t.Run("Check that the stateful set becomes ready", mongodbtests.StatefulSetBecomesReady(&mdb, wait.Timeout(20*time.Minute))) + t.Run("Check the number of arbiters", mongodbtests.AutomationConfigReplicaSetsHaveExpectedArbiters(&mdb, numberArbiters)) + +} diff --git a/test/e2e/util/wait/wait.go b/test/e2e/util/wait/wait.go index 907a3925b..95d70ce1e 100644 --- a/test/e2e/util/wait/wait.go +++ b/test/e2e/util/wait/wait.go @@ -38,6 +38,14 @@ func ForMongoDBToReachPhase(t *testing.T, mdb *mdbv1.MongoDBCommunity, phase mdb }) } +// ForMongoDBMessageStatus waits until the given MongoDB resource gets the expected message status +func ForMongoDBMessageStatus(t *testing.T, mdb *mdbv1.MongoDBCommunity, retryInterval, timeout time.Duration, message string) error { + return waitForMongoDBCondition(mdb, retryInterval, timeout, func(db mdbv1.MongoDBCommunity) bool { + t.Logf("current message: %s, waiting for message: %s", db.Status.Message, message) + return db.Status.Message == message + }) +} + // waitForMongoDBCondition polls and waits for a given condition to be true func waitForMongoDBCondition(mdb *mdbv1.MongoDBCommunity, retryInterval, timeout time.Duration, condition func(mdbv1.MongoDBCommunity) bool) error { mdbNew := mdbv1.MongoDBCommunity{} From 6615ac7cda7c127e3b671b03b8b0f12b317d2c32 Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Thu, 24 Jun 2021 13:04:44 +0100 Subject: [PATCH 329/790] Updated e2e.yml to include workflow_dispatch (#592) --- .github/workflows/e2e.yml | 1 + ...community.mongodb.com_mongodbcommunity.yaml | 2 +- docs/contributing.md | 18 ++++++++++++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index ee0b23095..fbf8119f7 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -6,6 +6,7 @@ on: push: branches: - master + workflow_dispatch: jobs: setup: runs-on: ubuntu-latest diff --git a/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml b/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml index e1abda907..12695642a 100644 --- a/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml +++ b/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml @@ -54,7 +54,7 @@ spec: nullable: true type: object arbiters: - description: Arbiters is the number arbiters (each counted as a member) + description: Arbiters is the number of arbiters (each counted as a member) in the replica set type: integer featureCompatibilityVersion: diff --git a/docs/contributing.md b/docs/contributing.md index fb1fbbad5..5ab97d9f5 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -202,6 +202,8 @@ This will run the `replica_set` E2E test which is a simple test which installs a MongoDB Replica Set and asserts that the deployed server can be connected to. + + ### Run the test locally with go test & Telepresence ```sh make e2e-telepresence test= @@ -214,6 +216,22 @@ Note: you must install [telepresence](https://www.getambassador.io/docs/telepres If on MacOS, you can run `make install-prerequisites-macos` which will perform the installation. +### Running with Github Actions + +Run a single test + +```sh +make e2e-gh test= +``` + +Run all tests. + +* Navigate to the Actions tab on the github repo +* `Run E2E` > `Run Workflow` > `Your Branch` + +Note: the code must be pushed to a remote branch before this will work. + + ## Troubleshooting When you run a test locally, if the `e2e-test` pod is present, you will have to first manually delete it; failing to do so will cause the `e2e-test` pod to fail. From 7b6430846d5ff5876b7b11ff5549880bdb146aec Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Thu, 24 Jun 2021 17:21:11 +0100 Subject: [PATCH 330/790] Added required env vars for make run to work correctly (#590) --- docs/RELEASE_NOTES.md | 10 +++------- docs/contributing.md | 5 +++++ scripts/dev/get_e2e_env_vars.py | 2 ++ 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/docs/RELEASE_NOTES.md b/docs/RELEASE_NOTES.md index f34d1360b..6b9872668 100644 --- a/docs/RELEASE_NOTES.md +++ b/docs/RELEASE_NOTES.md @@ -1,17 +1,13 @@ -# MongoDB Kubernetes Operator 0.6.2 +# MongoDB Kubernetes Operator 0.6.3 ## Kubernetes Operator - Changes - - stability improvements when changing version of MongoDB. - - increased number of concurrent resources the operator can act on. - - mongodb will now send its log to stdout by default. - - changed the default values for `MONGODB_REPO_URL` and `MONGODB_IMAGE` in the operator deployment - - added the support of arbiters for the replica sets. + - Members of a Replica Set can be configured as arbiters. ## Updated Image Tags -- mongodb-kubernetes-operator:0.6.2 +- mongodb-kubernetes-operator:0.6.3 _All the images can be found in:_ diff --git a/docs/contributing.md b/docs/contributing.md index 5ab97d9f5..2bf3a10a6 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -147,6 +147,11 @@ To remove the operator and any created resources you can run make undeploy ``` +Alternatively, you can run the operator locally with + +```sh +make run +``` # Running Tests diff --git a/scripts/dev/get_e2e_env_vars.py b/scripts/dev/get_e2e_env_vars.py index 3dbd55c34..48a44e2b6 100755 --- a/scripts/dev/get_e2e_env_vars.py +++ b/scripts/dev/get_e2e_env_vars.py @@ -27,6 +27,8 @@ def _get_e2e_test_envs(dev_config: DevConfig) -> Dict[str, str]: "READINESS_PROBE_IMAGE": f"{dev_config.repo_url}/{dev_config.readiness_probe_image}", "PERFORM_CLEANUP": "true" if cleanup else "false", "WATCH_NAMESPACE": dev_config.namespace, + "MONGODB_IMAGE": "mongo", + "MONGODB_REPO_URL": "docker.io", } From f73783da92acd3902e6c49f2abbfa5a09d280436 Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Fri, 25 Jun 2021 09:50:15 +0100 Subject: [PATCH 331/790] Remove safe-to-test label any time a PR is updated (#593) --- .github/workflows/remove-label.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .github/workflows/remove-label.yml diff --git a/.github/workflows/remove-label.yml b/.github/workflows/remove-label.yml new file mode 100644 index 000000000..60316ff49 --- /dev/null +++ b/.github/workflows/remove-label.yml @@ -0,0 +1,13 @@ +name: Remove Label +on: [ pull_request ] +jobs: + remove-safe-to-test-label: + runs-on: ubuntu-latest + name: Remove Label + steps: + - name: + uses: buildsville/add-remove-label@v1 + with: + token: ${{secrets.GITHUB_TOKEN}} + label: safe-to-test + type: remove From fb205a37567b9cc8edc89503bc8f3ddcba249317 Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Fri, 25 Jun 2021 14:27:59 +0100 Subject: [PATCH 332/790] Run GithubActions correctly from forks (Templating of GithubActions) (#576) --- .action_templates/e2e-fork-template.yaml | 25 ++++++ .action_templates/e2e-pr-template.yaml | 27 +++++++ .action_templates/e2e-single-template.yaml | 18 +++++ .../events/labeled-pull-request-target.yaml | 7 ++ .../events/on-pull-request-master.yaml | 5 ++ .action_templates/events/on-push-master.yaml | 5 ++ .../events/single-e2e-workflow-dispatch.yaml | 13 ++++ .../events/workflow-dispatch.yaml | 1 + .action_templates/jobs/e2e-success.yaml | 8 ++ .action_templates/jobs/setup.yaml | 12 +++ .action_templates/jobs/single-test.yaml | 3 + .action_templates/jobs/tests.yaml | 78 +++++++++++++++++++ .../build-and-push-development-images.yaml | 6 ++ .action_templates/steps/cancel-previous.yaml | 4 + .action_templates/steps/checkout-fork.yaml | 8 ++ .action_templates/steps/checkout.yaml | 2 + .../dump-and-upload-diagnostics-always.yaml | 12 +++ .../steps/dump-and-upload-diagnostics.yaml | 13 ++++ .action_templates/steps/quay-login.yaml | 6 ++ .action_templates/steps/run-test-matrix.yaml | 9 +++ .action_templates/steps/run-test-single.yaml | 3 + .action_templates/steps/save-run-status.yaml | 3 + .action_templates/steps/set-run-status.yaml | 17 ++++ .../steps/setup-and-install-python.yaml | 11 +++ .../steps/setup-kind-cluster.yaml | 7 ++ .github/workflows/release-images.yml | 3 +- requirements.txt | 2 + scripts/dev/generate_github_actions.py | 45 +++++++++++ scripts/git-hooks/pre-commit | 6 ++ 29 files changed, 358 insertions(+), 1 deletion(-) create mode 100644 .action_templates/e2e-fork-template.yaml create mode 100644 .action_templates/e2e-pr-template.yaml create mode 100644 .action_templates/e2e-single-template.yaml create mode 100644 .action_templates/events/labeled-pull-request-target.yaml create mode 100644 .action_templates/events/on-pull-request-master.yaml create mode 100644 .action_templates/events/on-push-master.yaml create mode 100644 .action_templates/events/single-e2e-workflow-dispatch.yaml create mode 100644 .action_templates/events/workflow-dispatch.yaml create mode 100644 .action_templates/jobs/e2e-success.yaml create mode 100644 .action_templates/jobs/setup.yaml create mode 100644 .action_templates/jobs/single-test.yaml create mode 100644 .action_templates/jobs/tests.yaml create mode 100644 .action_templates/steps/build-and-push-development-images.yaml create mode 100644 .action_templates/steps/cancel-previous.yaml create mode 100644 .action_templates/steps/checkout-fork.yaml create mode 100644 .action_templates/steps/checkout.yaml create mode 100644 .action_templates/steps/dump-and-upload-diagnostics-always.yaml create mode 100644 .action_templates/steps/dump-and-upload-diagnostics.yaml create mode 100644 .action_templates/steps/quay-login.yaml create mode 100644 .action_templates/steps/run-test-matrix.yaml create mode 100644 .action_templates/steps/run-test-single.yaml create mode 100644 .action_templates/steps/save-run-status.yaml create mode 100644 .action_templates/steps/set-run-status.yaml create mode 100644 .action_templates/steps/setup-and-install-python.yaml create mode 100644 .action_templates/steps/setup-kind-cluster.yaml create mode 100755 scripts/dev/generate_github_actions.py diff --git a/.action_templates/e2e-fork-template.yaml b/.action_templates/e2e-fork-template.yaml new file mode 100644 index 000000000..7c68f2674 --- /dev/null +++ b/.action_templates/e2e-fork-template.yaml @@ -0,0 +1,25 @@ +name: Run E2E Fork +jobs: + - template: setup + if: github.event.pull_request.head.repo.full_name != 'mongodb/mongodb-kubernetes-operator' && contains(github.event.pull_request.labels.*.name, 'safe-to-test') + steps: + - template: cancel-previous + - template: checkout-fork + - template: setup-and-install-python + - template: quay-login + - template: build-and-push-development-images + - template: tests + steps: + - template: cancel-previous + - template: checkout-fork + - template: set-run-status + - template: setup-and-install-python + - template: setup-kind-cluster + if: steps.last_run_status.outputs.last_run_status != 'success' + - template: run-test-matrix + - template: save-run-status + - template: dump-and-upload-diagnostics + - template: e2e-success + +events: + - template: labeled-pull-request-target diff --git a/.action_templates/e2e-pr-template.yaml b/.action_templates/e2e-pr-template.yaml new file mode 100644 index 000000000..4ee419ae7 --- /dev/null +++ b/.action_templates/e2e-pr-template.yaml @@ -0,0 +1,27 @@ +name: Run E2E +jobs: + - template: setup + if: github.event.pull_request.head.repo.full_name == 'mongodb/mongodb-kubernetes-operator' + steps: + - template: cancel-previous + - template: checkout + - template: setup-and-install-python + - template: quay-login + - template: build-and-push-development-images + - template: tests + steps: + - template: cancel-previous + - template: checkout + - template: set-run-status + - template: setup-and-install-python + - template: setup-kind-cluster + if: steps.last_run_status.outputs.last_run_status != 'success' + - template: run-test-matrix + - template: save-run-status + - template: dump-and-upload-diagnostics + - template: e2e-success + +events: + - template: on-pull-request-master + - template: on-push-master + - template: workflow-dispatch diff --git a/.action_templates/e2e-single-template.yaml b/.action_templates/e2e-single-template.yaml new file mode 100644 index 000000000..6c21d9cdc --- /dev/null +++ b/.action_templates/e2e-single-template.yaml @@ -0,0 +1,18 @@ +name: Run Single E2E +jobs: + - template: setup + steps: + - template: checkout + - template: setup-and-install-python + - template: quay-login + - template: build-and-push-development-images + - template: single-test + steps: + - template: checkout + - template: setup-and-install-python + - template: setup-kind-cluster + - template: run-test-single + - template: dump-and-upload-diagnostics-always + +events: + - template: single-e2e-workflow-dispatch diff --git a/.action_templates/events/labeled-pull-request-target.yaml b/.action_templates/events/labeled-pull-request-target.yaml new file mode 100644 index 000000000..1e7743cd8 --- /dev/null +++ b/.action_templates/events/labeled-pull-request-target.yaml @@ -0,0 +1,7 @@ +# pull_request_target means that the secrets of this repo will be used. +pull_request_target: + types: [labeled] + branches: + - master + paths-ignore: + - 'docs/**' diff --git a/.action_templates/events/on-pull-request-master.yaml b/.action_templates/events/on-pull-request-master.yaml new file mode 100644 index 000000000..9107a3d91 --- /dev/null +++ b/.action_templates/events/on-pull-request-master.yaml @@ -0,0 +1,5 @@ +pull_request: + branches: + - master + paths-ignore: + - 'docs/**' diff --git a/.action_templates/events/on-push-master.yaml b/.action_templates/events/on-push-master.yaml new file mode 100644 index 000000000..844e045c3 --- /dev/null +++ b/.action_templates/events/on-push-master.yaml @@ -0,0 +1,5 @@ +push: + branches: + - master + paths-ignore: + - 'docs/**' diff --git a/.action_templates/events/single-e2e-workflow-dispatch.yaml b/.action_templates/events/single-e2e-workflow-dispatch.yaml new file mode 100644 index 000000000..01cc9fcae --- /dev/null +++ b/.action_templates/events/single-e2e-workflow-dispatch.yaml @@ -0,0 +1,13 @@ +workflow_dispatch: + inputs: + distro: + description: 'Distro to run test' + required: true + default: "ubuntu" + test-name: + description: 'Name of test to run' + required: true + cluster-wide: + description: 'Whether or not the test is cluster wide' + required: true + default: "false" diff --git a/.action_templates/events/workflow-dispatch.yaml b/.action_templates/events/workflow-dispatch.yaml new file mode 100644 index 000000000..5de950ef3 --- /dev/null +++ b/.action_templates/events/workflow-dispatch.yaml @@ -0,0 +1 @@ +workflow_dispatch: {} diff --git a/.action_templates/jobs/e2e-success.yaml b/.action_templates/jobs/e2e-success.yaml new file mode 100644 index 000000000..d4aac0eab --- /dev/null +++ b/.action_templates/jobs/e2e-success.yaml @@ -0,0 +1,8 @@ +e2e-success: + if: always() + needs: [tests] + runs-on: ubuntu-latest + steps: + - name: Check E2E Result + if: needs.tests.result != 'success' + run: exit 1 diff --git a/.action_templates/jobs/setup.yaml b/.action_templates/jobs/setup.yaml new file mode 100644 index 000000000..cd4dbf073 --- /dev/null +++ b/.action_templates/jobs/setup.yaml @@ -0,0 +1,12 @@ +setup: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - pipeline-argument: operator-ubi + - pipeline-argument: version-post-start-hook-init + - pipeline-argument: readiness-probe-init + - pipeline-argument: agent-ubi + - pipeline-argument: agent-ubuntu + - pipeline-argument: e2e diff --git a/.action_templates/jobs/single-test.yaml b/.action_templates/jobs/single-test.yaml new file mode 100644 index 000000000..b06a8a918 --- /dev/null +++ b/.action_templates/jobs/single-test.yaml @@ -0,0 +1,3 @@ +single-test: + runs-on: ubuntu-latest + needs: [setup] diff --git a/.action_templates/jobs/tests.yaml b/.action_templates/jobs/tests.yaml new file mode 100644 index 000000000..cf05f4288 --- /dev/null +++ b/.action_templates/jobs/tests.yaml @@ -0,0 +1,78 @@ +tests: + runs-on: ubuntu-latest + needs: [setup] + strategy: + fail-fast: false + matrix: + include: + - test-name: replica_set + distro: ubuntu + - test-name: replica_set_recovery + distro: ubuntu + - test-name: replica_set_recovery + distro: ubuntu + - test-name: replica_set_mongod_readiness + distro: ubuntu + - test-name: replica_set_scale + distro: ubuntu + - test-name: replica_set_scale_down + distro: ubuntu + - test-name: replica_set_change_version + distro: ubuntu + - test-name: feature_compatibility_version + distro: ubuntu + - test-name: feature_compatibility_version_upgrade + distro: ubuntu + - test-name: replica_set_tls + distro: ubuntu + - test-name: replica_set_tls_upgrade + distro: ubuntu + - test-name: statefulset_arbitrary_config + distro: ubuntu + - test-name: statefulset_arbitrary_config_update + distro: ubuntu + - test-name: replica_set_mongod_config + distro: ubuntu + - test-name: replica_set_cross_namespace_deploy + distro: ubuntu + cluster-wide: true + - test-name: replica_set_custom_role + distro: ubuntu + - test-name: replica_set_arbiter + distro: ubuntu + + - test-name: replica_set + distro: ubi + - test-name: replica_set_recovery + distro: ubi + - test-name: replica_set_recovery + distro: ubi + - test-name: replica_set_mongod_readiness + distro: ubi + - test-name: replica_set_scale + distro: ubi + - test-name: replica_set_scale_down + distro: ubi + - test-name: replica_set_change_version + distro: ubi + - test-name: feature_compatibility_version + distro: ubi + - test-name: feature_compatibility_version_upgrade + distro: ubi + - test-name: replica_set_tls + distro: ubi + - test-name: replica_set_tls_upgrade + distro: ubi + - test-name: statefulset_arbitrary_config + distro: ubi + - test-name: statefulset_arbitrary_config_update + distro: ubi + - test-name: replica_set_mongod_config + distro: ubi + - test-name: replica_set_cross_namespace_deploy + distro: ubi + cluster-wide: true + - test-name: replica_set_custom_role + distro: ubi + - test-name: replica_set_arbiter + distro: ubi diff --git a/.action_templates/steps/build-and-push-development-images.yaml b/.action_templates/steps/build-and-push-development-images.yaml new file mode 100644 index 000000000..063ae446a --- /dev/null +++ b/.action_templates/steps/build-and-push-development-images.yaml @@ -0,0 +1,6 @@ +- name: Build and Push Images + run: | + python pipeline.py --image-name ${{ matrix.pipeline-argument }} --release false + env: + MONGODB_COMMUNITY_CONFIG: "${{ github.workspace }}/scripts/ci/config.json" + version_id: "${{ github.run_id }}" diff --git a/.action_templates/steps/cancel-previous.yaml b/.action_templates/steps/cancel-previous.yaml new file mode 100644 index 000000000..e15ae86c0 --- /dev/null +++ b/.action_templates/steps/cancel-previous.yaml @@ -0,0 +1,4 @@ +- name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.9.0 + with: + access_token: ${{ github.token }} diff --git a/.action_templates/steps/checkout-fork.yaml b/.action_templates/steps/checkout-fork.yaml new file mode 100644 index 000000000..f5ce58770 --- /dev/null +++ b/.action_templates/steps/checkout-fork.yaml @@ -0,0 +1,8 @@ +# We checkout the forked repository code. +# Because we are using pull_request_target the Github Secrets will be passed +# So code should be reviewed before labeling as "safe-to-test" +- name: Checkout Code + uses: actions/checkout@v2 + with: + ref: ${{github.event.pull_request.head.ref}} + repository: ${{github.event.pull_request.head.repo.full_name}} diff --git a/.action_templates/steps/checkout.yaml b/.action_templates/steps/checkout.yaml new file mode 100644 index 000000000..6edd51386 --- /dev/null +++ b/.action_templates/steps/checkout.yaml @@ -0,0 +1,2 @@ +- name: Checkout Code + uses: actions/checkout@v2 diff --git a/.action_templates/steps/dump-and-upload-diagnostics-always.yaml b/.action_templates/steps/dump-and-upload-diagnostics-always.yaml new file mode 100644 index 000000000..61cfff7db --- /dev/null +++ b/.action_templates/steps/dump-and-upload-diagnostics-always.yaml @@ -0,0 +1,12 @@ +- name: Dump Diagnostics + if: always() + continue-on-error: true + run: scripts/ci/dump_diagnostics.sh default # default since kind is running in the default namespace + +- name: Upload Diagnostics + if: always() + uses: actions/upload-artifact@v2 + continue-on-error: true + with: + name: "${{ github.event.inputs.test-name }}-${{ github.event.inputs.distro }}-diagnostics" + path: "${{ github.workspace }}/diagnostics" diff --git a/.action_templates/steps/dump-and-upload-diagnostics.yaml b/.action_templates/steps/dump-and-upload-diagnostics.yaml new file mode 100644 index 000000000..d8c23e3dd --- /dev/null +++ b/.action_templates/steps/dump-and-upload-diagnostics.yaml @@ -0,0 +1,13 @@ +- name: Dump Diagnostics + id: dump_diagnostics + if: always() && steps.e2e_test.outcome == 'failure' + continue-on-error: true + run: scripts/ci/dump_diagnostics.sh default # default since kind is running in the default namespace + +- name: Upload Diagnostics + if: always() && steps.dump_diagnostics.outcome == 'success' + uses: actions/upload-artifact@v2 + continue-on-error: true + with: + name: "${{ matrix.test-name }}-${{ matrix.distro }}-diagnostics" + path: "${{ github.workspace }}/diagnostics" diff --git a/.action_templates/steps/quay-login.yaml b/.action_templates/steps/quay-login.yaml new file mode 100644 index 000000000..85073e70b --- /dev/null +++ b/.action_templates/steps/quay-login.yaml @@ -0,0 +1,6 @@ +- name: Login to Quay.io + uses: docker/login-action@v1 + with: + registry: quay.io + username: ${{ secrets.QUAY_USERNAME }} + password: ${{ secrets.QUAY_ROBOT_TOKEN }} diff --git a/.action_templates/steps/run-test-matrix.yaml b/.action_templates/steps/run-test-matrix.yaml new file mode 100644 index 000000000..9c572a89c --- /dev/null +++ b/.action_templates/steps/run-test-matrix.yaml @@ -0,0 +1,9 @@ +- name: Run Test + id: e2e_test + if: steps.last_run_status.outputs.last_run_status != 'success' + run: | + cluster_wide=${{ matrix.cluster-wide }} + if [ -z "$cluster_wide" ]; then + cluster_wide="false" + fi + python3 ./scripts/dev/e2e.py --test ${{ matrix.test-name }} --tag ${{ github.run_id }} --config_file ./scripts/ci/config.json --distro ${{ matrix.distro }} --cluster-wide ${cluster_wide} diff --git a/.action_templates/steps/run-test-single.yaml b/.action_templates/steps/run-test-single.yaml new file mode 100644 index 000000000..453425961 --- /dev/null +++ b/.action_templates/steps/run-test-single.yaml @@ -0,0 +1,3 @@ +- name: Run Test Single + run: | + python3 ./scripts/dev/e2e.py --test ${{ github.event.inputs.test-name }} --tag ${{ github.run_id }} --config_file ./scripts/ci/config.json --distro ${{ github.event.inputs.distro }} --cluster-wide ${{ github.event.inputs.cluster-wide }} diff --git a/.action_templates/steps/save-run-status.yaml b/.action_templates/steps/save-run-status.yaml new file mode 100644 index 000000000..84845013b --- /dev/null +++ b/.action_templates/steps/save-run-status.yaml @@ -0,0 +1,3 @@ +- name: Save run status + if: always() + run: echo "::set-output name=last_run_status::${{ steps.e2e_test.outcome }}" > last_run_status diff --git a/.action_templates/steps/set-run-status.yaml b/.action_templates/steps/set-run-status.yaml new file mode 100644 index 000000000..40bb34079 --- /dev/null +++ b/.action_templates/steps/set-run-status.yaml @@ -0,0 +1,17 @@ +- name: Set default run status + run: echo "::set-output name=last_run_status::pending" > last_run_status + + # Tracking of the state of the previous test run is a workaround to the fact that it is not + # possible to re-run a single failed job, only re-running the entire workflow is currently possible. + # This workaround skips jobs if they have already passed. + # see https://github.com/actions/runner/issues/432 +- name: Restore last run status + id: last_run + uses: actions/cache@v2 + with: + path: last_run_status + key: ${{ github.run_id }}-${{ matrix.test-name }}-${{ matrix.distro }} + +- name: Set last run status + id: last_run_status + run: cat last_run_status diff --git a/.action_templates/steps/setup-and-install-python.yaml b/.action_templates/steps/setup-and-install-python.yaml new file mode 100644 index 000000000..420dc05fc --- /dev/null +++ b/.action_templates/steps/setup-and-install-python.yaml @@ -0,0 +1,11 @@ +- name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: '3.8' +- name: Cache Dependencies + uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ hashFiles('requirements.txt') }} +- name: Install Python Dependencies + run: pip install -r requirements.txt diff --git a/.action_templates/steps/setup-kind-cluster.yaml b/.action_templates/steps/setup-kind-cluster.yaml new file mode 100644 index 000000000..43faf02ef --- /dev/null +++ b/.action_templates/steps/setup-kind-cluster.yaml @@ -0,0 +1,7 @@ +- name: Setup Kind Cluster + uses: engineerd/setup-kind@v0.5.0 + with: + version: "v0.8.1" + +- name: Install CRD + run: kubectl apply -f config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml diff --git a/.github/workflows/release-images.yml b/.github/workflows/release-images.yml index c58be708e..94f72f8e7 100644 --- a/.github/workflows/release-images.yml +++ b/.github/workflows/release-images.yml @@ -3,6 +3,7 @@ name: Release Images on: pull_request_review: types: [submitted] + workflow_dispatch: jobs: release-images: @@ -85,7 +86,7 @@ jobs: - name: Create Github Release uses: ncipollo/release-action@v1 with: - tag: ${{ steps.release_tag.outputs.OUTPUT }} + tag: "v${{ steps.release_tag.outputs.OUTPUT }}" name: MongoDB Kubernetes Operator bodyFile: "${{ github.workspace }}/docs/RELEASE_NOTES.md" draft: true diff --git a/requirements.txt b/requirements.txt index 7ae27f692..2bb4843b5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ git+https://github.com/mongodb/sonar@0.0.11#egg=sonar +git+https://github.com/chatton/GithubActionTemplates@v0.0.2 docker==4.3.1 kubernetes==11.0.0 jinja2==2.11.3 @@ -11,4 +12,5 @@ pymongo==3.11.4 dnspython==2.0.0 requests==2.24.0 pyyaml==5.4.1 +ruamel.yaml==0.17.9 rsa>=4.7 # not directly required, pinned by Snyk to avoid a vulnerability diff --git a/scripts/dev/generate_github_actions.py b/scripts/dev/generate_github_actions.py new file mode 100755 index 000000000..37cfef84a --- /dev/null +++ b/scripts/dev/generate_github_actions.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python +from typing import Dict + +from ghat.template import template_github_action +import sys +import io +import ruamel.yaml + +yaml = ruamel.yaml.YAML() + +template_mapping = { + ".action_templates/e2e-fork-template.yaml": ".github/workflows/e2e-fork.yml", + ".action_templates/e2e-pr-template.yaml": ".github/workflows/e2e.yml", + ".action_templates/e2e-single-template.yaml": ".github/workflows/e2e-dispatch.yml", +} + + +def _prepend_auto_generated_message(github_action: Dict) -> str: + s = io.StringIO() + yaml.dump(github_action, s) + s.seek(0) + return """ +################################################################################## +# +# This file is automatically generated using templates. Changes to this file +# should happen through editing the templates under .action_templates/* +# Manual edits will be overwritten. +# +################################################################################## + +{}""".format( + s.read() + ) + + +def main() -> int: + for template in template_mapping: + github_action = template_github_action(template) + with open(template_mapping[template], "w+") as f: + f.write(_prepend_auto_generated_message(github_action)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/git-hooks/pre-commit b/scripts/git-hooks/pre-commit index 902efe332..39eda83df 100755 --- a/scripts/git-hooks/pre-commit +++ b/scripts/git-hooks/pre-commit @@ -73,6 +73,12 @@ function black_formatting() done } +function generate_github_actions(){ + scripts/dev/generate_github_actions.py + git add .github/workflows +} + +#generate_github_actions generate_crd go_imports black_formatting From 23ce9494ea421ce6f075b0c9ce5b2cb0c7e12df3 Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Fri, 25 Jun 2021 15:39:11 +0100 Subject: [PATCH 333/790] Generate Github Actions with Pre Commit hook (#594) --- .github/workflows/e2e-dispatch.yml | 172 ++++++------ .github/workflows/e2e-fork.yml | 241 +++++++++++++++++ .github/workflows/e2e.yml | 410 +++++++++++++++-------------- scripts/git-hooks/pre-commit | 2 +- 4 files changed, 548 insertions(+), 277 deletions(-) create mode 100644 .github/workflows/e2e-fork.yml diff --git a/.github/workflows/e2e-dispatch.yml b/.github/workflows/e2e-dispatch.yml index 93a3aaacf..510dfa86b 100644 --- a/.github/workflows/e2e-dispatch.yml +++ b/.github/workflows/e2e-dispatch.yml @@ -1,103 +1,115 @@ + +################################################################################## +# +# This file is automatically generated using templates. Changes to this file +# should happen through editing the templates under .action_templates/* +# Manual edits will be overwritten. +# +################################################################################## + name: Run Single E2E on: + # template: .action_templates/events/single-e2e-workflow-dispatch.yaml workflow_dispatch: inputs: distro: - description: 'Distro to run test' + description: Distro to run test required: true - default: "ubuntu" + default: ubuntu test-name: - description: 'Name of test to run' + description: Name of test to run required: true cluster-wide: - description: 'Whether or not the test is cluster wide' + description: Whether or not the test is cluster wide required: true - default: "false" - + default: 'false' jobs: + # template: .action_templates/jobs/setup.yaml setup: runs-on: ubuntu-latest strategy: fail-fast: false matrix: include: - - pipeline-argument: operator-ubi - - pipeline-argument: version-post-start-hook-init - - pipeline-argument: readiness-probe-init - - pipeline-argument: agent-ubi - - pipeline-argument: agent-ubuntu - - pipeline-argument: e2e + - pipeline-argument: operator-ubi + - pipeline-argument: version-post-start-hook-init + - pipeline-argument: readiness-probe-init + - pipeline-argument: agent-ubi + - pipeline-argument: agent-ubuntu + - pipeline-argument: e2e steps: - - name: Checkout Code - uses: actions/checkout@v2 - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: '3.8' - - - uses: actions/cache@v2 - with: - path: ~/.cache/pip - key: ${{ hashFiles('requirements.txt') }} - - - name: Install Python Dependencies - run: pip install -r requirements.txt - - - name: Login to Quay.io - uses: docker/login-action@v1 - with: - registry: quay.io - username: ${{ secrets.QUAY_USERNAME }} - password: ${{ secrets.QUAY_ROBOT_TOKEN }} - - - name: Build and Push Images - run: python pipeline.py --image-name ${{ matrix.pipeline-argument }} --release false - env: - MONGODB_COMMUNITY_CONFIG: "${{ github.workspace }}/scripts/ci/config.json" - version_id: "${{ github.run_id }}" - - tests: + # template: .action_templates/steps/checkout.yaml + - name: Checkout Code + uses: actions/checkout@v2 + # template: .action_templates/steps/setup-and-install-python.yaml + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: '3.8' + - name: Cache Dependencies + uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ hashFiles('requirements.txt') }} + - name: Install Python Dependencies + run: pip install -r requirements.txt + # template: .action_templates/steps/quay-login.yaml + - name: Login to Quay.io + uses: docker/login-action@v1 + with: + registry: quay.io + username: ${{ secrets.QUAY_USERNAME }} + password: ${{ secrets.QUAY_ROBOT_TOKEN }} + # template: .action_templates/steps/build-and-push-development-images.yaml + - name: Build and Push Images + run: | + python pipeline.py --image-name ${{ matrix.pipeline-argument }} --release false + env: + MONGODB_COMMUNITY_CONFIG: ${{ github.workspace }}/scripts/ci/config.json + version_id: ${{ github.run_id }} + # template: .action_templates/jobs/single-test.yaml + single-test: runs-on: ubuntu-latest needs: [setup] - steps: - - name: Checkout Code - uses: actions/checkout@v2 - - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: '3.8' - - - name: Cache Dependencies - uses: actions/cache@v2 - with: - path: ~/.cache/pip - key: ${{ hashFiles('requirements.txt') }} - - - name: Install Python Dependencies - run: pip install -r requirements.txt - - - name: Setup Kind Cluster - uses: engineerd/setup-kind@v0.5.0 - with: - version: "v0.8.1" - - - name: Install CRD - run: kubectl apply -f config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml - - - name: Run Test - run: python3 ./scripts/dev/e2e.py --test ${{ github.event.inputs.test-name }} --tag ${{ github.run_id }} --config_file ./scripts/ci/config.json --distro ${{ github.event.inputs.distro }} --cluster-wide ${{ github.event.inputs.cluster-wide }} + # template: .action_templates/steps/checkout.yaml + - name: Checkout Code + uses: actions/checkout@v2 + # template: .action_templates/steps/setup-and-install-python.yaml + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: '3.8' + - name: Cache Dependencies + uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ hashFiles('requirements.txt') }} + - name: Install Python Dependencies + run: pip install -r requirements.txt + # template: .action_templates/steps/setup-kind-cluster.yaml + - name: Setup Kind Cluster + uses: engineerd/setup-kind@v0.5.0 + with: + version: v0.8.1 - - name: Dump Diagnostics - if: always() - continue-on-error: true - run: scripts/ci/dump_diagnostics.sh default # default since kind is running in the default namespace + - name: Install CRD + run: kubectl apply -f config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml + # template: .action_templates/steps/run-test-single.yaml + - name: Run Test Single + run: | + python3 ./scripts/dev/e2e.py --test ${{ github.event.inputs.test-name }} --tag ${{ github.run_id }} --config_file ./scripts/ci/config.json --distro ${{ github.event.inputs.distro }} --cluster-wide ${{ github.event.inputs.cluster-wide }} + # template: .action_templates/steps/dump-and-upload-diagnostics-always.yaml + - name: Dump Diagnostics + if: always() + continue-on-error: true + run: scripts/ci/dump_diagnostics.sh default # default since kind is running in the default namespace - - name: Upload Diagnostics - if: always() - uses: actions/upload-artifact@v2 - continue-on-error: true - with: - name: "${{ github.event.inputs.test-name }}-${{ github.event.inputs.distro }}-diagnostics" - path: "${{ github.workspace }}/diagnostics" + - name: Upload Diagnostics + if: always() + uses: actions/upload-artifact@v2 + continue-on-error: true + with: + name: ${{ github.event.inputs.test-name }}-${{ github.event.inputs.distro + }}-diagnostics + path: ${{ github.workspace }}/diagnostics diff --git a/.github/workflows/e2e-fork.yml b/.github/workflows/e2e-fork.yml new file mode 100644 index 000000000..91e456774 --- /dev/null +++ b/.github/workflows/e2e-fork.yml @@ -0,0 +1,241 @@ + +################################################################################## +# +# This file is automatically generated using templates. Changes to this file +# should happen through editing the templates under .action_templates/* +# Manual edits will be overwritten. +# +################################################################################## + +name: Run E2E Fork +on: + # template: .action_templates/events/labeled-pull-request-target.yaml + pull_request_target: + types: [labeled] + branches: + - master + paths-ignore: + - docs/** +jobs: + # template: .action_templates/jobs/setup.yaml + setup: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - pipeline-argument: operator-ubi + - pipeline-argument: version-post-start-hook-init + - pipeline-argument: readiness-probe-init + - pipeline-argument: agent-ubi + - pipeline-argument: agent-ubuntu + - pipeline-argument: e2e + if: github.event.pull_request.head.repo.full_name != 'mongodb/mongodb-kubernetes-operator' + && contains(github.event.pull_request.labels.*.name, 'safe-to-test') + steps: + # template: .action_templates/steps/cancel-previous.yaml + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.9.0 + with: + access_token: ${{ github.token }} + # template: .action_templates/steps/checkout-fork.yaml + - name: Checkout Code + uses: actions/checkout@v2 + with: + ref: ${{github.event.pull_request.head.ref}} + repository: ${{github.event.pull_request.head.repo.full_name}} + # template: .action_templates/steps/setup-and-install-python.yaml + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: '3.8' + - name: Cache Dependencies + uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ hashFiles('requirements.txt') }} + - name: Install Python Dependencies + run: pip install -r requirements.txt + # template: .action_templates/steps/quay-login.yaml + - name: Login to Quay.io + uses: docker/login-action@v1 + with: + registry: quay.io + username: ${{ secrets.QUAY_USERNAME }} + password: ${{ secrets.QUAY_ROBOT_TOKEN }} + # template: .action_templates/steps/build-and-push-development-images.yaml + - name: Build and Push Images + run: | + python pipeline.py --image-name ${{ matrix.pipeline-argument }} --release false + env: + MONGODB_COMMUNITY_CONFIG: ${{ github.workspace }}/scripts/ci/config.json + version_id: ${{ github.run_id }} + # template: .action_templates/jobs/tests.yaml + tests: + runs-on: ubuntu-latest + needs: [setup] + strategy: + fail-fast: false + matrix: + include: + - test-name: replica_set + distro: ubuntu + - test-name: replica_set_recovery + distro: ubuntu + - test-name: replica_set_recovery + distro: ubuntu + - test-name: replica_set_mongod_readiness + distro: ubuntu + - test-name: replica_set_scale + distro: ubuntu + - test-name: replica_set_scale_down + distro: ubuntu + - test-name: replica_set_change_version + distro: ubuntu + - test-name: feature_compatibility_version + distro: ubuntu + - test-name: feature_compatibility_version_upgrade + distro: ubuntu + - test-name: replica_set_tls + distro: ubuntu + - test-name: replica_set_tls_upgrade + distro: ubuntu + - test-name: statefulset_arbitrary_config + distro: ubuntu + - test-name: statefulset_arbitrary_config_update + distro: ubuntu + - test-name: replica_set_mongod_config + distro: ubuntu + - test-name: replica_set_cross_namespace_deploy + distro: ubuntu + cluster-wide: true + - test-name: replica_set_custom_role + distro: ubuntu + - test-name: replica_set_arbiter + distro: ubuntu + + - test-name: replica_set + distro: ubi + - test-name: replica_set_recovery + distro: ubi + - test-name: replica_set_recovery + distro: ubi + - test-name: replica_set_mongod_readiness + distro: ubi + - test-name: replica_set_scale + distro: ubi + - test-name: replica_set_scale_down + distro: ubi + - test-name: replica_set_change_version + distro: ubi + - test-name: feature_compatibility_version + distro: ubi + - test-name: feature_compatibility_version_upgrade + distro: ubi + - test-name: replica_set_tls + distro: ubi + - test-name: replica_set_tls_upgrade + distro: ubi + - test-name: statefulset_arbitrary_config + distro: ubi + - test-name: statefulset_arbitrary_config_update + distro: ubi + - test-name: replica_set_mongod_config + distro: ubi + - test-name: replica_set_cross_namespace_deploy + distro: ubi + cluster-wide: true + - test-name: replica_set_custom_role + distro: ubi + - test-name: replica_set_arbiter + distro: ubi + steps: + # template: .action_templates/steps/cancel-previous.yaml + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.9.0 + with: + access_token: ${{ github.token }} + # template: .action_templates/steps/checkout-fork.yaml + - name: Checkout Code + uses: actions/checkout@v2 + with: + ref: ${{github.event.pull_request.head.ref}} + repository: ${{github.event.pull_request.head.repo.full_name}} + # template: .action_templates/steps/set-run-status.yaml + - name: Set default run status + run: echo "::set-output name=last_run_status::pending" > last_run_status + + # Tracking of the state of the previous test run is a workaround to the fact that it is not + # possible to re-run a single failed job, only re-running the entire workflow is currently possible. + # This workaround skips jobs if they have already passed. + # see https://github.com/actions/runner/issues/432 + - name: Restore last run status + id: last_run + uses: actions/cache@v2 + with: + path: last_run_status + key: ${{ github.run_id }}-${{ matrix.test-name }}-${{ matrix.distro }} + + - name: Set last run status + id: last_run_status + run: cat last_run_status + # template: .action_templates/steps/setup-and-install-python.yaml + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: '3.8' + - name: Cache Dependencies + uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ hashFiles('requirements.txt') }} + - name: Install Python Dependencies + run: pip install -r requirements.txt + # template: .action_templates/steps/setup-kind-cluster.yaml + - name: Setup Kind Cluster + uses: engineerd/setup-kind@v0.5.0 + with: + version: v0.8.1 + + if: steps.last_run_status.outputs.last_run_status != 'success' + - name: Install CRD + run: kubectl apply -f config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml + if: steps.last_run_status.outputs.last_run_status != 'success' + # template: .action_templates/steps/run-test-matrix.yaml + - name: Run Test + id: e2e_test + if: steps.last_run_status.outputs.last_run_status != 'success' + run: | + cluster_wide=${{ matrix.cluster-wide }} + if [ -z "$cluster_wide" ]; then + cluster_wide="false" + fi + python3 ./scripts/dev/e2e.py --test ${{ matrix.test-name }} --tag ${{ github.run_id }} --config_file ./scripts/ci/config.json --distro ${{ matrix.distro }} --cluster-wide ${cluster_wide} + # template: .action_templates/steps/save-run-status.yaml + - name: Save run status + if: always() + run: echo "::set-output name=last_run_status::${{ steps.e2e_test.outcome }}" + > last_run_status + # template: .action_templates/steps/dump-and-upload-diagnostics.yaml + - name: Dump Diagnostics + id: dump_diagnostics + if: always() && steps.e2e_test.outcome == 'failure' + continue-on-error: true + run: scripts/ci/dump_diagnostics.sh default # default since kind is running in the default namespace + + - name: Upload Diagnostics + if: always() && steps.dump_diagnostics.outcome == 'success' + uses: actions/upload-artifact@v2 + continue-on-error: true + with: + name: ${{ matrix.test-name }}-${{ matrix.distro }}-diagnostics + path: ${{ github.workspace }}/diagnostics + # template: .action_templates/jobs/e2e-success.yaml + e2e-success: + if: always() + needs: [tests] + runs-on: ubuntu-latest + steps: + - name: Check E2E Result + if: needs.tests.result != 'success' + run: exit 1 diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index fbf8119f7..14fe77af0 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -1,59 +1,79 @@ + +################################################################################## +# +# This file is automatically generated using templates. Changes to this file +# should happen through editing the templates under .action_templates/* +# Manual edits will be overwritten. +# +################################################################################## + name: Run E2E on: + # template: .action_templates/events/on-pull-request-master.yaml pull_request: branches: - - master + - master + paths-ignore: + - docs/** + # template: .action_templates/events/on-push-master.yaml push: branches: - - master - workflow_dispatch: + - master + paths-ignore: + - docs/** + # template: .action_templates/events/workflow-dispatch.yaml + workflow_dispatch: {} jobs: + # template: .action_templates/jobs/setup.yaml setup: runs-on: ubuntu-latest strategy: fail-fast: false matrix: include: - - pipeline-argument: operator-ubi - - pipeline-argument: version-post-start-hook-init - - pipeline-argument: readiness-probe-init - - pipeline-argument: agent-ubi - - pipeline-argument: agent-ubuntu - - pipeline-argument: e2e + - pipeline-argument: operator-ubi + - pipeline-argument: version-post-start-hook-init + - pipeline-argument: readiness-probe-init + - pipeline-argument: agent-ubi + - pipeline-argument: agent-ubuntu + - pipeline-argument: e2e + if: github.event.pull_request.head.repo.full_name == 'mongodb/mongodb-kubernetes-operator' steps: - - name: Cancel Previous Runs - uses: styfle/cancel-workflow-action@0.9.0 - with: - access_token: ${{ github.token }} - - - name: Checkout Code - uses: actions/checkout@v2 - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: '3.8' - - - uses: actions/cache@v2 - with: - path: ~/.cache/pip - key: ${{ hashFiles('requirements.txt') }} - - - name: Install Python Dependencies - run: pip install -r requirements.txt - - - name: Login to Quay.io - uses: docker/login-action@v1 - with: - registry: quay.io - username: ${{ secrets.QUAY_USERNAME }} - password: ${{ secrets.QUAY_ROBOT_TOKEN }} - - - name: Build and Push Images - run: python pipeline.py --image-name ${{ matrix.pipeline-argument }} --release false - env: - MONGODB_COMMUNITY_CONFIG: "${{ github.workspace }}/scripts/ci/config.json" - version_id: "${{ github.run_id }}" - + # template: .action_templates/steps/cancel-previous.yaml + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.9.0 + with: + access_token: ${{ github.token }} + # template: .action_templates/steps/checkout.yaml + - name: Checkout Code + uses: actions/checkout@v2 + # template: .action_templates/steps/setup-and-install-python.yaml + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: '3.8' + - name: Cache Dependencies + uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ hashFiles('requirements.txt') }} + - name: Install Python Dependencies + run: pip install -r requirements.txt + # template: .action_templates/steps/quay-login.yaml + - name: Login to Quay.io + uses: docker/login-action@v1 + with: + registry: quay.io + username: ${{ secrets.QUAY_USERNAME }} + password: ${{ secrets.QUAY_ROBOT_TOKEN }} + # template: .action_templates/steps/build-and-push-development-images.yaml + - name: Build and Push Images + run: | + python pipeline.py --image-name ${{ matrix.pipeline-argument }} --release false + env: + MONGODB_COMMUNITY_CONFIG: ${{ github.workspace }}/scripts/ci/config.json + version_id: ${{ github.run_id }} + # template: .action_templates/jobs/tests.yaml tests: runs-on: ubuntu-latest needs: [setup] @@ -61,163 +81,161 @@ jobs: fail-fast: false matrix: include: - - test-name: replica_set - distro: ubuntu - - test-name: replica_set_recovery - distro: ubuntu - - test-name: replica_set_recovery - distro: ubuntu - - test-name: replica_set_mongod_readiness - distro: ubuntu - - test-name: replica_set_scale - distro: ubuntu - - test-name: replica_set_scale_down - distro: ubuntu - - test-name: replica_set_change_version - distro: ubuntu - - test-name: feature_compatibility_version - distro: ubuntu - - test-name: feature_compatibility_version_upgrade - distro: ubuntu - - test-name: replica_set_tls - distro: ubuntu - - test-name: replica_set_tls_upgrade - distro: ubuntu - - test-name: statefulset_arbitrary_config - distro: ubuntu - - test-name: statefulset_arbitrary_config_update - distro: ubuntu - - test-name: replica_set_mongod_config - distro: ubuntu - - test-name: replica_set_cross_namespace_deploy - distro: ubuntu - cluster-wide: true - - test-name: replica_set_custom_role - distro: ubuntu - - test-name: replica_set_arbiter - distro: ubuntu - - - test-name: replica_set - distro: ubi - - test-name: replica_set_recovery - distro: ubi - - test-name: replica_set_recovery - distro: ubi - - test-name: replica_set_mongod_readiness - distro: ubi - - test-name: replica_set_scale - distro: ubi - - test-name: replica_set_scale_down - distro: ubi - - test-name: replica_set_change_version - distro: ubi - - test-name: feature_compatibility_version - distro: ubi - - test-name: feature_compatibility_version_upgrade - distro: ubi - - test-name: replica_set_tls - distro: ubi - - test-name: replica_set_tls_upgrade - distro: ubi - - test-name: statefulset_arbitrary_config - distro: ubi - - test-name: statefulset_arbitrary_config_update - distro: ubi - - test-name: replica_set_mongod_config - distro: ubi - - test-name: replica_set_cross_namespace_deploy - distro: ubi - cluster-wide: true - - test-name: replica_set_custom_role - distro: ubi - - test-name: replica_set_arbiter - distro: ubi - - + - test-name: replica_set + distro: ubuntu + - test-name: replica_set_recovery + distro: ubuntu + - test-name: replica_set_recovery + distro: ubuntu + - test-name: replica_set_mongod_readiness + distro: ubuntu + - test-name: replica_set_scale + distro: ubuntu + - test-name: replica_set_scale_down + distro: ubuntu + - test-name: replica_set_change_version + distro: ubuntu + - test-name: feature_compatibility_version + distro: ubuntu + - test-name: feature_compatibility_version_upgrade + distro: ubuntu + - test-name: replica_set_tls + distro: ubuntu + - test-name: replica_set_tls_upgrade + distro: ubuntu + - test-name: statefulset_arbitrary_config + distro: ubuntu + - test-name: statefulset_arbitrary_config_update + distro: ubuntu + - test-name: replica_set_mongod_config + distro: ubuntu + - test-name: replica_set_cross_namespace_deploy + distro: ubuntu + cluster-wide: true + - test-name: replica_set_custom_role + distro: ubuntu + - test-name: replica_set_arbiter + distro: ubuntu + + - test-name: replica_set + distro: ubi + - test-name: replica_set_recovery + distro: ubi + - test-name: replica_set_recovery + distro: ubi + - test-name: replica_set_mongod_readiness + distro: ubi + - test-name: replica_set_scale + distro: ubi + - test-name: replica_set_scale_down + distro: ubi + - test-name: replica_set_change_version + distro: ubi + - test-name: feature_compatibility_version + distro: ubi + - test-name: feature_compatibility_version_upgrade + distro: ubi + - test-name: replica_set_tls + distro: ubi + - test-name: replica_set_tls_upgrade + distro: ubi + - test-name: statefulset_arbitrary_config + distro: ubi + - test-name: statefulset_arbitrary_config_update + distro: ubi + - test-name: replica_set_mongod_config + distro: ubi + - test-name: replica_set_cross_namespace_deploy + distro: ubi + cluster-wide: true + - test-name: replica_set_custom_role + distro: ubi + - test-name: replica_set_arbiter + distro: ubi steps: - - name: Cancel Previous Runs - uses: styfle/cancel-workflow-action@0.9.0 - with: - access_token: ${{ github.token }} - - - name: Checkout Code - uses: actions/checkout@v2 - - - name: Set default run status - run: echo "::set-output name=last_run_status::pending" > last_run_status - - # Tracking of the state of the previous test run is a workaround to the fact that it is not - # possible to re-run a single failed job, only re-running the entire workflow is currently possible. - # This workaround skips jobs if they have already passed. - # see https://github.com/actions/runner/issues/432 - - name: Restore last run status - id: last_run - uses: actions/cache@v2 - with: - path: last_run_status - key: ${{ github.run_id }}-${{ matrix.test-name }}-${{ matrix.distro }} - - - name: Set last run status - id: last_run_status - run: cat last_run_status - - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: '3.8' - - - name: Cache Dependencies - uses: actions/cache@v2 - with: - path: ~/.cache/pip - key: ${{ hashFiles('requirements.txt') }} - - - name: Install Python Dependencies - run: pip install -r requirements.txt - - - name: Setup Kind Cluster - if: steps.last_run_status.outputs.last_run_status != 'success' - uses: engineerd/setup-kind@v0.5.0 - with: - version: "v0.8.1" - - - name: Install CRD - if: steps.last_run_status.outputs.last_run_status != 'success' - run: kubectl apply -f config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml - - - name: Run Test - id: e2e_test - if: steps.last_run_status.outputs.last_run_status != 'success' - run: | - cluster_wide=${{ matrix.cluster-wide }} - if [ -z "$cluster_wide" ]; then - cluster_wide="false" - fi - python3 ./scripts/dev/e2e.py --test ${{ matrix.test-name }} --tag ${{ github.run_id }} --config_file ./scripts/ci/config.json --distro ${{ matrix.distro }} --cluster-wide ${cluster_wide} - - - name: Save run status - if: always() - run: echo "::set-output name=last_run_status::${{ steps.e2e_test.outcome }}" > last_run_status - - - name: Dump Diagnostics - id: dump_diagnostics - if: always() && steps.e2e_test.outcome == 'failure' - continue-on-error: true - run: scripts/ci/dump_diagnostics.sh default # default since kind is running in the default namespace - - - name: Upload Diagnostics - if: always() && steps.dump_diagnostics.outcome == 'success' - uses: actions/upload-artifact@v2 - continue-on-error: true - with: - name: "${{ matrix.test-name }}-${{ matrix.distro }}-diagnostics" - path: "${{ github.workspace }}/diagnostics" - + # template: .action_templates/steps/cancel-previous.yaml + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.9.0 + with: + access_token: ${{ github.token }} + # template: .action_templates/steps/checkout.yaml + - name: Checkout Code + uses: actions/checkout@v2 + # template: .action_templates/steps/set-run-status.yaml + - name: Set default run status + run: echo "::set-output name=last_run_status::pending" > last_run_status + + # Tracking of the state of the previous test run is a workaround to the fact that it is not + # possible to re-run a single failed job, only re-running the entire workflow is currently possible. + # This workaround skips jobs if they have already passed. + # see https://github.com/actions/runner/issues/432 + - name: Restore last run status + id: last_run + uses: actions/cache@v2 + with: + path: last_run_status + key: ${{ github.run_id }}-${{ matrix.test-name }}-${{ matrix.distro }} + + - name: Set last run status + id: last_run_status + run: cat last_run_status + # template: .action_templates/steps/setup-and-install-python.yaml + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: '3.8' + - name: Cache Dependencies + uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ hashFiles('requirements.txt') }} + - name: Install Python Dependencies + run: pip install -r requirements.txt + # template: .action_templates/steps/setup-kind-cluster.yaml + - name: Setup Kind Cluster + uses: engineerd/setup-kind@v0.5.0 + with: + version: v0.8.1 + + if: steps.last_run_status.outputs.last_run_status != 'success' + - name: Install CRD + run: kubectl apply -f config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml + if: steps.last_run_status.outputs.last_run_status != 'success' + # template: .action_templates/steps/run-test-matrix.yaml + - name: Run Test + id: e2e_test + if: steps.last_run_status.outputs.last_run_status != 'success' + run: | + cluster_wide=${{ matrix.cluster-wide }} + if [ -z "$cluster_wide" ]; then + cluster_wide="false" + fi + python3 ./scripts/dev/e2e.py --test ${{ matrix.test-name }} --tag ${{ github.run_id }} --config_file ./scripts/ci/config.json --distro ${{ matrix.distro }} --cluster-wide ${cluster_wide} + # template: .action_templates/steps/save-run-status.yaml + - name: Save run status + if: always() + run: echo "::set-output name=last_run_status::${{ steps.e2e_test.outcome }}" + > last_run_status + # template: .action_templates/steps/dump-and-upload-diagnostics.yaml + - name: Dump Diagnostics + id: dump_diagnostics + if: always() && steps.e2e_test.outcome == 'failure' + continue-on-error: true + run: scripts/ci/dump_diagnostics.sh default # default since kind is running in the default namespace + + - name: Upload Diagnostics + if: always() && steps.dump_diagnostics.outcome == 'success' + uses: actions/upload-artifact@v2 + continue-on-error: true + with: + name: ${{ matrix.test-name }}-${{ matrix.distro }}-diagnostics + path: ${{ github.workspace }}/diagnostics + # template: .action_templates/jobs/e2e-success.yaml e2e-success: if: always() needs: [tests] runs-on: ubuntu-latest steps: - - name: Check E2E Result - if: needs.tests.result != 'success' - run: exit 1 + - name: Check E2E Result + if: needs.tests.result != 'success' + run: exit 1 diff --git a/scripts/git-hooks/pre-commit b/scripts/git-hooks/pre-commit index 39eda83df..7d2f5070c 100755 --- a/scripts/git-hooks/pre-commit +++ b/scripts/git-hooks/pre-commit @@ -78,7 +78,7 @@ function generate_github_actions(){ git add .github/workflows } -#generate_github_actions +generate_github_actions generate_crd go_imports black_formatting From 8592e9c28f6ecf0251aa106dd27cac724030cc2f Mon Sep 17 00:00:00 2001 From: FlorentinaSimlinger <60526092+FlorentinaSimlinger@users.noreply.github.com> Date: Mon, 28 Jun 2021 12:44:23 +0100 Subject: [PATCH 334/790] CLOUDP-82702: Reduce operator role permissions (#599) Co-authored-by: Rajdeep Das --- config/rbac/role.yaml | 40 +---------------------------------- deploy/clusterwide/role.yaml | 41 ++---------------------------------- docs/RELEASE_NOTES.md | 1 + 3 files changed, 4 insertions(+), 78 deletions(-) diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 4cb3c74c1..6a9c42070 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -2,7 +2,6 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: - creationTimestamp: null name: mongodb-kubernetes-operator rules: - apiGroups: @@ -10,10 +9,6 @@ rules: resources: - pods - services - - services/finalizers - - endpoints - - persistentvolumeclaims - - events - configmaps - secrets verbs: @@ -27,9 +22,6 @@ rules: - apiGroups: - apps resources: - - deployments - - daemonsets - - replicasets - statefulsets verbs: - create @@ -39,34 +31,6 @@ rules: - patch - update - watch -- apiGroups: - - monitoring.coreos.com - resources: - - servicemonitors - verbs: - - get - - create -- apiGroups: - - apps - resourceNames: - - mongodb-kubernetes-operator - resources: - - deployments/finalizers - verbs: - - update -- apiGroups: - - "" - resources: - - pods - verbs: - - get -- apiGroups: - - apps - resources: - - replicasets - - deployments - verbs: - - get - apiGroups: - mongodbcommunity.mongodb.com resources: @@ -75,10 +39,8 @@ rules: - mongodbcommunity/spec - mongodbcommunity/finalizers verbs: - - create - - delete - get - - list - patch + - list - update - watch diff --git a/deploy/clusterwide/role.yaml b/deploy/clusterwide/role.yaml index 6a134d463..bddf8378e 100644 --- a/deploy/clusterwide/role.yaml +++ b/deploy/clusterwide/role.yaml @@ -1,7 +1,6 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - creationTimestamp: null name: mongodb-kubernetes-operator rules: - apiGroups: @@ -9,10 +8,6 @@ rules: resources: - pods - services - - services/finalizers - - endpoints - - persistentvolumeclaims - - events - configmaps - secrets verbs: @@ -26,9 +21,6 @@ rules: - apiGroups: - apps resources: - - deployments - - daemonsets - - replicasets - statefulsets verbs: - create @@ -38,45 +30,16 @@ rules: - patch - update - watch -- apiGroups: - - monitoring.coreos.com - resources: - - servicemonitors - verbs: - - get - - create -- apiGroups: - - apps - resourceNames: - - mongodb-kubernetes-operator - resources: - - deployments/finalizers - verbs: - - update -- apiGroups: - - "" - resources: - - pods - verbs: - - get -- apiGroups: - - apps - resources: - - replicasets - - deployments - verbs: - - get - apiGroups: - mongodbcommunity.mongodb.com resources: - mongodbcommunity - mongodbcommunity/status - mongodbcommunity/spec + - mongodbcommunity/finalizers verbs: - - create - - delete - get - - list - patch + - list - update - watch diff --git a/docs/RELEASE_NOTES.md b/docs/RELEASE_NOTES.md index 6b9872668..d1fa02ef3 100644 --- a/docs/RELEASE_NOTES.md +++ b/docs/RELEASE_NOTES.md @@ -4,6 +4,7 @@ - Changes - Members of a Replica Set can be configured as arbiters. + - Reduce the number of permissions for operator role. ## Updated Image Tags From 00cea65c94203b5686efb044e751cb8e4cb466c2 Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Mon, 28 Jun 2021 13:53:08 +0100 Subject: [PATCH 335/790] Fix Dependabot PRs (#596) --- .action_templates/e2e-fork-template.yaml | 5 +++-- .action_templates/e2e-pr-template.yaml | 3 ++- ...-pull-request-target.yaml => pull-request-target.yaml} | 2 +- .github/workflows/e2e-fork.yml | 8 ++++---- .github/workflows/e2e.yml | 3 ++- requirements.txt | 2 +- 6 files changed, 13 insertions(+), 10 deletions(-) rename .action_templates/events/{labeled-pull-request-target.yaml => pull-request-target.yaml} (84%) diff --git a/.action_templates/e2e-fork-template.yaml b/.action_templates/e2e-fork-template.yaml index 7c68f2674..e14111f7e 100644 --- a/.action_templates/e2e-fork-template.yaml +++ b/.action_templates/e2e-fork-template.yaml @@ -1,7 +1,8 @@ name: Run E2E Fork jobs: - template: setup - if: github.event.pull_request.head.repo.full_name != 'mongodb/mongodb-kubernetes-operator' && contains(github.event.pull_request.labels.*.name, 'safe-to-test') + # dependabot gets a read only github token, and so must use pull_request_target instead of pull_request. + if: github.actor == 'dependabot[bot]' || (github.event.pull_request.head.repo.full_name != github.repository && contains(github.event.pull_request.labels.*.name, 'safe-to-test')) steps: - template: cancel-previous - template: checkout-fork @@ -22,4 +23,4 @@ jobs: - template: e2e-success events: - - template: labeled-pull-request-target + - template: pull-request-target diff --git a/.action_templates/e2e-pr-template.yaml b/.action_templates/e2e-pr-template.yaml index 4ee419ae7..ae5ab7ef9 100644 --- a/.action_templates/e2e-pr-template.yaml +++ b/.action_templates/e2e-pr-template.yaml @@ -1,7 +1,8 @@ name: Run E2E jobs: - template: setup - if: github.event.pull_request.head.repo.full_name == 'mongodb/mongodb-kubernetes-operator' + # run on master, or if a PR is being created from a branch. + if: github.ref == 'refs/heads/master' || (github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]') steps: - template: cancel-previous - template: checkout diff --git a/.action_templates/events/labeled-pull-request-target.yaml b/.action_templates/events/pull-request-target.yaml similarity index 84% rename from .action_templates/events/labeled-pull-request-target.yaml rename to .action_templates/events/pull-request-target.yaml index 1e7743cd8..9d412d81a 100644 --- a/.action_templates/events/labeled-pull-request-target.yaml +++ b/.action_templates/events/pull-request-target.yaml @@ -1,6 +1,6 @@ # pull_request_target means that the secrets of this repo will be used. pull_request_target: - types: [labeled] + types: [labeled, opened] branches: - master paths-ignore: diff --git a/.github/workflows/e2e-fork.yml b/.github/workflows/e2e-fork.yml index 91e456774..f9227e5bb 100644 --- a/.github/workflows/e2e-fork.yml +++ b/.github/workflows/e2e-fork.yml @@ -9,9 +9,9 @@ name: Run E2E Fork on: - # template: .action_templates/events/labeled-pull-request-target.yaml + # template: .action_templates/events/pull-request-target.yaml pull_request_target: - types: [labeled] + types: [labeled, opened] branches: - master paths-ignore: @@ -30,8 +30,8 @@ jobs: - pipeline-argument: agent-ubi - pipeline-argument: agent-ubuntu - pipeline-argument: e2e - if: github.event.pull_request.head.repo.full_name != 'mongodb/mongodb-kubernetes-operator' - && contains(github.event.pull_request.labels.*.name, 'safe-to-test') + if: github.actor == 'dependabot[bot]' || (github.event.pull_request.head.repo.full_name + != github.repository && contains(github.event.pull_request.labels.*.name, 'safe-to-test')) steps: # template: .action_templates/steps/cancel-previous.yaml - name: Cancel Previous Runs diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 14fe77af0..2c69f3b65 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -37,7 +37,8 @@ jobs: - pipeline-argument: agent-ubi - pipeline-argument: agent-ubuntu - pipeline-argument: e2e - if: github.event.pull_request.head.repo.full_name == 'mongodb/mongodb-kubernetes-operator' + if: github.ref == 'refs/heads/master' || (github.event.pull_request.head.repo.full_name + == github.repository && github.actor != 'dependabot[bot]') steps: # template: .action_templates/steps/cancel-previous.yaml - name: Cancel Previous Runs diff --git a/requirements.txt b/requirements.txt index 2bb4843b5..f1ff4a184 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ git+https://github.com/mongodb/sonar@0.0.11#egg=sonar -git+https://github.com/chatton/GithubActionTemplates@v0.0.2 +git+https://github.com/chatton/GithubActionTemplates@v0.0.4 docker==4.3.1 kubernetes==11.0.0 jinja2==2.11.3 From 05ef852e61ad8e3ee724e86dd0dcd9ba6913d7e5 Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Mon, 28 Jun 2021 15:28:14 +0100 Subject: [PATCH 336/790] Add github context action (#600) --- .action_templates/e2e-fork-template.yaml | 1 + .action_templates/e2e-pr-template.yaml | 1 + .action_templates/e2e-single-template.yaml | 1 + .action_templates/jobs/display-github-context.yaml | 8 ++++++++ .github/workflows/e2e-dispatch.yml | 9 +++++++++ .github/workflows/e2e-fork.yml | 9 +++++++++ .github/workflows/e2e.yml | 9 +++++++++ 7 files changed, 38 insertions(+) create mode 100644 .action_templates/jobs/display-github-context.yaml diff --git a/.action_templates/e2e-fork-template.yaml b/.action_templates/e2e-fork-template.yaml index e14111f7e..da7057f73 100644 --- a/.action_templates/e2e-fork-template.yaml +++ b/.action_templates/e2e-fork-template.yaml @@ -1,5 +1,6 @@ name: Run E2E Fork jobs: + - template: display-github-context - template: setup # dependabot gets a read only github token, and so must use pull_request_target instead of pull_request. if: github.actor == 'dependabot[bot]' || (github.event.pull_request.head.repo.full_name != github.repository && contains(github.event.pull_request.labels.*.name, 'safe-to-test')) diff --git a/.action_templates/e2e-pr-template.yaml b/.action_templates/e2e-pr-template.yaml index ae5ab7ef9..c151b195d 100644 --- a/.action_templates/e2e-pr-template.yaml +++ b/.action_templates/e2e-pr-template.yaml @@ -1,5 +1,6 @@ name: Run E2E jobs: + - template: display-github-context - template: setup # run on master, or if a PR is being created from a branch. if: github.ref == 'refs/heads/master' || (github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]') diff --git a/.action_templates/e2e-single-template.yaml b/.action_templates/e2e-single-template.yaml index 6c21d9cdc..03e9c511e 100644 --- a/.action_templates/e2e-single-template.yaml +++ b/.action_templates/e2e-single-template.yaml @@ -1,5 +1,6 @@ name: Run Single E2E jobs: + - template: display-github-context - template: setup steps: - template: checkout diff --git a/.action_templates/jobs/display-github-context.yaml b/.action_templates/jobs/display-github-context.yaml new file mode 100644 index 000000000..37ecb1972 --- /dev/null +++ b/.action_templates/jobs/display-github-context.yaml @@ -0,0 +1,8 @@ +action-context: + if: always() + runs-on: ubuntu-latest + steps: + - name: Dump GitHub context + env: + GITHUB_CONTEXT: ${{ toJSON(github) }} + run: echo "$GITHUB_CONTEXT" diff --git a/.github/workflows/e2e-dispatch.yml b/.github/workflows/e2e-dispatch.yml index 510dfa86b..4c1b355ae 100644 --- a/.github/workflows/e2e-dispatch.yml +++ b/.github/workflows/e2e-dispatch.yml @@ -24,6 +24,15 @@ on: required: true default: 'false' jobs: + # template: .action_templates/jobs/display-github-context.yaml + action-context: + if: always() + runs-on: ubuntu-latest + steps: + - name: Dump GitHub context + env: + GITHUB_CONTEXT: ${{ toJSON(github) }} + run: echo "$GITHUB_CONTEXT" # template: .action_templates/jobs/setup.yaml setup: runs-on: ubuntu-latest diff --git a/.github/workflows/e2e-fork.yml b/.github/workflows/e2e-fork.yml index f9227e5bb..f40ac6f01 100644 --- a/.github/workflows/e2e-fork.yml +++ b/.github/workflows/e2e-fork.yml @@ -17,6 +17,15 @@ on: paths-ignore: - docs/** jobs: + # template: .action_templates/jobs/display-github-context.yaml + action-context: + if: always() + runs-on: ubuntu-latest + steps: + - name: Dump GitHub context + env: + GITHUB_CONTEXT: ${{ toJSON(github) }} + run: echo "$GITHUB_CONTEXT" # template: .action_templates/jobs/setup.yaml setup: runs-on: ubuntu-latest diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 2c69f3b65..87852bc65 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -24,6 +24,15 @@ on: # template: .action_templates/events/workflow-dispatch.yaml workflow_dispatch: {} jobs: + # template: .action_templates/jobs/display-github-context.yaml + action-context: + if: always() + runs-on: ubuntu-latest + steps: + - name: Dump GitHub context + env: + GITHUB_CONTEXT: ${{ toJSON(github) }} + run: echo "$GITHUB_CONTEXT" # template: .action_templates/jobs/setup.yaml setup: runs-on: ubuntu-latest From 16d0fdc6b7dc4fc5e14e49683affe2509fffd1d6 Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Mon, 28 Jun 2021 15:58:56 +0100 Subject: [PATCH 337/790] Use dependencies label for dependabot (#601) --- .action_templates/e2e-fork-template.yaml | 2 +- .action_templates/events/pull-request-target.yaml | 2 +- .github/workflows/e2e-fork.yml | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.action_templates/e2e-fork-template.yaml b/.action_templates/e2e-fork-template.yaml index da7057f73..df3fde82e 100644 --- a/.action_templates/e2e-fork-template.yaml +++ b/.action_templates/e2e-fork-template.yaml @@ -3,7 +3,7 @@ jobs: - template: display-github-context - template: setup # dependabot gets a read only github token, and so must use pull_request_target instead of pull_request. - if: github.actor == 'dependabot[bot]' || (github.event.pull_request.head.repo.full_name != github.repository && contains(github.event.pull_request.labels.*.name, 'safe-to-test')) + if: contains(github.event.pull_request.labels.*.name, 'dependencies') || contains(github.event.pull_request.labels.*.name, 'safe-to-test') steps: - template: cancel-previous - template: checkout-fork diff --git a/.action_templates/events/pull-request-target.yaml b/.action_templates/events/pull-request-target.yaml index 9d412d81a..1e7743cd8 100644 --- a/.action_templates/events/pull-request-target.yaml +++ b/.action_templates/events/pull-request-target.yaml @@ -1,6 +1,6 @@ # pull_request_target means that the secrets of this repo will be used. pull_request_target: - types: [labeled, opened] + types: [labeled] branches: - master paths-ignore: diff --git a/.github/workflows/e2e-fork.yml b/.github/workflows/e2e-fork.yml index f40ac6f01..02e1bfb5f 100644 --- a/.github/workflows/e2e-fork.yml +++ b/.github/workflows/e2e-fork.yml @@ -11,7 +11,7 @@ name: Run E2E Fork on: # template: .action_templates/events/pull-request-target.yaml pull_request_target: - types: [labeled, opened] + types: [labeled] branches: - master paths-ignore: @@ -39,8 +39,8 @@ jobs: - pipeline-argument: agent-ubi - pipeline-argument: agent-ubuntu - pipeline-argument: e2e - if: github.actor == 'dependabot[bot]' || (github.event.pull_request.head.repo.full_name - != github.repository && contains(github.event.pull_request.labels.*.name, 'safe-to-test')) + if: contains(github.event.pull_request.labels.*.name, 'dependencies') || contains(github.event.pull_request.labels.*.name, + 'safe-to-test') steps: # template: .action_templates/steps/cancel-previous.yaml - name: Cancel Previous Runs From 6a9b587a8a42b9915dfb8b2b9e212a90f1dddfe3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Jun 2021 17:07:42 +0100 Subject: [PATCH 338/790] Bump k8s.io/apimachinery from 0.21.1 to 0.21.2 (#583) --- go.mod | 2 +- go.sum | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index bfaa64c8a..519af65be 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( go.mongodb.org/mongo-driver v1.5.3 go.uber.org/zap v1.17.0 k8s.io/api v0.21.1 - k8s.io/apimachinery v0.21.1 + k8s.io/apimachinery v0.21.2 k8s.io/client-go v0.21.1 sigs.k8s.io/controller-runtime v0.9.0 sigs.k8s.io/yaml v1.2.0 diff --git a/go.sum b/go.sum index a0798fcb7..c963c853f 100644 --- a/go.sum +++ b/go.sum @@ -614,6 +614,7 @@ golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40 h1:JWgyZ1qgdTaF3N3oxC+MdTV7qvEEgHo3otj+HB5CM7Q= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -788,8 +789,9 @@ k8s.io/api v0.21.1 h1:94bbZ5NTjdINJEdzOkpS4vdPhkb1VFpTYC9zh43f75c= k8s.io/api v0.21.1/go.mod h1:FstGROTmsSHBarKc8bylzXih8BLNYTiS3TZcsoEDg2s= k8s.io/apiextensions-apiserver v0.21.1 h1:AA+cnsb6w7SZ1vD32Z+zdgfXdXY8X9uGX5bN6EoPEIo= k8s.io/apiextensions-apiserver v0.21.1/go.mod h1:KESQFCGjqVcVsZ9g0xX5bacMjyX5emuWcS2arzdEouA= -k8s.io/apimachinery v0.21.1 h1:Q6XuHGlj2xc+hlMCvqyYfbv3H7SRGn2c8NycxJquDVs= k8s.io/apimachinery v0.21.1/go.mod h1:jbreFvJo3ov9rj7eWT7+sYiRx+qZuCYXwWT1bcDswPY= +k8s.io/apimachinery v0.21.2 h1:vezUc/BHqWlQDnZ+XkrpXSmnANSLbpnlpwo0Lhk0gpc= +k8s.io/apimachinery v0.21.2/go.mod h1:CdTY8fU/BlvAbJ2z/8kBwimGki5Zp8/fbVuLY8gJumM= k8s.io/apiserver v0.21.1/go.mod h1:nLLYZvMWn35glJ4/FZRhzLG/3MPxAaZTgV4FJZdr+tY= k8s.io/client-go v0.21.1 h1:bhblWYLZKUu+pm50plvQF8WpY6TXdRRtcS/K9WauOj4= k8s.io/client-go v0.21.1/go.mod h1:/kEw4RgW+3xnBGzvp9IWxKSNA+lXn3A7AuH3gdOAzLs= From 87244a47bc8b7bb9c3deffb0160e588463ae22e9 Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Wed, 30 Jun 2021 12:30:27 +0100 Subject: [PATCH 339/790] Use "run" script instead of GH Action to install kind (#609) --- .action_templates/steps/setup-kind-cluster.yaml | 7 ++++--- .github/workflows/e2e-dispatch.yml | 7 ++++--- .github/workflows/e2e-fork.yml | 7 ++++--- .github/workflows/e2e.yml | 7 ++++--- 4 files changed, 16 insertions(+), 12 deletions(-) diff --git a/.action_templates/steps/setup-kind-cluster.yaml b/.action_templates/steps/setup-kind-cluster.yaml index 43faf02ef..0f97cf1be 100644 --- a/.action_templates/steps/setup-kind-cluster.yaml +++ b/.action_templates/steps/setup-kind-cluster.yaml @@ -1,7 +1,8 @@ - name: Setup Kind Cluster - uses: engineerd/setup-kind@v0.5.0 - with: - version: "v0.8.1" + run: | + curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.11.1/kind-linux-amd64 + chmod +x ./kind + ./kind create cluster - name: Install CRD run: kubectl apply -f config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml diff --git a/.github/workflows/e2e-dispatch.yml b/.github/workflows/e2e-dispatch.yml index 4c1b355ae..9f040340b 100644 --- a/.github/workflows/e2e-dispatch.yml +++ b/.github/workflows/e2e-dispatch.yml @@ -98,9 +98,10 @@ jobs: run: pip install -r requirements.txt # template: .action_templates/steps/setup-kind-cluster.yaml - name: Setup Kind Cluster - uses: engineerd/setup-kind@v0.5.0 - with: - version: v0.8.1 + run: | + curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.11.1/kind-linux-amd64 + chmod +x ./kind + ./kind create cluster - name: Install CRD run: kubectl apply -f config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml diff --git a/.github/workflows/e2e-fork.yml b/.github/workflows/e2e-fork.yml index 02e1bfb5f..37901714d 100644 --- a/.github/workflows/e2e-fork.yml +++ b/.github/workflows/e2e-fork.yml @@ -202,9 +202,10 @@ jobs: run: pip install -r requirements.txt # template: .action_templates/steps/setup-kind-cluster.yaml - name: Setup Kind Cluster - uses: engineerd/setup-kind@v0.5.0 - with: - version: v0.8.1 + run: | + curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.11.1/kind-linux-amd64 + chmod +x ./kind + ./kind create cluster if: steps.last_run_status.outputs.last_run_status != 'success' - name: Install CRD diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 87852bc65..edbd6ceda 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -203,9 +203,10 @@ jobs: run: pip install -r requirements.txt # template: .action_templates/steps/setup-kind-cluster.yaml - name: Setup Kind Cluster - uses: engineerd/setup-kind@v0.5.0 - with: - version: v0.8.1 + run: | + curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.11.1/kind-linux-amd64 + chmod +x ./kind + ./kind create cluster if: steps.last_run_status.outputs.last_run_status != 'success' - name: Install CRD From 88593ea97da949a9f9a24e7ce60ed91a53d8782d Mon Sep 17 00:00:00 2001 From: FlorentinaSimlinger <60526092+FlorentinaSimlinger@users.noreply.github.com> Date: Wed, 30 Jun 2021 16:42:22 +0100 Subject: [PATCH 340/790] CLOUDP-91632: E2E tests should fail fast if the operator pod doesn't reach READY state (#607) --- test/e2e/setup/setup.go | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/test/e2e/setup/setup.go b/test/e2e/setup/setup.go index 96e08f373..d59b57c80 100644 --- a/test/e2e/setup/setup.go +++ b/test/e2e/setup/setup.go @@ -7,6 +7,7 @@ import ( "io/ioutil" "path" "testing" + "time" "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/envvar" @@ -18,6 +19,8 @@ import ( rbacv1 "k8s.io/api/rbac/v1" apiErrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/yaml" @@ -151,10 +154,12 @@ func deployOperator() error { if err := buildKubernetesResourceFromYamlFile(path.Join(roleDir(), "role_binding.yaml"), &rbacv1.RoleBinding{}, withNamespace(testConfig.namespace)); err != nil { return errors.Errorf("error building operator role binding: %s", err) } - fmt.Println("Successfully created the operator Role Binding") + + dep := &appsv1.Deployment{} + if err := buildKubernetesResourceFromYamlFile(path.Join(deployDir(), "manager.yaml"), - &appsv1.Deployment{}, + dep, withNamespace(testConfig.namespace), withOperatorImage(testConfig.operatorImage), withEnvVar("WATCH_NAMESPACE", watchNamespace), @@ -164,10 +169,36 @@ func deployOperator() error { ); err != nil { return errors.Errorf("error building operator deployment: %s", err) } + + if err := wait.PollImmediate(time.Second, 30*time.Second, hasDeploymentRequiredReplicas(dep)); err != nil { + return errors.New("error building operator deployment: the deployment does not have the required replicas") + } + fmt.Println("Successfully created the operator Deployment") return nil } +// hasDeploymentRequiredReplicas returns a condition function that indicates whether the given deployment +// currently has the required amount of replicas in the ready state as specified in spec.replicas +func hasDeploymentRequiredReplicas(dep *appsv1.Deployment) wait.ConditionFunc { + return func() (bool, error) { + err := e2eutil.TestClient.Get(context.TODO(), + types.NamespacedName{Name: dep.Name, + Namespace: e2eutil.OperatorNamespace}, + dep) + if err != nil { + if apiErrors.IsNotFound(err) { + return false, nil + } + return false, errors.Errorf("error getting operator deployment: %s", err) + } + if dep.Status.ReadyReplicas == *dep.Spec.Replicas { + return true, nil + } + return false, nil + } +} + // buildKubernetesResourceFromYamlFile will create the kubernetes resource defined in yamlFilePath. All of the functional options // provided will be applied before creation. func buildKubernetesResourceFromYamlFile(yamlFilePath string, obj client.Object, options ...func(obj runtime.Object)) error { From 10b4411ceb6dd9c4287cdd810745f2ceb22990e6 Mon Sep 17 00:00:00 2001 From: Rajdeep Das Date: Thu, 1 Jul 2021 17:09:38 +0200 Subject: [PATCH 341/790] Move the CreateorUpdateService to community repo (#611) --- pkg/kube/service/service.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/pkg/kube/service/service.go b/pkg/kube/service/service.go index 72da6fa1c..7270b370c 100644 --- a/pkg/kube/service/service.go +++ b/pkg/kube/service/service.go @@ -93,3 +93,27 @@ func Merge(dest corev1.Service, source corev1.Service) corev1.Service { dest.Spec.ExternalTrafficPolicy = source.Spec.ExternalTrafficPolicy return dest } + +// CreateOrUpdateService will create or update a service in Kubernetes. +func CreateOrUpdateService(getUpdateCreator GetUpdateCreator, desiredService corev1.Service) error { + namespacedName := types.NamespacedName{Namespace: desiredService.ObjectMeta.Namespace, Name: desiredService.ObjectMeta.Name} + existingService, err := getUpdateCreator.GetService(namespacedName) + + if err != nil { + if apiErrors.IsNotFound(err) { + err = getUpdateCreator.CreateService(desiredService) + if err != nil { + return err + } + } else { + return err + } + } else { + mergedService := Merge(existingService, desiredService) + err = getUpdateCreator.UpdateService(mergedService) + if err != nil { + return err + } + } + return nil +} From 4f74d5721c82d5bb2664d53e1307051ae6b56560 Mon Sep 17 00:00:00 2001 From: killian2k Date: Fri, 2 Jul 2021 08:11:36 +0200 Subject: [PATCH 342/790] Cloudp 78288 scram sha 1 user (#598) * Modified the changes in the release notes and incremented version to 0.6.3 also in the release notes * Added the feature but does not work completely * Now the unit tests are passing * Added the e2e tests and they pass * Clean a bit the code * cleaning code * Modify the tests * correct a comment * Add the tests and modify the release notes * Clean code for PR * Update controllers/validation/validation.go Co-authored-by: Cian Hatton * updates based on PR comments * simplified the bloc * modified based on PR review * small change with the order of two var declarations * Correct the unit tests * Correct yaml files Co-authored-by: Cian Hatton --- api/v1/mongodbcommunity_types.go | 39 +++++- ...ommunity.mongodb.com_mongodbcommunity.yaml | 2 + controllers/replicaset_controller_test.go | 8 ++ .../testdata/custom_storage_class.yaml | 3 + controllers/testdata/tolerations_example.yaml | 3 + .../testdata/volume_claim_templates_mdb.yaml | 3 + controllers/validation/validation.go | 37 +++-- docs/RELEASE_NOTES.md | 1 + .../scram/scram_enabler_test.go | 60 ++++++++- .../replica_set_authentication_test.go | 126 ++++++++++++++++++ ...tatefulset_arbitrary_config_update_test.go | 4 +- test/e2e/tlstests/tlstests.go | 8 +- test/e2e/util/mongotester/mongotester.go | 24 ++++ 13 files changed, 297 insertions(+), 21 deletions(-) create mode 100644 test/e2e/replica_set_authentication/replica_set_authentication_test.go diff --git a/api/v1/mongodbcommunity_types.go b/api/v1/mongodbcommunity_types.go index 8c9250115..f4c488d54 100644 --- a/api/v1/mongodbcommunity_types.go +++ b/api/v1/mongodbcommunity_types.go @@ -40,6 +40,11 @@ const ( defaultPasswordKey = "password" ) +// SCRAM-SHA-256 and SCRAM-SHA-1 are the supported auth modes. +const ( + defaultMode AuthMode = "SCRAM-SHA-256" +) + // MongoDBCommunitySpec defines the desired state of MongoDB type MongoDBCommunitySpec struct { // Members is the number of members in the replica set @@ -346,9 +351,22 @@ type Authentication struct { IgnoreUnknownUsers *bool `json:"ignoreUnknownUsers,omitempty"` } -// +kubebuilder:validation:Enum=SCRAM +// +kubebuilder:validation:Enum=SCRAM;SCRAM-SHA-256;SCRAM-SHA-1 type AuthMode string +// ConvertAuthModeToAuthMechanism acts as a map but is immutable. It allows users to use different labels to describe the +// same authentication mode. +func ConvertAuthModeToAuthMechanism(authModeLabel AuthMode) string { + switch authModeLabel { + case "SCRAM", "SCRAM-SHA-256": + return scram.Sha256 + case "SCRAM-SHA-1": + return scram.Sha1 + default: + return "" + } +} + // MongoDBCommunityStatus defines the observed state of MongoDB type MongoDBCommunityStatus struct { MongoURI string `json:"mongoUri"` @@ -402,12 +420,27 @@ func (m MongoDBCommunity) GetScramOptions() scram.Options { ignoreUnknownUsers = *m.Spec.Security.Authentication.IgnoreUnknownUsers } + authModes := m.Spec.Security.Authentication.Modes + defaultAuthMechanism := ConvertAuthModeToAuthMechanism(defaultMode) + autoAuthMechanism := ConvertAuthModeToAuthMechanism(authModes[0]) + + authMechanisms := make([]string, len(authModes)) + + for i, authMode := range authModes { + if authMech := ConvertAuthModeToAuthMechanism(authMode); authMech != "" { + authMechanisms[i] = authMech + if authMech == defaultAuthMechanism { + autoAuthMechanism = defaultAuthMechanism + } + } + } + return scram.Options{ AuthoritativeSet: !ignoreUnknownUsers, KeyFile: scram.AutomationAgentKeyFilePathInContainer, - AutoAuthMechanisms: []string{scram.Sha256}, + AutoAuthMechanisms: authMechanisms, AgentName: scram.AgentName, - AutoAuthMechanism: scram.Sha256, + AutoAuthMechanism: autoAuthMechanism, } } diff --git a/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml b/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml index 12695642a..5adb08761 100644 --- a/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml +++ b/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml @@ -91,6 +91,8 @@ spec: items: enum: - SCRAM + - SCRAM-SHA-256 + - SCRAM-SHA-1 type: string type: array required: diff --git a/controllers/replicaset_controller_test.go b/controllers/replicaset_controller_test.go index 5ed2c55f0..804f5f4d9 100644 --- a/controllers/replicaset_controller_test.go +++ b/controllers/replicaset_controller_test.go @@ -53,6 +53,11 @@ func newTestReplicaSet() mdbv1.MongoDBCommunity { Spec: mdbv1.MongoDBCommunitySpec{ Members: 3, Version: "4.2.2", + Security: mdbv1.Security{ + Authentication: mdbv1.Authentication{ + Modes: []mdbv1.AuthMode{"SCRAM"}, + }, + }, }, } } @@ -88,6 +93,9 @@ func newTestReplicaSetWithTLS() mdbv1.MongoDBCommunity { Members: 3, Version: "4.2.2", Security: mdbv1.Security{ + Authentication: mdbv1.Authentication{ + Modes: []mdbv1.AuthMode{"SCRAM"}, + }, TLS: mdbv1.TLS{ Enabled: true, CaConfigMap: mdbv1.LocalObjectReference{ diff --git a/controllers/testdata/custom_storage_class.yaml b/controllers/testdata/custom_storage_class.yaml index d3c5c59cc..9740ce4ec 100644 --- a/controllers/testdata/custom_storage_class.yaml +++ b/controllers/testdata/custom_storage_class.yaml @@ -6,6 +6,9 @@ spec: members: 3 type: ReplicaSet version: "4.2.6" + security: + authentication: + modes: ["SCRAM"] statefulSet: spec: volumeClaimTemplates: diff --git a/controllers/testdata/tolerations_example.yaml b/controllers/testdata/tolerations_example.yaml index 88e808ec3..b8c303ae7 100644 --- a/controllers/testdata/tolerations_example.yaml +++ b/controllers/testdata/tolerations_example.yaml @@ -6,6 +6,9 @@ spec: members: 3 type: ReplicaSet version: "4.2.6" + security: + authentication: + modes: ["SCRAM"] statefulSet: spec: template: diff --git a/controllers/testdata/volume_claim_templates_mdb.yaml b/controllers/testdata/volume_claim_templates_mdb.yaml index 797cc4b8b..1e20915b9 100644 --- a/controllers/testdata/volume_claim_templates_mdb.yaml +++ b/controllers/testdata/volume_claim_templates_mdb.yaml @@ -6,6 +6,9 @@ spec: members: 3 type: ReplicaSet version: "4.2.6" + security: + authentication: + modes: ["SCRAM"] statefulSet: spec: volumeClaimTemplates: diff --git a/controllers/validation/validation.go b/controllers/validation/validation.go index 7be3707aa..6890d3e17 100644 --- a/controllers/validation/validation.go +++ b/controllers/validation/validation.go @@ -11,15 +11,7 @@ import ( // ValidateInitalSpec checks if the resource initial Spec is valid func ValidateInitalSpec(mdb mdbv1.MongoDBCommunity) error { - if err := validateUsers(mdb); err != nil { - return err - } - - if err := validateArbiterSpec(mdb); err != nil { - return err - } - - return nil + return validateSpec(mdb) } // ValidateUpdate validates that the new Spec, corresponding to the existing one is still valid @@ -27,7 +19,11 @@ func ValidateUpdate(mdb mdbv1.MongoDBCommunity, oldSpec mdbv1.MongoDBCommunitySp if oldSpec.Security.TLS.Enabled && !mdb.Spec.Security.TLS.Enabled { return errors.New("TLS can't be set to disabled after it has been enabled") } + return validateSpec(mdb) +} +// validateSpec validates the specs of the given resource definition. +func validateSpec(mdb mdbv1.MongoDBCommunity) error { if err := validateUsers(mdb); err != nil { return err } @@ -36,6 +32,10 @@ func ValidateUpdate(mdb mdbv1.MongoDBCommunity, oldSpec mdbv1.MongoDBCommunitySp return err } + if err := validateAuthModeSpec(mdb); err != nil { + return err + } + return nil } @@ -76,3 +76,22 @@ func validateArbiterSpec(mdb mdbv1.MongoDBCommunity) error { return nil } + +// validateAuthModeSpec checks that the list of modes does not contain duplicates. +func validateAuthModeSpec(mdb mdbv1.MongoDBCommunity) error { + allModes := mdb.Spec.Security.Authentication.Modes + + // Check that no auth is defined more than once + mapModes := make(map[mdbv1.AuthMode]struct{}) + for i, mode := range allModes { + if value := mdbv1.ConvertAuthModeToAuthMechanism(mode); value == "" { + return fmt.Errorf("unexpected value (%q) defined for supported authentication modes", value) + } + mapModes[allModes[i]] = struct{}{} + } + if len(mapModes) != len(allModes) { + return fmt.Errorf("some authentication modes are declared twice or more") + } + + return nil +} diff --git a/docs/RELEASE_NOTES.md b/docs/RELEASE_NOTES.md index d1fa02ef3..d6abb63c5 100644 --- a/docs/RELEASE_NOTES.md +++ b/docs/RELEASE_NOTES.md @@ -5,6 +5,7 @@ - Changes - Members of a Replica Set can be configured as arbiters. - Reduce the number of permissions for operator role. + - Support SHA-1 as an authentication method. ## Updated Image Tags diff --git a/pkg/authentication/scram/scram_enabler_test.go b/pkg/authentication/scram/scram_enabler_test.go index 869418bde..90bd351ae 100644 --- a/pkg/authentication/scram/scram_enabler_test.go +++ b/pkg/authentication/scram/scram_enabler_test.go @@ -8,15 +8,16 @@ import ( ) func TestScramAutomationConfig(t *testing.T) { + + // Case 1: Both SHA-256 and SHA-1 auth := automationconfig.Auth{} opts := Options{ AuthoritativeSet: false, KeyFile: AutomationAgentKeyFilePathInContainer, - AutoAuthMechanisms: []string{Sha256}, + AutoAuthMechanisms: []string{Sha256, Sha1}, AgentName: "mms-automation", AutoAuthMechanism: Sha256, } - err := configureScramInAutomationConfig(&auth, "password", "keyfilecontents", []automationconfig.MongoDBUser{}, opts) assert.NoError(t, err) @@ -24,6 +25,33 @@ func TestScramAutomationConfig(t *testing.T) { assert.Equal(t, AgentName, auth.AutoUser) assert.Equal(t, "keyfilecontents", auth.Key) assert.Equal(t, "password", auth.AutoPwd) + assert.Equal(t, Sha256, auth.AutoAuthMechanism) + assert.Len(t, auth.DeploymentAuthMechanisms, 2) + assert.Len(t, auth.AutoAuthMechanisms, 2) + assert.Equal(t, []string{Sha256, Sha1}, auth.DeploymentAuthMechanisms) + assert.Equal(t, []string{Sha256, Sha1}, auth.AutoAuthMechanisms) + assert.Equal(t, AutomationAgentKeyFilePathInContainer, auth.KeyFile) + assert.Equal(t, automationAgentWindowsKeyFilePath, auth.KeyFileWindows) + }) + t.Run("Subsequent configuration doesn't add to deployment auth mechanisms", func(t *testing.T) { + err := configureScramInAutomationConfig(&auth, "password", "keyfilecontents", []automationconfig.MongoDBUser{}, opts) + assert.NoError(t, err) + assert.Equal(t, []string{Sha256, Sha1}, auth.DeploymentAuthMechanisms) + }) + + // Case 2: only SHA-256 + auth = automationconfig.Auth{} + opts = Options{ + AuthoritativeSet: false, + KeyFile: AutomationAgentKeyFilePathInContainer, + AutoAuthMechanisms: []string{Sha256}, + AgentName: "mms-automation", + AutoAuthMechanism: Sha256, + } + err = configureScramInAutomationConfig(&auth, "password", "keyfilecontents", []automationconfig.MongoDBUser{}, opts) + assert.NoError(t, err) + + t.Run("Authentication is correctly configured", func(t *testing.T) { assert.Equal(t, Sha256, auth.AutoAuthMechanism) assert.Len(t, auth.DeploymentAuthMechanisms, 1) assert.Len(t, auth.AutoAuthMechanisms, 1) @@ -32,10 +60,36 @@ func TestScramAutomationConfig(t *testing.T) { assert.Equal(t, AutomationAgentKeyFilePathInContainer, auth.KeyFile) assert.Equal(t, automationAgentWindowsKeyFilePath, auth.KeyFileWindows) }) - t.Run("Subsequent configuration doesn't add to deployment auth mechanisms", func(t *testing.T) { err := configureScramInAutomationConfig(&auth, "password", "keyfilecontents", []automationconfig.MongoDBUser{}, opts) assert.NoError(t, err) assert.Equal(t, []string{Sha256}, auth.DeploymentAuthMechanisms) }) + + // Case 1: only SHA-1 + auth = automationconfig.Auth{} + opts = Options{ + AuthoritativeSet: false, + KeyFile: AutomationAgentKeyFilePathInContainer, + AutoAuthMechanisms: []string{Sha1}, + AgentName: "mms-automation", + AutoAuthMechanism: Sha1, + } + err = configureScramInAutomationConfig(&auth, "password", "keyfilecontents", []automationconfig.MongoDBUser{}, opts) + assert.NoError(t, err) + + t.Run("Authentication is correctly configured", func(t *testing.T) { + assert.Equal(t, Sha1, auth.AutoAuthMechanism) + assert.Len(t, auth.DeploymentAuthMechanisms, 1) + assert.Len(t, auth.AutoAuthMechanisms, 1) + assert.Equal(t, []string{Sha1}, auth.DeploymentAuthMechanisms) + assert.Equal(t, []string{Sha1}, auth.AutoAuthMechanisms) + assert.Equal(t, AutomationAgentKeyFilePathInContainer, auth.KeyFile) + assert.Equal(t, automationAgentWindowsKeyFilePath, auth.KeyFileWindows) + }) + t.Run("Subsequent configuration doesn't add to deployment auth mechanisms", func(t *testing.T) { + err := configureScramInAutomationConfig(&auth, "password", "keyfilecontents", []automationconfig.MongoDBUser{}, opts) + assert.NoError(t, err) + assert.Equal(t, []string{Sha1}, auth.DeploymentAuthMechanisms) + }) } diff --git a/test/e2e/replica_set_authentication/replica_set_authentication_test.go b/test/e2e/replica_set_authentication/replica_set_authentication_test.go new file mode 100644 index 000000000..29a3dc8cc --- /dev/null +++ b/test/e2e/replica_set_authentication/replica_set_authentication_test.go @@ -0,0 +1,126 @@ +package replica_set_authentication + +import ( + "fmt" + "os" + "testing" + + mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" + . "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/util/mongotester" + "go.mongodb.org/mongo-driver/bson/primitive" + + e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" + "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/mongodbtests" + setup "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" +) + +func TestMain(m *testing.M) { + code, err := e2eutil.RunTest(m) + if err != nil { + fmt.Println(err) + } + os.Exit(code) +} + +func TestReplicaSetAuthentication(t *testing.T) { + ctx := setup.Setup(t) + defer ctx.Teardown() + + mdb, user := e2eutil.NewTestMongoDB(ctx, "mdb0", "") + pw, err := setup.GeneratePasswordForUser(ctx, user, "") + if err != nil { + t.Fatal(err) + } + t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) + + // Run all the possible configuration using sha256 or sha1 + t.Run("Auth test with SHA-256", testConfigAuthentication(mdb, user, pw)) + t.Run("Auth test with SHA-256 and SHA-1", testConfigAuthentication(mdb, user, pw, withSha1())) + t.Run("Auth test with SHA-256 (using label)", testConfigAuthentication(mdb, user, pw, withLabeledSha256())) + t.Run("Auth test with SHA-256 (using label) and SHA-1", testConfigAuthentication(mdb, user, pw, withSha1(), withLabeledSha256())) + t.Run("Auth test with SHA-1", testConfigAuthentication(mdb, user, pw, withSha1(), withoutSha256())) +} + +type authOptions struct { + sha256, sha1, useLabelForSha256 bool +} + +func withoutSha256() func(*authOptions) { + return func(opts *authOptions) { + opts.sha256 = false + } +} +func withLabeledSha256() func(*authOptions) { + return func(opts *authOptions) { + opts.sha256 = true + opts.useLabelForSha256 = true + } +} +func withSha1() func(*authOptions) { + return func(opts *authOptions) { + opts.sha1 = true + } +} + +// testConfigAuthentication run the tests using the autho ptions to update mdb and then checks that the resources are correctly configured +func testConfigAuthentication(mdb mdbv1.MongoDBCommunity, user mdbv1.MongoDBUser, pw string, allOptions ...func(*authOptions)) func(t *testing.T) { + return func(t *testing.T) { + + pickedOpts := authOptions{ + sha256: true, + } + for _, opt := range allOptions { + opt(&pickedOpts) + } + t.Logf("Config: use Sha256: %t (use label: %t), use Sha1: %t", pickedOpts.sha256, pickedOpts.useLabelForSha256, pickedOpts.sha1) + + enabledMechanisms := primitive.A{"SCRAM-SHA-256"} + var acceptedModes []mdbv1.AuthMode + if pickedOpts.sha256 { + if pickedOpts.useLabelForSha256 { + acceptedModes = append(acceptedModes, "SCRAM") + } else { + acceptedModes = append(acceptedModes, "SCRAM-SHA-256") + } + } + if pickedOpts.sha1 { + acceptedModes = append(acceptedModes, "SCRAM-SHA-1") + if pickedOpts.sha256 { + enabledMechanisms = primitive.A{"SCRAM-SHA-256", "SCRAM-SHA-1"} + } else { + enabledMechanisms = primitive.A{"SCRAM-SHA-1"} + } + } + + err := e2eutil.UpdateMongoDBResource(&mdb, func(db *mdbv1.MongoDBCommunity) { + db.Spec.Security.Authentication.Modes = acceptedModes + }) + if err != nil { + t.Fatal(err) + } + + tester, err := FromResource(t, mdb) + if err != nil { + t.Fatal(err) + } + + t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb)) + if pickedOpts.sha256 { + t.Run("Test Basic Connectivity with accepted auth", tester.ConnectivitySucceeds(WithScramWithAuth(user.Name, pw, "SCRAM-SHA-256"))) + } else { + t.Run("Test Basic Connectivity with unaccepted auth", tester.ConnectivityFails(WithScramWithAuth(user.Name, pw, "SCRAM-SHA-256"))) + } + if pickedOpts.sha1 { + t.Run("Test Basic Connectivity with accepted auth", tester.ConnectivitySucceeds(WithScramWithAuth(user.Name, pw, "SCRAM-SHA-1"))) + } else { + t.Run("Test Basic Connectivity with unaccepted auth", tester.ConnectivityFails(WithScramWithAuth(user.Name, pw, "SCRAM-SHA-1"))) + } + + if pickedOpts.sha256 { + t.Run("Ensure Authentication", tester.EnsureAuthenticationWithAuthIsConfigured(3, enabledMechanisms, WithScramWithAuth(user.Name, pw, "SCRAM-SHA-256"))) + } + if pickedOpts.sha1 { + t.Run("Ensure Authentication", tester.EnsureAuthenticationWithAuthIsConfigured(3, enabledMechanisms, WithScramWithAuth(user.Name, pw, "SCRAM-SHA-1"))) + } + } +} diff --git a/test/e2e/statefulset_arbitrary_config_update/statefulset_arbitrary_config_update_test.go b/test/e2e/statefulset_arbitrary_config_update/statefulset_arbitrary_config_update_test.go index 143c71b77..da47555b4 100644 --- a/test/e2e/statefulset_arbitrary_config_update/statefulset_arbitrary_config_update_test.go +++ b/test/e2e/statefulset_arbitrary_config_update/statefulset_arbitrary_config_update_test.go @@ -7,7 +7,7 @@ import ( "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/util/mongotester" - v1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" + mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/mongodbtests" setup "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" @@ -47,7 +47,7 @@ func TestStatefulSetArbitraryConfig(t *testing.T) { overrideSpec := mdb.Spec.StatefulSetConfiguration overrideSpec.SpecWrapper.Spec.Template.Spec.Containers[1].ReadinessProbe = &corev1.Probe{TimeoutSeconds: 100} - err = e2eutil.UpdateMongoDBResource(&mdb, func(mdb *v1.MongoDBCommunity) { mdb.Spec.StatefulSetConfiguration = overrideSpec }) + err = e2eutil.UpdateMongoDBResource(&mdb, func(mdb *mdbv1.MongoDBCommunity) { mdb.Spec.StatefulSetConfiguration = overrideSpec }) assert.NoError(t, err) diff --git a/test/e2e/tlstests/tlstests.go b/test/e2e/tlstests/tlstests.go index ee0f26839..543af8355 100644 --- a/test/e2e/tlstests/tlstests.go +++ b/test/e2e/tlstests/tlstests.go @@ -8,15 +8,15 @@ import ( "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/secret" - v1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" + mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" "github.com/stretchr/testify/assert" ) // EnableTLS will upgrade an existing TLS cluster to use TLS. -func EnableTLS(mdb *v1.MongoDBCommunity, optional bool) func(*testing.T) { +func EnableTLS(mdb *mdbv1.MongoDBCommunity, optional bool) func(*testing.T) { return func(t *testing.T) { - err := e2eutil.UpdateMongoDBResource(mdb, func(db *v1.MongoDBCommunity) { + err := e2eutil.UpdateMongoDBResource(mdb, func(db *mdbv1.MongoDBCommunity) { db.Spec.Security.TLS = e2eutil.NewTestTLSConfig(optional) }) if err != nil { @@ -25,7 +25,7 @@ func EnableTLS(mdb *v1.MongoDBCommunity, optional bool) func(*testing.T) { } } -func RotateCertificate(mdb *v1.MongoDBCommunity) func(*testing.T) { +func RotateCertificate(mdb *mdbv1.MongoDBCommunity) func(*testing.T) { return func(t *testing.T) { // Load new certificate and key testDataDir := e2eutil.TlsTestDataDir() diff --git a/test/e2e/util/mongotester/mongotester.go b/test/e2e/util/mongotester/mongotester.go index 592088c16..82c5bfc7b 100644 --- a/test/e2e/util/mongotester/mongotester.go +++ b/test/e2e/util/mongotester/mongotester.go @@ -126,6 +126,10 @@ func (m *Tester) ScramIsConfigured(tries int, opts ...OptionApplier) func(t *tes return m.hasAdminParameter("authenticationMechanisms", primitive.A{"SCRAM-SHA-256"}, tries, opts...) } +func (m *Tester) ScramWithAuthIsConfigured(tries int, enabledMechanisms primitive.A, opts ...OptionApplier) func(t *testing.T) { + return m.hasAdminParameter("authenticationMechanisms", enabledMechanisms, tries, opts...) +} + func (m *Tester) EnsureAuthenticationIsConfigured(tries int, opts ...OptionApplier) func(t *testing.T) { return func(t *testing.T) { t.Run("Ensure keyFile authentication is configured", m.HasKeyfileAuth(tries, opts...)) @@ -133,6 +137,13 @@ func (m *Tester) EnsureAuthenticationIsConfigured(tries int, opts ...OptionAppli } } +func (m *Tester) EnsureAuthenticationWithAuthIsConfigured(tries int, enabledMechanisms primitive.A, opts ...OptionApplier) func(t *testing.T) { + return func(t *testing.T) { + t.Run("Ensure keyFile authentication is configured", m.HasKeyfileAuth(tries, opts...)) + t.Run(fmt.Sprintf("%q is configured", enabledMechanisms), m.ScramWithAuthIsConfigured(tries, enabledMechanisms, opts...)) + } +} + func (m *Tester) HasTlsMode(tlsMode string, tries int, opts ...OptionApplier) func(t *testing.T) { return m.hasAdminParameter("sslMode", tlsMode, tries, opts...) } @@ -398,6 +409,19 @@ func WithScram(username, password string) OptionApplier { } } +func WithScramWithAuth(username, password string, authenticationMechanism string) OptionApplier { + return clientOptionAdder{ + option: &options.ClientOptions{ + Auth: &options.Credential{ + AuthMechanism: authenticationMechanism, + AuthSource: "admin", + Username: username, + Password: password, + }, + }, + } +} + // WithHosts configures the hosts of the deployment func WithHosts(hosts []string) OptionApplier { return clientOptionAdder{ From 9e07b958c1595baf4c9f54d9088cfa3b227c8dcf Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Fri, 2 Jul 2021 09:13:24 +0100 Subject: [PATCH 343/790] Use PyPi library instead of Github import for github-action-templates (#612) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f1ff4a184..80510bc9b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ git+https://github.com/mongodb/sonar@0.0.11#egg=sonar -git+https://github.com/chatton/GithubActionTemplates@v0.0.4 +github-action-templates==0.0.4 docker==4.3.1 kubernetes==11.0.0 jinja2==2.11.3 From 9cd22b34756df0a770aaa2024508a46ed7a908cf Mon Sep 17 00:00:00 2001 From: Rajdeep Das Date: Fri, 2 Jul 2021 18:05:48 +0200 Subject: [PATCH 344/790] Replace logw with logf - logw doesn't print the Spec fields (#613) --- controllers/replica_set_controller.go | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/controllers/replica_set_controller.go b/controllers/replica_set_controller.go index 0998babd1..735afd9c4 100644 --- a/controllers/replica_set_controller.go +++ b/controllers/replica_set_controller.go @@ -129,7 +129,7 @@ func (r ReplicaSetReconciler) Reconcile(ctx context.Context, request reconcile.R } r.log = zap.S().With("ReplicaSet", request.NamespacedName) - r.log.Infow("Reconciling MongoDB", "MongoDB.Spec", mdb.Spec, "MongoDB.Status", mdb.Status) + r.log.Infof("Reconciling MongoDB") r.log.Debug("Validating MongoDB.Spec") if err := r.validateSpec(mdb); err != nil { @@ -240,11 +240,11 @@ func (r ReplicaSetReconciler) Reconcile(ctx context.Context, request reconcile.R } if res.RequeueAfter > 0 || res.Requeue { - r.log.Infow("Requeuing reconciliation", "MongoDB.Spec:", mdb.Spec, "MongoDB.Status:", mdb.Status) + r.log.Info("Requeuing reconciliation") return res, nil } - r.log.Infow("Successfully finished reconciliation", "MongoDB.Spec:", mdb.Spec, "MongoDB.Status:", mdb.Status) + r.log.Infof("Successfully finished reconciliation, MongoDB.Spec: %+v, MongoDB.Status: %+v", mdb.Spec, mdb.Status) return res, err } @@ -298,15 +298,6 @@ func (r *ReplicaSetReconciler) deployStatefulSet(mdb mdbv1.MongoDBCommunity) (bo isReady := statefulset.IsReady(currentSts, mdb.StatefulSetReplicasThisReconciliation()) - if isReady { - r.log.Infow("StatefulSet is ready", - "replicas", currentSts.Spec.Replicas, - "generation", currentSts.Generation, - "observedGeneration", currentSts.Status.ObservedGeneration, - "updateStrategy", currentSts.Spec.UpdateStrategy.Type, - ) - } - return isReady || currentSts.Spec.UpdateStrategy.Type == appsv1.OnDeleteStatefulSetStrategyType, nil } From ce3e180161afc60b232d7f3bf78d43efbd6a2ab6 Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Mon, 5 Jul 2021 19:56:08 +0100 Subject: [PATCH 345/790] Always Run on Workflow Dispatch (#618) --- .action_templates/e2e-pr-template.yaml | 4 ++-- .github/workflows/e2e.yml | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.action_templates/e2e-pr-template.yaml b/.action_templates/e2e-pr-template.yaml index c151b195d..35c6efac7 100644 --- a/.action_templates/e2e-pr-template.yaml +++ b/.action_templates/e2e-pr-template.yaml @@ -2,8 +2,8 @@ name: Run E2E jobs: - template: display-github-context - template: setup - # run on master, or if a PR is being created from a branch. - if: github.ref == 'refs/heads/master' || (github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]') + # run on master, or if a PR is being created from a branch, or if it has been manually triggered. + if: github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/master' || (github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]') steps: - template: cancel-previous - template: checkout diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index edbd6ceda..105f19c27 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -46,8 +46,9 @@ jobs: - pipeline-argument: agent-ubi - pipeline-argument: agent-ubuntu - pipeline-argument: e2e - if: github.ref == 'refs/heads/master' || (github.event.pull_request.head.repo.full_name - == github.repository && github.actor != 'dependabot[bot]') + if: github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/master' + || (github.event.pull_request.head.repo.full_name == github.repository && github.actor + != 'dependabot[bot]') steps: # template: .action_templates/steps/cancel-previous.yaml - name: Cancel Previous Runs From 092fe04768189602edabbdfa12fc7de0844b1269 Mon Sep 17 00:00:00 2001 From: Rodrigo Valin Date: Mon, 5 Jul 2021 20:32:20 +0100 Subject: [PATCH 346/790] CLOUDP-94049: Updates agent image to latest. (#615) --- .github/workflows/release-single-image.yml | 2 ++ release.json | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release-single-image.yml b/.github/workflows/release-single-image.yml index 3d89f4f62..b14715e59 100644 --- a/.github/workflows/release-single-image.yml +++ b/.github/workflows/release-single-image.yml @@ -45,6 +45,8 @@ jobs: run: python pipeline.py --image-name ${{ github.event.inputs.pipeline-argument }} --release true env: MONGODB_COMMUNITY_CONFIG: "${{ github.workspace }}/scripts/ci/config.json" + AWS_ACCESS_KEY_ID: "${{ secrets.AWS_ACCESS_KEY_ID }}" + AWS_SECRET_ACCESS_KEY: "${{ secrets.AWS_SECRET_ACCESS_KEY }}" - name: Add Supported Release if: steps.release_status.outputs.OUTPUT == 'unreleased' diff --git a/release.json b/release.json index b30731b28..9b64c594f 100644 --- a/release.json +++ b/release.json @@ -3,7 +3,7 @@ "version-upgrade-hook": "1.0.2", "readiness-probe": "1.0.4", "mongodb-agent": { - "version": "10.29.0.6830-1", - "tools_version": "100.2.0" + "version": "11.0.5.6963-1", + "tools_version": "100.3.1" } } From 8fce413edcff390606136d296aa89236a5e5773e Mon Sep 17 00:00:00 2001 From: Rajdeep Das Date: Tue, 6 Jul 2021 12:28:10 +0200 Subject: [PATCH 347/790] CLOUDP-65796: Update CRD from v1beta1 to v1 (#617) --- Makefile | 2 +- PROJECT | 2 +- api/v1/doc.go | 4 + api/v1/mongodbcommunity_types.go | 2 + ...ommunity.mongodb.com_mongodbcommunity.yaml | 576 +++++++++--------- config/crd/kustomizeconfig.yaml | 4 +- config/manager/kustomization.yaml | 2 +- docs/RELEASE_NOTES.md | 2 + 8 files changed, 302 insertions(+), 292 deletions(-) create mode 100644 api/v1/doc.go diff --git a/Makefile b/Makefile index fee8be3e4..a4820abb4 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ NAMESPACE := $(shell jq -r .namespace < ~/.community-operator-dev/config.json) IMG := $(REPO_URL)/$(OPERATOR_IMAGE) DOCKERFILE ?= operator # Produce CRDs that work back to Kubernetes 1.11 (no version conversion) -CRD_OPTIONS ?= "crd:trivialVersions=true,preserveUnknownFields=true,crdVersions=v1beta1" +CRD_OPTIONS ?= "crd:trivialVersions=true,crdVersions=v1" # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) ifeq (,$(shell go env GOBIN)) diff --git a/PROJECT b/PROJECT index 88d8efcf9..40a421d71 100644 --- a/PROJECT +++ b/PROJECT @@ -3,7 +3,7 @@ layout: go.kubebuilder.io/v3 projectName: mko-v1 repo: github.com/mongodb/mongodb-kubernetes-operator resources: -- crdVersion: v1beta1 +- crdVersion: v1 group: mongodbcommunity kind: MongoDBCommunity version: v1 diff --git a/api/v1/doc.go b/api/v1/doc.go new file mode 100644 index 000000000..a6a3905a8 --- /dev/null +++ b/api/v1/doc.go @@ -0,0 +1,4 @@ +package v1 + +// +k8s:deepcopy-gen=package +// +versionName=v1 diff --git a/api/v1/mongodbcommunity_types.go b/api/v1/mongodbcommunity_types.go index f4c488d54..29fda2df8 100644 --- a/api/v1/mongodbcommunity_types.go +++ b/api/v1/mongodbcommunity_types.go @@ -90,6 +90,7 @@ type MongoDBCommunitySpec struct { // configuration file: https://docs.mongodb.com/manual/reference/configuration-options/ // +kubebuilder:validation:Type=object // +optional + // +kubebuilder:pruning:PreserveUnknownFields // +nullable AdditionalMongodConfig MongodConfiguration `json:"additionalMongodConfig,omitempty"` } @@ -191,6 +192,7 @@ type AuthenticationRestriction struct { // StatefulSetConfiguration holds the optional custom StatefulSet // that should be merged into the operator created one. type StatefulSetConfiguration struct { + // +kubebuilder:pruning:PreserveUnknownFields SpecWrapper StatefulSetSpecWrapper `json:"spec"` } diff --git a/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml b/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml index 5adb08761..7db8ac068 100644 --- a/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml +++ b/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml @@ -1,6 +1,6 @@ --- -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: @@ -8,15 +8,6 @@ metadata: creationTimestamp: null name: mongodbcommunity.mongodbcommunity.mongodb.com spec: - additionalPrinterColumns: - - JSONPath: .status.phase - description: Current state of the MongoDB deployment - name: Phase - type: string - - JSONPath: .status.version - description: Version of MongoDB server - name: Version - type: string group: mongodbcommunity.mongodb.com names: kind: MongoDBCommunity @@ -26,307 +17,318 @@ spec: - mdbc singular: mongodbcommunity scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - description: MongoDBCommunity is the Schema for the mongodbs API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: MongoDBCommunitySpec defines the desired state of MongoDB - properties: - additionalMongodConfig: - description: 'AdditionalMongodConfig is additional configuration that - can be passed to each data-bearing mongod at runtime. Uses the same - structure as the mongod configuration file: https://docs.mongodb.com/manual/reference/configuration-options/' - nullable: true - type: object - arbiters: - description: Arbiters is the number of arbiters (each counted as a member) - in the replica set - type: integer - featureCompatibilityVersion: - description: FeatureCompatibilityVersion configures the feature compatibility - version that will be set for the deployment - type: string - members: - description: Members is the number of members in the replica set - type: integer - replicaSetHorizons: - description: ReplicaSetHorizons Add this parameter and values if you - need your database to be accessed outside of Kubernetes. This setting - allows you to provide different DNS settings within the Kubernetes - cluster and to the Kubernetes cluster. The Kubernetes Operator uses - split horizon DNS for replica set members. This feature allows communication - both within the Kubernetes cluster and from outside Kubernetes. - items: - additionalProperties: - type: string + versions: + - additionalPrinterColumns: + - description: Current state of the MongoDB deployment + jsonPath: .status.phase + name: Phase + type: string + - description: Version of MongoDB server + jsonPath: .status.version + name: Version + type: string + name: v1 + schema: + openAPIV3Schema: + description: MongoDBCommunity is the Schema for the mongodbs API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: MongoDBCommunitySpec defines the desired state of MongoDB + properties: + additionalMongodConfig: + description: 'AdditionalMongodConfig is additional configuration that + can be passed to each data-bearing mongod at runtime. Uses the same + structure as the mongod configuration file: https://docs.mongodb.com/manual/reference/configuration-options/' + nullable: true type: object - type: array - security: - description: Security configures security features, such as TLS, and - authentication settings for a deployment - properties: - authentication: - properties: - ignoreUnknownUsers: - nullable: true - type: boolean - modes: - description: Modes is an array specifying which authentication - methods should be enabled. - items: - enum: - - SCRAM - - SCRAM-SHA-256 - - SCRAM-SHA-1 - type: string - type: array - required: - - modes + x-kubernetes-preserve-unknown-fields: true + arbiters: + description: Arbiters is the number of arbiters (each counted as a + member) in the replica set + type: integer + featureCompatibilityVersion: + description: FeatureCompatibilityVersion configures the feature compatibility + version that will be set for the deployment + type: string + members: + description: Members is the number of members in the replica set + type: integer + replicaSetHorizons: + description: ReplicaSetHorizons Add this parameter and values if you + need your database to be accessed outside of Kubernetes. This setting + allows you to provide different DNS settings within the Kubernetes + cluster and to the Kubernetes cluster. The Kubernetes Operator uses + split horizon DNS for replica set members. This feature allows communication + both within the Kubernetes cluster and from outside Kubernetes. + items: + additionalProperties: + type: string type: object - roles: - description: User-specified custom MongoDB roles that should be - configured in the deployment. - items: - description: CustomRole defines a custom MongoDB role. + type: array + security: + description: Security configures security features, such as TLS, and + authentication settings for a deployment + properties: + authentication: properties: - authenticationRestrictions: - description: The authentication restrictions the server enforces - on the role. - items: - description: AuthenticationRestriction specifies a list - of IP addresses and CIDR ranges users are allowed to connect - to or from. - properties: - clientSource: - items: - type: string - type: array - serverAddress: - items: - type: string - type: array - required: - - clientSource - - serverAddress - type: object - type: array - db: - description: The database of the role. - type: string - privileges: - description: The privileges to grant the role. - items: - description: Privilege defines the actions a role is allowed - to perform on a given resource. - properties: - actions: - items: - type: string - type: array - resource: - description: Resource specifies specifies the resources - upon which a privilege permits actions. See https://docs.mongodb.com/manual/reference/resource-document - for more. - properties: - anyResource: - type: boolean - cluster: - type: boolean - collection: - type: string - db: - type: string - type: object - required: - - actions - - resource - type: object - type: array - role: - description: The name of the role. - type: string - roles: - description: An array of roles from which this role inherits - privileges. + ignoreUnknownUsers: + default: true + nullable: true + type: boolean + modes: + description: Modes is an array specifying which authentication + methods should be enabled. items: - description: Role is the database role this user should - have - properties: - db: - description: DB is the database the role can act on - type: string - name: - description: Name is the name of the role - type: string - required: - - db - - name - type: object + enum: + - SCRAM + - SCRAM-SHA-256 + - SCRAM-SHA-1 + type: string type: array required: - - db - - privileges - - role + - modes type: object - type: array - tls: - description: TLS configuration for both client-server and server-server - communication - properties: - caConfigMapRef: - description: CaConfigMap is a reference to a ConfigMap containing - the certificate for the CA which signed the server certificates - The certificate is expected to be available under the key - "ca.crt" + roles: + description: User-specified custom MongoDB roles that should be + configured in the deployment. + items: + description: CustomRole defines a custom MongoDB role. properties: - name: + authenticationRestrictions: + description: The authentication restrictions the server + enforces on the role. + items: + description: AuthenticationRestriction specifies a list + of IP addresses and CIDR ranges users are allowed to + connect to or from. + properties: + clientSource: + items: + type: string + type: array + serverAddress: + items: + type: string + type: array + required: + - clientSource + - serverAddress + type: object + type: array + db: + description: The database of the role. type: string - required: - - name - type: object - certificateKeySecretRef: - description: CertificateKeySecret is a reference to a Secret - containing a private key and certificate to use for TLS. The - key and cert are expected to be PEM encoded and available - at "tls.key" and "tls.crt". This is the same format used for - the standard "kubernetes.io/tls" Secret type, but no specific - type is required. - properties: - name: + privileges: + description: The privileges to grant the role. + items: + description: Privilege defines the actions a role is allowed + to perform on a given resource. + properties: + actions: + items: + type: string + type: array + resource: + description: Resource specifies specifies the resources + upon which a privilege permits actions. See https://docs.mongodb.com/manual/reference/resource-document + for more. + properties: + anyResource: + type: boolean + cluster: + type: boolean + collection: + type: string + db: + type: string + type: object + required: + - actions + - resource + type: object + type: array + role: + description: The name of the role. type: string + roles: + description: An array of roles from which this role inherits + privileges. + items: + description: Role is the database role this user should + have + properties: + db: + description: DB is the database the role can act on + type: string + name: + description: Name is the name of the role + type: string + required: + - db + - name + type: object + type: array required: - - name + - db + - privileges + - role type: object - enabled: - type: boolean - optional: - description: Optional configures if TLS should be required or - optional for connections - type: boolean - required: - - enabled - type: object - type: object - statefulSet: - description: StatefulSetConfiguration holds the optional custom StatefulSet - that should be merged into the operator created one. - properties: - spec: - type: object - required: - - spec - type: object - type: - description: Type defines which type of MongoDB deployment the resource - should create - enum: - - ReplicaSet - type: string - users: - description: Users specifies the MongoDB users that should be configured - in your deployment - items: - properties: - db: - description: DB is the database the user is stored in. Defaults - to "admin" - type: string - name: - description: Name is the username of the user - type: string - passwordSecretRef: - description: PasswordSecretRef is a reference to the secret containing - this user's password + type: array + tls: + description: TLS configuration for both client-server and server-server + communication properties: - key: - description: Key is the key in the secret storing this password. - Defaults to "password" - type: string - name: - description: Name is the name of the secret storing this user's - password - type: string + caConfigMapRef: + description: CaConfigMap is a reference to a ConfigMap containing + the certificate for the CA which signed the server certificates + The certificate is expected to be available under the key + "ca.crt" + properties: + name: + type: string + required: + - name + type: object + certificateKeySecretRef: + description: CertificateKeySecret is a reference to a Secret + containing a private key and certificate to use for TLS. + The key and cert are expected to be PEM encoded and available + at "tls.key" and "tls.crt". This is the same format used + for the standard "kubernetes.io/tls" Secret type, but no + specific type is required. + properties: + name: + type: string + required: + - name + type: object + enabled: + type: boolean + optional: + description: Optional configures if TLS should be required + or optional for connections + type: boolean required: - - name + - enabled type: object - roles: - description: Roles is an array of roles assigned to this user - items: - description: Role is the database role this user should have + type: object + statefulSet: + description: StatefulSetConfiguration holds the optional custom StatefulSet + that should be merged into the operator created one. + properties: + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - spec + type: object + type: + description: Type defines which type of MongoDB deployment the resource + should create + enum: + - ReplicaSet + type: string + users: + description: Users specifies the MongoDB users that should be configured + in your deployment + items: + properties: + db: + description: DB is the database the user is stored in. Defaults + to "admin" + type: string + name: + description: Name is the username of the user + type: string + passwordSecretRef: + description: PasswordSecretRef is a reference to the secret + containing this user's password properties: - db: - description: DB is the database the role can act on + key: + description: Key is the key in the secret storing this password. + Defaults to "password" type: string name: - description: Name is the name of the role + description: Name is the name of the secret storing this + user's password type: string required: - - db - name type: object - type: array - scramCredentialsSecretName: - description: ScramCredentialsSecretName appended by string "scram-credentials" - is the name of the secret object created by the mongoDB operator - for storing SCRAM credentials - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - required: - - name - - passwordSecretRef - - roles - - scramCredentialsSecretName - type: object - type: array - version: - description: Version defines which version of MongoDB will be used - type: string - required: - - security - - type - - users - - version - type: object - status: - description: MongoDBCommunityStatus defines the observed state of MongoDB - properties: - currentMongoDBMembers: - type: integer - currentStatefulSetReplicas: - type: integer - message: - type: string - mongoUri: - type: string - phase: - type: string - required: - - currentMongoDBMembers - - currentStatefulSetReplicas - - mongoUri - - phase - type: object - type: object - version: v1 - versions: - - name: v1 + roles: + description: Roles is an array of roles assigned to this user + items: + description: Role is the database role this user should have + properties: + db: + description: DB is the database the role can act on + type: string + name: + description: Name is the name of the role + type: string + required: + - db + - name + type: object + type: array + scramCredentialsSecretName: + description: ScramCredentialsSecretName appended by string "scram-credentials" + is the name of the secret object created by the mongoDB operator + for storing SCRAM credentials + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - name + - passwordSecretRef + - roles + - scramCredentialsSecretName + type: object + type: array + version: + description: Version defines which version of MongoDB will be used + type: string + required: + - security + - type + - users + - version + type: object + status: + description: MongoDBCommunityStatus defines the observed state of MongoDB + properties: + currentMongoDBMembers: + type: integer + currentStatefulSetReplicas: + type: integer + message: + type: string + mongoUri: + type: string + phase: + type: string + required: + - currentMongoDBMembers + - currentStatefulSetReplicas + - mongoUri + - phase + type: object + type: object served: true storage: true + subresources: + status: {} status: acceptedNames: kind: "" diff --git a/config/crd/kustomizeconfig.yaml b/config/crd/kustomizeconfig.yaml index 651b34fd4..ec5c150a9 100644 --- a/config/crd/kustomizeconfig.yaml +++ b/config/crd/kustomizeconfig.yaml @@ -4,13 +4,13 @@ nameReference: version: v1 fieldSpecs: - kind: CustomResourceDefinition - version: v1beta1 + version: v1 group: apiextensions.k8s.io path: spec/conversion/webhook/clientConfig/service/name namespace: - kind: CustomResourceDefinition - version: v1beta1 + version: v1 group: apiextensions.k8s.io path: spec/conversion/webhook/clientConfig/service/namespace create: false diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index cb74a8d0e..3af48cebf 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -4,7 +4,7 @@ resources: generatorOptions: disableNameSuffixHash: true -apiVersion: kustomize.config.k8s.io/v1beta1 +apiVersion: kustomize.config.k8s.io/v1 kind: Kustomization images: - name: mongodb-kubernetes-operator diff --git a/docs/RELEASE_NOTES.md b/docs/RELEASE_NOTES.md index d6abb63c5..9c5d43ca7 100644 --- a/docs/RELEASE_NOTES.md +++ b/docs/RELEASE_NOTES.md @@ -6,6 +6,8 @@ - Members of a Replica Set can be configured as arbiters. - Reduce the number of permissions for operator role. - Support SHA-1 as an authentication method. + - Upgraded `mongodbcommunity.mongodbcommunity.mongodb.com` CRD to `v1` from `v1beta1` + * Users upgrading their CRD from v1beta1 to v1 need to set: `spec.preserveUnknownFields` to `false` in the CRD file `config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml` before applying the CRD to the cluster. ## Updated Image Tags From 6b4162c9810acdec24b5c1c6d003daa9164f757d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 6 Jul 2021 14:22:07 +0200 Subject: [PATCH 348/790] Bump sigs.k8s.io/controller-runtime from 0.9.0 to 0.9.2 (#602) --- go.mod | 6 +++--- go.sum | 31 +++++++++++++++---------------- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/go.mod b/go.mod index 519af65be..a3b009001 100644 --- a/go.mod +++ b/go.mod @@ -15,10 +15,10 @@ require ( github.com/xdg/stringprep v1.0.3 go.mongodb.org/mongo-driver v1.5.3 go.uber.org/zap v1.17.0 - k8s.io/api v0.21.1 + k8s.io/api v0.21.2 k8s.io/apimachinery v0.21.2 - k8s.io/client-go v0.21.1 - sigs.k8s.io/controller-runtime v0.9.0 + k8s.io/client-go v0.21.2 + sigs.k8s.io/controller-runtime v0.9.2 sigs.k8s.io/yaml v1.2.0 ) diff --git a/go.sum b/go.sum index c963c853f..488510ebf 100644 --- a/go.sum +++ b/go.sum @@ -612,7 +612,6 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40 h1:JWgyZ1qgdTaF3N3oxC+MdTV7qvEEgHo3otj+HB5CM7Q= @@ -634,8 +633,9 @@ golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6 h1:Vv0JUPWTyeqUq42B2WJ1FeIDjjvGKoA2Ss+Ts0lAVbs= +golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -785,19 +785,18 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.21.1 h1:94bbZ5NTjdINJEdzOkpS4vdPhkb1VFpTYC9zh43f75c= -k8s.io/api v0.21.1/go.mod h1:FstGROTmsSHBarKc8bylzXih8BLNYTiS3TZcsoEDg2s= -k8s.io/apiextensions-apiserver v0.21.1 h1:AA+cnsb6w7SZ1vD32Z+zdgfXdXY8X9uGX5bN6EoPEIo= -k8s.io/apiextensions-apiserver v0.21.1/go.mod h1:KESQFCGjqVcVsZ9g0xX5bacMjyX5emuWcS2arzdEouA= -k8s.io/apimachinery v0.21.1/go.mod h1:jbreFvJo3ov9rj7eWT7+sYiRx+qZuCYXwWT1bcDswPY= +k8s.io/api v0.21.2 h1:vz7DqmRsXTCSa6pNxXwQ1IYeAZgdIsua+DZU+o+SX3Y= +k8s.io/api v0.21.2/go.mod h1:Lv6UGJZ1rlMI1qusN8ruAp9PUBFyBwpEHAdG24vIsiU= +k8s.io/apiextensions-apiserver v0.21.2 h1:+exKMRep4pDrphEafRvpEi79wTnCFMqKf8LBtlA3yrE= +k8s.io/apiextensions-apiserver v0.21.2/go.mod h1:+Axoz5/l3AYpGLlhJDfcVQzCerVYq3K3CvDMvw6X1RA= k8s.io/apimachinery v0.21.2 h1:vezUc/BHqWlQDnZ+XkrpXSmnANSLbpnlpwo0Lhk0gpc= k8s.io/apimachinery v0.21.2/go.mod h1:CdTY8fU/BlvAbJ2z/8kBwimGki5Zp8/fbVuLY8gJumM= -k8s.io/apiserver v0.21.1/go.mod h1:nLLYZvMWn35glJ4/FZRhzLG/3MPxAaZTgV4FJZdr+tY= -k8s.io/client-go v0.21.1 h1:bhblWYLZKUu+pm50plvQF8WpY6TXdRRtcS/K9WauOj4= -k8s.io/client-go v0.21.1/go.mod h1:/kEw4RgW+3xnBGzvp9IWxKSNA+lXn3A7AuH3gdOAzLs= -k8s.io/code-generator v0.21.1/go.mod h1:hUlps5+9QaTrKx+jiM4rmq7YmH8wPOIko64uZCHDh6Q= -k8s.io/component-base v0.21.1 h1:iLpj2btXbR326s/xNQWmPNGu0gaYSjzn7IN/5i28nQw= -k8s.io/component-base v0.21.1/go.mod h1:NgzFZ2qu4m1juby4TnrmpR8adRk6ka62YdH5DkIIyKA= +k8s.io/apiserver v0.21.2/go.mod h1:lN4yBoGyiNT7SC1dmNk0ue6a5Wi6O3SWOIw91TsucQw= +k8s.io/client-go v0.21.2 h1:Q1j4L/iMN4pTw6Y4DWppBoUxgKO8LbffEMVEV00MUp0= +k8s.io/client-go v0.21.2/go.mod h1:HdJ9iknWpbl3vMGtib6T2PyI/VYxiZfq936WNVHBRrA= +k8s.io/code-generator v0.21.2/go.mod h1:8mXJDCB7HcRo1xiEQstcguZkbxZaqeUOrO9SsicWs3U= +k8s.io/component-base v0.21.2 h1:EsnmFFoJ86cEywC0DoIkAUiEV6fjgauNugiw1lmIjs4= +k8s.io/component-base v0.21.2/go.mod h1:9lvmIThzdlrJj5Hp8Z/TOgIkdfsNARQ1pT+3PByuiuc= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= @@ -812,9 +811,9 @@ k8s.io/utils v0.0.0-20210527160623-6fdb442a123b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= -sigs.k8s.io/controller-runtime v0.9.0 h1:ZIZ/dtpboPSbZYY7uUz2OzrkaBTOThx2yekLtpGB+zY= -sigs.k8s.io/controller-runtime v0.9.0/go.mod h1:TgkfvrhhEw3PlI0BRL/5xM+89y3/yc0ZDfdbTl84si8= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.19/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= +sigs.k8s.io/controller-runtime v0.9.2 h1:MnCAsopQno6+hI9SgJHKddzXpmv2wtouZz6931Eax+Q= +sigs.k8s.io/controller-runtime v0.9.2/go.mod h1:TxzMCHyEUpaeuOiZx/bIdc2T81vfs/aKdvJt9wuu0zk= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.1.0 h1:C4r9BgJ98vrKnnVCjwCSXcWjWe0NKcUQkmzDXZXGwH8= sigs.k8s.io/structured-merge-diff/v4 v4.1.0/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= From 26b755f222d8c73cf2b8bc0cbd8e76a81b5b0019 Mon Sep 17 00:00:00 2001 From: FlorentinaSimlinger <60526092+FlorentinaSimlinger@users.noreply.github.com> Date: Tue, 6 Jul 2021 15:00:09 +0100 Subject: [PATCH 349/790] CLOUDP-93590: Service name configurable (#616) --- api/v1/mongodbcommunity_types.go | 7 +++++-- .../mongodb.com_v1_custom_volume_cr.yaml | 1 + docs/RELEASE_NOTES.md | 1 + test/e2e/mongodbtests/mongodbtests.go | 13 +++++++++++++ .../statefulset_arbitrary_config_test.go | 4 ++++ 5 files changed, 24 insertions(+), 2 deletions(-) diff --git a/api/v1/mongodbcommunity_types.go b/api/v1/mongodbcommunity_types.go index 29fda2df8..7f94bbcd4 100644 --- a/api/v1/mongodbcommunity_types.go +++ b/api/v1/mongodbcommunity_types.go @@ -524,9 +524,12 @@ func (m MongoDBCommunity) Hosts() []string { return hosts } -// ServiceName returns the name of the Service that should be created for -// this resource +// ServiceName returns the name of the Service that should be created for this resource func (m MongoDBCommunity) ServiceName() string { + serviceName := m.Spec.StatefulSetConfiguration.SpecWrapper.Spec.ServiceName + if serviceName != "" { + return serviceName + } return m.Name + "-svc" } diff --git a/config/samples/arbitrary_statefulset_configuration/mongodb.com_v1_custom_volume_cr.yaml b/config/samples/arbitrary_statefulset_configuration/mongodb.com_v1_custom_volume_cr.yaml index dbd7cc3a9..20447b2cf 100644 --- a/config/samples/arbitrary_statefulset_configuration/mongodb.com_v1_custom_volume_cr.yaml +++ b/config/samples/arbitrary_statefulset_configuration/mongodb.com_v1_custom_volume_cr.yaml @@ -23,6 +23,7 @@ spec: statefulSet: spec: + # Name for the service object created by the operator serviceName: example-openshift-mongodb-svc selector: {} # Specifies a size for the data volume different from the default 10Gi diff --git a/docs/RELEASE_NOTES.md b/docs/RELEASE_NOTES.md index 9c5d43ca7..13bb54b10 100644 --- a/docs/RELEASE_NOTES.md +++ b/docs/RELEASE_NOTES.md @@ -8,6 +8,7 @@ - Support SHA-1 as an authentication method. - Upgraded `mongodbcommunity.mongodbcommunity.mongodb.com` CRD to `v1` from `v1beta1` * Users upgrading their CRD from v1beta1 to v1 need to set: `spec.preserveUnknownFields` to `false` in the CRD file `config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml` before applying the CRD to the cluster. + - Made service name configurable in mongdb custom resource with statefulSet.spec.serviceName ## Updated Image Tags diff --git a/test/e2e/mongodbtests/mongodbtests.go b/test/e2e/mongodbtests/mongodbtests.go index 953c619f9..72399fc7e 100644 --- a/test/e2e/mongodbtests/mongodbtests.go +++ b/test/e2e/mongodbtests/mongodbtests.go @@ -289,6 +289,19 @@ func BasicFunctionality(mdb *mdbv1.MongoDBCommunity) func(*testing.T) { } } +// ServiceWithNameExists checks whether a service with the name serviceName exists +func ServiceWithNameExists(serviceName string, namespace string) func(t *testing.T) { + return func(t *testing.T) { + serviceNamespacedName := types.NamespacedName{Name: serviceName, Namespace: namespace} + srv := corev1.Service{} + err := e2eutil.TestClient.Get(context.TODO(), serviceNamespacedName, &srv) + if err != nil { + t.Fatal(err) + } + t.Logf("Service with name %s exists", serviceName) + } +} + // DeletePod will delete a pod that belongs to this MongoDB resource's StatefulSet func DeletePod(mdb *mdbv1.MongoDBCommunity, podNum int) func(*testing.T) { return func(t *testing.T) { diff --git a/test/e2e/statefulset_arbitrary_config/statefulset_arbitrary_config_test.go b/test/e2e/statefulset_arbitrary_config/statefulset_arbitrary_config_test.go index 527fdcc09..3881bb7ab 100644 --- a/test/e2e/statefulset_arbitrary_config/statefulset_arbitrary_config_test.go +++ b/test/e2e/statefulset_arbitrary_config/statefulset_arbitrary_config_test.go @@ -34,6 +34,9 @@ func TestStatefulSetArbitraryConfig(t *testing.T) { mdb.Spec.StatefulSetConfiguration.SpecWrapper.Spec.Template.Spec.Containers[1].ReadinessProbe = &corev1.Probe{TimeoutSeconds: 100} + customServiceName := "database" + mdb.Spec.StatefulSetConfiguration.SpecWrapper.Spec.ServiceName = customServiceName + tester, err := mongotester.FromResource(t, mdb) if err != nil { t.Fatal(err) @@ -41,6 +44,7 @@ func TestStatefulSetArbitraryConfig(t *testing.T) { t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb)) + t.Run("Test setting Service Name", mongodbtests.ServiceWithNameExists(customServiceName, mdb.Namespace)) t.Run("Test Basic Connectivity", tester.ConnectivitySucceeds()) t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 1)) t.Run("Container has been merged by name", mongodbtests.StatefulSetContainerConditionIsTrue(&mdb, "mongodb-agent", func(container corev1.Container) bool { From 360c6092a65599c111c66a8979840e0df027d7f8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 7 Jul 2021 10:22:28 +0200 Subject: [PATCH 350/790] Bump go.mongodb.org/mongo-driver from 1.5.3 to 1.5.4 (#619) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a3b009001..f75efca12 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/stretchr/objx v0.3.0 github.com/stretchr/testify v1.7.0 github.com/xdg/stringprep v1.0.3 - go.mongodb.org/mongo-driver v1.5.3 + go.mongodb.org/mongo-driver v1.5.4 go.uber.org/zap v1.17.0 k8s.io/api v0.21.2 k8s.io/apimachinery v0.21.2 diff --git a/go.sum b/go.sum index 488510ebf..4989d1ef5 100644 --- a/go.sum +++ b/go.sum @@ -457,8 +457,8 @@ go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= -go.mongodb.org/mongo-driver v1.5.3 h1:wWbFB6zaGHpzguF3f7tW94sVE8sFl3lHx8OZx/4OuFI= -go.mongodb.org/mongo-driver v1.5.3/go.mod h1:gRXCHX4Jo7J0IJ1oDQyUxF7jfy19UfxniMS4xxMmUqw= +go.mongodb.org/mongo-driver v1.5.4 h1:NPIBF/lxEcKNfWwoCJRX8+dMVwecWf9q3qUJkuh75oM= +go.mongodb.org/mongo-driver v1.5.4/go.mod h1:gRXCHX4Jo7J0IJ1oDQyUxF7jfy19UfxniMS4xxMmUqw= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= From eb097e476001465a6f903e73bd8527591dd0b86e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 7 Jul 2021 11:03:23 +0200 Subject: [PATCH 351/790] Bump go.uber.org/zap from 1.17.0 to 1.18.1 (#620) --- go.mod | 2 +- go.sum | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index f75efca12..108f0e488 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/stretchr/testify v1.7.0 github.com/xdg/stringprep v1.0.3 go.mongodb.org/mongo-driver v1.5.4 - go.uber.org/zap v1.17.0 + go.uber.org/zap v1.18.1 k8s.io/api v0.21.2 k8s.io/apimachinery v0.21.2 k8s.io/client-go v0.21.2 diff --git a/go.sum b/go.sum index 4989d1ef5..4807e961c 100644 --- a/go.sum +++ b/go.sum @@ -48,6 +48,8 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.34.28 h1:sscPpn/Ns3i0F4HPEWAVcwdIRaZZCuL7llJ2/60yPIk= github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= +github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -473,8 +475,9 @@ go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/ go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.17.0 h1:MTjgFu6ZLKvY6Pvaqk97GlxNBuMpV4Hy/3P6tRGlI2U= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +go.uber.org/zap v1.18.1 h1:CSUJ2mjFszzEWt4CdKISEuChVIXGBn3lAPwkRGyVrc4= +go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= From 4cbc17ae010dc9fd2deb6ca9c3c538ba7cff2574 Mon Sep 17 00:00:00 2001 From: Rajdeep Das Date: Thu, 8 Jul 2021 09:34:35 +0200 Subject: [PATCH 352/790] CLOUDP-94119: Do not mark the CR as failed on secret absence, it could be deleted at later stage after when scram is set (#622) --- api/v1/mongodbcommunity_types.go | 4 ---- controllers/mongodb_users.go | 10 +++++++++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/api/v1/mongodbcommunity_types.go b/api/v1/mongodbcommunity_types.go index 7f94bbcd4..f75800b15 100644 --- a/api/v1/mongodbcommunity_types.go +++ b/api/v1/mongodbcommunity_types.go @@ -558,10 +558,6 @@ func (m MongoDBCommunity) NamespacedName() types.NamespacedName { return types.NamespacedName{Name: m.Name, Namespace: m.Namespace} } -func (m MongoDBCommunity) GetAgentScramCredentialsNamespacedName() types.NamespacedName { - return types.NamespacedName{Name: fmt.Sprintf("%s-agent-scram-credentials", m.Name), Namespace: m.Namespace} -} - func (m MongoDBCommunity) DesiredReplicas() int { return m.Spec.Members } diff --git a/controllers/mongodb_users.go b/controllers/mongodb_users.go index 480332da7..f04248777 100644 --- a/controllers/mongodb_users.go +++ b/controllers/mongodb_users.go @@ -16,7 +16,15 @@ func (r ReplicaSetReconciler) ensureUserResources(mdb mdbv1.MongoDBCommunity) er secretNamespacedName := types.NamespacedName{Name: user.PasswordSecretName, Namespace: mdb.Namespace} if _, err := secret.ReadKey(r.client, user.PasswordSecretKey, secretNamespacedName); err != nil { if apiErrors.IsNotFound(err) { - return fmt.Errorf(`user password secret "%s" not found: %s`, secretNamespacedName, err) + // check for SCRAM secret as well + + scramSecretName := types.NamespacedName{Name: user.ScramCredentialsSecretName, Namespace: mdb.Namespace} + _, err = r.client.GetSecret(scramSecretName) + if apiErrors.IsNotFound(err) { + return fmt.Errorf(`user password secret: %s and scram secret: %s not found`, secretNamespacedName, scramSecretName) + } + r.log.Errorf(`user password secret "%s" not found: %s`, secretNamespacedName, err) + continue } return err } From 133731dab96743a5c92f0522df4e97e57c43df5e Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Fri, 9 Jul 2021 10:14:35 +0100 Subject: [PATCH 353/790] Updated how to release docs (#625) --- .github/workflows/create-release-pr.yml | 36 ------------------------- docs/how-to-release.md | 20 +++++++------- 2 files changed, 9 insertions(+), 47 deletions(-) delete mode 100644 .github/workflows/create-release-pr.yml diff --git a/.github/workflows/create-release-pr.yml b/.github/workflows/create-release-pr.yml deleted file mode 100644 index 05d35a8cd..000000000 --- a/.github/workflows/create-release-pr.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: Create Release PR -on: - workflow_dispatch - -jobs: - create-release-pr: - runs-on: ubuntu-latest - steps: - - name: Checkout Code - uses: actions/checkout@v2 - - name: Determine Release Version - id: release_version - run: | - OUTPUT=$(jq -r '."mongodb-kubernetes-operator"' < $GITHUB_WORKSPACE/release.json) - echo "::set-output name=OUTPUT::$OUTPUT" - DATE=$(date '+%Y-%m-%d-%H-%M-%S') - echo "::set-output name=DATE::$DATE" - - uses: actions/setup-python@v2 - with: - python-version: '3.6' - architecture: 'x64' - - run: pip install -r requirements.txt - - name: Create Release PR - uses: technote-space/create-pr-action@v2 - with: - EXECUTE_COMMANDS: | - python scripts/ci/update_release.py - COMMIT_MESSAGE: 'Release Operator v${{ steps.release_version.outputs.OUTPUT }}' - COMMIT_NAME: 'GitHub Actions' - COMMIT_EMAIL: 'support@mongodb.com' - # include date in the pr branch name to ensure duplicate branches to not get created. - PR_BRANCH_NAME: 'release-v${{steps.release_version.outputs.OUTPUT}}-${{ steps.release_version.outputs.DATE }}' - PR_TITLE: 'Release MongoDB Kubernetes Operator v${{ steps.release_version.outputs.OUTPUT }}' - - # Note: sonar github is a repo scoped token, which allows this action to trigger other actions. - GITHUB_TOKEN: ${{ secrets.SONAR_GITHUB_TOKEN }} diff --git a/docs/how-to-release.md b/docs/how-to-release.md index 98b71a964..e0a825cba 100644 --- a/docs/how-to-release.md +++ b/docs/how-to-release.md @@ -1,17 +1,15 @@ ## How to Release -* Ensure the versions specified in [release.json](../release.json) are correct. - * Prepare a PR to bump and versions as required. - -* Ensure that [the release notes](./RELEASE_NOTES.md) are up to date for this release. - * Review the [tickets for this release](https://jira.mongodb.org/issues?jql=project%20%3D%20CLOUDP%20AND%20component%20%20%3D%20"Kubernetes%20Community"%20%20AND%20status%20in%20(Resolved%2C%20Closed)%20AND%20fixVersion%20%3D%20kube-community-0.6.0%20) (ensure relevant fix version is in the jql query) - -* Run the `Create Release PR` GitHub Action - * In the GitHub UI: - * `Actions` > `Create Release PR` > `Run Workflow` (on master) - -* Review and Approve the release PR that is created by this action. +* Prepare release PR: + * Update any changing versions in [release.json](../release.json). + * Ensure that [the release notes](./RELEASE_NOTES.md) are up to date for this release. + * Run `python scripts/ci/update_release.py` to update the relevant yaml manifests. + * Commit all changes. + * Create a PR with the title `Release MongoDB Kubernetes Operator v` (the title must match this pattern) + +* Have this PR Reviewed and Approved. * Upon approval, all new images for this release will be built and released, and a Github release draft will be created. + * No need to merge this PR, when all images are successfully released, the PR will be merged. * Review and publish the new GitHub release draft. From 3979d23e2b4fbf5f77fca886076449642544161d Mon Sep 17 00:00:00 2001 From: FlorentinaSimlinger <60526092+FlorentinaSimlinger@users.noreply.github.com> Date: Fri, 9 Jul 2021 11:43:13 +0100 Subject: [PATCH 354/790] CLOUDP-90175: cleanup all resources (#624) --- scripts/dev/e2e.py | 67 +++++++++++++++++++++++++++++------------ test/e2e/setup/setup.go | 21 +++++++------ 2 files changed, 59 insertions(+), 29 deletions(-) diff --git a/scripts/dev/e2e.py b/scripts/dev/e2e.py index 8917a05d1..23093e72c 100644 --- a/scripts/dev/e2e.py +++ b/scripts/dev/e2e.py @@ -11,6 +11,9 @@ import yaml TEST_POD_NAME = "e2e-test" +TEST_CLUSTER_ROLE_NAME = "e2e-test" +TEST_CLUSTER_ROLE_BINDING_NAME = "e2e-test" +TEST_SERVICE_ACCOUNT_NAME = "e2e-test" def load_yaml_from_file(path: str) -> Dict: @@ -32,27 +35,29 @@ def _load_test_role_binding() -> Dict: def _prepare_test_environment(config_file: str) -> None: """ - _prepare_testrunner_environment ensures the ServiceAccount, - Role and ClusterRole and bindings are created for the test runner. + _prepare_test_environment ensures that the old test pod is deleted + and that namespace, cluster role, cluster role binding and service account + are created for the test pod. """ rbacv1 = client.RbacAuthorizationV1Api() corev1 = client.CoreV1Api() dev_config = load_config(config_file) + _delete_test_pod(config_file) + print("Creating Namespace") k8s_conditions.ignore_if_already_exists( lambda: corev1.create_namespace( client.V1Namespace(metadata=dict(name=dev_config.namespace)) ) ) - _delete_test_pod(config_file) - print("Creating Role") + print("Creating Cluster Role") k8s_conditions.ignore_if_already_exists( lambda: rbacv1.create_cluster_role(_load_test_role()) ) - print("Creating Role Binding") + print("Creating Cluster Role Binding") role_binding = _load_test_role_binding() # set namespace specified in config.json role_binding["subjects"][0]["namespace"] = dev_config.namespace @@ -61,7 +66,7 @@ def _prepare_test_environment(config_file: str) -> None: lambda: rbacv1.create_cluster_role_binding(role_binding) ) - print("Creating ServiceAccount") + print("Creating Service Account") k8s_conditions.ignore_if_already_exists( lambda: corev1.create_namespaced_service_account( dev_config.namespace, _load_test_service_account() @@ -69,18 +74,6 @@ def _prepare_test_environment(config_file: str) -> None: ) -def _delete_test_pod(config_file: str) -> None: - """ - _delete_testrunner_pod deletes the test runner pod - if it already exists. - """ - dev_config = load_config(config_file) - corev1 = client.CoreV1Api() - k8s_conditions.ignore_if_doesnt_exist( - lambda: corev1.delete_namespaced_pod(TEST_POD_NAME, dev_config.namespace) - ) - - def create_test_pod(args: argparse.Namespace, dev_config: DevConfig) -> None: corev1 = client.CoreV1Api() test_pod = { @@ -158,7 +151,7 @@ def create_test_pod(args: argparse.Namespace, dev_config: DevConfig) -> None: timeout=60, exceptions_to_ignore=ApiException, ): - raise Exception("Could not create test_runner pod!") + raise Exception("Could not create test pod!") def wait_for_pod_to_be_running( @@ -178,6 +171,41 @@ def wait_for_pod_to_be_running( print("Pod is running") +def _delete_test_environment(config_file: str) -> None: + """ + _delete_test_environment ensures that the cluster role, cluster role binding and service account + for the test pod are deleted. + """ + rbacv1 = client.RbacAuthorizationV1Api() + corev1 = client.CoreV1Api() + dev_config = load_config(config_file) + + k8s_conditions.ignore_if_doesnt_exist( + lambda: rbacv1.delete_cluster_role(TEST_CLUSTER_ROLE_NAME) + ) + + k8s_conditions.ignore_if_doesnt_exist( + lambda: rbacv1.delete_cluster_role_binding(TEST_CLUSTER_ROLE_BINDING_NAME) + ) + + k8s_conditions.ignore_if_doesnt_exist( + lambda: corev1.delete_namespaced_service_account( + TEST_SERVICE_ACCOUNT_NAME, dev_config.namespace + ) + ) + + +def _delete_test_pod(config_file: str) -> None: + """ + _delete_test_pod deletes the test pod. + """ + dev_config = load_config(config_file) + corev1 = client.CoreV1Api() + k8s_conditions.ignore_if_doesnt_exist( + lambda: corev1.delete_namespaced_pod(TEST_POD_NAME, dev_config.namespace) + ) + + def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser() parser.add_argument("--test", help="Name of the test to run") @@ -246,6 +274,7 @@ def main() -> int: exceptions_to_ignore=ApiException, ): return 1 + _delete_test_environment(args.config_file) return 0 diff --git a/test/e2e/setup/setup.go b/test/e2e/setup/setup.go index d59b57c80..7feb4fa0b 100644 --- a/test/e2e/setup/setup.go +++ b/test/e2e/setup/setup.go @@ -46,7 +46,7 @@ func Setup(t *testing.T) *e2eutil.Context { t.Fatal(err) } - if err := deployOperator(); err != nil { + if err := deployOperator(ctx); err != nil { t.Fatal(err) } @@ -130,7 +130,8 @@ func deployDir() string { return envvar.GetEnvOrDefault(deployDirEnv, "/workspace/config/manager") } -func deployOperator() error { +func deployOperator(ctx *e2eutil.Context) error { + testConfig := loadTestConfigFromEnv() e2eutil.OperatorNamespace = testConfig.namespace @@ -141,24 +142,24 @@ func deployOperator() error { } fmt.Printf("Setting namespace to watch to %s\n", watchNamespace) - if err := buildKubernetesResourceFromYamlFile(path.Join(roleDir(), "role.yaml"), &rbacv1.Role{}, withNamespace(testConfig.namespace)); err != nil { + if err := buildKubernetesResourceFromYamlFile(ctx, path.Join(roleDir(), "role.yaml"), &rbacv1.Role{}, withNamespace(testConfig.namespace)); err != nil { return errors.Errorf("error building operator role: %s", err) } fmt.Println("Successfully created the operator Role") - if err := buildKubernetesResourceFromYamlFile(path.Join(roleDir(), "service_account.yaml"), &corev1.ServiceAccount{}, withNamespace(testConfig.namespace)); err != nil { + if err := buildKubernetesResourceFromYamlFile(ctx, path.Join(roleDir(), "service_account.yaml"), &corev1.ServiceAccount{}, withNamespace(testConfig.namespace)); err != nil { return errors.Errorf("error building operator service account: %s", err) } fmt.Println("Successfully created the operator Service Account") - if err := buildKubernetesResourceFromYamlFile(path.Join(roleDir(), "role_binding.yaml"), &rbacv1.RoleBinding{}, withNamespace(testConfig.namespace)); err != nil { + if err := buildKubernetesResourceFromYamlFile(ctx, path.Join(roleDir(), "role_binding.yaml"), &rbacv1.RoleBinding{}, withNamespace(testConfig.namespace)); err != nil { return errors.Errorf("error building operator role binding: %s", err) } fmt.Println("Successfully created the operator Role Binding") dep := &appsv1.Deployment{} - if err := buildKubernetesResourceFromYamlFile(path.Join(deployDir(), "manager.yaml"), + if err := buildKubernetesResourceFromYamlFile(ctx, path.Join(deployDir(), "manager.yaml"), dep, withNamespace(testConfig.namespace), withOperatorImage(testConfig.operatorImage), @@ -201,7 +202,7 @@ func hasDeploymentRequiredReplicas(dep *appsv1.Deployment) wait.ConditionFunc { // buildKubernetesResourceFromYamlFile will create the kubernetes resource defined in yamlFilePath. All of the functional options // provided will be applied before creation. -func buildKubernetesResourceFromYamlFile(yamlFilePath string, obj client.Object, options ...func(obj runtime.Object)) error { +func buildKubernetesResourceFromYamlFile(ctx *e2eutil.Context, yamlFilePath string, obj client.Object, options ...func(obj runtime.Object)) error { data, err := ioutil.ReadFile(yamlFilePath) if err != nil { return errors.Errorf("error reading file: %s", err) @@ -215,7 +216,7 @@ func buildKubernetesResourceFromYamlFile(yamlFilePath string, obj client.Object, opt(obj) } - return createOrUpdate(obj) + return createOrUpdate(ctx, obj) } // marshalRuntimeObjectFromYAMLBytes accepts the bytes of a yaml resource @@ -228,8 +229,8 @@ func marshalRuntimeObjectFromYAMLBytes(bytes []byte, obj runtime.Object) error { return json.Unmarshal(jsonBytes, &obj) } -func createOrUpdate(obj client.Object) error { - if err := e2eutil.TestClient.Create(context.TODO(), obj, &e2eutil.CleanupOptions{}); err != nil { +func createOrUpdate(ctx *e2eutil.Context, obj client.Object) error { + if err := e2eutil.TestClient.Create(context.TODO(), obj, &e2eutil.CleanupOptions{TestContext: ctx}); err != nil { if apiErrors.IsAlreadyExists(err) { return e2eutil.TestClient.Update(context.TODO(), obj) } From 0256e7a40eec68d476a3347780b38f0f90f4fa64 Mon Sep 17 00:00:00 2001 From: killian2k Date: Fri, 9 Jul 2021 12:53:27 +0200 Subject: [PATCH 355/790] Fixing the make deploy command (#626) --- Makefile | 2 +- config/manager/kustomization.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index a4820abb4..205c8461c 100644 --- a/Makefile +++ b/Makefile @@ -124,7 +124,7 @@ controller-gen: # Download kustomize locally if necessary KUSTOMIZE = $(shell pwd)/bin/kustomize kustomize: - $(call go-get-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v3@v3.8.7) + $(call go-get-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v4@v4.2.0) # go-get-tool will 'go get' any package $2 and install it to $1. PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST)))) diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 3af48cebf..cb74a8d0e 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -4,7 +4,7 @@ resources: generatorOptions: disableNameSuffixHash: true -apiVersion: kustomize.config.k8s.io/v1 +apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization images: - name: mongodb-kubernetes-operator From b15e712b902c86c97a182ad580c0d35bb00c4f94 Mon Sep 17 00:00:00 2001 From: killian2k Date: Fri, 9 Jul 2021 14:04:07 +0200 Subject: [PATCH 356/790] CLOUDP-90457 Create kubelinter-check.yml (#605) * Create kubelinter-check.yml Added the kubelinter action + reduce the CPU request when doing the e2e tests. Now, the manager.yaml and the openshift operator are in the mongodb namespace * correction kubelinter * add setup go * use cat * adding script to manage the kube-linter * correct script to manage the kube-linter * correct script to manage the kube-linter 2 * list folders in github_workspace * list folders in github_workspace and parent * add dump github context * add dump github context * Makes script more readable and add checkout * list hidden files * fix script file * fix script file 2 * fix script file 2 * add yamls to test_and_integrated_action * add yamls to test_and_integrated_action * Add the exception for specific error * Solve the yaml error files with kube-linter * fix tests * remove useless files * modifier e2e role * Modified a few files, readiness not working * modified the manager file TEMPORARLY to see if the modifications that will be made on openshift may have issues * Solved a few issues, removed probe * Solved a few issues, removed probe 2 * Fix typo * Fix typo 2 * Tests passes locally * fixed role * removed sec context * set manager to default workspace * Fixing minimum CPU required * PModify the e2e cpu requests to lower values * Now tests should pass * reduced cpu amount * PModify the e2e cpu requests to lower values again * added security to manager.yaml * corrected files * Update kubelinter-check.yml * Update role.yaml Removed spaces in last line * Update role.yaml * Update role.yaml * Throw an error when withCPURequest() or other with...() function is called with incorrect params. Changed namespace from default to mongodb * Cancel the tests on mongodb namespace * Update kustomization.yaml * Update operator_openshift.yaml * Update .github/config_files/config_lint.yaml Co-authored-by: Cian Hatton * Update service_account.yaml * Update kubelinter-check.yml * Update service_account.yaml Co-authored-by: Cian Hatton --- .github/config_files/config_lint.yaml | 13 +++++ .../config_files/config_lint_openshift.yaml | 16 ++++++ .github/workflows/kubelinter-check.yml | 44 +++++++++++++++ config/manager/manager.yaml | 30 +++++++++++ config/rbac/role.yaml | 1 + deploy/clusterwide/role.yaml | 14 +++-- deploy/clusterwide/role_binding.yaml | 2 +- deploy/e2e/service_account.yaml | 4 ++ deploy/openshift/operator_openshift.yaml | 30 ++++++++++- test/e2e/setup/setup.go | 53 +++++++++++++++---- 10 files changed, 192 insertions(+), 15 deletions(-) create mode 100644 .github/config_files/config_lint.yaml create mode 100644 .github/config_files/config_lint_openshift.yaml create mode 100644 .github/workflows/kubelinter-check.yml diff --git a/.github/config_files/config_lint.yaml b/.github/config_files/config_lint.yaml new file mode 100644 index 000000000..95db1fb9d --- /dev/null +++ b/.github/config_files/config_lint.yaml @@ -0,0 +1,13 @@ +checks: + addAllBuiltIn: true + +#Reasons to exclude: + # non-existent-service-account because the service account is created in another file + # minimum-three-replicas because the deployment contains only 1 replica of the operator + # no-readiness-probe & no-liveness-probe because for now, it brings nothing to add these probes + # because they will not check whether the operator is actually ready/living + exclude: + - "non-existent-service-account" + - "minimum-three-replicas" + - "no-liveness-probe" + - "no-readiness-probe" diff --git a/.github/config_files/config_lint_openshift.yaml b/.github/config_files/config_lint_openshift.yaml new file mode 100644 index 000000000..dd5c29bb2 --- /dev/null +++ b/.github/config_files/config_lint_openshift.yaml @@ -0,0 +1,16 @@ +checks: + addAllBuiltIn: true + + #Reasons to exclude + # non-existent-service-account because the service account is created in another file + # minimum-three-replicas because the deployment contains only 1 replica of the operator + # no-readiness-probe & no-liveness-probe because for now it brings nothing to add theses probes + # because they will not check whether the operator is actually ready/living + # run-as-non-root & no-read-only-root-fs because the security is managed somewhere else + exclude: + - "non-existent-service-account" + - "minimum-three-replicas" + - "no-liveness-probe" + - "no-readiness-probe" + - "run-as-non-root" + - "no-read-only-root-fs" diff --git a/.github/workflows/kubelinter-check.yml b/.github/workflows/kubelinter-check.yml new file mode 100644 index 000000000..21a1e539e --- /dev/null +++ b/.github/workflows/kubelinter-check.yml @@ -0,0 +1,44 @@ +name: Kubelinter-check + +on: + push: + branches: + - master + paths-ignore: + - docs/** + pull_request: + branches: + - master + workflow_dispatch: {} + +jobs: + Kubelinter-check: + name: Run Kube-linter check + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@v2 + + - name: Scan directory ./deploy/clusterwide/ with kube-linter + uses: stackrox/kube-linter-action@v1 + with: + directory: deploy/clusterwide + config: ${GITHUB_WORKSPACE}/.github/config_files/config_lint.yaml + + - name: Scan directory ./deploy/openshift/ with kube-linter + uses: stackrox/kube-linter-action@v1 + with: + directory: deploy/openshift + config: ${GITHUB_WORKSPACE}/.github/config_files/config_lint_openshift.yaml + + - name: Scan directory ./config/manager/ with kube-linter + uses: stackrox/kube-linter-action@v1 + with: + directory: config/manager/manager.yaml + config: ${GITHUB_WORKSPACE}/.github/config_files/config_lint.yaml + + - name: Scan directory ./config/samples/ with kube-linter + uses: stackrox/kube-linter-action@v1 + with: + directory: config/samples + config: ${GITHUB_WORKSPACE}/.github/config_files/config_lint.yaml diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 3dbdc9ad3..09c5e3d71 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -2,16 +2,35 @@ apiVersion: apps/v1 kind: Deployment metadata: name: mongodb-kubernetes-operator + namespace: mongodb + labels: + owner: mongodb + annotations: + email: "support@mongodb.com" spec: replicas: 1 selector: matchLabels: name: mongodb-kubernetes-operator + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 template: metadata: labels: name: mongodb-kubernetes-operator spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: name + operator: In + values: + - mongodb-kubernetes-operator + topologyKey: "kubernetes.io/hostname" containers: - command: - /usr/local/bin/entrypoint @@ -39,4 +58,15 @@ spec: image: quay.io/mongodb/mongodb-kubernetes-operator:0.6.2 imagePullPolicy: Always name: mongodb-kubernetes-operator + resources: + limits: + cpu: 1100m + memory: 1Gi + requests: + cpu: 500m + memory: 200Mi + securityContext: + readOnlyRootFilesystem: true + runAsUser: 2000 + serviceAccountName: mongodb-kubernetes-operator diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 6a9c42070..d5ba4af61 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -3,6 +3,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: mongodb-kubernetes-operator + namespace: mongodb rules: - apiGroups: - "" diff --git a/deploy/clusterwide/role.yaml b/deploy/clusterwide/role.yaml index bddf8378e..6cfeaa876 100644 --- a/deploy/clusterwide/role.yaml +++ b/deploy/clusterwide/role.yaml @@ -6,10 +6,8 @@ rules: - apiGroups: - "" resources: - - pods - services - configmaps - - secrets verbs: - create - delete @@ -22,8 +20,18 @@ rules: - apps resources: - statefulsets + verbs: + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - pods verbs: - - create - delete - get - list diff --git a/deploy/clusterwide/role_binding.yaml b/deploy/clusterwide/role_binding.yaml index 2096a8b65..a3a4ddc00 100644 --- a/deploy/clusterwide/role_binding.yaml +++ b/deploy/clusterwide/role_binding.yaml @@ -5,7 +5,7 @@ metadata: subjects: - kind: ServiceAccount name: mongodb-kubernetes-operator - namespace: default + namespace: mongodb roleRef: kind: ClusterRole name: mongodb-kubernetes-operator diff --git a/deploy/e2e/service_account.yaml b/deploy/e2e/service_account.yaml index 07bb7f0f4..e94e44181 100644 --- a/deploy/e2e/service_account.yaml +++ b/deploy/e2e/service_account.yaml @@ -2,3 +2,7 @@ apiVersion: v1 kind: ServiceAccount metadata: name: e2e-test + # namespace is 'default' for the e2e tests because the TLS certificates + # generated used as test fixture (in the TLS tests) only work with the + # default namespace. + namespace: default diff --git a/deploy/openshift/operator_openshift.yaml b/deploy/openshift/operator_openshift.yaml index 0f5bf582e..a15edb68c 100644 --- a/deploy/openshift/operator_openshift.yaml +++ b/deploy/openshift/operator_openshift.yaml @@ -2,18 +2,38 @@ apiVersion: apps/v1 kind: Deployment metadata: name: mongodb-kubernetes-operator + namespace: mongodb + labels: + owner: mongodb + annotations: + email: "support@mongodb.com" spec: replicas: 1 selector: matchLabels: name: mongodb-kubernetes-operator + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 template: metadata: labels: name: mongodb-kubernetes-operator spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: name + operator: In + values: + - mongodb-kubernetes-operator + topologyKey: "kubernetes.io/hostname" containers: - - command: + - name: mongodb-kubernetes-operator + command: - mongodb-kubernetes-operator env: - name: WATCH_NAMESPACE @@ -40,5 +60,11 @@ spec: value: docker.io image: quay.io/mongodb/mongodb-kubernetes-operator:0.6.2 imagePullPolicy: Always - name: mongodb-kubernetes-operator + resources: + limits: + cpu: 1100m + memory: 1Gi + requests: + cpu: 500m + memory: 200Mi serviceAccountName: mongodb-kubernetes-operator diff --git a/test/e2e/setup/setup.go b/test/e2e/setup/setup.go index 7feb4fa0b..5ee2e412b 100644 --- a/test/e2e/setup/setup.go +++ b/test/e2e/setup/setup.go @@ -18,6 +18,7 @@ import ( corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" apiErrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" @@ -167,6 +168,7 @@ func deployOperator(ctx *e2eutil.Context) error { withEnvVar(construct.AgentImageEnv, testConfig.agentImage), withEnvVar(construct.ReadinessProbeImageEnv, testConfig.readinessProbeImage), withEnvVar(construct.VersionUpgradeHookImageEnv, testConfig.versionUpgradeHookImage), + withCPURequest("50m"), ); err != nil { return errors.Errorf("error building operator deployment: %s", err) } @@ -202,7 +204,7 @@ func hasDeploymentRequiredReplicas(dep *appsv1.Deployment) wait.ConditionFunc { // buildKubernetesResourceFromYamlFile will create the kubernetes resource defined in yamlFilePath. All of the functional options // provided will be applied before creation. -func buildKubernetesResourceFromYamlFile(ctx *e2eutil.Context, yamlFilePath string, obj client.Object, options ...func(obj runtime.Object)) error { +func buildKubernetesResourceFromYamlFile(ctx *e2eutil.Context, yamlFilePath string, obj client.Object, options ...func(obj runtime.Object) error) error { data, err := ioutil.ReadFile(yamlFilePath) if err != nil { return errors.Errorf("error reading file: %s", err) @@ -213,7 +215,9 @@ func buildKubernetesResourceFromYamlFile(ctx *e2eutil.Context, yamlFilePath stri } for _, opt := range options { - opt(obj) + if err := opt(obj); err != nil { + return err + } } return createOrUpdate(ctx, obj) @@ -239,11 +243,33 @@ func createOrUpdate(ctx *e2eutil.Context, obj client.Object) error { return nil } +// withCPURequest assumes that the underlying type is an appsv1.Deployment. +// it returns a function which will change the amount +// requested for the CPUresource. There will be +// no effect when used with a non-deployment type +func withCPURequest(cpuRequest string) func(runtime.Object) error { + return func(obj runtime.Object) error { + dep, ok := obj.(*appsv1.Deployment) + if !ok { + return errors.Errorf("withCPURequest() called on a non-deployment object") + } + quantityCPU, okCPU := resource.ParseQuantity(cpuRequest) + if okCPU != nil { + return okCPU + } + for _, cont := range dep.Spec.Template.Spec.Containers { + cont.Resources.Requests["cpu"] = quantityCPU + } + + return nil + } +} + // withNamespace returns a function which will assign the namespace // of the underlying type to the value specified. We can // add new types here as required. -func withNamespace(ns string) func(runtime.Object) { - return func(obj runtime.Object) { +func withNamespace(ns string) func(runtime.Object) error { + return func(obj runtime.Object) error { switch v := obj.(type) { case *rbacv1.Role: v.Namespace = ns @@ -255,18 +281,24 @@ func withNamespace(ns string) func(runtime.Object) { v.Namespace = ns case *appsv1.Deployment: v.Namespace = ns + default: + return errors.Errorf("withNamespace() called on a non supported object") } + + return nil } } -func withEnvVar(key, val string) func(obj runtime.Object) { - return func(obj runtime.Object) { +func withEnvVar(key, val string) func(obj runtime.Object) error { + return func(obj runtime.Object) error { if testPod, ok := obj.(*corev1.Pod); ok { testPod.Spec.Containers[0].Env = updateEnvVarList(testPod.Spec.Containers[0].Env, key, val) } if testDeployment, ok := obj.(*appsv1.Deployment); ok { testDeployment.Spec.Template.Spec.Containers[0].Env = updateEnvVarList(testDeployment.Spec.Template.Spec.Containers[0].Env, key, val) } + + return nil } } @@ -282,11 +314,14 @@ func updateEnvVarList(envVarList []corev1.EnvVar, key, val string) []corev1.EnvV // withOperatorImage assumes that the underlying type is an appsv1.Deployment // which has the operator container as the first container. There will be -// no effect when used with a non-deployment type -func withOperatorImage(image string) func(runtime.Object) { - return func(obj runtime.Object) { +// an error return when used with a non-deployment type +func withOperatorImage(image string) func(runtime.Object) error { + return func(obj runtime.Object) error { if dep, ok := obj.(*appsv1.Deployment); ok { dep.Spec.Template.Spec.Containers[0].Image = image + return nil } + + return fmt.Errorf("withOperatorImage() called on a non-deployment object") } } From f8cdd9c611d8f792e372887c4bd2ef86da6b5f96 Mon Sep 17 00:00:00 2001 From: killian2k Date: Fri, 9 Jul 2021 16:55:48 +0200 Subject: [PATCH 357/790] Release MongoDB Kubernetes Operator v0.7.0 (#627) --- config/manager/manager.yaml | 17 ++++++++--------- deploy/openshift/operator_openshift.yaml | 20 ++++++++++---------- docs/RELEASE_NOTES.md | 6 +++--- release.json | 2 +- 4 files changed, 22 insertions(+), 23 deletions(-) diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 09c5e3d71..74d203982 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -1,21 +1,21 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: mongodb-kubernetes-operator - namespace: mongodb + annotations: + email: support@mongodb.com labels: owner: mongodb - annotations: - email: "support@mongodb.com" + name: mongodb-kubernetes-operator + namespace: mongodb spec: replicas: 1 selector: matchLabels: name: mongodb-kubernetes-operator strategy: - type: RollingUpdate rollingUpdate: maxUnavailable: 1 + type: RollingUpdate template: metadata: labels: @@ -30,7 +30,7 @@ spec: operator: In values: - mongodb-kubernetes-operator - topologyKey: "kubernetes.io/hostname" + topologyKey: kubernetes.io/hostname containers: - command: - /usr/local/bin/entrypoint @@ -46,7 +46,7 @@ spec: - name: OPERATOR_NAME value: mongodb-kubernetes-operator - name: AGENT_IMAGE - value: quay.io/mongodb/mongodb-agent:10.29.0.6830-1 + value: quay.io/mongodb/mongodb-agent:11.0.5.6963-1 - name: VERSION_UPGRADE_HOOK_IMAGE value: quay.io/mongodb/mongodb-kubernetes-operator-version-upgrade-post-start-hook:1.0.2 - name: READINESS_PROBE_IMAGE @@ -55,7 +55,7 @@ spec: value: mongo - name: MONGODB_REPO_URL value: docker.io - image: quay.io/mongodb/mongodb-kubernetes-operator:0.6.2 + image: quay.io/mongodb/mongodb-kubernetes-operator:0.7.0 imagePullPolicy: Always name: mongodb-kubernetes-operator resources: @@ -68,5 +68,4 @@ spec: securityContext: readOnlyRootFilesystem: true runAsUser: 2000 - serviceAccountName: mongodb-kubernetes-operator diff --git a/deploy/openshift/operator_openshift.yaml b/deploy/openshift/operator_openshift.yaml index a15edb68c..1b54b9354 100644 --- a/deploy/openshift/operator_openshift.yaml +++ b/deploy/openshift/operator_openshift.yaml @@ -1,21 +1,21 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: mongodb-kubernetes-operator - namespace: mongodb + annotations: + email: support@mongodb.com labels: owner: mongodb - annotations: - email: "support@mongodb.com" + name: mongodb-kubernetes-operator + namespace: mongodb spec: replicas: 1 selector: matchLabels: name: mongodb-kubernetes-operator strategy: - type: RollingUpdate rollingUpdate: maxUnavailable: 1 + type: RollingUpdate template: metadata: labels: @@ -30,10 +30,9 @@ spec: operator: In values: - mongodb-kubernetes-operator - topologyKey: "kubernetes.io/hostname" + topologyKey: kubernetes.io/hostname containers: - - name: mongodb-kubernetes-operator - command: + - command: - mongodb-kubernetes-operator env: - name: WATCH_NAMESPACE @@ -49,7 +48,7 @@ spec: - name: OPERATOR_NAME value: mongodb-kubernetes-operator - name: AGENT_IMAGE - value: quay.io/mongodb/mongodb-agent:10.29.0.6830-1 + value: quay.io/mongodb/mongodb-agent:11.0.5.6963-1 - name: READINESS_PROBE_IMAGE value: quay.io/mongodb/mongodb-kubernetes-readinessprobe:1.0.4 - name: VERSION_UPGRADE_HOOK_IMAGE @@ -58,8 +57,9 @@ spec: value: mongo - name: MONGODB_REPO_URL value: docker.io - image: quay.io/mongodb/mongodb-kubernetes-operator:0.6.2 + image: quay.io/mongodb/mongodb-kubernetes-operator:0.7.0 imagePullPolicy: Always + name: mongodb-kubernetes-operator resources: limits: cpu: 1100m diff --git a/docs/RELEASE_NOTES.md b/docs/RELEASE_NOTES.md index 13bb54b10..7b09cebb4 100644 --- a/docs/RELEASE_NOTES.md +++ b/docs/RELEASE_NOTES.md @@ -1,4 +1,4 @@ -# MongoDB Kubernetes Operator 0.6.3 +# MongoDB Kubernetes Operator 0.7.0 ## Kubernetes Operator @@ -7,12 +7,12 @@ - Reduce the number of permissions for operator role. - Support SHA-1 as an authentication method. - Upgraded `mongodbcommunity.mongodbcommunity.mongodb.com` CRD to `v1` from `v1beta1` - * Users upgrading their CRD from v1beta1 to v1 need to set: `spec.preserveUnknownFields` to `false` in the CRD file `config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml` before applying the CRD to the cluster. + - Users upgrading their CRD from v1beta1 to v1 need to set: `spec.preserveUnknownFields` to `false` in the CRD file `config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml` before applying the CRD to the cluster. - Made service name configurable in mongdb custom resource with statefulSet.spec.serviceName ## Updated Image Tags -- mongodb-kubernetes-operator:0.6.3 +- mongodb-kubernetes-operator:0.7.0 _All the images can be found in:_ diff --git a/release.json b/release.json index 9b64c594f..81c06f76c 100644 --- a/release.json +++ b/release.json @@ -1,5 +1,5 @@ { - "mongodb-kubernetes-operator": "0.6.2", + "mongodb-kubernetes-operator": "0.7.0", "version-upgrade-hook": "1.0.2", "readiness-probe": "1.0.4", "mongodb-agent": { From 829a38471bea02a0f6be25789843914589c95eb8 Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Fri, 9 Jul 2021 16:01:24 +0100 Subject: [PATCH 358/790] updated release actions (#628) --- .github/workflows/comment-release-pr.yml | 21 ++++++++++++++++ .github/workflows/release-images.yml | 32 +----------------------- docs/how-to-release.md | 2 +- 3 files changed, 23 insertions(+), 32 deletions(-) create mode 100644 .github/workflows/comment-release-pr.yml diff --git a/.github/workflows/comment-release-pr.yml b/.github/workflows/comment-release-pr.yml new file mode 100644 index 000000000..58026f6ed --- /dev/null +++ b/.github/workflows/comment-release-pr.yml @@ -0,0 +1,21 @@ +name: Link Github Releases +on: + pull_request: + types: [closed] + +jobs: + comment: + # only link releases on release PRs + if: startsWith(github.event.pull_request.title, 'Release MongoDB Kubernetes Operator') + runs-on: ubuntu-latest + steps: + - uses: actions/github-script@v3 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + github.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: 'Review and publish the release here: https://github.com/mongodb/mongodb-kubernetes-operator/releases + }) diff --git a/.github/workflows/release-images.yml b/.github/workflows/release-images.yml index 94f72f8e7..672e25cb4 100644 --- a/.github/workflows/release-images.yml +++ b/.github/workflows/release-images.yml @@ -64,7 +64,7 @@ jobs: ATLAS_DATABASE: "${{ secrets.ATLAS_DATABASE }}" ATLAS_CONNECTION_STRING: "${{ secrets.ATLAS_CONNECTION_STRING }}" - merge-release-pr-and-draft-github-release: + create-draft-release: runs-on: ubuntu-latest needs: [release-images] steps: @@ -75,14 +75,6 @@ jobs: run: | OUTPUT=$(jq -r '."mongodb-kubernetes-operator"' < $GITHUB_WORKSPACE/release.json) echo "::set-output name=OUTPUT::$OUTPUT" - - name: Merge Release PR - uses: pascalgn/automerge-action@v0.14.2 - env: - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - MERGE_LABELS: "" - MERGE_DELETE_BRANCH: true - MERGE_METHOD: squash - MERGE_COMMIT_MESSAGE: "Release MongoDB Kubernetes Operator v${{ steps.release_tag.outputs.OUTPUT }}" - name: Create Github Release uses: ncipollo/release-action@v1 with: @@ -91,25 +83,3 @@ jobs: bodyFile: "${{ github.workspace }}/docs/RELEASE_NOTES.md" draft: true token: ${{ secrets.GITHUB_TOKEN }} - - comment: - runs-on: ubuntu-latest - needs: [merge-release-pr-and-draft-github-release] - steps: - - name: Checkout Code - uses: actions/checkout@v2 - - name: Determine Release Version - id: release_version - run: | - VERSION=$(jq -r '."mongodb-kubernetes-operator"' < $GITHUB_WORKSPACE/release.json) - echo "::set-output name=version::${VERSION}" - - uses: actions/github-script@v3 - with: - github-token: ${{secrets.GITHUB_TOKEN}} - script: | - github.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: 'Review and publish the release here: https://github.com/mongodb/mongodb-kubernetes-operator/releases/tag/v${{ steps.release_version.outputs.version }}' - }) diff --git a/docs/how-to-release.md b/docs/how-to-release.md index e0a825cba..137d4e8b3 100644 --- a/docs/how-to-release.md +++ b/docs/how-to-release.md @@ -10,6 +10,6 @@ * Have this PR Reviewed and Approved. * Upon approval, all new images for this release will be built and released, and a Github release draft will be created. - * No need to merge this PR, when all images are successfully released, the PR will be merged. + * Once tests have passed, merge the release PR. * Review and publish the new GitHub release draft. From 1b5925c18368e5015efc32a00cf3f9680ca8a32c Mon Sep 17 00:00:00 2001 From: Priyadarshi Lahiri <1254077+priyolahiri@users.noreply.github.com> Date: Mon, 12 Jul 2021 16:16:07 +0200 Subject: [PATCH 359/790] CLOUDP-90359: Tested split-horizon and added docs (#629) * CLOUDP-90359: Tested split-horizon and added docs Tested split horizon, added docs and example manifests for deploying MongoDB with external access. * CLOUDP-90359: Added kube-linter annotation to ignore Added kube-linter annotation to ignore external_services.yaml * CLOUDP-90359: Fixed annotation Fixed kube-linter annotation * Update docs/external_access.md Co-authored-by: Nikolas De Giorgis * CLOUDP-90359: Implemented PR feedback Removed namespaces from yml files, corrected cert retrieval step, misc other changes Co-authored-by: Nikolas De Giorgis --- .../cert-manager-certificate.yaml | 16 ++++ .../external_access/cert-manager-issuer.yaml | 8 ++ .../external_access/external_services.yaml | 53 +++++++++++ .../mongodb.com_v1_mongodbcommunity_cr.yaml | 46 ++++++++++ docs/external_access.md | 88 +++++++++++++++++++ 5 files changed, 211 insertions(+) create mode 100644 config/samples/external_access/cert-manager-certificate.yaml create mode 100644 config/samples/external_access/cert-manager-issuer.yaml create mode 100644 config/samples/external_access/external_services.yaml create mode 100644 config/samples/external_access/mongodb.com_v1_mongodbcommunity_cr.yaml create mode 100644 docs/external_access.md diff --git a/config/samples/external_access/cert-manager-certificate.yaml b/config/samples/external_access/cert-manager-certificate.yaml new file mode 100644 index 000000000..6551bcda3 --- /dev/null +++ b/config/samples/external_access/cert-manager-certificate.yaml @@ -0,0 +1,16 @@ +--- +apiVersion: cert-manager.io/v1alpha2 +kind: Certificate +metadata: + name: cert-manager-certificate +spec: + secretName: mongodb-tls + issuerRef: + name: ca-issuer + kind: Issuer + commonName: "*.-svc..svc.cluster.local" + dnsNames: + - "*.-svc..svc.cluster.local" + - + - + - diff --git a/config/samples/external_access/cert-manager-issuer.yaml b/config/samples/external_access/cert-manager-issuer.yaml new file mode 100644 index 000000000..578c343b0 --- /dev/null +++ b/config/samples/external_access/cert-manager-issuer.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: cert-manager.io/v1alpha2 +kind: Issuer +metadata: + name: ca-issuer +spec: + ca: + secretName: ca-key-pair diff --git a/config/samples/external_access/external_services.yaml b/config/samples/external_access/external_services.yaml new file mode 100644 index 000000000..b14f1a673 --- /dev/null +++ b/config/samples/external_access/external_services.yaml @@ -0,0 +1,53 @@ +--- +kind: Service +apiVersion: v1 +metadata: + name: external-mongo-service-0 + annotations: + kube-linter.io/ignore-all: "used for sample" +spec: + type: NodePort + selector: + app: -svc + statefulset.kubernetes.io/pod-name: -0 + ports: + - protocol: TCP + nodePort: 31181 + port: 31181 + targetPort: 27017 + + +--- +kind: Service +apiVersion: v1 +metadata: + name: external-mongo-service-1 + annotations: + kube-linter.io/ignore-all: "used for sample" +spec: + type: NodePort + selector: + app: -svc + statefulset.kubernetes.io/pod-name: -1 + ports: + - nodePort: 31182 + port: 31182 + targetPort: 27017 + + +--- +kind: Service +apiVersion: v1 +metadata: + name: external-mongo-service-2 + annotations: + kube-linter.io/ignore-all: "used for sample" +spec: + type: NodePort + selector: + app: -svc + statefulset.kubernetes.io/pod-name: -2 + ports: + - nodePort: 31183 + port: 31183 + targetPort: 27017 diff --git a/config/samples/external_access/mongodb.com_v1_mongodbcommunity_cr.yaml b/config/samples/external_access/mongodb.com_v1_mongodbcommunity_cr.yaml new file mode 100644 index 000000000..14151e70f --- /dev/null +++ b/config/samples/external_access/mongodb.com_v1_mongodbcommunity_cr.yaml @@ -0,0 +1,46 @@ +--- +apiVersion: mongodbcommunity.mongodb.com/v1 +kind: MongoDBCommunity +metadata: + name: +spec: + members: 3 + type: ReplicaSet + version: "4.2.6" + replicaSetHorizons: + - horizon: :31181 + - horizon: :31182 + - horizon: :31183 + security: + tls: + enabled: true + certificateKeySecretRef: + name: mongodb-tls + caConfigMapRef: + name: ca-config-map + authentication: + modes: ["SCRAM"] + users: + - name: my-user + db: admin + passwordSecretRef: # a reference to the secret that will be used to generate the user's password + name: my-user-password + roles: + - name: clusterAdmin + db: admin + - name: userAdminAnyDatabase + db: admin + scramCredentialsSecretName: my-scram + + +# the user credentials will be generated from this secret +# once the credentials are generated, this secret is no longer required +--- +apiVersion: v1 +kind: Secret +metadata: + name: my-user-password +type: Opaque +stringData: + password: + diff --git a/docs/external_access.md b/docs/external_access.md new file mode 100644 index 000000000..4d0812b55 --- /dev/null +++ b/docs/external_access.md @@ -0,0 +1,88 @@ +## Enabling External Access to MongoDB deployment + +This guide assumes that the operator is installed and a MongoDB deployment is yet to be done but you have a chosen namespace that you are installing into. We will install cert-manager and then generate certificates and configure split-horizon to support internal and external DNS names for configuring external access to the replicaset. + +### Install cert-manager + +```sh +kubectl create namespace cert-manager +helm repo add jetstack https://charts.jetstack.io +helm repo update +helm install \ + cert-manager jetstack/cert-manager \ + --namespace cert-manager \ + --version v1.3.1 \ + --set installCRDs=true +``` + +### Install mkcert and generate CA + +```sh +brew install mkcert # for Mac +#for Linux / Windows systems look at https://github.com/FiloSottile/mkcert +mkcert -install +``` + +Execute ```mkcert --CAROOT``` to note the location of the generated root CA key and cert. + +### Retrieve the CA and create configmaps and secrets + +Use the files that you found in the previous step. Replace `````` with your chosen namespace + +```sh +kubectl create configmap ca-config-map --from-file=ca.crt --namespace + +kubectl create secret tls ca-key-pair --cert=ca.crt --key=ca.key --namespace +``` + +### Create the Cert Manager issuer and secret + +Edit the file ```cert-manager-certificate.yaml``` to replace `````` with your MongoDB deployment name. Also replace ``````, ``````, and `````` with the external FQDNs of the MongoDB replicaset members. Please remember that you will have to add an equal number of entries for each member of the replicaset. + +Apply the manifests. Replace `````` with the namespace you are using for the deployment. + +```sh +kubectl apply -f config/samples/external_access/cert-manager-issuer.yaml --namespace +kubectl apply -f config/samples/external_access/cert-manager-certificate.yaml --namespace +``` + +### Create the MongoDB deployment + +Edit ```config/samples/external_access/mongodb.com_v1_mongodbcommunity_cr.yaml```. Replace with the desired MongoDB deployment name -- this should be the same as in the previous step. Replace ``````, ``````, and `````` with the external FQDNs of the MongoDB replicaset members. Please remember that you should have the same number of entries in this section as the number of your replicaset members. You can also edit the ports for external access to your preferred numbers in this section -- you will have to remember to change them in the next step too. Change `````` to your desired admin password for MongoDB. + +Apply the manifest. + +```sh +kubectl apply -f config/samples/external_access/mongodb.com_v1_mongodbcommunity_cr.yaml +``` + +Wait for the replicaset to be available. + +### Create the external NodePort services for accessing the MongoDB deployment from outside the Kubernetes cluster + +Edit ```config/samples/external_access/external_services.yaml``` and replace `````` with the MongoDB deployment name that you have used in the preceeding steps. You can change the ```nodePort``` and ```port``` to reflect the changes (if any) you have made in the previous steps. + +Apply the manifest. + +```sh +kubectl apply -f config/samples/external_access/external_services.yaml +``` + +### Retrieve the certificates from a MongoDB replicaset member + +```sh +kubectl exec --namespace mcommunity -it -0 -c mongod -- bash +``` + +Once inside the container ```cat``` and copy the contents of the ```.pem``` file in ```/var/lib/tls/server``` into a file on your local system. + +### Connect to the MongoDB deployment from outside the Kubernetes cluster + +This is an example to connect to the MongoDB cluster with Mongo shell. Use the CA from ```mkcert``` and the certificate from the previous step. Replace the values in the command from the preceeding steps. + +```sh +mongosh --tls --tlsCAfile ca.crt --tlsCertificateKeyFile key.pem --username my-user --password mongodb://:31181,:31182,:31183 +``` + +### Conclusion +At this point, you should be able to connect to the MongoDB deployment from outside the cluster. Make sure that you can resolve to the FQDNs for the replicaset members where you have the Mongo client installed. From 7712d25ddaf31f4ce420ca9774f1c5a5e35763a6 Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Tue, 13 Jul 2021 10:58:21 +0100 Subject: [PATCH 360/790] CLOUDP-89483: tests for custom volumes (#603) --- .action_templates/jobs/tests.yaml | 4 + .../steps/setup-kind-cluster.yaml | 3 + .github/workflows/e2e-dispatch.yml | 3 + .github/workflows/e2e-fork.yml | 8 + .github/workflows/e2e.yml | 8 + test/e2e/mongodbtests/mongodbtests.go | 27 ++++ .../replica_set_custom_persistent_volume.go | 144 ++++++++++++++++++ 7 files changed, 197 insertions(+) create mode 100644 test/e2e/replica_set_custom_persistent_volume/replica_set_custom_persistent_volume.go diff --git a/.action_templates/jobs/tests.yaml b/.action_templates/jobs/tests.yaml index cf05f4288..9c893c38c 100644 --- a/.action_templates/jobs/tests.yaml +++ b/.action_templates/jobs/tests.yaml @@ -40,6 +40,8 @@ tests: distro: ubuntu - test-name: replica_set_arbiter distro: ubuntu + - test-name: replica_set_custom_persistent_volume + distro: ubuntu - test-name: replica_set distro: ubi @@ -76,3 +78,5 @@ tests: distro: ubi - test-name: replica_set_arbiter distro: ubi + - test-name: replica_set_custom_persistent_volume + distro: ubi diff --git a/.action_templates/steps/setup-kind-cluster.yaml b/.action_templates/steps/setup-kind-cluster.yaml index 0f97cf1be..b17558382 100644 --- a/.action_templates/steps/setup-kind-cluster.yaml +++ b/.action_templates/steps/setup-kind-cluster.yaml @@ -3,6 +3,9 @@ curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.11.1/kind-linux-amd64 chmod +x ./kind ./kind create cluster +- name: Create Directories + run: | + docker exec kind-control-plane mkdir -p /opt/data/mongo-data-0 /opt/data/mongo-data-1 /opt/data/mongo-data-2 /opt/data/mongo-logs-0 /opt/data/mongo-logs-1 /opt/data/mongo-logs-2 - name: Install CRD run: kubectl apply -f config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml diff --git a/.github/workflows/e2e-dispatch.yml b/.github/workflows/e2e-dispatch.yml index 9f040340b..cffb42f41 100644 --- a/.github/workflows/e2e-dispatch.yml +++ b/.github/workflows/e2e-dispatch.yml @@ -102,6 +102,9 @@ jobs: curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.11.1/kind-linux-amd64 chmod +x ./kind ./kind create cluster + - name: Create Directories + run: | + docker exec kind-control-plane mkdir -p /opt/data/mongo-data-0 /opt/data/mongo-data-1 /opt/data/mongo-data-2 /opt/data/mongo-logs-0 /opt/data/mongo-logs-1 /opt/data/mongo-logs-2 - name: Install CRD run: kubectl apply -f config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml diff --git a/.github/workflows/e2e-fork.yml b/.github/workflows/e2e-fork.yml index 37901714d..d6bb954bc 100644 --- a/.github/workflows/e2e-fork.yml +++ b/.github/workflows/e2e-fork.yml @@ -122,6 +122,8 @@ jobs: distro: ubuntu - test-name: replica_set_arbiter distro: ubuntu + - test-name: replica_set_custom_persistent_volume + distro: ubuntu - test-name: replica_set distro: ubi @@ -158,6 +160,8 @@ jobs: distro: ubi - test-name: replica_set_arbiter distro: ubi + - test-name: replica_set_custom_persistent_volume + distro: ubi steps: # template: .action_templates/steps/cancel-previous.yaml - name: Cancel Previous Runs @@ -206,6 +210,10 @@ jobs: curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.11.1/kind-linux-amd64 chmod +x ./kind ./kind create cluster + if: steps.last_run_status.outputs.last_run_status != 'success' + - name: Create Directories + run: | + docker exec kind-control-plane mkdir -p /opt/data/mongo-data-0 /opt/data/mongo-data-1 /opt/data/mongo-data-2 /opt/data/mongo-logs-0 /opt/data/mongo-logs-1 /opt/data/mongo-logs-2 if: steps.last_run_status.outputs.last_run_status != 'success' - name: Install CRD diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 105f19c27..4d695e8fc 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -127,6 +127,8 @@ jobs: distro: ubuntu - test-name: replica_set_arbiter distro: ubuntu + - test-name: replica_set_custom_persistent_volume + distro: ubuntu - test-name: replica_set distro: ubi @@ -163,6 +165,8 @@ jobs: distro: ubi - test-name: replica_set_arbiter distro: ubi + - test-name: replica_set_custom_persistent_volume + distro: ubi steps: # template: .action_templates/steps/cancel-previous.yaml - name: Cancel Previous Runs @@ -208,6 +212,10 @@ jobs: curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.11.1/kind-linux-amd64 chmod +x ./kind ./kind create cluster + if: steps.last_run_status.outputs.last_run_status != 'success' + - name: Create Directories + run: | + docker exec kind-control-plane mkdir -p /opt/data/mongo-data-0 /opt/data/mongo-data-1 /opt/data/mongo-data-2 /opt/data/mongo-logs-0 /opt/data/mongo-logs-1 /opt/data/mongo-logs-2 if: steps.last_run_status.outputs.last_run_status != 'success' - name: Install CRD diff --git a/test/e2e/mongodbtests/mongodbtests.go b/test/e2e/mongodbtests/mongodbtests.go index 72399fc7e..5b6b41056 100644 --- a/test/e2e/mongodbtests/mongodbtests.go +++ b/test/e2e/mongodbtests/mongodbtests.go @@ -156,6 +156,33 @@ func StatefulSetHasUpdateStrategy(mdb *mdbv1.MongoDBCommunity, strategy appsv1.S } } +// GetPersistentVolumes returns all persistent volumes on the cluster +func getPersistentVolumesList() (*corev1.PersistentVolumeList, error) { + return e2eutil.TestClient.CoreV1Client.PersistentVolumes().List(context.TODO(), metav1.ListOptions{}) +} + +func containsVolume(volumes []corev1.PersistentVolume, volumeName string) bool { + for _, v := range volumes { + if v.Name == volumeName { + return true + } + } + return false +} +func HasExpectedPersistentVolumes(volumes []corev1.PersistentVolume) func(t *testing.T) { + return func(t *testing.T) { + volumeList, err := getPersistentVolumesList() + actualVolumes := volumeList.Items + assert.NoError(t, err) + assert.Len(t, actualVolumes, len(volumes), + "The number of persistent volumes should be equal to the amount of volumes we created. Expected: %d, actual: %d", + len(volumes), len(actualVolumes)) + for _, v := range volumes { + assert.True(t, containsVolume(actualVolumes, v.Name)) + } + } +} + // MongoDBReachesRunningPhase ensure the MongoDB resource reaches the Running phase func MongoDBReachesRunningPhase(mdb *mdbv1.MongoDBCommunity) func(t *testing.T) { return func(t *testing.T) { diff --git a/test/e2e/replica_set_custom_persistent_volume/replica_set_custom_persistent_volume.go b/test/e2e/replica_set_custom_persistent_volume/replica_set_custom_persistent_volume.go new file mode 100644 index 000000000..5aaf92230 --- /dev/null +++ b/test/e2e/replica_set_custom_persistent_volume/replica_set_custom_persistent_volume.go @@ -0,0 +1,144 @@ +package replica_set_custom_persistent_volume + +import ( + "context" + "fmt" + "os" + "testing" + + v1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" + "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/mongodbtests" + . "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/util/mongotester" + "github.com/stretchr/testify/assert" + + e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" + setup "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" + corev1 "k8s.io/api/core/v1" + + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestMain(m *testing.M) { + code, err := e2eutil.RunTest(m) + if err != nil { + fmt.Println(err) + } + os.Exit(code) +} + +// getPersistentVolumeLocal returns a persistentVolume of type localPath and a "type" label. +func getPersistentVolumeLocal(name string, localPath string, label string) corev1.PersistentVolume { + return corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: e2eutil.OperatorNamespace, + Labels: map[string]string{"type": label}, + }, + Spec: corev1.PersistentVolumeSpec{ + PersistentVolumeSource: corev1.PersistentVolumeSource{ + Local: &corev1.LocalVolumeSource{ + Path: localPath, + }, + }, + AccessModes: []corev1.PersistentVolumeAccessMode{"ReadWriteOnce"}, + StorageClassName: "default", + NodeAffinity: &corev1.VolumeNodeAffinity{ + Required: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{ + { + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: "kubernetes.io/hostname", + Operator: "In", + Values: []string{"kind-control-plane"}, + }, + }, + }, + }, + }, + }, + }, + } +} + +// getVolumes returns two persistentVolumes for each of the `members` pod. +// one volume will be for the `data` claim and the other will be for the `logs` claim +func getVolumes(ctx *e2eutil.Context, volumeType string, members int) []corev1.PersistentVolume { + volumes := make([]corev1.PersistentVolume, members) + for i := 0; i < members; i++ { + volumes[i] = getPersistentVolumeLocal( + fmt.Sprintf("%s-volume-%d", volumeType, i), + fmt.Sprintf("/opt/data/mongo-%s-%d", volumeType, i), + volumeType, + ) + } + return volumes +} + +func getPvc(pvcType string, mdb v1.MongoDBCommunity) corev1.PersistentVolumeClaim { + name := "" + if pvcType == "logs" { + name = mdb.LogsVolumeName() + } else { + name = mdb.DataVolumeName() + } + defaultStorageClass := "default" + return corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"type": pvcType}, + }, + Resources: corev1.ResourceRequirements{ + Requests: map[corev1.ResourceName]resource.Quantity{"storage": *resource.NewScaledQuantity(int64(8), resource.Giga)}, + }, + StorageClassName: &defaultStorageClass, + }, + } +} + +func TestReplicaSetCustomPersistentVolumes(t *testing.T) { + ctx := setup.Setup(t) + defer ctx.Teardown() + + mdb, user := e2eutil.NewTestMongoDB(ctx, "mdb0", "") + mdb.Spec.StatefulSetConfiguration.SpecWrapper.Spec.VolumeClaimTemplates = []corev1.PersistentVolumeClaim{ + getPvc("data", mdb), + getPvc("logs", mdb), + } + volumesToCreate := getVolumes(ctx, "data", mdb.Spec.Members) + volumesToCreate = append(volumesToCreate, getVolumes(ctx, "logs", mdb.Spec.Members)...) + + for i := range volumesToCreate { + err := e2eutil.TestClient.Create(context.TODO(), &volumesToCreate[i], &e2eutil.CleanupOptions{TestContext: ctx}) + assert.NoError(t, err) + } + scramUser := mdb.GetScramUsers()[0] + + _, err := setup.GeneratePasswordForUser(ctx, user, "") + if err != nil { + t.Fatal(err) + } + + tester, err := FromResource(t, mdb) + if err != nil { + t.Fatal(err) + } + + t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) + t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb)) + t.Run("Keyfile authentication is configured", tester.HasKeyfileAuth(3)) + t.Run("Test Basic Connectivity", tester.ConnectivitySucceeds()) + t.Run("Test SRV Connectivity", tester.ConnectivitySucceeds(WithURI(mdb.MongoSRVURI()), WithoutTls(), WithReplicaSet((mdb.Name)))) + t.Run("Test Basic Connectivity with generated connection string secret", + tester.ConnectivitySucceeds(WithURI(mongodbtests.GetConnectionStringForUser(mdb, scramUser)))) + t.Run("Test SRV Connectivity with generated connection string secret", + tester.ConnectivitySucceeds(WithURI(mongodbtests.GetSrvConnectionStringForUser(mdb, scramUser)))) + t.Run("Ensure Authentication", tester.EnsureAuthenticationIsConfigured(3)) + t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 1)) + t.Run("Cluster has the expected persistent volumes", mongodbtests.HasExpectedPersistentVolumes(volumesToCreate)) +} From 37dec1c7ce038aeacc32cbe90ccc3c4c9d4c25cd Mon Sep 17 00:00:00 2001 From: killian2k Date: Wed, 14 Jul 2021 08:10:07 +0200 Subject: [PATCH 361/790] Cloudp 93925 add role for operator db (#632) * Fixing the make deploy command * A few modif to add resources * added the good files but does now pass tests * Error should be fixed * fix commit test * Clean the code * update replica_set_cross_ns test * Update replica_set_cross_namespace_deploy_test.go * rename variable * Update setup.go * update release --- config/rbac/role_binding_database.yaml | 12 +++++++ config/rbac/role_database.yaml | 20 ++++++++++++ config/rbac/service_account_database.yaml | 5 +++ .../construct/build_statefulset_test.go | 2 +- controllers/construct/mongodbstatefulset.go | 16 +++++----- docs/RELEASE_NOTES.md | 11 ++----- ...replica_set_cross_namespace_deploy_test.go | 32 +++++++++++++++++++ test/e2e/setup/setup.go | 11 +++++++ 8 files changed, 92 insertions(+), 17 deletions(-) create mode 100644 config/rbac/role_binding_database.yaml create mode 100644 config/rbac/role_database.yaml create mode 100644 config/rbac/service_account_database.yaml diff --git a/config/rbac/role_binding_database.yaml b/config/rbac/role_binding_database.yaml new file mode 100644 index 000000000..c8ff81f43 --- /dev/null +++ b/config/rbac/role_binding_database.yaml @@ -0,0 +1,12 @@ +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-database + namespace: mongodb +subjects: +- kind: ServiceAccount + name: mongodb-database +roleRef: + kind: Role + name: mongodb-database + apiGroup: rbac.authorization.k8s.io diff --git a/config/rbac/role_database.yaml b/config/rbac/role_database.yaml new file mode 100644 index 000000000..514690625 --- /dev/null +++ b/config/rbac/role_database.yaml @@ -0,0 +1,20 @@ +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-database + namespace: mongodb +rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - apiGroups: + - "" + resources: + - pods + verbs: + - patch + - delete + - get diff --git a/config/rbac/service_account_database.yaml b/config/rbac/service_account_database.yaml new file mode 100644 index 000000000..2d81c1bf7 --- /dev/null +++ b/config/rbac/service_account_database.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: mongodb-database + namespace: mongodb diff --git a/controllers/construct/build_statefulset_test.go b/controllers/construct/build_statefulset_test.go index f7ca3d7b4..7bd24da5d 100644 --- a/controllers/construct/build_statefulset_test.go +++ b/controllers/construct/build_statefulset_test.go @@ -84,7 +84,7 @@ func assertStatefulSetIsBuiltCorrectly(t *testing.T, mdb mdbv1.MongoDBCommunity, assert.Equal(t, mdb.ServiceName(), sts.Spec.ServiceName) assert.Equal(t, mdb.Name, sts.Name) assert.Equal(t, mdb.Namespace, sts.Namespace) - assert.Equal(t, operatorServiceAccountName, sts.Spec.Template.Spec.ServiceAccountName) + assert.Equal(t, mongodbDatabaseServiceAccountName, sts.Spec.Template.Spec.ServiceAccountName) assert.Len(t, sts.Spec.Template.Spec.Containers[0].Env, 4) assert.Len(t, sts.Spec.Template.Spec.Containers[1].Env, 1) diff --git a/controllers/construct/mongodbstatefulset.go b/controllers/construct/mongodbstatefulset.go index af04a7471..ee1c2df35 100644 --- a/controllers/construct/mongodbstatefulset.go +++ b/controllers/construct/mongodbstatefulset.go @@ -24,13 +24,13 @@ const ( AgentName = "mongodb-agent" MongodbName = "mongod" - versionUpgradeHookName = "mongod-posthook" - ReadinessProbeContainerName = "mongodb-agent-readinessprobe" - readinessProbePath = "/opt/scripts/readinessprobe" - agentHealthStatusFilePathEnv = "AGENT_STATUS_FILEPATH" - clusterFilePath = "/var/lib/automation/config/cluster-config.json" - operatorServiceAccountName = "mongodb-kubernetes-operator" - agentHealthStatusFilePathValue = "/var/log/mongodb-mms-automation/healthstatus/agent-health-status.json" + versionUpgradeHookName = "mongod-posthook" + ReadinessProbeContainerName = "mongodb-agent-readinessprobe" + readinessProbePath = "/opt/scripts/readinessprobe" + agentHealthStatusFilePathEnv = "AGENT_STATUS_FILEPATH" + clusterFilePath = "/var/lib/automation/config/cluster-config.json" + mongodbDatabaseServiceAccountName = "mongodb-database" + agentHealthStatusFilePathValue = "/var/log/mongodb-mms-automation/healthstatus/agent-health-status.json" MongodbRepoUrl = "MONGODB_REPO_URL" @@ -163,7 +163,7 @@ func BuildMongoDBReplicaSetStatefulSetModificationFunction(mdb MongoDBStatefulSe podtemplatespec.WithVolume(automationConfigVolume), podtemplatespec.WithVolume(scriptsVolume), podtemplatespec.WithVolume(keyFileVolume), - podtemplatespec.WithServiceAccount(operatorServiceAccountName), + podtemplatespec.WithServiceAccount(mongodbDatabaseServiceAccountName), podtemplatespec.WithContainer(AgentName, mongodbAgentContainer(mdb.AutomationConfigSecretName(), mongodbAgentVolumeMounts)), podtemplatespec.WithContainer(MongodbName, mongodbContainer(mdb.GetMongoDBVersion(), mongodVolumeMounts)), podtemplatespec.WithInitContainer(versionUpgradeHookName, versionUpgradeHookInit([]corev1.VolumeMount{hooksVolumeMount})), diff --git a/docs/RELEASE_NOTES.md b/docs/RELEASE_NOTES.md index 7b09cebb4..649393502 100644 --- a/docs/RELEASE_NOTES.md +++ b/docs/RELEASE_NOTES.md @@ -1,18 +1,13 @@ -# MongoDB Kubernetes Operator 0.7.0 +# MongoDB Kubernetes Operator 0.7.1 ## Kubernetes Operator - Changes - - Members of a Replica Set can be configured as arbiters. - - Reduce the number of permissions for operator role. - - Support SHA-1 as an authentication method. - - Upgraded `mongodbcommunity.mongodbcommunity.mongodb.com` CRD to `v1` from `v1beta1` - - Users upgrading their CRD from v1beta1 to v1 need to set: `spec.preserveUnknownFields` to `false` in the CRD file `config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml` before applying the CRD to the cluster. - - Made service name configurable in mongdb custom resource with statefulSet.spec.serviceName + - MongoDB database of the statefulSet is managed using distinct Role, ServiceAccount and RoleBinding. ## Updated Image Tags -- mongodb-kubernetes-operator:0.7.0 +- mongodb-kubernetes-operator:0.7.1 _All the images can be found in:_ diff --git a/test/e2e/replica_set_cross_namespace_deploy/replica_set_cross_namespace_deploy_test.go b/test/e2e/replica_set_cross_namespace_deploy/replica_set_cross_namespace_deploy_test.go index 118cb83fd..4dace6476 100644 --- a/test/e2e/replica_set_cross_namespace_deploy/replica_set_cross_namespace_deploy_test.go +++ b/test/e2e/replica_set_cross_namespace_deploy/replica_set_cross_namespace_deploy_test.go @@ -146,6 +146,38 @@ func TestCrossNamespaceDeploy(t *testing.T) { t.Fatal(err) } + err = e2eutil.EnsureServiceAccount(ctx, namespace, "mongodb-database") + if err != nil { + t.Fatal(err) + } + err = e2eutil.EnsureRole(ctx, namespace, "mongodb-database", []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"secrets"}, + Verbs: []string{"get"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"pods"}, + Verbs: []string{"delete", "get", "patch"}, + }, + }) + if err != nil { + t.Fatal(err) + } + err = e2eutil.EnsureRoleBinding(ctx, namespace, "mongodb-database", + []rbacv1.Subject{{ + Kind: "ServiceAccount", + Name: "mongodb-database", + }}, rbacv1.RoleRef{ + Kind: "Role", + APIGroup: "rbac.authorization.k8s.io", + Name: "mongodb-database", + }) + if err != nil { + t.Fatal(err) + } + mdb, user := e2eutil.NewTestMongoDB(ctx, "mdb0", namespace) _, err = setup.GeneratePasswordForUser(ctx, user, namespace) diff --git a/test/e2e/setup/setup.go b/test/e2e/setup/setup.go index 5ee2e412b..1eaace825 100644 --- a/test/e2e/setup/setup.go +++ b/test/e2e/setup/setup.go @@ -158,6 +158,17 @@ func deployOperator(ctx *e2eutil.Context) error { } fmt.Println("Successfully created the operator Role Binding") + if err := buildKubernetesResourceFromYamlFile(ctx, path.Join(roleDir(), "role_database.yaml"), &rbacv1.Role{}, withNamespace(testConfig.namespace)); err != nil { + return errors.Errorf("error building mongodb database role: %s", err) + } + if err := buildKubernetesResourceFromYamlFile(ctx, path.Join(roleDir(), "service_account_database.yaml"), &corev1.ServiceAccount{}, withNamespace(testConfig.namespace)); err != nil { + return errors.Errorf("error building mongodb database service account: %s", err) + } + if err := buildKubernetesResourceFromYamlFile(ctx, path.Join(roleDir(), "role_binding_database.yaml"), &rbacv1.RoleBinding{}, withNamespace(testConfig.namespace)); err != nil { + return errors.Errorf("error building mongodb database role binding: %s", err) + } + fmt.Println("Successfully created the role, service account and role binding for the database") + dep := &appsv1.Deployment{} if err := buildKubernetesResourceFromYamlFile(ctx, path.Join(deployDir(), "manager.yaml"), From dda6cebc6e33481832efeb1834e51d1582918fe2 Mon Sep 17 00:00:00 2001 From: FlorentinaSimlinger <60526092+FlorentinaSimlinger@users.noreply.github.com> Date: Wed, 14 Jul 2021 13:38:01 +0100 Subject: [PATCH 362/790] CLOUDP-78289: cr annotations update (#635) --- api/v1/mongodbcommunity_types.go | 8 ++++---- controllers/replica_set_controller.go | 13 ++++++------- controllers/validation/validation.go | 4 ++-- pkg/agent/agent_readiness.go | 4 ++-- 4 files changed, 14 insertions(+), 15 deletions(-) diff --git a/api/v1/mongodbcommunity_types.go b/api/v1/mongodbcommunity_types.go index f75800b15..cfec1a43c 100644 --- a/api/v1/mongodbcommunity_types.go +++ b/api/v1/mongodbcommunity_types.go @@ -592,12 +592,12 @@ func (m MongoDBCommunity) GetUpdateStrategyType() appsv1.StatefulSetUpdateStrate // IsChangingVersion returns true if an attempted version change is occurring. func (m MongoDBCommunity) IsChangingVersion() bool { - prevVersion := m.getPreviousVersion() - return prevVersion != "" && prevVersion != m.Spec.Version + lastVersion := m.getLastVersion() + return lastVersion != "" && lastVersion != m.Spec.Version } -// GetPreviousVersion returns the last MDB version the statefulset was configured with. -func (m MongoDBCommunity) getPreviousVersion() string { +// GetLastVersion returns the MDB version the statefulset was configured with. +func (m MongoDBCommunity) getLastVersion() string { return annotations.GetAnnotation(&m, annotations.LastAppliedMongoDBVersion) } diff --git a/controllers/replica_set_controller.go b/controllers/replica_set_controller.go index 735afd9c4..ef95cb6a4 100644 --- a/controllers/replica_set_controller.go +++ b/controllers/replica_set_controller.go @@ -464,10 +464,9 @@ func buildService(mdb mdbv1.MongoDBCommunity) corev1.Service { Build() } -// validateSpec checks if MongoDB resource Spec is valid. -// If there is no a previous Spec, then the function assumes this is the first version -// and runs the initial spec validations otherwise it validates that the new Spec, -// corresponding to the existing one is still valid +// validateSpec checks if the MongoDB resource Spec is valid. +// If there has not yet been a successful configuration, the function runs the intial Spec validations. Otherwise +// it checks that the attempted Spec is valid in relation to the Spec that resulted from that last successful configuration. func (r ReplicaSetReconciler) validateSpec(mdb mdbv1.MongoDBCommunity) error { lastSuccessfulConfigurationSaved, ok := mdb.Annotations[lastSuccessfulConfiguration] if !ok { @@ -475,13 +474,13 @@ func (r ReplicaSetReconciler) validateSpec(mdb mdbv1.MongoDBCommunity) error { return validation.ValidateInitalSpec(mdb) } - prevSpec := mdbv1.MongoDBCommunitySpec{} - err := json.Unmarshal([]byte(lastSuccessfulConfigurationSaved), &prevSpec) + lastSpec := mdbv1.MongoDBCommunitySpec{} + err := json.Unmarshal([]byte(lastSuccessfulConfigurationSaved), &lastSpec) if err != nil { return err } - return validation.ValidateUpdate(mdb, prevSpec) + return validation.ValidateUpdate(mdb, lastSpec) } func getCustomRolesModification(mdb mdbv1.MongoDBCommunity) (automationconfig.Modification, error) { diff --git a/controllers/validation/validation.go b/controllers/validation/validation.go index 6890d3e17..5510f5487 100644 --- a/controllers/validation/validation.go +++ b/controllers/validation/validation.go @@ -9,12 +9,12 @@ import ( "github.com/pkg/errors" ) -// ValidateInitalSpec checks if the resource initial Spec is valid +// ValidateInitalSpec checks if the resource's initial Spec is valid. func ValidateInitalSpec(mdb mdbv1.MongoDBCommunity) error { return validateSpec(mdb) } -// ValidateUpdate validates that the new Spec, corresponding to the existing one is still valid +// ValidateUpdate validates that the new Spec, corresponding to the existing one, is still valid. func ValidateUpdate(mdb mdbv1.MongoDBCommunity, oldSpec mdbv1.MongoDBCommunitySpec) error { if oldSpec.Security.TLS.Enabled && !mdb.Spec.Security.TLS.Enabled { return errors.New("TLS can't be set to disabled after it has been enabled") diff --git a/pkg/agent/agent_readiness.go b/pkg/agent/agent_readiness.go index 320c67957..fda6dc0cd 100644 --- a/pkg/agent/agent_readiness.go +++ b/pkg/agent/agent_readiness.go @@ -14,7 +14,7 @@ import ( const ( // podAnnotationAgentVersion is the Pod Annotation key which contains the current version of the Automation Config - // the Agent on the Pod is on now + // the Agent on the Pod is on now. podAnnotationAgentVersion = "agent.mongodb.com/version" ) @@ -53,7 +53,7 @@ func AllReachedGoalState(sts appsv1.StatefulSet, podGetter pod.Getter, desiredMe return true, nil } -// ReachedGoalState checks if a single Agent has reached the goal state. To do this it reads the Pod annotation +// ReachedGoalState checks if a single Agent has reached the goal state. To do this it reads the Pod annotation // to find out the current version the Agent is on. func ReachedGoalState(pod corev1.Pod, targetConfigVersion int, log *zap.SugaredLogger) bool { currentAgentVersion, ok := pod.Annotations[podAnnotationAgentVersion] From 72f059dca150bc7c4a48cbcc5860378f418f014d Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Fri, 16 Jul 2021 10:53:16 +0100 Subject: [PATCH 363/790] bump sonar version to 0.0.12 (#641) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 80510bc9b..2452c04e6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -git+https://github.com/mongodb/sonar@0.0.11#egg=sonar +git+https://github.com/mongodb/sonar@0.0.12#egg=sonar github-action-templates==0.0.4 docker==4.3.1 kubernetes==11.0.0 From c2035e4f2cdc4aafd3c9fea420191989ce86ce50 Mon Sep 17 00:00:00 2001 From: Rajdeep Das Date: Fri, 16 Jul 2021 12:17:36 +0200 Subject: [PATCH 364/790] Don't hardcode namespace in manager -- users can deploy in their own namespaces. (#640) --- config/manager/manager.yaml | 1 - config/rbac/role.yaml | 1 - config/rbac/role_binding_database.yaml | 1 - config/rbac/role_database.yaml | 1 - config/rbac/service_account_database.yaml | 1 - deploy/clusterwide/role_binding.yaml | 1 - deploy/openshift/operator_openshift.yaml | 1 - 7 files changed, 7 deletions(-) diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 74d203982..4f5a28c1c 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -6,7 +6,6 @@ metadata: labels: owner: mongodb name: mongodb-kubernetes-operator - namespace: mongodb spec: replicas: 1 selector: diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index d5ba4af61..6a9c42070 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -3,7 +3,6 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: mongodb-kubernetes-operator - namespace: mongodb rules: - apiGroups: - "" diff --git a/config/rbac/role_binding_database.yaml b/config/rbac/role_binding_database.yaml index c8ff81f43..b02a52db3 100644 --- a/config/rbac/role_binding_database.yaml +++ b/config/rbac/role_binding_database.yaml @@ -2,7 +2,6 @@ kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: mongodb-database - namespace: mongodb subjects: - kind: ServiceAccount name: mongodb-database diff --git a/config/rbac/role_database.yaml b/config/rbac/role_database.yaml index 514690625..eaeef740b 100644 --- a/config/rbac/role_database.yaml +++ b/config/rbac/role_database.yaml @@ -2,7 +2,6 @@ kind: Role apiVersion: rbac.authorization.k8s.io/v1 metadata: name: mongodb-database - namespace: mongodb rules: - apiGroups: - "" diff --git a/config/rbac/service_account_database.yaml b/config/rbac/service_account_database.yaml index 2d81c1bf7..b24ae9d58 100644 --- a/config/rbac/service_account_database.yaml +++ b/config/rbac/service_account_database.yaml @@ -2,4 +2,3 @@ apiVersion: v1 kind: ServiceAccount metadata: name: mongodb-database - namespace: mongodb diff --git a/deploy/clusterwide/role_binding.yaml b/deploy/clusterwide/role_binding.yaml index a3a4ddc00..d9be64667 100644 --- a/deploy/clusterwide/role_binding.yaml +++ b/deploy/clusterwide/role_binding.yaml @@ -5,7 +5,6 @@ metadata: subjects: - kind: ServiceAccount name: mongodb-kubernetes-operator - namespace: mongodb roleRef: kind: ClusterRole name: mongodb-kubernetes-operator diff --git a/deploy/openshift/operator_openshift.yaml b/deploy/openshift/operator_openshift.yaml index 1b54b9354..d7d5ffd07 100644 --- a/deploy/openshift/operator_openshift.yaml +++ b/deploy/openshift/operator_openshift.yaml @@ -6,7 +6,6 @@ metadata: labels: owner: mongodb name: mongodb-kubernetes-operator - namespace: mongodb spec: replicas: 1 selector: From 65db6a0d7716b6b6a7271e19bd38ff9a1b0aed3d Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Mon, 19 Jul 2021 09:43:19 +0100 Subject: [PATCH 365/790] fixed agent version incorrectly specified (#638) --- inventory.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/inventory.yaml b/inventory.yaml index dbd295447..62c062896 100644 --- a/inventory.yaml +++ b/inventory.yaml @@ -85,7 +85,7 @@ images: output: - registry: $(inputs.params.registry)/$(inputs.params.agent_image) - tag: $(inputs.params.version_id)-context + tag: $(inputs.params.agent_version)-context - name: agent-ubuntu-release task_type: docker_build @@ -103,7 +103,7 @@ images: output: - registry: $(inputs.params.registry)/$(inputs.params.agent_image) - tag: $(inputs.params.version_id) + tag: $(inputs.params.agent_version) - name: agent-ubi vars: @@ -189,7 +189,7 @@ images: output: - registry: $(inputs.params.registry)/$(inputs.params.agent_image) - tag: $(inputs.params.version_id)-context + tag: $(inputs.params.agent_version)-context - name: agent-ubi-release task_type: docker_build @@ -205,7 +205,7 @@ images: output: - registry: $(inputs.params.registry)/$(inputs.params.agent_image) - tag: $(inputs.params.version_id) + tag: $(inputs.params.agent_version) - name: readiness-probe-init vars: From 83964c8d5d4c87d8380a5459253a755d93cec1ea Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Tue, 20 Jul 2021 14:17:13 +0100 Subject: [PATCH 366/790] CLOUDP-95679: Added support for pem entry in tls secret (#646) Co-authored-by: Rajdeep Das --- .action_templates/jobs/tests.yaml | 4 + .github/workflows/e2e-fork.yml | 4 + .github/workflows/e2e.yml | 4 + api/v1/mongodbcommunity_types.go | 2 + ...ommunity.mongodb.com_mongodbcommunity.yaml | 6 +- controllers/mongodb_tls.go | 62 +++++++--- controllers/mongodb_tls_test.go | 109 ++++++++++++++---- docs/RELEASE_NOTES.md | 3 +- .../replica_set_tls/replica_set_tls_test.go | 2 +- .../replica_set_pem_file_test.go | 62 ++++++++++ .../replica_set_tls_rotate_test.go | 2 +- .../replica_set_tls_upgrade_test.go | 2 +- test/e2e/setup/setup.go | 42 ++++--- testdata/tls/server.pem | 53 +++++++++ 14 files changed, 301 insertions(+), 56 deletions(-) create mode 100644 test/e2e/replica_set_tls_pem_file/replica_set_pem_file_test.go create mode 100644 testdata/tls/server.pem diff --git a/.action_templates/jobs/tests.yaml b/.action_templates/jobs/tests.yaml index 9c893c38c..037749fab 100644 --- a/.action_templates/jobs/tests.yaml +++ b/.action_templates/jobs/tests.yaml @@ -25,6 +25,8 @@ tests: distro: ubuntu - test-name: replica_set_tls distro: ubuntu + - test-name: replica_set_tls_pem_file + distro: ubuntu - test-name: replica_set_tls_upgrade distro: ubuntu - test-name: statefulset_arbitrary_config @@ -65,6 +67,8 @@ tests: distro: ubi - test-name: replica_set_tls_upgrade distro: ubi + - test-name: replica_set_tls_pem_file + distro: ubi - test-name: statefulset_arbitrary_config distro: ubi - test-name: statefulset_arbitrary_config_update diff --git a/.github/workflows/e2e-fork.yml b/.github/workflows/e2e-fork.yml index d6bb954bc..0e2f6b0b9 100644 --- a/.github/workflows/e2e-fork.yml +++ b/.github/workflows/e2e-fork.yml @@ -107,6 +107,8 @@ jobs: distro: ubuntu - test-name: replica_set_tls distro: ubuntu + - test-name: replica_set_tls_pem_file + distro: ubuntu - test-name: replica_set_tls_upgrade distro: ubuntu - test-name: statefulset_arbitrary_config @@ -147,6 +149,8 @@ jobs: distro: ubi - test-name: replica_set_tls_upgrade distro: ubi + - test-name: replica_set_tls_pem_file + distro: ubi - test-name: statefulset_arbitrary_config distro: ubi - test-name: statefulset_arbitrary_config_update diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 4d695e8fc..d00ca1800 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -112,6 +112,8 @@ jobs: distro: ubuntu - test-name: replica_set_tls distro: ubuntu + - test-name: replica_set_tls_pem_file + distro: ubuntu - test-name: replica_set_tls_upgrade distro: ubuntu - test-name: statefulset_arbitrary_config @@ -152,6 +154,8 @@ jobs: distro: ubi - test-name: replica_set_tls_upgrade distro: ubi + - test-name: replica_set_tls_pem_file + distro: ubi - test-name: statefulset_arbitrary_config distro: ubi - test-name: statefulset_arbitrary_config_update diff --git a/api/v1/mongodbcommunity_types.go b/api/v1/mongodbcommunity_types.go index cfec1a43c..33d3f8005 100644 --- a/api/v1/mongodbcommunity_types.go +++ b/api/v1/mongodbcommunity_types.go @@ -321,6 +321,8 @@ type TLS struct { // CertificateKeySecret is a reference to a Secret containing a private key and certificate to use for TLS. // The key and cert are expected to be PEM encoded and available at "tls.key" and "tls.crt". // This is the same format used for the standard "kubernetes.io/tls" Secret type, but no specific type is required. + // Alternatively, an entry tls.pem, containing the concatenation of cert and key, can be provided. + // If all of tls.pem, tls.crt and tls.key are present, the tls.pem one needs to be equal to the concatenation of tls.crt and tls.key // +optional CertificateKeySecret LocalObjectReference `json:"certificateKeySecretRef"` diff --git a/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml b/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml index 7db8ac068..1bec50498 100644 --- a/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml +++ b/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml @@ -207,7 +207,11 @@ spec: The key and cert are expected to be PEM encoded and available at "tls.key" and "tls.crt". This is the same format used for the standard "kubernetes.io/tls" Secret type, but no - specific type is required. + specific type is required. Alternatively, an entry tls.pem, + containing the concatenation of cert and key, can be provided. + If all of tls.pem, tls.crt and tls.key are present, the + tls.pem one needs to be equal to the concatenation of tls.crt + and tls.key properties: name: type: string diff --git a/controllers/mongodb_tls.go b/controllers/mongodb_tls.go index eda541f46..7ca9d6b8e 100644 --- a/controllers/mongodb_tls.go +++ b/controllers/mongodb_tls.go @@ -5,8 +5,6 @@ import ( "fmt" "strings" - "github.com/pkg/errors" - "github.com/mongodb/mongodb-kubernetes-operator/controllers/construct" "github.com/mongodb/mongodb-kubernetes-operator/pkg/automationconfig" @@ -28,6 +26,7 @@ const ( tlsOperatorSecretMountPath = "/var/lib/tls/server/" //nolint tlsSecretCertName = "tls.crt" //nolint tlsSecretKeyName = "tls.key" + tlsSecretPemName = "tls.pem" ) // validateTLSConfig will check that the configured ConfigMap and Secret exist and that they have the correct fields. @@ -56,7 +55,7 @@ func (r *ReplicaSetReconciler) validateTLSConfig(mdb mdbv1.MongoDBCommunity) (bo } // Ensure Secret exists - secretData, err := secret.ReadStringData(r.client, mdb.TLSSecretNamespacedName()) + _, err = secret.ReadStringData(r.client, mdb.TLSSecretNamespacedName()) if err != nil { if apiErrors.IsNotFound(err) { r.log.Warnf(`Secret "%s" not found`, mdb.TLSSecretNamespacedName()) @@ -66,13 +65,11 @@ func (r *ReplicaSetReconciler) validateTLSConfig(mdb mdbv1.MongoDBCommunity) (bo return false, err } - // Ensure Secret has "tls.crt" and "tls.key" fields - if key, ok := secretData[tlsSecretKeyName]; !ok || key == "" { - r.log.Warnf(`Secret "%s" should have a key in field "%s"`, mdb.TLSSecretNamespacedName(), tlsSecretKeyName) - return false, nil - } - if cert, ok := secretData[tlsSecretCertName]; !ok || cert == "" { - r.log.Warnf(`Secret "%s" should have a certificate in field "%s"`, mdb.TLSSecretNamespacedName(), tlsSecretKeyName) + // validate whether the secret contains "tls.crt" and "tls.key", or it contains "tls.pem" + // if it contains all three, then the pem entry should be equal to the concatenation of crt and key + _, err = getPemOrConcatenatedCrtAndKey(r.client, mdb) + if err != nil { + r.log.Warnf(err.Error()) return false, nil } @@ -90,7 +87,7 @@ func getTLSConfigModification(getUpdateCreator secret.GetUpdateCreator, mdb mdbv return automationconfig.NOOP(), nil } - certKey, err := getCertAndKey(getUpdateCreator, mdb) + certKey, err := getPemOrConcatenatedCrtAndKey(getUpdateCreator, mdb) if err != nil { return automationconfig.NOOP(), err } @@ -99,18 +96,27 @@ func getTLSConfigModification(getUpdateCreator secret.GetUpdateCreator, mdb mdbv } // getCertAndKey will fetch the certificate and key from the user-provided Secret. -func getCertAndKey(getter secret.Getter, mdb mdbv1.MongoDBCommunity) (string, error) { +func getCertAndKey(getter secret.Getter, mdb mdbv1.MongoDBCommunity) string { cert, err := secret.ReadKey(getter, tlsSecretCertName, mdb.TLSSecretNamespacedName()) if err != nil { - return "", err + return "" } key, err := secret.ReadKey(getter, tlsSecretKeyName, mdb.TLSSecretNamespacedName()) if err != nil { - return "", err + return "" } - return combineCertificateAndKey(cert, key), nil + return combineCertificateAndKey(cert, key) +} + +// getPem will fetch the pem from the user-provided secret +func getPem(getter secret.Getter, mdb mdbv1.MongoDBCommunity) string { + pem, err := secret.ReadKey(getter, tlsSecretPemName, mdb.TLSSecretNamespacedName()) + if err != nil { + return "" + } + return pem } func combineCertificateAndKey(cert, key string) string { @@ -119,12 +125,34 @@ func combineCertificateAndKey(cert, key string) string { return fmt.Sprintf("%s\n%s", trimmedCert, trimmedKey) } +// getPemOrConcatenatedCrtAndKey will get the final PEM to write to the secret. +// This is either the tls.pem entry in the given secret, or the concatenation +// of tls.crt and tls.key +// It performs a basic validation on the entries. +func getPemOrConcatenatedCrtAndKey(getter secret.Getter, mdb mdbv1.MongoDBCommunity) (string, error) { + certKey := getCertAndKey(getter, mdb) + pem := getPem(getter, mdb) + if certKey == "" && pem == "" { + return "", fmt.Errorf(`Neither "%s" nor the pair "%s"/"%s" were present in the TLS secret`, tlsSecretPemName, tlsSecretCertName, tlsSecretKeyName) + } + if certKey == "" { + return pem, nil + } + if pem == "" { + return certKey, nil + } + if certKey != pem { + return "", fmt.Errorf(`If all of "%s", "%s" and "%s" are present in the secret, the entry for "%s" must be equal to the concatenation of "%s" with "%s"`, tlsSecretCertName, tlsSecretKeyName, tlsSecretPemName, tlsSecretPemName, tlsSecretCertName, tlsSecretKeyName) + } + return certKey, nil +} + // ensureTLSSecret will create or update the operator-managed Secret containing // the concatenated certificate and key from the user-provided Secret. func ensureTLSSecret(getUpdateCreator secret.GetUpdateCreator, mdb mdbv1.MongoDBCommunity) error { - certKey, err := getCertAndKey(getUpdateCreator, mdb) + certKey, err := getPemOrConcatenatedCrtAndKey(getUpdateCreator, mdb) if err != nil { - return errors.Errorf("could not get cert and key: %s", err) + return err } // Calculate file name from certificate and key fileName := tlsOperatorSecretFileName(certKey) diff --git a/controllers/mongodb_tls_test.go b/controllers/mongodb_tls_test.go index 650fd264f..983d343c4 100644 --- a/controllers/mongodb_tls_test.go +++ b/controllers/mongodb_tls_test.go @@ -23,7 +23,10 @@ func TestStatefulSet_IsCorrectlyConfiguredWithTLS(t *testing.T) { mdb := newTestReplicaSetWithTLS() mgr := client.NewManager(&mdb) - err := createTLSSecretAndConfigMap(mgr.GetClient(), mdb) + client := mdbClient.NewClient(mgr.GetClient()) + err := createTLSSecret(client, mdb, "CERT", "KEY", "") + assert.NoError(t, err) + err = createTLSConfigMap(client, mdb) assert.NoError(t, err) r := NewReconciler(mgr) @@ -82,7 +85,9 @@ func TestStatefulSet_IsCorrectlyConfiguredWithTLS(t *testing.T) { func TestAutomationConfig_IsCorrectlyConfiguredWithTLS(t *testing.T) { createAC := func(mdb mdbv1.MongoDBCommunity) automationconfig.AutomationConfig { client := mdbClient.NewClient(client.NewManager(&mdb).GetClient()) - err := createTLSSecretAndConfigMap(client, mdb) + err := createTLSSecret(client, mdb, "CERT", "KEY", "") + assert.NoError(t, err) + err = createTLSConfigMap(client, mdb) assert.NoError(t, err) tlsModification, err := getTLSConfigModification(client, mdb) @@ -151,7 +156,9 @@ func TestTLSOperatorSecret(t *testing.T) { t.Run("Secret is created if it doesn't exist", func(t *testing.T) { mdb := newTestReplicaSetWithTLS() c := mdbClient.NewClient(client.NewManager(&mdb).GetClient()) - err := createTLSSecretAndConfigMap(c, mdb) + err := createTLSSecret(c, mdb, "CERT", "KEY", "") + assert.NoError(t, err) + err = createTLSConfigMap(c, mdb) assert.NoError(t, err) r := NewReconciler(client.NewManagerWithClient(c)) @@ -159,7 +166,7 @@ func TestTLSOperatorSecret(t *testing.T) { err = r.ensureTLSResources(mdb) assert.NoError(t, err) - // Operator-managed secret should have been created and contain the + // Operator-managed secret should have been created and contains the // concatenated certificate and key. expectedCertificateKey := "CERT\nKEY" certificateKey, err := secret.ReadKey(c, tlsOperatorSecretFileName(expectedCertificateKey), mdb.TLSOperatorSecretNamespacedName()) @@ -170,7 +177,9 @@ func TestTLSOperatorSecret(t *testing.T) { t.Run("Secret is updated if it already exists", func(t *testing.T) { mdb := newTestReplicaSetWithTLS() k8sclient := mdbClient.NewClient(client.NewManager(&mdb).GetClient()) - err := createTLSSecretAndConfigMap(k8sclient, mdb) + err := createTLSSecret(k8sclient, mdb, "CERT", "KEY", "") + assert.NoError(t, err) + err = createTLSConfigMap(k8sclient, mdb) assert.NoError(t, err) // Create operator-managed secret @@ -215,29 +224,89 @@ func TestCombineCertificateAndKey(t *testing.T) { } } -func createTLSSecretAndConfigMap(c k8sClient.Client, mdb mdbv1.MongoDBCommunity) error { - s := secret.Builder(). - SetName(mdb.Spec.Security.TLS.CertificateKeySecret.Name). - SetNamespace(mdb.Namespace). - SetField("tls.crt", "CERT"). - SetField("tls.key", "KEY"). - Build() +func TestPemSupport(t *testing.T) { + t.Run("Success if only pem is provided", func(t *testing.T) { + mdb := newTestReplicaSetWithTLS() + c := mdbClient.NewClient(client.NewManager(&mdb).GetClient()) + err := createTLSSecret(c, mdb, "", "", "CERT\nKEY") + assert.NoError(t, err) + err = createTLSConfigMap(c, mdb) + assert.NoError(t, err) - err := c.Create(context.TODO(), &s) - if err != nil { - return err - } + r := NewReconciler(client.NewManagerWithClient(c)) + + err = r.ensureTLSResources(mdb) + assert.NoError(t, err) + + // Operator-managed secret should have been created and contains the + // concatenated certificate and key. + expectedCertificateKey := "CERT\nKEY" + certificateKey, err := secret.ReadKey(c, tlsOperatorSecretFileName(expectedCertificateKey), mdb.TLSOperatorSecretNamespacedName()) + assert.NoError(t, err) + assert.Equal(t, expectedCertificateKey, certificateKey) + }) + t.Run("Success if pem is equal to cert+key", func(t *testing.T) { + mdb := newTestReplicaSetWithTLS() + c := mdbClient.NewClient(client.NewManager(&mdb).GetClient()) + err := createTLSSecret(c, mdb, "CERT", "KEY", "CERT\nKEY") + assert.NoError(t, err) + err = createTLSConfigMap(c, mdb) + assert.NoError(t, err) + + r := NewReconciler(client.NewManagerWithClient(c)) + + err = r.ensureTLSResources(mdb) + assert.NoError(t, err) + + // Operator-managed secret should have been created and contains the + // concatenated certificate and key. + expectedCertificateKey := "CERT\nKEY" + certificateKey, err := secret.ReadKey(c, tlsOperatorSecretFileName(expectedCertificateKey), mdb.TLSOperatorSecretNamespacedName()) + assert.NoError(t, err) + assert.Equal(t, expectedCertificateKey, certificateKey) + }) + t.Run("Failure if pem is different from cert+key", func(t *testing.T) { + mdb := newTestReplicaSetWithTLS() + c := mdbClient.NewClient(client.NewManager(&mdb).GetClient()) + err := createTLSSecret(c, mdb, "CERT1", "KEY1", "CERT\nKEY") + assert.NoError(t, err) + err = createTLSConfigMap(c, mdb) + assert.NoError(t, err) + + r := NewReconciler(client.NewManagerWithClient(c)) + + err = r.ensureTLSResources(mdb) + assert.Error(t, err) + assert.Contains(t, err.Error(), `If all of "tls.crt", "tls.key" and "tls.pem" are present in the secret, the entry for "tls.pem" must be equal to the concatenation of "tls.crt" with "tls.key"`) + + }) +} +func createTLSConfigMap(c k8sClient.Client, mdb mdbv1.MongoDBCommunity) error { configMap := configmap.Builder(). SetName(mdb.Spec.Security.TLS.CaConfigMap.Name). SetNamespace(mdb.Namespace). SetField("ca.crt", "CERT"). Build() - err = c.Create(context.TODO(), &configMap) - if err != nil { - return err + return c.Create(context.TODO(), &configMap) +} + +func createTLSSecret(c k8sClient.Client, mdb mdbv1.MongoDBCommunity, crt string, key string, pem string) error { + sBuilder := secret.Builder(). + SetName(mdb.Spec.Security.TLS.CertificateKeySecret.Name). + SetNamespace(mdb.Namespace) + + if crt != "" { + sBuilder.SetField(tlsSecretCertName, crt) + } + if key != "" { + sBuilder.SetField(tlsSecretKeyName, key) + } + if pem != "" { + sBuilder.SetField(tlsSecretPemName, pem) } - return nil + s := sBuilder.Build() + return c.Create(context.TODO(), &s) } diff --git a/docs/RELEASE_NOTES.md b/docs/RELEASE_NOTES.md index 649393502..e2b5667f6 100644 --- a/docs/RELEASE_NOTES.md +++ b/docs/RELEASE_NOTES.md @@ -4,7 +4,8 @@ - Changes - MongoDB database of the statefulSet is managed using distinct Role, ServiceAccount and RoleBinding. - + - TLS Secret can also contain a single "tls.pem" entry, containing the concatenation of the certificate and key + - If a TLS secret contains all of "tls.key", "tls.crt" and "tls.pem" entries, the operator will raise an error if the "tls.pem" one is not equal to the concatenation of "tls.crt" with "tls.key" ## Updated Image Tags - mongodb-kubernetes-operator:0.7.1 diff --git a/test/e2e/replica_set_tls/replica_set_tls_test.go b/test/e2e/replica_set_tls/replica_set_tls_test.go index 67241f779..ba35611ee 100644 --- a/test/e2e/replica_set_tls/replica_set_tls_test.go +++ b/test/e2e/replica_set_tls/replica_set_tls_test.go @@ -33,7 +33,7 @@ func TestReplicaSetTLS(t *testing.T) { t.Fatal(err) } - if err := setup.CreateTLSResources(mdb.Namespace, ctx); err != nil { + if err := setup.CreateTLSResources(mdb.Namespace, ctx, setup.CertKeyPair); err != nil { t.Fatalf("Failed to set up TLS resources: %s", err) } diff --git a/test/e2e/replica_set_tls_pem_file/replica_set_pem_file_test.go b/test/e2e/replica_set_tls_pem_file/replica_set_pem_file_test.go new file mode 100644 index 000000000..6b7d1121c --- /dev/null +++ b/test/e2e/replica_set_tls_pem_file/replica_set_pem_file_test.go @@ -0,0 +1,62 @@ +package replica_set_tls + +import ( + "fmt" + "os" + "testing" + + . "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/util/mongotester" + + e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" + "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/mongodbtests" + setup "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" +) + +func TestMain(m *testing.M) { + code, err := e2eutil.RunTest(m) + if err != nil { + fmt.Println(err) + } + os.Exit(code) +} + +func TestReplicaSetTLS(t *testing.T) { + ctx := setup.Setup(t) + defer ctx.Teardown() + + mdb, user := e2eutil.NewTestMongoDB(ctx, "mdb-tls", "") + scramUser := mdb.GetScramUsers()[0] + mdb.Spec.Security.TLS = e2eutil.NewTestTLSConfig(false) + + _, err := setup.GeneratePasswordForUser(ctx, user, "") + if err != nil { + t.Fatal(err) + } + + if err := setup.CreateTLSResources(mdb.Namespace, ctx, setup.Pem); err != nil { + t.Fatalf("Failed to set up TLS resources: %s", err) + } + + tester, err := FromResource(t, mdb) + if err != nil { + t.Fatal(err) + } + + t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) + t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb)) + mongodbtests.SkipTestIfLocal(t, "Ensure MongoDB TLS Configuration", func(t *testing.T) { + t.Run("Has TLS Mode", tester.HasTlsMode("requireSSL", 60, WithTls())) + t.Run("Basic Connectivity Succeeds", tester.ConnectivitySucceeds(WithTls())) + t.Run("SRV Connectivity Succeeds", tester.ConnectivitySucceeds(WithURI(mdb.MongoSRVURI()), WithTls())) + t.Run("Basic Connectivity With Generated Connection String Secret Succeeds", + tester.ConnectivitySucceeds(WithURI(mongodbtests.GetConnectionStringForUser(mdb, scramUser)), WithTls())) + t.Run("SRV Connectivity With Generated Connection String Secret Succeeds", + tester.ConnectivitySucceeds(WithURI(mongodbtests.GetSrvConnectionStringForUser(mdb, scramUser)), WithTls())) + t.Run("Connectivity Fails", tester.ConnectivityFails(WithoutTls())) + t.Run("Ensure authentication is configured", tester.EnsureAuthenticationIsConfigured(3, WithTls())) + }) + t.Run("TLS is disabled", mongodbtests.DisableTLS(&mdb)) + t.Run("MongoDB Reaches Failed Phase", mongodbtests.MongoDBReachesFailedPhase(&mdb)) + t.Run("TLS is enabled", mongodbtests.EnableTLS(&mdb)) + t.Run("MongoDB Reaches Running Phase", mongodbtests.MongoDBReachesRunningPhase(&mdb)) +} diff --git a/test/e2e/replica_set_tls_rotate/replica_set_tls_rotate_test.go b/test/e2e/replica_set_tls_rotate/replica_set_tls_rotate_test.go index 07f496cc8..d083023d9 100644 --- a/test/e2e/replica_set_tls_rotate/replica_set_tls_rotate_test.go +++ b/test/e2e/replica_set_tls_rotate/replica_set_tls_rotate_test.go @@ -34,7 +34,7 @@ func TestReplicaSetTLSRotate(t *testing.T) { t.Fatal(err) } - if err := setup.CreateTLSResources(mdb.Namespace, ctx); err != nil { + if err := setup.CreateTLSResources(mdb.Namespace, ctx, setup.CertKeyPair); err != nil { t.Fatalf("Failed to set up TLS resources: %s", err) } tester, err := FromResource(t, mdb) diff --git a/test/e2e/replica_set_tls_upgrade/replica_set_tls_upgrade_test.go b/test/e2e/replica_set_tls_upgrade/replica_set_tls_upgrade_test.go index 8ed5ba7ee..88849b896 100644 --- a/test/e2e/replica_set_tls_upgrade/replica_set_tls_upgrade_test.go +++ b/test/e2e/replica_set_tls_upgrade/replica_set_tls_upgrade_test.go @@ -33,7 +33,7 @@ func TestReplicaSetTLSUpgrade(t *testing.T) { t.Fatal(err) } - if err := setup.CreateTLSResources(mdb.Namespace, ctx); err != nil { + if err := setup.CreateTLSResources(mdb.Namespace, ctx, setup.CertKeyPair); err != nil { t.Fatalf("Failed to set up TLS resources: %s", err) } diff --git a/test/e2e/setup/setup.go b/test/e2e/setup/setup.go index 1eaace825..4f991ab43 100644 --- a/test/e2e/setup/setup.go +++ b/test/e2e/setup/setup.go @@ -34,10 +34,15 @@ import ( mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" ) +type tlsSecretType string + const ( performCleanupEnv = "PERFORM_CLEANUP" deployDirEnv = "DEPLOY_DIR" roleDirEnv = "ROLE_DIR" + + CertKeyPair tlsSecretType = "CERTKEYPAIR" + Pem tlsSecretType = "PEM" ) func Setup(t *testing.T) *e2eutil.Context { @@ -56,7 +61,7 @@ func Setup(t *testing.T) *e2eutil.Context { // CreateTLSResources will setup the CA ConfigMap and cert-key Secret necessary for TLS // The certificates and keys are stored in testdata/tls -func CreateTLSResources(namespace string, ctx *e2eutil.Context) error { //nolint +func CreateTLSResources(namespace string, ctx *e2eutil.Context, secretType tlsSecretType) error { tlsConfig := e2eutil.NewTestTLSConfig(false) // Create CA ConfigMap @@ -77,22 +82,31 @@ func CreateTLSResources(namespace string, ctx *e2eutil.Context) error { //nolint return err } - // Create server key and certificate secret - cert, err := ioutil.ReadFile(path.Join(testDataDir, "server.crt")) - if err != nil { - return err + certKeySecretBuilder := secret.Builder(). + SetName(tlsConfig.CertificateKeySecret.Name). + SetNamespace(namespace) + + if secretType == CertKeyPair { + // Create server key and certificate secret + cert, err := ioutil.ReadFile(path.Join(testDataDir, "server.crt")) + if err != nil { + return err + } + key, err := ioutil.ReadFile(path.Join(testDataDir, "server.key")) + if err != nil { + return err + } + certKeySecretBuilder.SetField("tls.crt", string(cert)).SetField("tls.key", string(key)) } - key, err := ioutil.ReadFile(path.Join(testDataDir, "server.key")) - if err != nil { - return err + if secretType == Pem { + pem, err := ioutil.ReadFile(path.Join(testDataDir, "server.pem")) + if err != nil { + return err + } + certKeySecretBuilder.SetField("tls.pem", string(pem)) } - certKeySecret := secret.Builder(). - SetName(tlsConfig.CertificateKeySecret.Name). - SetNamespace(namespace). - SetField("tls.crt", string(cert)). - SetField("tls.key", string(key)). - Build() + certKeySecret := certKeySecretBuilder.Build() return e2eutil.TestClient.Create(context.TODO(), &certKeySecret, &e2eutil.CleanupOptions{TestContext: ctx}) } diff --git a/testdata/tls/server.pem b/testdata/tls/server.pem new file mode 100644 index 000000000..fa38bc24c --- /dev/null +++ b/testdata/tls/server.pem @@ -0,0 +1,53 @@ +-----BEGIN CERTIFICATE----- +MIIEWjCCAkKgAwIBAgIUKXYivNfzneHnf77o/hmGJPPZmiMwDQYJKoZIhvcNAQEL +BQAwJDEQMA4GA1UECgwHTW9uZ29EQjEQMA4GA1UEAwwHUm9vdCBDQTAeFw0yMTA0 +MjgxMjI0MjRaFw0zNTAxMDUxMjI0MjRaMDkxCzAJBgNVBAYTAlVTMQswCQYDVQQI +DAJOWTELMAkGA1UEBwwCTlkxEDAOBgNVBAoMB01vbmdvREIwggEiMA0GCSqGSIb3 +DQEBAQUAA4IBDwAwggEKAoIBAQDMgVLGC5blAlvcmbggfnmFZ0wHAstOxbjOPija +53TzvKi9L2Smrwf5/RtRQSZ6cgNfTLzDbz+jKHn5v0jWqSW5TzWSL1VcDiYSoito ++RwJcRmrLBuceqP8anUjCgqmDH5xFL2w+QNh9knGdOvbUkGr+gaUxeQxNclup8jV +v9qyRva2an8MB7VbSG8ZVDVkcBkH2xlO+S1ITl/SPBXKsDbOB/hWEAqkOoEom5lQ +6a4IjUYU8HUhebzERH71Jhgc53hcs1RourMLQmAZQoqy7E8On6B/jZxMqq4HsqiQ +PYV/FLPlT12hgKMBZ6POwIFxEueTGVuGzHU37aoxPwNT7cp/AgMBAAGjbzBtMB8G +A1UdIwQYMBaAFNKgRS9B32CXr3UdG9LwGmMUi6WbMAkGA1UdEwQCMAAwCwYDVR0P +BAQDAgTwMDIGA1UdEQQrMCmCJyoubWRiLXRscy1zdmMuZGVmYXVsdC5zdmMuY2x1 +c3Rlci5sb2NhbDANBgkqhkiG9w0BAQsFAAOCAgEAeeVcqSuI3UjmThAufNN5I+Z5 +jIUyU/kTcOHUr5hDA83+W8IuEHo/g+ZsvtCVqTqiXNd5Ehn5rdO+YB8fqXC2jgUr +VLbel87qdxqTwdZ6pO3X0StO1AuSN/ZydnfZqRyI7fJn28A0fzTHP5AZdOAYtGBR +nld9omH85p2EsZkhtdsZpRPr11mQoFnJ9lGcz2z/6GRbrlEYrM9nU4Ij8cBAlhrM +hkqNpQT56XM1QxJ6MdEwYQv4Fbkr5Aa75NGyb0m6uQNYDPyXgvvkSZ+lZTXBhVl1 +5GouRqRMe+hlGPYL4VKy23PAwag7dNlQ1GQLur+pWkXfHLdKIaoPLDwFx4m8PWGr +rErXOXKKYZIw+xsQYKOXNePMM3/bRlEZTt52wrBEDB3LNhNkuKB8J1+/dfE557l1 +5/Gyt+MuRAq/gi+ffR7KxuzYDipGSUmmWzFF/5LyOCAS9lKi8xyKzsYpdDDkcx8k +aC86zOjYseMKytk2hgOmNPjva9iG4mlQQ/S7FgOn01jJadpu9X0zVgmq7uKIemUM +6ts8qEK3zIGir10FfT0zxwaXQOMLMHrLvELGJEhHJPTQDMjPopKVEXtk9Upeveas +PK3QLsn3xE2XytH2HJnAHL3GR1nLT3HgdyrOlJlV37ZPXr3di7nfQQM4UJoghyWH +JZ6umbgvvVMWeLFX/IE= +-----END CERTIFICATE----- +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEAzIFSxguW5QJb3Jm4IH55hWdMBwLLTsW4zj4o2ud087yovS9k +pq8H+f0bUUEmenIDX0y8w28/oyh5+b9I1qkluU81ki9VXA4mEqIraPkcCXEZqywb +nHqj/Gp1IwoKpgx+cRS9sPkDYfZJxnTr21JBq/oGlMXkMTXJbqfI1b/askb2tmp/ +DAe1W0hvGVQ1ZHAZB9sZTvktSE5f0jwVyrA2zgf4VhAKpDqBKJuZUOmuCI1GFPB1 +IXm8xER+9SYYHOd4XLNUaLqzC0JgGUKKsuxPDp+gf42cTKquB7KokD2FfxSz5U9d +oYCjAWejzsCBcRLnkxlbhsx1N+2qMT8DU+3KfwIDAQABAoIBAQC7HjVbim0l25Or +9Gb6LF8KhiqVW6Qkzls7Mrr1GMT045FNkRi6PvrAbSvanA8WCE43m6I3/AmxQy7g +Knr+FsSymtw8htzGnxeNAx9PLGfP59GBwpj9A2YaZloJln2J03K6Cy1JyX6j2tNE +J+VKxyfZsKrm427Y7AsEGbd0hNgZN5s9l70q0FSCkFcb37b497k0gYcE+63wEaq3 +FHGoYvbjUVKqp1YpVQyALlHk2toDMOOVBt4MQzP6RsVQJ3LY7K0ZYlNu83EWutsJ +oIMjDwMoCpDtFqrUDzCgbYoDPAaREOBFJZcUrqQ3oTMCo8qEZgiinOVQks6vqnpd +vke/qfsRAoGBAPcm/4AkVeRCmEmR16U9K8pk2KyOJxbXvSvLPO55ytAHSeHEQYaE +FevTOYj+Whd5B/OWOcGXrvby0OpzfEizpE/cLyCGPQqONh5RyJUeG9mzmSCGfJKw +dru99Sg2njU+ZYmHtf6FtY6RGZ4OrwiifVzk/slGE9r0LJt0uVJ/Db83AoGBANPT +fWAetG/JJVG8RoQnddHzZhpmJAnqQt6QbKiYZ/WsH5mchsuJg2oybY9uf9TL1OMy +yxhCie1vFBBRD1s6j06btqF38i9D2H6R55i2PtP5AKFD6S9wucpFRiR0A5r5r69V +KwnYA1fu0uA6tYw457f0vS8NfIaiEDmERfiy4qL5AoGBANYsXUzWL/hWHVHjqFPw +5nnFWl5t8UHCQpQo0ux1bmNHbabPQ1kmLTjnGfy1La0ZnOJhVDuHDn/Be3kwCouV +4NWzoMM2kL8M7ajohkFyjf/hutiMsncLpFidDE2ExySspaDAkd22UNbytphZcSSy +aqCNcJ1KtPoQjndIdzAeGfORAoGABbbm4vjxFTLv9syFens2CnvufTfUMRBIzYhH +5iR2aYJDN/mpCUSkbvD9U6k/eZYmIBr2r6jb37PnbqlBKMzjoNNCkgiSWAQUixWU +keIYv88v3Snf2I/J81L7GXCnyD6EJs69Yn6ZWH3w4muzCh1e4u+PSv2qJleo6GRR +Hux0gMECgYBN0BaeyUPqRLgq3JsrPTK0VQ+8J5+3la7wZWU5vCr2cvLftzb7Devj +m5K901mFCPdtO5LJ8OdeOi1PHnG/+WCfuwDitN8OufPJ+tdSteG+F9XIu5sTMGLB +QJeIyHolsPZhW4OA3C7p6uZHAeDIqIpkv8j7974cLBWhGlQJx9403A== +-----END RSA PRIVATE KEY----- From 254e7eabc637b9b21634473d9fb5fc868e1ff9c4 Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Wed, 21 Jul 2021 11:43:56 +0100 Subject: [PATCH 367/790] Add test labels to test resources and cleanup before running a new test (#647) --- Makefile | 9 ++++++--- pkg/kube/configmap/configmap_builder.go | 12 ++++++++++++ scripts/dev/e2e.py | 1 + test/e2e/e2eutil.go | 12 +++++++++++- test/e2e/setup/setup.go | 4 ++++ 5 files changed, 34 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 205c8461c..a555f370d 100644 --- a/Makefile +++ b/Makefile @@ -67,7 +67,7 @@ vet: # Run e2e tests locally using go build while also setting up a proxy in the shell to allow # the test to run as if it were inside the cluster. This enables mongodb connectivity while running locally. -e2e-telepresence: install +e2e-telepresence: cleanup-e2e install telepresence connect; \ telepresence status; \ eval $$(scripts/dev/get_e2e_env_vars.py $(cleanup)); \ @@ -75,12 +75,12 @@ e2e-telepresence: install telepresence quit # Run e2e test by deploying test image in kubernetes. -e2e-k8s: install e2e-image +e2e-k8s: cleanup-e2e install e2e-image python scripts/dev/e2e.py --perform-cleanup --test $(test) # Run e2e test locally. # e.g. make e2e test=replica_set cleanup=true -e2e: install +e2e: cleanup-e2e install eval $$(scripts/dev/get_e2e_env_vars.py $(cleanup)); \ go test -v -short -timeout=30m -failfast ./test/e2e/$(test) @@ -88,6 +88,9 @@ e2e: install e2e-gh: scripts/dev/run_e2e_gh.sh $(test) +cleanup-e2e: + kubectl delete mdbc,all -l e2e-test=true + # Generate code generate: controller-gen $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." diff --git a/pkg/kube/configmap/configmap_builder.go b/pkg/kube/configmap/configmap_builder.go index 5341158bd..f0b175f4f 100644 --- a/pkg/kube/configmap/configmap_builder.go +++ b/pkg/kube/configmap/configmap_builder.go @@ -10,6 +10,7 @@ type builder struct { name string namespace string ownerReferences []metav1.OwnerReference + labels map[string]string } func (b *builder) SetName(name string) *builder { @@ -32,12 +33,22 @@ func (b *builder) SetOwnerReferences(ownerReferences []metav1.OwnerReference) *b return b } +func (b *builder) SetLabels(labels map[string]string) *builder { + newLabels := make(map[string]string) + for k, v := range labels { + newLabels[k] = v + } + b.labels = newLabels + return b +} + func (b builder) Build() corev1.ConfigMap { return corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: b.name, Namespace: b.namespace, OwnerReferences: b.ownerReferences, + Labels: b.labels, }, Data: b.data, } @@ -47,5 +58,6 @@ func Builder() *builder { return &builder{ data: map[string]string{}, ownerReferences: []metav1.OwnerReference{}, + labels: map[string]string{}, } } diff --git a/scripts/dev/e2e.py b/scripts/dev/e2e.py index 23093e72c..7b5a82462 100644 --- a/scripts/dev/e2e.py +++ b/scripts/dev/e2e.py @@ -81,6 +81,7 @@ def create_test_pod(args: argparse.Namespace, dev_config: DevConfig) -> None: "metadata": { "name": TEST_POD_NAME, "namespace": dev_config.namespace, + "labels": {"e2e-test": "true"}, }, "spec": { "restartPolicy": "Never", diff --git a/test/e2e/e2eutil.go b/test/e2e/e2eutil.go index dcac0399b..ca4921341 100644 --- a/test/e2e/e2eutil.go +++ b/test/e2e/e2eutil.go @@ -21,6 +21,13 @@ import ( const testDataDirEnv = "TEST_DATA_DIR" +// TestLabels should be applied to all resources created by tests. +func TestLabels() map[string]string { + return map[string]string{ + "e2e-test": "true", + } +} + func TestDataDir() string { return envvar.GetEnvOrDefault(testDataDirEnv, "/workspace/testdata") } @@ -51,6 +58,7 @@ func NewTestMongoDB(ctx *Context, name string, namespace string) (mdbv1.MongoDBC ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: mongodbNamespace, + Labels: TestLabels(), }, Spec: mdbv1.MongoDBCommunitySpec{ Members: 3, @@ -154,6 +162,7 @@ func NewTestTLSConfig(optional bool) mdbv1.TLS { func ensureObject(ctx *Context, obj k8sClient.Object) error { key := k8sClient.ObjectKeyFromObject(obj) + obj.SetLabels(TestLabels()) err := TestClient.Get(context.TODO(), key, obj) if err != nil { @@ -178,7 +187,8 @@ func ensureObject(ctx *Context, obj k8sClient.Object) error { func EnsureNamespace(ctx *Context, namespace string) error { return ensureObject(ctx, &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ - Name: namespace, + Name: namespace, + Labels: TestLabels(), }, }) } diff --git a/test/e2e/setup/setup.go b/test/e2e/setup/setup.go index 4f991ab43..5f44cd325 100644 --- a/test/e2e/setup/setup.go +++ b/test/e2e/setup/setup.go @@ -75,6 +75,7 @@ func CreateTLSResources(namespace string, ctx *e2eutil.Context, secretType tlsSe SetName(tlsConfig.CaConfigMap.Name). SetNamespace(namespace). SetField("ca.crt", string(ca)). + SetLabels(e2eutil.TestLabels()). Build() err = e2eutil.TestClient.Create(context.TODO(), &caConfigMap, &e2eutil.CleanupOptions{TestContext: ctx}) @@ -84,6 +85,7 @@ func CreateTLSResources(namespace string, ctx *e2eutil.Context, secretType tlsSe certKeySecretBuilder := secret.Builder(). SetName(tlsConfig.CertificateKeySecret.Name). + SetLabels(e2eutil.TestLabels()). SetNamespace(namespace) if secretType == CertKeyPair { @@ -132,6 +134,7 @@ func GeneratePasswordForUser(ctx *e2eutil.Context, mdbu mdbv1.MongoDBUser, names SetName(mdbu.PasswordSecretRef.Name). SetNamespace(nsp). SetField(passwordKey, password). + SetLabels(e2eutil.TestLabels()). Build() return password, e2eutil.TestClient.Create(context.TODO(), &passwordSecret, &e2eutil.CleanupOptions{TestContext: ctx}) @@ -245,6 +248,7 @@ func buildKubernetesResourceFromYamlFile(ctx *e2eutil.Context, yamlFilePath stri } } + obj.SetLabels(e2eutil.TestLabels()) return createOrUpdate(ctx, obj) } From 5e56466b08f17e7e526c6e32e73ffb178e03b45e Mon Sep 17 00:00:00 2001 From: FlorentinaSimlinger <60526092+FlorentinaSimlinger@users.noreply.github.com> Date: Wed, 21 Jul 2021 16:00:22 +0100 Subject: [PATCH 368/790] CLOUDP-89479: PodAntiAffinity sample CR (#648) Co-authored-by: Rajdeep Das --- ...1_mongodbcommunity_cr_podantiaffinity.yaml | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 config/samples/mongodb.com_v1_mongodbcommunity_cr_podantiaffinity.yaml diff --git a/config/samples/mongodb.com_v1_mongodbcommunity_cr_podantiaffinity.yaml b/config/samples/mongodb.com_v1_mongodbcommunity_cr_podantiaffinity.yaml new file mode 100644 index 000000000..20088c1f3 --- /dev/null +++ b/config/samples/mongodb.com_v1_mongodbcommunity_cr_podantiaffinity.yaml @@ -0,0 +1,58 @@ +--- +apiVersion: mongodbcommunity.mongodb.com/v1 +kind: MongoDBCommunity +metadata: + name: example-mongodb +spec: + members: 3 + type: ReplicaSet + version: "4.2.6" + security: + authentication: + modes: ["SCRAM"] + users: + - name: my-user + db: admin + passwordSecretRef: # a reference to the secret that will be used to generate the user's password + name: my-user-password + roles: + - name: clusterAdmin + db: admin + - name: userAdminAnyDatabase + db: admin + scramCredentialsSecretName: my-scram + statefulSet: + spec: + selector: + matchLabels: + app: mongodb + template: + metadata: + # label the pod which is used by the "labelSelector" in podAntiAffinty + # you can label it witch some other labels as well -- make sure it change the podAntiAffinity labelselector accordingly + labels: + app: mongodb + spec: + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - mongodb + topologyKey: kubernetes.io/hostname + +# the user credentials will be generated from this secret +# once the credentials are generated, this secret is no longer required +--- +apiVersion: v1 +kind: Secret +metadata: + name: my-user-password +type: Opaque +stringData: + password: From 0a6edf72afa15eed53a02402623861b3175aa013 Mon Sep 17 00:00:00 2001 From: killian2k Date: Fri, 23 Jul 2021 12:04:29 +0200 Subject: [PATCH 369/790] Cloudp 95948 community code base with helm (#649) * First version of helm * Fixed the issues * corrected chart.yaml * Fixing super lint action * Fixing super lint action 2 * Delete temp * cleaning * clean the code --- .github/workflows/main.yaml | 1 + helm-chart/Chart.yaml | 14 ++++ helm-chart/templates/tests/CRDs.yaml | 78 +++++++++++++++++++ helm-chart/templates/tests/config.yaml | 0 .../templates/tests/database_roles.yaml | 48 ++++++++++++ .../templates/tests/operator_roles.yaml | 76 ++++++++++++++++++ helm-chart/values.yaml | 40 ++++++++++ 7 files changed, 257 insertions(+) create mode 100644 helm-chart/Chart.yaml create mode 100644 helm-chart/templates/tests/CRDs.yaml create mode 100644 helm-chart/templates/tests/config.yaml create mode 100644 helm-chart/templates/tests/database_roles.yaml create mode 100644 helm-chart/templates/tests/operator_roles.yaml create mode 100644 helm-chart/values.yaml diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 7813fbc6f..bd0b7e0d4 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -68,5 +68,6 @@ jobs: VALIDATE_YAML: true VALIDATE_PYTHON: true VALIDATE_BASH: true + FILTER_REGEX_EXCLUDE: "/helm-chart/templates/*" # VALIDATE_GO: true This is currently broken: https://github.com/github/super-linter/issues/143 ... diff --git a/helm-chart/Chart.yaml b/helm-chart/Chart.yaml new file mode 100644 index 000000000..8828285bf --- /dev/null +++ b/helm-chart/Chart.yaml @@ -0,0 +1,14 @@ +apiVersion: "v2" +name: mongodb-kubernetes-operator +description: MongoDB Kubernetes Community Operator +version: 0.7.0 +type: application +appVersion: "0.7.0" +kubeVersion: '>=1.16-0' +keywords: +- mongodb +- database +- nosql +home: https://github.com/mongodb/mongodb-kubernetes-operator/ +maintainers: +- name: Cloud Team diff --git a/helm-chart/templates/tests/CRDs.yaml b/helm-chart/templates/tests/CRDs.yaml new file mode 100644 index 000000000..91ce48851 --- /dev/null +++ b/helm-chart/templates/tests/CRDs.yaml @@ -0,0 +1,78 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + email: support@mongodb.com + labels: + owner: mongodb + name: {{ .Values.operator.name }} + {{- if .Values.namespace }} + namespace: {{ .Values.namespace }} + {{- end }} +spec: + replicas: 1 + selector: + matchLabels: + name: {{ .Values.operator.name }} + strategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + labels: + name: {{ .Values.operator.name }} + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: name + operator: In + values: + - {{ .Values.operator.name }} + topologyKey: kubernetes.io/hostname + containers: + - command: + - /usr/local/bin/entrypoint + env: + - name: WATCH_NAMESPACE +{{- if .Values.operator.watchNamespace}} + value: "{{ .Values.operator.watchNamespace }}" +{{- else }} + valueFrom: + fieldRef: + fieldPath: metadata.namespace +{{- end }} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: OPERATOR_NAME + value: {{ .Values.operator.name }} + - name: AGENT_IMAGE + value: "{{ .Values.registry.agent }}/{{ .Values.agent.name }}:{{ .Values.agent.version }}" + - name: VERSION_UPGRADE_HOOK_IMAGE + value: "{{ .Values.registry.version_upgrade_hook }}/{{ .Values.version_upgrade_hook.name }}:{{ .Values.version_upgrade_hook.version }}" + - name: READINESS_PROBE_IMAGE + value: "{{ .Values.registry.readiness_probe }}/{{ .Values.readiness_probe.name }}:{{ .Values.readiness_probe.version }}" + - name: MONGODB_IMAGE + value: {{ .Values.mongodb.name }} + - name: MONGODB_REPO_URL + value: {{ .Values.mongodb.repo }} + image: {{ .Values.registry.operator }}/{{ .Values.operator.operator_image_name }}:{{ .Values.operator.version }}{{ .Values.build }} + imagePullPolicy: {{ .Values.registry.pullPolicy}} + name: {{ .Values.operator.deployment_name }} + resources: + limits: + cpu: 1100m + memory: 1Gi + requests: + cpu: 500m + memory: 200Mi + securityContext: + readOnlyRootFilesystem: true + runAsUser: 2000 + serviceAccountName: {{ .Values.operator.name }} diff --git a/helm-chart/templates/tests/config.yaml b/helm-chart/templates/tests/config.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/helm-chart/templates/tests/database_roles.yaml b/helm-chart/templates/tests/database_roles.yaml new file mode 100644 index 000000000..a91f6903d --- /dev/null +++ b/helm-chart/templates/tests/database_roles.yaml @@ -0,0 +1,48 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.database.name }} + {{- if .Values.namespace }} + namespace: {{ .Values.namespace }} + {{- end }} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ .Values.database.name }} + {{- if .Values.namespace }} + namespace: {{ .Values.namespace }} + {{- end }} +rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - apiGroups: + - "" + resources: + - pods + verbs: + - patch + - delete + - get + +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.database.name }} + {{- if .Values.namespace }} + namespace: {{ .Values.namespace }} + {{- end }} +subjects: + - kind: ServiceAccount + name: {{ .Values.database.name }} +roleRef: + kind: Role + name: {{ .Values.database.name }} + apiGroup: rbac.authorization.k8s.io diff --git a/helm-chart/templates/tests/operator_roles.yaml b/helm-chart/templates/tests/operator_roles.yaml new file mode 100644 index 000000000..96c4f67da --- /dev/null +++ b/helm-chart/templates/tests/operator_roles.yaml @@ -0,0 +1,76 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.operator.name }} + {{- if .Values.namespace }} + namespace: {{ .Values.namespace }} + {{- end }} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: {{ if eq (.Values.operator.watchNamespace | default "") "*" }} ClusterRole {{ else }} Role {{ end }} +metadata: + name: {{ .Values.operator.name }} + {{- if not (eq (.Values.operator.watchNamespace | default "*") "*") }} + namespace: {{ .Values.operator.watchNamespace }} + {{- else }} + namespace: {{ .Values.namespace }} + {{- end }} +rules: +- apiGroups: + - "" + resources: + - pods + - services + - configmaps + - secrets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - apps + resources: + - statefulsets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - mongodbcommunity.mongodb.com + resources: + - mongodbcommunity + - mongodbcommunity/status + - mongodbcommunity/spec + - mongodbcommunity/finalizers + verbs: + - get + - patch + - list + - update + - watch + +--- +kind: {{ if eq (.Values.operator.watchNamespace | default "") "*" }} ClusterRoleBinding {{ else }} RoleBinding {{ end }} +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.operator.name }} + {{- if .Values.namespace }} + namespace: {{ .Values.namespace }} + {{- end }} +subjects: +- kind: ServiceAccount + name: {{ .Values.operator.name }} +roleRef: + kind: {{ if eq (.Values.operator.watchNamespace | default "") "*" }} ClusterRole {{ else }} Role {{ end }} + name: {{ .Values.operator.name }} + apiGroup: rbac.authorization.k8s.io diff --git a/helm-chart/values.yaml b/helm-chart/values.yaml new file mode 100644 index 000000000..52aff8fce --- /dev/null +++ b/helm-chart/values.yaml @@ -0,0 +1,40 @@ +# Name of the Namespace to use +namespace: mongodb + +## Operator +operator: + # Name that will be assigned to most of internal Kubernetes objects like Deployment, ServiceAccount, Role etc. + name: mongodb-kubernetes-operator + + # Name of the operator image + operator_image_name: mongodb-kubernetes-operator + + # Name of the deployment of the operator pod + deployment_name: mongodb-kubernetes-operator + + # Version of mongodb-kubernetes-operator + version: 0.7.0 + +## Operator's database +database: + name: mongodb-database + +agent: + name: mongodb-agent + version: 11.0.5.6963-1 +version_upgrade_hook: + name: mongodb-kubernetes-operator-version-upgrade-post-start-hook + version: 1.0.2 +readiness_probe: + name: mongodb-kubernetes-readinessprobe + version: 1.0.4 +mongodb: + name: mongo + repo: docker.io + +registry: + agent: quay.io/mongodb + version_upgrade_hook: quay.io/mongodb + readiness_probe: quay.io/mongodb + operator: quay.io/mongodb + pullPolicy: Always From 7d19eb914c0d56f87f985d17c381ef40ae43ef22 Mon Sep 17 00:00:00 2001 From: John Williams <55147273+jwilliams-mongo@users.noreply.github.com> Date: Fri, 23 Jul 2021 08:49:20 -0500 Subject: [PATCH 370/790] =?UTF-8?q?(DOCSP-17097)(DOCSP-17262):=20improved?= =?UTF-8?q?=20connection=20example=20and=20added=20arb=E2=80=A6=20(#650)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/deploy-configure.md | 84 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 79 insertions(+), 5 deletions(-) diff --git a/docs/deploy-configure.md b/docs/deploy-configure.md index 427e72619..98369c658 100644 --- a/docs/deploy-configure.md +++ b/docs/deploy-configure.md @@ -6,6 +6,7 @@ The [`/config/samples`](../config/samples) directory contains example MongoDB re - [Deploy a Replica Set](#deploy-a-replica-set) - [Scale a Replica Set](#scale-a-replica-set) +- [Add Arbiters to a Replica Set](#add-arbiters-to-a-replica-set) - [Upgrade your MongoDB Resource Version and Feature Compatibility Version](#upgrade-your-mongodb-resource-version-and-feature-compatibility-version) - [Example](#example) - [Deploy Replica Sets on OpenShift](#deploy-replica-sets-on-openshift) @@ -40,8 +41,8 @@ To deploy your first replica set: **NOTE**: The following command requires [jq](https://stedolan.github.io/jq/) version 1.6 or higher.

```sh - kubectl get secret -- -n mongodb -o json | \ - jq -r '.data | with_entries(.value |= @base64d)' + kubectl get secret -- -n \ + -o json | jq -r '.data | with_entries(.value |= @base64d)' ``` The command returns the replica set's standard and DNS seed list [connection strings](https://docs.mongodb.com/manual/reference/connection-string/#connection-string-formats) in addition to the user's name and password: @@ -69,13 +70,25 @@ To deploy your first replica set: name: -- key: connectionString.standardSrv -5. Use one of the connection strings returned in the previous step to connect to the replica set: +5. Connect to one of your application's pods in the Kubernetes cluster: + + **NOTE**: You can access your replica set only from a pod in the same Kubernetes cluster. You can't access your replica set from outside of the Kubernetes cluster. + + ``` + kubectl -n exec --stdin --tty -- /bin/bash + ``` + + When you connect to your application pod, a shell prompt appears for your application's container: + ``` - mongo "mongodb+srv://:@example-mongodb-svc.mongodb.svc.cluster.local/admin?ssl=true" + user@app:~$ ``` - **NOTE**: You can access each `mongod` process in the replica set only from within a pod running in the cluster. +6. Use one of the connection strings returned in step 4 to connect to the replica set. The following example uses [`mongosh`](https://docs.mongodb.com/mongodb-shell/) to connect to a replica set: + ``` + mongosh "mongodb+srv://:@example-mongodb-svc.mongodb.svc.cluster.local/admin?ssl=true" + ``` ## Scale a Replica Set @@ -121,6 +134,67 @@ To scale a replica set: might take several minutes to remove the StatefulSet replicas for the members that you remove from the replica set. +## Add Arbiters to a Replica Set + +To add [arbiters](https://docs.mongodb.com/manual/core/replica-set-arbiter/) to your replica set, add the `spec.arbiters` field to your MongoDB resource definition. + +The value of the `spec.arbiters` field must be: + +- a positive integer, and +- less than the value of the `spec.members` field. + +**NOTE**: At least one replica set member must not be an arbiter. + +Consider the following MongoDB resource definition example: + +```yaml +apiVersion: mongodbcommunity.mongodb.com/v1 +kind: MongoDBCommunity +metadata: + name: example-mongodb +spec: + members: 3 + type: ReplicaSet + version: "4.2.7" +``` + +To add arbiters: + +1. Edit the resource definition. + + Add the `spec.arbiters` field and assign its value to the number of arbiters that you want the replica set to have. + + ```yaml + apiVersion: mongodbcommunity.mongodb.com/v1 + kind: MongoDBCommunity + metadata: + name: example-mongodb + spec: + members: 3 + type: ReplicaSet + arbiters: 3 + version: "4.2.7" + ``` + + If necessary, update the value of the `spec.members` field to ensure that you have at least one member that is not an arbiter: + + ```yaml + apiVersion: mongodbcommunity.mongodb.com/v1 + kind: MongoDBCommunity + metadata: + name: example-mongodb + spec: + members: 5 + type: ReplicaSet + arbiters: 3 + version: "4.2.7" + ``` + +2. Reapply the configuration to Kubernetes: + ``` + kubectl apply -f .yaml --namespace + ``` + ## Upgrade your MongoDB Resource Version and Feature Compatibility Version You can upgrade the major, minor, and/or feature compatibility versions of your MongoDB resource. These settings are configured in your resource definition YAML file. From 9af0c37f6ac56f33e3f4a16baca84466cc94a4a7 Mon Sep 17 00:00:00 2001 From: Rajdeep Das Date: Mon, 26 Jul 2021 13:12:30 +0200 Subject: [PATCH 371/790] Add SetData method to configmap builder (#653) --- controllers/mongodb_tls_test.go | 2 +- pkg/kube/client/mocked_client_test.go | 4 +++- pkg/kube/configmap/configmap_builder.go | 9 ++++++++- pkg/kube/configmap/configmap_test.go | 18 +++++++++--------- test/e2e/setup/setup.go | 2 +- 5 files changed, 22 insertions(+), 13 deletions(-) diff --git a/controllers/mongodb_tls_test.go b/controllers/mongodb_tls_test.go index 983d343c4..49260c1f3 100644 --- a/controllers/mongodb_tls_test.go +++ b/controllers/mongodb_tls_test.go @@ -286,7 +286,7 @@ func createTLSConfigMap(c k8sClient.Client, mdb mdbv1.MongoDBCommunity) error { configMap := configmap.Builder(). SetName(mdb.Spec.Security.TLS.CaConfigMap.Name). SetNamespace(mdb.Namespace). - SetField("ca.crt", "CERT"). + SetDataField("ca.crt", "CERT"). Build() return c.Create(context.TODO(), &configMap) diff --git a/pkg/kube/client/mocked_client_test.go b/pkg/kube/client/mocked_client_test.go index 968e7fe38..a27d00518 100644 --- a/pkg/kube/client/mocked_client_test.go +++ b/pkg/kube/client/mocked_client_test.go @@ -17,7 +17,8 @@ func TestMockedClient(t *testing.T) { cm := configmap.Builder(). SetName("cm-name"). SetNamespace("cm-namespace"). - SetField("field-1", "value-1"). + SetDataField("field-1", "value-1"). + SetData(map[string]string{"key-2": "field-2"}). Build() err := mockedClient.Create(context.TODO(), &cm) @@ -28,6 +29,7 @@ func TestMockedClient(t *testing.T) { assert.NoError(t, err) assert.Equal(t, "cm-namespace", newCm.Namespace) assert.Equal(t, "cm-name", newCm.Name) + assert.Equal(t, newCm.Data, map[string]string{"field-1": "value-1", "key-2": "field-2"}) svc := service.Builder(). SetName("svc-name"). diff --git a/pkg/kube/configmap/configmap_builder.go b/pkg/kube/configmap/configmap_builder.go index f0b175f4f..7c93d9331 100644 --- a/pkg/kube/configmap/configmap_builder.go +++ b/pkg/kube/configmap/configmap_builder.go @@ -23,7 +23,7 @@ func (b *builder) SetNamespace(namespace string) *builder { return b } -func (b *builder) SetField(key, value string) *builder { +func (b *builder) SetDataField(key, value string) *builder { b.data[key] = value return b } @@ -42,6 +42,13 @@ func (b *builder) SetLabels(labels map[string]string) *builder { return b } +func (b *builder) SetData(data map[string]string) *builder { + for k, v := range data { + b.SetDataField(k, v) + } + return b +} + func (b builder) Build() corev1.ConfigMap { return corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ diff --git a/pkg/kube/configmap/configmap_test.go b/pkg/kube/configmap/configmap_test.go index 4471152e5..2e53864f9 100644 --- a/pkg/kube/configmap/configmap_test.go +++ b/pkg/kube/configmap/configmap_test.go @@ -33,8 +33,8 @@ func TestReadKey(t *testing.T) { Builder(). SetName("name"). SetNamespace("namespace"). - SetField("key1", "value1"). - SetField("key2", "value2"). + SetDataField("key1", "value1"). + SetDataField("key2", "value2"). Build(), ) @@ -55,8 +55,8 @@ func TestReadData(t *testing.T) { Builder(). SetName("name"). SetNamespace("namespace"). - SetField("key1", "value1"). - SetField("key2", "value2"). + SetDataField("key1", "value1"). + SetDataField("key2", "value2"). Build(), ) @@ -75,7 +75,7 @@ func TestReadFileLikeField(t *testing.T) { Builder(). SetName("name"). SetNamespace("namespace"). - SetField("key1", "value1=1\nvalue2=2"). + SetDataField("key1", "value1=1\nvalue2=2"). Build(), ) @@ -90,7 +90,7 @@ func TestReadFileLikeField_InvalidExternalKey(t *testing.T) { Builder(). SetName("name"). SetNamespace("namespace"). - SetField("key1", "value1=1\nvalue2=2"). + SetDataField("key1", "value1=1\nvalue2=2"). Build(), ) @@ -104,7 +104,7 @@ func TestReadFileLikeField_InvalidInternalKey(t *testing.T) { Builder(). SetName("name"). SetNamespace("namespace"). - SetField("key1", "value1=1\nvalue2=2"). + SetDataField("key1", "value1=1\nvalue2=2"). Build(), ) @@ -140,8 +140,8 @@ func TestUpdateField(t *testing.T) { Builder(). SetName("name"). SetNamespace("namespace"). - SetField("field1", "value1"). - SetField("field2", "value2"). + SetDataField("field1", "value1"). + SetDataField("field2", "value2"). Build(), ) err := UpdateField(getUpdater, nsName("namespace", "name"), "field1", "newValue") diff --git a/test/e2e/setup/setup.go b/test/e2e/setup/setup.go index 5f44cd325..9a16b63b5 100644 --- a/test/e2e/setup/setup.go +++ b/test/e2e/setup/setup.go @@ -74,7 +74,7 @@ func CreateTLSResources(namespace string, ctx *e2eutil.Context, secretType tlsSe caConfigMap := configmap.Builder(). SetName(tlsConfig.CaConfigMap.Name). SetNamespace(namespace). - SetField("ca.crt", string(ca)). + SetDataField("ca.crt", string(ca)). SetLabels(e2eutil.TestLabels()). Build() From c7ef9c21ad3222b2cc7c62b9cd45d39c42426e77 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 27 Jul 2021 13:52:02 +0100 Subject: [PATCH 372/790] Bump go.mongodb.org/mongo-driver from 1.5.4 to 1.7.0 (#656) Bumps [go.mongodb.org/mongo-driver](https://github.com/mongodb/mongo-go-driver) from 1.5.4 to 1.7.0. - [Release notes](https://github.com/mongodb/mongo-go-driver/releases) - [Commits](https://github.com/mongodb/mongo-go-driver/compare/v1.5.4...v1.7.0) --- updated-dependencies: - dependency-name: go.mongodb.org/mongo-driver dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 11 ++--------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index 108f0e488..f21557afb 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/stretchr/objx v0.3.0 github.com/stretchr/testify v1.7.0 github.com/xdg/stringprep v1.0.3 - go.mongodb.org/mongo-driver v1.5.4 + go.mongodb.org/mongo-driver v1.7.0 go.uber.org/zap v1.18.1 k8s.io/api v0.21.2 k8s.io/apimachinery v0.21.2 diff --git a/go.sum b/go.sum index 4807e961c..e6d2e60d5 100644 --- a/go.sum +++ b/go.sum @@ -46,8 +46,6 @@ github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hC github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go v1.34.28 h1:sscPpn/Ns3i0F4HPEWAVcwdIRaZZCuL7llJ2/60yPIk= -github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -128,7 +126,6 @@ github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8 github.com/go-openapi/spec v0.19.5/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk= github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= @@ -261,10 +258,6 @@ github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= -github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= -github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= @@ -459,8 +452,8 @@ go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= -go.mongodb.org/mongo-driver v1.5.4 h1:NPIBF/lxEcKNfWwoCJRX8+dMVwecWf9q3qUJkuh75oM= -go.mongodb.org/mongo-driver v1.5.4/go.mod h1:gRXCHX4Jo7J0IJ1oDQyUxF7jfy19UfxniMS4xxMmUqw= +go.mongodb.org/mongo-driver v1.7.0 h1:hHrvOBWlWB2c7+8Gh/Xi5jj82AgidK/t7KVXBZ+IyUA= +go.mongodb.org/mongo-driver v1.7.0/go.mod h1:Q4oFMbo1+MSNqICAdYMlC/zSTrwCogR4R8NzkI+yfU8= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= From d694f402f6d188ee1cc83186e5ec4c300b7b65ad Mon Sep 17 00:00:00 2001 From: Ciprian Tibulca Date: Tue, 27 Jul 2021 15:00:53 +0100 Subject: [PATCH 373/790] bump go dependencies (#660) --- go.mod | 10 +++++----- go.sum | 20 ++++++++++++-------- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/go.mod b/go.mod index f21557afb..d9403cd41 100644 --- a/go.mod +++ b/go.mod @@ -9,16 +9,16 @@ require ( github.com/imdario/mergo v0.3.12 github.com/klauspost/compress v1.9.8 // indirect github.com/pkg/errors v0.9.1 - github.com/spf13/cast v1.3.1 + github.com/spf13/cast v1.4.0 github.com/stretchr/objx v0.3.0 github.com/stretchr/testify v1.7.0 github.com/xdg/stringprep v1.0.3 go.mongodb.org/mongo-driver v1.7.0 go.uber.org/zap v1.18.1 - k8s.io/api v0.21.2 - k8s.io/apimachinery v0.21.2 - k8s.io/client-go v0.21.2 - sigs.k8s.io/controller-runtime v0.9.2 + k8s.io/api v0.21.3 + k8s.io/apimachinery v0.21.3 + k8s.io/client-go v0.21.3 + sigs.k8s.io/controller-runtime v0.9.3 sigs.k8s.io/yaml v1.2.0 ) diff --git a/go.sum b/go.sum index e6d2e60d5..27b5e1f70 100644 --- a/go.sum +++ b/go.sum @@ -405,8 +405,8 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= -github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.4.0 h1:WhlbjwB9EGCc8W5Rxdkus+wmH2ASRwwTJk6tgHKwdqQ= +github.com/spf13/cast v1.4.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= @@ -781,15 +781,18 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.21.2 h1:vz7DqmRsXTCSa6pNxXwQ1IYeAZgdIsua+DZU+o+SX3Y= k8s.io/api v0.21.2/go.mod h1:Lv6UGJZ1rlMI1qusN8ruAp9PUBFyBwpEHAdG24vIsiU= +k8s.io/api v0.21.3 h1:cblWILbLO8ar+Fj6xdDGr603HRsf8Wu9E9rngJeprZQ= +k8s.io/api v0.21.3/go.mod h1:hUgeYHUbBp23Ue4qdX9tR8/ANi/g3ehylAqDn9NWVOg= k8s.io/apiextensions-apiserver v0.21.2 h1:+exKMRep4pDrphEafRvpEi79wTnCFMqKf8LBtlA3yrE= k8s.io/apiextensions-apiserver v0.21.2/go.mod h1:+Axoz5/l3AYpGLlhJDfcVQzCerVYq3K3CvDMvw6X1RA= -k8s.io/apimachinery v0.21.2 h1:vezUc/BHqWlQDnZ+XkrpXSmnANSLbpnlpwo0Lhk0gpc= k8s.io/apimachinery v0.21.2/go.mod h1:CdTY8fU/BlvAbJ2z/8kBwimGki5Zp8/fbVuLY8gJumM= +k8s.io/apimachinery v0.21.3 h1:3Ju4nvjCngxxMYby0BimUk+pQHPOQp3eCGChk5kfVII= +k8s.io/apimachinery v0.21.3/go.mod h1:H/IM+5vH9kZRNJ4l3x/fXP/5bOPJaVP/guptnZPeCFI= k8s.io/apiserver v0.21.2/go.mod h1:lN4yBoGyiNT7SC1dmNk0ue6a5Wi6O3SWOIw91TsucQw= -k8s.io/client-go v0.21.2 h1:Q1j4L/iMN4pTw6Y4DWppBoUxgKO8LbffEMVEV00MUp0= k8s.io/client-go v0.21.2/go.mod h1:HdJ9iknWpbl3vMGtib6T2PyI/VYxiZfq936WNVHBRrA= +k8s.io/client-go v0.21.3 h1:J9nxZTOmvkInRDCzcSNQmPJbDYN/PjlxXT9Mos3HcLg= +k8s.io/client-go v0.21.3/go.mod h1:+VPhCgTsaFmGILxR/7E1N0S+ryO010QBeNCv5JwRGYU= k8s.io/code-generator v0.21.2/go.mod h1:8mXJDCB7HcRo1xiEQstcguZkbxZaqeUOrO9SsicWs3U= k8s.io/component-base v0.21.2 h1:EsnmFFoJ86cEywC0DoIkAUiEV6fjgauNugiw1lmIjs4= k8s.io/component-base v0.21.2/go.mod h1:9lvmIThzdlrJj5Hp8Z/TOgIkdfsNARQ1pT+3PByuiuc= @@ -808,11 +811,12 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.19/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= -sigs.k8s.io/controller-runtime v0.9.2 h1:MnCAsopQno6+hI9SgJHKddzXpmv2wtouZz6931Eax+Q= -sigs.k8s.io/controller-runtime v0.9.2/go.mod h1:TxzMCHyEUpaeuOiZx/bIdc2T81vfs/aKdvJt9wuu0zk= +sigs.k8s.io/controller-runtime v0.9.3 h1:n075bHQ1wb8hpX7C27pNrqsb0fj8mcfCQfNX+oKTbYE= +sigs.k8s.io/controller-runtime v0.9.3/go.mod h1:TxzMCHyEUpaeuOiZx/bIdc2T81vfs/aKdvJt9wuu0zk= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.1.0 h1:C4r9BgJ98vrKnnVCjwCSXcWjWe0NKcUQkmzDXZXGwH8= sigs.k8s.io/structured-merge-diff/v4 v4.1.0/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.1.2 h1:Hr/htKFmJEbtMgS/UD0N+gtgctAqz81t3nu+sPzynno= +sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= From 39ddc054ee4b0fff9fef35f8c6b04effd2de53cf Mon Sep 17 00:00:00 2001 From: FlorentinaSimlinger <60526092+FlorentinaSimlinger@users.noreply.github.com> Date: Tue, 27 Jul 2021 16:35:50 +0100 Subject: [PATCH 374/790] CLOUDP-95181: version is missing (#655) Co-authored-by: Rajdeep Das --- api/v1/mongodbcommunity_types.go | 1 + ...ommunity.mongodb.com_mongodbcommunity.yaml | 3 +++ controllers/mongodb_status_options.go | 22 ++++++++++++++++- controllers/mongodb_status_options_test.go | 24 +++++++++++++++---- controllers/replica_set_controller.go | 3 ++- deploy/e2e/service_account.yaml | 4 ++-- test/e2e/mongodbtests/mongodbtests.go | 1 + .../replica_set_recovery_test.go | 1 + .../replica_set_scaling_test.go | 2 ++ .../replica_set_scale_down_test.go | 1 + 10 files changed, 53 insertions(+), 9 deletions(-) diff --git a/api/v1/mongodbcommunity_types.go b/api/v1/mongodbcommunity_types.go index 33d3f8005..af6263615 100644 --- a/api/v1/mongodbcommunity_types.go +++ b/api/v1/mongodbcommunity_types.go @@ -375,6 +375,7 @@ func ConvertAuthModeToAuthMechanism(authModeLabel AuthMode) string { type MongoDBCommunityStatus struct { MongoURI string `json:"mongoUri"` Phase Phase `json:"phase"` + Version string `json:"version"` CurrentStatefulSetReplicas int `json:"currentStatefulSetReplicas"` CurrentMongoDBMembers int `json:"currentMongoDBMembers"` diff --git a/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml b/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml index 1bec50498..38834247c 100644 --- a/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml +++ b/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml @@ -322,11 +322,14 @@ spec: type: string phase: type: string + version: + type: string required: - currentMongoDBMembers - currentStatefulSetReplicas - mongoUri - phase + - version type: object type: object served: true diff --git a/controllers/mongodb_status_options.go b/controllers/mongodb_status_options.go index 105596ba8..1e4074198 100644 --- a/controllers/mongodb_status_options.go +++ b/controllers/mongodb_status_options.go @@ -33,7 +33,7 @@ func (o *optionBuilder) GetOptions() []status.Option { return o.options } -// options returns an initialized optionBuilder +// statusOptions returns an initialized optionBuilder func statusOptions() *optionBuilder { return &optionBuilder{ options: []status.Option{}, @@ -60,6 +60,26 @@ func (m mongoUriOption) GetResult() (reconcile.Result, error) { return result.OK() } +func (o *optionBuilder) withVersion(version string) *optionBuilder { + o.options = append(o.options, + versionOption{ + version: version, + }) + return o +} + +type versionOption struct { + version string +} + +func (v versionOption) ApplyOption(mdb *mdbv1.MongoDBCommunity) { + mdb.Status.Version = v.version +} + +func (v versionOption) GetResult() (reconcile.Result, error) { + return result.OK() +} + func (o *optionBuilder) withPhase(phase mdbv1.Phase, retryAfter int) *optionBuilder { o.options = append(o.options, phaseOption{ diff --git a/controllers/mongodb_status_options_test.go b/controllers/mongodb_status_options_test.go index 7484cd6fe..9041c8d99 100644 --- a/controllers/mongodb_status_options_test.go +++ b/controllers/mongodb_status_options_test.go @@ -9,9 +9,11 @@ import ( "github.com/stretchr/testify/assert" ) +const testVersion string = "4.2.6" + func TestMongoUriOption_ApplyOption(t *testing.T) { - mdb := newReplicaSet(3, "my-rs", "my-ns") + mdb := newReplicaSet(3, testVersion, "my-rs", "my-ns") opt := mongoUriOption{ mongoUri: "my-uri", @@ -23,7 +25,7 @@ func TestMongoUriOption_ApplyOption(t *testing.T) { } func TestOptionBuilder_RunningPhase(t *testing.T) { - mdb := newReplicaSet(3, "my-rs", "my-ns") + mdb := newReplicaSet(3, testVersion, "my-rs", "my-ns") statusOptions().withRunningPhase().GetOptions()[0].ApplyOption(&mdb) @@ -31,7 +33,7 @@ func TestOptionBuilder_RunningPhase(t *testing.T) { } func TestOptionBuilder_PendingPhase(t *testing.T) { - mdb := newReplicaSet(3, "my-rs", "my-ns") + mdb := newReplicaSet(3, testVersion, "my-rs", "my-ns") statusOptions().withPendingPhase(10).GetOptions()[0].ApplyOption(&mdb) @@ -39,14 +41,25 @@ func TestOptionBuilder_PendingPhase(t *testing.T) { } func TestOptionBuilder_FailedPhase(t *testing.T) { - mdb := newReplicaSet(3, "my-rs", "my-ns") + mdb := newReplicaSet(3, testVersion, "my-rs", "my-ns") statusOptions().withFailedPhase().GetOptions()[0].ApplyOption(&mdb) assert.Equal(t, mdbv1.Failed, mdb.Status.Phase) } -func newReplicaSet(members int, name, namespace string) mdbv1.MongoDBCommunity { +func TestVersion_ApplyOption(t *testing.T) { + mdb := newReplicaSet(3, testVersion, "my-rs", "my-ns") + + opt := versionOption{ + version: testVersion, + } + opt.ApplyOption(&mdb) + + assert.Equal(t, testVersion, mdb.Status.Version, "Status should be updated") +} + +func newReplicaSet(members int, version string, name, namespace string) mdbv1.MongoDBCommunity { return mdbv1.MongoDBCommunity{ TypeMeta: metav1.TypeMeta{}, ObjectMeta: metav1.ObjectMeta{ @@ -55,6 +68,7 @@ func newReplicaSet(members int, name, namespace string) mdbv1.MongoDBCommunity { }, Spec: mdbv1.MongoDBCommunitySpec{ Members: members, + Version: version, }, } } diff --git a/controllers/replica_set_controller.go b/controllers/replica_set_controller.go index ef95cb6a4..c15a1b2cd 100644 --- a/controllers/replica_set_controller.go +++ b/controllers/replica_set_controller.go @@ -224,7 +224,8 @@ func (r ReplicaSetReconciler) Reconcile(ctx context.Context, request reconcile.R withMongoDBMembers(mdb.AutomationConfigMembersThisReconciliation()). withStatefulSetReplicas(mdb.StatefulSetReplicasThisReconciliation()). withMessage(None, ""). - withRunningPhase(), + withRunningPhase(). + withVersion(mdb.GetMongoDBVersion()), ) if err != nil { r.log.Errorf("Error updating the status of the MongoDB resource: %s", err) diff --git a/deploy/e2e/service_account.yaml b/deploy/e2e/service_account.yaml index e94e44181..ebc746c83 100644 --- a/deploy/e2e/service_account.yaml +++ b/deploy/e2e/service_account.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: ServiceAccount metadata: name: e2e-test - # namespace is 'default' for the e2e tests because the TLS certificates - # generated used as test fixture (in the TLS tests) only work with the + # namespace is 'default' for the e2e tests because the TLS certificates + # generated used as test fixture (in the TLS tests) only work with the # default namespace. namespace: default diff --git a/test/e2e/mongodbtests/mongodbtests.go b/test/e2e/mongodbtests/mongodbtests.go index 5b6b41056..3fc43c801 100644 --- a/test/e2e/mongodbtests/mongodbtests.go +++ b/test/e2e/mongodbtests/mongodbtests.go @@ -310,6 +310,7 @@ func BasicFunctionality(mdb *mdbv1.MongoDBCommunity) func(*testing.T) { mdbv1.MongoDBCommunityStatus{ MongoURI: mdb.MongoURI(), Phase: mdbv1.Running, + Version: mdb.GetMongoDBVersion(), CurrentMongoDBMembers: mdb.Spec.Members, CurrentStatefulSetReplicas: mdb.Spec.Members, })) diff --git a/test/e2e/replica_set_recovery/replica_set_recovery_test.go b/test/e2e/replica_set_recovery/replica_set_recovery_test.go index 9f83460b4..8000a3261 100644 --- a/test/e2e/replica_set_recovery/replica_set_recovery_test.go +++ b/test/e2e/replica_set_recovery/replica_set_recovery_test.go @@ -58,6 +58,7 @@ func TestReplicaSetRecovery(t *testing.T) { mdbv1.MongoDBCommunityStatus{ MongoURI: mdb.MongoURI(), Phase: mdbv1.Running, + Version: mdb.GetMongoDBVersion(), CurrentMongoDBMembers: 3, CurrentStatefulSetReplicas: 3, })) diff --git a/test/e2e/replica_set_scale/replica_set_scaling_test.go b/test/e2e/replica_set_scale/replica_set_scaling_test.go index 19d5d535d..3f027452a 100644 --- a/test/e2e/replica_set_scale/replica_set_scaling_test.go +++ b/test/e2e/replica_set_scale/replica_set_scaling_test.go @@ -53,6 +53,7 @@ func TestReplicaSetScaleUp(t *testing.T) { mdbv1.MongoDBCommunityStatus{ MongoURI: mdb.MongoURI(), Phase: mdbv1.Running, + Version: mdb.GetMongoDBVersion(), CurrentMongoDBMembers: 5, CurrentStatefulSetReplicas: 5, })) @@ -66,6 +67,7 @@ func TestReplicaSetScaleUp(t *testing.T) { // mdbv1.MongoDBStatus{ // MongoURI: mdb.MongoURI(), // Phase: mdbv1.Running, + // Version: mdb.GetMongoDBVersion(), // CurrentMongoDBMembers: 3, // CurrentStatefulSetReplicas: 3, // })) diff --git a/test/e2e/replica_set_scale_down/replica_set_scale_down_test.go b/test/e2e/replica_set_scale_down/replica_set_scale_down_test.go index 59cdbdfd4..6ee3a76e7 100644 --- a/test/e2e/replica_set_scale_down/replica_set_scale_down_test.go +++ b/test/e2e/replica_set_scale_down/replica_set_scale_down_test.go @@ -56,6 +56,7 @@ func TestReplicaSetScaleDown(t *testing.T) { mdbv1.MongoDBCommunityStatus{ MongoURI: mdb.MongoURI(), Phase: mdbv1.Running, + Version: mdb.GetMongoDBVersion(), CurrentMongoDBMembers: 1, CurrentStatefulSetReplicas: 1, })) From 48f79710cf73787de7c4d8f70b965cd68ae74b3f Mon Sep 17 00:00:00 2001 From: killian2k Date: Fri, 30 Jul 2021 15:30:17 +0200 Subject: [PATCH 375/790] Cloudp 95948 integrate helm to makefile (#654) * First version of helm * Fixed the issues * corrected chart.yaml * Fixing super lint action * Fixing super lint action 2 * Delete temp * cleaning * clean the code * Modify the makefile. Working on MacOS * Install helm using git command * modified way to install helm * File saved in a temp folder * Correcting the makefile * cleaning makefile * Remove kustomize * fixing makefile * debugged makefile * Modified name release chart * fix bug RELEASE_NAME_HELM * Now operator is running * Fixed bugs * removed build * Update Makefile --- Makefile | 60 +++++++++++++------ helm-chart/templates/tests/config.yaml | 0 .../tests/{CRDs.yaml => operator.yaml} | 2 +- 3 files changed, 43 insertions(+), 19 deletions(-) delete mode 100644 helm-chart/templates/tests/config.yaml rename helm-chart/templates/tests/{CRDs.yaml => operator.yaml} (97%) diff --git a/Makefile b/Makefile index a555f370d..259d94ffc 100644 --- a/Makefile +++ b/Makefile @@ -4,10 +4,13 @@ SHELL := /bin/bash REPO_URL := $(shell jq -r .repo_url < ~/.community-operator-dev/config.json) OPERATOR_IMAGE := $(shell jq -r .operator_image < ~/.community-operator-dev/config.json) NAMESPACE := $(shell jq -r .namespace < ~/.community-operator-dev/config.json) -IMG := $(REPO_URL)/$(OPERATOR_IMAGE) +UPGRADE_HOOK_IMG := $(shell jq -r .version_upgrade_hook_image < ~/.community-operator-dev/config.json) +READINESS_PROBE_IMG := $(shell jq -r .readiness_probe_image < ~/.community-operator-dev/config.json) + DOCKERFILE ?= operator # Produce CRDs that work back to Kubernetes 1.11 (no version conversion) CRD_OPTIONS ?= "crd:trivialVersions=true,crdVersions=v1" +RELEASE_NAME_HELM ?= community-operator-chart # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) ifeq (,$(shell go env GOBIN)) @@ -31,27 +34,35 @@ manager: generate fmt vet go build -o bin/manager ./cmd/manager/main.go # Run against the configured Kubernetes cluster in ~/.kube/config -run: install - $(KUSTOMIZE) build config/local_run | kubectl apply -n $(NAMESPACE) -f - +run: install install-chart eval $$(scripts/dev/get_e2e_env_vars.py $(cleanup)); \ go run ./cmd/manager/main.go # Install CRDs into a cluster -install: manifests kustomize - $(KUSTOMIZE) build config/crd | kubectl apply -f - +install: manifests helm install-crd + + +install-crd: + kubectl apply -f config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml + +install-chart: + helm upgrade --install --set namespace=$(NAMESPACE),version_upgrade_hook.name=$(UPGRADE_HOOK_IMG),readiness_probe.name=$(READINESS_PROBE_IMG),registry.operator=$(REPO_URL),operator.operator_image_name=$(OPERATOR_IMAGE),operator.version=latest $(RELEASE_NAME_HELM) helm-chart + + +uninstall-crd: + kubectl delete crd mongodbcommunity.mongodbcommunity.mongodb.com + +uninstall-chart: + $(HELM) uninstall $(RELEASE_NAME_HELM) -n $(NAMESPACE) # Uninstall CRDs from a cluster -uninstall: manifests kustomize - $(KUSTOMIZE) build config/crd | kubectl delete -f - +uninstall: manifests helm uninstall-chart uninstall-crd # Deploy controller in the configured Kubernetes cluster in ~/.kube/config -deploy: manifests kustomize - cd config/manager && $(KUSTOMIZE) edit set image quay.io/mongodb/mongodb-kubernetes-operator=$(IMG):latest - $(KUSTOMIZE) build config/default | kubectl apply -n $(NAMESPACE) -f - +deploy: manifests helm install-chart install-crd # UnDeploy controller from the configured Kubernetes cluster in ~/.kube/config -undeploy: - $(KUSTOMIZE) build config/default | kubectl delete -f - +undeploy: uninstall-chart uninstall-crd # Generate manifests e.g. CRD, RBAC etc. manifests: controller-gen @@ -71,11 +82,11 @@ e2e-telepresence: cleanup-e2e install telepresence connect; \ telepresence status; \ eval $$(scripts/dev/get_e2e_env_vars.py $(cleanup)); \ - go test -v -timeout=30m -failfast ./test/e2e/$(test); \ + go test -v -timeout=30m -failfast ./test/e2e/$(test); \ telepresence quit # Run e2e test by deploying test image in kubernetes. -e2e-k8s: cleanup-e2e install e2e-image +e2e-k8s: install-crd cleanup-e2e install e2e-image python scripts/dev/e2e.py --perform-cleanup --test $(test) # Run e2e test locally. @@ -124,10 +135,23 @@ CONTROLLER_GEN = $(shell pwd)/bin/controller-gen controller-gen: $(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.4.1) -# Download kustomize locally if necessary -KUSTOMIZE = $(shell pwd)/bin/kustomize -kustomize: - $(call go-get-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v4@v4.2.0) +# Download helm locally if necessary +HELM = /usr/local/bin/helm +helm: + $(call install-helm) + + +define install-helm +@[ -f $(HELM) ] || { \ +set -e ;\ +TMP_DIR=$$(mktemp -d) ;\ +cd $$TMP_DIR ;\ +curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 ;\ +chmod 700 get_helm.sh ;\ +./get_helm.sh ;\ +rm -rf $(TMP_DIR) ;\ +} +endef # go-get-tool will 'go get' any package $2 and install it to $1. PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST)))) diff --git a/helm-chart/templates/tests/config.yaml b/helm-chart/templates/tests/config.yaml deleted file mode 100644 index e69de29bb..000000000 diff --git a/helm-chart/templates/tests/CRDs.yaml b/helm-chart/templates/tests/operator.yaml similarity index 97% rename from helm-chart/templates/tests/CRDs.yaml rename to helm-chart/templates/tests/operator.yaml index 91ce48851..b20ff4f41 100644 --- a/helm-chart/templates/tests/CRDs.yaml +++ b/helm-chart/templates/tests/operator.yaml @@ -62,7 +62,7 @@ spec: value: {{ .Values.mongodb.name }} - name: MONGODB_REPO_URL value: {{ .Values.mongodb.repo }} - image: {{ .Values.registry.operator }}/{{ .Values.operator.operator_image_name }}:{{ .Values.operator.version }}{{ .Values.build }} + image: {{ .Values.registry.operator }}/{{ .Values.operator.operator_image_name }}:{{ .Values.operator.version }} imagePullPolicy: {{ .Values.registry.pullPolicy}} name: {{ .Values.operator.deployment_name }} resources: From 944225395ad61080dcd29002e035c6438f631912 Mon Sep 17 00:00:00 2001 From: FlorentinaSimlinger <60526092+FlorentinaSimlinger@users.noreply.github.com> Date: Tue, 3 Aug 2021 16:29:39 +0100 Subject: [PATCH 376/790] CLOUDP-91931: TLS recreate mdbc test (#667) --- .../replica_set_tls_recreate_mdbc_test.go | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 test/e2e/replica_set_tls_recreate_mdbc/replica_set_tls_recreate_mdbc_test.go diff --git a/test/e2e/replica_set_tls_recreate_mdbc/replica_set_tls_recreate_mdbc_test.go b/test/e2e/replica_set_tls_recreate_mdbc/replica_set_tls_recreate_mdbc_test.go new file mode 100644 index 000000000..7a1d1edf5 --- /dev/null +++ b/test/e2e/replica_set_tls_recreate_mdbc/replica_set_tls_recreate_mdbc_test.go @@ -0,0 +1,72 @@ +package replica_set_tls + +import ( + "context" + "fmt" + "os" + "testing" + + . "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/util/mongotester" + + e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" + "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/mongodbtests" + setup "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" +) + +func TestMain(m *testing.M) { + code, err := e2eutil.RunTest(m) + if err != nil { + fmt.Println(err) + } + os.Exit(code) +} + +func TestReplicaSetTLSRecreateMdbc(t *testing.T) { + ctx := setup.Setup(t) + defer ctx.Teardown() + + mdb1, user := e2eutil.NewTestMongoDB(ctx, "mdb-tls", "") + scramUser := mdb1.GetScramUsers()[0] + mdb1.Spec.Security.TLS = e2eutil.NewTestTLSConfig(false) + + _, err := setup.GeneratePasswordForUser(ctx, user, "") + if err != nil { + t.Fatal(err) + } + + if err := setup.CreateTLSResources(mdb1.Namespace, ctx, setup.CertKeyPair); err != nil { + t.Fatalf("Failed to set up TLS resources: %s", err) + } + + t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb1, ctx)) + t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb1)) + + if err := e2eutil.TestClient.Delete(context.TODO(), &mdb1); err != nil { + t.Fatalf("Failed to delete first test MongoDB: %s", err) + } + + mdb2, _ := e2eutil.NewTestMongoDB(ctx, "mdb-tls", "") + mdb2.Spec.Security.TLS = e2eutil.NewTestTLSConfig(false) + tester1, err := FromResource(t, mdb2) + if err != nil { + t.Fatal(err) + } + + t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb2, ctx)) + t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb2)) + mongodbtests.SkipTestIfLocal(t, "Ensure MongoDB TLS Configuration", func(t *testing.T) { + t.Run("Has TLS Mode", tester1.HasTlsMode("requireSSL", 60, WithTls())) + t.Run("Basic Connectivity Succeeds", tester1.ConnectivitySucceeds(WithTls())) + t.Run("SRV Connectivity Succeeds", tester1.ConnectivitySucceeds(WithURI(mdb2.MongoSRVURI()), WithTls())) + t.Run("Basic Connectivity With Generated Connection String Secret Succeeds", + tester1.ConnectivitySucceeds(WithURI(mongodbtests.GetConnectionStringForUser(mdb2, scramUser)), WithTls())) + t.Run("SRV Connectivity With Generated Connection String Secret Succeeds", + tester1.ConnectivitySucceeds(WithURI(mongodbtests.GetSrvConnectionStringForUser(mdb2, scramUser)), WithTls())) + t.Run("Connectivity Fails", tester1.ConnectivityFails(WithoutTls())) + t.Run("Ensure authentication is configured", tester1.EnsureAuthenticationIsConfigured(3, WithTls())) + }) + t.Run("TLS is disabled", mongodbtests.DisableTLS(&mdb2)) + t.Run("MongoDB Reaches Failed Phase", mongodbtests.MongoDBReachesFailedPhase(&mdb2)) + t.Run("TLS is enabled", mongodbtests.EnableTLS(&mdb2)) + t.Run("MongoDB Reaches Running Phase", mongodbtests.MongoDBReachesRunningPhase(&mdb2)) +} From 34d430d22e2fe4d54675416296088fe59b3174ca Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Wed, 4 Aug 2021 16:55:55 +0100 Subject: [PATCH 377/790] Fix clusterwide issues (#672) --- deploy/clusterwide/role.yaml | 4 +++- deploy/clusterwide/role_binding.yaml | 1 + docs/install-upgrade.md | 8 +++++--- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/deploy/clusterwide/role.yaml b/deploy/clusterwide/role.yaml index 6cfeaa876..de8abc63c 100644 --- a/deploy/clusterwide/role.yaml +++ b/deploy/clusterwide/role.yaml @@ -8,6 +8,7 @@ rules: resources: - services - configmaps + - secrets verbs: - create - delete @@ -20,7 +21,8 @@ rules: - apps resources: - statefulsets - verbs: + verbs: + - create - delete - get - list diff --git a/deploy/clusterwide/role_binding.yaml b/deploy/clusterwide/role_binding.yaml index d9be64667..11713808f 100644 --- a/deploy/clusterwide/role_binding.yaml +++ b/deploy/clusterwide/role_binding.yaml @@ -4,6 +4,7 @@ metadata: name: mongodb-kubernetes-operator subjects: - kind: ServiceAccount + namespace: name: mongodb-kubernetes-operator roleRef: kind: ClusterRole diff --git a/docs/install-upgrade.md b/docs/install-upgrade.md index e21909035..84b929c68 100644 --- a/docs/install-upgrade.md +++ b/docs/install-upgrade.md @@ -62,19 +62,21 @@ To configure the Operator to watch resources in other namespaces: value: "*" ``` -2. Run the following command to create cluster-wide roles and role-bindings in the default namespace: +2. Modify the [clusterRoleBinding](../deploy/clusterwide/role_binding.yaml) namespace value for the serviceAccount `mongodb-kubernetes-operator` to the namespace in which the operator is deployed. + +3. Run the following command to create cluster-wide roles and role-bindings in the default namespace: ```sh kubectl apply -f deploy/clusterwide ``` -3. For each namespace that you want the Operator to watch, run the following +4. For each namespace that you want the Operator to watch, run the following commands to deploy a Role, RoleBinding and ServiceAccount in that namespace: ```sh kubectl apply -k config/rbac --namespace ``` -4. [Install the operator](#procedure). +5. [Install the operator](#procedure). ### Configure the MongoDB Docker Image or Container Registry From b75fa423b9b20b6fc4f509d7528e0fff458ca4e0 Mon Sep 17 00:00:00 2001 From: killian2k Date: Thu, 5 Aug 2021 11:03:30 +0200 Subject: [PATCH 378/790] Fix makefile helm (#669) * fix the makefile * fixing the makefile run command, the clean-up command and removing the test folder of the helm chart * fixing the makefile run command, the clean-up command and removing the test folder of the helm chart --- Makefile | 18 ++++++++++++++---- .../templates/{tests => }/database_roles.yaml | 0 helm-chart/templates/{tests => }/operator.yaml | 0 .../templates/{tests => }/operator_roles.yaml | 0 4 files changed, 14 insertions(+), 4 deletions(-) rename helm-chart/templates/{tests => }/database_roles.yaml (100%) rename helm-chart/templates/{tests => }/operator.yaml (100%) rename helm-chart/templates/{tests => }/operator_roles.yaml (100%) diff --git a/Makefile b/Makefile index 259d94ffc..c137833a3 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,8 @@ NAMESPACE := $(shell jq -r .namespace < ~/.community-operator-dev/config.json) UPGRADE_HOOK_IMG := $(shell jq -r .version_upgrade_hook_image < ~/.community-operator-dev/config.json) READINESS_PROBE_IMG := $(shell jq -r .readiness_probe_image < ~/.community-operator-dev/config.json) +STRING_SET_VALUES := --set namespace=$(NAMESPACE),version_upgrade_hook.name=$(UPGRADE_HOOK_IMG),readiness_probe.name=$(READINESS_PROBE_IMG),registry.operator=$(REPO_URL),operator.operator_image_name=$(OPERATOR_IMAGE),operator.version=latest + DOCKERFILE ?= operator # Produce CRDs that work back to Kubernetes 1.11 (no version conversion) CRD_OPTIONS ?= "crd:trivialVersions=true,crdVersions=v1" @@ -34,7 +36,7 @@ manager: generate fmt vet go build -o bin/manager ./cmd/manager/main.go # Run against the configured Kubernetes cluster in ~/.kube/config -run: install install-chart +run: install install-rbac eval $$(scripts/dev/get_e2e_env_vars.py $(cleanup)); \ go run ./cmd/manager/main.go @@ -46,8 +48,11 @@ install-crd: kubectl apply -f config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml install-chart: - helm upgrade --install --set namespace=$(NAMESPACE),version_upgrade_hook.name=$(UPGRADE_HOOK_IMG),readiness_probe.name=$(READINESS_PROBE_IMG),registry.operator=$(REPO_URL),operator.operator_image_name=$(OPERATOR_IMAGE),operator.version=latest $(RELEASE_NAME_HELM) helm-chart + helm upgrade --install $(STRING_SET_VALUES) $(RELEASE_NAME_HELM) helm-chart +install-rbac: + $(HELM) template $(STRING_SET_VALUES) -s templates/database_roles.yaml helm-chart | kubectl apply -f - + $(HELM) template $(STRING_SET_VALUES) -s templates/operator_roles.yaml helm-chart | kubectl apply -f - uninstall-crd: kubectl delete crd mongodbcommunity.mongodbcommunity.mongodb.com @@ -55,6 +60,11 @@ uninstall-crd: uninstall-chart: $(HELM) uninstall $(RELEASE_NAME_HELM) -n $(NAMESPACE) +uninstall-rbac: + $(HELM) template $(STRING_SET_VALUES) -s templates/database_roles.yaml helm-chart | kubectl delete -f - + $(HELM) template $(STRING_SET_VALUES) -s templates/operator_roles.yaml helm-chart | kubectl delete -f - + + # Uninstall CRDs from a cluster uninstall: manifests helm uninstall-chart uninstall-crd @@ -86,7 +96,7 @@ e2e-telepresence: cleanup-e2e install telepresence quit # Run e2e test by deploying test image in kubernetes. -e2e-k8s: install-crd cleanup-e2e install e2e-image +e2e-k8s: cleanup-e2e install e2e-image python scripts/dev/e2e.py --perform-cleanup --test $(test) # Run e2e test locally. @@ -100,7 +110,7 @@ e2e-gh: scripts/dev/run_e2e_gh.sh $(test) cleanup-e2e: - kubectl delete mdbc,all -l e2e-test=true + kubectl delete mdbc,all -l e2e-test=true | true # Generate code generate: controller-gen diff --git a/helm-chart/templates/tests/database_roles.yaml b/helm-chart/templates/database_roles.yaml similarity index 100% rename from helm-chart/templates/tests/database_roles.yaml rename to helm-chart/templates/database_roles.yaml diff --git a/helm-chart/templates/tests/operator.yaml b/helm-chart/templates/operator.yaml similarity index 100% rename from helm-chart/templates/tests/operator.yaml rename to helm-chart/templates/operator.yaml diff --git a/helm-chart/templates/tests/operator_roles.yaml b/helm-chart/templates/operator_roles.yaml similarity index 100% rename from helm-chart/templates/tests/operator_roles.yaml rename to helm-chart/templates/operator_roles.yaml From ed9ec900804bef5f1fb278a39b29038d0feac604 Mon Sep 17 00:00:00 2001 From: Rajdeep Das Date: Thu, 5 Aug 2021 18:36:49 +0200 Subject: [PATCH 379/790] Move copy secret(from one cluster to another) functionality to community codebase. (#674) --- pkg/kube/secret/secret.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/pkg/kube/secret/secret.go b/pkg/kube/secret/secret.go index 52bf253b9..87eaa0a14 100644 --- a/pkg/kube/secret/secret.go +++ b/pkg/kube/secret/secret.go @@ -133,3 +133,19 @@ func EnsureSecretWithKey(secretGetUpdateCreateDeleter GetUpdateCreateDeleter, ns } return string(existingSecret.Data[key]), nil } + +// CopySecret copies secret object(data) from one cluster client to another, the from and to cluster-client can belong to the same or different clusters +func CopySecret(fromClient Getter, toClient GetUpdateCreator, sourceSecretNsName, destNsName types.NamespacedName) error { + s, err := fromClient.GetSecret(sourceSecretNsName) + if err != nil { + return err + } + + secretCopy := Builder(). + SetName(destNsName.Name). + SetNamespace(destNsName.Namespace). + SetByteData(s.Data). + Build() + + return CreateOrUpdate(toClient, secretCopy) +} From 472f280a2861c6fd71c575603669c799d74d0a76 Mon Sep 17 00:00:00 2001 From: Ciprian Tibulca Date: Mon, 9 Aug 2021 12:38:26 +0100 Subject: [PATCH 380/790] Add support for secret data type (#678) --- pkg/kube/secret/secret.go | 1 + pkg/kube/secret/secret_builder.go | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/pkg/kube/secret/secret.go b/pkg/kube/secret/secret.go index 87eaa0a14..52bdebcf2 100644 --- a/pkg/kube/secret/secret.go +++ b/pkg/kube/secret/secret.go @@ -145,6 +145,7 @@ func CopySecret(fromClient Getter, toClient GetUpdateCreator, sourceSecretNsName SetName(destNsName.Name). SetNamespace(destNsName.Namespace). SetByteData(s.Data). + SetDataType(s.Type). Build() return CreateOrUpdate(toClient, secretCopy) diff --git a/pkg/kube/secret/secret_builder.go b/pkg/kube/secret/secret_builder.go index 6cb7ad652..19bb2c082 100644 --- a/pkg/kube/secret/secret_builder.go +++ b/pkg/kube/secret/secret_builder.go @@ -7,6 +7,7 @@ import ( type builder struct { data map[string][]byte + dataType corev1.SecretType labels map[string]string name string namespace string @@ -59,6 +60,11 @@ func (b *builder) SetStringData(stringData map[string]string) *builder { return b } +func (b *builder) SetDataType(dataType corev1.SecretType) *builder { + b.dataType = dataType + return b +} + func (b builder) Build() corev1.Secret { return corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ @@ -68,6 +74,7 @@ func (b builder) Build() corev1.Secret { Labels: b.labels, }, Data: b.data, + Type: b.dataType, } } From 09ae2b9d9068a367dde7d5333946a72d43971822 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Aug 2021 16:18:50 +0100 Subject: [PATCH 381/790] Bump sigs.k8s.io/controller-runtime from 0.9.3 to 0.9.5 (#665) --- go.mod | 2 +- go.sum | 37 ++++++++++++++++--------------------- 2 files changed, 17 insertions(+), 22 deletions(-) diff --git a/go.mod b/go.mod index d9403cd41..c0c422f05 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( k8s.io/api v0.21.3 k8s.io/apimachinery v0.21.3 k8s.io/client-go v0.21.3 - sigs.k8s.io/controller-runtime v0.9.3 + sigs.k8s.io/controller-runtime v0.9.5 sigs.k8s.io/yaml v1.2.0 ) diff --git a/go.sum b/go.sum index 27b5e1f70..11fdf4e09 100644 --- a/go.sum +++ b/go.sum @@ -338,15 +338,14 @@ github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E= github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.13.0 h1:7lLHu94wT9Ij0o6EWWclhu0aOh32VxhkwEJvzuWPeak= -github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY= +github.com/onsi/gomega v1.14.0 h1:ep6kpPVwmr/nTbklSx2nrLNSIO62DoYAhnPNIMhK8gI= +github.com/onsi/gomega v1.14.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= @@ -468,7 +467,6 @@ go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/ go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.18.1 h1:CSUJ2mjFszzEWt4CdKISEuChVIXGBn3lAPwkRGyVrc4= go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -610,8 +608,9 @@ golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40 h1:JWgyZ1qgdTaF3N3oxC+MdTV7qvEEgHo3otj+HB5CM7Q= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE= @@ -630,8 +629,8 @@ golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6 h1:Vv0JUPWTyeqUq42B2WJ1FeIDjjvGKoA2Ss+Ts0lAVbs= -golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs= +golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -781,21 +780,18 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.21.2/go.mod h1:Lv6UGJZ1rlMI1qusN8ruAp9PUBFyBwpEHAdG24vIsiU= k8s.io/api v0.21.3 h1:cblWILbLO8ar+Fj6xdDGr603HRsf8Wu9E9rngJeprZQ= k8s.io/api v0.21.3/go.mod h1:hUgeYHUbBp23Ue4qdX9tR8/ANi/g3ehylAqDn9NWVOg= -k8s.io/apiextensions-apiserver v0.21.2 h1:+exKMRep4pDrphEafRvpEi79wTnCFMqKf8LBtlA3yrE= -k8s.io/apiextensions-apiserver v0.21.2/go.mod h1:+Axoz5/l3AYpGLlhJDfcVQzCerVYq3K3CvDMvw6X1RA= -k8s.io/apimachinery v0.21.2/go.mod h1:CdTY8fU/BlvAbJ2z/8kBwimGki5Zp8/fbVuLY8gJumM= +k8s.io/apiextensions-apiserver v0.21.3 h1:+B6biyUWpqt41kz5x6peIsljlsuwvNAp/oFax/j2/aY= +k8s.io/apiextensions-apiserver v0.21.3/go.mod h1:kl6dap3Gd45+21Jnh6utCx8Z2xxLm8LGDkprcd+KbsE= k8s.io/apimachinery v0.21.3 h1:3Ju4nvjCngxxMYby0BimUk+pQHPOQp3eCGChk5kfVII= k8s.io/apimachinery v0.21.3/go.mod h1:H/IM+5vH9kZRNJ4l3x/fXP/5bOPJaVP/guptnZPeCFI= -k8s.io/apiserver v0.21.2/go.mod h1:lN4yBoGyiNT7SC1dmNk0ue6a5Wi6O3SWOIw91TsucQw= -k8s.io/client-go v0.21.2/go.mod h1:HdJ9iknWpbl3vMGtib6T2PyI/VYxiZfq936WNVHBRrA= +k8s.io/apiserver v0.21.3/go.mod h1:eDPWlZG6/cCCMj/JBcEpDoK+I+6i3r9GsChYBHSbAzU= k8s.io/client-go v0.21.3 h1:J9nxZTOmvkInRDCzcSNQmPJbDYN/PjlxXT9Mos3HcLg= k8s.io/client-go v0.21.3/go.mod h1:+VPhCgTsaFmGILxR/7E1N0S+ryO010QBeNCv5JwRGYU= -k8s.io/code-generator v0.21.2/go.mod h1:8mXJDCB7HcRo1xiEQstcguZkbxZaqeUOrO9SsicWs3U= -k8s.io/component-base v0.21.2 h1:EsnmFFoJ86cEywC0DoIkAUiEV6fjgauNugiw1lmIjs4= -k8s.io/component-base v0.21.2/go.mod h1:9lvmIThzdlrJj5Hp8Z/TOgIkdfsNARQ1pT+3PByuiuc= +k8s.io/code-generator v0.21.3/go.mod h1:K3y0Bv9Cz2cOW2vXUrNZlFbflhuPvuadW6JdnN6gGKo= +k8s.io/component-base v0.21.3 h1:4WuuXY3Npa+iFfi2aDRiOz+anhNvRfye0859ZgfC5Og= +k8s.io/component-base v0.21.3/go.mod h1:kkuhtfEHeZM6LkX0saqSK8PbdO7A0HigUngmhhrwfGQ= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= @@ -805,16 +801,15 @@ k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 h1:vEx13qjvaZ4yfObSSXW7BrMc/KQBBT/Jyee8XtLf4x0= k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20210527160623-6fdb442a123b h1:MSqsVQ3pZvPGTqCjptfimO2WjG7A9un2zcpiHkA6M/s= -k8s.io/utils v0.0.0-20210527160623-6fdb442a123b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20210722164352-7f3ee0f31471 h1:DnzUXII7sVg1FJ/4JX6YDRJfLNAC7idRatPwe07suiI= +k8s.io/utils v0.0.0-20210722164352-7f3ee0f31471/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.19/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= -sigs.k8s.io/controller-runtime v0.9.3 h1:n075bHQ1wb8hpX7C27pNrqsb0fj8mcfCQfNX+oKTbYE= -sigs.k8s.io/controller-runtime v0.9.3/go.mod h1:TxzMCHyEUpaeuOiZx/bIdc2T81vfs/aKdvJt9wuu0zk= +sigs.k8s.io/controller-runtime v0.9.5 h1:WThcFE6cqctTn2jCZprLICO6BaKZfhsT37uAapTNfxc= +sigs.k8s.io/controller-runtime v0.9.5/go.mod h1:q6PpkM5vqQubEKUKOM6qr06oXGzOBcCby1DA9FbyZeA= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.1.0/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.1.2 h1:Hr/htKFmJEbtMgS/UD0N+gtgctAqz81t3nu+sPzynno= sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= From 0d8b9e60216a9ef8a7b131a19a967915c5889bf8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Aug 2021 18:56:37 +0100 Subject: [PATCH 382/790] Bump go.mongodb.org/mongo-driver from 1.7.0 to 1.7.1 (#679) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index c0c422f05..c31b11caa 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/stretchr/objx v0.3.0 github.com/stretchr/testify v1.7.0 github.com/xdg/stringprep v1.0.3 - go.mongodb.org/mongo-driver v1.7.0 + go.mongodb.org/mongo-driver v1.7.1 go.uber.org/zap v1.18.1 k8s.io/api v0.21.3 k8s.io/apimachinery v0.21.3 diff --git a/go.sum b/go.sum index 11fdf4e09..09790b3e1 100644 --- a/go.sum +++ b/go.sum @@ -451,8 +451,8 @@ go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= -go.mongodb.org/mongo-driver v1.7.0 h1:hHrvOBWlWB2c7+8Gh/Xi5jj82AgidK/t7KVXBZ+IyUA= -go.mongodb.org/mongo-driver v1.7.0/go.mod h1:Q4oFMbo1+MSNqICAdYMlC/zSTrwCogR4R8NzkI+yfU8= +go.mongodb.org/mongo-driver v1.7.1 h1:jwqTeEM3x6L9xDXrCxN0Hbg7vdGfPBOTIkr0+/LYZDA= +go.mongodb.org/mongo-driver v1.7.1/go.mod h1:Q4oFMbo1+MSNqICAdYMlC/zSTrwCogR4R8NzkI+yfU8= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= From 814f47266d9fd839edbc41015b9318017abd0b37 Mon Sep 17 00:00:00 2001 From: FlorentinaSimlinger <60526092+FlorentinaSimlinger@users.noreply.github.com> Date: Tue, 10 Aug 2021 11:40:44 +0100 Subject: [PATCH 383/790] delete version from status.required, add omitempty to Version (#684) --- api/v1/mongodbcommunity_types.go | 2 +- ...ommunity.mongodb.com_mongodbcommunity.yaml | 623 +++++++++--------- 2 files changed, 325 insertions(+), 300 deletions(-) diff --git a/api/v1/mongodbcommunity_types.go b/api/v1/mongodbcommunity_types.go index af6263615..5a9b9c50a 100644 --- a/api/v1/mongodbcommunity_types.go +++ b/api/v1/mongodbcommunity_types.go @@ -54,7 +54,7 @@ type MongoDBCommunitySpec struct { // +kubebuilder:validation:Enum=ReplicaSet Type Type `json:"type"` // Version defines which version of MongoDB will be used - Version string `json:"version"` + Version string `json:"version,omitempty"` // Arbiters is the number of arbiters (each counted as a member) in the replica set // +optional diff --git a/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml b/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml index 38834247c..c82e93d2d 100644 --- a/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml +++ b/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml @@ -1,4 +1,3 @@ - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition @@ -14,328 +13,354 @@ spec: listKind: MongoDBCommunityList plural: mongodbcommunity shortNames: - - mdbc + - mdbc singular: mongodbcommunity scope: Namespaced versions: - - additionalPrinterColumns: - - description: Current state of the MongoDB deployment - jsonPath: .status.phase - name: Phase - type: string - - description: Version of MongoDB server - jsonPath: .status.version - name: Version - type: string - name: v1 - schema: - openAPIV3Schema: - description: MongoDBCommunity is the Schema for the mongodbs API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: MongoDBCommunitySpec defines the desired state of MongoDB - properties: - additionalMongodConfig: - description: 'AdditionalMongodConfig is additional configuration that - can be passed to each data-bearing mongod at runtime. Uses the same - structure as the mongod configuration file: https://docs.mongodb.com/manual/reference/configuration-options/' - nullable: true - type: object - x-kubernetes-preserve-unknown-fields: true - arbiters: - description: Arbiters is the number of arbiters (each counted as a - member) in the replica set - type: integer - featureCompatibilityVersion: - description: FeatureCompatibilityVersion configures the feature compatibility - version that will be set for the deployment - type: string - members: - description: Members is the number of members in the replica set - type: integer - replicaSetHorizons: - description: ReplicaSetHorizons Add this parameter and values if you - need your database to be accessed outside of Kubernetes. This setting - allows you to provide different DNS settings within the Kubernetes - cluster and to the Kubernetes cluster. The Kubernetes Operator uses - split horizon DNS for replica set members. This feature allows communication - both within the Kubernetes cluster and from outside Kubernetes. - items: - additionalProperties: - type: string + - additionalPrinterColumns: + - description: Current state of the MongoDB deployment + jsonPath: .status.phase + name: Phase + type: string + - description: Version of MongoDB server + jsonPath: .status.version + name: Version + type: string + name: v1 + schema: + openAPIV3Schema: + description: MongoDBCommunity is the Schema for the mongodbs API + properties: + apiVersion: + description: + "APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources" + type: string + kind: + description: + "Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds" + type: string + metadata: + type: object + spec: + description: MongoDBCommunitySpec defines the desired state of MongoDB + properties: + additionalMongodConfig: + description: + "AdditionalMongodConfig is additional configuration that + can be passed to each data-bearing mongod at runtime. Uses the same + structure as the mongod configuration file: https://docs.mongodb.com/manual/reference/configuration-options/" + nullable: true type: object - type: array - security: - description: Security configures security features, such as TLS, and - authentication settings for a deployment - properties: - authentication: - properties: - ignoreUnknownUsers: - default: true - nullable: true - type: boolean - modes: - description: Modes is an array specifying which authentication - methods should be enabled. - items: - enum: - - SCRAM - - SCRAM-SHA-256 - - SCRAM-SHA-1 - type: string - type: array - required: - - modes + x-kubernetes-preserve-unknown-fields: true + arbiters: + description: + Arbiters is the number of arbiters (each counted as a + member) in the replica set + type: integer + featureCompatibilityVersion: + description: + FeatureCompatibilityVersion configures the feature compatibility + version that will be set for the deployment + type: string + members: + description: Members is the number of members in the replica set + type: integer + replicaSetHorizons: + description: + ReplicaSetHorizons Add this parameter and values if you + need your database to be accessed outside of Kubernetes. This setting + allows you to provide different DNS settings within the Kubernetes + cluster and to the Kubernetes cluster. The Kubernetes Operator uses + split horizon DNS for replica set members. This feature allows communication + both within the Kubernetes cluster and from outside Kubernetes. + items: + additionalProperties: + type: string type: object - roles: - description: User-specified custom MongoDB roles that should be - configured in the deployment. - items: - description: CustomRole defines a custom MongoDB role. + type: array + security: + description: + Security configures security features, such as TLS, and + authentication settings for a deployment + properties: + authentication: properties: - authenticationRestrictions: - description: The authentication restrictions the server - enforces on the role. - items: - description: AuthenticationRestriction specifies a list - of IP addresses and CIDR ranges users are allowed to - connect to or from. - properties: - clientSource: - items: - type: string - type: array - serverAddress: - items: - type: string - type: array - required: - - clientSource - - serverAddress - type: object - type: array - db: - description: The database of the role. - type: string - privileges: - description: The privileges to grant the role. - items: - description: Privilege defines the actions a role is allowed - to perform on a given resource. - properties: - actions: - items: - type: string - type: array - resource: - description: Resource specifies specifies the resources - upon which a privilege permits actions. See https://docs.mongodb.com/manual/reference/resource-document - for more. - properties: - anyResource: - type: boolean - cluster: - type: boolean - collection: - type: string - db: - type: string - type: object - required: - - actions - - resource - type: object - type: array - role: - description: The name of the role. - type: string - roles: - description: An array of roles from which this role inherits - privileges. + ignoreUnknownUsers: + default: true + nullable: true + type: boolean + modes: + description: + Modes is an array specifying which authentication + methods should be enabled. items: - description: Role is the database role this user should - have - properties: - db: - description: DB is the database the role can act on - type: string - name: - description: Name is the name of the role - type: string - required: - - db - - name - type: object + enum: + - SCRAM + - SCRAM-SHA-256 + - SCRAM-SHA-1 + type: string type: array required: - - db - - privileges - - role + - modes type: object - type: array - tls: - description: TLS configuration for both client-server and server-server - communication - properties: - caConfigMapRef: - description: CaConfigMap is a reference to a ConfigMap containing - the certificate for the CA which signed the server certificates - The certificate is expected to be available under the key - "ca.crt" + roles: + description: + User-specified custom MongoDB roles that should be + configured in the deployment. + items: + description: CustomRole defines a custom MongoDB role. properties: - name: + authenticationRestrictions: + description: + The authentication restrictions the server + enforces on the role. + items: + description: + AuthenticationRestriction specifies a list + of IP addresses and CIDR ranges users are allowed to + connect to or from. + properties: + clientSource: + items: + type: string + type: array + serverAddress: + items: + type: string + type: array + required: + - clientSource + - serverAddress + type: object + type: array + db: + description: The database of the role. type: string - required: - - name - type: object - certificateKeySecretRef: - description: CertificateKeySecret is a reference to a Secret - containing a private key and certificate to use for TLS. - The key and cert are expected to be PEM encoded and available - at "tls.key" and "tls.crt". This is the same format used - for the standard "kubernetes.io/tls" Secret type, but no - specific type is required. Alternatively, an entry tls.pem, - containing the concatenation of cert and key, can be provided. - If all of tls.pem, tls.crt and tls.key are present, the - tls.pem one needs to be equal to the concatenation of tls.crt - and tls.key - properties: - name: + privileges: + description: The privileges to grant the role. + items: + description: + Privilege defines the actions a role is allowed + to perform on a given resource. + properties: + actions: + items: + type: string + type: array + resource: + description: + Resource specifies specifies the resources + upon which a privilege permits actions. See https://docs.mongodb.com/manual/reference/resource-document + for more. + properties: + anyResource: + type: boolean + cluster: + type: boolean + collection: + type: string + db: + type: string + type: object + required: + - actions + - resource + type: object + type: array + role: + description: The name of the role. type: string + roles: + description: + An array of roles from which this role inherits + privileges. + items: + description: + Role is the database role this user should + have + properties: + db: + description: DB is the database the role can act on + type: string + name: + description: Name is the name of the role + type: string + required: + - db + - name + type: object + type: array required: - - name + - db + - privileges + - role type: object - enabled: - type: boolean - optional: - description: Optional configures if TLS should be required - or optional for connections - type: boolean - required: - - enabled - type: object - type: object - statefulSet: - description: StatefulSetConfiguration holds the optional custom StatefulSet - that should be merged into the operator created one. - properties: - spec: - type: object - x-kubernetes-preserve-unknown-fields: true - required: - - spec - type: object - type: - description: Type defines which type of MongoDB deployment the resource - should create - enum: - - ReplicaSet - type: string - users: - description: Users specifies the MongoDB users that should be configured - in your deployment - items: - properties: - db: - description: DB is the database the user is stored in. Defaults - to "admin" - type: string - name: - description: Name is the username of the user - type: string - passwordSecretRef: - description: PasswordSecretRef is a reference to the secret - containing this user's password + type: array + tls: + description: + TLS configuration for both client-server and server-server + communication properties: - key: - description: Key is the key in the secret storing this password. - Defaults to "password" - type: string - name: - description: Name is the name of the secret storing this - user's password - type: string + caConfigMapRef: + description: + CaConfigMap is a reference to a ConfigMap containing + the certificate for the CA which signed the server certificates + The certificate is expected to be available under the key + "ca.crt" + properties: + name: + type: string + required: + - name + type: object + certificateKeySecretRef: + description: + CertificateKeySecret is a reference to a Secret + containing a private key and certificate to use for TLS. + The key and cert are expected to be PEM encoded and available + at "tls.key" and "tls.crt". This is the same format used + for the standard "kubernetes.io/tls" Secret type, but no + specific type is required. Alternatively, an entry tls.pem, + containing the concatenation of cert and key, can be provided. + If all of tls.pem, tls.crt and tls.key are present, the + tls.pem one needs to be equal to the concatenation of tls.crt + and tls.key + properties: + name: + type: string + required: + - name + type: object + enabled: + type: boolean + optional: + description: + Optional configures if TLS should be required + or optional for connections + type: boolean required: - - name + - enabled type: object - roles: - description: Roles is an array of roles assigned to this user - items: - description: Role is the database role this user should have + type: object + statefulSet: + description: + StatefulSetConfiguration holds the optional custom StatefulSet + that should be merged into the operator created one. + properties: + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - spec + type: object + type: + description: + Type defines which type of MongoDB deployment the resource + should create + enum: + - ReplicaSet + type: string + users: + description: + Users specifies the MongoDB users that should be configured + in your deployment + items: + properties: + db: + description: + DB is the database the user is stored in. Defaults + to "admin" + type: string + name: + description: Name is the username of the user + type: string + passwordSecretRef: + description: + PasswordSecretRef is a reference to the secret + containing this user's password properties: - db: - description: DB is the database the role can act on + key: + description: + Key is the key in the secret storing this password. + Defaults to "password" type: string name: - description: Name is the name of the role + description: + Name is the name of the secret storing this + user's password type: string required: - - db - - name + - name type: object - type: array - scramCredentialsSecretName: - description: ScramCredentialsSecretName appended by string "scram-credentials" - is the name of the secret object created by the mongoDB operator - for storing SCRAM credentials - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - required: - - name - - passwordSecretRef - - roles - - scramCredentialsSecretName - type: object - type: array - version: - description: Version defines which version of MongoDB will be used - type: string - required: - - security - - type - - users - - version - type: object - status: - description: MongoDBCommunityStatus defines the observed state of MongoDB - properties: - currentMongoDBMembers: - type: integer - currentStatefulSetReplicas: - type: integer - message: - type: string - mongoUri: - type: string - phase: - type: string - version: - type: string - required: - - currentMongoDBMembers - - currentStatefulSetReplicas - - mongoUri - - phase - - version - type: object - type: object - served: true - storage: true - subresources: - status: {} + roles: + description: Roles is an array of roles assigned to this user + items: + description: Role is the database role this user should have + properties: + db: + description: DB is the database the role can act on + type: string + name: + description: Name is the name of the role + type: string + required: + - db + - name + type: object + type: array + scramCredentialsSecretName: + description: + ScramCredentialsSecretName appended by string "scram-credentials" + is the name of the secret object created by the mongoDB operator + for storing SCRAM credentials + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - name + - passwordSecretRef + - roles + - scramCredentialsSecretName + type: object + type: array + version: + description: Version defines which version of MongoDB will be used + type: string + required: + - security + - type + - users + - version + type: object + status: + description: MongoDBCommunityStatus defines the observed state of MongoDB + properties: + currentMongoDBMembers: + type: integer + currentStatefulSetReplicas: + type: integer + message: + type: string + mongoUri: + type: string + phase: + type: string + version: + type: string + required: + - currentMongoDBMembers + - currentStatefulSetReplicas + - mongoUri + - phase + type: object + type: object + served: true + storage: true + subresources: + status: {} status: acceptedNames: kind: "" From b55bff7eb45af7758895f62a0550f4a2f27154d9 Mon Sep 17 00:00:00 2001 From: killian2k Date: Mon, 16 Aug 2021 17:17:36 +0200 Subject: [PATCH 384/790] Correcting a small fix (#675) --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index c137833a3..53de83bca 100644 --- a/Makefile +++ b/Makefile @@ -48,7 +48,7 @@ install-crd: kubectl apply -f config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml install-chart: - helm upgrade --install $(STRING_SET_VALUES) $(RELEASE_NAME_HELM) helm-chart + $(HELM) upgrade --install $(STRING_SET_VALUES) $(RELEASE_NAME_HELM) helm-chart install-rbac: $(HELM) template $(STRING_SET_VALUES) -s templates/database_roles.yaml helm-chart | kubectl apply -f - From a57f9a970efabd94adb5c73624faa0b9fabbf81f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Aug 2021 16:19:19 +0100 Subject: [PATCH 385/790] Bump go.uber.org/zap from 1.18.1 to 1.19.0 (#681) --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index c31b11caa..4e6b14fb9 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/stretchr/testify v1.7.0 github.com/xdg/stringprep v1.0.3 go.mongodb.org/mongo-driver v1.7.1 - go.uber.org/zap v1.18.1 + go.uber.org/zap v1.19.0 k8s.io/api v0.21.3 k8s.io/apimachinery v0.21.3 k8s.io/client-go v0.21.3 diff --git a/go.sum b/go.sum index 09790b3e1..57e6f0c8a 100644 --- a/go.sum +++ b/go.sum @@ -467,8 +467,9 @@ go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/ go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.18.1 h1:CSUJ2mjFszzEWt4CdKISEuChVIXGBn3lAPwkRGyVrc4= go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= +go.uber.org/zap v1.19.0 h1:mZQZefskPPCMIBCSEH0v2/iUqqLrYtaeqwD6FUGUnFE= +go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= From df176e34a874dd5ace6083145b1b7b6644e36674 Mon Sep 17 00:00:00 2001 From: FlorentinaSimlinger <60526092+FlorentinaSimlinger@users.noreply.github.com> Date: Mon, 16 Aug 2021 17:01:25 +0100 Subject: [PATCH 386/790] CLOUDP-96891: operator does not recreate sts on being deleted (#671) --- controllers/replica_set_controller.go | 1 + test/e2e/mongodbtests/mongodbtests.go | 17 +++++++ .../statefulset_delete_test.go | 48 +++++++++++++++++++ 3 files changed, 66 insertions(+) create mode 100644 test/e2e/statefulset_delete/statefulset_delete_test.go diff --git a/controllers/replica_set_controller.go b/controllers/replica_set_controller.go index c15a1b2cd..473465490 100644 --- a/controllers/replica_set_controller.go +++ b/controllers/replica_set_controller.go @@ -86,6 +86,7 @@ func (r *ReplicaSetReconciler) SetupWithManager(mgr ctrl.Manager) error { WithOptions(controller.Options{MaxConcurrentReconciles: 3}). For(&mdbv1.MongoDBCommunity{}, builder.WithPredicates(predicates.OnlyOnSpecChange())). Watches(&source.Kind{Type: &corev1.Secret{}}, r.secretWatcher). + Owns(&appsv1.StatefulSet{}). Complete(r) } diff --git a/test/e2e/mongodbtests/mongodbtests.go b/test/e2e/mongodbtests/mongodbtests.go index 3fc43c801..681c37919 100644 --- a/test/e2e/mongodbtests/mongodbtests.go +++ b/test/e2e/mongodbtests/mongodbtests.go @@ -342,6 +342,23 @@ func DeletePod(mdb *mdbv1.MongoDBCommunity, podNum int) func(*testing.T) { } } +// DeleteStatefulSet provides a wrapper to delete appsv1.StatefulSet types +func DeleteStatefulSet(mdb *mdbv1.MongoDBCommunity) func(*testing.T) { + return func(t *testing.T) { + sts := appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: mdb.Name, + Namespace: mdb.Namespace, + }, + } + if err := e2eutil.TestClient.Delete(context.TODO(), &sts); err != nil { + t.Fatal(err) + } + + t.Logf("StatefulSet %s/%s deleted", sts.ObjectMeta.Namespace, sts.ObjectMeta.Name) + } +} + // Status compares the given status to the actual status of the MongoDB resource func Status(mdb *mdbv1.MongoDBCommunity, expectedStatus mdbv1.MongoDBCommunityStatus) func(t *testing.T) { return func(t *testing.T) { diff --git a/test/e2e/statefulset_delete/statefulset_delete_test.go b/test/e2e/statefulset_delete/statefulset_delete_test.go new file mode 100644 index 000000000..052015212 --- /dev/null +++ b/test/e2e/statefulset_delete/statefulset_delete_test.go @@ -0,0 +1,48 @@ +package statefulset_delete + +import ( + "fmt" + "os" + "testing" + + mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" + e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" + "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/mongodbtests" + setup "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" +) + +func TestMain(m *testing.M) { + code, err := e2eutil.RunTest(m) + if err != nil { + fmt.Println(err) + } + os.Exit(code) +} + +func TestStatefulSetDelete(t *testing.T) { + ctx := setup.Setup(t) + defer ctx.Teardown() + + mdb, user := e2eutil.NewTestMongoDB(ctx, "mdb0", "") + + _, err := setup.GeneratePasswordForUser(ctx, user, "") + if err != nil { + t.Fatal(err) + } + + t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) + t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb)) + + t.Run("Operator recreates StatefulSet", func(t *testing.T) { + t.Run("Delete Statefulset", mongodbtests.DeleteStatefulSet(&mdb)) + t.Run("Test Replica Set Recovers", mongodbtests.StatefulSetBecomesReady(&mdb)) + t.Run("MongoDB Reaches Running Phase", mongodbtests.MongoDBReachesRunningPhase(&mdb)) + t.Run("Test Status Was Updated", mongodbtests.Status(&mdb, + mdbv1.MongoDBCommunityStatus{ + MongoURI: mdb.MongoURI(), + Phase: mdbv1.Running, + Version: mdb.GetMongoDBVersion(), + CurrentMongoDBMembers: mdb.DesiredReplicas(), + })) + }) +} From 9e02574020ca8903e8bbd26a3beedfca9a395242 Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Tue, 17 Aug 2021 15:07:31 +0100 Subject: [PATCH 387/790] Add all helm values to makefile (#686) --- Makefile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 53de83bca..e70b866f4 100644 --- a/Makefile +++ b/Makefile @@ -6,8 +6,10 @@ OPERATOR_IMAGE := $(shell jq -r .operator_image < ~/.community-operator-dev/conf NAMESPACE := $(shell jq -r .namespace < ~/.community-operator-dev/config.json) UPGRADE_HOOK_IMG := $(shell jq -r .version_upgrade_hook_image < ~/.community-operator-dev/config.json) READINESS_PROBE_IMG := $(shell jq -r .readiness_probe_image < ~/.community-operator-dev/config.json) +REGISTRY := $(shell jq -r .repo_url < ~/.community-operator-dev/config.json) +AGENT_IMAGE_NAME := $(shell jq -r .agent_image_ubuntu < ~/.community-operator-dev/config.json) -STRING_SET_VALUES := --set namespace=$(NAMESPACE),version_upgrade_hook.name=$(UPGRADE_HOOK_IMG),readiness_probe.name=$(READINESS_PROBE_IMG),registry.operator=$(REPO_URL),operator.operator_image_name=$(OPERATOR_IMAGE),operator.version=latest +STRING_SET_VALUES := --set namespace=$(NAMESPACE),version_upgrade_hook.name=$(UPGRADE_HOOK_IMG),readiness_probe.name=$(READINESS_PROBE_IMG),registry.operator=$(REPO_URL),operator.operator_image_name=$(OPERATOR_IMAGE),operator.version=latest,registry.agent=$(REGISTRY),registry.version_upgrade_hook=$(REGISTRY),registry.readiness_probe=$(REGISTRY),registry.operator=$(REGISTRY),version_upgrade_hook.version=latest,readiness_probe.version=latest,agent.version=latest,agent.name=$(AGENT_IMAGE_NAME) DOCKERFILE ?= operator # Produce CRDs that work back to Kubernetes 1.11 (no version conversion) @@ -42,12 +44,12 @@ run: install install-rbac # Install CRDs into a cluster install: manifests helm install-crd - install-crd: kubectl apply -f config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml install-chart: + helm upgrade --install $(STRING_SET_VALUES) $(RELEASE_NAME_HELM) helm-chart $(HELM) upgrade --install $(STRING_SET_VALUES) $(RELEASE_NAME_HELM) helm-chart install-rbac: @@ -57,7 +59,7 @@ install-rbac: uninstall-crd: kubectl delete crd mongodbcommunity.mongodbcommunity.mongodb.com -uninstall-chart: +uninstall-chart: $(HELM) uninstall $(RELEASE_NAME_HELM) -n $(NAMESPACE) uninstall-rbac: From 3f7b54a887c7a5fd2fa30a786323d9bba3fee786 Mon Sep 17 00:00:00 2001 From: Rajdeep Das Date: Fri, 20 Aug 2021 16:17:54 +0200 Subject: [PATCH 388/790] Make DefaultSecurityContext return a pointer to security context to align with corev1.SecurityContext (#693) --- pkg/kube/container/container_test.go | 2 +- pkg/kube/container/containers.go | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/kube/container/container_test.go b/pkg/kube/container/container_test.go index 5c08b14b9..b6a2d30c4 100644 --- a/pkg/kube/container/container_test.go +++ b/pkg/kube/container/container_test.go @@ -20,7 +20,7 @@ func TestContainer(t *testing.T) { WithImage("image"), WithImagePullPolicy(corev1.PullAlways), WithPorts([]corev1.ContainerPort{{Name: "port-1", ContainerPort: int32(1000)}}), - WithSecurityContext(corev1.SecurityContext{ + WithSecurityContext(&corev1.SecurityContext{ RunAsGroup: int64Ref(100), RunAsNonRoot: boolRef(true), }), diff --git a/pkg/kube/container/containers.go b/pkg/kube/container/containers.go index e49c5505d..c3c3aa200 100644 --- a/pkg/kube/container/containers.go +++ b/pkg/kube/container/containers.go @@ -178,17 +178,17 @@ func WithPorts(ports []corev1.ContainerPort) Modification { } // WithSecurityContext sets teh container's SecurityContext -func WithSecurityContext(context corev1.SecurityContext) Modification { +func WithSecurityContext(context *corev1.SecurityContext) Modification { return func(container *corev1.Container) { - container.SecurityContext = &context + container.SecurityContext = context } } // DefaultSecurityContext returns the default security context for containers. // It sets RunAsUser = 2000 and RunAsNonRoot = true -func DefaultSecurityContext() corev1.SecurityContext { +func DefaultSecurityContext() *corev1.SecurityContext { runAsNonRoot := true runAsUser := int64(2000) - return corev1.SecurityContext{RunAsUser: &runAsUser, RunAsNonRoot: &runAsNonRoot} + return &corev1.SecurityContext{RunAsUser: &runAsUser, RunAsNonRoot: &runAsNonRoot} } From b020bfdc97bce4f2a917c2c3f33349efbcf2b3eb Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Thu, 26 Aug 2021 11:17:31 +0100 Subject: [PATCH 389/790] Prevent filling of volumes (#694) --- cmd/readiness/main.go | 27 +++++++++++++------------ go.mod | 1 + go.sum | 2 ++ pkg/readiness/config/config.go | 36 ++++++++++++++++++++++++++++++++++ release.json | 2 +- 5 files changed, 55 insertions(+), 13 deletions(-) diff --git a/cmd/readiness/main.go b/cmd/readiness/main.go index 9e629822d..281660304 100644 --- a/cmd/readiness/main.go +++ b/cmd/readiness/main.go @@ -3,6 +3,8 @@ package main import ( "encoding/json" "fmt" + "go.uber.org/zap/zapcore" + "gopkg.in/natefinch/lumberjack.v2" "io" "io/ioutil" "os" @@ -196,28 +198,29 @@ func parseHealthStatus(reader io.Reader) (health.Status, error) { return health, err } +func initLogger(l *lumberjack.Logger) { + log := zap.New(zapcore.NewCore( + zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig()), + zapcore.AddSync(l), + zap.DebugLevel, + ), zap.Development()) + logger = log.Sugar() +} + func main() { clientSet, err := kubernetesClientset() if err != nil { panic(err) } - config, err := config.BuildFromEnvVariables(clientSet, isHeadlessMode()) + cfg, err := config.BuildFromEnvVariables(clientSet, isHeadlessMode()) if err != nil { panic(err) } - cfg := zap.NewDevelopmentConfig() - // In production we log to the file - cfg.OutputPaths = []string{ - config.LogFilePath, - } - log, err := cfg.Build() - if err != nil { - panic(err) - } - logger = log.Sugar() - ready, err := isPodReady(config) + initLogger(cfg.Logger) + + ready, err := isPodReady(cfg) if err != nil { panic(err) } diff --git a/go.mod b/go.mod index 4e6b14fb9..cbbf3f671 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/xdg/stringprep v1.0.3 go.mongodb.org/mongo-driver v1.7.1 go.uber.org/zap v1.19.0 + gopkg.in/natefinch/lumberjack.v2 v2.0.0 k8s.io/api v0.21.3 k8s.io/apimachinery v0.21.3 k8s.io/client-go v0.21.3 diff --git a/go.sum b/go.sum index 57e6f0c8a..6f05e0407 100644 --- a/go.sum +++ b/go.sum @@ -30,6 +30,7 @@ github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSY github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= @@ -755,6 +756,7 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= diff --git a/pkg/readiness/config/config.go b/pkg/readiness/config/config.go index 91216dbfb..46cd45103 100644 --- a/pkg/readiness/config/config.go +++ b/pkg/readiness/config/config.go @@ -2,8 +2,10 @@ package config import ( "fmt" + "gopkg.in/natefinch/lumberjack.v2" "io" "os" + "strconv" "strings" "k8s.io/client-go/kubernetes" @@ -17,6 +19,9 @@ const ( agentHealthStatusFilePathEnv = "AGENT_STATUS_FILEPATH" logPathEnv = "LOG_FILE_PATH" hostNameEnv = "HOSTNAME" + readinessProbeLoggerBackups = "READINESS_PROBE_LOGGER_BACKUPS" + readinessProbeLoggerMaxSize = "READINESS_PROBE_LOGGER_MAX_SIZE" + readinessProbeLoggerMaxAge = "READINESS_PROBE_LOGGER_MAX_AGE" ) type Config struct { @@ -26,6 +31,7 @@ type Config struct { AutomationConfigSecretName string HealthStatusReader io.Reader LogFilePath string + Logger *lumberjack.Logger } func BuildFromEnvVariables(clientSet kubernetes.Interface, isHeadless bool) (Config, error) { @@ -48,6 +54,14 @@ func BuildFromEnvVariables(clientSet kubernetes.Interface, isHeadless bool) (Con return Config{}, fmt.Errorf("the '%s' environment variable must be set", hostNameEnv) } } + + logger := &lumberjack.Logger{ + Filename: readinessProbeLogFilePath(), + MaxBackups: readIntOrDefault(readinessProbeLoggerBackups, 5), + MaxSize: readInt(readinessProbeLoggerMaxSize), + MaxAge: readInt(readinessProbeLoggerMaxAge), + } + // Note, that we shouldn't close the file here - it will be closed very soon by the 'ioutil.ReadAll' // in main.go file, err := os.Open(healthStatusFilePath) @@ -61,9 +75,14 @@ func BuildFromEnvVariables(clientSet kubernetes.Interface, isHeadless bool) (Con Hostname: hostname, HealthStatusReader: file, LogFilePath: logFilePath, + Logger: logger, }, nil } +func readinessProbeLogFilePath() string { + return getEnvOrDefault(logPathEnv, defaultLogPath) +} + func getEnvOrDefault(envVar, defaultValue string) string { value := strings.TrimSpace(os.Getenv(envVar)) if value == "" { @@ -71,3 +90,20 @@ func getEnvOrDefault(envVar, defaultValue string) string { } return value } + +// readInt returns the int value of an envvar of the given name. +// defaults to 0. +func readInt(envVarName string) int { + return readIntOrDefault(envVarName, 0) +} + +// readIntOrDefault returns the int value of an envvar of the given name. +// defaults to the given value if not specified. +func readIntOrDefault(envVarName string, defaultValue int) int { + envVar := getEnvOrDefault(envVarName, strconv.Itoa(defaultValue)) + intValue, err := strconv.Atoi(envVar) + if err != nil { + return defaultValue + } + return intValue +} diff --git a/release.json b/release.json index 81c06f76c..e2f9b8bad 100644 --- a/release.json +++ b/release.json @@ -1,7 +1,7 @@ { "mongodb-kubernetes-operator": "0.7.0", "version-upgrade-hook": "1.0.2", - "readiness-probe": "1.0.4", + "readiness-probe": "1.0.5", "mongodb-agent": { "version": "11.0.5.6963-1", "tools_version": "100.3.1" From 5bc140d928a7c2deef10d4be0affbefa0459123a Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Mon, 30 Aug 2021 11:36:45 +0100 Subject: [PATCH 390/790] CLOUDP-95948: Refactor E2E tests to use HELM (#689) --- .github/config_files/config_lint.yaml | 1 + .../config_files/config_lint_clusterwide.yaml | 18 ++ .../config_files/config_lint_openshift.yaml | 3 +- .github/workflows/kubelinter-check.yml | 6 +- Makefile | 7 +- go.mod | 1 + go.sum | 11 +- helm-chart/templates/operator.yaml | 8 +- helm-chart/templates/operator_roles.yaml | 3 + helm-chart/values.yaml | 12 +- pkg/helm/helm.go | 50 ++++ scripts/dev/get_e2e_env_vars.py | 3 + scripts/dev/templates/Dockerfile.e2e | 9 + scripts/dev/templates/Dockerfile.template | 3 + test/e2e/e2eutil.go | 57 ---- test/e2e/mongodbtests/mongodbtests.go | 2 +- ...replica_set_cross_namespace_deploy_test.go | 169 +++-------- test/e2e/setup/setup.go | 268 ++++++------------ test/e2e/util/wait/wait.go | 9 + 19 files changed, 248 insertions(+), 392 deletions(-) create mode 100644 .github/config_files/config_lint_clusterwide.yaml create mode 100644 pkg/helm/helm.go diff --git a/.github/config_files/config_lint.yaml b/.github/config_files/config_lint.yaml index 95db1fb9d..435bc8a7b 100644 --- a/.github/config_files/config_lint.yaml +++ b/.github/config_files/config_lint.yaml @@ -11,3 +11,4 @@ checks: - "minimum-three-replicas" - "no-liveness-probe" - "no-readiness-probe" + - "use-namespace" diff --git a/.github/config_files/config_lint_clusterwide.yaml b/.github/config_files/config_lint_clusterwide.yaml new file mode 100644 index 000000000..b69b5147d --- /dev/null +++ b/.github/config_files/config_lint_clusterwide.yaml @@ -0,0 +1,18 @@ +checks: + addAllBuiltIn: true + +#Reasons to exclude: + # non-existent-service-account because the service account is created in another file + # minimum-three-replicas because the deployment contains only 1 replica of the operator + # no-readiness-probe & no-liveness-probe because for now, it brings nothing to add these probes + # because they will not check whether the operator is actually ready/living. + # When using a clusterwide operator, it is required to be able to create StatefulSets and Secrets + # so we exclude "access-to-secrets" and "access-to-create-pods" + exclude: + - "non-existent-service-account" + - "minimum-three-replicas" + - "no-liveness-probe" + - "no-readiness-probe" + - "use-namespace" + - "access-to-secrets" + - "access-to-create-pods" diff --git a/.github/config_files/config_lint_openshift.yaml b/.github/config_files/config_lint_openshift.yaml index dd5c29bb2..34ff6e440 100644 --- a/.github/config_files/config_lint_openshift.yaml +++ b/.github/config_files/config_lint_openshift.yaml @@ -1,6 +1,6 @@ checks: addAllBuiltIn: true - + #Reasons to exclude # non-existent-service-account because the service account is created in another file # minimum-three-replicas because the deployment contains only 1 replica of the operator @@ -14,3 +14,4 @@ checks: - "no-readiness-probe" - "run-as-non-root" - "no-read-only-root-fs" + - "use-namespace" diff --git a/.github/workflows/kubelinter-check.yml b/.github/workflows/kubelinter-check.yml index 21a1e539e..a361eb59f 100644 --- a/.github/workflows/kubelinter-check.yml +++ b/.github/workflows/kubelinter-check.yml @@ -4,7 +4,7 @@ on: push: branches: - master - paths-ignore: + paths-ignore: - docs/** pull_request: branches: @@ -18,12 +18,12 @@ jobs: steps: - name: Checkout Code uses: actions/checkout@v2 - + - name: Scan directory ./deploy/clusterwide/ with kube-linter uses: stackrox/kube-linter-action@v1 with: directory: deploy/clusterwide - config: ${GITHUB_WORKSPACE}/.github/config_files/config_lint.yaml + config: ${GITHUB_WORKSPACE}/.github/config_files/config_lint_clusterwide.yaml - name: Scan directory ./deploy/openshift/ with kube-linter uses: stackrox/kube-linter-action@v1 diff --git a/Makefile b/Makefile index e70b866f4..4e332de52 100644 --- a/Makefile +++ b/Makefile @@ -9,12 +9,12 @@ READINESS_PROBE_IMG := $(shell jq -r .readiness_probe_image < ~/.community-opera REGISTRY := $(shell jq -r .repo_url < ~/.community-operator-dev/config.json) AGENT_IMAGE_NAME := $(shell jq -r .agent_image_ubuntu < ~/.community-operator-dev/config.json) -STRING_SET_VALUES := --set namespace=$(NAMESPACE),version_upgrade_hook.name=$(UPGRADE_HOOK_IMG),readiness_probe.name=$(READINESS_PROBE_IMG),registry.operator=$(REPO_URL),operator.operator_image_name=$(OPERATOR_IMAGE),operator.version=latest,registry.agent=$(REGISTRY),registry.version_upgrade_hook=$(REGISTRY),registry.readiness_probe=$(REGISTRY),registry.operator=$(REGISTRY),version_upgrade_hook.version=latest,readiness_probe.version=latest,agent.version=latest,agent.name=$(AGENT_IMAGE_NAME) +STRING_SET_VALUES := --set namespace=$(NAMESPACE),versionUpgradeHook.name=$(UPGRADE_HOOK_IMG),readinessProbe.name=$(READINESS_PROBE_IMG),registry.operator=$(REPO_URL),operator.operatorImageName=$(OPERATOR_IMAGE),operator.version=latest,registry.agent=$(REGISTRY),registry.versionUpgradeHook=$(REGISTRY),registry.readinessProbe=$(REGISTRY),registry.operator=$(REGISTRY),versionUpgradeHook.version=latest,readinessProbe.version=latest,agent.version=latest,agent.name=$(AGENT_IMAGE_NAME) DOCKERFILE ?= operator # Produce CRDs that work back to Kubernetes 1.11 (no version conversion) CRD_OPTIONS ?= "crd:trivialVersions=true,crdVersions=v1" -RELEASE_NAME_HELM ?= community-operator-chart +RELEASE_NAME_HELM ?= mongodb-kubernetes-operator # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) ifeq (,$(shell go env GOBIN)) @@ -49,8 +49,7 @@ install-crd: kubectl apply -f config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml install-chart: - helm upgrade --install $(STRING_SET_VALUES) $(RELEASE_NAME_HELM) helm-chart - $(HELM) upgrade --install $(STRING_SET_VALUES) $(RELEASE_NAME_HELM) helm-chart + $(HELM) upgrade --install $(STRING_SET_VALUES) $(RELEASE_NAME_HELM) helm-chart install-rbac: $(HELM) template $(STRING_SET_VALUES) -s templates/database_roles.yaml helm-chart | kubectl apply -f - diff --git a/go.mod b/go.mod index cbbf3f671..4e10570bc 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/xdg/stringprep v1.0.3 go.mongodb.org/mongo-driver v1.7.1 go.uber.org/zap v1.19.0 + golang.org/x/tools v0.1.1 // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 k8s.io/api v0.21.3 k8s.io/apimachinery v0.21.3 diff --git a/go.sum b/go.sum index 6f05e0407..76479898c 100644 --- a/go.sum +++ b/go.sum @@ -448,6 +448,7 @@ github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= @@ -516,6 +517,7 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -546,6 +548,7 @@ golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 h1:DzZ89McO9/gWPsQXS/FVKAlG02ZjaQ6AlZRBimEYOd0= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -562,8 +565,9 @@ golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -608,8 +612,10 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -677,8 +683,9 @@ golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.1 h1:wGiQel/hW0NnEkJUk8lbzkX2gFJU6PFxf1v5OlCfuOs= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/helm-chart/templates/operator.yaml b/helm-chart/templates/operator.yaml index b20ff4f41..a89b817b5 100644 --- a/helm-chart/templates/operator.yaml +++ b/helm-chart/templates/operator.yaml @@ -55,16 +55,16 @@ spec: - name: AGENT_IMAGE value: "{{ .Values.registry.agent }}/{{ .Values.agent.name }}:{{ .Values.agent.version }}" - name: VERSION_UPGRADE_HOOK_IMAGE - value: "{{ .Values.registry.version_upgrade_hook }}/{{ .Values.version_upgrade_hook.name }}:{{ .Values.version_upgrade_hook.version }}" + value: "{{ .Values.registry.versionUpgradeHook }}/{{ .Values.versionUpgradeHook.name }}:{{ .Values.versionUpgradeHook.version }}" - name: READINESS_PROBE_IMAGE - value: "{{ .Values.registry.readiness_probe }}/{{ .Values.readiness_probe.name }}:{{ .Values.readiness_probe.version }}" + value: "{{ .Values.registry.readinessProbe }}/{{ .Values.readinessProbe.name }}:{{ .Values.readinessProbe.version }}" - name: MONGODB_IMAGE value: {{ .Values.mongodb.name }} - name: MONGODB_REPO_URL value: {{ .Values.mongodb.repo }} - image: {{ .Values.registry.operator }}/{{ .Values.operator.operator_image_name }}:{{ .Values.operator.version }} + image: {{ .Values.registry.operator }}/{{ .Values.operator.operatorImageName }}:{{ .Values.operator.version }} imagePullPolicy: {{ .Values.registry.pullPolicy}} - name: {{ .Values.operator.deployment_name }} + name: {{ .Values.operator.deploymentName }} resources: limits: cpu: 1100m diff --git a/helm-chart/templates/operator_roles.yaml b/helm-chart/templates/operator_roles.yaml index 96c4f67da..5b872dfb1 100644 --- a/helm-chart/templates/operator_roles.yaml +++ b/helm-chart/templates/operator_roles.yaml @@ -70,6 +70,9 @@ metadata: subjects: - kind: ServiceAccount name: {{ .Values.operator.name }} +{{ if eq (.Values.operator.watchNamespace | default "") "*" }} + namespace: {{ .Values.namespace }} +{{ end }} roleRef: kind: {{ if eq (.Values.operator.watchNamespace | default "") "*" }} ClusterRole {{ else }} Role {{ end }} name: {{ .Values.operator.name }} diff --git a/helm-chart/values.yaml b/helm-chart/values.yaml index 52aff8fce..7d5693de9 100644 --- a/helm-chart/values.yaml +++ b/helm-chart/values.yaml @@ -7,10 +7,10 @@ operator: name: mongodb-kubernetes-operator # Name of the operator image - operator_image_name: mongodb-kubernetes-operator + operatorImageName: mongodb-kubernetes-operator # Name of the deployment of the operator pod - deployment_name: mongodb-kubernetes-operator + deploymentName: mongodb-kubernetes-operator # Version of mongodb-kubernetes-operator version: 0.7.0 @@ -22,10 +22,10 @@ database: agent: name: mongodb-agent version: 11.0.5.6963-1 -version_upgrade_hook: +versionUpgradeHook: name: mongodb-kubernetes-operator-version-upgrade-post-start-hook version: 1.0.2 -readiness_probe: +readinessProbe: name: mongodb-kubernetes-readinessprobe version: 1.0.4 mongodb: @@ -34,7 +34,7 @@ mongodb: registry: agent: quay.io/mongodb - version_upgrade_hook: quay.io/mongodb - readiness_probe: quay.io/mongodb + versionUpgradeHook: quay.io/mongodb + readinessProbe: quay.io/mongodb operator: quay.io/mongodb pullPolicy: Always diff --git a/pkg/helm/helm.go b/pkg/helm/helm.go new file mode 100644 index 000000000..347ce7346 --- /dev/null +++ b/pkg/helm/helm.go @@ -0,0 +1,50 @@ +package helm + +import ( + "fmt" + "os/exec" + "strings" +) + +// Uninstall uninstalls a helm chart of the given name. There is no error in the case +// of the helm chart not existing. +func Uninstall(chartName string) error { + helmArgs := []string{"uninstall", chartName} + return executeHelmCommand(helmArgs, isNotFoundMessage) +} + +// Install a helm chert at the given path with the given name and the provided set arguments. +func Install(chartPath, chartName string, args map[string]string) error { + helmArgs := []string{"install"} + helmArgs = append(helmArgs, mapToHelmArgs(args)...) + helmArgs = append(helmArgs, chartName, chartPath) + return executeHelmCommand(helmArgs, nil) +} + +func isNotFoundMessage(s string) bool { + return strings.Contains(s, "not found") +} + +// executeHelmCommand accepts a list of arguments that should be passed to the helm command +// and a predicate that when returning true, indicates that the error message should be ignored. +func executeHelmCommand(args []string, messagePredicate func(string) bool) error { + cmd := exec.Command("helm", args...) + output, err := cmd.CombinedOutput() + if err != nil { + if messagePredicate != nil && messagePredicate(string(output)){ + return nil + } + return fmt.Errorf("error executing command: %s %s", err, output) + } + return nil +} + +// mapToHelmArgs accepts a map of string to string and returns a list of arguments +// that can be passed to a shell helm command. +func mapToHelmArgs(m map[string]string) []string { + var args []string + for k, v := range m { + args = append(args, "--set", fmt.Sprintf("%s=%s", k, v)) + } + return args +} diff --git a/scripts/dev/get_e2e_env_vars.py b/scripts/dev/get_e2e_env_vars.py index 48a44e2b6..c4a018d90 100755 --- a/scripts/dev/get_e2e_env_vars.py +++ b/scripts/dev/get_e2e_env_vars.py @@ -1,6 +1,8 @@ #!/usr/bin/env python import sys from typing import Dict +import os.path + from dev_config import load_config, DevConfig @@ -29,6 +31,7 @@ def _get_e2e_test_envs(dev_config: DevConfig) -> Dict[str, str]: "WATCH_NAMESPACE": dev_config.namespace, "MONGODB_IMAGE": "mongo", "MONGODB_REPO_URL": "docker.io", + "HELM_CHART_PATH": os.path.abspath("./helm-chart"), } diff --git a/scripts/dev/templates/Dockerfile.e2e b/scripts/dev/templates/Dockerfile.e2e index 1952bfa96..03805305b 100644 --- a/scripts/dev/templates/Dockerfile.e2e +++ b/scripts/dev/templates/Dockerfile.e2e @@ -3,3 +3,12 @@ {% block build_binary -%} COPY testdata/tls/ testdata/tls/ {% endblock -%} + + +{% block install_helm -%} +RUN curl -LO "https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl" \ + && chmod +x ./kubectl \ + && mv ./kubectl /usr/local/bin/kubectl \ + && curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 \ + && chmod +x get_helm.sh && ./get_helm.sh +{% endblock -%} diff --git a/scripts/dev/templates/Dockerfile.template b/scripts/dev/templates/Dockerfile.template index 2f41a5596..ce5f06b3c 100644 --- a/scripts/dev/templates/Dockerfile.template +++ b/scripts/dev/templates/Dockerfile.template @@ -27,3 +27,6 @@ ADD . . {% block command -%} {% endblock -%} + +{% block install_helm -%} +{% endblock -%} diff --git a/test/e2e/e2eutil.go b/test/e2e/e2eutil.go index ca4921341..6ad4c1596 100644 --- a/test/e2e/e2eutil.go +++ b/test/e2e/e2eutil.go @@ -12,7 +12,6 @@ import ( mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" corev1 "k8s.io/api/core/v1" - rbacv1 "k8s.io/api/rbac/v1" apiErrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -192,59 +191,3 @@ func EnsureNamespace(ctx *Context, namespace string) error { }, }) } - -// EnsureServiceAccount checks that the given ServiceAccount exists and creates it if not. -func EnsureServiceAccount(ctx *Context, namespace string, svcAcctName string) error { - return ensureObject(ctx, &corev1.ServiceAccount{ - ObjectMeta: metav1.ObjectMeta{ - Name: svcAcctName, - Namespace: namespace, - }, - }) -} - -// EnsureRole checks that the given role exists and creates it with the given rules if not. -func EnsureRole(ctx *Context, namespace string, roleName string, rules []rbacv1.PolicyRule) error { - return ensureObject(ctx, &rbacv1.Role{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - Name: roleName, - }, - Rules: rules, - }) -} - -// EnsureClusterRole checks that the given cluster role exists and creates it with the given rules if not. -func EnsureClusterRole(ctx *Context, namespace string, roleName string, rules []rbacv1.PolicyRule) error { - return ensureObject(ctx, &rbacv1.ClusterRole{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - Name: roleName, - }, - Rules: rules, - }) -} - -// EnsureRoleBinding checks that the given role binding exists and creates it with the given subjects and roleRef if not. -func EnsureRoleBinding(ctx *Context, namespace string, roleBindingName string, subjects []rbacv1.Subject, roleRef rbacv1.RoleRef) error { - return ensureObject(ctx, &rbacv1.RoleBinding{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - Name: roleBindingName, - }, - Subjects: subjects, - RoleRef: roleRef, - }) -} - -// EnsureClusterRoleBinding checks that the given cluster role exists and creates it with the given subjects and roleRef if not. -func EnsureClusterRoleBinding(ctx *Context, namespace string, roleBindingName string, subjects []rbacv1.Subject, roleRef rbacv1.RoleRef) error { - return ensureObject(ctx, &rbacv1.ClusterRoleBinding{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - Name: roleBindingName, - }, - Subjects: subjects, - RoleRef: roleRef, - }) -} diff --git a/test/e2e/mongodbtests/mongodbtests.go b/test/e2e/mongodbtests/mongodbtests.go index 681c37919..67431dcd2 100644 --- a/test/e2e/mongodbtests/mongodbtests.go +++ b/test/e2e/mongodbtests/mongodbtests.go @@ -36,7 +36,7 @@ func SkipTestIfLocal(t *testing.T, msg string, f func(t *testing.T)) { func StatefulSetBecomesReady(mdb *mdbv1.MongoDBCommunity, opts ...wait.Configuration) func(t *testing.T) { defaultOpts := []wait.Configuration{ wait.RetryInterval(time.Second * 15), - wait.Timeout(time.Minute * 15), + wait.Timeout(time.Minute * 20), } defaultOpts = append(defaultOpts, opts...) return statefulSetIsReady(mdb, defaultOpts...) diff --git a/test/e2e/replica_set_cross_namespace_deploy/replica_set_cross_namespace_deploy_test.go b/test/e2e/replica_set_cross_namespace_deploy/replica_set_cross_namespace_deploy_test.go index 4dace6476..05293bcdc 100644 --- a/test/e2e/replica_set_cross_namespace_deploy/replica_set_cross_namespace_deploy_test.go +++ b/test/e2e/replica_set_cross_namespace_deploy/replica_set_cross_namespace_deploy_test.go @@ -1,7 +1,11 @@ package replica_set_cross_namespace_deploy import ( + "context" "fmt" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/types" "os" "testing" @@ -11,8 +15,6 @@ import ( e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/mongodbtests" setup "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" - - rbacv1 "k8s.io/api/rbac/v1" ) func TestMain(m *testing.M) { @@ -38,162 +40,75 @@ func TestCrossNamespaceDeploy(t *testing.T) { t.Fatal(err) } - err = e2eutil.EnsureServiceAccount(ctx, namespace, "mongodb-kubernetes-operator") - if err != nil { + if err := createDatabaseServiceAccountRoleAndRoleBinding(t, namespace); err != nil { t.Fatal(err) } - // Create a role in the test namespace - err = e2eutil.EnsureRole(ctx, namespace, "mongodb-kubernetes-operator", []rbacv1.PolicyRule{ - { - APIGroups: []string{""}, - Resources: []string{"pods", "services", "services/finalizers", "endpoints", "persistentvolumeclaims", "events", "configmaps", "secrets"}, - Verbs: []string{"create", "delete", "get", "list", "patch", "update", "watch"}, - }, - { - APIGroups: []string{"apps"}, - Resources: []string{"deployments", "daemonsets", "replicasets", "statefulsets"}, - Verbs: []string{"create", "delete", "get", "list", "patch", "update", "watch"}, - }, - { - APIGroups: []string{"monitoring.coreos.com"}, - Resources: []string{"servicemonitors"}, - Verbs: []string{"get", "create"}, - }, - { - APIGroups: []string{"apps"}, - ResourceNames: []string{"mongodb-kubernetes-operator"}, - Resources: []string{"deployments/finalizers"}, - Verbs: []string{"update"}, - }, - { - APIGroups: []string{"apps"}, - Resources: []string{"replicasets", "deployments"}, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{"mongodbcommunity.mongodb.com"}, - Resources: []string{"mongodbcommunity", "mongodbcommunity/status", "mongodbcommunity/spec"}, - Verbs: []string{"create", "delete", "get", "list", "patch", "update", "watch"}, - }, - }) - if err != nil { - t.Fatal(err) - } + mdb, user := e2eutil.NewTestMongoDB(ctx, "mdb0", namespace) - err = e2eutil.EnsureRoleBinding(ctx, namespace, "mongodb-kubernetes-operator", - []rbacv1.Subject{{ - Kind: "ServiceAccount", - Name: "mongodb-kubernetes-operator", - }}, rbacv1.RoleRef{ - Kind: "Role", - APIGroup: "rbac.authorization.k8s.io", - Name: "mongodb-kubernetes-operator", - }) + _, err = setup.GeneratePasswordForUser(ctx, user, namespace) if err != nil { t.Fatal(err) } - // Create a cluster role in the default (operator) namespace - err = e2eutil.EnsureClusterRole(ctx, e2eutil.OperatorNamespace, "mongodb-kubernetes-operator", []rbacv1.PolicyRule{ - { - APIGroups: []string{""}, - Resources: []string{"pods", "services", "services/finalizers", "endpoints", "persistentvolumeclaims", "events", "configmaps", "secrets"}, - Verbs: []string{"create", "delete", "get", "list", "patch", "update", "watch"}, - }, - { - APIGroups: []string{"apps"}, - Resources: []string{"deployments", "daemonsets", "replicasets", "statefulsets"}, - Verbs: []string{"create", "delete", "get", "list", "patch", "update", "watch"}, - }, - { - APIGroups: []string{"monitoring.coreos.com"}, - Resources: []string{"servicemonitors"}, - Verbs: []string{"get", "create"}, - }, - { - APIGroups: []string{"apps"}, - ResourceNames: []string{"mongodb-kubernetes-operator"}, - Resources: []string{"deployments/finalizers"}, - Verbs: []string{"update"}, - }, - { - APIGroups: []string{"apps"}, - Resources: []string{"replicasets", "deployments"}, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{"mongodbcommunity.mongodb.com"}, - Resources: []string{"mongodbcommunity", "mongodbcommunity/status", "mongodbcommunity/spec"}, - Verbs: []string{"create", "delete", "get", "list", "patch", "update", "watch"}, - }, - }) + tester, err := FromResource(t, mdb) if err != nil { t.Fatal(err) } - err = e2eutil.EnsureClusterRoleBinding(ctx, e2eutil.OperatorNamespace, "mongodb-kubernetes-operator", - []rbacv1.Subject{{ - Kind: "ServiceAccount", - Name: "mongodb-kubernetes-operator", - Namespace: e2eutil.OperatorNamespace, - }}, rbacv1.RoleRef{ - Kind: "ClusterRole", - APIGroup: "rbac.authorization.k8s.io", - Name: "mongodb-kubernetes-operator", - }) + t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) + t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb)) + t.Run("Keyfile authentication is configured", tester.HasKeyfileAuth(3)) + t.Run("Test Basic Connectivity", tester.ConnectivitySucceeds()) + t.Run("Ensure Authentication", tester.EnsureAuthenticationIsConfigured(3)) + t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 1)) +} + + +// createDatabaseServiceAccountRoleAndRoleBinding creates the ServiceAccount, Role and RoleBinding required +// for the database StatefulSet in the other namespace. +func createDatabaseServiceAccountRoleAndRoleBinding(t *testing.T, namespace string) error { + sa := corev1.ServiceAccount{} + err := e2eutil.TestClient.Get(context.TODO(), types.NamespacedName{Name: "mongodb-database", Namespace: e2eutil.OperatorNamespace}, &sa) if err != nil { t.Fatal(err) } - err = e2eutil.EnsureServiceAccount(ctx, namespace, "mongodb-database") + sa.Namespace = namespace + sa.ObjectMeta.ResourceVersion = "" + + err = e2eutil.TestClient.Create(context.TODO(), &sa, &e2eutil.CleanupOptions{}) if err != nil { t.Fatal(err) } - err = e2eutil.EnsureRole(ctx, namespace, "mongodb-database", []rbacv1.PolicyRule{ - { - APIGroups: []string{""}, - Resources: []string{"secrets"}, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"pods"}, - Verbs: []string{"delete", "get", "patch"}, - }, - }) + + role := rbacv1.Role{} + err = e2eutil.TestClient.Get(context.TODO(), types.NamespacedName{Name: "mongodb-database", Namespace: e2eutil.OperatorNamespace}, &role) if err != nil { t.Fatal(err) } - err = e2eutil.EnsureRoleBinding(ctx, namespace, "mongodb-database", - []rbacv1.Subject{{ - Kind: "ServiceAccount", - Name: "mongodb-database", - }}, rbacv1.RoleRef{ - Kind: "Role", - APIGroup: "rbac.authorization.k8s.io", - Name: "mongodb-database", - }) + + role.Namespace = namespace + role.ObjectMeta.ResourceVersion = "" + + err = e2eutil.TestClient.Create(context.TODO(), &role, &e2eutil.CleanupOptions{}) if err != nil { t.Fatal(err) } - mdb, user := e2eutil.NewTestMongoDB(ctx, "mdb0", namespace) - - _, err = setup.GeneratePasswordForUser(ctx, user, namespace) + rolebinding := rbacv1.RoleBinding{} + err = e2eutil.TestClient.Get(context.TODO(), types.NamespacedName{Name: "mongodb-database", Namespace: e2eutil.OperatorNamespace}, &rolebinding) if err != nil { t.Fatal(err) } - tester, err := FromResource(t, mdb) + rolebinding.Namespace = namespace + rolebinding.ObjectMeta.ResourceVersion = "" + + err = e2eutil.TestClient.Create(context.TODO(), &rolebinding, &e2eutil.CleanupOptions{}) if err != nil { t.Fatal(err) } - - t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) - t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb)) - t.Run("Keyfile authentication is configured", tester.HasKeyfileAuth(3)) - t.Run("Test Basic Connectivity", tester.ConnectivitySucceeds()) - t.Run("Ensure Authentication", tester.EnsureAuthenticationIsConfigured(3)) - t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 1)) + return nil } + diff --git a/test/e2e/setup/setup.go b/test/e2e/setup/setup.go index 9a16b63b5..ce64f162c 100644 --- a/test/e2e/setup/setup.go +++ b/test/e2e/setup/setup.go @@ -2,28 +2,23 @@ package setup import ( "context" - "encoding/json" "fmt" - "io/ioutil" - "path" - "testing" - "time" - + "github.com/mongodb/mongodb-kubernetes-operator/pkg/helm" "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/envvar" - - "github.com/mongodb/mongodb-kubernetes-operator/controllers/construct" - "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/generate" + waite2e "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/util/wait" "github.com/pkg/errors" + "io/ioutil" appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - rbacv1 "k8s.io/api/rbac/v1" apiErrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/yaml" + "path" + "strings" + "testing" + "time" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/generate" "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/secret" @@ -37,12 +32,10 @@ import ( type tlsSecretType string const ( - performCleanupEnv = "PERFORM_CLEANUP" - deployDirEnv = "DEPLOY_DIR" - roleDirEnv = "ROLE_DIR" - - CertKeyPair tlsSecretType = "CERTKEYPAIR" - Pem tlsSecretType = "PEM" + performCleanupEnv = "PERFORM_CLEANUP" + helmChartPathEnv = "HELM_CHART_PATH" + CertKeyPair tlsSecretType = "CERTKEYPAIR" + Pem tlsSecretType = "PEM" ) func Setup(t *testing.T) *e2eutil.Context { @@ -52,7 +45,7 @@ func Setup(t *testing.T) *e2eutil.Context { t.Fatal(err) } - if err := deployOperator(ctx); err != nil { + if err := deployOperator(); err != nil { t.Fatal(err) } @@ -140,18 +133,56 @@ func GeneratePasswordForUser(ctx *e2eutil.Context, mdbu mdbv1.MongoDBUser, names return password, e2eutil.TestClient.Create(context.TODO(), &passwordSecret, &e2eutil.CleanupOptions{TestContext: ctx}) } -func roleDir() string { - return envvar.GetEnvOrDefault(roleDirEnv, "/workspace/config/rbac") -} +// extractRegistryNameAndVersion splits a full image string and returns the individual components. +// this function expects the input to be in the form of some/registry/imagename:tag. +func extractRegistryNameAndVersion(fullImage string) (string, string, string) { + splitString := strings.Split(fullImage, "/") + registry := strings.Join(splitString[:len(splitString)-1], "/") -func deployDir() string { - return envvar.GetEnvOrDefault(deployDirEnv, "/workspace/config/manager") + splitString = strings.Split(splitString[len(splitString)-1], ":") + version := "latest" + if len(splitString) > 1 { + version = splitString[len(splitString)-1] + } + name := splitString[0] + return registry, name, version } -func deployOperator(ctx *e2eutil.Context) error { +// getHelmArgs returns a map of helm arguments that are required to install the operator. +func getHelmArgs(testConfig testConfig, watchNamespace string) map[string]string { + agentRegistry, agentName, agentVersion := extractRegistryNameAndVersion(testConfig.agentImage) + versionUpgradeHookRegistry, versionUpgradeHookName, versionUpgradeHookVersion := extractRegistryNameAndVersion(testConfig.versionUpgradeHookImage) + readinessProbeRegistry, readinessProbeName, readinessProbeVersion := extractRegistryNameAndVersion(testConfig.readinessProbeImage) + operatorRegistry, operatorName, operatorVersion := extractRegistryNameAndVersion(testConfig.operatorImage) - testConfig := loadTestConfigFromEnv() + helmArgs := make(map[string]string) + + helmArgs["namespace"] = testConfig.namespace + + helmArgs["operator.watchNamespace"] = watchNamespace + helmArgs["operator.operatorImageName"] = operatorName + helmArgs["operator.version"] = operatorVersion + + helmArgs["versionUpgradeHook.name"] = versionUpgradeHookName + helmArgs["versionUpgradeHook.version"] = versionUpgradeHookVersion + + helmArgs["readinessProbe.name"] = readinessProbeName + helmArgs["readinessProbe.version"] = readinessProbeVersion + + helmArgs["agent.version"] = agentVersion + helmArgs["agent.name"] = agentName + + helmArgs["registry.versionUpgradeHook"] = versionUpgradeHookRegistry + helmArgs["registry.operator"] = operatorRegistry + helmArgs["registry.agent"] = agentRegistry + helmArgs["registry.readinessProbe"] = readinessProbeRegistry + return helmArgs +} + +// deployOperator installs all resources required by the operator using helm. +func deployOperator() error { + testConfig := loadTestConfigFromEnv() e2eutil.OperatorNamespace = testConfig.namespace fmt.Printf("Setting operator namespace to %s\n", e2eutil.OperatorNamespace) watchNamespace := testConfig.namespace @@ -160,52 +191,40 @@ func deployOperator(ctx *e2eutil.Context) error { } fmt.Printf("Setting namespace to watch to %s\n", watchNamespace) - if err := buildKubernetesResourceFromYamlFile(ctx, path.Join(roleDir(), "role.yaml"), &rbacv1.Role{}, withNamespace(testConfig.namespace)); err != nil { - return errors.Errorf("error building operator role: %s", err) + chartName := "mongodb-kubernetes-operator" + if err := helm.Uninstall(chartName); err != nil { + return err } - fmt.Println("Successfully created the operator Role") - if err := buildKubernetesResourceFromYamlFile(ctx, path.Join(roleDir(), "service_account.yaml"), &corev1.ServiceAccount{}, withNamespace(testConfig.namespace)); err != nil { - return errors.Errorf("error building operator service account: %s", err) + helmChartPath := envvar.GetEnvOrDefault(helmChartPathEnv, "/workspace/helm-chart") + helmArgs := getHelmArgs(testConfig, watchNamespace) + if err := helm.Install(helmChartPath, chartName, helmArgs); err != nil { + return err } - fmt.Println("Successfully created the operator Service Account") - if err := buildKubernetesResourceFromYamlFile(ctx, path.Join(roleDir(), "role_binding.yaml"), &rbacv1.RoleBinding{}, withNamespace(testConfig.namespace)); err != nil { - return errors.Errorf("error building operator role binding: %s", err) + dep, err := waite2e.ForDeploymentToExist("mongodb-kubernetes-operator", time.Second*10, time.Minute*1, e2eutil.OperatorNamespace) + if err != nil { + return err } - fmt.Println("Successfully created the operator Role Binding") - if err := buildKubernetesResourceFromYamlFile(ctx, path.Join(roleDir(), "role_database.yaml"), &rbacv1.Role{}, withNamespace(testConfig.namespace)); err != nil { - return errors.Errorf("error building mongodb database role: %s", err) - } - if err := buildKubernetesResourceFromYamlFile(ctx, path.Join(roleDir(), "service_account_database.yaml"), &corev1.ServiceAccount{}, withNamespace(testConfig.namespace)); err != nil { - return errors.Errorf("error building mongodb database service account: %s", err) + quantityCPU, err := resource.ParseQuantity("50m") + if err != nil { + return err } - if err := buildKubernetesResourceFromYamlFile(ctx, path.Join(roleDir(), "role_binding_database.yaml"), &rbacv1.RoleBinding{}, withNamespace(testConfig.namespace)); err != nil { - return errors.Errorf("error building mongodb database role binding: %s", err) + + for _, cont := range dep.Spec.Template.Spec.Containers { + cont.Resources.Requests["cpu"] = quantityCPU } - fmt.Println("Successfully created the role, service account and role binding for the database") - - dep := &appsv1.Deployment{} - - if err := buildKubernetesResourceFromYamlFile(ctx, path.Join(deployDir(), "manager.yaml"), - dep, - withNamespace(testConfig.namespace), - withOperatorImage(testConfig.operatorImage), - withEnvVar("WATCH_NAMESPACE", watchNamespace), - withEnvVar(construct.AgentImageEnv, testConfig.agentImage), - withEnvVar(construct.ReadinessProbeImageEnv, testConfig.readinessProbeImage), - withEnvVar(construct.VersionUpgradeHookImageEnv, testConfig.versionUpgradeHookImage), - withCPURequest("50m"), - ); err != nil { - return errors.Errorf("error building operator deployment: %s", err) + + err = e2eutil.TestClient.Update(context.TODO(), &dep) + if err != nil { + return err } - if err := wait.PollImmediate(time.Second, 30*time.Second, hasDeploymentRequiredReplicas(dep)); err != nil { + if err := wait.PollImmediate(time.Second, 30*time.Second, hasDeploymentRequiredReplicas(&dep)); err != nil { return errors.New("error building operator deployment: the deployment does not have the required replicas") } - - fmt.Println("Successfully created the operator Deployment") + fmt.Println("Successfully installed the operator deployment") return nil } @@ -229,128 +248,3 @@ func hasDeploymentRequiredReplicas(dep *appsv1.Deployment) wait.ConditionFunc { return false, nil } } - -// buildKubernetesResourceFromYamlFile will create the kubernetes resource defined in yamlFilePath. All of the functional options -// provided will be applied before creation. -func buildKubernetesResourceFromYamlFile(ctx *e2eutil.Context, yamlFilePath string, obj client.Object, options ...func(obj runtime.Object) error) error { - data, err := ioutil.ReadFile(yamlFilePath) - if err != nil { - return errors.Errorf("error reading file: %s", err) - } - - if err := marshalRuntimeObjectFromYAMLBytes(data, obj); err != nil { - return errors.Errorf("error converting yaml bytes to service account: %s", err) - } - - for _, opt := range options { - if err := opt(obj); err != nil { - return err - } - } - - obj.SetLabels(e2eutil.TestLabels()) - return createOrUpdate(ctx, obj) -} - -// marshalRuntimeObjectFromYAMLBytes accepts the bytes of a yaml resource -// and unmarshals them into the provided runtime Object -func marshalRuntimeObjectFromYAMLBytes(bytes []byte, obj runtime.Object) error { - jsonBytes, err := yaml.YAMLToJSON(bytes) - if err != nil { - return err - } - return json.Unmarshal(jsonBytes, &obj) -} - -func createOrUpdate(ctx *e2eutil.Context, obj client.Object) error { - if err := e2eutil.TestClient.Create(context.TODO(), obj, &e2eutil.CleanupOptions{TestContext: ctx}); err != nil { - if apiErrors.IsAlreadyExists(err) { - return e2eutil.TestClient.Update(context.TODO(), obj) - } - return errors.Errorf("error creating %s in kubernetes: %s", obj.GetObjectKind(), err) - } - return nil -} - -// withCPURequest assumes that the underlying type is an appsv1.Deployment. -// it returns a function which will change the amount -// requested for the CPUresource. There will be -// no effect when used with a non-deployment type -func withCPURequest(cpuRequest string) func(runtime.Object) error { - return func(obj runtime.Object) error { - dep, ok := obj.(*appsv1.Deployment) - if !ok { - return errors.Errorf("withCPURequest() called on a non-deployment object") - } - quantityCPU, okCPU := resource.ParseQuantity(cpuRequest) - if okCPU != nil { - return okCPU - } - for _, cont := range dep.Spec.Template.Spec.Containers { - cont.Resources.Requests["cpu"] = quantityCPU - } - - return nil - } -} - -// withNamespace returns a function which will assign the namespace -// of the underlying type to the value specified. We can -// add new types here as required. -func withNamespace(ns string) func(runtime.Object) error { - return func(obj runtime.Object) error { - switch v := obj.(type) { - case *rbacv1.Role: - v.Namespace = ns - case *corev1.ServiceAccount: - v.Namespace = ns - case *rbacv1.RoleBinding: - v.Namespace = ns - case *corev1.Pod: - v.Namespace = ns - case *appsv1.Deployment: - v.Namespace = ns - default: - return errors.Errorf("withNamespace() called on a non supported object") - } - - return nil - } -} - -func withEnvVar(key, val string) func(obj runtime.Object) error { - return func(obj runtime.Object) error { - if testPod, ok := obj.(*corev1.Pod); ok { - testPod.Spec.Containers[0].Env = updateEnvVarList(testPod.Spec.Containers[0].Env, key, val) - } - if testDeployment, ok := obj.(*appsv1.Deployment); ok { - testDeployment.Spec.Template.Spec.Containers[0].Env = updateEnvVarList(testDeployment.Spec.Template.Spec.Containers[0].Env, key, val) - } - - return nil - } -} - -func updateEnvVarList(envVarList []corev1.EnvVar, key, val string) []corev1.EnvVar { - for index, envVar := range envVarList { - if envVar.Name == key { - envVarList[index] = corev1.EnvVar{Name: key, Value: val} - return envVarList - } - } - return append(envVarList, corev1.EnvVar{Name: key, Value: val}) -} - -// withOperatorImage assumes that the underlying type is an appsv1.Deployment -// which has the operator container as the first container. There will be -// an error return when used with a non-deployment type -func withOperatorImage(image string) func(runtime.Object) error { - return func(obj runtime.Object) error { - if dep, ok := obj.(*appsv1.Deployment); ok { - dep.Spec.Template.Spec.Containers[0].Image = image - return nil - } - - return fmt.Errorf("withOperatorImage() called on a non-deployment object") - } -} diff --git a/test/e2e/util/wait/wait.go b/test/e2e/util/wait/wait.go index 95d70ce1e..0bd256fb1 100644 --- a/test/e2e/util/wait/wait.go +++ b/test/e2e/util/wait/wait.go @@ -59,6 +59,15 @@ func waitForMongoDBCondition(mdb *mdbv1.MongoDBCommunity, retryInterval, timeout }) } + +// ForDeploymentToExist waits until a Deployment of the given name exists +// using the provided retryInterval and timeout +func ForDeploymentToExist(deployName string, retryInterval, timeout time.Duration, namespace string) (appsv1.Deployment, error) { + deploy := appsv1.Deployment{} + return deploy, waitForRuntimeObjectToExist(deployName, retryInterval, timeout, &deploy, namespace) +} + + // ForStatefulSetToExist waits until a StatefulSet of the given name exists // using the provided retryInterval and timeout func ForStatefulSetToExist(stsName string, retryInterval, timeout time.Duration, namespace string) (appsv1.StatefulSet, error) { From 934835e43cab9b38c075da58d1f8bc959b3f9e54 Mon Sep 17 00:00:00 2001 From: Rodrigo Valin Date: Mon, 30 Aug 2021 19:43:34 +0100 Subject: [PATCH 391/790] Updates go dependencies. (#700) --- go.mod | 10 +++++----- go.sum | 40 ++++++++++++++++++++++++++++------------ 2 files changed, 33 insertions(+), 17 deletions(-) diff --git a/go.mod b/go.mod index 4e10570bc..47c25e47c 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/imdario/mergo v0.3.12 github.com/klauspost/compress v1.9.8 // indirect github.com/pkg/errors v0.9.1 - github.com/spf13/cast v1.4.0 + github.com/spf13/cast v1.4.1 github.com/stretchr/objx v0.3.0 github.com/stretchr/testify v1.7.0 github.com/xdg/stringprep v1.0.3 @@ -17,10 +17,10 @@ require ( go.uber.org/zap v1.19.0 golang.org/x/tools v0.1.1 // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 - k8s.io/api v0.21.3 - k8s.io/apimachinery v0.21.3 - k8s.io/client-go v0.21.3 - sigs.k8s.io/controller-runtime v0.9.5 + k8s.io/api v0.22.1 + k8s.io/apimachinery v0.22.1 + k8s.io/client-go v0.22.1 + sigs.k8s.io/controller-runtime v0.9.6 sigs.k8s.io/yaml v1.2.0 ) diff --git a/go.sum b/go.sum index 76479898c..7936a975a 100644 --- a/go.sum +++ b/go.sum @@ -25,10 +25,13 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7 github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-autorest v13.3.2+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest v0.11.12/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= +github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= +github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -100,6 +103,7 @@ github.com/evanphx/json-patch v4.11.0+incompatible h1:glyUF9yIYtMHzn8xaKw5rMhdWc github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -163,8 +167,9 @@ github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4er github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= @@ -190,6 +195,7 @@ github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -216,6 +222,7 @@ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= +github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw= github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= @@ -245,7 +252,6 @@ github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= @@ -339,6 +345,7 @@ github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= @@ -405,8 +412,8 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.4.0 h1:WhlbjwB9EGCc8W5Rxdkus+wmH2ASRwwTJk6tgHKwdqQ= -github.com/spf13/cast v1.4.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA= +github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= @@ -549,8 +556,9 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 h1:DzZ89McO9/gWPsQXS/FVKAlG02ZjaQ6AlZRBimEYOd0= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20210520170846-37e1c6afe023 h1:ADo5wSpq2gqaCGQWzk7S5vd//0iyyLeAratkEoG5dLE= +golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -604,6 +612,7 @@ golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -617,6 +626,7 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -790,15 +800,18 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.21.3 h1:cblWILbLO8ar+Fj6xdDGr603HRsf8Wu9E9rngJeprZQ= k8s.io/api v0.21.3/go.mod h1:hUgeYHUbBp23Ue4qdX9tR8/ANi/g3ehylAqDn9NWVOg= +k8s.io/api v0.22.1 h1:ISu3tD/jRhYfSW8jI/Q1e+lRxkR7w9UwQEZ7FgslrwY= +k8s.io/api v0.22.1/go.mod h1:bh13rkTp3F1XEaLGykbyRD2QaTTzPm0e/BMd8ptFONY= k8s.io/apiextensions-apiserver v0.21.3 h1:+B6biyUWpqt41kz5x6peIsljlsuwvNAp/oFax/j2/aY= k8s.io/apiextensions-apiserver v0.21.3/go.mod h1:kl6dap3Gd45+21Jnh6utCx8Z2xxLm8LGDkprcd+KbsE= -k8s.io/apimachinery v0.21.3 h1:3Ju4nvjCngxxMYby0BimUk+pQHPOQp3eCGChk5kfVII= k8s.io/apimachinery v0.21.3/go.mod h1:H/IM+5vH9kZRNJ4l3x/fXP/5bOPJaVP/guptnZPeCFI= +k8s.io/apimachinery v0.22.1 h1:DTARnyzmdHMz7bFWFDDm22AM4pLWTQECMpRTFu2d2OM= +k8s.io/apimachinery v0.22.1/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0= k8s.io/apiserver v0.21.3/go.mod h1:eDPWlZG6/cCCMj/JBcEpDoK+I+6i3r9GsChYBHSbAzU= -k8s.io/client-go v0.21.3 h1:J9nxZTOmvkInRDCzcSNQmPJbDYN/PjlxXT9Mos3HcLg= k8s.io/client-go v0.21.3/go.mod h1:+VPhCgTsaFmGILxR/7E1N0S+ryO010QBeNCv5JwRGYU= +k8s.io/client-go v0.22.1 h1:jW0ZSHi8wW260FvcXHkIa0NLxFBQszTlhiAVsU5mopw= +k8s.io/client-go v0.22.1/go.mod h1:BquC5A4UOo4qVDUtoc04/+Nxp1MeHcVc1HJm1KmG8kk= k8s.io/code-generator v0.21.3/go.mod h1:K3y0Bv9Cz2cOW2vXUrNZlFbflhuPvuadW6JdnN6gGKo= k8s.io/component-base v0.21.3 h1:4WuuXY3Npa+iFfi2aDRiOz+anhNvRfye0859ZgfC5Og= k8s.io/component-base v0.21.3/go.mod h1:kkuhtfEHeZM6LkX0saqSK8PbdO7A0HigUngmhhrwfGQ= @@ -806,19 +819,22 @@ k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8 k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.8.0 h1:Q3gmuM9hKEjefWFFYF0Mat+YyFJvsUyYuwyNNJ5C9Ts= k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= -k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 h1:vEx13qjvaZ4yfObSSXW7BrMc/KQBBT/Jyee8XtLf4x0= +k8s.io/klog/v2 v2.9.0 h1:D7HV+n1V57XeZ0m6tdRkfknthUaM06VFbWldOFh8kzM= +k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE= +k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e h1:KLHHjkdQFomZy8+06csTWZ0m1343QqxZhR2LJ1OxCYM= +k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20210707171843-4b05e18ac7d9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210722164352-7f3ee0f31471 h1:DnzUXII7sVg1FJ/4JX6YDRJfLNAC7idRatPwe07suiI= k8s.io/utils v0.0.0-20210722164352-7f3ee0f31471/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.19/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= -sigs.k8s.io/controller-runtime v0.9.5 h1:WThcFE6cqctTn2jCZprLICO6BaKZfhsT37uAapTNfxc= -sigs.k8s.io/controller-runtime v0.9.5/go.mod h1:q6PpkM5vqQubEKUKOM6qr06oXGzOBcCby1DA9FbyZeA= +sigs.k8s.io/controller-runtime v0.9.6 h1:EevVMlgUj4fC1NVM4+DB3iPkWkmGRNarA66neqv9Qew= +sigs.k8s.io/controller-runtime v0.9.6/go.mod h1:q6PpkM5vqQubEKUKOM6qr06oXGzOBcCby1DA9FbyZeA= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.1.2 h1:Hr/htKFmJEbtMgS/UD0N+gtgctAqz81t3nu+sPzynno= sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= From 98b3c78aaca0c58b951156e6f4d7770fd913ebb6 Mon Sep 17 00:00:00 2001 From: Rodrigo Valin Date: Thu, 2 Sep 2021 11:24:20 +0100 Subject: [PATCH 392/790] CLOUDP-98103: Documents readiness probe values. (#702) --- .github/workflows/code-health.yml | 2 +- ...ngodbcommunity_readiness_probe_values.yaml | 43 ++++++++++++++++++ .../construct/build_statefulset_test.go | 1 - docs/deploy-configure.md | 44 +++++++++++++++++++ 4 files changed, 88 insertions(+), 2 deletions(-) create mode 100644 config/samples/mongodb.com_v1_mongodbcommunity_readiness_probe_values.yaml diff --git a/.github/workflows/code-health.yml b/.github/workflows/code-health.yml index 8be014241..dda57bd91 100644 --- a/.github/workflows/code-health.yml +++ b/.github/workflows/code-health.yml @@ -18,6 +18,6 @@ jobs: - uses: actions/checkout@v2 - name: Mypy linting - uses: jpetrucciani/mypy-check@master + uses: jpetrucciani/mypy-check@179fdad632bf3ccf4cabb7ee4307ef25e51d2f96 with: path: scripts/*/*.py diff --git a/config/samples/mongodb.com_v1_mongodbcommunity_readiness_probe_values.yaml b/config/samples/mongodb.com_v1_mongodbcommunity_readiness_probe_values.yaml new file mode 100644 index 000000000..e978125a7 --- /dev/null +++ b/config/samples/mongodb.com_v1_mongodbcommunity_readiness_probe_values.yaml @@ -0,0 +1,43 @@ +--- +apiVersion: mongodbcommunity.mongodb.com/v1 +kind: MongoDBCommunity +metadata: + name: mongodb-specify-readiness-probe-values +spec: + members: 3 + type: ReplicaSet + version: "4.4.0" + security: + authentication: + modes: ["SCRAM"] + users: + - name: my-user + db: admin + passwordSecretRef: # a reference to the secret that will be used to generate the user's password + name: my-user-password + roles: + - name: clusterAdmin + db: admin + - name: userAdminAnyDatabase + db: admin + scramCredentialsSecretName: my-scram + statefulSet: + spec: + template: + spec: + containers: + - name: mongodb-agent + readinessProbe: + failureThreshold: 50 + initialDelaySeconds: 10 + +# the user credentials will be generated from this secret +# once the credentials are generated, this secret is no longer required +--- +apiVersion: v1 +kind: Secret +metadata: + name: my-user-password +type: Opaque +stringData: + password: diff --git a/controllers/construct/build_statefulset_test.go b/controllers/construct/build_statefulset_test.go index 7bd24da5d..a68910ffd 100644 --- a/controllers/construct/build_statefulset_test.go +++ b/controllers/construct/build_statefulset_test.go @@ -93,7 +93,6 @@ func assertStatefulSetIsBuiltCorrectly(t *testing.T, mdb mdbv1.MongoDBCommunity, probe := agentContainer.ReadinessProbe assert.True(t, reflect.DeepEqual(probes.New(DefaultReadiness()), *probe)) assert.Equal(t, probes.New(DefaultReadiness()).FailureThreshold, probe.FailureThreshold) - assert.Equal(t, int32(5), probe.InitialDelaySeconds) assert.Len(t, agentContainer.VolumeMounts, 6) assert.NotNil(t, agentContainer.ReadinessProbe) diff --git a/docs/deploy-configure.md b/docs/deploy-configure.md index 98369c658..4fdd93bdc 100644 --- a/docs/deploy-configure.md +++ b/docs/deploy-configure.md @@ -328,3 +328,47 @@ To define a custom role: ``` kubectl apply -f .yaml --namespace ``` + + +## Specify Non-Default Values for Readiness Probe + +Under some circumstances it might be necessary to set your own custom values for +the `ReadinessProbe` used by the MongoDB Community Operator. To do so, you +should use the `statefulSet` attribute in `resource.spec`, as in the following +provided example [yaml +file](../config/samples/mongodb.com_v1_mongodbcommunity_readiness_probe_values.yaml). +Only those attributes passed will be set, for instance, given the following structure: + +```yaml +spec: + statefulSet: + spec: + template: + spec: + containers: + - name: mongodb-agent + readinessProbe: + failureThreshold: 40 + initialDelaySeconds: 5 +``` + +*Only* the values of `failureThreshold` and `initialDelaySeconds` will be set to +their custom, specified values. The rest of the attributes will be set to their +default values. + +*Please note that these are the actual values set by the Operator for our +MongoDB Custom Resources.* + +### When to specify custom values for the Readiness Probe + +In some cases, for instance, with a less than optimal download speed from the +image registry, it could be necessary for the Operator to tolerate a Pod that +has taken longer than expected to restart or upgrade to a different version of +MongoDB. In these cases we want the Kubernetes API to wait a little longer +before giving up, we could increase the value of `failureThreshold` to `60`. + +In other cases, if the Kubernetes API is slower than usual, we would increase +the value of `periodSeconds` to `20`, so the Kubernetes API will do half of the +requests it normally does (default value for `periodSeconds` is `10`). + +*Please note that these are referential values only!* From a6117f6a5b41746e0c6f3644adfb83fa0783b70f Mon Sep 17 00:00:00 2001 From: Rajdeep Das Date: Thu, 2 Sep 2021 16:29:21 +0200 Subject: [PATCH 393/790] Add WithAnnotations and WithVolumeClaimTemplates method to Statefulset construction (#705) --- pkg/kube/statefulset/statefulset.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/pkg/kube/statefulset/statefulset.go b/pkg/kube/statefulset/statefulset.go index 3314ddf0a..7e9fbab63 100644 --- a/pkg/kube/statefulset/statefulset.go +++ b/pkg/kube/statefulset/statefulset.go @@ -211,6 +211,13 @@ func WithLabels(labels map[string]string) Modification { set.Labels = copyMap(labels) } } + +func WithAnnotations(annotations map[string]string) Modification { + return func(set *appsv1.StatefulSet) { + set.Annotations = copyMap(annotations) + } +} + func WithMatchLabels(matchLabels map[string]string) Modification { return func(set *appsv1.StatefulSet) { if set.Spec.Selector == nil { @@ -280,6 +287,14 @@ func WithVolumeClaim(name string, f func(*corev1.PersistentVolumeClaim)) Modific } } +func WithVolumeClaimTemplates(pv []corev1.PersistentVolumeClaim) Modification { + pvCopy := make([]corev1.PersistentVolumeClaim, len(pv)) + copy(pvCopy, pv) + return func(set *appsv1.StatefulSet) { + set.Spec.VolumeClaimTemplates = pvCopy + } +} + func WithCustomSpecs(spec appsv1.StatefulSetSpec) Modification { return func(set *appsv1.StatefulSet) { set.Spec = merge.StatefulSetSpecs(set.Spec, spec) From 44ed501acaeb58b523d757025fbf292bec7e024d Mon Sep 17 00:00:00 2001 From: Ciprian Tibulca Date: Mon, 6 Sep 2021 12:59:56 +0100 Subject: [PATCH 394/790] CLOUDP-98684 helm chart - integrate cert-manager (#706) * CLOUDP-98684 helm chart - integrate cert-manager * CLOUDP-98684 helm chart - skip installing crds when running e2e tests * CLOUDP-98684 use the correct k8s api to read CA secret * CLOUDP-98684 rename helm cr --- Makefile | 4 + api/v1/mongodbcommunity_types.go | 14 +- ...ommunity.mongodb.com_mongodbcommunity.yaml | 634 +++++++++--------- controllers/mongodb_tls.go | 34 +- controllers/mongodb_tls_test.go | 4 + controllers/replicaset_controller_test.go | 2 +- ...ommunity.mongodb.com_mongodbcommunity.yaml | 355 ++++++++++ .../mongodbcommunity_cr_with_tls.yaml | 79 +++ helm-chart/values.yaml | 11 + pkg/helm/helm.go | 2 +- test/e2e/e2eutil.go | 2 +- 11 files changed, 806 insertions(+), 335 deletions(-) create mode 100644 helm-chart/crds/mongodbcommunity.mongodb.com_mongodbcommunity.yaml create mode 100644 helm-chart/templates/mongodbcommunity_cr_with_tls.yaml diff --git a/Makefile b/Makefile index 4e332de52..8f532f1ba 100644 --- a/Makefile +++ b/Makefile @@ -51,6 +51,9 @@ install-crd: install-chart: $(HELM) upgrade --install $(STRING_SET_VALUES) $(RELEASE_NAME_HELM) helm-chart +install-chart-with-tls-enabled: + $(HELM) upgrade --install --set createResource=true $(STRING_SET_VALUES) $(RELEASE_NAME_HELM) helm-chart + install-rbac: $(HELM) template $(STRING_SET_VALUES) -s templates/database_roles.yaml helm-chart | kubectl apply -f - $(HELM) template $(STRING_SET_VALUES) -s templates/operator_roles.yaml helm-chart | kubectl apply -f - @@ -78,6 +81,7 @@ undeploy: uninstall-chart uninstall-crd # Generate manifests e.g. CRD, RBAC etc. manifests: controller-gen $(CONTROLLER_GEN) $(CRD_OPTIONS) paths="./..." output:crd:artifacts:config=config/crd/bases + cp config/crd/bases/* helm-chart/crds # Run go fmt against code fmt: diff --git a/api/v1/mongodbcommunity_types.go b/api/v1/mongodbcommunity_types.go index 5a9b9c50a..60e091af2 100644 --- a/api/v1/mongodbcommunity_types.go +++ b/api/v1/mongodbcommunity_types.go @@ -326,10 +326,16 @@ type TLS struct { // +optional CertificateKeySecret LocalObjectReference `json:"certificateKeySecretRef"` + // CaCertificateSecret is a reference to a Secret containing the certificate for the CA which signed the server certificates + // The certificate is expected to be available under the key "ca.crt" + // +optional + CaCertificateSecret *LocalObjectReference `json:"caCertificateSecretRef"` + // CaConfigMap is a reference to a ConfigMap containing the certificate for the CA which signed the server certificates // The certificate is expected to be available under the key "ca.crt" + // This field is ignored when CaCertificateSecretRef is configured // +optional - CaConfigMap LocalObjectReference `json:"caConfigMapRef"` + CaConfigMap *LocalObjectReference `json:"caConfigMapRef"` } // LocalObjectReference is a reference to another Kubernetes object by name. @@ -540,6 +546,12 @@ func (m MongoDBCommunity) AutomationConfigSecretName() string { return m.Name + "-config" } +// TLSCaCertificateSecretNamespacedName will get the namespaced name of the Secret containing the CA certificate +// As the Secret will be mounted to our pods, it has to be in the same namespace as the MongoDB resource +func (m MongoDBCommunity) TLSCaCertificateSecretNamespacedName() types.NamespacedName { + return types.NamespacedName{Name: m.Spec.Security.TLS.CaCertificateSecret.Name, Namespace: m.Namespace} +} + // TLSConfigMapNamespacedName will get the namespaced name of the ConfigMap containing the CA certificate // As the ConfigMap will be mounted to our pods, it has to be in the same namespace as the MongoDB resource func (m MongoDBCommunity) TLSConfigMapNamespacedName() types.NamespacedName { diff --git a/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml b/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml index c82e93d2d..7e7f03e12 100644 --- a/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml +++ b/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml @@ -1,3 +1,4 @@ + --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition @@ -13,354 +14,339 @@ spec: listKind: MongoDBCommunityList plural: mongodbcommunity shortNames: - - mdbc + - mdbc singular: mongodbcommunity scope: Namespaced versions: - - additionalPrinterColumns: - - description: Current state of the MongoDB deployment - jsonPath: .status.phase - name: Phase - type: string - - description: Version of MongoDB server - jsonPath: .status.version - name: Version - type: string - name: v1 - schema: - openAPIV3Schema: - description: MongoDBCommunity is the Schema for the mongodbs API - properties: - apiVersion: - description: - "APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources" - type: string - kind: - description: - "Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds" - type: string - metadata: - type: object - spec: - description: MongoDBCommunitySpec defines the desired state of MongoDB - properties: - additionalMongodConfig: - description: - "AdditionalMongodConfig is additional configuration that - can be passed to each data-bearing mongod at runtime. Uses the same - structure as the mongod configuration file: https://docs.mongodb.com/manual/reference/configuration-options/" - nullable: true + - additionalPrinterColumns: + - description: Current state of the MongoDB deployment + jsonPath: .status.phase + name: Phase + type: string + - description: Version of MongoDB server + jsonPath: .status.version + name: Version + type: string + name: v1 + schema: + openAPIV3Schema: + description: MongoDBCommunity is the Schema for the mongodbs API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: MongoDBCommunitySpec defines the desired state of MongoDB + properties: + additionalMongodConfig: + description: 'AdditionalMongodConfig is additional configuration that + can be passed to each data-bearing mongod at runtime. Uses the same + structure as the mongod configuration file: https://docs.mongodb.com/manual/reference/configuration-options/' + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + arbiters: + description: Arbiters is the number of arbiters (each counted as a + member) in the replica set + type: integer + featureCompatibilityVersion: + description: FeatureCompatibilityVersion configures the feature compatibility + version that will be set for the deployment + type: string + members: + description: Members is the number of members in the replica set + type: integer + replicaSetHorizons: + description: ReplicaSetHorizons Add this parameter and values if you + need your database to be accessed outside of Kubernetes. This setting + allows you to provide different DNS settings within the Kubernetes + cluster and to the Kubernetes cluster. The Kubernetes Operator uses + split horizon DNS for replica set members. This feature allows communication + both within the Kubernetes cluster and from outside Kubernetes. + items: + additionalProperties: + type: string type: object - x-kubernetes-preserve-unknown-fields: true - arbiters: - description: - Arbiters is the number of arbiters (each counted as a - member) in the replica set - type: integer - featureCompatibilityVersion: - description: - FeatureCompatibilityVersion configures the feature compatibility - version that will be set for the deployment - type: string - members: - description: Members is the number of members in the replica set - type: integer - replicaSetHorizons: - description: - ReplicaSetHorizons Add this parameter and values if you - need your database to be accessed outside of Kubernetes. This setting - allows you to provide different DNS settings within the Kubernetes - cluster and to the Kubernetes cluster. The Kubernetes Operator uses - split horizon DNS for replica set members. This feature allows communication - both within the Kubernetes cluster and from outside Kubernetes. - items: - additionalProperties: - type: string + type: array + security: + description: Security configures security features, such as TLS, and + authentication settings for a deployment + properties: + authentication: + properties: + ignoreUnknownUsers: + default: true + nullable: true + type: boolean + modes: + description: Modes is an array specifying which authentication + methods should be enabled. + items: + enum: + - SCRAM + - SCRAM-SHA-256 + - SCRAM-SHA-1 + type: string + type: array + required: + - modes type: object - type: array - security: - description: - Security configures security features, such as TLS, and - authentication settings for a deployment - properties: - authentication: + roles: + description: User-specified custom MongoDB roles that should be + configured in the deployment. + items: + description: CustomRole defines a custom MongoDB role. properties: - ignoreUnknownUsers: - default: true - nullable: true - type: boolean - modes: - description: - Modes is an array specifying which authentication - methods should be enabled. + authenticationRestrictions: + description: The authentication restrictions the server + enforces on the role. items: - enum: - - SCRAM - - SCRAM-SHA-256 - - SCRAM-SHA-1 - type: string + description: AuthenticationRestriction specifies a list + of IP addresses and CIDR ranges users are allowed to + connect to or from. + properties: + clientSource: + items: + type: string + type: array + serverAddress: + items: + type: string + type: array + required: + - clientSource + - serverAddress + type: object + type: array + db: + description: The database of the role. + type: string + privileges: + description: The privileges to grant the role. + items: + description: Privilege defines the actions a role is allowed + to perform on a given resource. + properties: + actions: + items: + type: string + type: array + resource: + description: Resource specifies specifies the resources + upon which a privilege permits actions. See https://docs.mongodb.com/manual/reference/resource-document + for more. + properties: + anyResource: + type: boolean + cluster: + type: boolean + collection: + type: string + db: + type: string + type: object + required: + - actions + - resource + type: object + type: array + role: + description: The name of the role. + type: string + roles: + description: An array of roles from which this role inherits + privileges. + items: + description: Role is the database role this user should + have + properties: + db: + description: DB is the database the role can act on + type: string + name: + description: Name is the name of the role + type: string + required: + - db + - name + type: object type: array required: - - modes + - db + - privileges + - role type: object - roles: - description: - User-specified custom MongoDB roles that should be - configured in the deployment. - items: - description: CustomRole defines a custom MongoDB role. + type: array + tls: + description: TLS configuration for both client-server and server-server + communication + properties: + caCertificateSecretRef: + description: CaCertificateSecret is a reference to a Secret + containing the certificate for the CA which signed the server + certificates The certificate is expected to be available + under the key "ca.crt" properties: - authenticationRestrictions: - description: - The authentication restrictions the server - enforces on the role. - items: - description: - AuthenticationRestriction specifies a list - of IP addresses and CIDR ranges users are allowed to - connect to or from. - properties: - clientSource: - items: - type: string - type: array - serverAddress: - items: - type: string - type: array - required: - - clientSource - - serverAddress - type: object - type: array - db: - description: The database of the role. + name: type: string - privileges: - description: The privileges to grant the role. - items: - description: - Privilege defines the actions a role is allowed - to perform on a given resource. - properties: - actions: - items: - type: string - type: array - resource: - description: - Resource specifies specifies the resources - upon which a privilege permits actions. See https://docs.mongodb.com/manual/reference/resource-document - for more. - properties: - anyResource: - type: boolean - cluster: - type: boolean - collection: - type: string - db: - type: string - type: object - required: - - actions - - resource - type: object - type: array - role: - description: The name of the role. + required: + - name + type: object + caConfigMapRef: + description: CaConfigMap is a reference to a ConfigMap containing + the certificate for the CA which signed the server certificates + The certificate is expected to be available under the key + "ca.crt" This field is ignored when CaCertificateSecretRef + is configured + properties: + name: type: string - roles: - description: - An array of roles from which this role inherits - privileges. - items: - description: - Role is the database role this user should - have - properties: - db: - description: DB is the database the role can act on - type: string - name: - description: Name is the name of the role - type: string - required: - - db - - name - type: object - type: array required: - - db - - privileges - - role + - name type: object - type: array - tls: - description: - TLS configuration for both client-server and server-server - communication + certificateKeySecretRef: + description: CertificateKeySecret is a reference to a Secret + containing a private key and certificate to use for TLS. + The key and cert are expected to be PEM encoded and available + at "tls.key" and "tls.crt". This is the same format used + for the standard "kubernetes.io/tls" Secret type, but no + specific type is required. Alternatively, an entry tls.pem, + containing the concatenation of cert and key, can be provided. + If all of tls.pem, tls.crt and tls.key are present, the + tls.pem one needs to be equal to the concatenation of tls.crt + and tls.key + properties: + name: + type: string + required: + - name + type: object + enabled: + type: boolean + optional: + description: Optional configures if TLS should be required + or optional for connections + type: boolean + required: + - enabled + type: object + type: object + statefulSet: + description: StatefulSetConfiguration holds the optional custom StatefulSet + that should be merged into the operator created one. + properties: + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - spec + type: object + type: + description: Type defines which type of MongoDB deployment the resource + should create + enum: + - ReplicaSet + type: string + users: + description: Users specifies the MongoDB users that should be configured + in your deployment + items: + properties: + db: + description: DB is the database the user is stored in. Defaults + to "admin" + type: string + name: + description: Name is the username of the user + type: string + passwordSecretRef: + description: PasswordSecretRef is a reference to the secret + containing this user's password properties: - caConfigMapRef: - description: - CaConfigMap is a reference to a ConfigMap containing - the certificate for the CA which signed the server certificates - The certificate is expected to be available under the key - "ca.crt" - properties: - name: - type: string - required: - - name - type: object - certificateKeySecretRef: - description: - CertificateKeySecret is a reference to a Secret - containing a private key and certificate to use for TLS. - The key and cert are expected to be PEM encoded and available - at "tls.key" and "tls.crt". This is the same format used - for the standard "kubernetes.io/tls" Secret type, but no - specific type is required. Alternatively, an entry tls.pem, - containing the concatenation of cert and key, can be provided. - If all of tls.pem, tls.crt and tls.key are present, the - tls.pem one needs to be equal to the concatenation of tls.crt - and tls.key - properties: - name: - type: string - required: - - name - type: object - enabled: - type: boolean - optional: - description: - Optional configures if TLS should be required - or optional for connections - type: boolean + key: + description: Key is the key in the secret storing this password. + Defaults to "password" + type: string + name: + description: Name is the name of the secret storing this + user's password + type: string required: - - enabled - type: object - type: object - statefulSet: - description: - StatefulSetConfiguration holds the optional custom StatefulSet - that should be merged into the operator created one. - properties: - spec: + - name type: object - x-kubernetes-preserve-unknown-fields: true - required: - - spec - type: object - type: - description: - Type defines which type of MongoDB deployment the resource - should create - enum: - - ReplicaSet - type: string - users: - description: - Users specifies the MongoDB users that should be configured - in your deployment - items: - properties: - db: - description: - DB is the database the user is stored in. Defaults - to "admin" - type: string - name: - description: Name is the username of the user - type: string - passwordSecretRef: - description: - PasswordSecretRef is a reference to the secret - containing this user's password + roles: + description: Roles is an array of roles assigned to this user + items: + description: Role is the database role this user should have properties: - key: - description: - Key is the key in the secret storing this password. - Defaults to "password" + db: + description: DB is the database the role can act on type: string name: - description: - Name is the name of the secret storing this - user's password + description: Name is the name of the role type: string required: - - name + - db + - name type: object - roles: - description: Roles is an array of roles assigned to this user - items: - description: Role is the database role this user should have - properties: - db: - description: DB is the database the role can act on - type: string - name: - description: Name is the name of the role - type: string - required: - - db - - name - type: object - type: array - scramCredentialsSecretName: - description: - ScramCredentialsSecretName appended by string "scram-credentials" - is the name of the secret object created by the mongoDB operator - for storing SCRAM credentials - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - required: - - name - - passwordSecretRef - - roles - - scramCredentialsSecretName - type: object - type: array - version: - description: Version defines which version of MongoDB will be used - type: string - required: - - security - - type - - users - - version - type: object - status: - description: MongoDBCommunityStatus defines the observed state of MongoDB - properties: - currentMongoDBMembers: - type: integer - currentStatefulSetReplicas: - type: integer - message: - type: string - mongoUri: - type: string - phase: - type: string - version: - type: string - required: - - currentMongoDBMembers - - currentStatefulSetReplicas - - mongoUri - - phase - type: object - type: object - served: true - storage: true - subresources: - status: {} + type: array + scramCredentialsSecretName: + description: ScramCredentialsSecretName appended by string "scram-credentials" + is the name of the secret object created by the mongoDB operator + for storing SCRAM credentials + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - name + - passwordSecretRef + - roles + - scramCredentialsSecretName + type: object + type: array + version: + description: Version defines which version of MongoDB will be used + type: string + required: + - security + - type + - users + type: object + status: + description: MongoDBCommunityStatus defines the observed state of MongoDB + properties: + currentMongoDBMembers: + type: integer + currentStatefulSetReplicas: + type: integer + message: + type: string + mongoUri: + type: string + phase: + type: string + version: + type: string + required: + - currentMongoDBMembers + - currentStatefulSetReplicas + - mongoUri + - phase + - version + type: object + type: object + served: true + storage: true + subresources: + status: {} status: acceptedNames: kind: "" diff --git a/controllers/mongodb_tls.go b/controllers/mongodb_tls.go index 7ca9d6b8e..701ea9367 100644 --- a/controllers/mongodb_tls.go +++ b/controllers/mongodb_tls.go @@ -15,7 +15,9 @@ import ( "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/podtemplatespec" "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/statefulset" + v1 "k8s.io/api/core/v1" apiErrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" ) @@ -37,20 +39,33 @@ func (r *ReplicaSetReconciler) validateTLSConfig(mdb mdbv1.MongoDBCommunity) (bo r.log.Info("Ensuring TLS is correctly configured") - // Ensure CA ConfigMap exists - caData, err := configmap.ReadData(r.client, mdb.TLSConfigMapNamespacedName()) + // Ensure CA cert is configured + var caResourceName types.NamespacedName + var caResourceType string + var caData map[string]string + var err error + if mdb.Spec.Security.TLS.CaCertificateSecret != nil { + caResourceName = mdb.TLSCaCertificateSecretNamespacedName() + caResourceType = "Secret" + caData, err = secret.ReadStringData(r.client, caResourceName) + } else { + caResourceName = mdb.TLSConfigMapNamespacedName() + caResourceType = "ConfigMap" + caData, err = configmap.ReadData(r.client, caResourceName) + } + if err != nil { if apiErrors.IsNotFound(err) { - r.log.Warnf(`CA ConfigMap "%s" not found`, mdb.TLSConfigMapNamespacedName()) + r.log.Warnf(`CA %s "%s" not found`, caResourceType, caResourceName) return false, nil } return false, err } - // Ensure ConfigMap has a "ca.crt" field + // Ensure Secret or ConfigMap has a "ca.crt" field if cert, ok := caData[tlsCACertName]; !ok || cert == "" { - r.log.Warnf(`ConfigMap "%s" should have a CA certificate in field "%s"`, mdb.TLSConfigMapNamespacedName(), tlsCACertName) + r.log.Warnf(`%s "%s" should have a CA certificate in field "%s"`, caResourceType, caResourceName, tlsCACertName) return false, nil } @@ -210,9 +225,14 @@ func buildTLSPodSpecModification(mdb mdbv1.MongoDBCommunity) podtemplatespec.Mod return podtemplatespec.NOOP() } - // Configure a volume which mounts the CA certificate from a ConfigMap + // Configure a volume which mounts the CA certificate from either a Secret or a ConfigMap // The certificate is used by both mongod and the agent - caVolume := statefulset.CreateVolumeFromConfigMap("tls-ca", mdb.Spec.Security.TLS.CaConfigMap.Name) + var caVolume v1.Volume + if mdb.Spec.Security.TLS.CaCertificateSecret != nil { + caVolume = statefulset.CreateVolumeFromSecret("tls-ca", mdb.Spec.Security.TLS.CaCertificateSecret.Name) + } else { + caVolume = statefulset.CreateVolumeFromConfigMap("tls-ca", mdb.Spec.Security.TLS.CaConfigMap.Name) + } caVolumeMount := statefulset.CreateVolumeMount(caVolume.Name, tlsCAMountPath, statefulset.WithReadOnly(true)) // Configure a volume which mounts the secret holding the server key and certificate diff --git a/controllers/mongodb_tls_test.go b/controllers/mongodb_tls_test.go index 49260c1f3..82a503587 100644 --- a/controllers/mongodb_tls_test.go +++ b/controllers/mongodb_tls_test.go @@ -283,6 +283,10 @@ func TestPemSupport(t *testing.T) { } func createTLSConfigMap(c k8sClient.Client, mdb mdbv1.MongoDBCommunity) error { + if !mdb.Spec.Security.TLS.Enabled { + return nil + } + configMap := configmap.Builder(). SetName(mdb.Spec.Security.TLS.CaConfigMap.Name). SetNamespace(mdb.Namespace). diff --git a/controllers/replicaset_controller_test.go b/controllers/replicaset_controller_test.go index 804f5f4d9..033991720 100644 --- a/controllers/replicaset_controller_test.go +++ b/controllers/replicaset_controller_test.go @@ -98,7 +98,7 @@ func newTestReplicaSetWithTLS() mdbv1.MongoDBCommunity { }, TLS: mdbv1.TLS{ Enabled: true, - CaConfigMap: mdbv1.LocalObjectReference{ + CaConfigMap: &mdbv1.LocalObjectReference{ Name: "caConfigMap", }, CertificateKeySecret: mdbv1.LocalObjectReference{ diff --git a/helm-chart/crds/mongodbcommunity.mongodb.com_mongodbcommunity.yaml b/helm-chart/crds/mongodbcommunity.mongodb.com_mongodbcommunity.yaml new file mode 100644 index 000000000..7e7f03e12 --- /dev/null +++ b/helm-chart/crds/mongodbcommunity.mongodb.com_mongodbcommunity.yaml @@ -0,0 +1,355 @@ + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.4.1 + creationTimestamp: null + name: mongodbcommunity.mongodbcommunity.mongodb.com +spec: + group: mongodbcommunity.mongodb.com + names: + kind: MongoDBCommunity + listKind: MongoDBCommunityList + plural: mongodbcommunity + shortNames: + - mdbc + singular: mongodbcommunity + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Current state of the MongoDB deployment + jsonPath: .status.phase + name: Phase + type: string + - description: Version of MongoDB server + jsonPath: .status.version + name: Version + type: string + name: v1 + schema: + openAPIV3Schema: + description: MongoDBCommunity is the Schema for the mongodbs API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: MongoDBCommunitySpec defines the desired state of MongoDB + properties: + additionalMongodConfig: + description: 'AdditionalMongodConfig is additional configuration that + can be passed to each data-bearing mongod at runtime. Uses the same + structure as the mongod configuration file: https://docs.mongodb.com/manual/reference/configuration-options/' + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + arbiters: + description: Arbiters is the number of arbiters (each counted as a + member) in the replica set + type: integer + featureCompatibilityVersion: + description: FeatureCompatibilityVersion configures the feature compatibility + version that will be set for the deployment + type: string + members: + description: Members is the number of members in the replica set + type: integer + replicaSetHorizons: + description: ReplicaSetHorizons Add this parameter and values if you + need your database to be accessed outside of Kubernetes. This setting + allows you to provide different DNS settings within the Kubernetes + cluster and to the Kubernetes cluster. The Kubernetes Operator uses + split horizon DNS for replica set members. This feature allows communication + both within the Kubernetes cluster and from outside Kubernetes. + items: + additionalProperties: + type: string + type: object + type: array + security: + description: Security configures security features, such as TLS, and + authentication settings for a deployment + properties: + authentication: + properties: + ignoreUnknownUsers: + default: true + nullable: true + type: boolean + modes: + description: Modes is an array specifying which authentication + methods should be enabled. + items: + enum: + - SCRAM + - SCRAM-SHA-256 + - SCRAM-SHA-1 + type: string + type: array + required: + - modes + type: object + roles: + description: User-specified custom MongoDB roles that should be + configured in the deployment. + items: + description: CustomRole defines a custom MongoDB role. + properties: + authenticationRestrictions: + description: The authentication restrictions the server + enforces on the role. + items: + description: AuthenticationRestriction specifies a list + of IP addresses and CIDR ranges users are allowed to + connect to or from. + properties: + clientSource: + items: + type: string + type: array + serverAddress: + items: + type: string + type: array + required: + - clientSource + - serverAddress + type: object + type: array + db: + description: The database of the role. + type: string + privileges: + description: The privileges to grant the role. + items: + description: Privilege defines the actions a role is allowed + to perform on a given resource. + properties: + actions: + items: + type: string + type: array + resource: + description: Resource specifies specifies the resources + upon which a privilege permits actions. See https://docs.mongodb.com/manual/reference/resource-document + for more. + properties: + anyResource: + type: boolean + cluster: + type: boolean + collection: + type: string + db: + type: string + type: object + required: + - actions + - resource + type: object + type: array + role: + description: The name of the role. + type: string + roles: + description: An array of roles from which this role inherits + privileges. + items: + description: Role is the database role this user should + have + properties: + db: + description: DB is the database the role can act on + type: string + name: + description: Name is the name of the role + type: string + required: + - db + - name + type: object + type: array + required: + - db + - privileges + - role + type: object + type: array + tls: + description: TLS configuration for both client-server and server-server + communication + properties: + caCertificateSecretRef: + description: CaCertificateSecret is a reference to a Secret + containing the certificate for the CA which signed the server + certificates The certificate is expected to be available + under the key "ca.crt" + properties: + name: + type: string + required: + - name + type: object + caConfigMapRef: + description: CaConfigMap is a reference to a ConfigMap containing + the certificate for the CA which signed the server certificates + The certificate is expected to be available under the key + "ca.crt" This field is ignored when CaCertificateSecretRef + is configured + properties: + name: + type: string + required: + - name + type: object + certificateKeySecretRef: + description: CertificateKeySecret is a reference to a Secret + containing a private key and certificate to use for TLS. + The key and cert are expected to be PEM encoded and available + at "tls.key" and "tls.crt". This is the same format used + for the standard "kubernetes.io/tls" Secret type, but no + specific type is required. Alternatively, an entry tls.pem, + containing the concatenation of cert and key, can be provided. + If all of tls.pem, tls.crt and tls.key are present, the + tls.pem one needs to be equal to the concatenation of tls.crt + and tls.key + properties: + name: + type: string + required: + - name + type: object + enabled: + type: boolean + optional: + description: Optional configures if TLS should be required + or optional for connections + type: boolean + required: + - enabled + type: object + type: object + statefulSet: + description: StatefulSetConfiguration holds the optional custom StatefulSet + that should be merged into the operator created one. + properties: + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - spec + type: object + type: + description: Type defines which type of MongoDB deployment the resource + should create + enum: + - ReplicaSet + type: string + users: + description: Users specifies the MongoDB users that should be configured + in your deployment + items: + properties: + db: + description: DB is the database the user is stored in. Defaults + to "admin" + type: string + name: + description: Name is the username of the user + type: string + passwordSecretRef: + description: PasswordSecretRef is a reference to the secret + containing this user's password + properties: + key: + description: Key is the key in the secret storing this password. + Defaults to "password" + type: string + name: + description: Name is the name of the secret storing this + user's password + type: string + required: + - name + type: object + roles: + description: Roles is an array of roles assigned to this user + items: + description: Role is the database role this user should have + properties: + db: + description: DB is the database the role can act on + type: string + name: + description: Name is the name of the role + type: string + required: + - db + - name + type: object + type: array + scramCredentialsSecretName: + description: ScramCredentialsSecretName appended by string "scram-credentials" + is the name of the secret object created by the mongoDB operator + for storing SCRAM credentials + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - name + - passwordSecretRef + - roles + - scramCredentialsSecretName + type: object + type: array + version: + description: Version defines which version of MongoDB will be used + type: string + required: + - security + - type + - users + type: object + status: + description: MongoDBCommunityStatus defines the observed state of MongoDB + properties: + currentMongoDBMembers: + type: integer + currentStatefulSetReplicas: + type: integer + message: + type: string + mongoUri: + type: string + phase: + type: string + version: + type: string + required: + - currentMongoDBMembers + - currentStatefulSetReplicas + - mongoUri + - phase + - version + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/helm-chart/templates/mongodbcommunity_cr_with_tls.yaml b/helm-chart/templates/mongodbcommunity_cr_with_tls.yaml new file mode 100644 index 000000000..01b417e51 --- /dev/null +++ b/helm-chart/templates/mongodbcommunity_cr_with_tls.yaml @@ -0,0 +1,79 @@ +{{- if and .Values.createResource .Values.resource.tls.enabled .Values.resource.tls.useCertManager }} +# cert-manager resources +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: tls-selfsigned-issuer + namespace: {{ .Values.namespace }} +spec: + selfSigned: {} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: tls-selfsigned-ca + namespace: {{ .Values.namespace }} +spec: + isCA: true + commonName: "*.{{ .Values.resource.name }}-svc.mongodb.svc.cluster.local" + dnsNames: + - "*.{{ .Values.resource.name }}-svc.mongodb.svc.cluster.local" + secretName: {{ .Values.resource.tls.caCertificateSecretRef }} + privateKey: + algorithm: ECDSA + size: 256 + issuerRef: + name: tls-selfsigned-issuer + kind: Issuer +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: tls-ca-issuer + namespace: {{ .Values.namespace }} +spec: + ca: + secretName: {{ .Values.resource.tls.caCertificateSecretRef }} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: cert-manager-tls-certificate + namespace: {{ .Values.namespace }} +spec: + secretName: {{ .Values.resource.tls.certificateKeySecretRef }} + issuerRef: + name: tls-ca-issuer + kind: Issuer + duration: 8760h # 365 days + renewBefore: 720h # 30 days + commonName: "*.{{ .Values.resource.name }}-svc.mongodb.svc.cluster.local" + dnsNames: + - "*.{{ .Values.resource.name }}-svc.mongodb.svc.cluster.local" +{{- end }} + +# mongodb resources +{{- if .Values.createResource }} +--- +apiVersion: mongodbcommunity.mongodb.com/v1 +kind: MongoDBCommunity +metadata: + name: {{ .Values.resource.name }} + namespace: {{ .Values.namespace }} +spec: + members: {{ .Values.resource.members }} + type: ReplicaSet + version: {{ .Values.resource.version }} + security: + tls: + enabled: {{ .Values.resource.tls.enabled }} + {{- if .Values.resource.tls.enabled }} + certificateKeySecretRef: + name: {{ .Values.resource.tls.certificateKeySecretRef }} + caCertificateSecretRef: + name: {{ .Values.resource.tls.caCertificateSecretRef }} + {{- end }} + authentication: + modes: ["SCRAM"] + users: [] +{{- end }} \ No newline at end of file diff --git a/helm-chart/values.yaml b/helm-chart/values.yaml index 7d5693de9..95ea145c9 100644 --- a/helm-chart/values.yaml +++ b/helm-chart/values.yaml @@ -38,3 +38,14 @@ registry: readinessProbe: quay.io/mongodb operator: quay.io/mongodb pullPolicy: Always + +createResource: false +resource: + name: mongodb-replica-set + version: 4.4.0 + members: 3 + tls: + enabled: true + useCertManager: true + certificateKeySecretRef: tls-certificate + caCertificateSecretRef: tls-ca-key-pair diff --git a/pkg/helm/helm.go b/pkg/helm/helm.go index 347ce7346..3967181f8 100644 --- a/pkg/helm/helm.go +++ b/pkg/helm/helm.go @@ -15,7 +15,7 @@ func Uninstall(chartName string) error { // Install a helm chert at the given path with the given name and the provided set arguments. func Install(chartPath, chartName string, args map[string]string) error { - helmArgs := []string{"install"} + helmArgs := []string{"install", "--skip-crds"} helmArgs = append(helmArgs, mapToHelmArgs(args)...) helmArgs = append(helmArgs, chartName, chartPath) return executeHelmCommand(helmArgs, nil) diff --git a/test/e2e/e2eutil.go b/test/e2e/e2eutil.go index 6ad4c1596..5bedcf403 100644 --- a/test/e2e/e2eutil.go +++ b/test/e2e/e2eutil.go @@ -153,7 +153,7 @@ func NewTestTLSConfig(optional bool) mdbv1.TLS { CertificateKeySecret: mdbv1.LocalObjectReference{ Name: "test-tls-secret", }, - CaConfigMap: mdbv1.LocalObjectReference{ + CaConfigMap: &mdbv1.LocalObjectReference{ Name: "test-tls-ca", }, } From 9ec2dcf3b17a6e8d6cec6732ea2e5df1d8621c27 Mon Sep 17 00:00:00 2001 From: Rodrigo Valin Date: Wed, 8 Sep 2021 15:12:47 +0100 Subject: [PATCH 395/790] CLOUDP-99851: Update dependencies. (#710) --- api/v1/zz_generated.deepcopy.go | 13 ++- go.mod | 5 +- go.sum | 178 +++++++++++++++++--------------- 3 files changed, 107 insertions(+), 89 deletions(-) diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index 8e97f95a6..eade1c619 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -370,7 +370,7 @@ func (in *SecretKeyReference) DeepCopy() *SecretKeyReference { func (in *Security) DeepCopyInto(out *Security) { *out = *in in.Authentication.DeepCopyInto(&out.Authentication) - out.TLS = in.TLS + in.TLS.DeepCopyInto(&out.TLS) if in.Roles != nil { in, out := &in.Roles, &out.Roles *out = make([]CustomRole, len(*in)) @@ -416,7 +416,16 @@ func (in *StatefulSetSpecWrapper) DeepCopyInto(out *StatefulSetSpecWrapper) { func (in *TLS) DeepCopyInto(out *TLS) { *out = *in out.CertificateKeySecret = in.CertificateKeySecret - out.CaConfigMap = in.CaConfigMap + if in.CaCertificateSecret != nil { + in, out := &in.CaCertificateSecret, &out.CaCertificateSecret + *out = new(LocalObjectReference) + **out = **in + } + if in.CaConfigMap != nil { + in, out := &in.CaConfigMap, &out.CaConfigMap + *out = new(LocalObjectReference) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TLS. diff --git a/go.mod b/go.mod index 47c25e47c..338eb13f5 100644 --- a/go.mod +++ b/go.mod @@ -13,14 +13,13 @@ require ( github.com/stretchr/objx v0.3.0 github.com/stretchr/testify v1.7.0 github.com/xdg/stringprep v1.0.3 - go.mongodb.org/mongo-driver v1.7.1 + go.mongodb.org/mongo-driver v1.7.2 go.uber.org/zap v1.19.0 - golang.org/x/tools v0.1.1 // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 k8s.io/api v0.22.1 k8s.io/apimachinery v0.22.1 k8s.io/client-go v0.22.1 - sigs.k8s.io/controller-runtime v0.9.6 + sigs.k8s.io/controller-runtime v0.10.0 sigs.k8s.io/yaml v1.2.0 ) diff --git a/go.sum b/go.sum index 7936a975a..56eec4741 100644 --- a/go.sum +++ b/go.sum @@ -22,15 +22,13 @@ cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiy cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-autorest v13.3.2+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest/autorest v0.11.12/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= -github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= -github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= @@ -46,10 +44,12 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -61,6 +61,8 @@ github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJm github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= +github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= @@ -69,18 +71,19 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= +github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= +github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -89,24 +92,28 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= -github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.11.0+incompatible h1:glyUF9yIYtMHzn8xaKw5rMhdWcwsYV8dZHIq5567/xs= github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -123,14 +130,12 @@ github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/zapr v0.4.0 h1:uc1uML3hRYL9/ZZPdgHS/n8Nzo+eaYL/Efxkkamf7OM= github.com/go-logr/zapr v0.4.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= -github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= -github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= -github.com/go-openapi/spec v0.19.5/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk= -github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= @@ -158,12 +163,13 @@ github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWe github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -189,6 +195,7 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= @@ -215,25 +222,22 @@ github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw= github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= @@ -252,7 +256,6 @@ github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= @@ -267,9 +270,10 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -281,6 +285,7 @@ github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8 github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= @@ -293,20 +298,17 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= +github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= @@ -321,7 +323,7 @@ github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:F github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= -github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= +github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -340,20 +342,18 @@ github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.14.0 h1:ep6kpPVwmr/nTbklSx2nrLNSIO62DoYAhnPNIMhK8gI= -github.com/onsi/gomega v1.14.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= +github.com/onsi/gomega v1.15.0 h1:WjP/FQ/sk43MRmnEcT+MlDw2TFvkrXlprrPST/IudjU= +github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= @@ -387,11 +387,11 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -405,9 +405,11 @@ github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMB github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= @@ -415,10 +417,9 @@ github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkU github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA= github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= +github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= @@ -426,7 +427,6 @@ github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5q github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.3.0 h1:NGXK3lHquSN08v5vWalVI/L8XU9hdzE/G6xsrze47As= github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -439,9 +439,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.0.2 h1:akYIkZ28e6A96dkWNJQu3nmCzH3YfwMPQExUYDaRv7w= @@ -457,16 +456,32 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= -go.mongodb.org/mongo-driver v1.7.1 h1:jwqTeEM3x6L9xDXrCxN0Hbg7vdGfPBOTIkr0+/LYZDA= -go.mongodb.org/mongo-driver v1.7.1/go.mod h1:Q4oFMbo1+MSNqICAdYMlC/zSTrwCogR4R8NzkI+yfU8= +go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= +go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= +go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= +go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0= +go.etcd.io/etcd/pkg/v3 v3.5.0/go.mod h1:UzJGatBQ1lXChBkQF0AuAtkRQMYnHubxAEYIrC3MSsE= +go.etcd.io/etcd/raft/v3 v3.5.0/go.mod h1:UFOHSIvO/nKwd4lhkwabrTD3cqW5yVyYYf/KlD00Szc= +go.etcd.io/etcd/server/v3 v3.5.0/go.mod h1:3Ah5ruV+M+7RZr0+Y/5mNLwC+eQlni+mQmOVdCRJoS4= +go.mongodb.org/mongo-driver v1.7.2 h1:pFttQyIiJUHEn50YfZgC9ECjITMT44oiN36uArf/OFg= +go.mongodb.org/mongo-driver v1.7.2/go.mod h1:Q4oFMbo1+MSNqICAdYMlC/zSTrwCogR4R8NzkI+yfU8= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4= +go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= +go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM= +go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= +go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= +go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= +go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE= +go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE= +go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -476,7 +491,7 @@ go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/ go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= +go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.19.0 h1:mZQZefskPPCMIBCSEH0v2/iUqqLrYtaeqwD6FUGUnFE= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -485,7 +500,6 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -513,8 +527,9 @@ golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -523,7 +538,6 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -542,7 +556,6 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -553,8 +566,9 @@ golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210520170846-37e1c6afe023 h1:ADo5wSpq2gqaCGQWzk7S5vd//0iyyLeAratkEoG5dLE= @@ -593,10 +607,8 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -616,19 +628,19 @@ golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2 h1:c8PlLMqBbOHoqtjteWm5/kbe6rNY2pbRfbIMVnepueo= +golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE= @@ -638,11 +650,9 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -651,6 +661,7 @@ golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2M golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -665,7 +676,6 @@ golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= @@ -693,9 +703,8 @@ golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.1 h1:wGiQel/hW0NnEkJUk8lbzkX2gFJU6PFxf1v5OlCfuOs= -golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.2 h1:kRBLX7v7Af8W7Gdbbc908OJcdgtK8bOz9Uaj8/F1ACA= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -738,16 +747,24 @@ google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4 google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -767,7 +784,6 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= @@ -782,6 +798,7 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -800,44 +817,37 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.21.3/go.mod h1:hUgeYHUbBp23Ue4qdX9tR8/ANi/g3ehylAqDn9NWVOg= k8s.io/api v0.22.1 h1:ISu3tD/jRhYfSW8jI/Q1e+lRxkR7w9UwQEZ7FgslrwY= k8s.io/api v0.22.1/go.mod h1:bh13rkTp3F1XEaLGykbyRD2QaTTzPm0e/BMd8ptFONY= -k8s.io/apiextensions-apiserver v0.21.3 h1:+B6biyUWpqt41kz5x6peIsljlsuwvNAp/oFax/j2/aY= -k8s.io/apiextensions-apiserver v0.21.3/go.mod h1:kl6dap3Gd45+21Jnh6utCx8Z2xxLm8LGDkprcd+KbsE= -k8s.io/apimachinery v0.21.3/go.mod h1:H/IM+5vH9kZRNJ4l3x/fXP/5bOPJaVP/guptnZPeCFI= +k8s.io/apiextensions-apiserver v0.22.1 h1:YSJYzlFNFSfUle+yeEXX0lSQyLEoxoPJySRupepb0gE= +k8s.io/apiextensions-apiserver v0.22.1/go.mod h1:HeGmorjtRmRLE+Q8dJu6AYRoZccvCMsghwS8XTUYb2c= k8s.io/apimachinery v0.22.1 h1:DTARnyzmdHMz7bFWFDDm22AM4pLWTQECMpRTFu2d2OM= k8s.io/apimachinery v0.22.1/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0= -k8s.io/apiserver v0.21.3/go.mod h1:eDPWlZG6/cCCMj/JBcEpDoK+I+6i3r9GsChYBHSbAzU= -k8s.io/client-go v0.21.3/go.mod h1:+VPhCgTsaFmGILxR/7E1N0S+ryO010QBeNCv5JwRGYU= +k8s.io/apiserver v0.22.1/go.mod h1:2mcM6dzSt+XndzVQJX21Gx0/Klo7Aen7i0Ai6tIa400= k8s.io/client-go v0.22.1 h1:jW0ZSHi8wW260FvcXHkIa0NLxFBQszTlhiAVsU5mopw= k8s.io/client-go v0.22.1/go.mod h1:BquC5A4UOo4qVDUtoc04/+Nxp1MeHcVc1HJm1KmG8kk= -k8s.io/code-generator v0.21.3/go.mod h1:K3y0Bv9Cz2cOW2vXUrNZlFbflhuPvuadW6JdnN6gGKo= -k8s.io/component-base v0.21.3 h1:4WuuXY3Npa+iFfi2aDRiOz+anhNvRfye0859ZgfC5Og= -k8s.io/component-base v0.21.3/go.mod h1:kkuhtfEHeZM6LkX0saqSK8PbdO7A0HigUngmhhrwfGQ= +k8s.io/code-generator v0.22.1/go.mod h1:eV77Y09IopzeXOJzndrDyCI88UBok2h6WxAlBwpxa+o= +k8s.io/component-base v0.22.1 h1:SFqIXsEN3v3Kkr1bS6rstrs1wd45StJqbtgbQ4nRQdo= +k8s.io/component-base v0.22.1/go.mod h1:0D+Bl8rrnsPN9v0dyYvkqFfBeAd4u7n77ze+p8CMiPo= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/klog/v2 v2.9.0 h1:D7HV+n1V57XeZ0m6tdRkfknthUaM06VFbWldOFh8kzM= k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= -k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE= k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e h1:KLHHjkdQFomZy8+06csTWZ0m1343QqxZhR2LJ1OxCYM= k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= -k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210707171843-4b05e18ac7d9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20210722164352-7f3ee0f31471 h1:DnzUXII7sVg1FJ/4JX6YDRJfLNAC7idRatPwe07suiI= -k8s.io/utils v0.0.0-20210722164352-7f3ee0f31471/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20210802155522-efc7438f0176 h1:Mx0aa+SUAcNRQbs5jUzV8lkDlGFU8laZsY9jrcVX5SY= +k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.19/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= -sigs.k8s.io/controller-runtime v0.9.6 h1:EevVMlgUj4fC1NVM4+DB3iPkWkmGRNarA66neqv9Qew= -sigs.k8s.io/controller-runtime v0.9.6/go.mod h1:q6PpkM5vqQubEKUKOM6qr06oXGzOBcCby1DA9FbyZeA= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= +sigs.k8s.io/controller-runtime v0.10.0 h1:HgyZmMpjUOrtkaFtCnfxsR1bGRuFoAczSNbn2MoKj5U= +sigs.k8s.io/controller-runtime v0.10.0/go.mod h1:GCdh6kqV6IY4LK0JLwX0Zm6g233RtVGdb/f0+KSfprg= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.1.2 h1:Hr/htKFmJEbtMgS/UD0N+gtgctAqz81t3nu+sPzynno= sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= -sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= From b9487763041816f773d8dd01e1a5eb69dcf4a644 Mon Sep 17 00:00:00 2001 From: Ciprian Tibulca Date: Wed, 15 Sep 2021 11:02:19 +0100 Subject: [PATCH 396/790] CLOUDP-100089 make MongoDBCommunityStatus.Version optional for backwards compatibility (#718) --- api/v1/mongodbcommunity_types.go | 2 +- .../bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml | 1 - .../crds/mongodbcommunity.mongodb.com_mongodbcommunity.yaml | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/api/v1/mongodbcommunity_types.go b/api/v1/mongodbcommunity_types.go index 60e091af2..7c59b4d18 100644 --- a/api/v1/mongodbcommunity_types.go +++ b/api/v1/mongodbcommunity_types.go @@ -381,7 +381,7 @@ func ConvertAuthModeToAuthMechanism(authModeLabel AuthMode) string { type MongoDBCommunityStatus struct { MongoURI string `json:"mongoUri"` Phase Phase `json:"phase"` - Version string `json:"version"` + Version string `json:"version,omitempty"` CurrentStatefulSetReplicas int `json:"currentStatefulSetReplicas"` CurrentMongoDBMembers int `json:"currentMongoDBMembers"` diff --git a/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml b/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml index 7e7f03e12..ae01f98f8 100644 --- a/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml +++ b/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml @@ -340,7 +340,6 @@ spec: - currentStatefulSetReplicas - mongoUri - phase - - version type: object type: object served: true diff --git a/helm-chart/crds/mongodbcommunity.mongodb.com_mongodbcommunity.yaml b/helm-chart/crds/mongodbcommunity.mongodb.com_mongodbcommunity.yaml index 7e7f03e12..ae01f98f8 100644 --- a/helm-chart/crds/mongodbcommunity.mongodb.com_mongodbcommunity.yaml +++ b/helm-chart/crds/mongodbcommunity.mongodb.com_mongodbcommunity.yaml @@ -340,7 +340,6 @@ spec: - currentStatefulSetReplicas - mongoUri - phase - - version type: object type: object served: true From f2b3fd39a52d6d4563a31d61f95bf55d4958109b Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Mon, 20 Sep 2021 09:48:09 +0100 Subject: [PATCH 397/790] Added options to CreateVolumeFromConfigMap (#722) --- pkg/kube/statefulset/statefulset.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pkg/kube/statefulset/statefulset.go b/pkg/kube/statefulset/statefulset.go index 7e9fbab63..105cd00a4 100644 --- a/pkg/kube/statefulset/statefulset.go +++ b/pkg/kube/statefulset/statefulset.go @@ -82,8 +82,8 @@ type VolumeMountData struct { ReadOnly bool } -func CreateVolumeFromConfigMap(name, sourceName string) corev1.Volume { - return corev1.Volume{ +func CreateVolumeFromConfigMap(name, sourceName string, options ...func(v *corev1.Volume)) corev1.Volume { + volume := &corev1.Volume{ Name: name, VolumeSource: corev1.VolumeSource{ ConfigMap: &corev1.ConfigMapVolumeSource{ @@ -93,6 +93,11 @@ func CreateVolumeFromConfigMap(name, sourceName string) corev1.Volume { }, }, } + + for _, option := range options { + option(volume) + } + return *volume } func CreateVolumeFromSecret(name, sourceName string, options ...func(v *corev1.Volume)) corev1.Volume { From 299e3519c8ad1cd820a36e6d6d4f6c21f675e2e6 Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Tue, 21 Sep 2021 11:21:27 +0100 Subject: [PATCH 398/790] Added secret exists method (#728) --- pkg/helm/helm.go | 2 +- pkg/kube/configmap/configmap.go | 13 +++++++++++++ pkg/kube/configmap/configmap_test.go | 4 ++-- pkg/kube/secret/secret.go | 13 +++++++++++++ pkg/kube/secret/secret_test.go | 4 ++-- .../replica_set_cross_namespace_deploy_test.go | 7 +++---- test/e2e/util/wait/wait.go | 2 -- 7 files changed, 34 insertions(+), 11 deletions(-) diff --git a/pkg/helm/helm.go b/pkg/helm/helm.go index 3967181f8..8a65d8364 100644 --- a/pkg/helm/helm.go +++ b/pkg/helm/helm.go @@ -31,7 +31,7 @@ func executeHelmCommand(args []string, messagePredicate func(string) bool) error cmd := exec.Command("helm", args...) output, err := cmd.CombinedOutput() if err != nil { - if messagePredicate != nil && messagePredicate(string(output)){ + if messagePredicate != nil && messagePredicate(string(output)) { return nil } return fmt.Errorf("error executing command: %s %s", err, output) diff --git a/pkg/kube/configmap/configmap.go b/pkg/kube/configmap/configmap.go index 9e9444e40..5e5c2d2f2 100644 --- a/pkg/kube/configmap/configmap.go +++ b/pkg/kube/configmap/configmap.go @@ -129,3 +129,16 @@ func ReadFileLikeField(getter Getter, objectKey client.ObjectKey, externalKey st } return value, nil } + +// Exists return whether a configmap with the given namespaced name exists +func Exists(cmGetter Getter, nsName types.NamespacedName) (bool, error) { + _, err := cmGetter.GetConfigMap(nsName) + + if err != nil { + if apiErrors.IsNotFound(err) { + return false, nil + } + return false, err + } + return true, nil +} diff --git a/pkg/kube/configmap/configmap_test.go b/pkg/kube/configmap/configmap_test.go index 2e53864f9..b685180ad 100644 --- a/pkg/kube/configmap/configmap_test.go +++ b/pkg/kube/configmap/configmap_test.go @@ -124,13 +124,13 @@ func (c configMapGetUpdater) GetConfigMap(objectKey client.ObjectKey) (corev1.Co return corev1.ConfigMap{}, notFoundError() } -func (c configMapGetUpdater) UpdateConfigMap(cm corev1.ConfigMap) error { +func (c *configMapGetUpdater) UpdateConfigMap(cm corev1.ConfigMap) error { c.cm = cm return nil } func newGetUpdater(cm corev1.ConfigMap) GetUpdater { - return configMapGetUpdater{ + return &configMapGetUpdater{ cm: cm, } } diff --git a/pkg/kube/secret/secret.go b/pkg/kube/secret/secret.go index 52bdebcf2..2d14088b7 100644 --- a/pkg/kube/secret/secret.go +++ b/pkg/kube/secret/secret.go @@ -150,3 +150,16 @@ func CopySecret(fromClient Getter, toClient GetUpdateCreator, sourceSecretNsName return CreateOrUpdate(toClient, secretCopy) } + +// Exists return whether a secret with the given namespaced name exists +func Exists(secretGetter Getter, nsName types.NamespacedName) (bool, error) { + _, err := secretGetter.GetSecret(nsName) + + if err != nil { + if apiErrors.IsNotFound(err) { + return false, nil + } + return false, err + } + return true, nil +} diff --git a/pkg/kube/secret/secret_test.go b/pkg/kube/secret/secret_test.go index 7d9a7dcfb..3fc706acd 100644 --- a/pkg/kube/secret/secret_test.go +++ b/pkg/kube/secret/secret_test.go @@ -102,13 +102,13 @@ func (c secretGetUpdater) GetSecret(objectKey client.ObjectKey) (corev1.Secret, return corev1.Secret{}, notFoundError() } -func (c secretGetUpdater) UpdateSecret(s corev1.Secret) error { +func (c *secretGetUpdater) UpdateSecret(s corev1.Secret) error { c.secret = s return nil } func newGetUpdater(s corev1.Secret) GetUpdater { - return secretGetUpdater{ + return &secretGetUpdater{ secret: s, } } diff --git a/test/e2e/replica_set_cross_namespace_deploy/replica_set_cross_namespace_deploy_test.go b/test/e2e/replica_set_cross_namespace_deploy/replica_set_cross_namespace_deploy_test.go index 05293bcdc..afd961ee1 100644 --- a/test/e2e/replica_set_cross_namespace_deploy/replica_set_cross_namespace_deploy_test.go +++ b/test/e2e/replica_set_cross_namespace_deploy/replica_set_cross_namespace_deploy_test.go @@ -3,11 +3,12 @@ package replica_set_cross_namespace_deploy import ( "context" "fmt" + "os" + "testing" + corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/types" - "os" - "testing" "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/generate" . "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/util/mongotester" @@ -64,7 +65,6 @@ func TestCrossNamespaceDeploy(t *testing.T) { t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 1)) } - // createDatabaseServiceAccountRoleAndRoleBinding creates the ServiceAccount, Role and RoleBinding required // for the database StatefulSet in the other namespace. func createDatabaseServiceAccountRoleAndRoleBinding(t *testing.T, namespace string) error { @@ -111,4 +111,3 @@ func createDatabaseServiceAccountRoleAndRoleBinding(t *testing.T, namespace stri } return nil } - diff --git a/test/e2e/util/wait/wait.go b/test/e2e/util/wait/wait.go index 0bd256fb1..8a8a84c11 100644 --- a/test/e2e/util/wait/wait.go +++ b/test/e2e/util/wait/wait.go @@ -59,7 +59,6 @@ func waitForMongoDBCondition(mdb *mdbv1.MongoDBCommunity, retryInterval, timeout }) } - // ForDeploymentToExist waits until a Deployment of the given name exists // using the provided retryInterval and timeout func ForDeploymentToExist(deployName string, retryInterval, timeout time.Duration, namespace string) (appsv1.Deployment, error) { @@ -67,7 +66,6 @@ func ForDeploymentToExist(deployName string, retryInterval, timeout time.Duratio return deploy, waitForRuntimeObjectToExist(deployName, retryInterval, timeout, &deploy, namespace) } - // ForStatefulSetToExist waits until a StatefulSet of the given name exists // using the provided retryInterval and timeout func ForStatefulSetToExist(stsName string, retryInterval, timeout time.Duration, namespace string) (appsv1.StatefulSet, error) { From 11b19868c2a5e181b3286f55bde2217aad5cd596 Mon Sep 17 00:00:00 2001 From: Rajdeep Das Date: Tue, 21 Sep 2021 14:22:01 +0200 Subject: [PATCH 399/790] Bump Go dependencies (#729) --- go.mod | 8 ++++---- go.sum | 21 +++++++++++++-------- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 338eb13f5..b977b6943 100644 --- a/go.mod +++ b/go.mod @@ -14,11 +14,11 @@ require ( github.com/stretchr/testify v1.7.0 github.com/xdg/stringprep v1.0.3 go.mongodb.org/mongo-driver v1.7.2 - go.uber.org/zap v1.19.0 + go.uber.org/zap v1.19.1 gopkg.in/natefinch/lumberjack.v2 v2.0.0 - k8s.io/api v0.22.1 - k8s.io/apimachinery v0.22.1 - k8s.io/client-go v0.22.1 + k8s.io/api v0.22.2 + k8s.io/apimachinery v0.22.2 + k8s.io/client-go v0.22.2 sigs.k8s.io/controller-runtime v0.10.0 sigs.k8s.io/yaml v1.2.0 ) diff --git a/go.sum b/go.sum index 56eec4741..e4889331c 100644 --- a/go.sum +++ b/go.sum @@ -485,15 +485,17 @@ go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqe go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.1.11-0.20210813005559-691160354723 h1:sHOAIxRGBp443oHZIPB+HsUGaksVCXVQENPxwTfQdH4= +go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= -go.uber.org/zap v1.19.0 h1:mZQZefskPPCMIBCSEH0v2/iUqqLrYtaeqwD6FUGUnFE= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= +go.uber.org/zap v1.19.1 h1:ue41HOKd1vGURxrmeKIgELGb3jPW9DMUDGtsinblHwI= +go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -528,7 +530,6 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= @@ -703,8 +704,8 @@ golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.2 h1:kRBLX7v7Af8W7Gdbbc908OJcdgtK8bOz9Uaj8/F1ACA= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -817,15 +818,18 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.22.1 h1:ISu3tD/jRhYfSW8jI/Q1e+lRxkR7w9UwQEZ7FgslrwY= k8s.io/api v0.22.1/go.mod h1:bh13rkTp3F1XEaLGykbyRD2QaTTzPm0e/BMd8ptFONY= +k8s.io/api v0.22.2 h1:M8ZzAD0V6725Fjg53fKeTJxGsJvRbk4TEm/fexHMtfw= +k8s.io/api v0.22.2/go.mod h1:y3ydYpLJAaDI+BbSe2xmGcqxiWHmWjkEeIbiwHvnPR8= k8s.io/apiextensions-apiserver v0.22.1 h1:YSJYzlFNFSfUle+yeEXX0lSQyLEoxoPJySRupepb0gE= k8s.io/apiextensions-apiserver v0.22.1/go.mod h1:HeGmorjtRmRLE+Q8dJu6AYRoZccvCMsghwS8XTUYb2c= -k8s.io/apimachinery v0.22.1 h1:DTARnyzmdHMz7bFWFDDm22AM4pLWTQECMpRTFu2d2OM= k8s.io/apimachinery v0.22.1/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0= +k8s.io/apimachinery v0.22.2 h1:ejz6y/zNma8clPVfNDLnPbleBo6MpoFy/HBiBqCouVk= +k8s.io/apimachinery v0.22.2/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0= k8s.io/apiserver v0.22.1/go.mod h1:2mcM6dzSt+XndzVQJX21Gx0/Klo7Aen7i0Ai6tIa400= -k8s.io/client-go v0.22.1 h1:jW0ZSHi8wW260FvcXHkIa0NLxFBQszTlhiAVsU5mopw= k8s.io/client-go v0.22.1/go.mod h1:BquC5A4UOo4qVDUtoc04/+Nxp1MeHcVc1HJm1KmG8kk= +k8s.io/client-go v0.22.2 h1:DaSQgs02aCC1QcwUdkKZWOeaVsQjYvWv8ZazcZ6JcHc= +k8s.io/client-go v0.22.2/go.mod h1:sAlhrkVDf50ZHx6z4K0S40wISNTarf1r800F+RlCF6U= k8s.io/code-generator v0.22.1/go.mod h1:eV77Y09IopzeXOJzndrDyCI88UBok2h6WxAlBwpxa+o= k8s.io/component-base v0.22.1 h1:SFqIXsEN3v3Kkr1bS6rstrs1wd45StJqbtgbQ4nRQdo= k8s.io/component-base v0.22.1/go.mod h1:0D+Bl8rrnsPN9v0dyYvkqFfBeAd4u7n77ze+p8CMiPo= @@ -838,8 +842,9 @@ k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e h1:KLHHjkdQFomZy8+06csTWZ0m1343QqxZhR2LJ1OxCYM= k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= k8s.io/utils v0.0.0-20210707171843-4b05e18ac7d9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20210802155522-efc7438f0176 h1:Mx0aa+SUAcNRQbs5jUzV8lkDlGFU8laZsY9jrcVX5SY= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a h1:8dYfu/Fc9Gz2rNJKB9IQRGgQOh2clmRzNIPPY1xLY5g= +k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= From 8a3900c516e005714e85fa87989948f5a7994a21 Mon Sep 17 00:00:00 2001 From: Hamed Momeni <537522+2hamed@users.noreply.github.com> Date: Fri, 24 Sep 2021 12:00:57 +0200 Subject: [PATCH 400/790] fix deployment scopes link in install doc (#723) --- docs/install-upgrade.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/install-upgrade.md b/docs/install-upgrade.md index 84b929c68..15a42d5a5 100644 --- a/docs/install-upgrade.md +++ b/docs/install-upgrade.md @@ -22,7 +22,7 @@ Before you install the MongoDB Community Kubernetes Operator, you must: ``` git clone https://github.com/mongodb/mongodb-kubernetes-operator.git ``` -4. **Optional** Review the possible Operator [deployment scopes](#understand-mongodb-community-operator-deployment-scopes) and configure the Operator to watch other namespaces. +4. **Optional** Review the possible Operator [deployment scopes](#understand-deployment-scopes) and configure the Operator to watch other namespaces. 5. **Optional** Configure the [MongoDB Docker image or container registry](#configure-the-mongodb-docker-image-or-container-registry). ### Understand Deployment Scopes From 4131c4a8519712acd05499c95b80ae35459d2ca9 Mon Sep 17 00:00:00 2001 From: manuelfo Date: Fri, 24 Sep 2021 11:01:30 +0100 Subject: [PATCH 401/790] Update contributing.md (#719) Fix typo --- docs/contributing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/contributing.md b/docs/contributing.md index 2bf3a10a6..676a08e43 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -5,7 +5,7 @@ from a high perspective how everything works together. After our experience building the [Enterprise MongoDB Kubernetes Operator](https://github.com/mongodb/mongodb-enterprise-kubernetes), we have -realized that is is very important to have a clean environment to work, and as such we have +realized that it is very important to have a clean environment to work, and as such we have adopted a strategy that makes it easier for everyone to contribute. This strategy is based on using From 4b2e3128e8943fb49440eb1f45e9ff14379cf8af Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Fri, 24 Sep 2021 11:51:59 +0100 Subject: [PATCH 402/790] Remove non-isolated-pod check (#732) --- .github/config_files/config_lint.yaml | 1 + .github/config_files/config_lint_clusterwide.yaml | 1 + .github/config_files/config_lint_openshift.yaml | 1 + 3 files changed, 3 insertions(+) diff --git a/.github/config_files/config_lint.yaml b/.github/config_files/config_lint.yaml index 435bc8a7b..b1f1e5e4c 100644 --- a/.github/config_files/config_lint.yaml +++ b/.github/config_files/config_lint.yaml @@ -12,3 +12,4 @@ checks: - "no-liveness-probe" - "no-readiness-probe" - "use-namespace" + - "non-isolated-pod" diff --git a/.github/config_files/config_lint_clusterwide.yaml b/.github/config_files/config_lint_clusterwide.yaml index b69b5147d..44b93e5d2 100644 --- a/.github/config_files/config_lint_clusterwide.yaml +++ b/.github/config_files/config_lint_clusterwide.yaml @@ -16,3 +16,4 @@ checks: - "use-namespace" - "access-to-secrets" - "access-to-create-pods" + - "non-isolated-pod" diff --git a/.github/config_files/config_lint_openshift.yaml b/.github/config_files/config_lint_openshift.yaml index 34ff6e440..74cef6dc1 100644 --- a/.github/config_files/config_lint_openshift.yaml +++ b/.github/config_files/config_lint_openshift.yaml @@ -15,3 +15,4 @@ checks: - "run-as-non-root" - "no-read-only-root-fs" - "use-namespace" + - "non-isolated-pod" From 0344458addd7f41c48cacf6e028b39e4e80f72e0 Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Tue, 28 Sep 2021 16:46:00 +0100 Subject: [PATCH 403/790] Use new kubelinter action version and pin kubelinter (#733) --- .github/config_files/config_lint.yaml | 1 - .github/config_files/config_lint_clusterwide.yaml | 1 - .github/config_files/config_lint_openshift.yaml | 1 - .github/workflows/kubelinter-check.yml | 12 ++++++++---- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/config_files/config_lint.yaml b/.github/config_files/config_lint.yaml index b1f1e5e4c..435bc8a7b 100644 --- a/.github/config_files/config_lint.yaml +++ b/.github/config_files/config_lint.yaml @@ -12,4 +12,3 @@ checks: - "no-liveness-probe" - "no-readiness-probe" - "use-namespace" - - "non-isolated-pod" diff --git a/.github/config_files/config_lint_clusterwide.yaml b/.github/config_files/config_lint_clusterwide.yaml index 44b93e5d2..b69b5147d 100644 --- a/.github/config_files/config_lint_clusterwide.yaml +++ b/.github/config_files/config_lint_clusterwide.yaml @@ -16,4 +16,3 @@ checks: - "use-namespace" - "access-to-secrets" - "access-to-create-pods" - - "non-isolated-pod" diff --git a/.github/config_files/config_lint_openshift.yaml b/.github/config_files/config_lint_openshift.yaml index 74cef6dc1..34ff6e440 100644 --- a/.github/config_files/config_lint_openshift.yaml +++ b/.github/config_files/config_lint_openshift.yaml @@ -15,4 +15,3 @@ checks: - "run-as-non-root" - "no-read-only-root-fs" - "use-namespace" - - "non-isolated-pod" diff --git a/.github/workflows/kubelinter-check.yml b/.github/workflows/kubelinter-check.yml index a361eb59f..b4ef431fd 100644 --- a/.github/workflows/kubelinter-check.yml +++ b/.github/workflows/kubelinter-check.yml @@ -20,25 +20,29 @@ jobs: uses: actions/checkout@v2 - name: Scan directory ./deploy/clusterwide/ with kube-linter - uses: stackrox/kube-linter-action@v1 + uses: stackrox/kube-linter-action@v1.0.3 with: directory: deploy/clusterwide config: ${GITHUB_WORKSPACE}/.github/config_files/config_lint_clusterwide.yaml + version: "48442350" # Note: This is the id for release 0.2.3 returned from api.github.com/repos/stackrox/kube-linter/releases - name: Scan directory ./deploy/openshift/ with kube-linter - uses: stackrox/kube-linter-action@v1 + uses: stackrox/kube-linter-action@v1.0.3 with: directory: deploy/openshift config: ${GITHUB_WORKSPACE}/.github/config_files/config_lint_openshift.yaml + version: "48442350" - name: Scan directory ./config/manager/ with kube-linter - uses: stackrox/kube-linter-action@v1 + uses: stackrox/kube-linter-action@v1.0.3 with: directory: config/manager/manager.yaml config: ${GITHUB_WORKSPACE}/.github/config_files/config_lint.yaml + version: "48442350" - name: Scan directory ./config/samples/ with kube-linter - uses: stackrox/kube-linter-action@v1 + uses: stackrox/kube-linter-action@v1.0.3 with: directory: config/samples config: ${GITHUB_WORKSPACE}/.github/config_files/config_lint.yaml + version: "48442350" From ea694550860a7b724aa2ccc3bbe2a7806915ca70 Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Tue, 28 Sep 2021 16:46:16 +0100 Subject: [PATCH 404/790] Changed WithAnnotations to merge instead of setting (#736) --- pkg/kube/statefulset/statefulset.go | 2 +- pkg/kube/statefulset/statefulset_test.go | 30 ++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/pkg/kube/statefulset/statefulset.go b/pkg/kube/statefulset/statefulset.go index 105cd00a4..771908f5d 100644 --- a/pkg/kube/statefulset/statefulset.go +++ b/pkg/kube/statefulset/statefulset.go @@ -219,7 +219,7 @@ func WithLabels(labels map[string]string) Modification { func WithAnnotations(annotations map[string]string) Modification { return func(set *appsv1.StatefulSet) { - set.Annotations = copyMap(annotations) + set.Annotations = merge.StringToStringMap(set.Annotations, annotations) } } diff --git a/pkg/kube/statefulset/statefulset_test.go b/pkg/kube/statefulset/statefulset_test.go index 37e331252..bbd4c431b 100644 --- a/pkg/kube/statefulset/statefulset_test.go +++ b/pkg/kube/statefulset/statefulset_test.go @@ -205,3 +205,33 @@ func TestCreateVolumeMountWithMultipleOptions(t *testing.T) { assert.Equal(t, mount.SubPath, "our-subpath") assert.True(t, mount.ReadOnly) } + +func TestWithAnnotations(t *testing.T) { + sts, err := defaultStatefulSetBuilder().Build() + assert.NoError(t, err) + + assert.Len(t, sts.Annotations, 0) + + // Test that it works when there are no annotations + WithAnnotations(map[string]string{ + "foo": "bar", + })(&sts) + assert.Equal(t, "bar", sts.Annotations["foo"]) + + // test that WithAnnotations merges the maps + WithAnnotations(map[string]string{ + "bar": "baz", + })(&sts) + assert.Equal(t, "bar", sts.Annotations["foo"]) + assert.Equal(t, "baz", sts.Annotations["bar"]) + + // Test that we can override a key + WithAnnotations(map[string]string{ + "foo": "baz", + })(&sts) + assert.Equal(t, "baz", sts.Annotations["foo"]) + + // handles nil values gracefully + WithAnnotations(nil)(&sts) + assert.Len(t, sts.Annotations, 2) +} From c65a0a53047c3fef9321f4f0bdf1fd00cb634b13 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 29 Sep 2021 16:25:47 +0100 Subject: [PATCH 405/790] Bump sigs.k8s.io/yaml from 1.2.0 to 1.3.0 (#735) Bumps [sigs.k8s.io/yaml](https://github.com/kubernetes-sigs/yaml) from 1.2.0 to 1.3.0. - [Release notes](https://github.com/kubernetes-sigs/yaml/releases) - [Changelog](https://github.com/kubernetes-sigs/yaml/blob/master/RELEASE.md) - [Commits](https://github.com/kubernetes-sigs/yaml/compare/v1.2.0...v1.3.0) --- updated-dependencies: - dependency-name: sigs.k8s.io/yaml dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index b977b6943..9d87a4994 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( k8s.io/apimachinery v0.22.2 k8s.io/client-go v0.22.2 sigs.k8s.io/controller-runtime v0.10.0 - sigs.k8s.io/yaml v1.2.0 + sigs.k8s.io/yaml v1.3.0 ) replace github.com/docker/docker => github.com/moby/moby v17.12.0-ce-rc1.0.20200309214505-aa6a9891b09c+incompatible // Required by Helm diff --git a/go.sum b/go.sum index e4889331c..81b53902b 100644 --- a/go.sum +++ b/go.sum @@ -854,5 +854,6 @@ sigs.k8s.io/controller-runtime v0.10.0/go.mod h1:GCdh6kqV6IY4LK0JLwX0Zm6g233RtVG sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.1.2 h1:Hr/htKFmJEbtMgS/UD0N+gtgctAqz81t3nu+sPzynno= sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= -sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= From 48ae26a1e96266cf337a4fe3ac1093cc209cb2d2 Mon Sep 17 00:00:00 2001 From: Ciprian Tibulca Date: Wed, 29 Sep 2021 16:37:59 +0100 Subject: [PATCH 406/790] CLOUDP-99765 use cert-manager to generate tls certificates for e2e tests (#709) * CLOUDP-99765 use cert-manager to generate tls certificates for e2e tests * CLOUDP-99765 cert-manager config * CLOUDP-99765 update TLS rotate e2e test * CLOUDP-99765 remove e2e tester global state usage * CLOUDP-99765 fix replica_set_tls_recreate_mdbc test and include it with replica_set_tls_rotate in PR validation * CLOUDP-99765 use configured ns in cert-manager dns * CLOUDP-99765 update resources namespace used by e2e tests * CLOUDP-99765 PR comments --- .action_templates/jobs/tests.yaml | 12 +- .github/workflows/e2e-fork.yml | 12 +- .github/workflows/e2e.yml | 12 +- Makefile | 3 +- api/v1/mongodbcommunity_types.go | 4 +- deploy/e2e/role.yaml | 182 ++++++++++++++++++ deploy/e2e/role_binding.yaml | 2 +- deploy/e2e/service_account.yaml | 5 +- docs/contributing.md | 4 +- .../mongodbcommunity_cr_with_tls.yaml | 10 +- helm-chart/values.yaml | 3 + pkg/helm/helm.go | 20 +- scripts/dev/e2e.py | 6 +- test/e2e/e2eutil.go | 6 +- test/e2e/mongodbtests/mongodbtests.go | 10 + .../replica_set_tls/replica_set_tls_test.go | 24 ++- .../replica_set_pem_file_test.go | 24 ++- .../replica_set_tls_recreate_mdbc_test.go | 27 ++- .../replica_set_tls_rotate_test.go | 26 +-- .../replica_set_tls_upgrade_test.go | 22 +-- test/e2e/setup/setup.go | 128 ++++++------ test/e2e/setup/test_config.go | 51 +++-- test/e2e/tlstests/tlstests.go | 36 ++-- test/e2e/util/mongotester/mongotester.go | 82 ++++---- test/e2e/util/wait/wait.go | 31 ++- 25 files changed, 489 insertions(+), 253 deletions(-) diff --git a/.action_templates/jobs/tests.yaml b/.action_templates/jobs/tests.yaml index 037749fab..8cd5f1137 100644 --- a/.action_templates/jobs/tests.yaml +++ b/.action_templates/jobs/tests.yaml @@ -27,6 +27,10 @@ tests: distro: ubuntu - test-name: replica_set_tls_pem_file distro: ubuntu + - test-name: replica_set_tls_recreate_mdbc + distro: ubuntu + - test-name: replica_set_tls_rotate + distro: ubuntu - test-name: replica_set_tls_upgrade distro: ubuntu - test-name: statefulset_arbitrary_config @@ -65,10 +69,14 @@ tests: distro: ubi - test-name: replica_set_tls distro: ubi - - test-name: replica_set_tls_upgrade - distro: ubi - test-name: replica_set_tls_pem_file distro: ubi + - test-name: replica_set_tls_recreate_mdbc + distro: ubi + - test-name: replica_set_tls_rotate + distro: ubi + - test-name: replica_set_tls_upgrade + distro: ubi - test-name: statefulset_arbitrary_config distro: ubi - test-name: statefulset_arbitrary_config_update diff --git a/.github/workflows/e2e-fork.yml b/.github/workflows/e2e-fork.yml index 0e2f6b0b9..e5ec665ca 100644 --- a/.github/workflows/e2e-fork.yml +++ b/.github/workflows/e2e-fork.yml @@ -109,6 +109,10 @@ jobs: distro: ubuntu - test-name: replica_set_tls_pem_file distro: ubuntu + - test-name: replica_set_tls_recreate_mdbc + distro: ubuntu + - test-name: replica_set_tls_rotate + distro: ubuntu - test-name: replica_set_tls_upgrade distro: ubuntu - test-name: statefulset_arbitrary_config @@ -147,10 +151,14 @@ jobs: distro: ubi - test-name: replica_set_tls distro: ubi - - test-name: replica_set_tls_upgrade - distro: ubi - test-name: replica_set_tls_pem_file distro: ubi + - test-name: replica_set_tls_recreate_mdbc + distro: ubi + - test-name: replica_set_tls_rotate + distro: ubi + - test-name: replica_set_tls_upgrade + distro: ubi - test-name: statefulset_arbitrary_config distro: ubi - test-name: statefulset_arbitrary_config_update diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index d00ca1800..a63df064d 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -114,6 +114,10 @@ jobs: distro: ubuntu - test-name: replica_set_tls_pem_file distro: ubuntu + - test-name: replica_set_tls_recreate_mdbc + distro: ubuntu + - test-name: replica_set_tls_rotate + distro: ubuntu - test-name: replica_set_tls_upgrade distro: ubuntu - test-name: statefulset_arbitrary_config @@ -152,10 +156,14 @@ jobs: distro: ubi - test-name: replica_set_tls distro: ubi - - test-name: replica_set_tls_upgrade - distro: ubi - test-name: replica_set_tls_pem_file distro: ubi + - test-name: replica_set_tls_recreate_mdbc + distro: ubi + - test-name: replica_set_tls_rotate + distro: ubi + - test-name: replica_set_tls_upgrade + distro: ubi - test-name: statefulset_arbitrary_config distro: ubi - test-name: statefulset_arbitrary_config_update diff --git a/Makefile b/Makefile index 8f532f1ba..f39ece048 100644 --- a/Makefile +++ b/Makefile @@ -15,6 +15,7 @@ DOCKERFILE ?= operator # Produce CRDs that work back to Kubernetes 1.11 (no version conversion) CRD_OPTIONS ?= "crd:trivialVersions=true,crdVersions=v1" RELEASE_NAME_HELM ?= mongodb-kubernetes-operator +TEST_NAMESPACE ?= mongodb # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) ifeq (,$(shell go env GOBIN)) @@ -115,7 +116,7 @@ e2e-gh: scripts/dev/run_e2e_gh.sh $(test) cleanup-e2e: - kubectl delete mdbc,all -l e2e-test=true | true + kubectl delete mdbc,all -l e2e-test=true -n ${TEST_NAMESPACE} | true # Generate code generate: controller-gen diff --git a/api/v1/mongodbcommunity_types.go b/api/v1/mongodbcommunity_types.go index 7c59b4d18..2070ae84e 100644 --- a/api/v1/mongodbcommunity_types.go +++ b/api/v1/mongodbcommunity_types.go @@ -329,13 +329,13 @@ type TLS struct { // CaCertificateSecret is a reference to a Secret containing the certificate for the CA which signed the server certificates // The certificate is expected to be available under the key "ca.crt" // +optional - CaCertificateSecret *LocalObjectReference `json:"caCertificateSecretRef"` + CaCertificateSecret *LocalObjectReference `json:"caCertificateSecretRef,omitempty"` // CaConfigMap is a reference to a ConfigMap containing the certificate for the CA which signed the server certificates // The certificate is expected to be available under the key "ca.crt" // This field is ignored when CaCertificateSecretRef is configured // +optional - CaConfigMap *LocalObjectReference `json:"caConfigMapRef"` + CaConfigMap *LocalObjectReference `json:"caConfigMapRef,omitempty"` } // LocalObjectReference is a reference to another Kubernetes object by name. diff --git a/deploy/e2e/role.yaml b/deploy/e2e/role.yaml index f420945c7..ceb65bafa 100644 --- a/deploy/e2e/role.yaml +++ b/deploy/e2e/role.yaml @@ -104,3 +104,185 @@ rules: - patch - update - watch +# needed for cert-manager integration +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - list + - watch + - create + - delete + - update +- apiGroups: + - admissionregistration.k8s.io + resources: + - mutatingwebhookconfigurations + - validatingwebhookconfigurations + verbs: + - get + - list + - watch + - create + - delete + - update +- apiGroups: + - acme.cert-manager.io + resources: + - challenges + - challenges/finalizers + - challenges/status + - orders + - orders/finalizers + - orders/status + verbs: + - create + - delete + - update + - get + - list + - watch + - patch + - deletecollection +- apiGroups: + - cert-manager.io + resources: + - clusterissuers + - clusterissuers/status + - issuers + - issuers/status + - certificates + - certificaterequests + - certificaterequests/finalizers + - certificaterequests/status + - certificates/finalizers + - certificates/status + verbs: + - get + - list + - watch + - create + - update + - delete + - patch + - deletecollection +- apiGroups: + - cert-manager.io + resources: + - signers + resourceNames: + - clusterissuers.cert-manager.io/* + - issuers.cert-manager.io/* + verbs: + - approve +- apiGroups: + - networking.k8s.io + resources: + - ingresses + - ingresses/finalizers + verbs: + - get + - list + - watch + - create + - delete + - update +- apiGroups: + - networking.x-k8s.io + resources: + - httproutes + - gateways + - gateways/finalizers + - httproutes/finalizers + verbs: + - get + - list + - watch + - create + - delete + - update +- apiGroups: + - route.openshift.io + resources: + - routes/custom-host + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +- apiGroups: + - apiregistration.k8s.io + resources: + - apiservices + verbs: + - get + - list + - watch + - create + - delete + - update +- apiGroups: + - auditregistration.k8s.io + resources: + - auditsinks + verbs: + - get + - list + - watch + - create + - delete + - update +- apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests + - certificatesigningrequests/status + verbs: + - get + - list + - watch + - create + - delete + - update +- apiGroups: + - certificates.k8s.io + resources: + - signers + resourceNames: + - clusterissuers.cert-manager.io/* + - issuers.cert-manager.io/* + verbs: + - sign +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create +- apiGroups: + - coordination.k8s.io + resources: + - leases + resourceNames: + - cert-manager-controller + - cert-manager-cainjector-leader-election + - cert-manager-cainjector-leader-election-core + verbs: + - get + - update + - patch +- apiGroups: + - batch + resources: + - jobs + verbs: + - create + - delete + - get + - list + - update diff --git a/deploy/e2e/role_binding.yaml b/deploy/e2e/role_binding.yaml index f9a643252..17a3828b3 100644 --- a/deploy/e2e/role_binding.yaml +++ b/deploy/e2e/role_binding.yaml @@ -5,7 +5,7 @@ metadata: subjects: - kind: ServiceAccount name: e2e-test - namespace: default + namespace: mongodb roleRef: kind: ClusterRole name: e2e-test diff --git a/deploy/e2e/service_account.yaml b/deploy/e2e/service_account.yaml index ebc746c83..84fea363b 100644 --- a/deploy/e2e/service_account.yaml +++ b/deploy/e2e/service_account.yaml @@ -2,7 +2,4 @@ apiVersion: v1 kind: ServiceAccount metadata: name: e2e-test - # namespace is 'default' for the e2e tests because the TLS certificates - # generated used as test fixture (in the TLS tests) only work with the - # default namespace. - namespace: default + namespace: mongodb diff --git a/docs/contributing.md b/docs/contributing.md index 676a08e43..4661d1c3b 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -54,7 +54,7 @@ to be able to run properly. Create a json file with the following content: ```json { - "namespace": "default", + "namespace": "mongodb", "repo_url": "localhost:5000", "operator_image": "mongodb-kubernetes-operator", "e2e_image": "community-e2e", @@ -93,7 +93,7 @@ accessible from your Kubernetes cluster. ## Test Namespace You can change the namespace used for tests, if you are using `Kind`, for -instance, you can leave this as `default`. +instance, you can leave this as `mongodb`. ## Python Environment diff --git a/helm-chart/templates/mongodbcommunity_cr_with_tls.yaml b/helm-chart/templates/mongodbcommunity_cr_with_tls.yaml index 01b417e51..2157fcb2c 100644 --- a/helm-chart/templates/mongodbcommunity_cr_with_tls.yaml +++ b/helm-chart/templates/mongodbcommunity_cr_with_tls.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.createResource .Values.resource.tls.enabled .Values.resource.tls.useCertManager }} +{{- if and .Values.resource.tls.enabled .Values.resource.tls.useCertManager }} # cert-manager resources apiVersion: cert-manager.io/v1 kind: Issuer @@ -15,9 +15,9 @@ metadata: namespace: {{ .Values.namespace }} spec: isCA: true - commonName: "*.{{ .Values.resource.name }}-svc.mongodb.svc.cluster.local" + commonName: "*.{{ .Values.resource.name }}-svc.{{ .Values.namespace }}.svc.cluster.local" dnsNames: - - "*.{{ .Values.resource.name }}-svc.mongodb.svc.cluster.local" + - "*.{{ .Values.resource.name }}-svc.{{ .Values.namespace }}.svc.cluster.local" secretName: {{ .Values.resource.tls.caCertificateSecretRef }} privateKey: algorithm: ECDSA @@ -47,9 +47,9 @@ spec: kind: Issuer duration: 8760h # 365 days renewBefore: 720h # 30 days - commonName: "*.{{ .Values.resource.name }}-svc.mongodb.svc.cluster.local" + commonName: "*.{{ .Values.resource.name }}-svc.{{ .Values.namespace }}.svc.cluster.local" dnsNames: - - "*.{{ .Values.resource.name }}-svc.mongodb.svc.cluster.local" + - "*.{{ .Values.resource.name }}-svc.{{ .Values.namespace }}.svc.cluster.local" {{- end }} # mongodb resources diff --git a/helm-chart/values.yaml b/helm-chart/values.yaml index 95ea145c9..76eca1553 100644 --- a/helm-chart/values.yaml +++ b/helm-chart/values.yaml @@ -49,3 +49,6 @@ resource: useCertManager: true certificateKeySecretRef: tls-certificate caCertificateSecretRef: tls-ca-key-pair + certManager: + certDuration: 8760h # 365 days + renewCertBefore: 720h # 30 days diff --git a/pkg/helm/helm.go b/pkg/helm/helm.go index 8a65d8364..d6e97cc1f 100644 --- a/pkg/helm/helm.go +++ b/pkg/helm/helm.go @@ -8,16 +8,22 @@ import ( // Uninstall uninstalls a helm chart of the given name. There is no error in the case // of the helm chart not existing. -func Uninstall(chartName string) error { - helmArgs := []string{"uninstall", chartName} +func Uninstall(chartName string, namespace string) error { + helmArgs := []string{"uninstall", chartName, "-n", namespace} return executeHelmCommand(helmArgs, isNotFoundMessage) } // Install a helm chert at the given path with the given name and the provided set arguments. -func Install(chartPath, chartName string, args map[string]string) error { - helmArgs := []string{"install", "--skip-crds"} - helmArgs = append(helmArgs, mapToHelmArgs(args)...) +func Install(chartPath, chartName string, flags map[string]string, templateValues map[string]string) error { + helmArgs := []string{"install"} helmArgs = append(helmArgs, chartName, chartPath) + for flagKey, flagValue := range flags { + helmArgs = append(helmArgs, fmt.Sprintf("--%s", flagKey)) + if flagValue != "" { + helmArgs = append(helmArgs, flagValue) + } + } + helmArgs = append(helmArgs, mapToHelmValuesArg(templateValues)...) return executeHelmCommand(helmArgs, nil) } @@ -39,9 +45,9 @@ func executeHelmCommand(args []string, messagePredicate func(string) bool) error return nil } -// mapToHelmArgs accepts a map of string to string and returns a list of arguments +// mapToHelmValuesArg accepts a map of string to string and returns a list of arguments // that can be passed to a shell helm command. -func mapToHelmArgs(m map[string]string) []string { +func mapToHelmValuesArg(m map[string]string) []string { var args []string for k, v := range m { args = append(args, "--set", fmt.Sprintf("%s=%s", k, v)) diff --git a/scripts/dev/e2e.py b/scripts/dev/e2e.py index 7b5a82462..71c3631ea 100644 --- a/scripts/dev/e2e.py +++ b/scripts/dev/e2e.py @@ -67,9 +67,13 @@ def _prepare_test_environment(config_file: str) -> None: ) print("Creating Service Account") + service_account = _load_test_service_account() + # set namespace specified in config.json + service_account["metadata"]["namespace"] = dev_config.namespace + k8s_conditions.ignore_if_already_exists( lambda: corev1.create_namespaced_service_account( - dev_config.namespace, _load_test_service_account() + dev_config.namespace, service_account ) ) diff --git a/test/e2e/e2eutil.go b/test/e2e/e2eutil.go index 5bedcf403..4a139c525 100644 --- a/test/e2e/e2eutil.go +++ b/test/e2e/e2eutil.go @@ -151,10 +151,10 @@ func NewTestTLSConfig(optional bool) mdbv1.TLS { Enabled: true, Optional: optional, CertificateKeySecret: mdbv1.LocalObjectReference{ - Name: "test-tls-secret", + Name: "tls-certificate", }, - CaConfigMap: &mdbv1.LocalObjectReference{ - Name: "test-tls-ca", + CaCertificateSecret: &mdbv1.LocalObjectReference{ + Name: "tls-ca-key-pair", }, } } diff --git a/test/e2e/mongodbtests/mongodbtests.go b/test/e2e/mongodbtests/mongodbtests.go index 67431dcd2..318920818 100644 --- a/test/e2e/mongodbtests/mongodbtests.go +++ b/test/e2e/mongodbtests/mongodbtests.go @@ -102,6 +102,16 @@ func StatefulSetHasOwnerReference(mdb *mdbv1.MongoDBCommunity, expectedOwnerRefe } } +// StatefulSetIsDeleted ensures that the underlying stateful set is deleted +func StatefulSetIsDeleted(mdb *mdbv1.MongoDBCommunity) func(t *testing.T) { + return func(t *testing.T) { + err := wait.ForStatefulSetToBeDeleted(mdb.Name, time.Second*10, time.Minute*1, mdb.Namespace) + if err != nil { + t.Fatal(err) + } + } +} + func ServiceHasOwnerReference(mdb *mdbv1.MongoDBCommunity, expectedOwnerReference metav1.OwnerReference) func(t *testing.T) { return func(t *testing.T) { serviceNamespacedName := types.NamespacedName{Name: mdb.ServiceName(), Namespace: mdb.Namespace} diff --git a/test/e2e/replica_set_tls/replica_set_tls_test.go b/test/e2e/replica_set_tls/replica_set_tls_test.go index ba35611ee..b31a7f8ac 100644 --- a/test/e2e/replica_set_tls/replica_set_tls_test.go +++ b/test/e2e/replica_set_tls/replica_set_tls_test.go @@ -21,22 +21,20 @@ func TestMain(m *testing.M) { } func TestReplicaSetTLS(t *testing.T) { - ctx := setup.Setup(t) + resourceName := "mdb-tls" + + ctx, testConfig := setup.SetupWithTLS(t, resourceName) defer ctx.Teardown() - mdb, user := e2eutil.NewTestMongoDB(ctx, "mdb-tls", "") + mdb, user := e2eutil.NewTestMongoDB(ctx, resourceName, testConfig.Namespace) scramUser := mdb.GetScramUsers()[0] mdb.Spec.Security.TLS = e2eutil.NewTestTLSConfig(false) - _, err := setup.GeneratePasswordForUser(ctx, user, "") + _, err := setup.GeneratePasswordForUser(ctx, user, testConfig.Namespace) if err != nil { t.Fatal(err) } - if err := setup.CreateTLSResources(mdb.Namespace, ctx, setup.CertKeyPair); err != nil { - t.Fatalf("Failed to set up TLS resources: %s", err) - } - tester, err := FromResource(t, mdb) if err != nil { t.Fatal(err) @@ -45,15 +43,15 @@ func TestReplicaSetTLS(t *testing.T) { t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb)) mongodbtests.SkipTestIfLocal(t, "Ensure MongoDB TLS Configuration", func(t *testing.T) { - t.Run("Has TLS Mode", tester.HasTlsMode("requireSSL", 60, WithTls())) - t.Run("Basic Connectivity Succeeds", tester.ConnectivitySucceeds(WithTls())) - t.Run("SRV Connectivity Succeeds", tester.ConnectivitySucceeds(WithURI(mdb.MongoSRVURI()), WithTls())) + t.Run("Has TLS Mode", tester.HasTlsMode("requireSSL", 60, WithTls(mdb))) + t.Run("Basic Connectivity Succeeds", tester.ConnectivitySucceeds(WithTls(mdb))) + t.Run("SRV Connectivity Succeeds", tester.ConnectivitySucceeds(WithURI(mdb.MongoSRVURI()), WithTls(mdb))) t.Run("Basic Connectivity With Generated Connection String Secret Succeeds", - tester.ConnectivitySucceeds(WithURI(mongodbtests.GetConnectionStringForUser(mdb, scramUser)), WithTls())) + tester.ConnectivitySucceeds(WithURI(mongodbtests.GetConnectionStringForUser(mdb, scramUser)), WithTls(mdb))) t.Run("SRV Connectivity With Generated Connection String Secret Succeeds", - tester.ConnectivitySucceeds(WithURI(mongodbtests.GetSrvConnectionStringForUser(mdb, scramUser)), WithTls())) + tester.ConnectivitySucceeds(WithURI(mongodbtests.GetSrvConnectionStringForUser(mdb, scramUser)), WithTls(mdb))) t.Run("Connectivity Fails", tester.ConnectivityFails(WithoutTls())) - t.Run("Ensure authentication is configured", tester.EnsureAuthenticationIsConfigured(3, WithTls())) + t.Run("Ensure authentication is configured", tester.EnsureAuthenticationIsConfigured(3, WithTls(mdb))) }) t.Run("TLS is disabled", mongodbtests.DisableTLS(&mdb)) t.Run("MongoDB Reaches Failed Phase", mongodbtests.MongoDBReachesFailedPhase(&mdb)) diff --git a/test/e2e/replica_set_tls_pem_file/replica_set_pem_file_test.go b/test/e2e/replica_set_tls_pem_file/replica_set_pem_file_test.go index 6b7d1121c..b31a7f8ac 100644 --- a/test/e2e/replica_set_tls_pem_file/replica_set_pem_file_test.go +++ b/test/e2e/replica_set_tls_pem_file/replica_set_pem_file_test.go @@ -21,22 +21,20 @@ func TestMain(m *testing.M) { } func TestReplicaSetTLS(t *testing.T) { - ctx := setup.Setup(t) + resourceName := "mdb-tls" + + ctx, testConfig := setup.SetupWithTLS(t, resourceName) defer ctx.Teardown() - mdb, user := e2eutil.NewTestMongoDB(ctx, "mdb-tls", "") + mdb, user := e2eutil.NewTestMongoDB(ctx, resourceName, testConfig.Namespace) scramUser := mdb.GetScramUsers()[0] mdb.Spec.Security.TLS = e2eutil.NewTestTLSConfig(false) - _, err := setup.GeneratePasswordForUser(ctx, user, "") + _, err := setup.GeneratePasswordForUser(ctx, user, testConfig.Namespace) if err != nil { t.Fatal(err) } - if err := setup.CreateTLSResources(mdb.Namespace, ctx, setup.Pem); err != nil { - t.Fatalf("Failed to set up TLS resources: %s", err) - } - tester, err := FromResource(t, mdb) if err != nil { t.Fatal(err) @@ -45,15 +43,15 @@ func TestReplicaSetTLS(t *testing.T) { t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb)) mongodbtests.SkipTestIfLocal(t, "Ensure MongoDB TLS Configuration", func(t *testing.T) { - t.Run("Has TLS Mode", tester.HasTlsMode("requireSSL", 60, WithTls())) - t.Run("Basic Connectivity Succeeds", tester.ConnectivitySucceeds(WithTls())) - t.Run("SRV Connectivity Succeeds", tester.ConnectivitySucceeds(WithURI(mdb.MongoSRVURI()), WithTls())) + t.Run("Has TLS Mode", tester.HasTlsMode("requireSSL", 60, WithTls(mdb))) + t.Run("Basic Connectivity Succeeds", tester.ConnectivitySucceeds(WithTls(mdb))) + t.Run("SRV Connectivity Succeeds", tester.ConnectivitySucceeds(WithURI(mdb.MongoSRVURI()), WithTls(mdb))) t.Run("Basic Connectivity With Generated Connection String Secret Succeeds", - tester.ConnectivitySucceeds(WithURI(mongodbtests.GetConnectionStringForUser(mdb, scramUser)), WithTls())) + tester.ConnectivitySucceeds(WithURI(mongodbtests.GetConnectionStringForUser(mdb, scramUser)), WithTls(mdb))) t.Run("SRV Connectivity With Generated Connection String Secret Succeeds", - tester.ConnectivitySucceeds(WithURI(mongodbtests.GetSrvConnectionStringForUser(mdb, scramUser)), WithTls())) + tester.ConnectivitySucceeds(WithURI(mongodbtests.GetSrvConnectionStringForUser(mdb, scramUser)), WithTls(mdb))) t.Run("Connectivity Fails", tester.ConnectivityFails(WithoutTls())) - t.Run("Ensure authentication is configured", tester.EnsureAuthenticationIsConfigured(3, WithTls())) + t.Run("Ensure authentication is configured", tester.EnsureAuthenticationIsConfigured(3, WithTls(mdb))) }) t.Run("TLS is disabled", mongodbtests.DisableTLS(&mdb)) t.Run("MongoDB Reaches Failed Phase", mongodbtests.MongoDBReachesFailedPhase(&mdb)) diff --git a/test/e2e/replica_set_tls_recreate_mdbc/replica_set_tls_recreate_mdbc_test.go b/test/e2e/replica_set_tls_recreate_mdbc/replica_set_tls_recreate_mdbc_test.go index 7a1d1edf5..81cd47543 100644 --- a/test/e2e/replica_set_tls_recreate_mdbc/replica_set_tls_recreate_mdbc_test.go +++ b/test/e2e/replica_set_tls_recreate_mdbc/replica_set_tls_recreate_mdbc_test.go @@ -22,30 +22,29 @@ func TestMain(m *testing.M) { } func TestReplicaSetTLSRecreateMdbc(t *testing.T) { - ctx := setup.Setup(t) + resourceName := "mdb-tls" + + ctx, testConfig := setup.SetupWithTLS(t, resourceName) defer ctx.Teardown() - mdb1, user := e2eutil.NewTestMongoDB(ctx, "mdb-tls", "") + mdb1, user := e2eutil.NewTestMongoDB(ctx, resourceName, testConfig.Namespace) scramUser := mdb1.GetScramUsers()[0] mdb1.Spec.Security.TLS = e2eutil.NewTestTLSConfig(false) - _, err := setup.GeneratePasswordForUser(ctx, user, "") + _, err := setup.GeneratePasswordForUser(ctx, user, testConfig.Namespace) if err != nil { t.Fatal(err) } - if err := setup.CreateTLSResources(mdb1.Namespace, ctx, setup.CertKeyPair); err != nil { - t.Fatalf("Failed to set up TLS resources: %s", err) - } - t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb1, ctx)) t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb1)) if err := e2eutil.TestClient.Delete(context.TODO(), &mdb1); err != nil { t.Fatalf("Failed to delete first test MongoDB: %s", err) } + t.Run("Stateful Set Is Deleted", mongodbtests.StatefulSetIsDeleted(&mdb1)) - mdb2, _ := e2eutil.NewTestMongoDB(ctx, "mdb-tls", "") + mdb2, _ := e2eutil.NewTestMongoDB(ctx, resourceName, testConfig.Namespace) mdb2.Spec.Security.TLS = e2eutil.NewTestTLSConfig(false) tester1, err := FromResource(t, mdb2) if err != nil { @@ -55,15 +54,15 @@ func TestReplicaSetTLSRecreateMdbc(t *testing.T) { t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb2, ctx)) t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb2)) mongodbtests.SkipTestIfLocal(t, "Ensure MongoDB TLS Configuration", func(t *testing.T) { - t.Run("Has TLS Mode", tester1.HasTlsMode("requireSSL", 60, WithTls())) - t.Run("Basic Connectivity Succeeds", tester1.ConnectivitySucceeds(WithTls())) - t.Run("SRV Connectivity Succeeds", tester1.ConnectivitySucceeds(WithURI(mdb2.MongoSRVURI()), WithTls())) + t.Run("Has TLS Mode", tester1.HasTlsMode("requireSSL", 60, WithTls(mdb2))) + t.Run("Basic Connectivity Succeeds", tester1.ConnectivitySucceeds(WithTls(mdb2))) + t.Run("SRV Connectivity Succeeds", tester1.ConnectivitySucceeds(WithURI(mdb2.MongoSRVURI()), WithTls(mdb2))) t.Run("Basic Connectivity With Generated Connection String Secret Succeeds", - tester1.ConnectivitySucceeds(WithURI(mongodbtests.GetConnectionStringForUser(mdb2, scramUser)), WithTls())) + tester1.ConnectivitySucceeds(WithURI(mongodbtests.GetConnectionStringForUser(mdb2, scramUser)), WithTls(mdb2))) t.Run("SRV Connectivity With Generated Connection String Secret Succeeds", - tester1.ConnectivitySucceeds(WithURI(mongodbtests.GetSrvConnectionStringForUser(mdb2, scramUser)), WithTls())) + tester1.ConnectivitySucceeds(WithURI(mongodbtests.GetSrvConnectionStringForUser(mdb2, scramUser)), WithTls(mdb2))) t.Run("Connectivity Fails", tester1.ConnectivityFails(WithoutTls())) - t.Run("Ensure authentication is configured", tester1.EnsureAuthenticationIsConfigured(3, WithTls())) + t.Run("Ensure authentication is configured", tester1.EnsureAuthenticationIsConfigured(3, WithTls(mdb2))) }) t.Run("TLS is disabled", mongodbtests.DisableTLS(&mdb2)) t.Run("MongoDB Reaches Failed Phase", mongodbtests.MongoDBReachesFailedPhase(&mdb2)) diff --git a/test/e2e/replica_set_tls_rotate/replica_set_tls_rotate_test.go b/test/e2e/replica_set_tls_rotate/replica_set_tls_rotate_test.go index d083023d9..554619008 100644 --- a/test/e2e/replica_set_tls_rotate/replica_set_tls_rotate_test.go +++ b/test/e2e/replica_set_tls_rotate/replica_set_tls_rotate_test.go @@ -23,36 +23,40 @@ func TestMain(m *testing.M) { } func TestReplicaSetTLSRotate(t *testing.T) { - ctx := setup.Setup(t) + resourceName := "mdb-tls" + + ctx, testConfig := setup.SetupWithTLS(t, resourceName) defer ctx.Teardown() - mdb, user := e2eutil.NewTestMongoDB(ctx, "mdb-tls", "") + mdb, user := e2eutil.NewTestMongoDB(ctx, resourceName, testConfig.Namespace) mdb.Spec.Security.TLS = e2eutil.NewTestTLSConfig(false) - _, err := setup.GeneratePasswordForUser(ctx, user, "") + _, err := setup.GeneratePasswordForUser(ctx, user, testConfig.Namespace) if err != nil { t.Fatal(err) } - if err := setup.CreateTLSResources(mdb.Namespace, ctx, setup.CertKeyPair); err != nil { - t.Fatalf("Failed to set up TLS resources: %s", err) - } tester, err := FromResource(t, mdb) + if err != nil { + t.Fatal(err) + } + clientCert, err := GetClientCert(mdb) if err != nil { t.Fatal(err) } + initialCertSerialNumber := clientCert.SerialNumber t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb)) - t.Run("Wait for TLS to be enabled", tester.HasTlsMode("requireSSL", 60, WithTls())) - t.Run("Test Basic TLS Connectivity", tester.ConnectivitySucceeds(WithTls())) - t.Run("Ensure Authentication", tester.EnsureAuthenticationIsConfigured(3, WithTls())) + t.Run("Wait for TLS to be enabled", tester.HasTlsMode("requireSSL", 60, WithTls(mdb))) + t.Run("Test Basic TLS Connectivity", tester.ConnectivitySucceeds(WithTls(mdb))) + t.Run("Ensure Authentication", tester.EnsureAuthenticationIsConfigured(3, WithTls(mdb))) t.Run("Test TLS required", tester.ConnectivityFails(WithoutTls())) t.Run("MongoDB is reachable while certificate is rotated", func(t *testing.T) { - defer tester.StartBackgroundConnectivityTest(t, time.Second*10, WithTls())() + defer tester.StartBackgroundConnectivityTest(t, time.Second*10, WithTls(mdb))() t.Run("Update certificate secret", tlstests.RotateCertificate(&mdb)) - t.Run("Wait for certificate to be rotated", tester.WaitForRotatedCertificate()) + t.Run("Wait for certificate to be rotated", tester.WaitForRotatedCertificate(mdb, initialCertSerialNumber)) }) } diff --git a/test/e2e/replica_set_tls_upgrade/replica_set_tls_upgrade_test.go b/test/e2e/replica_set_tls_upgrade/replica_set_tls_upgrade_test.go index 88849b896..a0d873829 100644 --- a/test/e2e/replica_set_tls_upgrade/replica_set_tls_upgrade_test.go +++ b/test/e2e/replica_set_tls_upgrade/replica_set_tls_upgrade_test.go @@ -24,19 +24,17 @@ func TestMain(m *testing.M) { } func TestReplicaSetTLSUpgrade(t *testing.T) { - ctx := setup.Setup(t) + resourceName := "mdb-tls" + + ctx, testConfig := setup.SetupWithTLS(t, resourceName) defer ctx.Teardown() - mdb, user := e2eutil.NewTestMongoDB(ctx, "mdb-tls", "") - _, err := setup.GeneratePasswordForUser(ctx, user, "") + mdb, user := e2eutil.NewTestMongoDB(ctx, resourceName, testConfig.Namespace) + _, err := setup.GeneratePasswordForUser(ctx, user, testConfig.Namespace) if err != nil { t.Fatal(err) } - if err := setup.CreateTLSResources(mdb.Namespace, ctx, setup.CertKeyPair); err != nil { - t.Fatalf("Failed to set up TLS resources: %s", err) - } - tester, err := FromResource(t, mdb) if err != nil { t.Fatal(err) @@ -58,18 +56,18 @@ func TestReplicaSetTLSUpgrade(t *testing.T) { // Ensure MongoDB is reachable both with and without TLS t.Run("Test Basic Connectivity", tester.ConnectivitySucceeds(WithoutTls())) - t.Run("Test Basic TLS Connectivity", tester.ConnectivitySucceeds(WithTls())) - t.Run("Internal cluster keyfile authentication is enabled", tester.HasKeyfileAuth(3, WithTls())) + t.Run("Test Basic TLS Connectivity", tester.ConnectivitySucceeds(WithTls(mdb))) + t.Run("Internal cluster keyfile authentication is enabled", tester.HasKeyfileAuth(3, WithTls(mdb))) // Make TLS required t.Run("MongoDB is reachable over TLS while making TLS required", func(t *testing.T) { - defer tester.StartBackgroundConnectivityTest(t, time.Second*10, WithTls())() + defer tester.StartBackgroundConnectivityTest(t, time.Second*10, WithTls(mdb))() t.Run("Make TLS required", tlstests.EnableTLS(&mdb, false)) t.Run("Stateful Set Reaches Ready State, after setting TLS to requireSSL", mongodbtests.StatefulSetBecomesReady(&mdb)) - t.Run("Wait for TLS to be required", tester.HasTlsMode("requireSSL", 120, WithTls())) + t.Run("Wait for TLS to be required", tester.HasTlsMode("requireSSL", 120, WithTls(mdb))) }) // Ensure MongoDB is reachable only over TLS - t.Run("Test Basic TLS Connectivity", tester.ConnectivitySucceeds(WithTls())) + t.Run("Test Basic TLS Connectivity", tester.ConnectivitySucceeds(WithTls(mdb))) t.Run("Test TLS Required For Connectivity", tester.ConnectivityFails(WithoutTls())) } diff --git a/test/e2e/setup/setup.go b/test/e2e/setup/setup.go index ce64f162c..5ac58dbfe 100644 --- a/test/e2e/setup/setup.go +++ b/test/e2e/setup/setup.go @@ -3,27 +3,25 @@ package setup import ( "context" "fmt" + "strconv" + "strings" + "testing" + "time" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/helm" "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/envvar" waite2e "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/util/wait" "github.com/pkg/errors" - "io/ioutil" appsv1 "k8s.io/api/apps/v1" apiErrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" - "path" - "strings" - "testing" - "time" "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/generate" "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/secret" - "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/configmap" - e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" @@ -33,7 +31,6 @@ type tlsSecretType string const ( performCleanupEnv = "PERFORM_CLEANUP" - helmChartPathEnv = "HELM_CHART_PATH" CertKeyPair tlsSecretType = "CERTKEYPAIR" Pem tlsSecretType = "PEM" ) @@ -45,65 +42,31 @@ func Setup(t *testing.T) *e2eutil.Context { t.Fatal(err) } - if err := deployOperator(); err != nil { + config := loadTestConfigFromEnv() + if err := deployOperator(config, "mdb", false); err != nil { t.Fatal(err) } return ctx } -// CreateTLSResources will setup the CA ConfigMap and cert-key Secret necessary for TLS -// The certificates and keys are stored in testdata/tls -func CreateTLSResources(namespace string, ctx *e2eutil.Context, secretType tlsSecretType) error { - tlsConfig := e2eutil.NewTestTLSConfig(false) +func SetupWithTLS(t *testing.T, resourceName string) (*e2eutil.Context, TestConfig) { + ctx, err := e2eutil.NewContext(t, envvar.ReadBool(performCleanupEnv)) - // Create CA ConfigMap - testDataDir := e2eutil.TlsTestDataDir() - ca, err := ioutil.ReadFile(path.Join(testDataDir, "ca.crt")) if err != nil { - return nil + t.Fatal(err) } - caConfigMap := configmap.Builder(). - SetName(tlsConfig.CaConfigMap.Name). - SetNamespace(namespace). - SetDataField("ca.crt", string(ca)). - SetLabels(e2eutil.TestLabels()). - Build() - - err = e2eutil.TestClient.Create(context.TODO(), &caConfigMap, &e2eutil.CleanupOptions{TestContext: ctx}) - if err != nil { - return err + config := loadTestConfigFromEnv() + if err := deployCertManager(config); err != nil { + t.Fatal(err) } - certKeySecretBuilder := secret.Builder(). - SetName(tlsConfig.CertificateKeySecret.Name). - SetLabels(e2eutil.TestLabels()). - SetNamespace(namespace) - - if secretType == CertKeyPair { - // Create server key and certificate secret - cert, err := ioutil.ReadFile(path.Join(testDataDir, "server.crt")) - if err != nil { - return err - } - key, err := ioutil.ReadFile(path.Join(testDataDir, "server.key")) - if err != nil { - return err - } - certKeySecretBuilder.SetField("tls.crt", string(cert)).SetField("tls.key", string(key)) - } - if secretType == Pem { - pem, err := ioutil.ReadFile(path.Join(testDataDir, "server.pem")) - if err != nil { - return err - } - certKeySecretBuilder.SetField("tls.pem", string(pem)) + if err := deployOperator(config, resourceName, true); err != nil { + t.Fatal(err) } - certKeySecret := certKeySecretBuilder.Build() - - return e2eutil.TestClient.Create(context.TODO(), &certKeySecret, &e2eutil.CleanupOptions{TestContext: ctx}) + return ctx, config } // GeneratePasswordForUser will create a secret with a password for the given user @@ -149,15 +112,15 @@ func extractRegistryNameAndVersion(fullImage string) (string, string, string) { } // getHelmArgs returns a map of helm arguments that are required to install the operator. -func getHelmArgs(testConfig testConfig, watchNamespace string) map[string]string { - agentRegistry, agentName, agentVersion := extractRegistryNameAndVersion(testConfig.agentImage) - versionUpgradeHookRegistry, versionUpgradeHookName, versionUpgradeHookVersion := extractRegistryNameAndVersion(testConfig.versionUpgradeHookImage) - readinessProbeRegistry, readinessProbeName, readinessProbeVersion := extractRegistryNameAndVersion(testConfig.readinessProbeImage) - operatorRegistry, operatorName, operatorVersion := extractRegistryNameAndVersion(testConfig.operatorImage) +func getHelmArgs(testConfig TestConfig, watchNamespace string, resourceName string, withTLS bool) map[string]string { + agentRegistry, agentName, agentVersion := extractRegistryNameAndVersion(testConfig.AgentImage) + versionUpgradeHookRegistry, versionUpgradeHookName, versionUpgradeHookVersion := extractRegistryNameAndVersion(testConfig.VersionUpgradeHookImage) + readinessProbeRegistry, readinessProbeName, readinessProbeVersion := extractRegistryNameAndVersion(testConfig.ReadinessProbeImage) + operatorRegistry, operatorName, operatorVersion := extractRegistryNameAndVersion(testConfig.OperatorImage) helmArgs := make(map[string]string) - helmArgs["namespace"] = testConfig.namespace + helmArgs["namespace"] = testConfig.Namespace helmArgs["operator.watchNamespace"] = watchNamespace helmArgs["operator.operatorImageName"] = operatorName @@ -177,28 +140,36 @@ func getHelmArgs(testConfig testConfig, watchNamespace string) map[string]string helmArgs["registry.agent"] = agentRegistry helmArgs["registry.readinessProbe"] = readinessProbeRegistry + helmArgs["createResource"] = strconv.FormatBool(false) + helmArgs["resource.name"] = resourceName + helmArgs["resource.tls.enabled"] = strconv.FormatBool(withTLS) + helmArgs["resource.tls.useCertManager"] = strconv.FormatBool(withTLS) + return helmArgs } // deployOperator installs all resources required by the operator using helm. -func deployOperator() error { - testConfig := loadTestConfigFromEnv() - e2eutil.OperatorNamespace = testConfig.namespace +func deployOperator(config TestConfig, resourceName string, withTLS bool) error { + e2eutil.OperatorNamespace = config.Namespace fmt.Printf("Setting operator namespace to %s\n", e2eutil.OperatorNamespace) - watchNamespace := testConfig.namespace - if testConfig.clusterWide { + watchNamespace := config.Namespace + if config.ClusterWide { watchNamespace = "*" } fmt.Printf("Setting namespace to watch to %s\n", watchNamespace) - chartName := "mongodb-kubernetes-operator" - if err := helm.Uninstall(chartName); err != nil { + helmChartName := "mongodb-kubernetes-operator" + if err := helm.Uninstall(helmChartName, config.Namespace); err != nil { return err } - helmChartPath := envvar.GetEnvOrDefault(helmChartPathEnv, "/workspace/helm-chart") - helmArgs := getHelmArgs(testConfig, watchNamespace) - if err := helm.Install(helmChartPath, chartName, helmArgs); err != nil { + helmArgs := getHelmArgs(config, watchNamespace, resourceName, withTLS) + helmFlags := map[string]string{ + "skip-crds": "", + "namespace": config.Namespace, + "create-namespace": "", + } + if err := helm.Install(config.HelmChartPath, helmChartName, helmFlags, helmArgs); err != nil { return err } @@ -228,6 +199,25 @@ func deployOperator() error { return nil } +func deployCertManager(config TestConfig) error { + const helmChartName = "cert-manager" + if err := helm.Uninstall(helmChartName, config.CertManagerNamespace); err != nil { + return errors.Errorf("failed to uninstall cert-manager Helm chart: %s", err) + } + + charlUrl := fmt.Sprintf("https://charts.jetstack.io/charts/cert-manager-%s.tgz", config.CertManagerVersion) + flags := map[string]string{ + "version": config.CertManagerVersion, + "namespace": config.CertManagerNamespace, + "create-namespace": "", + } + values := map[string]string{"installCRDs": "true"} + if err := helm.Install(charlUrl, helmChartName, flags, values); err != nil { + return errors.Errorf("failed to install cert-manager Helm chart: %s", err) + } + return nil +} + // hasDeploymentRequiredReplicas returns a condition function that indicates whether the given deployment // currently has the required amount of replicas in the ready state as specified in spec.replicas func hasDeploymentRequiredReplicas(dep *appsv1.Deployment) wait.ConditionFunc { diff --git a/test/e2e/setup/test_config.go b/test/e2e/setup/test_config.go index 0dae114e7..288333969 100644 --- a/test/e2e/setup/test_config.go +++ b/test/e2e/setup/test_config.go @@ -6,30 +6,39 @@ import ( ) const ( - testNamespaceEnvName = "TEST_NAMESPACE" - operatorImageEnvName = "OPERATOR_IMAGE" - clusterWideEnvName = "CLUSTER_WIDE" - performCleanupEnvName = "PERFORM_CLEANUP" + testNamespaceEnvName = "TEST_NAMESPACE" + testCertManagerNamespaceEnvName = "TEST_CERT_MANAGER_NAMESPACE" + testCertManagerVersionEnvName = "TEST_CERT_MANAGER_VERSION" + operatorImageEnvName = "OPERATOR_IMAGE" + clusterWideEnvName = "CLUSTER_WIDE" + performCleanupEnvName = "PERFORM_CLEANUP" + helmChartPathEnvName = "HELM_CHART_PATH" ) -type testConfig struct { - namespace string - operatorImage string - versionUpgradeHookImage string - clusterWide bool - performCleanup bool - agentImage string - readinessProbeImage string +type TestConfig struct { + Namespace string + CertManagerNamespace string + CertManagerVersion string + OperatorImage string + VersionUpgradeHookImage string + ClusterWide bool + PerformCleanup bool + AgentImage string + ReadinessProbeImage string + HelmChartPath string } -func loadTestConfigFromEnv() testConfig { - return testConfig{ - namespace: envvar.GetEnvOrDefault(testNamespaceEnvName, "default"), - operatorImage: envvar.GetEnvOrDefault(operatorImageEnvName, "quay.io/mongodb/community-operator-dev:latest"), - versionUpgradeHookImage: envvar.GetEnvOrDefault(construct.VersionUpgradeHookImageEnv, "quay.io/mongodb/mongodb-kubernetes-operator-version-upgrade-post-start-hook:1.0.2"), - agentImage: envvar.GetEnvOrDefault(construct.AgentImageEnv, "quay.io/mongodb/mongodb-agent:10.29.0.6830-1"), // TODO: better way to decide default agent image. - clusterWide: envvar.ReadBool(clusterWideEnvName), - performCleanup: envvar.ReadBool(performCleanupEnvName), - readinessProbeImage: envvar.GetEnvOrDefault(construct.ReadinessProbeImageEnv, "quay.io/mongodb/mongodb-kubernetes-readinessprobe:1.0.3"), +func loadTestConfigFromEnv() TestConfig { + return TestConfig{ + Namespace: envvar.GetEnvOrDefault(testNamespaceEnvName, "mongodb"), + CertManagerNamespace: envvar.GetEnvOrDefault(testCertManagerNamespaceEnvName, "cert-manager"), + CertManagerVersion: envvar.GetEnvOrDefault(testCertManagerVersionEnvName, "v1.5.3"), + OperatorImage: envvar.GetEnvOrDefault(operatorImageEnvName, "quay.io/mongodb/community-operator-dev:latest"), + VersionUpgradeHookImage: envvar.GetEnvOrDefault(construct.VersionUpgradeHookImageEnv, "quay.io/mongodb/mongodb-kubernetes-operator-version-upgrade-post-start-hook:1.0.2"), + AgentImage: envvar.GetEnvOrDefault(construct.AgentImageEnv, "quay.io/mongodb/mongodb-agent:10.29.0.6830-1"), // TODO: better way to decide default agent image. + ClusterWide: envvar.ReadBool(clusterWideEnvName), + PerformCleanup: envvar.ReadBool(performCleanupEnvName), + ReadinessProbeImage: envvar.GetEnvOrDefault(construct.ReadinessProbeImageEnv, "quay.io/mongodb/mongodb-kubernetes-readinessprobe:1.0.3"), + HelmChartPath: envvar.GetEnvOrDefault(helmChartPathEnvName, "/workspace/helm-chart"), } } diff --git a/test/e2e/tlstests/tlstests.go b/test/e2e/tlstests/tlstests.go index 543af8355..e2c1f02ef 100644 --- a/test/e2e/tlstests/tlstests.go +++ b/test/e2e/tlstests/tlstests.go @@ -1,16 +1,17 @@ package tlstests import ( + "bytes" "context" - "io/ioutil" - "path" "testing" - - "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/secret" + "time" mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" ) // EnableTLS will upgrade an existing TLS cluster to use TLS. @@ -27,21 +28,24 @@ func EnableTLS(mdb *mdbv1.MongoDBCommunity, optional bool) func(*testing.T) { func RotateCertificate(mdb *mdbv1.MongoDBCommunity) func(*testing.T) { return func(t *testing.T) { - // Load new certificate and key - testDataDir := e2eutil.TlsTestDataDir() - cert, err := ioutil.ReadFile(path.Join(testDataDir, "server_rotated.crt")) - assert.NoError(t, err) - key, err := ioutil.ReadFile(path.Join(testDataDir, "server_rotated.key")) + certKeySecretName := types.NamespacedName{Name: mdb.Spec.Security.TLS.CertificateKeySecret.Name, Namespace: mdb.Namespace} + + currentSecret := corev1.Secret{} + err := e2eutil.TestClient.Get(context.TODO(), certKeySecretName, ¤tSecret) assert.NoError(t, err) - certKeySecret := secret.Builder(). - SetName(mdb.Spec.Security.TLS.CertificateKeySecret.Name). - SetNamespace(mdb.Namespace). - SetField("tls.crt", string(cert)). - SetField("tls.key", string(key)). - Build() + // delete current cert secret, cert-manager should generate a new one + err = e2eutil.TestClient.Delete(context.TODO(), ¤tSecret) + assert.NoError(t, err) - err = e2eutil.TestClient.Update(context.TODO(), &certKeySecret) + newSecret := corev1.Secret{} + err = wait.Poll(5*time.Second, 1*time.Minute, func() (done bool, err error) { + if err := e2eutil.TestClient.Get(context.TODO(), certKeySecretName, &newSecret); err != nil { + return false, nil + } + return true, nil + }) assert.NoError(t, err) + assert.False(t, bytes.Equal(currentSecret.Data["tls.crt"], newSecret.Data["tls.crt"])) } } diff --git a/test/e2e/util/mongotester/mongotester.go b/test/e2e/util/mongotester/mongotester.go index 82c5bfc7b..49188e829 100644 --- a/test/e2e/util/mongotester/mongotester.go +++ b/test/e2e/util/mongotester/mongotester.go @@ -4,22 +4,20 @@ import ( "context" "crypto/tls" "crypto/x509" + "encoding/pem" "fmt" - "io/ioutil" "math/big" - "path" "reflect" "testing" "time" + "github.com/pkg/errors" "github.com/stretchr/objx" "go.mongodb.org/mongo-driver/bson/primitive" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" - "github.com/pkg/errors" - mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" "github.com/mongodb/mongodb-kubernetes-operator/pkg/automationconfig" e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" @@ -30,36 +28,15 @@ import ( "k8s.io/apimachinery/pkg/util/wait" ) -// tlsConfig is the test tls fixture that we use for testing -// tls. -var tlsConfig *tls.Config = nil - -// initTls loads in the tls configuration fixture -func initTls() error { - if tlsConfig != nil { - return nil - } - var err error - tlsConfig, err = getClientTLSConfig() - if err != nil { - return err - } - return nil -} - type Tester struct { mongoClient *mongo.Client clientOpts []*options.ClientOptions } -func newTester(opts ...*options.ClientOptions) (*Tester, error) { - if err := initTls(); err != nil { - return nil, err - } - +func newTester(mdb mdbv1.MongoDBCommunity, opts ...*options.ClientOptions) *Tester { t := &Tester{} t.clientOpts = append(t.clientOpts, opts...) - return t, nil + return t } // OptionApplier is an interface which is able to accept a list @@ -90,16 +67,12 @@ func FromResource(t *testing.T, mdb mdbv1.MongoDBCommunity, opts ...OptionApplie clientOpts = WithScram(user.Name, string(passwordSecret.Data[user.PasswordSecretRef.Key])).ApplyOption(clientOpts...) } - if mdb.Spec.Security.TLS.Enabled { - clientOpts = WithTls().ApplyOption(clientOpts...) - } - // add any additional options for _, opt := range opts { clientOpts = opt.ApplyOption(clientOpts...) } - return newTester(clientOpts...) + return newTester(mdb, clientOpts...), nil } // ConnectivitySucceeds performs a basic check that ensures that it is possible @@ -254,19 +227,16 @@ func (m *Tester) connectivityCheck(shouldSucceed bool, opts ...OptionApplier) fu } } -func (m *Tester) WaitForRotatedCertificate() func(*testing.T) { +func (m *Tester) WaitForRotatedCertificate(mdb mdbv1.MongoDBCommunity, initialCertSerialNumber *big.Int) func(*testing.T) { return func(t *testing.T) { - // The rotated certificate has serial number 2 - expectedSerial := big.NewInt(2) - - tls, err := getClientTLSConfig() + tls, err := getClientTLSConfig(mdb) assert.NoError(t, err) // Reject all server certificates that don't have the expected serial number tls.VerifyPeerCertificate = func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error { cert := verifiedChains[0][0] - if expectedSerial.Cmp(cert.SerialNumber) != 0 { - return errors.Errorf("expected certificate serial number %s, got %s", expectedSerial, cert.SerialNumber) + if initialCertSerialNumber.Cmp(cert.SerialNumber) == 0 { + return errors.Errorf("certificate serial number has not changed: %s", cert.SerialNumber) } return nil } @@ -432,7 +402,12 @@ func WithHosts(hosts []string) OptionApplier { } // WithTls configures the client to use tls -func WithTls() OptionApplier { +func WithTls(mdb mdbv1.MongoDBCommunity) OptionApplier { + tlsConfig, err := getClientTLSConfig(mdb) + if err != nil { + panic(errors.Errorf("could not retrieve TLS config: %s", err)) + } + return withTls(tlsConfig) } @@ -470,20 +445,33 @@ func WithReplicaSet(rsname string) OptionApplier { } // getClientTLSConfig reads in the tls fixtures -func getClientTLSConfig() (*tls.Config, error) { - // Read the CA certificate from test data - testDataDir := e2eutil.TlsTestDataDir() - caPEM, err := ioutil.ReadFile(path.Join(testDataDir, "ca.crt")) - if err != nil { +func getClientTLSConfig(mdb mdbv1.MongoDBCommunity) (*tls.Config, error) { + caSecret := corev1.Secret{} + caSecretName := types.NamespacedName{Name: mdb.Spec.Security.TLS.CaCertificateSecret.Name, Namespace: mdb.Namespace} + if err := e2eutil.TestClient.Get(context.TODO(), caSecretName, &caSecret); err != nil { return nil, err } - + caPEM := caSecret.Data["ca.crt"] caPool := x509.NewCertPool() caPool.AppendCertsFromPEM(caPEM) - return &tls.Config{ //nolint RootCAs: caPool, }, nil + +} + +// GetClientCert reads the client key certificate +func GetClientCert(mdb mdbv1.MongoDBCommunity) (*x509.Certificate, error) { + certSecret := corev1.Secret{} + certSecretName := types.NamespacedName{Name: mdb.Spec.Security.TLS.CertificateKeySecret.Name, Namespace: mdb.Namespace} + if err := e2eutil.TestClient.Get(context.TODO(), certSecretName, &certSecret); err != nil { + return nil, err + } + block, _ := pem.Decode(certSecret.Data["tls.crt"]) + if block == nil { + return nil, errors.Errorf("error decoding client cert key") + } + return x509.ParseCertificate(block.Bytes) } // defaults returns the default connectivity options diff --git a/test/e2e/util/wait/wait.go b/test/e2e/util/wait/wait.go index 8a8a84c11..35f24f2e5 100644 --- a/test/e2e/util/wait/wait.go +++ b/test/e2e/util/wait/wait.go @@ -73,6 +73,13 @@ func ForStatefulSetToExist(stsName string, retryInterval, timeout time.Duration, return sts, waitForRuntimeObjectToExist(stsName, retryInterval, timeout, &sts, namespace) } +// ForStatefulSetToBeDeleted waits until a StatefulSet of the given name is deleted +// using the provided retryInterval and timeout +func ForStatefulSetToBeDeleted(stsName string, retryInterval, timeout time.Duration, namespace string) error { + sts := appsv1.StatefulSet{} + return waitForRuntimeObjectToBeDeleted(stsName, retryInterval, timeout, &sts, namespace) +} + // ForStatefulSetToHaveUpdateStrategy waits until all replicas of the StatefulSet with the given name // have reached the ready status func ForStatefulSetToHaveUpdateStrategy(t *testing.T, mdb *mdbv1.MongoDBCommunity, strategy appsv1.StatefulSetUpdateStrategyType, opts ...Configuration) error { @@ -148,10 +155,24 @@ func ForPodReadiness(t *testing.T, isReady bool, containerName string, timeout t // using the provided retryInterval and timeout provided. func waitForRuntimeObjectToExist(name string, retryInterval, timeout time.Duration, obj client.Object, namespace string) error { return wait.Poll(retryInterval, timeout, func() (done bool, err error) { - err = e2eutil.TestClient.Get(context.TODO(), types.NamespacedName{Name: name, Namespace: namespace}, obj) - if err != nil { - return false, client.IgnoreNotFound(err) - } - return true, nil + return runtimeObjectExists(name, obj, namespace) }) } + +// waitForRuntimeObjectToBeDeleted waits until a runtime.Object of the given name is deleted +// using the provided retryInterval and timeout provided. +func waitForRuntimeObjectToBeDeleted(name string, retryInterval, timeout time.Duration, obj client.Object, namespace string) error { + return wait.Poll(retryInterval, timeout, func() (done bool, err error) { + exists, err := runtimeObjectExists(name, obj, namespace) + return !exists, err + }) +} + +// runtimeObjectExists checks if a runtime.Object of the given name exists +func runtimeObjectExists(name string, obj client.Object, namespace string) (bool, error) { + err := e2eutil.TestClient.Get(context.TODO(), types.NamespacedName{Name: name, Namespace: namespace}, obj) + if err != nil { + return false, client.IgnoreNotFound(err) + } + return true, nil +} From 1aa7093d2cc977bc3b1f5a5fa7e1e902d37768c8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 29 Sep 2021 17:17:07 +0100 Subject: [PATCH 407/790] Bump sigs.k8s.io/controller-runtime from 0.10.0 to 0.10.1 (#734) Bumps [sigs.k8s.io/controller-runtime](https://github.com/kubernetes-sigs/controller-runtime) from 0.10.0 to 0.10.1. - [Release notes](https://github.com/kubernetes-sigs/controller-runtime/releases) - [Commits](https://github.com/kubernetes-sigs/controller-runtime/compare/v0.10.0...v0.10.1) --- updated-dependencies: - dependency-name: sigs.k8s.io/controller-runtime dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 21 ++++++++------------- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index 9d87a4994..92d944739 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( k8s.io/api v0.22.2 k8s.io/apimachinery v0.22.2 k8s.io/client-go v0.22.2 - sigs.k8s.io/controller-runtime v0.10.0 + sigs.k8s.io/controller-runtime v0.10.1 sigs.k8s.io/yaml v1.3.0 ) diff --git a/go.sum b/go.sum index 81b53902b..4760fa48e 100644 --- a/go.sum +++ b/go.sum @@ -818,21 +818,18 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.22.1/go.mod h1:bh13rkTp3F1XEaLGykbyRD2QaTTzPm0e/BMd8ptFONY= k8s.io/api v0.22.2 h1:M8ZzAD0V6725Fjg53fKeTJxGsJvRbk4TEm/fexHMtfw= k8s.io/api v0.22.2/go.mod h1:y3ydYpLJAaDI+BbSe2xmGcqxiWHmWjkEeIbiwHvnPR8= -k8s.io/apiextensions-apiserver v0.22.1 h1:YSJYzlFNFSfUle+yeEXX0lSQyLEoxoPJySRupepb0gE= -k8s.io/apiextensions-apiserver v0.22.1/go.mod h1:HeGmorjtRmRLE+Q8dJu6AYRoZccvCMsghwS8XTUYb2c= -k8s.io/apimachinery v0.22.1/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0= +k8s.io/apiextensions-apiserver v0.22.2 h1:zK7qI8Ery7j2CaN23UCFaC1hj7dMiI87n01+nKuewd4= +k8s.io/apiextensions-apiserver v0.22.2/go.mod h1:2E0Ve/isxNl7tWLSUDgi6+cmwHi5fQRdwGVCxbC+KFA= k8s.io/apimachinery v0.22.2 h1:ejz6y/zNma8clPVfNDLnPbleBo6MpoFy/HBiBqCouVk= k8s.io/apimachinery v0.22.2/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0= -k8s.io/apiserver v0.22.1/go.mod h1:2mcM6dzSt+XndzVQJX21Gx0/Klo7Aen7i0Ai6tIa400= -k8s.io/client-go v0.22.1/go.mod h1:BquC5A4UOo4qVDUtoc04/+Nxp1MeHcVc1HJm1KmG8kk= +k8s.io/apiserver v0.22.2/go.mod h1:vrpMmbyjWrgdyOvZTSpsusQq5iigKNWv9o9KlDAbBHI= k8s.io/client-go v0.22.2 h1:DaSQgs02aCC1QcwUdkKZWOeaVsQjYvWv8ZazcZ6JcHc= k8s.io/client-go v0.22.2/go.mod h1:sAlhrkVDf50ZHx6z4K0S40wISNTarf1r800F+RlCF6U= -k8s.io/code-generator v0.22.1/go.mod h1:eV77Y09IopzeXOJzndrDyCI88UBok2h6WxAlBwpxa+o= -k8s.io/component-base v0.22.1 h1:SFqIXsEN3v3Kkr1bS6rstrs1wd45StJqbtgbQ4nRQdo= -k8s.io/component-base v0.22.1/go.mod h1:0D+Bl8rrnsPN9v0dyYvkqFfBeAd4u7n77ze+p8CMiPo= +k8s.io/code-generator v0.22.2/go.mod h1:eV77Y09IopzeXOJzndrDyCI88UBok2h6WxAlBwpxa+o= +k8s.io/component-base v0.22.2 h1:vNIvE0AIrLhjX8drH0BgCNJcR4QZxMXcJzBsDplDx9M= +k8s.io/component-base v0.22.2/go.mod h1:5Br2QhI9OTe79p+TzPe9JKNQYvEKbq9rTJDWllunGug= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= @@ -841,16 +838,14 @@ k8s.io/klog/v2 v2.9.0 h1:D7HV+n1V57XeZ0m6tdRkfknthUaM06VFbWldOFh8kzM= k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e h1:KLHHjkdQFomZy8+06csTWZ0m1343QqxZhR2LJ1OxCYM= k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= -k8s.io/utils v0.0.0-20210707171843-4b05e18ac7d9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a h1:8dYfu/Fc9Gz2rNJKB9IQRGgQOh2clmRzNIPPY1xLY5g= k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= -sigs.k8s.io/controller-runtime v0.10.0 h1:HgyZmMpjUOrtkaFtCnfxsR1bGRuFoAczSNbn2MoKj5U= -sigs.k8s.io/controller-runtime v0.10.0/go.mod h1:GCdh6kqV6IY4LK0JLwX0Zm6g233RtVGdb/f0+KSfprg= +sigs.k8s.io/controller-runtime v0.10.1 h1:+eLHgY/VrJWnfg6iXUqhCUqNXgPH1NZeP9drNAAgWlg= +sigs.k8s.io/controller-runtime v0.10.1/go.mod h1:CQp8eyUQZ/Q7PJvnIrB6/hgfTC1kBkGylwsLgOQi1WY= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.1.2 h1:Hr/htKFmJEbtMgS/UD0N+gtgctAqz81t3nu+sPzynno= sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= From 5a232bb3b428a418ad9346f18d5efac1da3d976b Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Wed, 6 Oct 2021 11:03:38 +0100 Subject: [PATCH 408/790] Added merge.StringToBoolMap (#741) --- pkg/util/merge/merge.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pkg/util/merge/merge.go b/pkg/util/merge/merge.go index adfe0a539..c532e1757 100644 --- a/pkg/util/merge/merge.go +++ b/pkg/util/merge/merge.go @@ -34,6 +34,19 @@ func StringToStringMap(map1, map2 map[string]string) map[string]string { return mergedMap } +// StringToBoolMap merges two string-to-bool maps together with the second map +// overriding any values also specified in the first. +func StringToBoolMap(map1, map2 map[string]bool) map[string]bool { + mergedMap := make(map[string]bool) + for k, v := range map1 { + mergedMap[k] = v + } + for k, v := range map2 { + mergedMap[k] = v + } + return mergedMap +} + // Containers merges two slices of containers merging each item by container name. func Containers(defaultContainers, overrideContainers []corev1.Container) []corev1.Container { mergedContainerMap := map[string]corev1.Container{} From ae1ca8fa4eb79cc4dc557752015c4520633d1d29 Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Tue, 12 Oct 2021 09:45:48 +0100 Subject: [PATCH 409/790] Add function to secret package (#749) --- pkg/kube/secret/secret.go | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/pkg/kube/secret/secret.go b/pkg/kube/secret/secret.go index 2d14088b7..3c1c2e383 100644 --- a/pkg/kube/secret/secret.go +++ b/pkg/kube/secret/secret.go @@ -1,6 +1,8 @@ package secret import ( + "reflect" + "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" @@ -71,11 +73,15 @@ func ReadStringData(getter Getter, key client.ObjectKey) (map[string]string, err return nil, err } + return dataToStringData(secret.Data), nil +} + +func dataToStringData(data map[string][]byte) map[string]string { stringData := make(map[string]string) - for k, v := range secret.Data { + for k, v := range data { stringData[k] = string(v) } - return stringData, nil + return stringData } // UpdateField updates a single field in the secret with the provided objectKey @@ -163,3 +169,23 @@ func Exists(secretGetter Getter, nsName types.NamespacedName) (bool, error) { } return true, nil } + +// CreateOrUpdateIfNeeded creates a secret if it doesn't exists, or updates it if needed. +func CreateOrUpdateIfNeeded(getUpdateCreator GetUpdateCreator, secret corev1.Secret) error { + // Check if the secret exists + olsSecret, err := getUpdateCreator.GetSecret(types.NamespacedName{Name: secret.Name, Namespace: secret.Namespace}) + if err != nil { + + if apiErrors.IsNotFound(err) { + return getUpdateCreator.CreateSecret(secret) + } + return err + } + + if reflect.DeepEqual(secret.StringData, dataToStringData(olsSecret.Data)) { + return nil + } + + // They are different so we need to update it + return getUpdateCreator.UpdateSecret(secret) +} From 4e429799d3eea09af14ba2a62beb89c6ed7e37b7 Mon Sep 17 00:00:00 2001 From: Rodrigo Valin Date: Wed, 13 Oct 2021 11:49:44 +0100 Subject: [PATCH 410/790] CLOUDP-101992: Adds helm chart as submodule (#745) --- .action_templates/steps/checkout-fork.yaml | 1 + .action_templates/steps/checkout.yaml | 2 + .github/workflows/e2e-dispatch.yml | 4 + .github/workflows/e2e-fork.yml | 2 + .github/workflows/e2e.yml | 4 + .github/workflows/main.yaml | 6 +- .gitmodules | 3 + Makefile | 16 +- helm-chart/Chart.yaml | 14 - ...ommunity.mongodb.com_mongodbcommunity.yaml | 354 ------------------ helm-chart/templates/database_roles.yaml | 48 --- .../mongodbcommunity_cr_with_tls.yaml | 79 ---- helm-chart/templates/operator.yaml | 78 ---- helm-chart/templates/operator_roles.yaml | 79 ---- helm-chart/values.yaml | 54 --- helm-charts | 1 + scripts/dev/get_e2e_env_vars.py | 2 +- test/e2e/setup/test_config.go | 2 +- 18 files changed, 32 insertions(+), 717 deletions(-) create mode 100644 .gitmodules delete mode 100644 helm-chart/Chart.yaml delete mode 100644 helm-chart/crds/mongodbcommunity.mongodb.com_mongodbcommunity.yaml delete mode 100644 helm-chart/templates/database_roles.yaml delete mode 100644 helm-chart/templates/mongodbcommunity_cr_with_tls.yaml delete mode 100644 helm-chart/templates/operator.yaml delete mode 100644 helm-chart/templates/operator_roles.yaml delete mode 100644 helm-chart/values.yaml create mode 160000 helm-charts diff --git a/.action_templates/steps/checkout-fork.yaml b/.action_templates/steps/checkout-fork.yaml index f5ce58770..579100bf2 100644 --- a/.action_templates/steps/checkout-fork.yaml +++ b/.action_templates/steps/checkout-fork.yaml @@ -6,3 +6,4 @@ with: ref: ${{github.event.pull_request.head.ref}} repository: ${{github.event.pull_request.head.repo.full_name}} + submodules: true diff --git a/.action_templates/steps/checkout.yaml b/.action_templates/steps/checkout.yaml index 6edd51386..8d3e07859 100644 --- a/.action_templates/steps/checkout.yaml +++ b/.action_templates/steps/checkout.yaml @@ -1,2 +1,4 @@ - name: Checkout Code uses: actions/checkout@v2 + with: + submodules: true diff --git a/.github/workflows/e2e-dispatch.yml b/.github/workflows/e2e-dispatch.yml index cffb42f41..a6d4ce45a 100644 --- a/.github/workflows/e2e-dispatch.yml +++ b/.github/workflows/e2e-dispatch.yml @@ -50,6 +50,8 @@ jobs: # template: .action_templates/steps/checkout.yaml - name: Checkout Code uses: actions/checkout@v2 + with: + submodules: true # template: .action_templates/steps/setup-and-install-python.yaml - name: Setup Python uses: actions/setup-python@v2 @@ -84,6 +86,8 @@ jobs: # template: .action_templates/steps/checkout.yaml - name: Checkout Code uses: actions/checkout@v2 + with: + submodules: true # template: .action_templates/steps/setup-and-install-python.yaml - name: Setup Python uses: actions/setup-python@v2 diff --git a/.github/workflows/e2e-fork.yml b/.github/workflows/e2e-fork.yml index e5ec665ca..fd37e1227 100644 --- a/.github/workflows/e2e-fork.yml +++ b/.github/workflows/e2e-fork.yml @@ -53,6 +53,7 @@ jobs: with: ref: ${{github.event.pull_request.head.ref}} repository: ${{github.event.pull_request.head.repo.full_name}} + submodules: true # template: .action_templates/steps/setup-and-install-python.yaml - name: Setup Python uses: actions/setup-python@v2 @@ -186,6 +187,7 @@ jobs: with: ref: ${{github.event.pull_request.head.ref}} repository: ${{github.event.pull_request.head.repo.full_name}} + submodules: true # template: .action_templates/steps/set-run-status.yaml - name: Set default run status run: echo "::set-output name=last_run_status::pending" > last_run_status diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index a63df064d..07e74fe0a 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -58,6 +58,8 @@ jobs: # template: .action_templates/steps/checkout.yaml - name: Checkout Code uses: actions/checkout@v2 + with: + submodules: true # template: .action_templates/steps/setup-and-install-python.yaml - name: Setup Python uses: actions/setup-python@v2 @@ -188,6 +190,8 @@ jobs: # template: .action_templates/steps/checkout.yaml - name: Checkout Code uses: actions/checkout@v2 + with: + submodules: true # template: .action_templates/steps/set-run-status.yaml - name: Set default run status run: echo "::set-output name=last_run_status::pending" > last_run_status diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index bd0b7e0d4..f278b0dc8 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -37,6 +37,9 @@ jobs: ########################## - name: Checkout Code uses: actions/checkout@v2 + with: + # Make sure we also get the helm-charts submodule! + submodules: true - name: Install missing python packages run: sudo apt-get install -y --no-install-recommends python3-venv python3-setuptools @@ -68,6 +71,5 @@ jobs: VALIDATE_YAML: true VALIDATE_PYTHON: true VALIDATE_BASH: true - FILTER_REGEX_EXCLUDE: "/helm-chart/templates/*" + FILTER_REGEX_EXCLUDE: "/helm-charts/charts/community-operator/templates/*" # VALIDATE_GO: true This is currently broken: https://github.com/github/super-linter/issues/143 -... diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..80d9434c7 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "helm-charts"] + path = helm-charts + url = git@github.com:mongodb/helm-charts.git diff --git a/Makefile b/Makefile index f39ece048..e04d7c761 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,8 @@ READINESS_PROBE_IMG := $(shell jq -r .readiness_probe_image < ~/.community-opera REGISTRY := $(shell jq -r .repo_url < ~/.community-operator-dev/config.json) AGENT_IMAGE_NAME := $(shell jq -r .agent_image_ubuntu < ~/.community-operator-dev/config.json) +HELM_CHART ?= ./helm-charts/charts/community-operator + STRING_SET_VALUES := --set namespace=$(NAMESPACE),versionUpgradeHook.name=$(UPGRADE_HOOK_IMG),readinessProbe.name=$(READINESS_PROBE_IMG),registry.operator=$(REPO_URL),operator.operatorImageName=$(OPERATOR_IMAGE),operator.version=latest,registry.agent=$(REGISTRY),registry.versionUpgradeHook=$(REGISTRY),registry.readinessProbe=$(REGISTRY),registry.operator=$(REGISTRY),versionUpgradeHook.version=latest,readinessProbe.version=latest,agent.version=latest,agent.name=$(AGENT_IMAGE_NAME) DOCKERFILE ?= operator @@ -50,14 +52,14 @@ install-crd: kubectl apply -f config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml install-chart: - $(HELM) upgrade --install $(STRING_SET_VALUES) $(RELEASE_NAME_HELM) helm-chart + $(HELM) upgrade --install $(STRING_SET_VALUES) $(RELEASE_NAME_HELM) $(HELM_CHART) install-chart-with-tls-enabled: - $(HELM) upgrade --install --set createResource=true $(STRING_SET_VALUES) $(RELEASE_NAME_HELM) helm-chart + $(HELM) upgrade --install --set createResource=true $(STRING_SET_VALUES) $(RELEASE_NAME_HELM) $(HELM_CHART) install-rbac: - $(HELM) template $(STRING_SET_VALUES) -s templates/database_roles.yaml helm-chart | kubectl apply -f - - $(HELM) template $(STRING_SET_VALUES) -s templates/operator_roles.yaml helm-chart | kubectl apply -f - + $(HELM) template $(STRING_SET_VALUES) -s templates/database_roles.yaml $(HELM_CHART) | kubectl apply -f - + $(HELM) template $(STRING_SET_VALUES) -s templates/operator_roles.yaml $(HELM_CHART) | kubectl apply -f - uninstall-crd: kubectl delete crd mongodbcommunity.mongodbcommunity.mongodb.com @@ -66,8 +68,8 @@ uninstall-chart: $(HELM) uninstall $(RELEASE_NAME_HELM) -n $(NAMESPACE) uninstall-rbac: - $(HELM) template $(STRING_SET_VALUES) -s templates/database_roles.yaml helm-chart | kubectl delete -f - - $(HELM) template $(STRING_SET_VALUES) -s templates/operator_roles.yaml helm-chart | kubectl delete -f - + $(HELM) template $(STRING_SET_VALUES) -s templates/database_roles.yaml $(HELM_CHART) | kubectl delete -f - + $(HELM) template $(STRING_SET_VALUES) -s templates/operator_roles.yaml $(HELM_CHART) | kubectl delete -f - # Uninstall CRDs from a cluster @@ -82,7 +84,7 @@ undeploy: uninstall-chart uninstall-crd # Generate manifests e.g. CRD, RBAC etc. manifests: controller-gen $(CONTROLLER_GEN) $(CRD_OPTIONS) paths="./..." output:crd:artifacts:config=config/crd/bases - cp config/crd/bases/* helm-chart/crds + cp config/crd/bases/* $(HELM_CHART)/crds # Run go fmt against code fmt: diff --git a/helm-chart/Chart.yaml b/helm-chart/Chart.yaml deleted file mode 100644 index 8828285bf..000000000 --- a/helm-chart/Chart.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: "v2" -name: mongodb-kubernetes-operator -description: MongoDB Kubernetes Community Operator -version: 0.7.0 -type: application -appVersion: "0.7.0" -kubeVersion: '>=1.16-0' -keywords: -- mongodb -- database -- nosql -home: https://github.com/mongodb/mongodb-kubernetes-operator/ -maintainers: -- name: Cloud Team diff --git a/helm-chart/crds/mongodbcommunity.mongodb.com_mongodbcommunity.yaml b/helm-chart/crds/mongodbcommunity.mongodb.com_mongodbcommunity.yaml deleted file mode 100644 index ae01f98f8..000000000 --- a/helm-chart/crds/mongodbcommunity.mongodb.com_mongodbcommunity.yaml +++ /dev/null @@ -1,354 +0,0 @@ - ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.4.1 - creationTimestamp: null - name: mongodbcommunity.mongodbcommunity.mongodb.com -spec: - group: mongodbcommunity.mongodb.com - names: - kind: MongoDBCommunity - listKind: MongoDBCommunityList - plural: mongodbcommunity - shortNames: - - mdbc - singular: mongodbcommunity - scope: Namespaced - versions: - - additionalPrinterColumns: - - description: Current state of the MongoDB deployment - jsonPath: .status.phase - name: Phase - type: string - - description: Version of MongoDB server - jsonPath: .status.version - name: Version - type: string - name: v1 - schema: - openAPIV3Schema: - description: MongoDBCommunity is the Schema for the mongodbs API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: MongoDBCommunitySpec defines the desired state of MongoDB - properties: - additionalMongodConfig: - description: 'AdditionalMongodConfig is additional configuration that - can be passed to each data-bearing mongod at runtime. Uses the same - structure as the mongod configuration file: https://docs.mongodb.com/manual/reference/configuration-options/' - nullable: true - type: object - x-kubernetes-preserve-unknown-fields: true - arbiters: - description: Arbiters is the number of arbiters (each counted as a - member) in the replica set - type: integer - featureCompatibilityVersion: - description: FeatureCompatibilityVersion configures the feature compatibility - version that will be set for the deployment - type: string - members: - description: Members is the number of members in the replica set - type: integer - replicaSetHorizons: - description: ReplicaSetHorizons Add this parameter and values if you - need your database to be accessed outside of Kubernetes. This setting - allows you to provide different DNS settings within the Kubernetes - cluster and to the Kubernetes cluster. The Kubernetes Operator uses - split horizon DNS for replica set members. This feature allows communication - both within the Kubernetes cluster and from outside Kubernetes. - items: - additionalProperties: - type: string - type: object - type: array - security: - description: Security configures security features, such as TLS, and - authentication settings for a deployment - properties: - authentication: - properties: - ignoreUnknownUsers: - default: true - nullable: true - type: boolean - modes: - description: Modes is an array specifying which authentication - methods should be enabled. - items: - enum: - - SCRAM - - SCRAM-SHA-256 - - SCRAM-SHA-1 - type: string - type: array - required: - - modes - type: object - roles: - description: User-specified custom MongoDB roles that should be - configured in the deployment. - items: - description: CustomRole defines a custom MongoDB role. - properties: - authenticationRestrictions: - description: The authentication restrictions the server - enforces on the role. - items: - description: AuthenticationRestriction specifies a list - of IP addresses and CIDR ranges users are allowed to - connect to or from. - properties: - clientSource: - items: - type: string - type: array - serverAddress: - items: - type: string - type: array - required: - - clientSource - - serverAddress - type: object - type: array - db: - description: The database of the role. - type: string - privileges: - description: The privileges to grant the role. - items: - description: Privilege defines the actions a role is allowed - to perform on a given resource. - properties: - actions: - items: - type: string - type: array - resource: - description: Resource specifies specifies the resources - upon which a privilege permits actions. See https://docs.mongodb.com/manual/reference/resource-document - for more. - properties: - anyResource: - type: boolean - cluster: - type: boolean - collection: - type: string - db: - type: string - type: object - required: - - actions - - resource - type: object - type: array - role: - description: The name of the role. - type: string - roles: - description: An array of roles from which this role inherits - privileges. - items: - description: Role is the database role this user should - have - properties: - db: - description: DB is the database the role can act on - type: string - name: - description: Name is the name of the role - type: string - required: - - db - - name - type: object - type: array - required: - - db - - privileges - - role - type: object - type: array - tls: - description: TLS configuration for both client-server and server-server - communication - properties: - caCertificateSecretRef: - description: CaCertificateSecret is a reference to a Secret - containing the certificate for the CA which signed the server - certificates The certificate is expected to be available - under the key "ca.crt" - properties: - name: - type: string - required: - - name - type: object - caConfigMapRef: - description: CaConfigMap is a reference to a ConfigMap containing - the certificate for the CA which signed the server certificates - The certificate is expected to be available under the key - "ca.crt" This field is ignored when CaCertificateSecretRef - is configured - properties: - name: - type: string - required: - - name - type: object - certificateKeySecretRef: - description: CertificateKeySecret is a reference to a Secret - containing a private key and certificate to use for TLS. - The key and cert are expected to be PEM encoded and available - at "tls.key" and "tls.crt". This is the same format used - for the standard "kubernetes.io/tls" Secret type, but no - specific type is required. Alternatively, an entry tls.pem, - containing the concatenation of cert and key, can be provided. - If all of tls.pem, tls.crt and tls.key are present, the - tls.pem one needs to be equal to the concatenation of tls.crt - and tls.key - properties: - name: - type: string - required: - - name - type: object - enabled: - type: boolean - optional: - description: Optional configures if TLS should be required - or optional for connections - type: boolean - required: - - enabled - type: object - type: object - statefulSet: - description: StatefulSetConfiguration holds the optional custom StatefulSet - that should be merged into the operator created one. - properties: - spec: - type: object - x-kubernetes-preserve-unknown-fields: true - required: - - spec - type: object - type: - description: Type defines which type of MongoDB deployment the resource - should create - enum: - - ReplicaSet - type: string - users: - description: Users specifies the MongoDB users that should be configured - in your deployment - items: - properties: - db: - description: DB is the database the user is stored in. Defaults - to "admin" - type: string - name: - description: Name is the username of the user - type: string - passwordSecretRef: - description: PasswordSecretRef is a reference to the secret - containing this user's password - properties: - key: - description: Key is the key in the secret storing this password. - Defaults to "password" - type: string - name: - description: Name is the name of the secret storing this - user's password - type: string - required: - - name - type: object - roles: - description: Roles is an array of roles assigned to this user - items: - description: Role is the database role this user should have - properties: - db: - description: DB is the database the role can act on - type: string - name: - description: Name is the name of the role - type: string - required: - - db - - name - type: object - type: array - scramCredentialsSecretName: - description: ScramCredentialsSecretName appended by string "scram-credentials" - is the name of the secret object created by the mongoDB operator - for storing SCRAM credentials - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - required: - - name - - passwordSecretRef - - roles - - scramCredentialsSecretName - type: object - type: array - version: - description: Version defines which version of MongoDB will be used - type: string - required: - - security - - type - - users - type: object - status: - description: MongoDBCommunityStatus defines the observed state of MongoDB - properties: - currentMongoDBMembers: - type: integer - currentStatefulSetReplicas: - type: integer - message: - type: string - mongoUri: - type: string - phase: - type: string - version: - type: string - required: - - currentMongoDBMembers - - currentStatefulSetReplicas - - mongoUri - - phase - type: object - type: object - served: true - storage: true - subresources: - status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/helm-chart/templates/database_roles.yaml b/helm-chart/templates/database_roles.yaml deleted file mode 100644 index a91f6903d..000000000 --- a/helm-chart/templates/database_roles.yaml +++ /dev/null @@ -1,48 +0,0 @@ ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ .Values.database.name }} - {{- if .Values.namespace }} - namespace: {{ .Values.namespace }} - {{- end }} - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: {{ .Values.database.name }} - {{- if .Values.namespace }} - namespace: {{ .Values.namespace }} - {{- end }} -rules: - - apiGroups: - - "" - resources: - - secrets - verbs: - - get - - apiGroups: - - "" - resources: - - pods - verbs: - - patch - - delete - - get - ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: {{ .Values.database.name }} - {{- if .Values.namespace }} - namespace: {{ .Values.namespace }} - {{- end }} -subjects: - - kind: ServiceAccount - name: {{ .Values.database.name }} -roleRef: - kind: Role - name: {{ .Values.database.name }} - apiGroup: rbac.authorization.k8s.io diff --git a/helm-chart/templates/mongodbcommunity_cr_with_tls.yaml b/helm-chart/templates/mongodbcommunity_cr_with_tls.yaml deleted file mode 100644 index 2157fcb2c..000000000 --- a/helm-chart/templates/mongodbcommunity_cr_with_tls.yaml +++ /dev/null @@ -1,79 +0,0 @@ -{{- if and .Values.resource.tls.enabled .Values.resource.tls.useCertManager }} -# cert-manager resources -apiVersion: cert-manager.io/v1 -kind: Issuer -metadata: - name: tls-selfsigned-issuer - namespace: {{ .Values.namespace }} -spec: - selfSigned: {} ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: tls-selfsigned-ca - namespace: {{ .Values.namespace }} -spec: - isCA: true - commonName: "*.{{ .Values.resource.name }}-svc.{{ .Values.namespace }}.svc.cluster.local" - dnsNames: - - "*.{{ .Values.resource.name }}-svc.{{ .Values.namespace }}.svc.cluster.local" - secretName: {{ .Values.resource.tls.caCertificateSecretRef }} - privateKey: - algorithm: ECDSA - size: 256 - issuerRef: - name: tls-selfsigned-issuer - kind: Issuer ---- -apiVersion: cert-manager.io/v1 -kind: Issuer -metadata: - name: tls-ca-issuer - namespace: {{ .Values.namespace }} -spec: - ca: - secretName: {{ .Values.resource.tls.caCertificateSecretRef }} ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: cert-manager-tls-certificate - namespace: {{ .Values.namespace }} -spec: - secretName: {{ .Values.resource.tls.certificateKeySecretRef }} - issuerRef: - name: tls-ca-issuer - kind: Issuer - duration: 8760h # 365 days - renewBefore: 720h # 30 days - commonName: "*.{{ .Values.resource.name }}-svc.{{ .Values.namespace }}.svc.cluster.local" - dnsNames: - - "*.{{ .Values.resource.name }}-svc.{{ .Values.namespace }}.svc.cluster.local" -{{- end }} - -# mongodb resources -{{- if .Values.createResource }} ---- -apiVersion: mongodbcommunity.mongodb.com/v1 -kind: MongoDBCommunity -metadata: - name: {{ .Values.resource.name }} - namespace: {{ .Values.namespace }} -spec: - members: {{ .Values.resource.members }} - type: ReplicaSet - version: {{ .Values.resource.version }} - security: - tls: - enabled: {{ .Values.resource.tls.enabled }} - {{- if .Values.resource.tls.enabled }} - certificateKeySecretRef: - name: {{ .Values.resource.tls.certificateKeySecretRef }} - caCertificateSecretRef: - name: {{ .Values.resource.tls.caCertificateSecretRef }} - {{- end }} - authentication: - modes: ["SCRAM"] - users: [] -{{- end }} \ No newline at end of file diff --git a/helm-chart/templates/operator.yaml b/helm-chart/templates/operator.yaml deleted file mode 100644 index a89b817b5..000000000 --- a/helm-chart/templates/operator.yaml +++ /dev/null @@ -1,78 +0,0 @@ ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - annotations: - email: support@mongodb.com - labels: - owner: mongodb - name: {{ .Values.operator.name }} - {{- if .Values.namespace }} - namespace: {{ .Values.namespace }} - {{- end }} -spec: - replicas: 1 - selector: - matchLabels: - name: {{ .Values.operator.name }} - strategy: - rollingUpdate: - maxUnavailable: 1 - type: RollingUpdate - template: - metadata: - labels: - name: {{ .Values.operator.name }} - spec: - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: name - operator: In - values: - - {{ .Values.operator.name }} - topologyKey: kubernetes.io/hostname - containers: - - command: - - /usr/local/bin/entrypoint - env: - - name: WATCH_NAMESPACE -{{- if .Values.operator.watchNamespace}} - value: "{{ .Values.operator.watchNamespace }}" -{{- else }} - valueFrom: - fieldRef: - fieldPath: metadata.namespace -{{- end }} - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: OPERATOR_NAME - value: {{ .Values.operator.name }} - - name: AGENT_IMAGE - value: "{{ .Values.registry.agent }}/{{ .Values.agent.name }}:{{ .Values.agent.version }}" - - name: VERSION_UPGRADE_HOOK_IMAGE - value: "{{ .Values.registry.versionUpgradeHook }}/{{ .Values.versionUpgradeHook.name }}:{{ .Values.versionUpgradeHook.version }}" - - name: READINESS_PROBE_IMAGE - value: "{{ .Values.registry.readinessProbe }}/{{ .Values.readinessProbe.name }}:{{ .Values.readinessProbe.version }}" - - name: MONGODB_IMAGE - value: {{ .Values.mongodb.name }} - - name: MONGODB_REPO_URL - value: {{ .Values.mongodb.repo }} - image: {{ .Values.registry.operator }}/{{ .Values.operator.operatorImageName }}:{{ .Values.operator.version }} - imagePullPolicy: {{ .Values.registry.pullPolicy}} - name: {{ .Values.operator.deploymentName }} - resources: - limits: - cpu: 1100m - memory: 1Gi - requests: - cpu: 500m - memory: 200Mi - securityContext: - readOnlyRootFilesystem: true - runAsUser: 2000 - serviceAccountName: {{ .Values.operator.name }} diff --git a/helm-chart/templates/operator_roles.yaml b/helm-chart/templates/operator_roles.yaml deleted file mode 100644 index 5b872dfb1..000000000 --- a/helm-chart/templates/operator_roles.yaml +++ /dev/null @@ -1,79 +0,0 @@ ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ .Values.operator.name }} - {{- if .Values.namespace }} - namespace: {{ .Values.namespace }} - {{- end }} - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: {{ if eq (.Values.operator.watchNamespace | default "") "*" }} ClusterRole {{ else }} Role {{ end }} -metadata: - name: {{ .Values.operator.name }} - {{- if not (eq (.Values.operator.watchNamespace | default "*") "*") }} - namespace: {{ .Values.operator.watchNamespace }} - {{- else }} - namespace: {{ .Values.namespace }} - {{- end }} -rules: -- apiGroups: - - "" - resources: - - pods - - services - - configmaps - - secrets - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - apps - resources: - - statefulsets - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - mongodbcommunity.mongodb.com - resources: - - mongodbcommunity - - mongodbcommunity/status - - mongodbcommunity/spec - - mongodbcommunity/finalizers - verbs: - - get - - patch - - list - - update - - watch - ---- -kind: {{ if eq (.Values.operator.watchNamespace | default "") "*" }} ClusterRoleBinding {{ else }} RoleBinding {{ end }} -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: {{ .Values.operator.name }} - {{- if .Values.namespace }} - namespace: {{ .Values.namespace }} - {{- end }} -subjects: -- kind: ServiceAccount - name: {{ .Values.operator.name }} -{{ if eq (.Values.operator.watchNamespace | default "") "*" }} - namespace: {{ .Values.namespace }} -{{ end }} -roleRef: - kind: {{ if eq (.Values.operator.watchNamespace | default "") "*" }} ClusterRole {{ else }} Role {{ end }} - name: {{ .Values.operator.name }} - apiGroup: rbac.authorization.k8s.io diff --git a/helm-chart/values.yaml b/helm-chart/values.yaml deleted file mode 100644 index 76eca1553..000000000 --- a/helm-chart/values.yaml +++ /dev/null @@ -1,54 +0,0 @@ -# Name of the Namespace to use -namespace: mongodb - -## Operator -operator: - # Name that will be assigned to most of internal Kubernetes objects like Deployment, ServiceAccount, Role etc. - name: mongodb-kubernetes-operator - - # Name of the operator image - operatorImageName: mongodb-kubernetes-operator - - # Name of the deployment of the operator pod - deploymentName: mongodb-kubernetes-operator - - # Version of mongodb-kubernetes-operator - version: 0.7.0 - -## Operator's database -database: - name: mongodb-database - -agent: - name: mongodb-agent - version: 11.0.5.6963-1 -versionUpgradeHook: - name: mongodb-kubernetes-operator-version-upgrade-post-start-hook - version: 1.0.2 -readinessProbe: - name: mongodb-kubernetes-readinessprobe - version: 1.0.4 -mongodb: - name: mongo - repo: docker.io - -registry: - agent: quay.io/mongodb - versionUpgradeHook: quay.io/mongodb - readinessProbe: quay.io/mongodb - operator: quay.io/mongodb - pullPolicy: Always - -createResource: false -resource: - name: mongodb-replica-set - version: 4.4.0 - members: 3 - tls: - enabled: true - useCertManager: true - certificateKeySecretRef: tls-certificate - caCertificateSecretRef: tls-ca-key-pair - certManager: - certDuration: 8760h # 365 days - renewCertBefore: 720h # 30 days diff --git a/helm-charts b/helm-charts new file mode 160000 index 000000000..ec377b07a --- /dev/null +++ b/helm-charts @@ -0,0 +1 @@ +Subproject commit ec377b07a54aa26c9b0d2fccf65609347bc6ff1a diff --git a/scripts/dev/get_e2e_env_vars.py b/scripts/dev/get_e2e_env_vars.py index c4a018d90..7f0435fe4 100755 --- a/scripts/dev/get_e2e_env_vars.py +++ b/scripts/dev/get_e2e_env_vars.py @@ -31,7 +31,7 @@ def _get_e2e_test_envs(dev_config: DevConfig) -> Dict[str, str]: "WATCH_NAMESPACE": dev_config.namespace, "MONGODB_IMAGE": "mongo", "MONGODB_REPO_URL": "docker.io", - "HELM_CHART_PATH": os.path.abspath("./helm-chart"), + "HELM_CHART_PATH": os.path.abspath("./helm-charts/charts/community-operator"), } diff --git a/test/e2e/setup/test_config.go b/test/e2e/setup/test_config.go index 288333969..b45aa7c21 100644 --- a/test/e2e/setup/test_config.go +++ b/test/e2e/setup/test_config.go @@ -39,6 +39,6 @@ func loadTestConfigFromEnv() TestConfig { ClusterWide: envvar.ReadBool(clusterWideEnvName), PerformCleanup: envvar.ReadBool(performCleanupEnvName), ReadinessProbeImage: envvar.GetEnvOrDefault(construct.ReadinessProbeImageEnv, "quay.io/mongodb/mongodb-kubernetes-readinessprobe:1.0.3"), - HelmChartPath: envvar.GetEnvOrDefault(helmChartPathEnvName, "/workspace/helm-chart"), + HelmChartPath: envvar.GetEnvOrDefault(helmChartPathEnvName, "/workspace/helm-charts/charts/community-operator"), } } From d0d546636a764f2bcbee7a7920bfcec2b54fa677 Mon Sep 17 00:00:00 2001 From: Rajdeep Das Date: Thu, 21 Oct 2021 10:19:22 +0200 Subject: [PATCH 411/790] Fix agent health-status file not being present as info instead of error (#754) --- cmd/versionhook/main.go | 2 +- release.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/versionhook/main.go b/cmd/versionhook/main.go index 11ddcf3f2..a7c339261 100644 --- a/cmd/versionhook/main.go +++ b/cmd/versionhook/main.go @@ -127,7 +127,7 @@ func waitForAgentHealthStatus() (agent.Health, error) { func getAgentHealthStatus() (agent.Health, error) { f, err := os.Open(os.Getenv(agentStatusFilePathEnv)) if err != nil { - return agent.Health{}, errors.Errorf("could not open file: %s", err) + return agent.Health{}, err } defer f.Close() diff --git a/release.json b/release.json index e2f9b8bad..8eb4811c5 100644 --- a/release.json +++ b/release.json @@ -1,6 +1,6 @@ { "mongodb-kubernetes-operator": "0.7.0", - "version-upgrade-hook": "1.0.2", + "version-upgrade-hook": "1.0.3", "readiness-probe": "1.0.5", "mongodb-agent": { "version": "11.0.5.6963-1", From 5f15071f081fbe17904ba6f6782638411d514df3 Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Thu, 21 Oct 2021 09:38:30 +0100 Subject: [PATCH 412/790] CLOUDP-101809: Respect value of storage.dbPath (#753) --- api/v1/mongodbcommunity_types.go | 9 ++++ .../construct/build_statefulset_test.go | 21 ++++++++- controllers/construct/mongodbstatefulset.go | 28 +++++++++--- controllers/replica_set_controller.go | 8 ++-- controllers/replicaset_controller_test.go | 45 ++++++++++++++++--- controllers/testdata/specify_data_dir.yaml | 26 +++++++++++ docs/RELEASE_NOTES.md | 6 +++ .../automation_config_builder.go | 14 +++++- .../replica_set_mongod_config_test.go | 23 +++++++--- 9 files changed, 156 insertions(+), 24 deletions(-) create mode 100644 controllers/testdata/specify_data_dir.yaml diff --git a/api/v1/mongodbcommunity_types.go b/api/v1/mongodbcommunity_types.go index 2070ae84e..9621c8243 100644 --- a/api/v1/mongodbcommunity_types.go +++ b/api/v1/mongodbcommunity_types.go @@ -3,6 +3,7 @@ package v1 import ( "encoding/json" "fmt" + "github.com/stretchr/objx" "net/url" "strings" @@ -405,6 +406,14 @@ type MongoDBCommunity struct { Status MongoDBCommunityStatus `json:"status,omitempty"` } +func (m MongoDBCommunity) GetMongodConfiguration() map[string]interface{} { + mongodConfig := objx.New(map[string]interface{}{}) + for k, v := range m.Spec.AdditionalMongodConfig.Object { + mongodConfig.Set(k, v) + } + return mongodConfig +} + func (m MongoDBCommunity) GetAgentPasswordSecretNamespacedName() types.NamespacedName { return types.NamespacedName{Name: m.Name + "-agent-password", Namespace: m.Namespace} } diff --git a/controllers/construct/build_statefulset_test.go b/controllers/construct/build_statefulset_test.go index a68910ffd..77d2ba2cf 100644 --- a/controllers/construct/build_statefulset_test.go +++ b/controllers/construct/build_statefulset_test.go @@ -61,7 +61,7 @@ func TestMultipleCalls_DoNotCauseSideEffects(t *testing.T) { } func TestMongod_Container(t *testing.T) { - c := container.New(mongodbContainer("4.2", []corev1.VolumeMount{})) + c := container.New(mongodbContainer("4.2", []corev1.VolumeMount{}, map[string]interface{}{})) t.Run("Has correct Env vars", func(t *testing.T) { assert.Len(t, c.Env, 1) @@ -78,6 +78,25 @@ func TestMongod_Container(t *testing.T) { }) } +func TestGetDbPath(t *testing.T) { + t.Run("Test default is used if unspecifed", func(t *testing.T) { + m := map[string]interface{}{} + path := GetDBDataDir(m) + assert.Equal(t, defaultDataDir, path) + }) + + t.Run("Test storage.dbPath is used if specified", func(t *testing.T) { + m := map[string]interface{}{ + "storage": map[string]interface{}{ + "dbPath": "/data/db", + }, + } + + path := GetDBDataDir(m) + assert.Equal(t, "/data/db", path) + }) +} + func assertStatefulSetIsBuiltCorrectly(t *testing.T, mdb mdbv1.MongoDBCommunity, sts *appsv1.StatefulSet) { assert.Len(t, sts.Spec.Template.Spec.Containers, 2) assert.Len(t, sts.Spec.Template.Spec.InitContainers, 2) diff --git a/controllers/construct/mongodbstatefulset.go b/controllers/construct/mongodbstatefulset.go index ee1c2df35..df8438929 100644 --- a/controllers/construct/mongodbstatefulset.go +++ b/controllers/construct/mongodbstatefulset.go @@ -2,6 +2,7 @@ package construct import ( "fmt" + "github.com/stretchr/objx" "os" "strings" @@ -43,8 +44,9 @@ const ( ReadinessProbeImageEnv = "READINESS_PROBE_IMAGE" ManagedSecurityContextEnv = "MANAGED_SECURITY_CONTEXT" - automationconfFilePath = "/data/automation-mongod.conf" - keyfileFilePath = "/var/lib/mongodb-mms-automation/authentication/keyfile" + defaultDataDir = "/data" + automationMongodConfFileName = "automation-mongod.conf" + keyfileFilePath = "/var/lib/mongodb-mms-automation/authentication/keyfile" automationAgentOptions = " -skipMongoStart -noDaemonize -useLocalMongoDbTools" @@ -82,6 +84,9 @@ type MongoDBStatefulSetOwner interface { DataVolumeName() string // LogsVolumeName returns the name that the data volume should have LogsVolumeName() string + + // GetMongodConfiguration returns the MongoDB configuration for each member. + GetMongodConfiguration() map[string]interface{} } // BuildMongoDBReplicaSetStatefulSetModificationFunction builds the parts of the replica set that are common between every resource that implements @@ -122,14 +127,14 @@ func BuildMongoDBReplicaSetStatefulSetModificationFunction(mdb MongoDBStatefulSe singleModeVolumeClaim := func(s *appsv1.StatefulSet) {} if mdb.HasSeparateDataAndLogsVolumes() { logVolumeMount := statefulset.CreateVolumeMount(mdb.LogsVolumeName(), automationconfig.DefaultAgentLogPath) - dataVolumeMount := statefulset.CreateVolumeMount(mdb.DataVolumeName(), "/data") + dataVolumeMount := statefulset.CreateVolumeMount(mdb.DataVolumeName(), GetDBDataDir(mdb.GetMongodConfiguration())) dataVolumeClaim = statefulset.WithVolumeClaim(mdb.DataVolumeName(), dataPvc(mdb.DataVolumeName())) logVolumeClaim = statefulset.WithVolumeClaim(mdb.LogsVolumeName(), logsPvc(mdb.LogsVolumeName())) mongodbAgentVolumeMounts = append(mongodbAgentVolumeMounts, dataVolumeMount, logVolumeMount) mongodVolumeMounts = append(mongodVolumeMounts, dataVolumeMount, logVolumeMount) } else { mounts := []corev1.VolumeMount{ - statefulset.CreateVolumeMount(mdb.DataVolumeName(), "/data", statefulset.WithSubPath("data")), + statefulset.CreateVolumeMount(mdb.DataVolumeName(), GetDBDataDir(mdb.GetMongodConfiguration()), statefulset.WithSubPath("data")), statefulset.CreateVolumeMount(mdb.DataVolumeName(), automationconfig.DefaultAgentLogPath, statefulset.WithSubPath("logs")), } mongodbAgentVolumeMounts = append(mongodbAgentVolumeMounts, mounts...) @@ -165,7 +170,7 @@ func BuildMongoDBReplicaSetStatefulSetModificationFunction(mdb MongoDBStatefulSe podtemplatespec.WithVolume(keyFileVolume), podtemplatespec.WithServiceAccount(mongodbDatabaseServiceAccountName), podtemplatespec.WithContainer(AgentName, mongodbAgentContainer(mdb.AutomationConfigSecretName(), mongodbAgentVolumeMounts)), - podtemplatespec.WithContainer(MongodbName, mongodbContainer(mdb.GetMongoDBVersion(), mongodVolumeMounts)), + podtemplatespec.WithContainer(MongodbName, mongodbContainer(mdb.GetMongoDBVersion(), mongodVolumeMounts, mdb.GetMongodConfiguration())), podtemplatespec.WithInitContainer(versionUpgradeHookName, versionUpgradeHookInit([]corev1.VolumeMount{hooksVolumeMount})), podtemplatespec.WithInitContainer(ReadinessProbeContainerName, readinessProbeInit([]corev1.VolumeMount{scriptsVolumeMount})), ), @@ -276,7 +281,16 @@ func getMongoDBImage(version string) string { return fmt.Sprintf("%s/%s:%s", repoUrl, mongoImageName, version) } -func mongodbContainer(version string, volumeMounts []corev1.VolumeMount) container.Modification { +// GetDBDataDir returns the db path which should be used. +func GetDBDataDir(additionalMongoDBConfig objx.Map) string { + if additionalMongoDBConfig == nil { + return defaultDataDir + } + return additionalMongoDBConfig.Get("storage.dbPath").Str(defaultDataDir) +} + +func mongodbContainer(version string, volumeMounts []corev1.VolumeMount, additionalMongoDBConfig map[string]interface{}) container.Modification { + filePath := GetDBDataDir(additionalMongoDBConfig) + "/" + automationMongodConfFileName mongoDbCommand := fmt.Sprintf(` #run post-start hook to handle version changes /hooks/version-upgrade @@ -291,7 +305,7 @@ tail -F /var/log/mongodb-mms-automation/mongodb.log > /dev/stdout & # start mongod with this configuration exec mongod -f %s; -`, automationconfFilePath, keyfileFilePath, automationconfFilePath) +`, filePath, keyfileFilePath, filePath) containerCommand := []string{ "/bin/sh", diff --git a/controllers/replica_set_controller.go b/controllers/replica_set_controller.go index 473465490..4c4e1f8a0 100644 --- a/controllers/replica_set_controller.go +++ b/controllers/replica_set_controller.go @@ -4,6 +4,8 @@ import ( "context" "encoding/json" "fmt" + "github.com/imdario/mergo" + "github.com/stretchr/objx" "os" "github.com/mongodb/mongodb-kubernetes-operator/controllers/predicates" @@ -25,13 +27,10 @@ import ( "github.com/pkg/errors" - "github.com/imdario/mergo" - "github.com/mongodb/mongodb-kubernetes-operator/pkg/authentication/scram" - "github.com/stretchr/objx" - "github.com/mongodb/mongodb-kubernetes-operator/controllers/construct" "github.com/mongodb/mongodb-kubernetes-operator/controllers/validation" "github.com/mongodb/mongodb-kubernetes-operator/controllers/watch" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/authentication/scram" "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/annotations" "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/podtemplatespec" @@ -441,6 +440,7 @@ func buildAutomationConfig(mdb mdbv1.MongoDBCommunity, auth automationconfig.Aut SetFCV(mdb.Spec.FeatureCompatibilityVersion). SetOptions(automationconfig.Options{DownloadBase: "/var/lib/mongodb-mms-automation"}). SetAuth(auth). + SetDataDir(construct.GetDBDataDir(mdb.GetMongodConfiguration())). AddModifications(getMongodConfigModification(mdb)). AddModifications(modifications...). Build() diff --git a/controllers/replicaset_controller_test.go b/controllers/replicaset_controller_test.go index 033991720..111153583 100644 --- a/controllers/replicaset_controller_test.go +++ b/controllers/replicaset_controller_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/container" "io/ioutil" "os" "reflect" @@ -611,7 +612,7 @@ func TestReplicaSet_IsScaledUpToDesiredMembers_WhenFirstCreated(t *testing.T) { } func TestVolumeClaimTemplates_Configuration(t *testing.T) { - sts := performReconciliationAndGetStatefulSet(t, "volume_claim_templates_mdb.yaml") + sts, _ := performReconciliationAndGetStatefulSet(t, "volume_claim_templates_mdb.yaml") assert.Len(t, sts.Spec.VolumeClaimTemplates, 3) @@ -626,7 +627,7 @@ func TestVolumeClaimTemplates_Configuration(t *testing.T) { } func TestChangeDataVolume_Configuration(t *testing.T) { - sts := performReconciliationAndGetStatefulSet(t, "change_data_volume.yaml") + sts, _ := performReconciliationAndGetStatefulSet(t, "change_data_volume.yaml") assert.Len(t, sts.Spec.VolumeClaimTemplates, 2) dataVolume := sts.Spec.VolumeClaimTemplates[0] @@ -639,7 +640,7 @@ func TestChangeDataVolume_Configuration(t *testing.T) { } func TestCustomStorageClass_Configuration(t *testing.T) { - sts := performReconciliationAndGetStatefulSet(t, "custom_storage_class.yaml") + sts, _ := performReconciliationAndGetStatefulSet(t, "custom_storage_class.yaml") dataVolume := sts.Spec.VolumeClaimTemplates[0] @@ -655,7 +656,7 @@ func TestCustomStorageClass_Configuration(t *testing.T) { } func TestCustomTaintsAndTolerations_Configuration(t *testing.T) { - sts := performReconciliationAndGetStatefulSet(t, "tolerations_example.yaml") + sts, _ := performReconciliationAndGetStatefulSet(t, "tolerations_example.yaml") assert.Len(t, sts.Spec.Template.Spec.Tolerations, 2) assert.Equal(t, "example-key", sts.Spec.Template.Spec.Tolerations[0].Key) @@ -667,7 +668,39 @@ func TestCustomTaintsAndTolerations_Configuration(t *testing.T) { assert.Equal(t, corev1.TaintEffectNoExecute, sts.Spec.Template.Spec.Tolerations[1].Effect) } -func performReconciliationAndGetStatefulSet(t *testing.T, filePath string) appsv1.StatefulSet { +func TestCustomDataDir_Configuration(t *testing.T) { + sts, c := performReconciliationAndGetStatefulSet(t, "specify_data_dir.yaml") + + agentContainer := container.GetByName("mongodb-agent", sts.Spec.Template.Spec.Containers) + assert.NotNil(t, agentContainer) + assertVolumeMountPath(t, agentContainer.VolumeMounts, "data-volume", "/some/path/db") + + mongoContainer := container.GetByName("mongod", sts.Spec.Template.Spec.Containers) + assert.NotNil(t, mongoContainer) + + lastCommand := mongoContainer.Command[len(agentContainer.Command)-1] + assert.Contains(t, lastCommand, "/some/path/db", "startup command should be using the newly specified path") + + ac, err := automationconfig.ReadFromSecret(c, types.NamespacedName{Name: "example-mongodb-config", Namespace: "test-ns"}) + assert.NoError(t, err) + + for _, p := range ac.Processes { + actualStoragePath := p.Args26.Get("storage.dbPath").String() + assert.Equal(t, "/some/path/db", actualStoragePath, "process dbPath should have been set") + } +} + +func assertVolumeMountPath(t *testing.T, mounts []corev1.VolumeMount, name, path string) { + for _, v := range mounts { + if v.Name == name { + assert.Equal(t, path, v.MountPath) + return + } + } + t.Fatalf("volume with name %s was not present!", name) +} + +func performReconciliationAndGetStatefulSet(t *testing.T, filePath string) (appsv1.StatefulSet, client.Client) { mdb, err := loadTestFixture(filePath) assert.NoError(t, err) mgr := client.NewManager(&mdb) @@ -678,7 +711,7 @@ func performReconciliationAndGetStatefulSet(t *testing.T, filePath string) appsv sts, err := mgr.Client.GetStatefulSet(mdb.NamespacedName()) assert.NoError(t, err) - return sts + return sts, mgr.Client } func generatePasswordsForAllUsers(mdb mdbv1.MongoDBCommunity, c client.Client) error { diff --git a/controllers/testdata/specify_data_dir.yaml b/controllers/testdata/specify_data_dir.yaml new file mode 100644 index 000000000..d2b80012c --- /dev/null +++ b/controllers/testdata/specify_data_dir.yaml @@ -0,0 +1,26 @@ +--- +apiVersion: mongodbcommunity.mongodb.com/v1 +kind: MongoDBCommunity +metadata: + name: example-mongodb + namespace: test-ns +spec: + members: 3 + type: ReplicaSet + version: "4.2.6" + security: + authentication: + modes: ["SCRAM"] + users: + - name: my-user + db: admin + passwordSecretRef: + name: my-user-password + roles: + - name: clusterAdmin + db: admin + - name: userAdminAnyDatabase + db: admin + scramCredentialsSecretName: my-scram + additionalMongodConfig: + storage.dbPath: /some/path/db diff --git a/docs/RELEASE_NOTES.md b/docs/RELEASE_NOTES.md index e2b5667f6..0328cada6 100644 --- a/docs/RELEASE_NOTES.md +++ b/docs/RELEASE_NOTES.md @@ -6,6 +6,12 @@ - MongoDB database of the statefulSet is managed using distinct Role, ServiceAccount and RoleBinding. - TLS Secret can also contain a single "tls.pem" entry, containing the concatenation of the certificate and key - If a TLS secret contains all of "tls.key", "tls.crt" and "tls.pem" entries, the operator will raise an error if the "tls.pem" one is not equal to the concatenation of "tls.crt" with "tls.key" + +## MongoDBCommunity Resource +* Changes +* Specifying `spec.additionalMongodConfig.storage.dbPath` will now be respected correctly. + + ## Updated Image Tags - mongodb-kubernetes-operator:0.7.1 diff --git a/pkg/automationconfig/automation_config_builder.go b/pkg/automationconfig/automation_config_builder.go index 6bd466fcd..e922d2b58 100644 --- a/pkg/automationconfig/automation_config_builder.go +++ b/pkg/automationconfig/automation_config_builder.go @@ -47,6 +47,7 @@ type Builder struct { cafilePath string sslConfig *TLS tlsConfig *TLS + dataDir string } func NewBuilder() *Builder { @@ -108,6 +109,11 @@ func (b *Builder) SetName(name string) *Builder { return b } +func (b *Builder) SetDataDir(dataDir string) *Builder { + b.dataDir = dataDir + return b +} + func (b *Builder) SetFCV(fcv string) *Builder { b.fcv = fcv return b @@ -209,6 +215,12 @@ func (b *Builder) Build() (AutomationConfig, error) { if err := b.setFeatureCompatibilityVersionIfUpgradeIsHappening(); err != nil { return AutomationConfig{}, errors.Errorf("can't build the automation config: %s", err) } + + dataDir := DefaultMongoDBDataDir + if b.dataDir != "" { + dataDir = b.dataDir + } + totalVotes := 0 for i, h := range hostnames { @@ -226,7 +238,7 @@ func (b *Builder) Build() (AutomationConfig, error) { } process.SetPort(27017) - process.SetStoragePath(DefaultMongoDBDataDir) + process.SetStoragePath(dataDir) process.SetReplicaSetName(b.name) for _, mod := range b.processModifications { diff --git a/test/e2e/replica_set_mongod_config/replica_set_mongod_config_test.go b/test/e2e/replica_set_mongod_config/replica_set_mongod_config_test.go index 7e7beeab8..14328f3b3 100644 --- a/test/e2e/replica_set_mongod_config/replica_set_mongod_config_test.go +++ b/test/e2e/replica_set_mongod_config/replica_set_mongod_config_test.go @@ -37,11 +37,22 @@ func TestReplicaSet(t *testing.T) { t.Fatal(err) } - // Override the journal compressor setting - setting := "storage.wiredTiger.engineConfig.journalCompressor" - value := "zlib" + settings := []string{ + "storage.wiredTiger.engineConfig.journalCompressor", + "storage.dbPath", + } + + values := []string{ + "zlib", + "/some/path/db", + } + + // Override the journal compressor and dbPath settings mongodConfig := objx.New(map[string]interface{}{}) - mongodConfig.Set(setting, value) + for i := range settings { + mongodConfig.Set(settings[i], values[i]) + } + mdb.Spec.AdditionalMongodConfig.Object = mongodConfig t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) @@ -49,5 +60,7 @@ func TestReplicaSet(t *testing.T) { t.Run("Ensure Authentication", tester.EnsureAuthenticationIsConfigured(3)) t.Run("Test Basic Connectivity", tester.ConnectivitySucceeds()) t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 1)) - t.Run("Mongod config has been set", tester.EnsureMongodConfig(setting, value)) + for i := range settings { + t.Run(fmt.Sprintf("Mongod setting %s has been set", settings[i]), tester.EnsureMongodConfig(settings[i], values[i])) + } } From 6f390f2bc8f9af697b4f656e51bbcca6027446e9 Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Thu, 21 Oct 2021 12:49:00 +0100 Subject: [PATCH 413/790] Changed SetAnnotations method signature (#755) --- pkg/kube/annotations/annotations.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/kube/annotations/annotations.go b/pkg/kube/annotations/annotations.go index 5b46dda6d..79b7bfe05 100644 --- a/pkg/kube/annotations/annotations.go +++ b/pkg/kube/annotations/annotations.go @@ -26,25 +26,24 @@ const ( LastAppliedMongoDBVersion = "mongodb.com/v1.lastAppliedMongoDBVersion" ) -func GetAnnotation(object Versioned, key string) string { +func GetAnnotation(object client.Object, key string) string { value, ok := object.GetAnnotations()[key] if !ok { return "" } - return value } -func SetAnnotations(spec Versioned, annotations map[string]string, kubeClient client.Client) error { - currentObject := spec - err := kubeClient.Get(context.TODO(), spec.NamespacedName(), currentObject) +func SetAnnotations(object client.Object, annotations map[string]string, kubeClient client.Client) error { + currentObject := object + err := kubeClient.Get(context.TODO(), types.NamespacedName{Name: object.GetName(), Namespace: object.GetNamespace()}, currentObject) if err != nil { return err } // If the object has no annotations, we first need to create an empty entry in // metadata.annotations, otherwise the server will reject our request - payload := []patchValue{} + var payload []patchValue if currentObject.GetAnnotations() == nil || len(currentObject.GetAnnotations()) == 0 { payload = append(payload, patchValue{ Op: "replace", @@ -68,7 +67,7 @@ func SetAnnotations(spec Versioned, annotations map[string]string, kubeClient cl } patch := client.RawPatch(types.JSONPatchType, data) - return kubeClient.Patch(context.TODO(), spec, patch) + return kubeClient.Patch(context.TODO(), object, patch) } func UpdateLastAppliedMongoDBVersion(mdb Versioned, kubeClient client.Client) error { @@ -78,3 +77,4 @@ func UpdateLastAppliedMongoDBVersion(mdb Versioned, kubeClient client.Client) er return SetAnnotations(mdb, annotations, kubeClient) } + From 0df45f5385afa813d67336030e87921e720ef0a3 Mon Sep 17 00:00:00 2001 From: Hans Kristian Flaatten Date: Thu, 21 Oct 2021 19:28:53 +0200 Subject: [PATCH 414/790] Allow overriding clusterDomain for connection strings (#747) --- api/v1/mongodbcommunity_types.go | 38 ++++++++++++------- api/v1/mongodbcommunity_types_test.go | 9 +++-- controllers/mongodb_users.go | 6 +-- controllers/replica_set_controller.go | 4 +- test/e2e/mongodbtests/mongodbtests.go | 2 +- test/e2e/replica_set/replica_set_test.go | 2 +- .../replica_set_custom_persistent_volume.go | 2 +- .../replica_set_multiple_test.go | 4 +- .../replica_set_recovery_test.go | 2 +- .../replica_set_scaling_test.go | 4 +- .../replica_set_scale_down_test.go | 2 +- .../replica_set_tls/replica_set_tls_test.go | 2 +- .../replica_set_pem_file_test.go | 2 +- .../replica_set_tls_recreate_mdbc_test.go | 2 +- .../statefulset_delete_test.go | 2 +- test/e2e/util/mongotester/mongotester.go | 4 +- 16 files changed, 51 insertions(+), 36 deletions(-) diff --git a/api/v1/mongodbcommunity_types.go b/api/v1/mongodbcommunity_types.go index 9621c8243..8d8fbe223 100644 --- a/api/v1/mongodbcommunity_types.go +++ b/api/v1/mongodbcommunity_types.go @@ -46,6 +46,10 @@ const ( defaultMode AuthMode = "SCRAM-SHA-256" ) +const ( + defaultClusterDomain = "cluster.local" +) + // MongoDBCommunitySpec defines the desired state of MongoDB type MongoDBCommunitySpec struct { // Members is the number of members in the replica set @@ -498,32 +502,36 @@ func (m MongoDBCommunity) AutomationConfigMembersThisReconciliation() int { } // MongoURI returns a mongo uri which can be used to connect to this deployment -func (m MongoDBCommunity) MongoURI() string { - return fmt.Sprintf("mongodb://%s", strings.Join(m.Hosts(), ",")) +func (m MongoDBCommunity) MongoURI(clusterDomain string) string { + return fmt.Sprintf("mongodb://%s", strings.Join(m.Hosts(clusterDomain), ",")) } // MongoSRVURI returns a mongo srv uri which can be used to connect to this deployment -func (m MongoDBCommunity) MongoSRVURI() string { - clusterDomain := "svc.cluster.local" // TODO: make this configurable - return fmt.Sprintf("mongodb+srv://%s.%s.%s", m.ServiceName(), m.Namespace, clusterDomain) +func (m MongoDBCommunity) MongoSRVURI(clusterDomain string) string { + if clusterDomain == "" { + clusterDomain = defaultClusterDomain + } + return fmt.Sprintf("mongodb+srv://%s.%s.svc.%s", m.ServiceName(), m.Namespace, clusterDomain) } // MongoAuthUserURI returns a mongo uri which can be used to connect to this deployment // and includes the authentication data for the user -func (m MongoDBCommunity) MongoAuthUserURI(user scram.User, password string) string { +func (m MongoDBCommunity) MongoAuthUserURI(user scram.User, password string, clusterDomain string) string { return fmt.Sprintf("mongodb://%s:%s@%s/%s?ssl=%t", url.QueryEscape(user.Username), url.QueryEscape(password), - strings.Join(m.Hosts(), ","), + strings.Join(m.Hosts(clusterDomain), ","), user.Database, m.Spec.Security.TLS.Enabled) } // MongoAuthUserSRVURI returns a mongo srv uri which can be used to connect to this deployment // and includes the authentication data for the user -func (m MongoDBCommunity) MongoAuthUserSRVURI(user scram.User, password string) string { - clusterDomain := "svc.cluster.local" // TODO: make this configurable - return fmt.Sprintf("mongodb+srv://%s:%s@%s.%s.%s/%s?ssl=%t", +func (m MongoDBCommunity) MongoAuthUserSRVURI(user scram.User, password string, clusterDomain string) string { + if clusterDomain == "" { + clusterDomain = defaultClusterDomain + } + return fmt.Sprintf("mongodb+srv://%s:%s@%s.%s.svc.%s/%s?ssl=%t", url.QueryEscape(user.Username), url.QueryEscape(password), m.ServiceName(), @@ -533,11 +541,15 @@ func (m MongoDBCommunity) MongoAuthUserSRVURI(user scram.User, password string) m.Spec.Security.TLS.Enabled) } -func (m MongoDBCommunity) Hosts() []string { +func (m MongoDBCommunity) Hosts(clusterDomain string) []string { hosts := make([]string, m.Spec.Members) - clusterDomain := "svc.cluster.local" // TODO: make this configurable + + if clusterDomain == "" { + clusterDomain = defaultClusterDomain + } + for i := 0; i < m.Spec.Members; i++ { - hosts[i] = fmt.Sprintf("%s-%d.%s.%s.%s:%d", m.Name, i, m.ServiceName(), m.Namespace, clusterDomain, 27017) + hosts[i] = fmt.Sprintf("%s-%d.%s.%s.svc.%s:%d", m.Name, i, m.ServiceName(), m.Namespace, clusterDomain, 27017) } return hosts } diff --git a/api/v1/mongodbcommunity_types_test.go b/api/v1/mongodbcommunity_types_test.go index 692e9732a..1880c523b 100644 --- a/api/v1/mongodbcommunity_types_test.go +++ b/api/v1/mongodbcommunity_types_test.go @@ -9,11 +9,14 @@ import ( func TestMongoDB_MongoURI(t *testing.T) { mdb := newReplicaSet(2, "my-rs", "my-namespace") - assert.Equal(t, mdb.MongoURI(), "mongodb://my-rs-0.my-rs-svc.my-namespace.svc.cluster.local:27017,my-rs-1.my-rs-svc.my-namespace.svc.cluster.local:27017") + assert.Equal(t, mdb.MongoURI(""), "mongodb://my-rs-0.my-rs-svc.my-namespace.svc.cluster.local:27017,my-rs-1.my-rs-svc.my-namespace.svc.cluster.local:27017") + assert.Equal(t, mdb.MongoURI("my.cluster"), "mongodb://my-rs-0.my-rs-svc.my-namespace.svc.my.cluster:27017,my-rs-1.my-rs-svc.my-namespace.svc.my.cluster:27017") mdb = newReplicaSet(1, "my-single-rs", "my-single-namespace") - assert.Equal(t, mdb.MongoURI(), "mongodb://my-single-rs-0.my-single-rs-svc.my-single-namespace.svc.cluster.local:27017") + assert.Equal(t, mdb.MongoURI(""), "mongodb://my-single-rs-0.my-single-rs-svc.my-single-namespace.svc.cluster.local:27017") + assert.Equal(t, mdb.MongoURI("my.cluster"), "mongodb://my-single-rs-0.my-single-rs-svc.my-single-namespace.svc.my.cluster:27017") mdb = newReplicaSet(5, "my-big-rs", "my-big-namespace") - assert.Equal(t, mdb.MongoURI(), "mongodb://my-big-rs-0.my-big-rs-svc.my-big-namespace.svc.cluster.local:27017,my-big-rs-1.my-big-rs-svc.my-big-namespace.svc.cluster.local:27017,my-big-rs-2.my-big-rs-svc.my-big-namespace.svc.cluster.local:27017,my-big-rs-3.my-big-rs-svc.my-big-namespace.svc.cluster.local:27017,my-big-rs-4.my-big-rs-svc.my-big-namespace.svc.cluster.local:27017") + assert.Equal(t, mdb.MongoURI(""), "mongodb://my-big-rs-0.my-big-rs-svc.my-big-namespace.svc.cluster.local:27017,my-big-rs-1.my-big-rs-svc.my-big-namespace.svc.cluster.local:27017,my-big-rs-2.my-big-rs-svc.my-big-namespace.svc.cluster.local:27017,my-big-rs-3.my-big-rs-svc.my-big-namespace.svc.cluster.local:27017,my-big-rs-4.my-big-rs-svc.my-big-namespace.svc.cluster.local:27017") + assert.Equal(t, mdb.MongoURI("my.cluster"), "mongodb://my-big-rs-0.my-big-rs-svc.my-big-namespace.svc.my.cluster:27017,my-big-rs-1.my-big-rs-svc.my-big-namespace.svc.my.cluster:27017,my-big-rs-2.my-big-rs-svc.my-big-namespace.svc.my.cluster:27017,my-big-rs-3.my-big-rs-svc.my-big-namespace.svc.my.cluster:27017,my-big-rs-4.my-big-rs-svc.my-big-namespace.svc.my.cluster:27017") } func TestGetScramCredentialsSecretName(t *testing.T) { diff --git a/controllers/mongodb_users.go b/controllers/mongodb_users.go index f04248777..112c46596 100644 --- a/controllers/mongodb_users.go +++ b/controllers/mongodb_users.go @@ -37,7 +37,7 @@ func (r ReplicaSetReconciler) ensureUserResources(mdb mdbv1.MongoDBCommunity) er // updateConnectionStringSecrets updates secrets where user specific connection strings are stored. // The client applications can mount these secrets and connect to the mongodb cluster -func (r ReplicaSetReconciler) updateConnectionStringSecrets(mdb mdbv1.MongoDBCommunity) error { +func (r ReplicaSetReconciler) updateConnectionStringSecrets(mdb mdbv1.MongoDBCommunity, clusterDomain string) error { for _, user := range mdb.GetScramUsers() { secretNamespacedName := types.NamespacedName{Name: user.PasswordSecretName, Namespace: mdb.Namespace} pwd, err := secret.ReadKey(r.client, user.PasswordSecretKey, secretNamespacedName) @@ -48,8 +48,8 @@ func (r ReplicaSetReconciler) updateConnectionStringSecrets(mdb mdbv1.MongoDBCom connectionStringSecret := secret.Builder(). SetName(user.GetConnectionStringSecretName(mdb)). SetNamespace(mdb.Namespace). - SetField("connectionString.standard", mdb.MongoAuthUserURI(user, pwd)). - SetField("connectionString.standardSrv", mdb.MongoAuthUserSRVURI(user, pwd)). + SetField("connectionString.standard", mdb.MongoAuthUserURI(user, pwd, clusterDomain)). + SetField("connectionString.standardSrv", mdb.MongoAuthUserSRVURI(user, pwd, clusterDomain)). SetField("username", user.Username). SetField("password", pwd). SetOwnerReferences(mdb.GetOwnerReferences()). diff --git a/controllers/replica_set_controller.go b/controllers/replica_set_controller.go index 4c4e1f8a0..114c827bd 100644 --- a/controllers/replica_set_controller.go +++ b/controllers/replica_set_controller.go @@ -220,7 +220,7 @@ func (r ReplicaSetReconciler) Reconcile(ctx context.Context, request reconcile.R res, err := status.Update(r.client.Status(), &mdb, statusOptions(). - withMongoURI(mdb.MongoURI()). + withMongoURI(mdb.MongoURI(os.Getenv(clusterDNSName))). withMongoDBMembers(mdb.AutomationConfigMembersThisReconciliation()). withStatefulSetReplicas(mdb.StatefulSetReplicasThisReconciliation()). withMessage(None, ""). @@ -232,7 +232,7 @@ func (r ReplicaSetReconciler) Reconcile(ctx context.Context, request reconcile.R return res, err } - if err := r.updateConnectionStringSecrets(mdb); err != nil { + if err := r.updateConnectionStringSecrets(mdb, os.Getenv(clusterDNSName)); err != nil { r.log.Errorf("Could not update connection string secrets: %s", err) } diff --git a/test/e2e/mongodbtests/mongodbtests.go b/test/e2e/mongodbtests/mongodbtests.go index 318920818..8964bbc00 100644 --- a/test/e2e/mongodbtests/mongodbtests.go +++ b/test/e2e/mongodbtests/mongodbtests.go @@ -318,7 +318,7 @@ func BasicFunctionality(mdb *mdbv1.MongoDBCommunity) func(*testing.T) { t.Run("Connection string secrets are configured", ConnectionStringSecretsAreConfigured(mdb, mdbOwnerReference)) t.Run("Test Status Was Updated", Status(mdb, mdbv1.MongoDBCommunityStatus{ - MongoURI: mdb.MongoURI(), + MongoURI: mdb.MongoURI(""), Phase: mdbv1.Running, Version: mdb.GetMongoDBVersion(), CurrentMongoDBMembers: mdb.Spec.Members, diff --git a/test/e2e/replica_set/replica_set_test.go b/test/e2e/replica_set/replica_set_test.go index 74249b524..e75f012c7 100644 --- a/test/e2e/replica_set/replica_set_test.go +++ b/test/e2e/replica_set/replica_set_test.go @@ -41,7 +41,7 @@ func TestReplicaSet(t *testing.T) { t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb)) t.Run("Keyfile authentication is configured", tester.HasKeyfileAuth(3)) t.Run("Test Basic Connectivity", tester.ConnectivitySucceeds()) - t.Run("Test SRV Connectivity", tester.ConnectivitySucceeds(WithURI(mdb.MongoSRVURI()), WithoutTls(), WithReplicaSet((mdb.Name)))) + t.Run("Test SRV Connectivity", tester.ConnectivitySucceeds(WithURI(mdb.MongoSRVURI("")), WithoutTls(), WithReplicaSet((mdb.Name)))) t.Run("Test Basic Connectivity with generated connection string secret", tester.ConnectivitySucceeds(WithURI(mongodbtests.GetConnectionStringForUser(mdb, scramUser)))) t.Run("Test SRV Connectivity with generated connection string secret", diff --git a/test/e2e/replica_set_custom_persistent_volume/replica_set_custom_persistent_volume.go b/test/e2e/replica_set_custom_persistent_volume/replica_set_custom_persistent_volume.go index 5aaf92230..c142b3482 100644 --- a/test/e2e/replica_set_custom_persistent_volume/replica_set_custom_persistent_volume.go +++ b/test/e2e/replica_set_custom_persistent_volume/replica_set_custom_persistent_volume.go @@ -133,7 +133,7 @@ func TestReplicaSetCustomPersistentVolumes(t *testing.T) { t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb)) t.Run("Keyfile authentication is configured", tester.HasKeyfileAuth(3)) t.Run("Test Basic Connectivity", tester.ConnectivitySucceeds()) - t.Run("Test SRV Connectivity", tester.ConnectivitySucceeds(WithURI(mdb.MongoSRVURI()), WithoutTls(), WithReplicaSet((mdb.Name)))) + t.Run("Test SRV Connectivity", tester.ConnectivitySucceeds(WithURI(mdb.MongoSRVURI("")), WithoutTls(), WithReplicaSet((mdb.Name)))) t.Run("Test Basic Connectivity with generated connection string secret", tester.ConnectivitySucceeds(WithURI(mongodbtests.GetConnectionStringForUser(mdb, scramUser)))) t.Run("Test SRV Connectivity with generated connection string secret", diff --git a/test/e2e/replica_set_multiple/replica_set_multiple_test.go b/test/e2e/replica_set_multiple/replica_set_multiple_test.go index d3396eb07..b6c173197 100644 --- a/test/e2e/replica_set_multiple/replica_set_multiple_test.go +++ b/test/e2e/replica_set_multiple/replica_set_multiple_test.go @@ -74,7 +74,7 @@ func TestReplicaSetMultiple(t *testing.T) { t.Run("AutomationConfig's version has been increased", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb0, 3)) t.Run("Test Status Was Updated", mongodbtests.Status(&mdb0, mdbv1.MongoDBCommunityStatus{ - MongoURI: mdb0.MongoURI(), + MongoURI: mdb0.MongoURI(""), Phase: mdbv1.Running, CurrentMongoDBMembers: 5, CurrentStatefulSetReplicas: 5, @@ -87,7 +87,7 @@ func TestReplicaSetMultiple(t *testing.T) { //t.Run("AutomationConfig's version has been increased", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb0, 3)) //t.Run("Test Status Was Updated", mongodbtests.Status(&mdb0, // mdbv1.MongoDBStatus{ - // MongoURI: mdb0.MongoURI(), + // MongoURI: mdb0.MongoURI(""), // Phase: mdbv1.Running, // CurrentMongoDBMembers: 5, // CurrentStatefulSetReplicas: 5, diff --git a/test/e2e/replica_set_recovery/replica_set_recovery_test.go b/test/e2e/replica_set_recovery/replica_set_recovery_test.go index 8000a3261..cd50d152b 100644 --- a/test/e2e/replica_set_recovery/replica_set_recovery_test.go +++ b/test/e2e/replica_set_recovery/replica_set_recovery_test.go @@ -56,7 +56,7 @@ func TestReplicaSetRecovery(t *testing.T) { t.Run("MongoDB Reaches Running Phase", mongodbtests.MongoDBReachesRunningPhase(&mdb)) t.Run("Test Status Was Updated", mongodbtests.Status(&mdb, mdbv1.MongoDBCommunityStatus{ - MongoURI: mdb.MongoURI(), + MongoURI: mdb.MongoURI(""), Phase: mdbv1.Running, Version: mdb.GetMongoDBVersion(), CurrentMongoDBMembers: 3, diff --git a/test/e2e/replica_set_scale/replica_set_scaling_test.go b/test/e2e/replica_set_scale/replica_set_scaling_test.go index 3f027452a..709fa99ff 100644 --- a/test/e2e/replica_set_scale/replica_set_scaling_test.go +++ b/test/e2e/replica_set_scale/replica_set_scaling_test.go @@ -51,7 +51,7 @@ func TestReplicaSetScaleUp(t *testing.T) { t.Run("AutomationConfig's version has been increased", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 3)) t.Run("Test Status Was Updated", mongodbtests.Status(&mdb, mdbv1.MongoDBCommunityStatus{ - MongoURI: mdb.MongoURI(), + MongoURI: mdb.MongoURI(""), Phase: mdbv1.Running, Version: mdb.GetMongoDBVersion(), CurrentMongoDBMembers: 5, @@ -65,7 +65,7 @@ func TestReplicaSetScaleUp(t *testing.T) { //t.Run("AutomationConfig's version has been increased", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 5)) //t.Run("Test Status Was Updated", mongodbtests.Status(&mdb, // mdbv1.MongoDBStatus{ - // MongoURI: mdb.MongoURI(), + // MongoURI: mdb.MongoURI(""), // Phase: mdbv1.Running, // Version: mdb.GetMongoDBVersion(), // CurrentMongoDBMembers: 3, diff --git a/test/e2e/replica_set_scale_down/replica_set_scale_down_test.go b/test/e2e/replica_set_scale_down/replica_set_scale_down_test.go index 6ee3a76e7..6aa5f8c4f 100644 --- a/test/e2e/replica_set_scale_down/replica_set_scale_down_test.go +++ b/test/e2e/replica_set_scale_down/replica_set_scale_down_test.go @@ -54,7 +54,7 @@ func TestReplicaSetScaleDown(t *testing.T) { t.Run("AutomationConfig's version has been increased", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 3)) t.Run("Test Status Was Updated", mongodbtests.Status(&mdb, mdbv1.MongoDBCommunityStatus{ - MongoURI: mdb.MongoURI(), + MongoURI: mdb.MongoURI(""), Phase: mdbv1.Running, Version: mdb.GetMongoDBVersion(), CurrentMongoDBMembers: 1, diff --git a/test/e2e/replica_set_tls/replica_set_tls_test.go b/test/e2e/replica_set_tls/replica_set_tls_test.go index b31a7f8ac..0f6509ebf 100644 --- a/test/e2e/replica_set_tls/replica_set_tls_test.go +++ b/test/e2e/replica_set_tls/replica_set_tls_test.go @@ -45,7 +45,7 @@ func TestReplicaSetTLS(t *testing.T) { mongodbtests.SkipTestIfLocal(t, "Ensure MongoDB TLS Configuration", func(t *testing.T) { t.Run("Has TLS Mode", tester.HasTlsMode("requireSSL", 60, WithTls(mdb))) t.Run("Basic Connectivity Succeeds", tester.ConnectivitySucceeds(WithTls(mdb))) - t.Run("SRV Connectivity Succeeds", tester.ConnectivitySucceeds(WithURI(mdb.MongoSRVURI()), WithTls(mdb))) + t.Run("SRV Connectivity Succeeds", tester.ConnectivitySucceeds(WithURI(mdb.MongoSRVURI("")), WithTls(mdb))) t.Run("Basic Connectivity With Generated Connection String Secret Succeeds", tester.ConnectivitySucceeds(WithURI(mongodbtests.GetConnectionStringForUser(mdb, scramUser)), WithTls(mdb))) t.Run("SRV Connectivity With Generated Connection String Secret Succeeds", diff --git a/test/e2e/replica_set_tls_pem_file/replica_set_pem_file_test.go b/test/e2e/replica_set_tls_pem_file/replica_set_pem_file_test.go index b31a7f8ac..0f6509ebf 100644 --- a/test/e2e/replica_set_tls_pem_file/replica_set_pem_file_test.go +++ b/test/e2e/replica_set_tls_pem_file/replica_set_pem_file_test.go @@ -45,7 +45,7 @@ func TestReplicaSetTLS(t *testing.T) { mongodbtests.SkipTestIfLocal(t, "Ensure MongoDB TLS Configuration", func(t *testing.T) { t.Run("Has TLS Mode", tester.HasTlsMode("requireSSL", 60, WithTls(mdb))) t.Run("Basic Connectivity Succeeds", tester.ConnectivitySucceeds(WithTls(mdb))) - t.Run("SRV Connectivity Succeeds", tester.ConnectivitySucceeds(WithURI(mdb.MongoSRVURI()), WithTls(mdb))) + t.Run("SRV Connectivity Succeeds", tester.ConnectivitySucceeds(WithURI(mdb.MongoSRVURI("")), WithTls(mdb))) t.Run("Basic Connectivity With Generated Connection String Secret Succeeds", tester.ConnectivitySucceeds(WithURI(mongodbtests.GetConnectionStringForUser(mdb, scramUser)), WithTls(mdb))) t.Run("SRV Connectivity With Generated Connection String Secret Succeeds", diff --git a/test/e2e/replica_set_tls_recreate_mdbc/replica_set_tls_recreate_mdbc_test.go b/test/e2e/replica_set_tls_recreate_mdbc/replica_set_tls_recreate_mdbc_test.go index 81cd47543..85f4e2776 100644 --- a/test/e2e/replica_set_tls_recreate_mdbc/replica_set_tls_recreate_mdbc_test.go +++ b/test/e2e/replica_set_tls_recreate_mdbc/replica_set_tls_recreate_mdbc_test.go @@ -56,7 +56,7 @@ func TestReplicaSetTLSRecreateMdbc(t *testing.T) { mongodbtests.SkipTestIfLocal(t, "Ensure MongoDB TLS Configuration", func(t *testing.T) { t.Run("Has TLS Mode", tester1.HasTlsMode("requireSSL", 60, WithTls(mdb2))) t.Run("Basic Connectivity Succeeds", tester1.ConnectivitySucceeds(WithTls(mdb2))) - t.Run("SRV Connectivity Succeeds", tester1.ConnectivitySucceeds(WithURI(mdb2.MongoSRVURI()), WithTls(mdb2))) + t.Run("SRV Connectivity Succeeds", tester1.ConnectivitySucceeds(WithURI(mdb2.MongoSRVURI("")), WithTls(mdb2))) t.Run("Basic Connectivity With Generated Connection String Secret Succeeds", tester1.ConnectivitySucceeds(WithURI(mongodbtests.GetConnectionStringForUser(mdb2, scramUser)), WithTls(mdb2))) t.Run("SRV Connectivity With Generated Connection String Secret Succeeds", diff --git a/test/e2e/statefulset_delete/statefulset_delete_test.go b/test/e2e/statefulset_delete/statefulset_delete_test.go index 052015212..2a2aedac0 100644 --- a/test/e2e/statefulset_delete/statefulset_delete_test.go +++ b/test/e2e/statefulset_delete/statefulset_delete_test.go @@ -39,7 +39,7 @@ func TestStatefulSetDelete(t *testing.T) { t.Run("MongoDB Reaches Running Phase", mongodbtests.MongoDBReachesRunningPhase(&mdb)) t.Run("Test Status Was Updated", mongodbtests.Status(&mdb, mdbv1.MongoDBCommunityStatus{ - MongoURI: mdb.MongoURI(), + MongoURI: mdb.MongoURI(""), Phase: mdbv1.Running, Version: mdb.GetMongoDBVersion(), CurrentMongoDBMembers: mdb.DesiredReplicas(), diff --git a/test/e2e/util/mongotester/mongotester.go b/test/e2e/util/mongotester/mongotester.go index 49188e829..8f69f6014 100644 --- a/test/e2e/util/mongotester/mongotester.go +++ b/test/e2e/util/mongotester/mongotester.go @@ -51,9 +51,9 @@ type OptionApplier interface { func FromResource(t *testing.T, mdb mdbv1.MongoDBCommunity, opts ...OptionApplier) (*Tester, error) { var clientOpts []*options.ClientOptions - clientOpts = WithHosts(mdb.Hosts()).ApplyOption(clientOpts...) + clientOpts = WithHosts(mdb.Hosts("")).ApplyOption(clientOpts...) - t.Logf("Configuring hosts: %s for MongoDB: %s", mdb.Hosts(), mdb.NamespacedName()) + t.Logf("Configuring hosts: %s for MongoDB: %s", mdb.Hosts(""), mdb.NamespacedName()) users := mdb.Spec.Users if len(users) == 1 { From f0c6ed12bb55174e19c65b24432dfa628303e6e5 Mon Sep 17 00:00:00 2001 From: Zev Isert Date: Fri, 22 Oct 2021 02:56:10 -0700 Subject: [PATCH 415/790] abriters report as healthy (#740) --- pkg/readiness/health/health.go | 15 ++++++++++++--- pkg/readiness/health/health_test.go | 6 +++--- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/pkg/readiness/health/health.go b/pkg/readiness/health/health.go index d538d2a91..8adf5ebf0 100644 --- a/pkg/readiness/health/health.go +++ b/pkg/readiness/health/health.go @@ -62,8 +62,7 @@ type StepStatus struct { } // isReadyState will return true, meaning a *ready state* in the sense that this Process can -// accept read operations. There are no other states in which the MongoDB server could that -// would mean a Ready State. +// accept read operations. // It returns true if the managed process is mongos or standalone (replicationStatusUndefined) // or if the agent doesn't publish the replica status (older agents) func (h processHealth) IsReadyState() bool { @@ -75,5 +74,15 @@ func (h processHealth) IsReadyState() bool { return true } - return status == replicationStatusPrimary || status == replicationStatusSecondary + switch status { + case + // There are no other states in which the MongoDB + // server could that would mean a Ready State. + replicationStatusPrimary, + replicationStatusSecondary, + replicationStatusArbiter: + return true + } + + return false } diff --git a/pkg/readiness/health/health_test.go b/pkg/readiness/health/health_test.go index 6690736b1..d80eacda6 100644 --- a/pkg/readiness/health/health_test.go +++ b/pkg/readiness/health/health_test.go @@ -6,10 +6,10 @@ import ( "github.com/stretchr/testify/assert" ) -// TestIsReadyState checks that Primary, Secondary and Undefined always result +// TestIsReadyState checks that Primary, Secondary, Arbiter, and Undefined always result // in Ready State. func TestIsReadyStateNotPrimaryNorSecondary(t *testing.T) { - status := []replicationStatus{replicationStatusUndefined, replicationStatusPrimary, replicationStatusSecondary} + status := []replicationStatus{replicationStatusUndefined, replicationStatusPrimary, replicationStatusSecondary, replicationStatusArbiter} for i := range status { h := processHealth{ReplicaStatus: &status[i]} @@ -21,7 +21,7 @@ func TestIsReadyStateNotPrimaryNorSecondary(t *testing.T) { func TestIsNotReady(t *testing.T) { status := []replicationStatus{ replicationStatusStartup, replicationStatusRecovering, replicationStatusStartup2, - replicationStatusUnknown, replicationStatusArbiter, replicationStatusDown, + replicationStatusUnknown, replicationStatusDown, replicationStatusRollback, replicationStatusRemoved, } From 90f5587022a91123238aee9a9430b9b47c764463 Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Fri, 22 Oct 2021 16:15:14 +0100 Subject: [PATCH 416/790] Enable PVC test (#690) Co-authored-by: Nikolas De Giorgis --- deploy/e2e/role.yaml | 7 +++++++ ...ume.go => replica_set_custom_persistent_volume_test.go} | 1 + 2 files changed, 8 insertions(+) rename test/e2e/replica_set_custom_persistent_volume/{replica_set_custom_persistent_volume.go => replica_set_custom_persistent_volume_test.go} (97%) diff --git a/deploy/e2e/role.yaml b/deploy/e2e/role.yaml index ceb65bafa..d769369ec 100644 --- a/deploy/e2e/role.yaml +++ b/deploy/e2e/role.yaml @@ -25,6 +25,13 @@ rules: - patch - update - watch +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - create + - list - apiGroups: - apps resources: diff --git a/test/e2e/replica_set_custom_persistent_volume/replica_set_custom_persistent_volume.go b/test/e2e/replica_set_custom_persistent_volume/replica_set_custom_persistent_volume_test.go similarity index 97% rename from test/e2e/replica_set_custom_persistent_volume/replica_set_custom_persistent_volume.go rename to test/e2e/replica_set_custom_persistent_volume/replica_set_custom_persistent_volume_test.go index c142b3482..2ef55932d 100644 --- a/test/e2e/replica_set_custom_persistent_volume/replica_set_custom_persistent_volume.go +++ b/test/e2e/replica_set_custom_persistent_volume/replica_set_custom_persistent_volume_test.go @@ -43,6 +43,7 @@ func getPersistentVolumeLocal(name string, localPath string, label string) corev }, AccessModes: []corev1.PersistentVolumeAccessMode{"ReadWriteOnce"}, StorageClassName: "default", + Capacity: corev1.ResourceList{corev1.ResourceStorage: *resource.NewScaledQuantity(int64(8), resource.Giga)}, NodeAffinity: &corev1.VolumeNodeAffinity{ Required: &corev1.NodeSelector{ NodeSelectorTerms: []corev1.NodeSelectorTerm{ From 62a2ee3fe5b4e38c3d3fc2bf22783c3fcc1efd32 Mon Sep 17 00:00:00 2001 From: Rodrigo Valin Date: Wed, 27 Oct 2021 13:25:59 +0100 Subject: [PATCH 417/790] Release MongoDB Kubernetes Operator v0.7.1 (#760) --- config/manager/manager.yaml | 6 +++--- controllers/replica_set_controller.go | 8 ++++---- deploy/openshift/operator_openshift.yaml | 6 +++--- docs/RELEASE_NOTES.md | 3 +++ release.json | 4 ++-- 5 files changed, 15 insertions(+), 12 deletions(-) diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 4f5a28c1c..6a43ad3be 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -47,14 +47,14 @@ spec: - name: AGENT_IMAGE value: quay.io/mongodb/mongodb-agent:11.0.5.6963-1 - name: VERSION_UPGRADE_HOOK_IMAGE - value: quay.io/mongodb/mongodb-kubernetes-operator-version-upgrade-post-start-hook:1.0.2 + value: quay.io/mongodb/mongodb-kubernetes-operator-version-upgrade-post-start-hook:1.0.3 - name: READINESS_PROBE_IMAGE - value: quay.io/mongodb/mongodb-kubernetes-readinessprobe:1.0.4 + value: quay.io/mongodb/mongodb-kubernetes-readinessprobe:1.0.6 - name: MONGODB_IMAGE value: mongo - name: MONGODB_REPO_URL value: docker.io - image: quay.io/mongodb/mongodb-kubernetes-operator:0.7.0 + image: quay.io/mongodb/mongodb-kubernetes-operator:0.7.1 imagePullPolicy: Always name: mongodb-kubernetes-operator resources: diff --git a/controllers/replica_set_controller.go b/controllers/replica_set_controller.go index 114c827bd..1aa109eed 100644 --- a/controllers/replica_set_controller.go +++ b/controllers/replica_set_controller.go @@ -53,7 +53,7 @@ import ( ) const ( - clusterDNSName = "CLUSTER_DNS_NAME" + clusterDomain = "CLUSTER_DOMAIN" lastSuccessfulConfiguration = "mongodb.com/v1.lastSuccessfulConfiguration" lastAppliedMongoDBVersion = "mongodb.com/v1.lastAppliedMongoDBVersion" @@ -220,7 +220,7 @@ func (r ReplicaSetReconciler) Reconcile(ctx context.Context, request reconcile.R res, err := status.Update(r.client.Status(), &mdb, statusOptions(). - withMongoURI(mdb.MongoURI(os.Getenv(clusterDNSName))). + withMongoURI(mdb.MongoURI(os.Getenv(clusterDomain))). withMongoDBMembers(mdb.AutomationConfigMembersThisReconciliation()). withStatefulSetReplicas(mdb.StatefulSetReplicasThisReconciliation()). withMessage(None, ""). @@ -232,7 +232,7 @@ func (r ReplicaSetReconciler) Reconcile(ctx context.Context, request reconcile.R return res, err } - if err := r.updateConnectionStringSecrets(mdb, os.Getenv(clusterDNSName)); err != nil { + if err := r.updateConnectionStringSecrets(mdb, os.Getenv(clusterDomain)); err != nil { r.log.Errorf("Could not update connection string secrets: %s", err) } @@ -425,7 +425,7 @@ func (r ReplicaSetReconciler) ensureAutomationConfig(mdb mdbv1.MongoDBCommunity) } func buildAutomationConfig(mdb mdbv1.MongoDBCommunity, auth automationconfig.Auth, currentAc automationconfig.AutomationConfig, modifications ...automationconfig.Modification) (automationconfig.AutomationConfig, error) { - domain := getDomain(mdb.ServiceName(), mdb.Namespace, os.Getenv(clusterDNSName)) + domain := getDomain(mdb.ServiceName(), mdb.Namespace, os.Getenv(clusterDomain)) zap.S().Debugw("AutomationConfigMembersThisReconciliation", "mdb.AutomationConfigMembersThisReconciliation()", mdb.AutomationConfigMembersThisReconciliation()) return automationconfig.NewBuilder(). diff --git a/deploy/openshift/operator_openshift.yaml b/deploy/openshift/operator_openshift.yaml index d7d5ffd07..6f0fb3976 100644 --- a/deploy/openshift/operator_openshift.yaml +++ b/deploy/openshift/operator_openshift.yaml @@ -49,14 +49,14 @@ spec: - name: AGENT_IMAGE value: quay.io/mongodb/mongodb-agent:11.0.5.6963-1 - name: READINESS_PROBE_IMAGE - value: quay.io/mongodb/mongodb-kubernetes-readinessprobe:1.0.4 + value: quay.io/mongodb/mongodb-kubernetes-readinessprobe:1.0.6 - name: VERSION_UPGRADE_HOOK_IMAGE - value: quay.io/mongodb/mongodb-kubernetes-operator-version-upgrade-post-start-hook:1.0.2 + value: quay.io/mongodb/mongodb-kubernetes-operator-version-upgrade-post-start-hook:1.0.3 - name: MONGODB_IMAGE value: mongo - name: MONGODB_REPO_URL value: docker.io - image: quay.io/mongodb/mongodb-kubernetes-operator:0.7.0 + image: quay.io/mongodb/mongodb-kubernetes-operator:0.7.1 imagePullPolicy: Always name: mongodb-kubernetes-operator resources: diff --git a/docs/RELEASE_NOTES.md b/docs/RELEASE_NOTES.md index 0328cada6..b78216d03 100644 --- a/docs/RELEASE_NOTES.md +++ b/docs/RELEASE_NOTES.md @@ -6,6 +6,9 @@ - MongoDB database of the statefulSet is managed using distinct Role, ServiceAccount and RoleBinding. - TLS Secret can also contain a single "tls.pem" entry, containing the concatenation of the certificate and key - If a TLS secret contains all of "tls.key", "tls.crt" and "tls.pem" entries, the operator will raise an error if the "tls.pem" one is not equal to the concatenation of "tls.crt" with "tls.key" + - Readinessprobe reports MongoDB running as Arbitrer as _Running_ & _Healthy_. + - The `CLUSTER_DOMAIN` environment variable can be set on the Operator Pod, to configure the Kubernetes cluster's Domain, + in case this one differs from the default `cluster.local`. ## MongoDBCommunity Resource * Changes diff --git a/release.json b/release.json index 8eb4811c5..f608c9437 100644 --- a/release.json +++ b/release.json @@ -1,7 +1,7 @@ { - "mongodb-kubernetes-operator": "0.7.0", + "mongodb-kubernetes-operator": "0.7.1", "version-upgrade-hook": "1.0.3", - "readiness-probe": "1.0.5", + "readiness-probe": "1.0.6", "mongodb-agent": { "version": "11.0.5.6963-1", "tools_version": "100.3.1" From 70e06b2d47ecb5952af240c5528ccfc2e2fe3e04 Mon Sep 17 00:00:00 2001 From: Rodrigo Valin Date: Fri, 29 Oct 2021 14:10:58 +0100 Subject: [PATCH 418/790] Adds missing roles to database. (#762) --- config/rbac/kustomization.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index c110cb8e3..f1fe88a33 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -2,3 +2,6 @@ resources: - role.yaml - role_binding.yaml - service_account.yaml +- service_account_database.yaml +- role_binding_database.yaml +- role_database.yaml From 660fd49bc3420ae15908175bcfebc23b9b32f3da Mon Sep 17 00:00:00 2001 From: Rodrigo Valin Date: Fri, 29 Oct 2021 18:45:32 +0100 Subject: [PATCH 419/790] CLOUDP-98794: Fixes Openshift install. (#765) --- deploy/openshift/operator_openshift.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/openshift/operator_openshift.yaml b/deploy/openshift/operator_openshift.yaml index 6f0fb3976..3805ad64f 100644 --- a/deploy/openshift/operator_openshift.yaml +++ b/deploy/openshift/operator_openshift.yaml @@ -32,7 +32,7 @@ spec: topologyKey: kubernetes.io/hostname containers: - command: - - mongodb-kubernetes-operator + - /usr/local/bin/entrypoint env: - name: WATCH_NAMESPACE valueFrom: From c9b86fd83de6bead7de20b36aa3cd8023ad700cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=A9o=20Bouvard?= Date: Mon, 1 Nov 2021 16:14:34 +0100 Subject: [PATCH 420/790] Add spec.version limitations to deploy docs (#758) (#766) --- docs/deploy-configure.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/deploy-configure.md b/docs/deploy-configure.md index 4fdd93bdc..3f0bf2107 100644 --- a/docs/deploy-configure.md +++ b/docs/deploy-configure.md @@ -199,7 +199,7 @@ To add arbiters: You can upgrade the major, minor, and/or feature compatibility versions of your MongoDB resource. These settings are configured in your resource definition YAML file. -- To upgrade your resource's major and/or minor versions, set the `spec.version` setting to the desired MongoDB version. +- To upgrade your resource's major and/or minor versions, set the `spec.version` setting to the desired MongoDB version. Make sure to specify a full image tag, such as `5.0.3`. Setting the `spec.version` to loosely-defined tags such as `5.0` is not currently supported. - To modify your resource's [feature compatibility version](https://docs.mongodb.com/manual/reference/command/setFeatureCompatibilityVersion/), set the `spec.featureCompatibilityVersion` setting to the desired version. From 459913268c1cccf483882f31d3983e75bf461814 Mon Sep 17 00:00:00 2001 From: Rodrigo Valin Date: Wed, 3 Nov 2021 15:34:24 +0000 Subject: [PATCH 421/790] Uses latest version of helm-charts. (#763) --- helm-charts | 2 +- pkg/helm/helm.go | 6 ++++++ test/e2e/mongodbtests/mongodbtests.go | 2 +- test/e2e/setup/setup.go | 8 +++++++- 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/helm-charts b/helm-charts index ec377b07a..1f2dc0339 160000 --- a/helm-charts +++ b/helm-charts @@ -1 +1 @@ -Subproject commit ec377b07a54aa26c9b0d2fccf65609347bc6ff1a +Subproject commit 1f2dc033920cec758104ff2d3749cb3ea5a95cdd diff --git a/pkg/helm/helm.go b/pkg/helm/helm.go index d6e97cc1f..35392f7f2 100644 --- a/pkg/helm/helm.go +++ b/pkg/helm/helm.go @@ -13,6 +13,12 @@ func Uninstall(chartName string, namespace string) error { return executeHelmCommand(helmArgs, isNotFoundMessage) } +// DependencyUpdate downloads dependencies for a Chart. +func DependencyUpdate(chartPath string) error { + helmArgs := []string{"dependency", "update", chartPath} + return executeHelmCommand(helmArgs, nil) +} + // Install a helm chert at the given path with the given name and the provided set arguments. func Install(chartPath, chartName string, flags map[string]string, templateValues map[string]string) error { helmArgs := []string{"install"} diff --git a/test/e2e/mongodbtests/mongodbtests.go b/test/e2e/mongodbtests/mongodbtests.go index 8964bbc00..885c85bb6 100644 --- a/test/e2e/mongodbtests/mongodbtests.go +++ b/test/e2e/mongodbtests/mongodbtests.go @@ -220,7 +220,7 @@ func AutomationConfigSecretExists(mdb *mdbv1.MongoDBCommunity) func(t *testing.T s, err := wait.ForSecretToExist(mdb.AutomationConfigSecretName(), time.Second*5, time.Minute*1, mdb.Namespace) assert.NoError(t, err) - t.Logf("Secret %s/%s was successfully created", mdb.AutomationConfigSecretName(), mdb.Namespace) + t.Logf("Secret %s/%s was successfully created", mdb.Namespace, mdb.AutomationConfigSecretName()) assert.Contains(t, s.Data, automationconfig.ConfigKey) t.Log("The Secret contained the automation config") diff --git a/test/e2e/setup/setup.go b/test/e2e/setup/setup.go index 5ac58dbfe..64bbb15e2 100644 --- a/test/e2e/setup/setup.go +++ b/test/e2e/setup/setup.go @@ -140,6 +140,8 @@ func getHelmArgs(testConfig TestConfig, watchNamespace string, resourceName stri helmArgs["registry.agent"] = agentRegistry helmArgs["registry.readinessProbe"] = readinessProbeRegistry + helmArgs["community-operator-crds.enabled"] = strconv.FormatBool(false) + helmArgs["createResource"] = strconv.FormatBool(false) helmArgs["resource.name"] = resourceName helmArgs["resource.tls.enabled"] = strconv.FormatBool(withTLS) @@ -165,10 +167,14 @@ func deployOperator(config TestConfig, resourceName string, withTLS bool) error helmArgs := getHelmArgs(config, watchNamespace, resourceName, withTLS) helmFlags := map[string]string{ - "skip-crds": "", "namespace": config.Namespace, "create-namespace": "", } + + if err := helm.DependencyUpdate(config.HelmChartPath); err != nil { + return err + } + if err := helm.Install(config.HelmChartPath, helmChartName, helmFlags, helmArgs); err != nil { return err } From 6f64c824c1b1a1e7c5c65208014493bdf7af0a6f Mon Sep 17 00:00:00 2001 From: Rodrigo Valin Date: Wed, 3 Nov 2021 19:20:40 +0000 Subject: [PATCH 422/790] Makefile uses configured local config. (#770) --- Makefile | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index e04d7c761..98d40de8b 100644 --- a/Makefile +++ b/Makefile @@ -1,13 +1,15 @@ SHELL := /bin/bash +MONGODB_COMMUNITY_CONFIG ?= $(HOME)/.community-operator-dev/config.json + # Image URL to use all building/pushing image targets -REPO_URL := $(shell jq -r .repo_url < ~/.community-operator-dev/config.json) -OPERATOR_IMAGE := $(shell jq -r .operator_image < ~/.community-operator-dev/config.json) -NAMESPACE := $(shell jq -r .namespace < ~/.community-operator-dev/config.json) -UPGRADE_HOOK_IMG := $(shell jq -r .version_upgrade_hook_image < ~/.community-operator-dev/config.json) -READINESS_PROBE_IMG := $(shell jq -r .readiness_probe_image < ~/.community-operator-dev/config.json) -REGISTRY := $(shell jq -r .repo_url < ~/.community-operator-dev/config.json) -AGENT_IMAGE_NAME := $(shell jq -r .agent_image_ubuntu < ~/.community-operator-dev/config.json) +REPO_URL := $(shell jq -r .repo_url < $(MONGODB_COMMUNITY_CONFIG)) +OPERATOR_IMAGE := $(shell jq -r .operator_image < $(MONGODB_COMMUNITY_CONFIG)) +NAMESPACE := $(shell jq -r .namespace < $(MONGODB_COMMUNITY_CONFIG)) +UPGRADE_HOOK_IMG := $(shell jq -r .version_upgrade_hook_image < $(MONGODB_COMMUNITY_CONFIG)) +READINESS_PROBE_IMG := $(shell jq -r .readiness_probe_image < $(MONGODB_COMMUNITY_CONFIG)) +REGISTRY := $(shell jq -r .repo_url < $(MONGODB_COMMUNITY_CONFIG)) +AGENT_IMAGE_NAME := $(shell jq -r .agent_image_ubuntu < $(MONGODB_COMMUNITY_CONFIG)) HELM_CHART ?= ./helm-charts/charts/community-operator From e8210c8dd0e692eb27a4caa48ec6c62b9a3e228a Mon Sep 17 00:00:00 2001 From: Rodrigo Valin Date: Thu, 11 Nov 2021 09:38:05 +0000 Subject: [PATCH 423/790] Bumps helm-charts. (#775) --- helm-charts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm-charts b/helm-charts index 1f2dc0339..7c7641eb3 160000 --- a/helm-charts +++ b/helm-charts @@ -1 +1 @@ -Subproject commit 1f2dc033920cec758104ff2d3749cb3ea5a95cdd +Subproject commit 7c7641eb3b5ca6c716e44afae9bf99351f7b644d From 170696401cd962797e1676e6a64e4e9e68108e0f Mon Sep 17 00:00:00 2001 From: Rodrigo Valin Date: Thu, 11 Nov 2021 10:31:28 +0000 Subject: [PATCH 424/790] Updates multiple libraries. (#777) --- go.mod | 11 +++++------ go.sum | 22 ++++++++++++---------- pkg/kube/annotations/annotations.go | 1 - 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/go.mod b/go.mod index 92d944739..2ada65b45 100644 --- a/go.mod +++ b/go.mod @@ -7,19 +7,18 @@ require ( github.com/go-logr/logr v0.4.0 github.com/hashicorp/go-multierror v1.1.1 github.com/imdario/mergo v0.3.12 - github.com/klauspost/compress v1.9.8 // indirect github.com/pkg/errors v0.9.1 github.com/spf13/cast v1.4.1 github.com/stretchr/objx v0.3.0 github.com/stretchr/testify v1.7.0 github.com/xdg/stringprep v1.0.3 - go.mongodb.org/mongo-driver v1.7.2 + go.mongodb.org/mongo-driver v1.7.4 go.uber.org/zap v1.19.1 gopkg.in/natefinch/lumberjack.v2 v2.0.0 - k8s.io/api v0.22.2 - k8s.io/apimachinery v0.22.2 - k8s.io/client-go v0.22.2 - sigs.k8s.io/controller-runtime v0.10.1 + k8s.io/api v0.22.3 + k8s.io/apimachinery v0.22.3 + k8s.io/client-go v0.22.3 + sigs.k8s.io/controller-runtime v0.10.2 sigs.k8s.io/yaml v1.3.0 ) diff --git a/go.sum b/go.sum index 4760fa48e..c0bae52c0 100644 --- a/go.sum +++ b/go.sum @@ -288,9 +288,8 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.9.8 h1:VMAMUUOh+gaxKTMk+zqbjsSjsIcUcL/LF4o63i82QyA= -github.com/klauspost/compress v1.9.8/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -464,8 +463,8 @@ go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lL go.etcd.io/etcd/pkg/v3 v3.5.0/go.mod h1:UzJGatBQ1lXChBkQF0AuAtkRQMYnHubxAEYIrC3MSsE= go.etcd.io/etcd/raft/v3 v3.5.0/go.mod h1:UFOHSIvO/nKwd4lhkwabrTD3cqW5yVyYYf/KlD00Szc= go.etcd.io/etcd/server/v3 v3.5.0/go.mod h1:3Ah5ruV+M+7RZr0+Y/5mNLwC+eQlni+mQmOVdCRJoS4= -go.mongodb.org/mongo-driver v1.7.2 h1:pFttQyIiJUHEn50YfZgC9ECjITMT44oiN36uArf/OFg= -go.mongodb.org/mongo-driver v1.7.2/go.mod h1:Q4oFMbo1+MSNqICAdYMlC/zSTrwCogR4R8NzkI+yfU8= +go.mongodb.org/mongo-driver v1.7.4 h1:sllcioag8Mec0LYkftYWq+cKNPIR4Kqq3iv9ZXY0g/E= +go.mongodb.org/mongo-driver v1.7.4/go.mod h1:NqaYOwnXWr5Pm7AOpO5QFxKJ503nbMse/R79oO62zWg= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -818,15 +817,18 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.22.2 h1:M8ZzAD0V6725Fjg53fKeTJxGsJvRbk4TEm/fexHMtfw= k8s.io/api v0.22.2/go.mod h1:y3ydYpLJAaDI+BbSe2xmGcqxiWHmWjkEeIbiwHvnPR8= +k8s.io/api v0.22.3 h1:wOoES2GoSkUsdped2RB4zYypPqWtvprGoKCENTOOjP4= +k8s.io/api v0.22.3/go.mod h1:azgiXFiXqiWyLCfI62/eYBOu19rj2LKmIhFPP4+33fs= k8s.io/apiextensions-apiserver v0.22.2 h1:zK7qI8Ery7j2CaN23UCFaC1hj7dMiI87n01+nKuewd4= k8s.io/apiextensions-apiserver v0.22.2/go.mod h1:2E0Ve/isxNl7tWLSUDgi6+cmwHi5fQRdwGVCxbC+KFA= -k8s.io/apimachinery v0.22.2 h1:ejz6y/zNma8clPVfNDLnPbleBo6MpoFy/HBiBqCouVk= k8s.io/apimachinery v0.22.2/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0= +k8s.io/apimachinery v0.22.3 h1:mrvBG5CZnEfwgpVqWcrRKvdsYECTrhAR6cApAgdsflk= +k8s.io/apimachinery v0.22.3/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0= k8s.io/apiserver v0.22.2/go.mod h1:vrpMmbyjWrgdyOvZTSpsusQq5iigKNWv9o9KlDAbBHI= -k8s.io/client-go v0.22.2 h1:DaSQgs02aCC1QcwUdkKZWOeaVsQjYvWv8ZazcZ6JcHc= k8s.io/client-go v0.22.2/go.mod h1:sAlhrkVDf50ZHx6z4K0S40wISNTarf1r800F+RlCF6U= +k8s.io/client-go v0.22.3 h1:6onkOSc+YNdwq5zXE0wFXicq64rrym+mXwHu/CPVGO4= +k8s.io/client-go v0.22.3/go.mod h1:ElDjYf8gvZsKDYexmsmnMQ0DYO8W9RwBjfQ1PI53yow= k8s.io/code-generator v0.22.2/go.mod h1:eV77Y09IopzeXOJzndrDyCI88UBok2h6WxAlBwpxa+o= k8s.io/component-base v0.22.2 h1:vNIvE0AIrLhjX8drH0BgCNJcR4QZxMXcJzBsDplDx9M= k8s.io/component-base v0.22.2/go.mod h1:5Br2QhI9OTe79p+TzPe9JKNQYvEKbq9rTJDWllunGug= @@ -844,8 +846,8 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= -sigs.k8s.io/controller-runtime v0.10.1 h1:+eLHgY/VrJWnfg6iXUqhCUqNXgPH1NZeP9drNAAgWlg= -sigs.k8s.io/controller-runtime v0.10.1/go.mod h1:CQp8eyUQZ/Q7PJvnIrB6/hgfTC1kBkGylwsLgOQi1WY= +sigs.k8s.io/controller-runtime v0.10.2 h1:jW8qiY+yMnnPx6O9hu63tgcwaKzd1yLYui+mpvClOOc= +sigs.k8s.io/controller-runtime v0.10.2/go.mod h1:CQp8eyUQZ/Q7PJvnIrB6/hgfTC1kBkGylwsLgOQi1WY= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.1.2 h1:Hr/htKFmJEbtMgS/UD0N+gtgctAqz81t3nu+sPzynno= sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= diff --git a/pkg/kube/annotations/annotations.go b/pkg/kube/annotations/annotations.go index 79b7bfe05..f4d923738 100644 --- a/pkg/kube/annotations/annotations.go +++ b/pkg/kube/annotations/annotations.go @@ -77,4 +77,3 @@ func UpdateLastAppliedMongoDBVersion(mdb Versioned, kubeClient client.Client) er return SetAnnotations(mdb, annotations, kubeClient) } - From cd411b999c4d737644b5e5f9ee21ccaa0498c901 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 11 Nov 2021 11:09:06 +0000 Subject: [PATCH 425/790] Bump sigs.k8s.io/controller-runtime from 0.10.2 to 0.10.3 (#778) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 2ada65b45..1dabdcf20 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( k8s.io/api v0.22.3 k8s.io/apimachinery v0.22.3 k8s.io/client-go v0.22.3 - sigs.k8s.io/controller-runtime v0.10.2 + sigs.k8s.io/controller-runtime v0.10.3 sigs.k8s.io/yaml v1.3.0 ) diff --git a/go.sum b/go.sum index c0bae52c0..2aebb7d9f 100644 --- a/go.sum +++ b/go.sum @@ -846,8 +846,8 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= -sigs.k8s.io/controller-runtime v0.10.2 h1:jW8qiY+yMnnPx6O9hu63tgcwaKzd1yLYui+mpvClOOc= -sigs.k8s.io/controller-runtime v0.10.2/go.mod h1:CQp8eyUQZ/Q7PJvnIrB6/hgfTC1kBkGylwsLgOQi1WY= +sigs.k8s.io/controller-runtime v0.10.3 h1:s5Ttmw/B4AuIbwrXD3sfBkXwnPMMWrqpVj4WRt1dano= +sigs.k8s.io/controller-runtime v0.10.3/go.mod h1:CQp8eyUQZ/Q7PJvnIrB6/hgfTC1kBkGylwsLgOQi1WY= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.1.2 h1:Hr/htKFmJEbtMgS/UD0N+gtgctAqz81t3nu+sPzynno= sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= From 93091662af59cdb8a792c29acf665ba974688317 Mon Sep 17 00:00:00 2001 From: Rajdeep Das Date: Tue, 16 Nov 2021 15:17:15 +0100 Subject: [PATCH 426/790] Read agent API Key on startup (#781) --- controllers/construct/mongodbstatefulset.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/controllers/construct/mongodbstatefulset.go b/controllers/construct/mongodbstatefulset.go index df8438929..6f4351376 100644 --- a/controllers/construct/mongodbstatefulset.go +++ b/controllers/construct/mongodbstatefulset.go @@ -2,10 +2,11 @@ package construct import ( "fmt" - "github.com/stretchr/objx" "os" "strings" + "github.com/stretchr/objx" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/automationconfig" "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/container" "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/persistentvolumeclaim" @@ -51,6 +52,7 @@ const ( automationAgentOptions = " -skipMongoStart -noDaemonize -useLocalMongoDbTools" MongodbUserCommand = `current_uid=$(id -u) +AGENT_API_KEY="$(cat /mongodb-automation/agent-api-key/agentApiKey)" declare -r current_uid if ! grep -q "${current_uid}" /etc/passwd ; then sed -e "s/^mongodb:/builder:/" /etc/passwd > /tmp/passwd From 2e4578c0d313eb330e513a066877b1b91b0e5088 Mon Sep 17 00:00:00 2001 From: Rajdeep Das Date: Thu, 18 Nov 2021 11:45:20 +0100 Subject: [PATCH 427/790] Add "WithVolumes" API to podspec template (#782) This would enable us to patch volumes more cleanly, currently, it would be used to configure volumes when vault is configured or not. --- pkg/kube/podtemplatespec/podspec_template.go | 16 +++++++++ .../podtemplatespec/podspec_template_test.go | 33 +++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/pkg/kube/podtemplatespec/podspec_template.go b/pkg/kube/podtemplatespec/podspec_template.go index 6ce66b936..c50d7204e 100644 --- a/pkg/kube/podtemplatespec/podspec_template.go +++ b/pkg/kube/podtemplatespec/podspec_template.go @@ -107,6 +107,22 @@ func WithServiceAccount(serviceAccountName string) Modification { } } +// WithVolumes appends the given volumes to the existing volume, it ensures not +// to override existing volumes +func WithVolumes(volumes []corev1.Volume) Modification { + return func(template *corev1.PodTemplateSpec) { + present := make(map[string]struct{}) + for _, v := range template.Spec.Volumes { + present[v.Name] = struct{}{} + } + for _, v := range volumes { + if _, ok := present[v.Name]; !ok { + template.Spec.Volumes = append(template.Spec.Volumes, v) + } + } + } +} + // WithVolume ensures the given volume exists func WithVolume(volume corev1.Volume) Modification { return func(template *corev1.PodTemplateSpec) { diff --git a/pkg/kube/podtemplatespec/podspec_template_test.go b/pkg/kube/podtemplatespec/podspec_template_test.go index e0238da23..a5f6c57fe 100644 --- a/pkg/kube/podtemplatespec/podspec_template_test.go +++ b/pkg/kube/podtemplatespec/podspec_template_test.go @@ -341,6 +341,39 @@ func TestMergeVolumes_DoesNotAddDuplicatesWithSameName(t *testing.T) { assert.Equal(t, "new-volume-3", mergedPodSpecTemplate.Spec.Volumes[2].Name) } +func TestAddVolumes(t *testing.T) { + volumeModification := WithVolume(corev1.Volume{ + Name: "new-volume", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "old-host-path", + }, + }}, + ) + + toAddVolumes := []corev1.Volume{ + { + Name: "new-volume", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "new-host-path", + }, + }, + }, + { + Name: "new-volume-2", + }, + } + + volumesModification := WithVolumes(toAddVolumes) + + p := New(volumeModification, volumesModification) + assert.Len(t, p.Spec.Volumes, 2) + assert.Equal(t, p.Spec.Volumes[0].Name, "new-volume") + assert.Equal(t, p.Spec.Volumes[1].Name, "new-volume-2") + +} + func int64Ref(i int64) *int64 { return &i } From b1750002ca416069f3f85e5518e7874b463a12fd Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Thu, 18 Nov 2021 10:47:37 +0000 Subject: [PATCH 428/790] Separate Kube Secret Client (#783) --- pkg/kube/client/client.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/kube/client/client.go b/pkg/kube/client/client.go index 70c9fc387..f70048226 100644 --- a/pkg/kube/client/client.go +++ b/pkg/kube/client/client.go @@ -25,15 +25,19 @@ func NewClient(c k8sClient.Client) Client { type Client interface { k8sClient.Client + KubernetesSecretClient // TODO: remove this function, add mongodb package which has GetAndUpdate function GetAndUpdate(nsName types.NamespacedName, obj k8sClient.Object, updateFunc func()) error configmap.GetUpdateCreateDeleter service.GetUpdateCreateDeleter - secret.GetUpdateCreateDeleter statefulset.GetUpdateCreateDeleter pod.Getter } +type KubernetesSecretClient interface { + secret.GetUpdateCreateDeleter +} + type client struct { k8sClient.Client } From 76795206a6c2277308c4bed95a710c838d951972 Mon Sep 17 00:00:00 2001 From: Rodrigo Valin Date: Thu, 18 Nov 2021 15:03:12 +0000 Subject: [PATCH 429/790] Release MongoDB Kubernetes Operator v0.7.2 (#780) --- config/manager/manager.yaml | 2 +- deploy/openshift/operator_openshift.yaml | 2 +- docs/RELEASE_NOTES.md | 17 +++------ docs/how-to-release.md | 2 + helm-charts | 2 +- release.json | 2 +- scripts/ci/update_release.py | 47 ++++++++++++++++++++---- 7 files changed, 51 insertions(+), 23 deletions(-) diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 6a43ad3be..d55db68d7 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -54,7 +54,7 @@ spec: value: mongo - name: MONGODB_REPO_URL value: docker.io - image: quay.io/mongodb/mongodb-kubernetes-operator:0.7.1 + image: quay.io/mongodb/mongodb-kubernetes-operator:0.7.2 imagePullPolicy: Always name: mongodb-kubernetes-operator resources: diff --git a/deploy/openshift/operator_openshift.yaml b/deploy/openshift/operator_openshift.yaml index 3805ad64f..a82fd68d9 100644 --- a/deploy/openshift/operator_openshift.yaml +++ b/deploy/openshift/operator_openshift.yaml @@ -56,7 +56,7 @@ spec: value: mongo - name: MONGODB_REPO_URL value: docker.io - image: quay.io/mongodb/mongodb-kubernetes-operator:0.7.1 + image: quay.io/mongodb/mongodb-kubernetes-operator:0.7.2 imagePullPolicy: Always name: mongodb-kubernetes-operator resources: diff --git a/docs/RELEASE_NOTES.md b/docs/RELEASE_NOTES.md index b78216d03..640123540 100644 --- a/docs/RELEASE_NOTES.md +++ b/docs/RELEASE_NOTES.md @@ -1,23 +1,18 @@ -# MongoDB Kubernetes Operator 0.7.1 +# MongoDB Kubernetes Operator 0.7.2 ## Kubernetes Operator -- Changes - - MongoDB database of the statefulSet is managed using distinct Role, ServiceAccount and RoleBinding. - - TLS Secret can also contain a single "tls.pem" entry, containing the concatenation of the certificate and key - - If a TLS secret contains all of "tls.key", "tls.crt" and "tls.pem" entries, the operator will raise an error if the "tls.pem" one is not equal to the concatenation of "tls.crt" with "tls.key" - - Readinessprobe reports MongoDB running as Arbitrer as _Running_ & _Healthy_. - - The `CLUSTER_DOMAIN` environment variable can be set on the Operator Pod, to configure the Kubernetes cluster's Domain, - in case this one differs from the default `cluster.local`. +- Bug fixes + - Adds missing roles for Database Pods. + - Fixes OpenShift install. ## MongoDBCommunity Resource -* Changes -* Specifying `spec.additionalMongodConfig.storage.dbPath` will now be respected correctly. +* No changes ## Updated Image Tags -- mongodb-kubernetes-operator:0.7.1 +- mongodb-kubernetes-operator:0.7.2 _All the images can be found in:_ diff --git a/docs/how-to-release.md b/docs/how-to-release.md index 137d4e8b3..9f42c5f5a 100644 --- a/docs/how-to-release.md +++ b/docs/how-to-release.md @@ -5,6 +5,8 @@ * Update any changing versions in [release.json](../release.json). * Ensure that [the release notes](./RELEASE_NOTES.md) are up to date for this release. * Run `python scripts/ci/update_release.py` to update the relevant yaml manifests. + * Copy `CRD`s to Helm Chart + - `cp config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml helm-charts/charts/community-operator-crds/templates/mongodbcommunity.mongodb.com_mongodbcommunity.yaml` * Commit all changes. * Create a PR with the title `Release MongoDB Kubernetes Operator v` (the title must match this pattern) diff --git a/helm-charts b/helm-charts index 7c7641eb3..625081d8e 160000 --- a/helm-charts +++ b/helm-charts @@ -1 +1 @@ -Subproject commit 7c7641eb3b5ca6c716e44afae9bf99351f7b644d +Subproject commit 625081d8ea02e434e55e877a565815abb2254286 diff --git a/release.json b/release.json index f608c9437..d7074e5eb 100644 --- a/release.json +++ b/release.json @@ -1,5 +1,5 @@ { - "mongodb-kubernetes-operator": "0.7.1", + "mongodb-kubernetes-operator": "0.7.2", "version-upgrade-hook": "1.0.3", "readiness-probe": "1.0.6", "mongodb-agent": { diff --git a/scripts/ci/update_release.py b/scripts/ci/update_release.py index 511f9d220..1b2d9f2ff 100755 --- a/scripts/ci/update_release.py +++ b/scripts/ci/update_release.py @@ -2,16 +2,23 @@ import json import sys -import yaml -from typing import Dict +from typing import Dict, Callable + +import ruamel.yaml + +yaml = ruamel.yaml.YAML() + RELATIVE_PATH_TO_MANAGER_YAML = "config/manager/manager.yaml" RELATIVE_PATH_TO_OPENSHIFT_MANAGER_YAML = "deploy/openshift/operator_openshift.yaml" +RELATIVE_PATH_TO_CHART_VALUES = "helm-charts/charts/community-operator/values.yaml" +RELATIVE_PATH_TO_CHART = "helm-charts/charts/community-operator/Chart.yaml" + def _load_yaml_file(path: str) -> Dict: with open(path, "r") as f: - return yaml.safe_load(f.read()) + return yaml.load(f.read()) def _dump_yaml(operator: Dict, path: str) -> None: @@ -19,10 +26,10 @@ def _dump_yaml(operator: Dict, path: str) -> None: yaml.dump(operator, f) -def update_and_write_file(path: str) -> None: +def update_and_write_file(path: str, update_function: Callable) -> None: release = _load_release() yaml_file = _load_yaml_file(path) - _update_operator_deployment(yaml_file, release) + update_function(yaml_file, release) _dump_yaml(yaml_file, path) @@ -36,7 +43,7 @@ def _replace_tag(image: str, new_tag: str) -> str: return split_image[0] + ":" + new_tag -def _update_operator_deployment(operator_deployment: Dict, release: Dict) -> None: +def update_operator_deployment(operator_deployment: Dict, release: Dict) -> None: operator_container = operator_deployment["spec"]["template"]["spec"]["containers"][ 0 ] @@ -55,9 +62,33 @@ def _update_operator_deployment(operator_deployment: Dict, release: Dict) -> Non ) +def update_chart_values(values: Dict, release: Dict) -> None: + values["agent"]["version"] = release["mongodb-agent"]["version"] + values["versionUpgradeHook"]["version"] = release["version-upgrade-hook"] + values["readinessProbe"]["version"] = release["readiness-probe"] + values["operator"]["version"] = release["mongodb-kubernetes-operator"] + + +def update_chart(chart: Dict, release: Dict) -> None: + chart["version"] = release["mongodb-kubernetes-operator"] + chart["appVersion"] = release["mongodb-kubernetes-operator"] + + for dependency in chart["dependencies"]: + if dependency["name"] == "community-operator-crds": + dependency["version"] = release["mongodb-kubernetes-operator"] + + def main() -> int: - update_and_write_file(RELATIVE_PATH_TO_MANAGER_YAML) - update_and_write_file(RELATIVE_PATH_TO_OPENSHIFT_MANAGER_YAML) + # Updating local files + update_and_write_file(RELATIVE_PATH_TO_MANAGER_YAML, update_operator_deployment) + update_and_write_file( + RELATIVE_PATH_TO_OPENSHIFT_MANAGER_YAML, update_operator_deployment + ) + + # Updating Helm Chart files + update_and_write_file(RELATIVE_PATH_TO_CHART_VALUES, update_chart_values) + update_and_write_file(RELATIVE_PATH_TO_CHART, update_chart) + return 0 From 316be0034c8a8ae4324257b3fe84c2425555022d Mon Sep 17 00:00:00 2001 From: Rajdeep Das Date: Tue, 30 Nov 2021 17:18:25 +0100 Subject: [PATCH 430/790] Pass optional string in different methods (#794) This is needed for vault secret backend, if in future we need to pass more params, we can consider wrapping it in a struct --- pkg/authentication/scram/mock_types_test.go | 8 ++++---- pkg/automationconfig/automation_config_secret_test.go | 6 +++--- pkg/kube/client/client.go | 8 ++++---- pkg/kube/secret/secret.go | 8 ++++---- pkg/kube/secret/secret_test.go | 6 +++--- 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/pkg/authentication/scram/mock_types_test.go b/pkg/authentication/scram/mock_types_test.go index fbbd662fc..a4a2e17e3 100644 --- a/pkg/authentication/scram/mock_types_test.go +++ b/pkg/authentication/scram/mock_types_test.go @@ -11,21 +11,21 @@ type mockSecretGetUpdateCreateDeleter struct { secrets map[client.ObjectKey]corev1.Secret } -func (c mockSecretGetUpdateCreateDeleter) DeleteSecret(objectKey client.ObjectKey) error { +func (c mockSecretGetUpdateCreateDeleter) DeleteSecret(objectKey client.ObjectKey, path ...string) error { delete(c.secrets, objectKey) return nil } -func (c mockSecretGetUpdateCreateDeleter) UpdateSecret(s corev1.Secret) error { +func (c mockSecretGetUpdateCreateDeleter) UpdateSecret(s corev1.Secret, path ...string) error { c.secrets[types.NamespacedName{Name: s.Name, Namespace: s.Namespace}] = s return nil } -func (c mockSecretGetUpdateCreateDeleter) CreateSecret(secret corev1.Secret) error { +func (c mockSecretGetUpdateCreateDeleter) CreateSecret(secret corev1.Secret, path ...string) error { return c.UpdateSecret(secret) } -func (c mockSecretGetUpdateCreateDeleter) GetSecret(objectKey client.ObjectKey) (corev1.Secret, error) { +func (c mockSecretGetUpdateCreateDeleter) GetSecret(objectKey client.ObjectKey, path ...string) (corev1.Secret, error) { if s, ok := c.secrets[objectKey]; !ok { return corev1.Secret{}, notFoundError() } else { diff --git a/pkg/automationconfig/automation_config_secret_test.go b/pkg/automationconfig/automation_config_secret_test.go index 0450fc954..efaa6d7e4 100644 --- a/pkg/automationconfig/automation_config_secret_test.go +++ b/pkg/automationconfig/automation_config_secret_test.go @@ -83,7 +83,7 @@ type mockSecretGetUpdateCreator struct { secret *corev1.Secret } -func (m *mockSecretGetUpdateCreator) GetSecret(objectKey client.ObjectKey) (corev1.Secret, error) { +func (m *mockSecretGetUpdateCreator) GetSecret(objectKey client.ObjectKey, path ...string) (corev1.Secret, error) { if m.secret != nil { if objectKey.Name == m.secret.Name && objectKey.Namespace == m.secret.Namespace { return *m.secret, nil @@ -92,12 +92,12 @@ func (m *mockSecretGetUpdateCreator) GetSecret(objectKey client.ObjectKey) (core return corev1.Secret{}, notFoundError() } -func (m *mockSecretGetUpdateCreator) UpdateSecret(secret corev1.Secret) error { +func (m *mockSecretGetUpdateCreator) UpdateSecret(secret corev1.Secret, path ...string) error { m.secret = &secret return nil } -func (m *mockSecretGetUpdateCreator) CreateSecret(secret corev1.Secret) error { +func (m *mockSecretGetUpdateCreator) CreateSecret(secret corev1.Secret, path ...string) error { if m.secret == nil { m.secret = &secret return nil diff --git a/pkg/kube/client/client.go b/pkg/kube/client/client.go index f70048226..a0c6d5c6e 100644 --- a/pkg/kube/client/client.go +++ b/pkg/kube/client/client.go @@ -95,7 +95,7 @@ func (c client) GetPod(objectKey k8sClient.ObjectKey) (corev1.Pod, error) { } // GetSecret provides a thin wrapper and client.Client to access corev1.Secret types -func (c client) GetSecret(objectKey k8sClient.ObjectKey) (corev1.Secret, error) { +func (c client) GetSecret(objectKey k8sClient.ObjectKey, path ...string) (corev1.Secret, error) { s := corev1.Secret{} if err := c.Get(context.TODO(), objectKey, &s); err != nil { return corev1.Secret{}, err @@ -104,17 +104,17 @@ func (c client) GetSecret(objectKey k8sClient.ObjectKey) (corev1.Secret, error) } // UpdateSecret provides a thin wrapper and client.Client to update corev1.Secret types -func (c client) UpdateSecret(secret corev1.Secret) error { +func (c client) UpdateSecret(secret corev1.Secret, path ...string) error { return c.Update(context.TODO(), &secret) } // CreateSecret provides a thin wrapper and client.Client to create corev1.Secret types -func (c client) CreateSecret(secret corev1.Secret) error { +func (c client) CreateSecret(secret corev1.Secret, path ...string) error { return c.Create(context.TODO(), &secret) } // DeleteSecret provides a thin wrapper and client.Client to delete corev1.Secret types -func (c client) DeleteSecret(key k8sClient.ObjectKey) error { +func (c client) DeleteSecret(key k8sClient.ObjectKey, path ...string) error { s := corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: key.Name, diff --git a/pkg/kube/secret/secret.go b/pkg/kube/secret/secret.go index 3c1c2e383..b98921d62 100644 --- a/pkg/kube/secret/secret.go +++ b/pkg/kube/secret/secret.go @@ -13,19 +13,19 @@ import ( ) type Getter interface { - GetSecret(objectKey client.ObjectKey) (corev1.Secret, error) + GetSecret(objectKey client.ObjectKey, path ...string) (corev1.Secret, error) } type Updater interface { - UpdateSecret(secret corev1.Secret) error + UpdateSecret(secret corev1.Secret, path ...string) error } type Creator interface { - CreateSecret(secret corev1.Secret) error + CreateSecret(secret corev1.Secret, path ...string) error } type Deleter interface { - DeleteSecret(objectKey client.ObjectKey) error + DeleteSecret(objectKey client.ObjectKey, path ...string) error } type GetUpdater interface { diff --git a/pkg/kube/secret/secret_test.go b/pkg/kube/secret/secret_test.go index 3fc706acd..84f3a4d35 100644 --- a/pkg/kube/secret/secret_test.go +++ b/pkg/kube/secret/secret_test.go @@ -15,7 +15,7 @@ type secretGetter struct { secret corev1.Secret } -func (c secretGetter) GetSecret(objectKey client.ObjectKey) (corev1.Secret, error) { +func (c secretGetter) GetSecret(objectKey client.ObjectKey, path ...string) (corev1.Secret, error) { if c.secret.Name == objectKey.Name && c.secret.Namespace == objectKey.Namespace { return c.secret, nil } @@ -95,14 +95,14 @@ type secretGetUpdater struct { secret corev1.Secret } -func (c secretGetUpdater) GetSecret(objectKey client.ObjectKey) (corev1.Secret, error) { +func (c secretGetUpdater) GetSecret(objectKey client.ObjectKey, path ...string) (corev1.Secret, error) { if c.secret.Name == objectKey.Name && c.secret.Namespace == objectKey.Namespace { return c.secret, nil } return corev1.Secret{}, notFoundError() } -func (c *secretGetUpdater) UpdateSecret(s corev1.Secret) error { +func (c *secretGetUpdater) UpdateSecret(s corev1.Secret, path ...string) error { c.secret = s return nil } From 4ff30216cc10b414c36e4995b757ddb9e5545011 Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Wed, 1 Dec 2021 09:50:53 +0000 Subject: [PATCH 431/790] Revert "Pass optional string in different methods (#794)" (#796) --- pkg/authentication/scram/mock_types_test.go | 8 ++++---- pkg/automationconfig/automation_config_secret_test.go | 6 +++--- pkg/kube/client/client.go | 8 ++++---- pkg/kube/secret/secret.go | 8 ++++---- pkg/kube/secret/secret_test.go | 6 +++--- 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/pkg/authentication/scram/mock_types_test.go b/pkg/authentication/scram/mock_types_test.go index a4a2e17e3..fbbd662fc 100644 --- a/pkg/authentication/scram/mock_types_test.go +++ b/pkg/authentication/scram/mock_types_test.go @@ -11,21 +11,21 @@ type mockSecretGetUpdateCreateDeleter struct { secrets map[client.ObjectKey]corev1.Secret } -func (c mockSecretGetUpdateCreateDeleter) DeleteSecret(objectKey client.ObjectKey, path ...string) error { +func (c mockSecretGetUpdateCreateDeleter) DeleteSecret(objectKey client.ObjectKey) error { delete(c.secrets, objectKey) return nil } -func (c mockSecretGetUpdateCreateDeleter) UpdateSecret(s corev1.Secret, path ...string) error { +func (c mockSecretGetUpdateCreateDeleter) UpdateSecret(s corev1.Secret) error { c.secrets[types.NamespacedName{Name: s.Name, Namespace: s.Namespace}] = s return nil } -func (c mockSecretGetUpdateCreateDeleter) CreateSecret(secret corev1.Secret, path ...string) error { +func (c mockSecretGetUpdateCreateDeleter) CreateSecret(secret corev1.Secret) error { return c.UpdateSecret(secret) } -func (c mockSecretGetUpdateCreateDeleter) GetSecret(objectKey client.ObjectKey, path ...string) (corev1.Secret, error) { +func (c mockSecretGetUpdateCreateDeleter) GetSecret(objectKey client.ObjectKey) (corev1.Secret, error) { if s, ok := c.secrets[objectKey]; !ok { return corev1.Secret{}, notFoundError() } else { diff --git a/pkg/automationconfig/automation_config_secret_test.go b/pkg/automationconfig/automation_config_secret_test.go index efaa6d7e4..0450fc954 100644 --- a/pkg/automationconfig/automation_config_secret_test.go +++ b/pkg/automationconfig/automation_config_secret_test.go @@ -83,7 +83,7 @@ type mockSecretGetUpdateCreator struct { secret *corev1.Secret } -func (m *mockSecretGetUpdateCreator) GetSecret(objectKey client.ObjectKey, path ...string) (corev1.Secret, error) { +func (m *mockSecretGetUpdateCreator) GetSecret(objectKey client.ObjectKey) (corev1.Secret, error) { if m.secret != nil { if objectKey.Name == m.secret.Name && objectKey.Namespace == m.secret.Namespace { return *m.secret, nil @@ -92,12 +92,12 @@ func (m *mockSecretGetUpdateCreator) GetSecret(objectKey client.ObjectKey, path return corev1.Secret{}, notFoundError() } -func (m *mockSecretGetUpdateCreator) UpdateSecret(secret corev1.Secret, path ...string) error { +func (m *mockSecretGetUpdateCreator) UpdateSecret(secret corev1.Secret) error { m.secret = &secret return nil } -func (m *mockSecretGetUpdateCreator) CreateSecret(secret corev1.Secret, path ...string) error { +func (m *mockSecretGetUpdateCreator) CreateSecret(secret corev1.Secret) error { if m.secret == nil { m.secret = &secret return nil diff --git a/pkg/kube/client/client.go b/pkg/kube/client/client.go index a0c6d5c6e..f70048226 100644 --- a/pkg/kube/client/client.go +++ b/pkg/kube/client/client.go @@ -95,7 +95,7 @@ func (c client) GetPod(objectKey k8sClient.ObjectKey) (corev1.Pod, error) { } // GetSecret provides a thin wrapper and client.Client to access corev1.Secret types -func (c client) GetSecret(objectKey k8sClient.ObjectKey, path ...string) (corev1.Secret, error) { +func (c client) GetSecret(objectKey k8sClient.ObjectKey) (corev1.Secret, error) { s := corev1.Secret{} if err := c.Get(context.TODO(), objectKey, &s); err != nil { return corev1.Secret{}, err @@ -104,17 +104,17 @@ func (c client) GetSecret(objectKey k8sClient.ObjectKey, path ...string) (corev1 } // UpdateSecret provides a thin wrapper and client.Client to update corev1.Secret types -func (c client) UpdateSecret(secret corev1.Secret, path ...string) error { +func (c client) UpdateSecret(secret corev1.Secret) error { return c.Update(context.TODO(), &secret) } // CreateSecret provides a thin wrapper and client.Client to create corev1.Secret types -func (c client) CreateSecret(secret corev1.Secret, path ...string) error { +func (c client) CreateSecret(secret corev1.Secret) error { return c.Create(context.TODO(), &secret) } // DeleteSecret provides a thin wrapper and client.Client to delete corev1.Secret types -func (c client) DeleteSecret(key k8sClient.ObjectKey, path ...string) error { +func (c client) DeleteSecret(key k8sClient.ObjectKey) error { s := corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: key.Name, diff --git a/pkg/kube/secret/secret.go b/pkg/kube/secret/secret.go index b98921d62..3c1c2e383 100644 --- a/pkg/kube/secret/secret.go +++ b/pkg/kube/secret/secret.go @@ -13,19 +13,19 @@ import ( ) type Getter interface { - GetSecret(objectKey client.ObjectKey, path ...string) (corev1.Secret, error) + GetSecret(objectKey client.ObjectKey) (corev1.Secret, error) } type Updater interface { - UpdateSecret(secret corev1.Secret, path ...string) error + UpdateSecret(secret corev1.Secret) error } type Creator interface { - CreateSecret(secret corev1.Secret, path ...string) error + CreateSecret(secret corev1.Secret) error } type Deleter interface { - DeleteSecret(objectKey client.ObjectKey, path ...string) error + DeleteSecret(objectKey client.ObjectKey) error } type GetUpdater interface { diff --git a/pkg/kube/secret/secret_test.go b/pkg/kube/secret/secret_test.go index 84f3a4d35..3fc706acd 100644 --- a/pkg/kube/secret/secret_test.go +++ b/pkg/kube/secret/secret_test.go @@ -15,7 +15,7 @@ type secretGetter struct { secret corev1.Secret } -func (c secretGetter) GetSecret(objectKey client.ObjectKey, path ...string) (corev1.Secret, error) { +func (c secretGetter) GetSecret(objectKey client.ObjectKey) (corev1.Secret, error) { if c.secret.Name == objectKey.Name && c.secret.Namespace == objectKey.Namespace { return c.secret, nil } @@ -95,14 +95,14 @@ type secretGetUpdater struct { secret corev1.Secret } -func (c secretGetUpdater) GetSecret(objectKey client.ObjectKey, path ...string) (corev1.Secret, error) { +func (c secretGetUpdater) GetSecret(objectKey client.ObjectKey) (corev1.Secret, error) { if c.secret.Name == objectKey.Name && c.secret.Namespace == objectKey.Namespace { return c.secret, nil } return corev1.Secret{}, notFoundError() } -func (c *secretGetUpdater) UpdateSecret(s corev1.Secret, path ...string) error { +func (c *secretGetUpdater) UpdateSecret(s corev1.Secret) error { c.secret = s return nil } From 60964de803f510f8064c6676ab2ae65ff598efe0 Mon Sep 17 00:00:00 2001 From: Rajdeep Das Date: Wed, 1 Dec 2021 15:07:59 +0100 Subject: [PATCH 432/790] Configure SCRAM secret exist to check Vault as well (#797) --- pkg/authentication/scram/scram.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/pkg/authentication/scram/scram.go b/pkg/authentication/scram/scram.go index 2887f7d11..8d33bbc21 100644 --- a/pkg/authentication/scram/scram.go +++ b/pkg/authentication/scram/scram.go @@ -3,6 +3,7 @@ package scram import ( "encoding/base64" "fmt" + "strings" "github.com/pkg/errors" @@ -145,7 +146,7 @@ func ensureScramCredentials(getUpdateCreator secret.GetUpdateCreator, user User, password, err := secret.ReadKey(getUpdateCreator, user.PasswordSecretKey, types.NamespacedName{Name: user.PasswordSecretName, Namespace: mdbNamespacedName.Namespace}) if err != nil { // if the password is deleted, that's fine we can read from the stored credentials that were previously generated - if apiErrors.IsNotFound(err) { + if SecretNotExist(err) { zap.S().Debugf("password secret was not found, reading from credentials from secret/%s", user.ScramCredentialsSecretName) return readExistingCredentials(getUpdateCreator, mdbNamespacedName, user.ScramCredentialsSecretName) } @@ -188,7 +189,7 @@ func needToGenerateNewCredentials(secretGetter secret.Getter, username, scramCre s, err := secretGetter.GetSecret(types.NamespacedName{Name: scramCredentialsSecretName, Namespace: mdbNamespacedName.Namespace}) if err != nil { // haven't generated credentials yet, so we are changing password - if apiErrors.IsNotFound(err) { + if SecretNotExist(err) { zap.S().Debugf("No existing credentials found, generating new credentials") return true, nil } @@ -342,3 +343,10 @@ func convertMongoDBUserToAutomationConfigUser(secretGetUpdateCreateDeleter secre func (u User) GetConnectionStringSecretName(mdb Configurable) string { return fmt.Sprintf("%s-%s-%s", mdb.NamespacedName().Name, u.Database, u.Username) } + +func SecretNotExist(err error) bool { + if err == nil { + return false + } + return apiErrors.IsNotFound(err) || strings.Contains(err.Error(), "secret not found") +} From 9f8a0ff26a6e82286daee74ae1ef3b1f5d769d80 Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Wed, 1 Dec 2021 14:40:10 +0000 Subject: [PATCH 433/790] Change interface for automation config (#798) --- api/v1/mongodbcommunity_types.go | 7 +++++- controllers/construct/mongodbstatefulset.go | 22 ++++++++++------ pkg/authentication/scram/scram.go | 13 ++-------- .../automation_config_secret.go | 12 +++++---- pkg/kube/podtemplatespec/podspec_template.go | 7 +++++- pkg/kube/secret/secret.go | 8 ++++++ pkg/readiness/headless/headless.go | 25 +++++++++++++++++-- 7 files changed, 67 insertions(+), 27 deletions(-) diff --git a/api/v1/mongodbcommunity_types.go b/api/v1/mongodbcommunity_types.go index 8d8fbe223..d1f489327 100644 --- a/api/v1/mongodbcommunity_types.go +++ b/api/v1/mongodbcommunity_types.go @@ -3,10 +3,11 @@ package v1 import ( "encoding/json" "fmt" - "github.com/stretchr/objx" "net/url" "strings" + "github.com/stretchr/objx" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/authentication/scram" "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/annotations" @@ -653,6 +654,10 @@ func (m MongoDBCommunity) LogsVolumeName() string { return "logs-volume" } +func (m MongoDBCommunity) NeedsAutomationConfigVolume() bool { + return true +} + type automationConfigReplicasScaler struct { current, desired int } diff --git a/controllers/construct/mongodbstatefulset.go b/controllers/construct/mongodbstatefulset.go index 6f4351376..ef31271d1 100644 --- a/controllers/construct/mongodbstatefulset.go +++ b/controllers/construct/mongodbstatefulset.go @@ -89,6 +89,9 @@ type MongoDBStatefulSetOwner interface { // GetMongodConfiguration returns the MongoDB configuration for each member. GetMongodConfiguration() map[string]interface{} + + // NeedsAutomationConfigVolume returns whether the statefuslet needs to have a volume for the automationconfig. + NeedsAutomationConfigVolume() bool } // BuildMongoDBReplicaSetStatefulSetModificationFunction builds the parts of the replica set that are common between every resource that implements @@ -114,15 +117,20 @@ func BuildMongoDBReplicaSetStatefulSetModificationFunction(mdb MongoDBStatefulSe scriptsVolume := statefulset.CreateVolumeFromEmptyDir("agent-scripts") scriptsVolumeMount := statefulset.CreateVolumeMount(scriptsVolume.Name, "/opt/scripts", statefulset.WithReadOnly(false)) - automationConfigVolume := statefulset.CreateVolumeFromSecret("automation-config", mdb.AutomationConfigSecretName()) - automationConfigVolumeMount := statefulset.CreateVolumeMount(automationConfigVolume.Name, "/var/lib/automation/config", statefulset.WithReadOnly(true)) - keyFileNsName := mdb.GetAgentKeyfileSecretNamespacedName() keyFileVolume := statefulset.CreateVolumeFromEmptyDir(keyFileNsName.Name) keyFileVolumeVolumeMount := statefulset.CreateVolumeMount(keyFileVolume.Name, "/var/lib/mongodb-mms-automation/authentication", statefulset.WithReadOnly(false)) keyFileVolumeVolumeMountMongod := statefulset.CreateVolumeMount(keyFileVolume.Name, "/var/lib/mongodb-mms-automation/authentication", statefulset.WithReadOnly(false)) - mongodbAgentVolumeMounts := []corev1.VolumeMount{agentHealthStatusVolumeMount, automationConfigVolumeMount, scriptsVolumeMount, keyFileVolumeVolumeMount} + mongodbAgentVolumeMounts := []corev1.VolumeMount{agentHealthStatusVolumeMount, scriptsVolumeMount, keyFileVolumeVolumeMount} + + automationConfigVolumeFunc := podtemplatespec.NOOP() + if mdb.NeedsAutomationConfigVolume() { + automationConfigVolume := statefulset.CreateVolumeFromSecret("automation-config", mdb.AutomationConfigSecretName()) + automationConfigVolumeFunc = podtemplatespec.WithVolume(automationConfigVolume) + automationConfigVolumeMount := statefulset.CreateVolumeMount(automationConfigVolume.Name, "/var/lib/automation/config", statefulset.WithReadOnly(true)) + mongodbAgentVolumeMounts = append(mongodbAgentVolumeMounts, automationConfigVolumeMount) + } mongodVolumeMounts := []corev1.VolumeMount{mongodHealthStatusVolumeMount, hooksVolumeMount, keyFileVolumeVolumeMountMongod} dataVolumeClaim := statefulset.NOOP() logVolumeClaim := statefulset.NOOP() @@ -167,7 +175,7 @@ func BuildMongoDBReplicaSetStatefulSetModificationFunction(mdb MongoDBStatefulSe podtemplatespec.WithPodLabels(labels), podtemplatespec.WithVolume(healthStatusVolume), podtemplatespec.WithVolume(hooksVolume), - podtemplatespec.WithVolume(automationConfigVolume), + automationConfigVolumeFunc, podtemplatespec.WithVolume(scriptsVolume), podtemplatespec.WithVolume(keyFileVolume), podtemplatespec.WithServiceAccount(mongodbDatabaseServiceAccountName), @@ -180,11 +188,11 @@ func BuildMongoDBReplicaSetStatefulSetModificationFunction(mdb MongoDBStatefulSe } func BaseAgentCommand() string { - return "agent/mongodb-agent -cluster=" + clusterFilePath + " -healthCheckFilePath=" + agentHealthStatusFilePathValue + " -serveStatusPort=5000" + return "agent/mongodb-agent -healthCheckFilePath=" + agentHealthStatusFilePathValue + " -serveStatusPort=5000" } func AutomationAgentCommand() []string { - return []string{"/bin/bash", "-c", MongodbUserCommand + BaseAgentCommand() + automationAgentOptions} + return []string{"/bin/bash", "-c", MongodbUserCommand + BaseAgentCommand() + " -cluster=" + clusterFilePath + automationAgentOptions} } func mongodbAgentContainer(automationConfigSecretName string, volumeMounts []corev1.VolumeMount) container.Modification { diff --git a/pkg/authentication/scram/scram.go b/pkg/authentication/scram/scram.go index 8d33bbc21..92e1fd21d 100644 --- a/pkg/authentication/scram/scram.go +++ b/pkg/authentication/scram/scram.go @@ -3,7 +3,6 @@ package scram import ( "encoding/base64" "fmt" - "strings" "github.com/pkg/errors" @@ -14,7 +13,6 @@ import ( "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/secret" "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/generate" - apiErrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" ) @@ -146,7 +144,7 @@ func ensureScramCredentials(getUpdateCreator secret.GetUpdateCreator, user User, password, err := secret.ReadKey(getUpdateCreator, user.PasswordSecretKey, types.NamespacedName{Name: user.PasswordSecretName, Namespace: mdbNamespacedName.Namespace}) if err != nil { // if the password is deleted, that's fine we can read from the stored credentials that were previously generated - if SecretNotExist(err) { + if secret.SecretNotExist(err) { zap.S().Debugf("password secret was not found, reading from credentials from secret/%s", user.ScramCredentialsSecretName) return readExistingCredentials(getUpdateCreator, mdbNamespacedName, user.ScramCredentialsSecretName) } @@ -189,7 +187,7 @@ func needToGenerateNewCredentials(secretGetter secret.Getter, username, scramCre s, err := secretGetter.GetSecret(types.NamespacedName{Name: scramCredentialsSecretName, Namespace: mdbNamespacedName.Namespace}) if err != nil { // haven't generated credentials yet, so we are changing password - if SecretNotExist(err) { + if secret.SecretNotExist(err) { zap.S().Debugf("No existing credentials found, generating new credentials") return true, nil } @@ -343,10 +341,3 @@ func convertMongoDBUserToAutomationConfigUser(secretGetUpdateCreateDeleter secre func (u User) GetConnectionStringSecretName(mdb Configurable) string { return fmt.Sprintf("%s-%s-%s", mdb.NamespacedName().Name, u.Database, u.Username) } - -func SecretNotExist(err error) bool { - if err == nil { - return false - } - return apiErrors.IsNotFound(err) || strings.Contains(err.Error(), "secret not found") -} diff --git a/pkg/automationconfig/automation_config_secret.go b/pkg/automationconfig/automation_config_secret.go index c2bf81576..c27aa766e 100644 --- a/pkg/automationconfig/automation_config_secret.go +++ b/pkg/automationconfig/automation_config_secret.go @@ -3,10 +3,7 @@ package automationconfig import ( "encoding/json" - "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/secret" - apiErrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" ) @@ -18,7 +15,10 @@ const ConfigKey = "cluster-config.json" func ReadFromSecret(secretGetter secret.Getter, secretNsName types.NamespacedName) (AutomationConfig, error) { acSecret, err := secretGetter.GetSecret(secretNsName) if err != nil { - return AutomationConfig{}, client.IgnoreNotFound(err) + if secret.SecretNotExist(err) { + err = nil + } + return AutomationConfig{}, err } return FromBytes(acSecret.Data[ConfigKey]) } @@ -30,7 +30,7 @@ func ReadFromSecret(secretGetter secret.Getter, secretNsName types.NamespacedNam func EnsureSecret(secretGetUpdateCreator secret.GetUpdateCreator, secretNsName types.NamespacedName, owner []metav1.OwnerReference, desiredAutomationConfig AutomationConfig) (AutomationConfig, error) { existingSecret, err := secretGetUpdateCreator.GetSecret(secretNsName) if err != nil { - if apiErrors.IsNotFound(err) { + if secret.SecretNotExist(err) { return createNewAutomationConfigSecret(secretGetUpdateCreator, secretNsName, owner, desiredAutomationConfig) } return AutomationConfig{}, err @@ -60,6 +60,8 @@ func EnsureSecret(secretGetUpdateCreator secret.GetUpdateCreator, secretNsName t existingSecret.Data[ConfigKey] = acBytes } + existingSecret.Name = secretNsName.Name + existingSecret.Namespace = secretNsName.Namespace return desiredAutomationConfig, secretGetUpdateCreator.UpdateSecret(existingSecret) } diff --git a/pkg/kube/podtemplatespec/podspec_template.go b/pkg/kube/podtemplatespec/podspec_template.go index c50d7204e..e00ec80e5 100644 --- a/pkg/kube/podtemplatespec/podspec_template.go +++ b/pkg/kube/podtemplatespec/podspec_template.go @@ -231,7 +231,12 @@ func WithAnnotations(annotations map[string]string) Modification { annotations = map[string]string{} } return func(podTemplateSpec *corev1.PodTemplateSpec) { - podTemplateSpec.Annotations = annotations + if podTemplateSpec.Annotations == nil { + podTemplateSpec.Annotations = map[string]string{} + } + for k, v := range annotations { + podTemplateSpec.Annotations[k] = v + } } } diff --git a/pkg/kube/secret/secret.go b/pkg/kube/secret/secret.go index 3c1c2e383..9d4af5cb7 100644 --- a/pkg/kube/secret/secret.go +++ b/pkg/kube/secret/secret.go @@ -2,6 +2,7 @@ package secret import ( "reflect" + "strings" "github.com/pkg/errors" @@ -189,3 +190,10 @@ func CreateOrUpdateIfNeeded(getUpdateCreator GetUpdateCreator, secret corev1.Sec // They are different so we need to update it return getUpdateCreator.UpdateSecret(secret) } + +func SecretNotExist(err error) bool { + if err == nil { + return false + } + return apiErrors.IsNotFound(err) || strings.Contains(err.Error(), "secret not found") +} diff --git a/pkg/readiness/headless/headless.go b/pkg/readiness/headless/headless.go index 14f82aa72..4f68975d1 100644 --- a/pkg/readiness/headless/headless.go +++ b/pkg/readiness/headless/headless.go @@ -1,6 +1,10 @@ package headless import ( + "io/ioutil" + "os" + "strconv" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/readiness/config" "github.com/mongodb/mongodb-kubernetes-operator/pkg/readiness/health" "github.com/mongodb/mongodb-kubernetes-operator/pkg/readiness/pod" @@ -15,9 +19,26 @@ import ( // https://kubernetes.io/docs/tasks/access-application-cluster/access-cluster/#accessing-the-api-from-a-pod) // though passing the namespace as an environment variable makes the code simpler for testing and saves an IO operation func PerformCheckHeadlessMode(health health.Status, conf config.Config) (bool, error) { - targetVersion, err := secret.ReadAutomationConfigVersionFromSecret(conf.Namespace, conf.ClientSet, conf.AutomationConfigSecretName) + var targetVersion int64 + var err error + targetVersion, err = secret.ReadAutomationConfigVersionFromSecret(conf.Namespace, conf.ClientSet, conf.AutomationConfigSecretName) if err != nil { - return false, err + file, err := os.Open("/var/lib/automation/config/acVersion/version") + if err != nil { + return false, err + } + defer file.Close() + data, err := ioutil.ReadAll(file) + + if err != nil { + return false, err + } + + targetVersion, err = strconv.ParseInt(string(data), 10, 64) + + if err != nil { + return false, err + } } currentAgentVersion := readCurrentAgentInfo(health, targetVersion) From ad1af8dae522ebf7b78f92c45e9aeca50c622ab9 Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Wed, 1 Dec 2021 15:02:32 +0000 Subject: [PATCH 434/790] Release Readiness Probe 1.0.7 (#799) --- release.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release.json b/release.json index d7074e5eb..692db1395 100644 --- a/release.json +++ b/release.json @@ -1,7 +1,7 @@ { "mongodb-kubernetes-operator": "0.7.2", "version-upgrade-hook": "1.0.3", - "readiness-probe": "1.0.6", + "readiness-probe": "1.0.7", "mongodb-agent": { "version": "11.0.5.6963-1", "tools_version": "100.3.1" From bbb5ca14130eba88475379bd990dfc77005f5eae Mon Sep 17 00:00:00 2001 From: Rajdeep Das Date: Wed, 1 Dec 2021 17:00:53 +0100 Subject: [PATCH 435/790] Configure secret with Secret Client (#800) --- pkg/kube/secret/secret.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/kube/secret/secret.go b/pkg/kube/secret/secret.go index 9d4af5cb7..4f5e213a2 100644 --- a/pkg/kube/secret/secret.go +++ b/pkg/kube/secret/secret.go @@ -99,7 +99,7 @@ func UpdateField(getUpdater GetUpdater, objectKey client.ObjectKey, key, value s func CreateOrUpdate(getUpdateCreator GetUpdateCreator, secret corev1.Secret) error { _, err := getUpdateCreator.GetSecret(types.NamespacedName{Name: secret.Name, Namespace: secret.Namespace}) if err != nil { - if apiErrors.IsNotFound(err) { + if SecretNotExist(err) { return getUpdateCreator.CreateSecret(secret) } return err @@ -123,7 +123,7 @@ func HasAllKeys(secret corev1.Secret, keys ...string) bool { func EnsureSecretWithKey(secretGetUpdateCreateDeleter GetUpdateCreateDeleter, nsName types.NamespacedName, ownerReferences []metav1.OwnerReference, key, value string) (string, error) { existingSecret, err0 := secretGetUpdateCreateDeleter.GetSecret(nsName) if err0 != nil { - if apiErrors.IsNotFound(err0) { + if SecretNotExist(err0) { s := Builder(). SetNamespace(nsName.Namespace). SetName(nsName.Name). From 63b3567c83bdadfca3f089e57a3943617085dd99 Mon Sep 17 00:00:00 2001 From: Rodrigo Valin Date: Thu, 2 Dec 2021 11:05:59 +0000 Subject: [PATCH 436/790] CLOUDP-105706: Arbiters go into own Statefulset (#785) --- api/v1/mongodbcommunity_types.go | 94 ++++++++++++++-- ...ommunity.mongodb.com_mongodbcommunity.yaml | 11 +- controllers/mongodb_status_options.go | 38 +++++++ controllers/replica_set_controller.go | 106 +++++++++++++++--- docs/RELEASE_NOTES.md | 15 ++- pkg/automationconfig/automation_config.go | 17 ++- .../automation_config_builder.go | 77 ++++++++++--- .../automation_config_test.go | 102 ++++++++--------- pkg/util/scale/scale.go | 11 +- pkg/util/versions/versions.go | 3 +- test/e2e/mongodbtests/mongodbtests.go | 42 ++++++- .../replica_set_arbiter_test.go | 12 +- test/e2e/util/wait/wait.go | 36 +++++- 13 files changed, 440 insertions(+), 124 deletions(-) diff --git a/api/v1/mongodbcommunity_types.go b/api/v1/mongodbcommunity_types.go index d1f489327..cfdb7ebd9 100644 --- a/api/v1/mongodbcommunity_types.go +++ b/api/v1/mongodbcommunity_types.go @@ -62,7 +62,9 @@ type MongoDBCommunitySpec struct { // Version defines which version of MongoDB will be used Version string `json:"version,omitempty"` - // Arbiters is the number of arbiters (each counted as a member) in the replica set + // Arbiters is the number of arbiters to add to the Replica Set. + // It is not recommended to have more than one arbiter per Replica Set. + // More info: https://docs.mongodb.com/manual/tutorial/add-replica-set-arbiter/ // +optional Arbiters int `json:"arbiters"` @@ -392,6 +394,9 @@ type MongoDBCommunityStatus struct { CurrentStatefulSetReplicas int `json:"currentStatefulSetReplicas"` CurrentMongoDBMembers int `json:"currentMongoDBMembers"` + CurrentStatefulSetArbitersReplicas int `json:"currentStatefulSetArbitersReplicas"` + CurrentMongoDBArbiters int `json:"currentMongoDBArbiters"` + Message string `json:"message,omitempty"` } @@ -493,12 +498,39 @@ func (m MongoDBCommunity) GetScramUsers() []scram.User { return users } +// IsStillScaling returns true if this resource is currently scaling, +// considering both arbiters and regular members. +func (m MongoDBCommunity) IsStillScaling() bool { + return scale.IsStillScaling(m) || scale.IsStillScaling(automationConfigReplicasScaler{ + current: m.CurrentArbiters(), + desired: m.DesiredArbiters(), + }) +} + +// AutomationConfigMembersThisReconciliation determines the correct number of +// automation config replica set members based on our desired number, and our +// current number. func (m MongoDBCommunity) AutomationConfigMembersThisReconciliation() int { - // determine the correct number of automation config replica set members - // based on our desired number, and our current number return scale.ReplicasThisReconciliation(automationConfigReplicasScaler{ - desired: m.Spec.Members, current: m.Status.CurrentMongoDBMembers, + desired: m.Spec.Members, + }) +} + +// AutomationConfigArbitersThisReconciliation determines the correct number of +// automation config replica set arbiters based on our desired number, and our +// current number. +// +// Will not update arbiters until members have reached desired number. +func (m MongoDBCommunity) AutomationConfigArbitersThisReconciliation() int { + if m.AutomationConfigMembersThisReconciliation() < m.Spec.Members { + return m.Status.CurrentMongoDBArbiters + } + + return scale.ReplicasThisReconciliation(automationConfigReplicasScaler{ + desired: m.Spec.Arbiters, + current: m.Status.CurrentMongoDBArbiters, + forceIndividualScaling: true, }) } @@ -555,7 +587,7 @@ func (m MongoDBCommunity) Hosts(clusterDomain string) []string { return hosts } -// ServiceName returns the name of the Service that should be created for this resource +// ServiceName returns the name of the Service that should be created for this resource. func (m MongoDBCommunity) ServiceName() string { serviceName := m.Spec.StatefulSetConfiguration.SpecWrapper.Spec.ServiceName if serviceName != "" { @@ -564,6 +596,15 @@ func (m MongoDBCommunity) ServiceName() string { return m.Name + "-svc" } +// ArbiterServiceName returns the name of the Service for the Arbiters for this resource. +func (m MongoDBCommunity) ArbiterServiceName() string { + serviceName := m.Spec.StatefulSetConfiguration.SpecWrapper.Spec.ServiceName + if serviceName != "" { + return serviceName + "-arb-svc" + } + return m.Name + "-arb-svc" +} + func (m MongoDBCommunity) AutomationConfigSecretName() string { return m.Name + "-config" } @@ -603,6 +644,28 @@ func (m MongoDBCommunity) CurrentReplicas() int { return m.Status.CurrentStatefulSetReplicas } +// ForcedIndividualScaling if set to true, will always scale the deployment 1 by +// 1, even if the resource has been just created. +// +// The reason for this is that we have 2 types of resources that are scaled at +// different times: a) Regular members, which can be scaled from 0->n, for +// instance, when the resource was just created; and b) Arbiters, which will be +// scaled from 0->M 1 by 1 at all times. +// +// This was done to simplify the process of scaling arbiters, *after* members +// have reached the desired amount of replicas. +func (m MongoDBCommunity) ForcedIndividualScaling() bool { + return false +} + +func (m MongoDBCommunity) DesiredArbiters() int { + return m.Spec.Arbiters +} + +func (m MongoDBCommunity) CurrentArbiters() int { + return m.Status.CurrentStatefulSetArbitersReplicas +} + func (m MongoDBCommunity) GetMongoDBVersion() string { return m.Spec.Version } @@ -615,7 +678,19 @@ func (m MongoDBCommunity) GetMongoDBVersionForAnnotation() string { } func (m *MongoDBCommunity) StatefulSetReplicasThisReconciliation() int { - return scale.ReplicasThisReconciliation(m) + return scale.ReplicasThisReconciliation(automationConfigReplicasScaler{ + desired: m.DesiredReplicas(), + current: m.CurrentReplicas(), + forceIndividualScaling: false, + }) +} + +func (m *MongoDBCommunity) StatefulSetArbitersThisReconciliation() int { + return scale.ReplicasThisReconciliation(automationConfigReplicasScaler{ + desired: m.DesiredArbiters(), + current: m.CurrentArbiters(), + forceIndividualScaling: true, + }) } // GetUpdateStrategyType returns the type of RollingUpgradeStrategy that the @@ -659,7 +734,8 @@ func (m MongoDBCommunity) NeedsAutomationConfigVolume() bool { } type automationConfigReplicasScaler struct { - current, desired int + current, desired int + forceIndividualScaling bool } func (a automationConfigReplicasScaler) DesiredReplicas() int { @@ -670,6 +746,10 @@ func (a automationConfigReplicasScaler) CurrentReplicas() int { return a.current } +func (a automationConfigReplicasScaler) ForcedIndividualScaling() bool { + return a.forceIndividualScaling +} + // +kubebuilder:object:root=true // MongoDBCommunityList contains a list of MongoDB diff --git a/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml b/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml index ae01f98f8..222e94520 100644 --- a/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml +++ b/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml @@ -55,8 +55,9 @@ spec: type: object x-kubernetes-preserve-unknown-fields: true arbiters: - description: Arbiters is the number of arbiters (each counted as a - member) in the replica set + description: 'Arbiters is the number of arbiters to add to the Replica + Set. It is not recommended to have more than one arbiter per Replica + Set. More info: https://docs.mongodb.com/manual/tutorial/add-replica-set-arbiter/' type: integer featureCompatibilityVersion: description: FeatureCompatibilityVersion configures the feature compatibility @@ -323,8 +324,12 @@ spec: status: description: MongoDBCommunityStatus defines the observed state of MongoDB properties: + currentMongoDBArbiters: + type: integer currentMongoDBMembers: type: integer + currentStatefulSetArbitersReplicas: + type: integer currentStatefulSetReplicas: type: integer message: @@ -336,7 +341,9 @@ spec: version: type: string required: + - currentMongoDBArbiters - currentMongoDBMembers + - currentStatefulSetArbitersReplicas - currentStatefulSetReplicas - mongoUri - phase diff --git a/controllers/mongodb_status_options.go b/controllers/mongodb_status_options.go index 1e4074198..5961bdcbd 100644 --- a/controllers/mongodb_status_options.go +++ b/controllers/mongodb_status_options.go @@ -132,6 +132,20 @@ func (o *optionBuilder) withStatefulSetReplicas(members int) *optionBuilder { return o } +func (o *optionBuilder) withMongoDBArbiters(arbiters int) *optionBuilder { + o.options = append(o.options, mongoDBArbitersOption{ + mongoDBArbiters: arbiters, + }) + return o +} + +func (o *optionBuilder) withStatefulSetArbiters(arbiters int) *optionBuilder { + o.options = append(o.options, statefulSetArbitersOption{ + arbiters: arbiters, + }) + return o +} + func (o *optionBuilder) withMessage(severityLevel severity, msg string) *optionBuilder { if apierrors.IsTransientMessage(msg) { severityLevel = Debug @@ -203,3 +217,27 @@ func (s statefulSetReplicasOption) ApplyOption(mdb *mdbv1.MongoDBCommunity) { func (s statefulSetReplicasOption) GetResult() (reconcile.Result, error) { return result.OK() } + +type mongoDBArbitersOption struct { + mongoDBArbiters int +} + +func (a mongoDBArbitersOption) ApplyOption(mdb *mdbv1.MongoDBCommunity) { + mdb.Status.CurrentMongoDBArbiters = a.mongoDBArbiters +} + +func (a mongoDBArbitersOption) GetResult() (reconcile.Result, error) { + return result.OK() +} + +type statefulSetArbitersOption struct { + arbiters int +} + +func (s statefulSetArbitersOption) ApplyOption(mdb *mdbv1.MongoDBCommunity) { + mdb.Status.CurrentStatefulSetArbitersReplicas = s.arbiters +} + +func (s statefulSetArbitersOption) GetResult() (reconcile.Result, error) { + return result.OK() +} diff --git a/controllers/replica_set_controller.go b/controllers/replica_set_controller.go index 1aa109eed..aadeba54f 100644 --- a/controllers/replica_set_controller.go +++ b/controllers/replica_set_controller.go @@ -4,9 +4,10 @@ import ( "context" "encoding/json" "fmt" + "os" + "github.com/imdario/mergo" "github.com/stretchr/objx" - "os" "github.com/mongodb/mongodb-kubernetes-operator/controllers/predicates" "sigs.k8s.io/controller-runtime/pkg/builder" @@ -141,10 +142,19 @@ func (r ReplicaSetReconciler) Reconcile(ctx context.Context, request reconcile.R } r.log.Debug("Ensuring the service exists") - if err := r.ensureService(mdb); err != nil { + if err := r.ensureService(mdb, false); err != nil { + return status.Update(r.client.Status(), &mdb, + statusOptions(). + withMessage(Error, fmt.Sprintf("Error ensuring the service (members) exists: %s", err)). + withFailedPhase(), + ) + } + + r.log.Debug("Ensuring the service for Arbiters exists") + if err := r.ensureService(mdb, true); err != nil { return status.Update(r.client.Status(), &mdb, statusOptions(). - withMessage(Error, fmt.Sprintf("Error ensuring the service exists: %s", err)). + withMessage(Error, fmt.Sprintf("Error ensuring the service (arbiters) exists: %s", err)). withFailedPhase(), ) } @@ -208,12 +218,14 @@ func (r ReplicaSetReconciler) Reconcile(ctx context.Context, request reconcile.R ) } - if scale.IsStillScaling(mdb) { + if mdb.IsStillScaling() { return status.Update(r.client.Status(), &mdb, statusOptions(). withMongoDBMembers(mdb.AutomationConfigMembersThisReconciliation()). withMessage(Info, fmt.Sprintf("Performing scaling operation, currentMembers=%d, desiredMembers=%d", mdb.CurrentReplicas(), mdb.DesiredReplicas())). withStatefulSetReplicas(mdb.StatefulSetReplicasThisReconciliation()). + withStatefulSetArbiters(mdb.StatefulSetArbitersThisReconciliation()). + withMongoDBArbiters(mdb.AutomationConfigArbitersThisReconciliation()). withPendingPhase(10), ) } @@ -223,6 +235,8 @@ func (r ReplicaSetReconciler) Reconcile(ctx context.Context, request reconcile.R withMongoURI(mdb.MongoURI(os.Getenv(clusterDomain))). withMongoDBMembers(mdb.AutomationConfigMembersThisReconciliation()). withStatefulSetReplicas(mdb.StatefulSetReplicasThisReconciliation()). + withStatefulSetArbiters(mdb.StatefulSetArbitersThisReconciliation()). + withMongoDBArbiters(mdb.AutomationConfigArbitersThisReconciliation()). withMessage(None, ""). withRunningPhase(). withVersion(mdb.GetMongoDBVersion()), @@ -283,13 +297,26 @@ func (r *ReplicaSetReconciler) ensureTLSResources(mdb mdbv1.MongoDBCommunity) er } // deployStatefulSet deploys the backing StatefulSet of the MongoDBCommunity resource. +// +// When `Spec.Arbiters` > 0, a second StatefulSet will be created, with the amount +// of Pods corresponding to the amount of expected arbiters. +// // The returned boolean indicates that the StatefulSet is ready. func (r *ReplicaSetReconciler) deployStatefulSet(mdb mdbv1.MongoDBCommunity) (bool, error) { r.log.Info("Creating/Updating StatefulSet") - if err := r.createOrUpdateStatefulSet(mdb); err != nil { + if err := r.createOrUpdateStatefulSet(mdb, false); err != nil { return false, errors.Errorf("error creating/updating StatefulSet: %s", err) } + if mdb.Spec.Arbiters > 0 { + r.log.Info("Creating/Updating StatefulSet for Arbiters") + if err := r.createOrUpdateStatefulSet(mdb, true); err != nil { + return false, errors.Errorf("error creating/updating StatefulSet: %s", err) + } + } else { + r.log.Info("Arbiters set to 0, not creating another STS") + } + currentSts, err := r.client.GetStatefulSet(mdb.NamespacedName()) if err != nil { return false, errors.Errorf("error getting StatefulSet: %s", err) @@ -383,24 +410,40 @@ func (r *ReplicaSetReconciler) deployMongoDBReplicaSet(mdb mdbv1.MongoDBCommunit }) } -func (r *ReplicaSetReconciler) ensureService(mdb mdbv1.MongoDBCommunity) error { - svc := buildService(mdb) +// ensureService creates a Service unless it already exists. +// +// The Service definition is built from the `mdb` resource. If `isArbiter` is set to true, the Service +// will be created for the arbiters Statefulset. +func (r *ReplicaSetReconciler) ensureService(mdb mdbv1.MongoDBCommunity, isArbiter bool) error { + svc := buildService(mdb, isArbiter) err := r.client.Create(context.TODO(), &svc) if err != nil && apiErrors.IsAlreadyExists(err) { r.log.Infof("The service already exists... moving forward: %s", err) return nil } + return err } -func (r *ReplicaSetReconciler) createOrUpdateStatefulSet(mdb mdbv1.MongoDBCommunity) error { +func (r *ReplicaSetReconciler) createOrUpdateStatefulSet(mdb mdbv1.MongoDBCommunity, isArbiter bool) error { set := appsv1.StatefulSet{} - err := r.client.Get(context.TODO(), mdb.NamespacedName(), &set) + + name := mdb.NamespacedName() + if isArbiter { + name.Name = name.Name + "-arb" + } + + err := r.client.Get(context.TODO(), name, &set) err = k8sClient.IgnoreNotFound(err) if err != nil { return errors.Errorf("error getting StatefulSet: %s", err) } + buildStatefulSetModificationFunction(mdb)(&set) + if isArbiter { + buildArbitersModificationFunction(mdb)(&set) + } + if _, err = statefulset.CreateOrUpdate(r.client, set); err != nil { return errors.Errorf("error creating/updating StatefulSet: %s", err) } @@ -421,19 +464,27 @@ func (r ReplicaSetReconciler) ensureAutomationConfig(mdb mdbv1.MongoDBCommunity) mdb.GetOwnerReferences(), ac, ) - } func buildAutomationConfig(mdb mdbv1.MongoDBCommunity, auth automationconfig.Auth, currentAc automationconfig.AutomationConfig, modifications ...automationconfig.Modification) (automationconfig.AutomationConfig, error) { domain := getDomain(mdb.ServiceName(), mdb.Namespace, os.Getenv(clusterDomain)) + arbiterDomain := getDomain(mdb.ArbiterServiceName(), mdb.Namespace, os.Getenv(clusterDomain)) + zap.S().Debugw("AutomationConfigMembersThisReconciliation", "mdb.AutomationConfigMembersThisReconciliation()", mdb.AutomationConfigMembersThisReconciliation()) + arbitersCount := mdb.AutomationConfigArbitersThisReconciliation() + if mdb.AutomationConfigMembersThisReconciliation() < mdb.Spec.Members { + // Have not reached desired amount of members yet, should not scale arbiters + arbitersCount = mdb.Status.CurrentMongoDBArbiters + } + return automationconfig.NewBuilder(). SetTopology(automationconfig.ReplicaSetTopology). SetName(mdb.Name). SetDomain(domain). + SetArbiterDomain(arbiterDomain). SetMembers(mdb.AutomationConfigMembersThisReconciliation()). - SetArbiters(mdb.Spec.Arbiters). + SetArbiters(arbitersCount). SetReplicaSetHorizons(mdb.Spec.ReplicaSetHorizons). SetPreviousAutomationConfig(currentAc). SetMongoDBVersion(mdb.Spec.Version). @@ -448,13 +499,21 @@ func buildAutomationConfig(mdb mdbv1.MongoDBCommunity, auth automationconfig.Aut // buildService creates a Service that will be used for the Replica Set StatefulSet // that allows all the members of the STS to see each other. -// TODO: Make sure this Service is as minimal as possible, to not interfere with -// future implementations and Service Discovery mechanisms we might implement. -func buildService(mdb mdbv1.MongoDBCommunity) corev1.Service { +// +// If `isArbiter` is true, the function will create a Service suitable for the +// Arbiter's StatefulSet. +func buildService(mdb mdbv1.MongoDBCommunity, isArbiter bool) corev1.Service { label := make(map[string]string) - label["app"] = mdb.ServiceName() + + name := mdb.ServiceName() + if isArbiter { + name = mdb.ArbiterServiceName() + } + + label["app"] = name + return service.Builder(). - SetName(mdb.ServiceName()). + SetName(name). SetNamespace(mdb.Namespace). SetSelector(label). SetServiceType(corev1.ServiceTypeClusterIP). @@ -517,13 +576,18 @@ func (r ReplicaSetReconciler) buildAutomationConfig(mdb mdbv1.MongoDBCommunity) return automationconfig.AutomationConfig{}, errors.Errorf("could not configure scram authentication: %s", err) } - return buildAutomationConfig( + automationConfig, err := buildAutomationConfig( mdb, auth, currentAC, tlsModification, customRolesModification, ) + if err != nil { + return automationconfig.AutomationConfig{}, errors.Errorf("could not create an automation config: %s", err) + } + + return automationConfig, nil } // getMongodConfigModification will merge the additional configuration in the CRD @@ -561,6 +625,14 @@ func buildStatefulSetModificationFunction(mdb mdbv1.MongoDBCommunity) statefulse ) } +func buildArbitersModificationFunction(mdb mdbv1.MongoDBCommunity) statefulset.Modification { + return statefulset.Apply( + statefulset.WithReplicas(mdb.StatefulSetArbitersThisReconciliation()), + statefulset.WithServiceName(mdb.ArbiterServiceName()), + statefulset.WithName(mdb.Name+"-arb"), + ) +} + func getDomain(service, namespace, clusterName string) string { if clusterName == "" { clusterName = "cluster.local" diff --git a/docs/RELEASE_NOTES.md b/docs/RELEASE_NOTES.md index 640123540..21ca2cd4e 100644 --- a/docs/RELEASE_NOTES.md +++ b/docs/RELEASE_NOTES.md @@ -1,18 +1,21 @@ -# MongoDB Kubernetes Operator 0.7.2 +# MongoDB Kubernetes Operator 0.7.3 ## Kubernetes Operator -- Bug fixes - - Adds missing roles for Database Pods. - - Fixes OpenShift install. +- Changes + - The Operator can correctly scale arbiters up and down. When arbiters are + enabled (this is, when `spec.arbiters > 0`), a new StatefulSet will be + created to hold the Pods that will act as arbiters. The new StatefulSet will + be named `-arb`. ## MongoDBCommunity Resource -* No changes + +- No changes. ## Updated Image Tags -- mongodb-kubernetes-operator:0.7.2 +- mongodb-kubernetes-operator:0.7.3 _All the images can be found in:_ diff --git a/pkg/automationconfig/automation_config.go b/pkg/automationconfig/automation_config.go index b0c3ba79b..9739a0966 100644 --- a/pkg/automationconfig/automation_config.go +++ b/pkg/automationconfig/automation_config.go @@ -135,22 +135,21 @@ type ReplicaSetMember struct { type ReplicaSetHorizons map[string]string -func newReplicaSetMember(p Process, id int, horizons ReplicaSetHorizons, totalVotesSoFar int, numberArbiters int) ReplicaSetMember { +// newReplicaSetMember returns a ReplicaSetMember. +func newReplicaSetMember(name string, id int, horizons ReplicaSetHorizons, isArbiter bool, isVotingMember bool) ReplicaSetMember { // ensure that the number of voting members in the replica set is not more than 7 // as this is the maximum number of voting members. - votes := 1 - priority := 1 + votes := 0 + priority := 0 - isArbiter := totalVotesSoFar < numberArbiters - - if totalVotesSoFar > maxVotingMembers { - votes = 0 - priority = 0 + if isVotingMember { + votes = 1 + priority = 1 } return ReplicaSetMember{ Id: id, - Host: p.Name, + Host: name, Priority: priority, ArbiterOnly: isArbiter, Votes: votes, diff --git a/pkg/automationconfig/automation_config_builder.go b/pkg/automationconfig/automation_config_builder.go index e922d2b58..c86e60425 100644 --- a/pkg/automationconfig/automation_config_builder.go +++ b/pkg/automationconfig/automation_config_builder.go @@ -14,8 +14,9 @@ import ( type Topology string const ( - ReplicaSetTopology Topology = "ReplicaSet" - maxVotingMembers int = 7 + ReplicaSetTopology Topology = "ReplicaSet" + maxVotingMembers int = 7 + arbitersStartingIndex int = 100 ) type Modification func(*AutomationConfig) @@ -31,6 +32,7 @@ type Builder struct { members int arbiters int domain string + arbiterDomain string name string fcv string topology Topology @@ -104,6 +106,11 @@ func (b *Builder) SetDomain(domain string) *Builder { return b } +func (b *Builder) SetArbiterDomain(domain string) *Builder { + b.arbiterDomain = domain + return b +} + func (b *Builder) SetName(name string) *Builder { b.name = name return b @@ -204,13 +211,21 @@ func (b *Builder) setFeatureCompatibilityVersionIfUpgradeIsHappening() error { } func (b *Builder) Build() (AutomationConfig, error) { - hostnames := make([]string, b.members) + hostnames := make([]string, b.members+b.arbiters) + + // Create hostnames for data-bearing nodes. They start from 0 for i := 0; i < b.members; i++ { hostnames[i] = fmt.Sprintf("%s-%d.%s", b.name, i, b.domain) } - members := make([]ReplicaSetMember, b.members) - processes := make([]Process, b.members) + // Create hostnames for arbiters. They are added right after the regular members + for i := b.members; i < b.arbiters+b.members; i++ { + // Arbiters will be in b.name-arb-svc service + hostnames[i] = fmt.Sprintf("%s-arb-%d.%s", b.name, i-b.members, b.arbiterDomain) + } + + members := make([]ReplicaSetMember, b.members+b.arbiters) + processes := make([]Process, b.members+b.arbiters) if err := b.setFeatureCompatibilityVersionIfUpgradeIsHappening(); err != nil { return AutomationConfig{}, errors.Errorf("can't build the automation config: %s", err) @@ -221,22 +236,36 @@ func (b *Builder) Build() (AutomationConfig, error) { dataDir = b.dataDir } - totalVotes := 0 for i, h := range hostnames { + // Arbiters start counting from b.members and up + isArbiter := i >= b.members + replicaSetIndex := i + processIndex := i + + if isArbiter { + processIndex = i - b.members + // The arbiter's index will start on `arbitersStartingIndex` and increase + // from there. These ids must be kept constant if the data-bearing nodes + // change indexes, if for instance, they are scaled up and down. + // + replicaSetIndex = arbitersStartingIndex + processIndex + } + fcv := versions.CalculateFeatureCompatibilityVersion(b.mongodbVersion) + if b.fcv != "" { + fcv = b.fcv + } + + // TODO: Replace with a Builder for Process. process := &Process{ - Name: toProcessName(b.name, i), + Name: toProcessName(b.name, processIndex, isArbiter), HostName: h, - FeatureCompatibilityVersion: versions.CalculateFeatureCompatibilityVersion(b.mongodbVersion), + FeatureCompatibilityVersion: fcv, ProcessType: Mongod, Version: b.mongodbVersion, AuthSchemaVersion: 5, } - if b.fcv != "" { - process.FeatureCompatibilityVersion = b.fcv - } - process.SetPort(27017) process.SetStoragePath(dataDir) process.SetReplicaSetName(b.name) @@ -244,16 +273,24 @@ func (b *Builder) Build() (AutomationConfig, error) { for _, mod := range b.processModifications { mod(i, process) } - processes[i] = *process + var horizon ReplicaSetHorizons if b.replicaSetHorizons != nil { - members[i] = newReplicaSetMember(*process, i, b.replicaSetHorizons[i], totalVotes, b.arbiters) - } else { - members[i] = newReplicaSetMember(*process, i, nil, totalVotes, b.arbiters) + horizon = b.replicaSetHorizons[i] + } + + isVotingMember := true + if !isArbiter && i >= (maxVotingMembers-b.arbiters) { + // Arbiters can't be non-voting members + // If there are more than 7 (maxVotingMembers) members on this Replica Set + // those that lose right to vote should be the data-bearing nodes, not the + // arbiters. + isVotingMember = false } - totalVotes += members[i].Votes + // TODO: Replace with a Builder for ReplicaSetMember. + members[i] = newReplicaSetMember(process.Name, replicaSetIndex, horizon, isArbiter, isVotingMember) } if b.auth == nil { @@ -274,6 +311,7 @@ func (b *Builder) Build() (AutomationConfig, error) { Id: b.name, Members: members, ProtocolVersion: "1", + NumberArbiters: b.arbiters, }, }, MonitoringVersions: b.monitoringVersions, @@ -311,7 +349,10 @@ func (b *Builder) Build() (AutomationConfig, error) { return currentAc, nil } -func toProcessName(name string, index int) string { +func toProcessName(name string, index int, isArbiter bool) string { + if isArbiter { + return fmt.Sprintf("%s-arb-%d", name, index) + } return fmt.Sprintf("%s-%d", name, index) } diff --git a/pkg/automationconfig/automation_config_test.go b/pkg/automationconfig/automation_config_test.go index 85bc013fc..a9681a3c8 100644 --- a/pkg/automationconfig/automation_config_test.go +++ b/pkg/automationconfig/automation_config_test.go @@ -42,7 +42,7 @@ func TestBuildAutomationConfig(t *testing.T) { assert.Equal(t, fmt.Sprintf("my-rs-%d.my-ns.svc.cluster.local", i), p.HostName) assert.Equal(t, DefaultMongoDBDataDir, p.Args26.Get("storage.dbPath").Data()) assert.Equal(t, "my-rs", p.Args26.Get("replication.replSetName").Data()) - assert.Equal(t, toProcessName("my-rs", i), p.Name) + assert.Equal(t, toProcessName("my-rs", i, false), p.Name) assert.Equal(t, "4.2.0", p.Version) assert.Equal(t, "4.0", p.FeatureCompatibilityVersion) } @@ -64,10 +64,11 @@ func TestBuildAutomationConfig(t *testing.T) { func TestBuildAutomationConfigArbiters(t *testing.T) { // Test no arbiter (field specified) - noArbiters := 0 + numArbiters := 0 + numMembers := 4 ac, err := NewBuilder(). - SetMembers(4). - SetArbiters(noArbiters). + SetMembers(numMembers). + SetArbiters(numArbiters). Build() assert.NoError(t, err) @@ -79,7 +80,7 @@ func TestBuildAutomationConfigArbiters(t *testing.T) { // Test no arbiter (field NOT specified) ac, err = NewBuilder(). - SetMembers(4). + SetMembers(numMembers). Build() assert.NoError(t, err) @@ -90,79 +91,72 @@ func TestBuildAutomationConfigArbiters(t *testing.T) { } // Test only one arbiter - noArbiters = 1 + numArbiters = 1 + numMembers = 4 ac, err = NewBuilder(). - SetMembers(4). - SetArbiters(noArbiters). + SetMembers(numMembers). + SetArbiters(numArbiters). Build() assert.NoError(t, err) rs = ac.ReplicaSets[0] - for i, member := range rs.Members { - if i < noArbiters { - assert.True(t, member.ArbiterOnly, "The first member should be an arbiter") - } else { - assert.False(t, member.ArbiterOnly, "These members should not be arbiters") - } - } + assert.Len(t, rs.Members, numMembers+numArbiters) + assert.False(t, rs.Members[0].ArbiterOnly) + assert.False(t, rs.Members[1].ArbiterOnly) + assert.False(t, rs.Members[2].ArbiterOnly) + assert.False(t, rs.Members[3].ArbiterOnly) + assert.True(t, rs.Members[4].ArbiterOnly) // Test with multiple arbiters - noArbiters = 2 - ac, err = NewBuilder(). - SetMembers(4). - SetArbiters(noArbiters). - Build() - - assert.NoError(t, err) - - rs = ac.ReplicaSets[0] - for i, member := range rs.Members { - if i < noArbiters { - assert.True(t, member.ArbiterOnly, "The first two members should be arbiters") - } else { - assert.False(t, member.ArbiterOnly, "These members should not be arbiters") - } - } - - //Test With only Arbiters - // The error will be generated when the reconcile loop is called (tested in the e2e tests). - noArbiters = 4 + numArbiters = 2 + numMembers = 4 ac, err = NewBuilder(). - SetMembers(noArbiters). - SetArbiters(noArbiters). + SetMembers(numMembers). + SetArbiters(numArbiters). Build() assert.NoError(t, err) rs = ac.ReplicaSets[0] for i, member := range rs.Members { - if i < noArbiters { - assert.True(t, member.ArbiterOnly, "The first two members should be arbiters") + if i < numMembers { + assert.False(t, member.ArbiterOnly, "First members should not be arbiters") } else { - assert.False(t, member.ArbiterOnly, "These members should not be arbiters") + assert.True(t, member.ArbiterOnly, "Last members should be arbiters") + assert.Equal(t, member.Id, 100+i-numMembers) } } - //Test With more Arbiters than members - // The error will be generated when the reconcile loop is called (tested in the e2e tests). - noMembers := 3 - noArbiters = noMembers + 1 + // Test arbiters should be able to vote + numArbiters = 2 + numMembers = 10 ac, err = NewBuilder(). - SetMembers(noMembers). - SetArbiters(noArbiters). + SetMembers(numMembers). + SetArbiters(numArbiters). Build() assert.NoError(t, err) - rs = ac.ReplicaSets[0] - for i, member := range rs.Members { - if i < noArbiters { - assert.True(t, member.ArbiterOnly, "The first two members should be arbiters") - } else { - assert.False(t, member.ArbiterOnly, "These members should not be arbiters") - } - } + m := ac.ReplicaSets[0].Members + + // First 5 data-bearing nodes have votes + assert.Equal(t, 1, m[0].Votes) + assert.Equal(t, 1, m[1].Votes) + assert.Equal(t, 1, m[2].Votes) + assert.Equal(t, 1, m[3].Votes) + assert.Equal(t, 1, m[4].Votes) + + // From 6th data-bearing nodes, they won'thave any votes + assert.Equal(t, 0, m[5].Votes) + assert.Equal(t, 0, m[6].Votes) + assert.Equal(t, 0, m[7].Votes) + assert.Equal(t, 0, m[8].Votes) + assert.Equal(t, 0, m[9].Votes) + + // Arbiters always have votes + assert.Equal(t, 1, m[10].Votes) + assert.Equal(t, 1, m[11].Votes) } func TestReplicaSetHorizons(t *testing.T) { diff --git a/pkg/util/scale/scale.go b/pkg/util/scale/scale.go index 66aab6a21..cc534fb27 100644 --- a/pkg/util/scale/scale.go +++ b/pkg/util/scale/scale.go @@ -5,6 +5,7 @@ package scale type ReplicaSetScaler interface { DesiredReplicas() int CurrentReplicas() int + ForcedIndividualScaling() bool } // ReplicasThisReconciliation returns the number of replicas that should be configured @@ -12,10 +13,18 @@ type ReplicaSetScaler interface { func ReplicasThisReconciliation(replicaSetScaler ReplicaSetScaler) int { // the current replica set members will be 0 when we are creating a new deployment // if this is the case, we want to jump straight to the desired members and not make changes incrementally - if replicaSetScaler.CurrentReplicas() == 0 || replicaSetScaler.CurrentReplicas() == replicaSetScaler.DesiredReplicas() { + + if replicaSetScaler.CurrentReplicas() == replicaSetScaler.DesiredReplicas() { return replicaSetScaler.DesiredReplicas() } + if !replicaSetScaler.ForcedIndividualScaling() { + // Short-circuit to scale up all at once + if replicaSetScaler.CurrentReplicas() == 0 { + return replicaSetScaler.DesiredReplicas() + } + } + if IsScalingDown(replicaSetScaler) { return replicaSetScaler.CurrentReplicas() - 1 } diff --git a/pkg/util/versions/versions.go b/pkg/util/versions/versions.go index 413065dfd..40be0ae0c 100644 --- a/pkg/util/versions/versions.go +++ b/pkg/util/versions/versions.go @@ -13,8 +13,7 @@ func CalculateFeatureCompatibilityVersion(versionStr string) string { return "" } - baseVersion, _ := semver.Make("3.4.0") - if v1.GTE(baseVersion) { + if v1.GTE(semver.MustParse("3.4.0")) { return fmt.Sprintf("%d.%d", v1.Major, v1.Minor) } diff --git a/test/e2e/mongodbtests/mongodbtests.go b/test/e2e/mongodbtests/mongodbtests.go index 885c85bb6..1e0bf4660 100644 --- a/test/e2e/mongodbtests/mongodbtests.go +++ b/test/e2e/mongodbtests/mongodbtests.go @@ -42,6 +42,17 @@ func StatefulSetBecomesReady(mdb *mdbv1.MongoDBCommunity, opts ...wait.Configura return statefulSetIsReady(mdb, defaultOpts...) } +// ArbitersStatefulSetBecomesReady ensures that the underlying stateful set +// reaches the running state. +func ArbitersStatefulSetBecomesReady(mdb *mdbv1.MongoDBCommunity, opts ...wait.Configuration) func(t *testing.T) { + defaultOpts := []wait.Configuration{ + wait.RetryInterval(time.Second * 15), + wait.Timeout(time.Minute * 20), + } + defaultOpts = append(defaultOpts, opts...) + return arbitersStatefulSetIsReady(mdb, defaultOpts...) +} + // StatefulSetBecomesUnready ensures the underlying stateful set reaches // the unready state. func StatefulSetBecomesUnready(mdb *mdbv1.MongoDBCommunity, opts ...wait.Configuration) func(t *testing.T) { @@ -66,8 +77,8 @@ func StatefulSetIsReadyAfterScaleDown(mdb *mdbv1.MongoDBCommunity) func(t *testi } } -// StatefulSetIsReady ensures that the underlying stateful set -// reaches the running state +// statefulSetIsReady ensures that the underlying stateful set +// reaches the running state. func statefulSetIsReady(mdb *mdbv1.MongoDBCommunity, opts ...wait.Configuration) func(t *testing.T) { return func(t *testing.T) { err := wait.ForStatefulSetToBeReady(t, mdb, opts...) @@ -78,6 +89,18 @@ func statefulSetIsReady(mdb *mdbv1.MongoDBCommunity, opts ...wait.Configuration) } } +// arbitersStatefulSetIsReady ensures that the underlying stateful set +// reaches the running state. +func arbitersStatefulSetIsReady(mdb *mdbv1.MongoDBCommunity, opts ...wait.Configuration) func(t *testing.T) { + return func(t *testing.T) { + err := wait.ForArbitersStatefulSetToBeReady(t, mdb, opts...) + if err != nil { + t.Fatal(err) + } + t.Logf("StatefulSet %s/%s is ready!", mdb.Namespace, mdb.Name) + } +} + // statefulSetIsNotReady ensures that the underlying stateful set reaches the unready state. func statefulSetIsNotReady(mdb *mdbv1.MongoDBCommunity, opts ...wait.Configuration) func(t *testing.T) { return func(t *testing.T) { @@ -379,7 +402,7 @@ func Status(mdb *mdbv1.MongoDBCommunity, expectedStatus mdbv1.MongoDBCommunitySt } } -// Scale update the MongoDB with a new number of members and updates the resource +// Scale update the MongoDB with a new number of members and updates the resource. func Scale(mdb *mdbv1.MongoDBCommunity, newMembers int) func(*testing.T) { return func(t *testing.T) { t.Logf("Scaling Mongodb %s, to %d members", mdb.Name, newMembers) @@ -392,6 +415,19 @@ func Scale(mdb *mdbv1.MongoDBCommunity, newMembers int) func(*testing.T) { } } +// Scale update the MongoDB with a new number of arbiters and updates the resource. +func ScaleArbiters(mdb *mdbv1.MongoDBCommunity, newArbiters int) func(*testing.T) { + return func(t *testing.T) { + t.Logf("Scaling Mongodb %s, to %d members", mdb.Name, newArbiters) + err := e2eutil.UpdateMongoDBResource(mdb, func(db *mdbv1.MongoDBCommunity) { + db.Spec.Arbiters = newArbiters + }) + if err != nil { + t.Fatal(err) + } + } +} + // DisableTLS changes the tls.enabled attribute to false. func DisableTLS(mdb *mdbv1.MongoDBCommunity) func(*testing.T) { return tls(mdb, false) diff --git a/test/e2e/replica_set_arbiter/replica_set_arbiter_test.go b/test/e2e/replica_set_arbiter/replica_set_arbiter_test.go index 04a357a16..3e769e964 100644 --- a/test/e2e/replica_set_arbiter/replica_set_arbiter_test.go +++ b/test/e2e/replica_set_arbiter/replica_set_arbiter_test.go @@ -52,8 +52,7 @@ func TestReplicaSetArbiter(t *testing.T) { t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) t.Run("Check status for case 2", mongodbtests.StatefulSetMessageIsReceived(&mdb, ctx, desiredStatus)) - // Valid case 1 - numberArbiters = 1 + numberArbiters = 0 numberMembers = 3 mdb, user = e2eutil.NewTestMongoDB(ctx, "mdb2", "") mdb.Spec.Arbiters = numberArbiters @@ -66,4 +65,13 @@ func TestReplicaSetArbiter(t *testing.T) { t.Run("Check that the stateful set becomes ready", mongodbtests.StatefulSetBecomesReady(&mdb, wait.Timeout(20*time.Minute))) t.Run("Check the number of arbiters", mongodbtests.AutomationConfigReplicaSetsHaveExpectedArbiters(&mdb, numberArbiters)) + // Arbiters need to be less than regular members + t.Run("Scale MongoDB Up to 2 Arbiters", mongodbtests.ScaleArbiters(&mdb, 2)) + t.Run("Arbiters Stateful Set Scaled Up Correctly", mongodbtests.ArbitersStatefulSetBecomesReady(&mdb)) + t.Run("MongoDB Reaches Running Phase", mongodbtests.MongoDBReachesRunningPhase(&mdb)) + + t.Run("Scale MongoDB Up to 0 Arbiters", mongodbtests.ScaleArbiters(&mdb, 0)) + t.Run("Arbiters Stateful Set Scaled Up Correctly", mongodbtests.ArbitersStatefulSetBecomesReady(&mdb)) + t.Run("MongoDB Reaches Running Phase", mongodbtests.MongoDBReachesRunningPhase(&mdb)) + t.Run("Check the number of arbiters", mongodbtests.AutomationConfigReplicaSetsHaveExpectedArbiters(&mdb, 0)) } diff --git a/test/e2e/util/wait/wait.go b/test/e2e/util/wait/wait.go index 35f24f2e5..042dc46c0 100644 --- a/test/e2e/util/wait/wait.go +++ b/test/e2e/util/wait/wait.go @@ -16,6 +16,13 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) +type StatefulSetType int + +const ( + MembersStatefulSet StatefulSetType = iota + ArbitersStatefulSet +) + // ForConfigMapToExist waits until a ConfigMap of the given name exists // using the provided retryInterval and timeout func ForConfigMapToExist(cmName string, retryInterval, timeout time.Duration) (corev1.ConfigMap, error) { @@ -107,6 +114,15 @@ func ForStatefulSetToBeUnready(t *testing.T, mdb *mdbv1.MongoDBCommunity, opts . }) } +// ForArbitersStatefulSetToBeReady waits until all replicas of the StatefulSet with the given name +// have reached the ready status. +func ForArbitersStatefulSetToBeReady(t *testing.T, mdb *mdbv1.MongoDBCommunity, opts ...Configuration) error { + options := newOptions(opts...) + return waitForStatefulSetConditionWithSpecificSts(t, mdb, MembersStatefulSet, options, func(sts appsv1.StatefulSet) bool { + return statefulset.IsReady(sts, mdb.Spec.Members) + }) +} + // ForStatefulSetToBeReadyAfterScaleDown waits for just the ready replicas to be correct // and does not account for the updated replicas func ForStatefulSetToBeReadyAfterScaleDown(t *testing.T, mdb *mdbv1.MongoDBCommunity, opts ...Configuration) error { @@ -116,25 +132,39 @@ func ForStatefulSetToBeReadyAfterScaleDown(t *testing.T, mdb *mdbv1.MongoDBCommu }) } -func waitForStatefulSetCondition(t *testing.T, mdb *mdbv1.MongoDBCommunity, waitOpts Options, condition func(set appsv1.StatefulSet) bool) error { +func waitForStatefulSetConditionWithSpecificSts(t *testing.T, mdb *mdbv1.MongoDBCommunity, statefulSetType StatefulSetType, waitOpts Options, condition func(set appsv1.StatefulSet) bool) error { _, err := ForStatefulSetToExist(mdb.Name, waitOpts.RetryInterval, waitOpts.Timeout, mdb.Namespace) if err != nil { return errors.Errorf("error waiting for stateful set to be created: %s", err) } sts := appsv1.StatefulSet{} + name := mdb.NamespacedName() + if statefulSetType == ArbitersStatefulSet { + name = types.NamespacedName{Name: mdb.Name + "-arb", Namespace: mdb.Namespace} + } return wait.Poll(waitOpts.RetryInterval, waitOpts.Timeout, func() (done bool, err error) { - err = e2eutil.TestClient.Get(context.TODO(), mdb.NamespacedName(), &sts) + err = e2eutil.TestClient.Get(context.TODO(), name, &sts) if err != nil { return false, err } t.Logf("Waiting for %s to have %d replicas. Current ready replicas: %d, Current updated replicas: %d, Current generation: %d, Observed Generation: %d\n", - mdb.Name, mdb.Spec.Members, sts.Status.ReadyReplicas, sts.Status.UpdatedReplicas, sts.Generation, sts.Status.ObservedGeneration) + name, mdb.Spec.Members, sts.Status.ReadyReplicas, sts.Status.UpdatedReplicas, sts.Generation, sts.Status.ObservedGeneration) ready := condition(sts) return ready, nil }) } +func waitForStatefulSetCondition(t *testing.T, mdb *mdbv1.MongoDBCommunity, waitOpts Options, condition func(set appsv1.StatefulSet) bool) error { + // uses members statefulset + return waitForStatefulSetConditionWithSpecificSts(t, mdb, MembersStatefulSet, waitOpts, condition) +} + +func waitForStatefulSetConditionArbiters(t *testing.T, mdb *mdbv1.MongoDBCommunity, waitOpts Options, condition func(set appsv1.StatefulSet) bool) error { + // uses members statefulset + return waitForStatefulSetConditionWithSpecificSts(t, mdb, ArbitersStatefulSet, waitOpts, condition) +} + func ForPodReadiness(t *testing.T, isReady bool, containerName string, timeout time.Duration, pod corev1.Pod) error { return wait.Poll(time.Second*3, timeout, func() (done bool, err error) { err = e2eutil.TestClient.Get(context.TODO(), types.NamespacedName{Name: pod.Name, Namespace: pod.Namespace}, &pod) From 3ef0777ba6781ec37e2a66cf961017011dd9f1fc Mon Sep 17 00:00:00 2001 From: Rodrigo Valin Date: Thu, 2 Dec 2021 13:16:42 +0000 Subject: [PATCH 437/790] Update dependencies. (#801) --- go.mod | 7 ++++--- go.sum | 18 ++++++++++-------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index 1dabdcf20..c81001097 100644 --- a/go.mod +++ b/go.mod @@ -14,10 +14,11 @@ require ( github.com/xdg/stringprep v1.0.3 go.mongodb.org/mongo-driver v1.7.4 go.uber.org/zap v1.19.1 + golang.org/x/net v0.0.0-20210614182718-04defd469f4e // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 - k8s.io/api v0.22.3 - k8s.io/apimachinery v0.22.3 - k8s.io/client-go v0.22.3 + k8s.io/api v0.22.4 + k8s.io/apimachinery v0.22.4 + k8s.io/client-go v0.22.4 sigs.k8s.io/controller-runtime v0.10.3 sigs.k8s.io/yaml v1.3.0 ) diff --git a/go.sum b/go.sum index 2aebb7d9f..486e3dc1c 100644 --- a/go.sum +++ b/go.sum @@ -571,8 +571,9 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= -golang.org/x/net v0.0.0-20210520170846-37e1c6afe023 h1:ADo5wSpq2gqaCGQWzk7S5vd//0iyyLeAratkEoG5dLE= golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q= +golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -818,17 +819,17 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/api v0.22.2/go.mod h1:y3ydYpLJAaDI+BbSe2xmGcqxiWHmWjkEeIbiwHvnPR8= -k8s.io/api v0.22.3 h1:wOoES2GoSkUsdped2RB4zYypPqWtvprGoKCENTOOjP4= -k8s.io/api v0.22.3/go.mod h1:azgiXFiXqiWyLCfI62/eYBOu19rj2LKmIhFPP4+33fs= +k8s.io/api v0.22.4 h1:UvyHW0ezB2oIgHAxlYoo6UJQObYXU7awuNarwoHEOjw= +k8s.io/api v0.22.4/go.mod h1:Rgs+9gIGYC5laXQSZZ9JqT5NevNgoGiOdVWi1BAB3qk= k8s.io/apiextensions-apiserver v0.22.2 h1:zK7qI8Ery7j2CaN23UCFaC1hj7dMiI87n01+nKuewd4= k8s.io/apiextensions-apiserver v0.22.2/go.mod h1:2E0Ve/isxNl7tWLSUDgi6+cmwHi5fQRdwGVCxbC+KFA= k8s.io/apimachinery v0.22.2/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0= -k8s.io/apimachinery v0.22.3 h1:mrvBG5CZnEfwgpVqWcrRKvdsYECTrhAR6cApAgdsflk= -k8s.io/apimachinery v0.22.3/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0= +k8s.io/apimachinery v0.22.4 h1:9uwcvPpukBw/Ri0EUmWz+49cnFtaoiyEhQTK+xOe7Ck= +k8s.io/apimachinery v0.22.4/go.mod h1:yU6oA6Gnax9RrxGzVvPFFJ+mpnW6PBSqp0sx0I0HHW0= k8s.io/apiserver v0.22.2/go.mod h1:vrpMmbyjWrgdyOvZTSpsusQq5iigKNWv9o9KlDAbBHI= k8s.io/client-go v0.22.2/go.mod h1:sAlhrkVDf50ZHx6z4K0S40wISNTarf1r800F+RlCF6U= -k8s.io/client-go v0.22.3 h1:6onkOSc+YNdwq5zXE0wFXicq64rrym+mXwHu/CPVGO4= -k8s.io/client-go v0.22.3/go.mod h1:ElDjYf8gvZsKDYexmsmnMQ0DYO8W9RwBjfQ1PI53yow= +k8s.io/client-go v0.22.4 h1:aAQ1Wk+I3bjCNk35YWUqbaueqrIonkfDPJSPDDe8Kfg= +k8s.io/client-go v0.22.4/go.mod h1:Yzw4e5e7h1LNHA4uqnMVrpEpUs1hJOiuBsJKIlRCHDA= k8s.io/code-generator v0.22.2/go.mod h1:eV77Y09IopzeXOJzndrDyCI88UBok2h6WxAlBwpxa+o= k8s.io/component-base v0.22.2 h1:vNIvE0AIrLhjX8drH0BgCNJcR4QZxMXcJzBsDplDx9M= k8s.io/component-base v0.22.2/go.mod h1:5Br2QhI9OTe79p+TzPe9JKNQYvEKbq9rTJDWllunGug= @@ -838,8 +839,9 @@ k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.9.0 h1:D7HV+n1V57XeZ0m6tdRkfknthUaM06VFbWldOFh8kzM= k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= -k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e h1:KLHHjkdQFomZy8+06csTWZ0m1343QqxZhR2LJ1OxCYM= k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= +k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c h1:jvamsI1tn9V0S8jicyX82qaFC0H/NKxv2e5mbqsgR80= +k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a h1:8dYfu/Fc9Gz2rNJKB9IQRGgQOh2clmRzNIPPY1xLY5g= k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= From a65882f792320b236c0c950a2f3ee5d1903ac4b6 Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Tue, 14 Dec 2021 13:56:33 +0000 Subject: [PATCH 438/790] CLOUDP-107433: correctly default to admin db (#812) --- api/v1/mongodbcommunity_types.go | 11 +++++++++-- test/e2e/e2eutil.go | 1 - 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/api/v1/mongodbcommunity_types.go b/api/v1/mongodbcommunity_types.go index cfdb7ebd9..a40d9494c 100644 --- a/api/v1/mongodbcommunity_types.go +++ b/api/v1/mongodbcommunity_types.go @@ -27,7 +27,8 @@ import ( type Type string const ( - ReplicaSet Type = "ReplicaSet" + ReplicaSet Type = "ReplicaSet" + defaultDBForUser string = "admin" ) type Phase string @@ -481,14 +482,20 @@ func (m MongoDBCommunity) GetScramUsers() []scram.User { for i, u := range m.Spec.Users { roles := make([]scram.Role, len(u.Roles)) for j, r := range u.Roles { + roles[j] = scram.Role{ Name: r.Name, Database: r.DB, } } + + db := u.DB + if db == "" { + db = defaultDBForUser + } users[i] = scram.User{ Username: u.Name, - Database: u.DB, + Database: db, Roles: roles, PasswordSecretKey: u.GetPasswordSecretKey(), PasswordSecretName: u.PasswordSecretRef.Name, diff --git a/test/e2e/e2eutil.go b/test/e2e/e2eutil.go index 4a139c525..da8aca1de 100644 --- a/test/e2e/e2eutil.go +++ b/test/e2e/e2eutil.go @@ -72,7 +72,6 @@ func NewTestMongoDB(ctx *Context, name string, namespace string) (mdbv1.MongoDBC Users: []mdbv1.MongoDBUser{ { Name: fmt.Sprintf("%s-user", name), - DB: "admin", PasswordSecretRef: mdbv1.SecretKeyReference{ Key: fmt.Sprintf("%s-password", name), Name: fmt.Sprintf("%s-%s-password-secret", name, ctx.ExecutionId), From 73a6e370ddbb922b9da703800babc2f3e6521197 Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Tue, 14 Dec 2021 14:03:46 +0000 Subject: [PATCH 439/790] Remove ubuntu github actions (#814) --- .action_templates/jobs/setup.yaml | 1 - .action_templates/jobs/tests.yaml | 44 ----------------------------- .github/workflows/e2e-dispatch.yml | 1 - .github/workflows/e2e-fork.yml | 45 ------------------------------ .github/workflows/e2e.yml | 45 ------------------------------ helm-charts | 2 +- 6 files changed, 1 insertion(+), 137 deletions(-) diff --git a/.action_templates/jobs/setup.yaml b/.action_templates/jobs/setup.yaml index cd4dbf073..3e7a70fd3 100644 --- a/.action_templates/jobs/setup.yaml +++ b/.action_templates/jobs/setup.yaml @@ -8,5 +8,4 @@ setup: - pipeline-argument: version-post-start-hook-init - pipeline-argument: readiness-probe-init - pipeline-argument: agent-ubi - - pipeline-argument: agent-ubuntu - pipeline-argument: e2e diff --git a/.action_templates/jobs/tests.yaml b/.action_templates/jobs/tests.yaml index 8cd5f1137..c4e81e0a3 100644 --- a/.action_templates/jobs/tests.yaml +++ b/.action_templates/jobs/tests.yaml @@ -5,50 +5,6 @@ tests: fail-fast: false matrix: include: - - test-name: replica_set - distro: ubuntu - - test-name: replica_set_recovery - distro: ubuntu - - test-name: replica_set_recovery - distro: ubuntu - - test-name: replica_set_mongod_readiness - distro: ubuntu - - test-name: replica_set_scale - distro: ubuntu - - test-name: replica_set_scale_down - distro: ubuntu - - test-name: replica_set_change_version - distro: ubuntu - - test-name: feature_compatibility_version - distro: ubuntu - - test-name: feature_compatibility_version_upgrade - distro: ubuntu - - test-name: replica_set_tls - distro: ubuntu - - test-name: replica_set_tls_pem_file - distro: ubuntu - - test-name: replica_set_tls_recreate_mdbc - distro: ubuntu - - test-name: replica_set_tls_rotate - distro: ubuntu - - test-name: replica_set_tls_upgrade - distro: ubuntu - - test-name: statefulset_arbitrary_config - distro: ubuntu - - test-name: statefulset_arbitrary_config_update - distro: ubuntu - - test-name: replica_set_mongod_config - distro: ubuntu - - test-name: replica_set_cross_namespace_deploy - distro: ubuntu - cluster-wide: true - - test-name: replica_set_custom_role - distro: ubuntu - - test-name: replica_set_arbiter - distro: ubuntu - - test-name: replica_set_custom_persistent_volume - distro: ubuntu - - test-name: replica_set distro: ubi - test-name: replica_set_recovery diff --git a/.github/workflows/e2e-dispatch.yml b/.github/workflows/e2e-dispatch.yml index a6d4ce45a..fef98a5b8 100644 --- a/.github/workflows/e2e-dispatch.yml +++ b/.github/workflows/e2e-dispatch.yml @@ -44,7 +44,6 @@ jobs: - pipeline-argument: version-post-start-hook-init - pipeline-argument: readiness-probe-init - pipeline-argument: agent-ubi - - pipeline-argument: agent-ubuntu - pipeline-argument: e2e steps: # template: .action_templates/steps/checkout.yaml diff --git a/.github/workflows/e2e-fork.yml b/.github/workflows/e2e-fork.yml index fd37e1227..b88a335ff 100644 --- a/.github/workflows/e2e-fork.yml +++ b/.github/workflows/e2e-fork.yml @@ -37,7 +37,6 @@ jobs: - pipeline-argument: version-post-start-hook-init - pipeline-argument: readiness-probe-init - pipeline-argument: agent-ubi - - pipeline-argument: agent-ubuntu - pipeline-argument: e2e if: contains(github.event.pull_request.labels.*.name, 'dependencies') || contains(github.event.pull_request.labels.*.name, 'safe-to-test') @@ -88,50 +87,6 @@ jobs: fail-fast: false matrix: include: - - test-name: replica_set - distro: ubuntu - - test-name: replica_set_recovery - distro: ubuntu - - test-name: replica_set_recovery - distro: ubuntu - - test-name: replica_set_mongod_readiness - distro: ubuntu - - test-name: replica_set_scale - distro: ubuntu - - test-name: replica_set_scale_down - distro: ubuntu - - test-name: replica_set_change_version - distro: ubuntu - - test-name: feature_compatibility_version - distro: ubuntu - - test-name: feature_compatibility_version_upgrade - distro: ubuntu - - test-name: replica_set_tls - distro: ubuntu - - test-name: replica_set_tls_pem_file - distro: ubuntu - - test-name: replica_set_tls_recreate_mdbc - distro: ubuntu - - test-name: replica_set_tls_rotate - distro: ubuntu - - test-name: replica_set_tls_upgrade - distro: ubuntu - - test-name: statefulset_arbitrary_config - distro: ubuntu - - test-name: statefulset_arbitrary_config_update - distro: ubuntu - - test-name: replica_set_mongod_config - distro: ubuntu - - test-name: replica_set_cross_namespace_deploy - distro: ubuntu - cluster-wide: true - - test-name: replica_set_custom_role - distro: ubuntu - - test-name: replica_set_arbiter - distro: ubuntu - - test-name: replica_set_custom_persistent_volume - distro: ubuntu - - test-name: replica_set distro: ubi - test-name: replica_set_recovery diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 07e74fe0a..fc82c4acf 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -44,7 +44,6 @@ jobs: - pipeline-argument: version-post-start-hook-init - pipeline-argument: readiness-probe-init - pipeline-argument: agent-ubi - - pipeline-argument: agent-ubuntu - pipeline-argument: e2e if: github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/master' || (github.event.pull_request.head.repo.full_name == github.repository && github.actor @@ -94,50 +93,6 @@ jobs: fail-fast: false matrix: include: - - test-name: replica_set - distro: ubuntu - - test-name: replica_set_recovery - distro: ubuntu - - test-name: replica_set_recovery - distro: ubuntu - - test-name: replica_set_mongod_readiness - distro: ubuntu - - test-name: replica_set_scale - distro: ubuntu - - test-name: replica_set_scale_down - distro: ubuntu - - test-name: replica_set_change_version - distro: ubuntu - - test-name: feature_compatibility_version - distro: ubuntu - - test-name: feature_compatibility_version_upgrade - distro: ubuntu - - test-name: replica_set_tls - distro: ubuntu - - test-name: replica_set_tls_pem_file - distro: ubuntu - - test-name: replica_set_tls_recreate_mdbc - distro: ubuntu - - test-name: replica_set_tls_rotate - distro: ubuntu - - test-name: replica_set_tls_upgrade - distro: ubuntu - - test-name: statefulset_arbitrary_config - distro: ubuntu - - test-name: statefulset_arbitrary_config_update - distro: ubuntu - - test-name: replica_set_mongod_config - distro: ubuntu - - test-name: replica_set_cross_namespace_deploy - distro: ubuntu - cluster-wide: true - - test-name: replica_set_custom_role - distro: ubuntu - - test-name: replica_set_arbiter - distro: ubuntu - - test-name: replica_set_custom_persistent_volume - distro: ubuntu - - test-name: replica_set distro: ubi - test-name: replica_set_recovery diff --git a/helm-charts b/helm-charts index 625081d8e..40c089fda 160000 --- a/helm-charts +++ b/helm-charts @@ -1 +1 @@ -Subproject commit 625081d8ea02e434e55e877a565815abb2254286 +Subproject commit 40c089fdaf0cbe3367686e73a3e57667f065ebae From c5993cef8337ea7bf9b1d23e2b9a2a6ce1f5b391 Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Tue, 14 Dec 2021 17:09:22 +0000 Subject: [PATCH 440/790] update helm repo (#817) --- helm-charts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm-charts b/helm-charts index 40c089fda..9e67e7119 160000 --- a/helm-charts +++ b/helm-charts @@ -1 +1 @@ -Subproject commit 40c089fdaf0cbe3367686e73a3e57667f065ebae +Subproject commit 9e67e7119750353f633951a5fbdd34200d00058d From 5a91339811c2b9d9001ea1d6f2b11804b156089a Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Tue, 14 Dec 2021 17:15:13 +0000 Subject: [PATCH 441/790] CLOUDP-78291: host example (#815) --- .../mongodb.com_v1_hostpath.yaml | 245 ++++++++++++++++++ 1 file changed, 245 insertions(+) create mode 100644 config/samples/arbitrary_statefulset_configuration/mongodb.com_v1_hostpath.yaml diff --git a/config/samples/arbitrary_statefulset_configuration/mongodb.com_v1_hostpath.yaml b/config/samples/arbitrary_statefulset_configuration/mongodb.com_v1_hostpath.yaml new file mode 100644 index 000000000..17600d919 --- /dev/null +++ b/config/samples/arbitrary_statefulset_configuration/mongodb.com_v1_hostpath.yaml @@ -0,0 +1,245 @@ +# This example deploys a 3 members ReplicaSet with HostPath volumes +apiVersion: mongodbcommunity.mongodb.com/v1 +kind: MongoDBCommunity +metadata: + name: mdb0 +spec: + members: 3 + security: + authentication: + modes: + - SCRAM + statefulSet: + spec: + template: + spec: + # Hostpath volumes are owned by root + # but MongoDB containers run as non root + # so we use an init container to change the owner of + # the directory (init containers run as root) + initContainers: + - command: + - chown + - -R + - "2000" + - /data + image: busybox + volumeMounts: + - mountPath: /data + name: data-volume + securityContext: + runAsNonRoot: false + runAsUser: 0 + runAsGroup: 0 + name: change-dir-permissions + volumeClaimTemplates: + - metadata: + name: data-volume + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 8G + selector: + matchLabels: + # We set this labels when creating the volume + # (see below) + type: data + storageClassName: default + - metadata: + name: logs-volume + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 8G + selector: + matchLabels: + type: logs + storageClassName: default + type: ReplicaSet + users: + - name: my-user + db: admin + passwordSecretRef: # a reference to the secret that will be used to generate the user's password + name: my-user-password + roles: + - name: clusterAdmin + db: admin + - name: userAdminAnyDatabase + db: admin + scramCredentialsSecretName: my-scram + version: 4.4.0 +--- +apiVersion: v1 +kind: Secret +metadata: + name: my-user-password +type: Opaque +stringData: + password: +--- +# Here we create 6 PVs: two for each ReplicaSet member +# (one for data, one for logs) +apiVersion: v1 +items: +- apiVersion: v1 + kind: PersistentVolume + metadata: + labels: + type: data + name: data-volume-0 + spec: + accessModes: + - ReadWriteOnce + capacity: + storage: 8G + hostPath: + path: /opt/data/mongo-data-0 + type: "" + nodeAffinity: + required: + # This is just an example for matchexpression + # This field is required depends on the specific + # of the environment the resource is deployed in + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: + - amd64 + persistentVolumeReclaimPolicy: Retain + storageClassName: default + volumeMode: Filesystem +- apiVersion: v1 + kind: PersistentVolume + metadata: + labels: + type: data + name: data-volume-1 + spec: + accessModes: + - ReadWriteOnce + capacity: + storage: 8G + hostPath: + path: /opt/data/mongo-data-1 + type: "" + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: + - amd64 + persistentVolumeReclaimPolicy: Retain + storageClassName: default + volumeMode: Filesystem + status: + phase: Available +- apiVersion: v1 + kind: PersistentVolume + metadata: + labels: + type: data + name: data-volume-2 + spec: + accessModes: + - ReadWriteOnce + capacity: + storage: 8G + hostPath: + path: /opt/data/mongo-data-2 + type: "" + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: + - amd64 + persistentVolumeReclaimPolicy: Retain + storageClassName: default + volumeMode: Filesystem +- apiVersion: v1 + kind: PersistentVolume + metadata: + labels: + type: logs + name: logs-volume-0 + spec: + accessModes: + - ReadWriteOnce + capacity: + storage: 8G + hostPath: + path: /opt/data/mongo-logs-0 + type: "" + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: + - amd64 + persistentVolumeReclaimPolicy: Retain + storageClassName: default + volumeMode: Filesystem +- apiVersion: v1 + kind: PersistentVolume + metadata: + labels: + type: logs + name: logs-volume-1 + spec: + accessModes: + - ReadWriteOnce + capacity: + storage: 8G + hostPath: + path: /opt/data/mongo-logs-1 + type: "" + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: + - amd64 + persistentVolumeReclaimPolicy: Retain + storageClassName: default + volumeMode: Filesystem +- apiVersion: v1 + kind: PersistentVolume + metadata: + labels: + type: logs + name: logs-volume-2 + + spec: + accessModes: + - ReadWriteOnce + capacity: + storage: 8G + hostPath: + path: /opt/data/mongo-logs-2 + type: "" + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: + - amd64 + persistentVolumeReclaimPolicy: Retain + storageClassName: default + volumeMode: Filesystem +kind: List +--- From 79fce2fe6bb6c178b90f7405552e9af49d76d285 Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Tue, 14 Dec 2021 17:27:03 +0000 Subject: [PATCH 442/790] Cloudp 107436 enable partial merging of custom automation config (#806) --- api/v1/mongodbcommunity_types.go | 18 +++++ api/v1/zz_generated.deepcopy.go | 41 ++++++++++++ ...ommunity.mongodb.com_mongodbcommunity.yaml | 22 +++++++ ..._mongodbcommunity_disabled_process_cr.yaml | 38 +++++++++++ controllers/replica_set_controller.go | 22 +++++++ pkg/automationconfig/automation_config.go | 1 + pkg/util/merge/merge_automationconfigs.go | 39 +++++++++++ .../merge/merge_automationconfigs_test.go | 66 +++++++++++++++++++ 8 files changed, 247 insertions(+) create mode 100644 config/samples/mongodb.com_v1_mongodbcommunity_disabled_process_cr.yaml create mode 100644 pkg/util/merge/merge_automationconfigs.go create mode 100644 pkg/util/merge/merge_automationconfigs_test.go diff --git a/api/v1/mongodbcommunity_types.go b/api/v1/mongodbcommunity_types.go index a40d9494c..d58f1291e 100644 --- a/api/v1/mongodbcommunity_types.go +++ b/api/v1/mongodbcommunity_types.go @@ -102,6 +102,10 @@ type MongoDBCommunitySpec struct { // +kubebuilder:pruning:PreserveUnknownFields // +nullable AdditionalMongodConfig MongodConfiguration `json:"additionalMongodConfig,omitempty"` + + // AutomationConfigOverride is merged on top of the operator created automation config. Processes are merged + // by name. Currently Only the process.disabled field is supported. + AutomationConfigOverride *AutomationConfigOverride `json:"automationConfig,omitempty"` } // ReplicaSetHorizonConfiguration holds the split horizon DNS settings for @@ -198,6 +202,20 @@ type AuthenticationRestriction struct { ServerAddress []string `json:"serverAddress"` } +// AutomationConfigOverride contains fields which will be overridden in the operator created config. +type AutomationConfigOverride struct { + Processes []OverrideProcess `json:"processes"` +} + +// Note: We do not use the automationconfig.Process type directly here as unmarshalling cannot happen directly +// with the Args26 which is a map[string]interface{} + +// OverrideProcess contains fields that we can override on the AutomationConfig processes. +type OverrideProcess struct { + Name string `json:"name"` + Disabled bool `json:"disabled"` +} + // StatefulSetConfiguration holds the optional custom StatefulSet // that should be merged into the operator created one. type StatefulSetConfiguration struct { diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index eade1c619..62da7cc94 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -1,3 +1,4 @@ +//go:build !ignore_autogenerated // +build !ignore_autogenerated /* @@ -75,6 +76,26 @@ func (in *AuthenticationRestriction) DeepCopy() *AuthenticationRestriction { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AutomationConfigOverride) DeepCopyInto(out *AutomationConfigOverride) { + *out = *in + if in.Processes != nil { + in, out := &in.Processes, &out.Processes + *out = make([]OverrideProcess, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AutomationConfigOverride. +func (in *AutomationConfigOverride) DeepCopy() *AutomationConfigOverride { + if in == nil { + return nil + } + out := new(AutomationConfigOverride) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CustomRole) DeepCopyInto(out *CustomRole) { *out = *in @@ -209,6 +230,11 @@ func (in *MongoDBCommunitySpec) DeepCopyInto(out *MongoDBCommunitySpec) { } in.StatefulSetConfiguration.DeepCopyInto(&out.StatefulSetConfiguration) in.AdditionalMongodConfig.DeepCopyInto(&out.AdditionalMongodConfig) + if in.AutomationConfigOverride != nil { + in, out := &in.AutomationConfigOverride, &out.AutomationConfigOverride + *out = new(AutomationConfigOverride) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MongoDBCommunitySpec. @@ -263,6 +289,21 @@ func (in *MongodConfiguration) DeepCopyInto(out *MongodConfiguration) { *out = *clone } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OverrideProcess) DeepCopyInto(out *OverrideProcess) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OverrideProcess. +func (in *OverrideProcess) DeepCopy() *OverrideProcess { + if in == nil { + return nil + } + out := new(OverrideProcess) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Privilege) DeepCopyInto(out *Privilege) { *out = *in diff --git a/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml b/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml index 222e94520..9d9178a33 100644 --- a/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml +++ b/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml @@ -59,6 +59,28 @@ spec: Set. It is not recommended to have more than one arbiter per Replica Set. More info: https://docs.mongodb.com/manual/tutorial/add-replica-set-arbiter/' type: integer + automationConfig: + description: AutomationConfigOverride is merged on top of the operator + created automation config. Processes are merged by name. Currently + Only the process.disabled field is supported. + properties: + processes: + items: + description: OverrideProcess contains fields that we can override + on the AutomationConfig processes. + properties: + disabled: + type: boolean + name: + type: string + required: + - disabled + - name + type: object + type: array + required: + - processes + type: object featureCompatibilityVersion: description: FeatureCompatibilityVersion configures the feature compatibility version that will be set for the deployment diff --git a/config/samples/mongodb.com_v1_mongodbcommunity_disabled_process_cr.yaml b/config/samples/mongodb.com_v1_mongodbcommunity_disabled_process_cr.yaml new file mode 100644 index 000000000..6bb42c608 --- /dev/null +++ b/config/samples/mongodb.com_v1_mongodbcommunity_disabled_process_cr.yaml @@ -0,0 +1,38 @@ +--- +apiVersion: mongodbcommunity.mongodb.com/v1 +kind: MongoDBCommunity +metadata: + name: example-mongodb +spec: + members: 3 + automationConfig: + processes: + - name: example-mongodb-1 + disabled: true + type: ReplicaSet + version: "4.2.6" + security: + authentication: + modes: ["SCRAM"] + users: + - name: my-user + db: admin + passwordSecretRef: # a reference to the secret that will be used to generate the user's password + name: my-user-password + roles: + - name: clusterAdmin + db: admin + - name: userAdminAnyDatabase + db: admin + scramCredentialsSecretName: my-scram + +# the user credentials will be generated from this secret +# once the credentials are generated, this secret is no longer required +--- +apiVersion: v1 +kind: Secret +metadata: + name: my-user-password +type: Opaque +stringData: + password: diff --git a/controllers/replica_set_controller.go b/controllers/replica_set_controller.go index aadeba54f..22680558c 100644 --- a/controllers/replica_set_controller.go +++ b/controllers/replica_set_controller.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/merge" "os" "github.com/imdario/mergo" @@ -587,9 +588,30 @@ func (r ReplicaSetReconciler) buildAutomationConfig(mdb mdbv1.MongoDBCommunity) return automationconfig.AutomationConfig{}, errors.Errorf("could not create an automation config: %s", err) } + if mdb.Spec.AutomationConfigOverride != nil { + automationConfig = merge.AutomationConfigs(automationConfig, overrideToAutomationConfig(*mdb.Spec.AutomationConfigOverride)) + } + return automationConfig, nil } +// overrideToAutomationConfig turns an automation config ovverride from the resource spec into an automation config +// which can be used to merge. +func overrideToAutomationConfig(override mdbv1.AutomationConfigOverride) automationconfig.AutomationConfig { + var processes []automationconfig.Process + for _, p := range override.Processes { + processes = append(processes, automationconfig.Process{ + Name: p.Name, + Disabled: p.Disabled, + }) + } + + // TODO: currently we are just merging processes. Other fields can be added here. + return automationconfig.AutomationConfig{ + Processes: processes, + } +} + // getMongodConfigModification will merge the additional configuration in the CRD // into the configuration set up by the operator. func getMongodConfigModification(mdb mdbv1.MongoDBCommunity) automationconfig.Modification { diff --git a/pkg/automationconfig/automation_config.go b/pkg/automationconfig/automation_config.go index 9739a0966..23dcf8364 100644 --- a/pkg/automationconfig/automation_config.go +++ b/pkg/automationconfig/automation_config.go @@ -45,6 +45,7 @@ type MonitoringVersion struct { type Process struct { Name string `json:"name"` + Disabled bool `json:"disabled"` HostName string `json:"hostname"` Args26 objx.Map `json:"args2_6"` FeatureCompatibilityVersion string `json:"featureCompatibilityVersion"` diff --git a/pkg/util/merge/merge_automationconfigs.go b/pkg/util/merge/merge_automationconfigs.go new file mode 100644 index 000000000..bf8eec550 --- /dev/null +++ b/pkg/util/merge/merge_automationconfigs.go @@ -0,0 +1,39 @@ +package merge + +import ( + "github.com/mongodb/mongodb-kubernetes-operator/pkg/automationconfig" +) + +// AutomationConfigs merges the values in "override" into the "original" Wrapper. +// Merging is done by name for processes. +func AutomationConfigs(original, override automationconfig.AutomationConfig) automationconfig.AutomationConfig { + original.Processes = mergeProcesses(original.Processes, override.Processes) + return original +} + +func mergeProcesses(original, override []automationconfig.Process) []automationconfig.Process { + mergedProcesses := append([]automationconfig.Process{}, original...) + for _, overrideProcess := range override { + correspondingIndex := getProcessIndexByName(overrideProcess.Name, original) + if correspondingIndex == -1 { + continue + } + mergedProcesses[correspondingIndex] = mergeProcess(original[correspondingIndex], overrideProcess) + } + return mergedProcesses +} + +func getProcessIndexByName(desiredProcessName string, originalProcesses []automationconfig.Process) int { + for i := range originalProcesses { + if originalProcesses[i].Name == desiredProcessName { + return i + } + } + return -1 +} + +func mergeProcess(original, override automationconfig.Process) automationconfig.Process { + // TODO: in order to override the disabled field, we just need this one field. We can handle all fields in a future change. + original.Disabled = override.Disabled + return original +} diff --git a/pkg/util/merge/merge_automationconfigs_test.go b/pkg/util/merge/merge_automationconfigs_test.go new file mode 100644 index 000000000..42adaa8f9 --- /dev/null +++ b/pkg/util/merge/merge_automationconfigs_test.go @@ -0,0 +1,66 @@ +package merge + +import ( + "github.com/mongodb/mongodb-kubernetes-operator/pkg/automationconfig" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestMergeAutomationConfigs(t *testing.T) { + original, err := automationconfig.NewBuilder(). + SetName("test-ac"). + SetMembers(3). + Build() + + assert.NoError(t, err) + override, err := automationconfig.NewBuilder(). + SetName("test-ac"). + SetMembers(3). + AddProcessModification(func(i int, process *automationconfig.Process) { + // set a single process to be disabled. + process.Disabled = i == 1 + }).Build() + + assert.NoError(t, err) + + for _, p := range original.Processes { + assert.False(t, p.Disabled) + } + + assert.False(t, override.Processes[0].Disabled) + assert.True(t, override.Processes[1].Disabled) + assert.False(t, override.Processes[2].Disabled) + + mergedAc := AutomationConfigs(original, override) + assert.False(t, mergedAc.Processes[0].Disabled) + assert.True(t, mergedAc.Processes[1].Disabled) + assert.False(t, mergedAc.Processes[2].Disabled) +} + +func TestMergeAutomationConfigs_NonExistentMember(t *testing.T) { + original, err := automationconfig.NewBuilder(). + SetName("test-ac"). + SetMembers(3). + Build() + + assert.NoError(t, err) + override, err := automationconfig.NewBuilder(). + SetName("test-ac-0"). + SetMembers(3). + AddProcessModification(func(i int, process *automationconfig.Process) { + process.Disabled = i == 1 + }).Build() + + assert.NoError(t, err) + + + assert.False(t, override.Processes[0].Disabled) + assert.True(t, override.Processes[1].Disabled) + assert.False(t, override.Processes[2].Disabled) + + mergedAc := AutomationConfigs(original, override) + + assert.False(t, mergedAc.Processes[0].Disabled) + assert.False(t, mergedAc.Processes[1].Disabled, "should not be updated as the name does not match.") + assert.False(t, mergedAc.Processes[2].Disabled) +} From 67b85c39aced6f4cef4466f652ace92d19014dda Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Wed, 15 Dec 2021 09:17:25 +0000 Subject: [PATCH 443/790] CLOUDP-104426: multiple users issue (#813) --- api/v1/mongodbcommunity_types.go | 1 + ...ommunity.mongodb.com_mongodbcommunity.yaml | 3 +- controllers/validation/validation.go | 34 ++++++++++++++++--- 3 files changed, 32 insertions(+), 6 deletions(-) diff --git a/api/v1/mongodbcommunity_types.go b/api/v1/mongodbcommunity_types.go index d58f1291e..3691454c8 100644 --- a/api/v1/mongodbcommunity_types.go +++ b/api/v1/mongodbcommunity_types.go @@ -291,6 +291,7 @@ type MongoDBUser struct { Roles []Role `json:"roles"` // ScramCredentialsSecretName appended by string "scram-credentials" is the name of the secret object created by the mongoDB operator for storing SCRAM credentials + // These secrets names must be different for each user in a deployment. // +kubebuilder:validation:Pattern=^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ ScramCredentialsSecretName string `json:"scramCredentialsSecretName"` } diff --git a/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml b/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml index 9d9178a33..3d7027534 100644 --- a/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml +++ b/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml @@ -325,7 +325,8 @@ spec: scramCredentialsSecretName: description: ScramCredentialsSecretName appended by string "scram-credentials" is the name of the secret object created by the mongoDB operator - for storing SCRAM credentials + for storing SCRAM credentials These secrets names must be + different for each user in a deployment. pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ type: string required: diff --git a/controllers/validation/validation.go b/controllers/validation/validation.go index 5510f5487..37d28541e 100644 --- a/controllers/validation/validation.go +++ b/controllers/validation/validation.go @@ -43,25 +43,49 @@ func validateSpec(mdb mdbv1.MongoDBCommunity) error { func validateUsers(mdb mdbv1.MongoDBCommunity) error { connectionStringSecretNameMap := map[string]scram.User{} nameCollisions := []string{} + + scramSecretNameMap := map[string]scram.User{} + scramSecretNameCollisions := []string{} + for _, user := range mdb.GetScramUsers() { - secretName := user.GetConnectionStringSecretName(mdb) - if previousUser, exists := connectionStringSecretNameMap[secretName]; exists { + + // Ensure no collisions in the connection string secret names + connectionStringSecretName := user.GetConnectionStringSecretName(mdb) + if previousUser, exists := connectionStringSecretNameMap[connectionStringSecretName]; exists { nameCollisions = append(nameCollisions, - fmt.Sprintf(`[secret name: "%s" for user: "%s", db: "%s" and user: "%s", db: "%s"]`, - secretName, + fmt.Sprintf(`[connection string secret name: "%s" for user: "%s", db: "%s" and user: "%s", db: "%s"]`, + connectionStringSecretName, previousUser.Username, previousUser.Database, user.Username, user.Database)) } else { - connectionStringSecretNameMap[secretName] = user + connectionStringSecretNameMap[connectionStringSecretName] = user + } + + // Ensure no collisions in the secret holding scram credentials + scramSecretName := user.ScramCredentialsSecretName + if previousUser, exists := scramSecretNameMap[scramSecretName]; exists { + scramSecretNameCollisions = append(scramSecretNameCollisions, + fmt.Sprintf(`[scram secret name: "%s" for user: "%s" and user: "%s"]`, + scramSecretName, + previousUser.Username, + user.Username)) + } else { + connectionStringSecretNameMap[connectionStringSecretName] = user } + } if len(nameCollisions) > 0 { return errors.Errorf("connection string secret names collision, update at least one of the users so that the resulted secret names (--) are unique: %s", strings.Join(nameCollisions, ", ")) } + if len(scramSecretNameCollisions) > 0 { + return errors.Errorf("scram credential secret names collision, update at least one of the users: %s", + strings.Join(scramSecretNameCollisions, ", ")) + } + return nil } From 30471cd43baef039347f5e18c2b7502a6c0a2ba0 Mon Sep 17 00:00:00 2001 From: Ciprian Tibulca Date: Tue, 21 Dec 2021 12:24:49 +0200 Subject: [PATCH 444/790] CLOUDP-107814 mark new arbiters related fields (PR 785) as optional to fix the reconcile process done by the latest released operator (0.7.2) (#828) --- api/v1/mongodbcommunity_types.go | 4 ++-- .../bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/api/v1/mongodbcommunity_types.go b/api/v1/mongodbcommunity_types.go index 3691454c8..8dd46bf93 100644 --- a/api/v1/mongodbcommunity_types.go +++ b/api/v1/mongodbcommunity_types.go @@ -414,8 +414,8 @@ type MongoDBCommunityStatus struct { CurrentStatefulSetReplicas int `json:"currentStatefulSetReplicas"` CurrentMongoDBMembers int `json:"currentMongoDBMembers"` - CurrentStatefulSetArbitersReplicas int `json:"currentStatefulSetArbitersReplicas"` - CurrentMongoDBArbiters int `json:"currentMongoDBArbiters"` + CurrentStatefulSetArbitersReplicas int `json:"currentStatefulSetArbitersReplicas,omitempty"` + CurrentMongoDBArbiters int `json:"currentMongoDBArbiters,omitempty"` Message string `json:"message,omitempty"` } diff --git a/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml b/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml index 3d7027534..ae35da308 100644 --- a/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml +++ b/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml @@ -364,9 +364,7 @@ spec: version: type: string required: - - currentMongoDBArbiters - currentMongoDBMembers - - currentStatefulSetArbitersReplicas - currentStatefulSetReplicas - mongoUri - phase From e6e0324f7f119fef4504889533d66c0d91af4a55 Mon Sep 17 00:00:00 2001 From: Rodrigo Valin Date: Wed, 5 Jan 2022 15:53:45 +0000 Subject: [PATCH 445/790] CLOUDP-105007: Bumps helm-charts. (#824) --- .github/workflows/main.yaml | 2 +- helm-charts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index f278b0dc8..b922a90d8 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -62,7 +62,7 @@ jobs: # Run: operator-sdk generate k8s - name: Lint Code Base - uses: docker://github/super-linter:v3 + uses: docker://github/super-linter:v4 env: VALIDATE_ALL_CODEBASE: true # Now we set the PYTHONPATH to the path of the dependencies *inside* the container diff --git a/helm-charts b/helm-charts index 9e67e7119..735367f4d 160000 --- a/helm-charts +++ b/helm-charts @@ -1 +1 @@ -Subproject commit 9e67e7119750353f633951a5fbdd34200d00058d +Subproject commit 735367f4db9933489ea36a53cc4e71797b7263b9 From 8342de19dfccac6a53ef201b48c7acc0b5de58bc Mon Sep 17 00:00:00 2001 From: mircea-cosbuc Date: Mon, 10 Jan 2022 17:10:13 +0100 Subject: [PATCH 446/790] Consistency and fixes in docs (#845) * Specify automation config is stored in `Secret` * Fix reference to old CRD name * Add repo links to files linked from external_access --- docs/architecture.md | 4 ++-- docs/build_operator_locally.md | 2 +- docs/external_access.md | 18 +++++++++--------- docs/install-upgrade.md | 2 +- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 71db0d3a8..4b66c1fe0 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -13,7 +13,7 @@ The MongoDB Community Kubernetes Operator is a [Custom Resource Definition](http You create and update MongoDB resources by defining a MongoDB resource definition. When you apply the MongoDB resource definition to your Kubernetes environment, the Operator: 1. Creates a [StatefulSet](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/) that contains one [pod](https://kubernetes.io/docs/concepts/workloads/pods/pod-overview/) for each [replica set](https://docs.mongodb.com/manual/replication/) member. -1. Writes the Automation configuration as a [ConfigMap](https://kubernetes.io/docs/concepts/configuration/configmap/) and mounts it to each pod. +1. Writes the Automation configuration as a [Secret](https://kubernetes.io/docs/concepts/configuration/secret/) and mounts it to each pod. 1. Creates one [init container](https://kubernetes.io/docs/concepts/workloads/pods/init-containers/) and two [containers](https://kubernetes.io/docs/concepts/containers/overview/) in each pod: - An init container which copies the `cmd/versionhook` binary to the main `mongod` container. This is run before `mongod` starts to handle [version upgrades](#example-mongodb-version-upgrade). @@ -25,7 +25,7 @@ You create and update MongoDB resources by defining a MongoDB resource definitio 1. Creates several volumes: - `data-volume` which is [persistent](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) and mounts to `/data` on both the server and agent containers. Stores server data as well as `automation-mongod.conf` written by the agent and some locks the agent needs. - - `automation-config` which is mounted from the previously generated ConfigMap to both the server and agent. Only lives as long as the pod. + - `automation-config` which is mounted from the previously generated `Secret` to both the server and agent. Only lives as long as the pod. - `healthstatus` which contains the agent's current status. This is shared with the `mongod` container where it's used by the pre-stop hook. Only lives as long as the pod. 1. Initiates the MongoDB Agent, which in turn creates the database configuration and launches the `mongod` process according to your [MongoDB resource definition](../deploy/crds/mongodb.com_v1_mongodbcommunity_cr.yaml). diff --git a/docs/build_operator_locally.md b/docs/build_operator_locally.md index 4fe827bec..cd5f6d22b 100644 --- a/docs/build_operator_locally.md +++ b/docs/build_operator_locally.md @@ -42,6 +42,6 @@ make all-images deploy Note: this will build and push the operator at `repo_url/mongodb-kubernetes-operator`, where `repo_url` is extracted from the [dev config file](./contributing.md#developing-locally) -5. Change the [manager yaml file](../config/manager/manager.yaml) `image` field to have the image you just built +5. Change the `image` field in the [manager.yaml](../config/manager/manager.yaml) file to have the image you just built 6. You can now deploy your resources following the [docs](../docs/README.md) diff --git a/docs/external_access.md b/docs/external_access.md index 4d0812b55..f72899398 100644 --- a/docs/external_access.md +++ b/docs/external_access.md @@ -30,14 +30,14 @@ Execute ```mkcert --CAROOT``` to note the location of the generated root CA key Use the files that you found in the previous step. Replace `````` with your chosen namespace ```sh -kubectl create configmap ca-config-map --from-file=ca.crt --namespace +kubectl create configmap ca-config-map --from-file=ca.crt= --namespace -kubectl create secret tls ca-key-pair --cert=ca.crt --key=ca.key --namespace +kubectl create secret tls ca-key-pair --cert= --key= --namespace ``` ### Create the Cert Manager issuer and secret -Edit the file ```cert-manager-certificate.yaml``` to replace `````` with your MongoDB deployment name. Also replace ``````, ``````, and `````` with the external FQDNs of the MongoDB replicaset members. Please remember that you will have to add an equal number of entries for each member of the replicaset. +Edit the file [cert-manager-certificate.yaml](../config/samples/external_access/cert-manager-certificate.yaml) to replace `````` with your MongoDB deployment name. Also replace ``````, ``````, and `````` with the external FQDNs of the MongoDB replicaset members. Please remember that you will have to add an equal number of entries for each member of the replicaset. Apply the manifests. Replace `````` with the namespace you are using for the deployment. @@ -48,30 +48,30 @@ kubectl apply -f config/samples/external_access/cert-manager-certificate.yaml -- ### Create the MongoDB deployment -Edit ```config/samples/external_access/mongodb.com_v1_mongodbcommunity_cr.yaml```. Replace with the desired MongoDB deployment name -- this should be the same as in the previous step. Replace ``````, ``````, and `````` with the external FQDNs of the MongoDB replicaset members. Please remember that you should have the same number of entries in this section as the number of your replicaset members. You can also edit the ports for external access to your preferred numbers in this section -- you will have to remember to change them in the next step too. Change `````` to your desired admin password for MongoDB. +Edit [mongodb.com_v1_mongodbcommunity_cr.yaml](../config/samples/external_access/mongodb.com_v1_mongodbcommunity_cr.yaml). Replace with the desired MongoDB deployment name -- this should be the same as in the previous step. Replace ``````, ``````, and `````` with the external FQDNs of the MongoDB replicaset members. Please remember that you should have the same number of entries in this section as the number of your replicaset members. You can also edit the ports for external access to your preferred numbers in this section -- you will have to remember to change them in the next step too. Change `````` to your desired admin password for MongoDB. Apply the manifest. ```sh -kubectl apply -f config/samples/external_access/mongodb.com_v1_mongodbcommunity_cr.yaml +kubectl apply -f config/samples/external_access/mongodb.com_v1_mongodbcommunity_cr.yaml --namespace ``` Wait for the replicaset to be available. ### Create the external NodePort services for accessing the MongoDB deployment from outside the Kubernetes cluster -Edit ```config/samples/external_access/external_services.yaml``` and replace `````` with the MongoDB deployment name that you have used in the preceeding steps. You can change the ```nodePort``` and ```port``` to reflect the changes (if any) you have made in the previous steps. +Edit [external_services.yaml](../config/samples/external_access/external_services.yaml) and replace `````` with the MongoDB deployment name that you have used in the preceeding steps. You can change the ```nodePort``` and ```port``` to reflect the changes (if any) you have made in the previous steps. Apply the manifest. ```sh -kubectl apply -f config/samples/external_access/external_services.yaml +kubectl apply -f config/samples/external_access/external_services.yaml --namespace ``` ### Retrieve the certificates from a MongoDB replicaset member ```sh -kubectl exec --namespace mcommunity -it -0 -c mongod -- bash +kubectl exec --namespace -it -0 -c mongod -- bash ``` Once inside the container ```cat``` and copy the contents of the ```.pem``` file in ```/var/lib/tls/server``` into a file on your local system. @@ -81,7 +81,7 @@ Once inside the container ```cat``` and copy the contents of the ```.pem``` file This is an example to connect to the MongoDB cluster with Mongo shell. Use the CA from ```mkcert``` and the certificate from the previous step. Replace the values in the command from the preceeding steps. ```sh -mongosh --tls --tlsCAfile ca.crt --tlsCertificateKeyFile key.pem --username my-user --password mongodb://:31181,:31182,:31183 +mongosh --tls --tlsCAFile ca.crt --tlsCertificateKeyFile key.pem --username my-user --password mongodb://:31181,:31182,:31183 ``` ### Conclusion diff --git a/docs/install-upgrade.md b/docs/install-upgrade.md index 15a42d5a5..186364c5d 100644 --- a/docs/install-upgrade.md +++ b/docs/install-upgrade.md @@ -182,7 +182,7 @@ Make sure you run commands in the correct namespace. ``` c. Delete the old customResourceDefinition. Not strictly needed but no need to keep it around anymore (unless you got more installations of operator in your cluster) ``` - kubectl delete crd mongodb.mongodb.com + kubectl delete crd mongodbcommunity.mongodbcommunity.mongodb.com ``` 3. In with the new From 59cfe33a28d244ed26d8985df089ac89bd703bc7 Mon Sep 17 00:00:00 2001 From: mircea-cosbuc Date: Wed, 12 Jan 2022 16:54:17 +0100 Subject: [PATCH 447/790] Make mongod port configurable (#848) * Make mongod port configurable * Adapt e2e mongod config test * Add custom port test with CR testdata * Configure net.port separately in e2e test * Cleanup e2e test * Fix typo in GetDBPort docstring * Use defaults from automationconfig package --- api/v1/mongodbcommunity_types.go | 36 ++++++++++++-- api/v1/mongodbcommunity_types_test.go | 25 ++++++++++ .../construct/build_statefulset_test.go | 21 +-------- controllers/construct/mongodbstatefulset.go | 22 +++------ controllers/replica_set_controller.go | 7 +-- controllers/replicaset_controller_test.go | 47 ++++++++++++++++++- controllers/testdata/specify_net_port.yaml | 26 ++++++++++ pkg/automationconfig/automation_config.go | 1 + .../automation_config_builder.go | 9 +++- .../merge/merge_automationconfigs_test.go | 4 +- test/e2e/mongodbtests/mongodbtests.go | 13 +++++ .../replica_set_mongod_config_test.go | 17 ++++--- 12 files changed, 175 insertions(+), 53 deletions(-) create mode 100644 controllers/testdata/specify_net_port.yaml diff --git a/api/v1/mongodbcommunity_types.go b/api/v1/mongodbcommunity_types.go index 8dd46bf93..3634084eb 100644 --- a/api/v1/mongodbcommunity_types.go +++ b/api/v1/mongodbcommunity_types.go @@ -276,6 +276,29 @@ func (m *MongodConfiguration) DeepCopy() *MongodConfiguration { } } +// NewMongodConfiguration returns an empty MongodConfiguration +func NewMongodConfiguration() MongodConfiguration { + return MongodConfiguration{Object: map[string]interface{}{}} +} + +// SetOption updated the MongodConfiguration with a new option +func (m MongodConfiguration) SetOption(key string, value interface{}) MongodConfiguration { + m.Object = objx.New(m.Object).Set(key, value) + return m +} + +// GetDBDataDir returns the db path which should be used. +func (m MongodConfiguration) GetDBDataDir() string { + return objx.New(m.Object).Get("storage.dbPath").Str(automationconfig.DefaultMongoDBDataDir) +} + +// GetDBPort returns the port that should be used for the mongod process. +// If port is not specified, the default port of 27017 will be used. +func (m MongodConfiguration) GetDBPort() int { + // When passed as a number "net.port" is unmarshalled into a float64 + return int(objx.New(m.Object).Get("net.port").Float64(float64(automationconfig.DefaultDBPort))) +} + type MongoDBUser struct { // Name is the username of the user Name string `json:"name"` @@ -436,10 +459,10 @@ type MongoDBCommunity struct { Status MongoDBCommunityStatus `json:"status,omitempty"` } -func (m MongoDBCommunity) GetMongodConfiguration() map[string]interface{} { - mongodConfig := objx.New(map[string]interface{}{}) +func (m MongoDBCommunity) GetMongodConfiguration() MongodConfiguration { + mongodConfig := NewMongodConfiguration() for k, v := range m.Spec.AdditionalMongodConfig.Object { - mongodConfig.Set(k, v) + mongodConfig.SetOption(k, v) } return mongodConfig } @@ -608,7 +631,12 @@ func (m MongoDBCommunity) Hosts(clusterDomain string) []string { } for i := 0; i < m.Spec.Members; i++ { - hosts[i] = fmt.Sprintf("%s-%d.%s.%s.svc.%s:%d", m.Name, i, m.ServiceName(), m.Namespace, clusterDomain, 27017) + hosts[i] = fmt.Sprintf("%s-%d.%s.%s.svc.%s:%d", + m.Name, i, + m.ServiceName(), + m.Namespace, + clusterDomain, + m.GetMongodConfiguration().GetDBPort()) } return hosts } diff --git a/api/v1/mongodbcommunity_types_test.go b/api/v1/mongodbcommunity_types_test.go index 1880c523b..4af93881c 100644 --- a/api/v1/mongodbcommunity_types_test.go +++ b/api/v1/mongodbcommunity_types_test.go @@ -17,6 +17,31 @@ func TestMongoDB_MongoURI(t *testing.T) { mdb = newReplicaSet(5, "my-big-rs", "my-big-namespace") assert.Equal(t, mdb.MongoURI(""), "mongodb://my-big-rs-0.my-big-rs-svc.my-big-namespace.svc.cluster.local:27017,my-big-rs-1.my-big-rs-svc.my-big-namespace.svc.cluster.local:27017,my-big-rs-2.my-big-rs-svc.my-big-namespace.svc.cluster.local:27017,my-big-rs-3.my-big-rs-svc.my-big-namespace.svc.cluster.local:27017,my-big-rs-4.my-big-rs-svc.my-big-namespace.svc.cluster.local:27017") assert.Equal(t, mdb.MongoURI("my.cluster"), "mongodb://my-big-rs-0.my-big-rs-svc.my-big-namespace.svc.my.cluster:27017,my-big-rs-1.my-big-rs-svc.my-big-namespace.svc.my.cluster:27017,my-big-rs-2.my-big-rs-svc.my-big-namespace.svc.my.cluster:27017,my-big-rs-3.my-big-rs-svc.my-big-namespace.svc.my.cluster:27017,my-big-rs-4.my-big-rs-svc.my-big-namespace.svc.my.cluster:27017") + mdb = newReplicaSet(2, "my-rs", "my-namespace") + mdb.Spec.AdditionalMongodConfig.Object = map[string]interface{}{ + "net.port": 40333., + } + assert.Equal(t, mdb.MongoURI(""), "mongodb://my-rs-0.my-rs-svc.my-namespace.svc.cluster.local:40333,my-rs-1.my-rs-svc.my-namespace.svc.cluster.local:40333") + assert.Equal(t, mdb.MongoURI("my.cluster"), "mongodb://my-rs-0.my-rs-svc.my-namespace.svc.my.cluster:40333,my-rs-1.my-rs-svc.my-namespace.svc.my.cluster:40333") +} + +func TestMongodConfiguration(t *testing.T) { + mc := NewMongodConfiguration() + assert.Equal(t, mc.Object, map[string]interface{}{}) + assert.Equal(t, mc.GetDBDataDir(), "/data") + assert.Equal(t, mc.GetDBPort(), 27017) + mc.SetOption("net.port", 40333.) + assert.Equal(t, mc.GetDBPort(), 40333) + mc.SetOption("storage", map[string]interface{}{"dbPath": "/other/data/path"}) + assert.Equal(t, mc.GetDBDataDir(), "/other/data/path") + assert.Equal(t, mc.Object, map[string]interface{}{ + "net": map[string]interface{}{ + "port": 40333., + }, + "storage": map[string]interface{}{ + "dbPath": "/other/data/path", + }, + }) } func TestGetScramCredentialsSecretName(t *testing.T) { diff --git a/controllers/construct/build_statefulset_test.go b/controllers/construct/build_statefulset_test.go index 77d2ba2cf..c55c5d17d 100644 --- a/controllers/construct/build_statefulset_test.go +++ b/controllers/construct/build_statefulset_test.go @@ -61,7 +61,7 @@ func TestMultipleCalls_DoNotCauseSideEffects(t *testing.T) { } func TestMongod_Container(t *testing.T) { - c := container.New(mongodbContainer("4.2", []corev1.VolumeMount{}, map[string]interface{}{})) + c := container.New(mongodbContainer("4.2", []corev1.VolumeMount{}, mdbv1.NewMongodConfiguration())) t.Run("Has correct Env vars", func(t *testing.T) { assert.Len(t, c.Env, 1) @@ -78,25 +78,6 @@ func TestMongod_Container(t *testing.T) { }) } -func TestGetDbPath(t *testing.T) { - t.Run("Test default is used if unspecifed", func(t *testing.T) { - m := map[string]interface{}{} - path := GetDBDataDir(m) - assert.Equal(t, defaultDataDir, path) - }) - - t.Run("Test storage.dbPath is used if specified", func(t *testing.T) { - m := map[string]interface{}{ - "storage": map[string]interface{}{ - "dbPath": "/data/db", - }, - } - - path := GetDBDataDir(m) - assert.Equal(t, "/data/db", path) - }) -} - func assertStatefulSetIsBuiltCorrectly(t *testing.T, mdb mdbv1.MongoDBCommunity, sts *appsv1.StatefulSet) { assert.Len(t, sts.Spec.Template.Spec.Containers, 2) assert.Len(t, sts.Spec.Template.Spec.InitContainers, 2) diff --git a/controllers/construct/mongodbstatefulset.go b/controllers/construct/mongodbstatefulset.go index ef31271d1..a929a9257 100644 --- a/controllers/construct/mongodbstatefulset.go +++ b/controllers/construct/mongodbstatefulset.go @@ -5,8 +5,6 @@ import ( "os" "strings" - "github.com/stretchr/objx" - "github.com/mongodb/mongodb-kubernetes-operator/pkg/automationconfig" "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/container" "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/persistentvolumeclaim" @@ -19,6 +17,7 @@ import ( appsv1 "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/types" + mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" corev1 "k8s.io/api/core/v1" ) @@ -45,7 +44,6 @@ const ( ReadinessProbeImageEnv = "READINESS_PROBE_IMAGE" ManagedSecurityContextEnv = "MANAGED_SECURITY_CONTEXT" - defaultDataDir = "/data" automationMongodConfFileName = "automation-mongod.conf" keyfileFilePath = "/var/lib/mongodb-mms-automation/authentication/keyfile" @@ -88,7 +86,7 @@ type MongoDBStatefulSetOwner interface { LogsVolumeName() string // GetMongodConfiguration returns the MongoDB configuration for each member. - GetMongodConfiguration() map[string]interface{} + GetMongodConfiguration() mdbv1.MongodConfiguration // NeedsAutomationConfigVolume returns whether the statefuslet needs to have a volume for the automationconfig. NeedsAutomationConfigVolume() bool @@ -137,14 +135,14 @@ func BuildMongoDBReplicaSetStatefulSetModificationFunction(mdb MongoDBStatefulSe singleModeVolumeClaim := func(s *appsv1.StatefulSet) {} if mdb.HasSeparateDataAndLogsVolumes() { logVolumeMount := statefulset.CreateVolumeMount(mdb.LogsVolumeName(), automationconfig.DefaultAgentLogPath) - dataVolumeMount := statefulset.CreateVolumeMount(mdb.DataVolumeName(), GetDBDataDir(mdb.GetMongodConfiguration())) + dataVolumeMount := statefulset.CreateVolumeMount(mdb.DataVolumeName(), mdb.GetMongodConfiguration().GetDBDataDir()) dataVolumeClaim = statefulset.WithVolumeClaim(mdb.DataVolumeName(), dataPvc(mdb.DataVolumeName())) logVolumeClaim = statefulset.WithVolumeClaim(mdb.LogsVolumeName(), logsPvc(mdb.LogsVolumeName())) mongodbAgentVolumeMounts = append(mongodbAgentVolumeMounts, dataVolumeMount, logVolumeMount) mongodVolumeMounts = append(mongodVolumeMounts, dataVolumeMount, logVolumeMount) } else { mounts := []corev1.VolumeMount{ - statefulset.CreateVolumeMount(mdb.DataVolumeName(), GetDBDataDir(mdb.GetMongodConfiguration()), statefulset.WithSubPath("data")), + statefulset.CreateVolumeMount(mdb.DataVolumeName(), mdb.GetMongodConfiguration().GetDBDataDir(), statefulset.WithSubPath("data")), statefulset.CreateVolumeMount(mdb.DataVolumeName(), automationconfig.DefaultAgentLogPath, statefulset.WithSubPath("logs")), } mongodbAgentVolumeMounts = append(mongodbAgentVolumeMounts, mounts...) @@ -291,16 +289,8 @@ func getMongoDBImage(version string) string { return fmt.Sprintf("%s/%s:%s", repoUrl, mongoImageName, version) } -// GetDBDataDir returns the db path which should be used. -func GetDBDataDir(additionalMongoDBConfig objx.Map) string { - if additionalMongoDBConfig == nil { - return defaultDataDir - } - return additionalMongoDBConfig.Get("storage.dbPath").Str(defaultDataDir) -} - -func mongodbContainer(version string, volumeMounts []corev1.VolumeMount, additionalMongoDBConfig map[string]interface{}) container.Modification { - filePath := GetDBDataDir(additionalMongoDBConfig) + "/" + automationMongodConfFileName +func mongodbContainer(version string, volumeMounts []corev1.VolumeMount, additionalMongoDBConfig mdbv1.MongodConfiguration) container.Modification { + filePath := additionalMongoDBConfig.GetDBDataDir() + "/" + automationMongodConfFileName mongoDbCommand := fmt.Sprintf(` #run post-start hook to handle version changes /hooks/version-upgrade diff --git a/controllers/replica_set_controller.go b/controllers/replica_set_controller.go index 22680558c..745d26ec8 100644 --- a/controllers/replica_set_controller.go +++ b/controllers/replica_set_controller.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "fmt" - "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/merge" "os" "github.com/imdario/mergo" @@ -18,6 +17,7 @@ import ( "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/container" "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/functions" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/merge" "github.com/mongodb/mongodb-kubernetes-operator/pkg/agent" @@ -492,7 +492,8 @@ func buildAutomationConfig(mdb mdbv1.MongoDBCommunity, auth automationconfig.Aut SetFCV(mdb.Spec.FeatureCompatibilityVersion). SetOptions(automationconfig.Options{DownloadBase: "/var/lib/mongodb-mms-automation"}). SetAuth(auth). - SetDataDir(construct.GetDBDataDir(mdb.GetMongodConfiguration())). + SetDataDir(mdb.GetMongodConfiguration().GetDBDataDir()). + SetPort(mdb.GetMongodConfiguration().GetDBPort()). AddModifications(getMongodConfigModification(mdb)). AddModifications(modifications...). Build() @@ -519,7 +520,7 @@ func buildService(mdb mdbv1.MongoDBCommunity, isArbiter bool) corev1.Service { SetSelector(label). SetServiceType(corev1.ServiceTypeClusterIP). SetClusterIP("None"). - SetPort(27017). + SetPort(int32(mdb.GetMongodConfiguration().GetDBPort())). SetPortName("mongodb"). SetPublishNotReadyAddresses(true). SetOwnerReferences(mdb.GetOwnerReferences()). diff --git a/controllers/replicaset_controller_test.go b/controllers/replicaset_controller_test.go index 111153583..c39d4fa79 100644 --- a/controllers/replicaset_controller_test.go +++ b/controllers/replicaset_controller_test.go @@ -4,13 +4,14 @@ import ( "context" "encoding/json" "fmt" - "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/container" "io/ioutil" "os" "reflect" "testing" "time" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/container" + "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/yaml" @@ -271,6 +272,37 @@ func TestService_isCorrectlyCreatedAndUpdated(t *testing.T) { assertReconciliationSuccessful(t, res, err) } +func TestService_usesCustomMongodPortWhenSpecified(t *testing.T) { + mdb := newTestReplicaSet() + + mongodConfig := objx.New(map[string]interface{}{}) + mongodConfig.Set("net.port", 1000.) + mdb.Spec.AdditionalMongodConfig.Object = mongodConfig + + mgr := client.NewManager(&mdb) + r := NewReconciler(mgr) + res, err := r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) + assertReconciliationSuccessful(t, res, err) + + svc := corev1.Service{} + err = mgr.GetClient().Get(context.TODO(), types.NamespacedName{Name: mdb.ServiceName(), Namespace: mdb.Namespace}, &svc) + assert.NoError(t, err) + assert.Equal(t, svc.Spec.Type, corev1.ServiceTypeClusterIP) + assert.Equal(t, svc.Spec.Selector["app"], mdb.ServiceName()) + assert.Len(t, svc.Spec.Ports, 1) + assert.Equal(t, svc.Spec.Ports[0], corev1.ServicePort{Port: 1000, Name: "mongodb"}) + + res, err = r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) + assertReconciliationSuccessful(t, res, err) +} + +func TestCustomNetPort_Configuration(t *testing.T) { + svc, _ := performReconciliationAndGetService(t, "specify_net_port.yaml") + assert.Equal(t, svc.Spec.Type, corev1.ServiceTypeClusterIP) + assert.Len(t, svc.Spec.Ports, 1) + assert.Equal(t, svc.Spec.Ports[0], corev1.ServicePort{Port: 40333, Name: "mongodb"}) +} + func TestAutomationConfig_versionIsBumpedOnChange(t *testing.T) { mdb := newTestReplicaSet() @@ -714,6 +746,19 @@ func performReconciliationAndGetStatefulSet(t *testing.T, filePath string) (apps return sts, mgr.Client } +func performReconciliationAndGetService(t *testing.T, filePath string) (corev1.Service, client.Client) { + mdb, err := loadTestFixture(filePath) + assert.NoError(t, err) + mgr := client.NewManager(&mdb) + assert.NoError(t, generatePasswordsForAllUsers(mdb, mgr.Client)) + r := NewReconciler(mgr) + res, err := r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: mdb.NamespacedName()}) + assertReconciliationSuccessful(t, res, err) + svc, err := mgr.Client.GetService(types.NamespacedName{Name: mdb.ServiceName(), Namespace: mdb.Namespace}) + assert.NoError(t, err) + return svc, mgr.Client +} + func generatePasswordsForAllUsers(mdb mdbv1.MongoDBCommunity, c client.Client) error { for _, user := range mdb.Spec.Users { diff --git a/controllers/testdata/specify_net_port.yaml b/controllers/testdata/specify_net_port.yaml new file mode 100644 index 000000000..f57d367dd --- /dev/null +++ b/controllers/testdata/specify_net_port.yaml @@ -0,0 +1,26 @@ +--- +apiVersion: mongodbcommunity.mongodb.com/v1 +kind: MongoDBCommunity +metadata: + name: example-mongodb + namespace: test-ns +spec: + members: 3 + type: ReplicaSet + version: "4.2.6" + security: + authentication: + modes: ["SCRAM"] + users: + - name: my-user + db: admin + passwordSecretRef: + name: my-user-password + roles: + - name: clusterAdmin + db: admin + - name: userAdminAnyDatabase + db: admin + scramCredentialsSecretName: my-scram + additionalMongodConfig: + net.port: 40333 diff --git a/pkg/automationconfig/automation_config.go b/pkg/automationconfig/automation_config.go index 23dcf8364..1580067be 100644 --- a/pkg/automationconfig/automation_config.go +++ b/pkg/automationconfig/automation_config.go @@ -11,6 +11,7 @@ import ( const ( Mongod ProcessType = "mongod" DefaultMongoDBDataDir string = "/data" + DefaultDBPort int = 27017 DefaultAgentLogPath string = "/var/log/mongodb-mms-automation" ) diff --git a/pkg/automationconfig/automation_config_builder.go b/pkg/automationconfig/automation_config_builder.go index c86e60425..cfc9eb2d6 100644 --- a/pkg/automationconfig/automation_config_builder.go +++ b/pkg/automationconfig/automation_config_builder.go @@ -50,6 +50,7 @@ type Builder struct { sslConfig *TLS tlsConfig *TLS dataDir string + port int } func NewBuilder() *Builder { @@ -61,6 +62,7 @@ func NewBuilder() *Builder { backupVersions: []BackupVersion{}, monitoringVersions: []MonitoringVersion{}, processModifications: []func(int, *Process){}, + port: DefaultDBPort, tlsConfig: nil, sslConfig: nil, } @@ -121,6 +123,11 @@ func (b *Builder) SetDataDir(dataDir string) *Builder { return b } +func (b *Builder) SetPort(port int) *Builder { + b.port = port + return b +} + func (b *Builder) SetFCV(fcv string) *Builder { b.fcv = fcv return b @@ -266,7 +273,7 @@ func (b *Builder) Build() (AutomationConfig, error) { AuthSchemaVersion: 5, } - process.SetPort(27017) + process.SetPort(b.port) process.SetStoragePath(dataDir) process.SetReplicaSetName(b.name) diff --git a/pkg/util/merge/merge_automationconfigs_test.go b/pkg/util/merge/merge_automationconfigs_test.go index 42adaa8f9..5e947eed0 100644 --- a/pkg/util/merge/merge_automationconfigs_test.go +++ b/pkg/util/merge/merge_automationconfigs_test.go @@ -1,9 +1,10 @@ package merge import ( + "testing" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/automationconfig" "github.com/stretchr/testify/assert" - "testing" ) func TestMergeAutomationConfigs(t *testing.T) { @@ -53,7 +54,6 @@ func TestMergeAutomationConfigs_NonExistentMember(t *testing.T) { assert.NoError(t, err) - assert.False(t, override.Processes[0].Disabled) assert.True(t, override.Processes[1].Disabled) assert.False(t, override.Processes[2].Disabled) diff --git a/test/e2e/mongodbtests/mongodbtests.go b/test/e2e/mongodbtests/mongodbtests.go index 1e0bf4660..94776ff80 100644 --- a/test/e2e/mongodbtests/mongodbtests.go +++ b/test/e2e/mongodbtests/mongodbtests.go @@ -147,6 +147,19 @@ func ServiceHasOwnerReference(mdb *mdbv1.MongoDBCommunity, expectedOwnerReferenc } } +func ServiceUsesCorrectPort(mdb *mdbv1.MongoDBCommunity, expectedPort int32) func(t *testing.T) { + return func(t *testing.T) { + serviceNamespacedName := types.NamespacedName{Name: mdb.ServiceName(), Namespace: mdb.Namespace} + svc := corev1.Service{} + err := e2eutil.TestClient.Get(context.TODO(), serviceNamespacedName, &svc) + if err != nil { + t.Fatal(err) + } + assert.Len(t, svc.Spec.Ports, 1) + assert.Equal(t, svc.Spec.Ports[0].Port, expectedPort) + } +} + func AgentSecretsHaveOwnerReference(mdb *mdbv1.MongoDBCommunity, expectedOwnerReference metav1.OwnerReference) func(t *testing.T) { checkSecret := func(t *testing.T, resourceNamespacedName types.NamespacedName) { secret := corev1.Secret{} diff --git a/test/e2e/replica_set_mongod_config/replica_set_mongod_config_test.go b/test/e2e/replica_set_mongod_config/replica_set_mongod_config_test.go index 14328f3b3..c70c02403 100644 --- a/test/e2e/replica_set_mongod_config/replica_set_mongod_config_test.go +++ b/test/e2e/replica_set_mongod_config/replica_set_mongod_config_test.go @@ -5,7 +5,7 @@ import ( "os" "testing" - "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/util/mongotester" + . "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/util/mongotester" e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/mongodbtests" @@ -32,11 +32,6 @@ func TestReplicaSet(t *testing.T) { t.Fatal(err) } - tester, err := mongotester.FromResource(t, mdb) - if err != nil { - t.Fatal(err) - } - settings := []string{ "storage.wiredTiger.engineConfig.journalCompressor", "storage.dbPath", @@ -53,8 +48,16 @@ func TestReplicaSet(t *testing.T) { mongodConfig.Set(settings[i], values[i]) } + // Override the net.port setting + mongodConfig.Set("net.port", 40333.) + mdb.Spec.AdditionalMongodConfig.Object = mongodConfig + tester, err := FromResource(t, mdb) + if err != nil { + t.Fatal(err) + } + t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb)) t.Run("Ensure Authentication", tester.EnsureAuthenticationIsConfigured(3)) @@ -63,4 +66,6 @@ func TestReplicaSet(t *testing.T) { for i := range settings { t.Run(fmt.Sprintf("Mongod setting %s has been set", settings[i]), tester.EnsureMongodConfig(settings[i], values[i])) } + t.Run("Mongod setting net.port has been set", tester.EnsureMongodConfig("net.port", int32(40333))) + t.Run("Service has the correct port", mongodbtests.ServiceUsesCorrectPort(&mdb, 40333)) } From 64314ff56bc64685e546f8ec2ae46f812d27402e Mon Sep 17 00:00:00 2001 From: haslersn Date: Thu, 13 Jan 2022 15:39:52 +0100 Subject: [PATCH 448/790] Cluster-wide deployment: Fix kustomize compatibility (#835) [672] added the subject namespace ``. However, this broke compatibility of that file with kustomize. kustomize will insert the subject namespace, if it's `default` or empty, but not if it's ``. [672]: https://github.com/mongodb/mongodb-kubernetes-operator/pull/672 Signed-off-by: Sebastian Hasler --- deploy/clusterwide/role_binding.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/clusterwide/role_binding.yaml b/deploy/clusterwide/role_binding.yaml index 11713808f..7617ec02d 100644 --- a/deploy/clusterwide/role_binding.yaml +++ b/deploy/clusterwide/role_binding.yaml @@ -4,7 +4,7 @@ metadata: name: mongodb-kubernetes-operator subjects: - kind: ServiceAccount - namespace: + # namespace: name: mongodb-kubernetes-operator roleRef: kind: ClusterRole From b47a8f8937f9e71cfa943db2e546b9bc12983794 Mon Sep 17 00:00:00 2001 From: Priyadarshi Lahiri <1254077+priyolahiri@users.noreply.github.com> Date: Thu, 13 Jan 2022 15:43:48 +0100 Subject: [PATCH 449/790] Update PULL_REQUEST_TEMPLATE.md --- .github/PULL_REQUEST_TEMPLATE.md | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 7d52039b2..f77733901 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -2,6 +2,5 @@ * [ ] Have you opened an Issue before filing this PR? * [ ] Have you signed our [CLA](https://www.mongodb.com/legal/contributor-agreement)? -* [ ] Have you signed all of your commits? * [ ] Have you checked to ensure there aren't other open [Pull Requests](../../../pulls) for the same update/change? * [ ] Put `closes #XXXX` in your comment to auto-close the issue that your PR fixes (if such). From 72a7198b9a9982bb663a5917ac3e7eeedb312c42 Mon Sep 17 00:00:00 2001 From: haslersn Date: Thu, 13 Jan 2022 17:58:55 +0100 Subject: [PATCH 450/790] Compatibility with Restricted PodSecurityPolicy (Operator Only) (#836) Setting the `RuntimeDefault` seccomp profile is required for running the operator under the Restricted policy of Pod Security Standards: https://kubernetes.io/docs/concepts/security/pod-security-standards/ Note that this commit only enables running the operator itself under the Restricted policy, and not the MongoDB StatefulSets created by the operator. For the latter, further changes are required: https://github.com/mongodb/mongodb-kubernetes-operator/issues/833 Signed-off-by: Sebastian Hasler --- config/manager/manager.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index d55db68d7..7de0d90ba 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -67,4 +67,7 @@ spec: securityContext: readOnlyRootFilesystem: true runAsUser: 2000 + securityContext: + seccompProfile: + type: RuntimeDefault serviceAccountName: mongodb-kubernetes-operator From 83dd773afc920d38bc711d5f984f83b2cd7c5fac Mon Sep 17 00:00:00 2001 From: Nikolas De Giorgis Date: Fri, 14 Jan 2022 09:56:57 +0000 Subject: [PATCH 451/790] CLOUDP-105593: Agent image to ubi8-minimal (#816) * CLOUDP-105593: Agent/ image/ to/ ubi8 * reintroduced package accidentally removed * wip --- scripts/dev/templates/agent/Dockerfile.ubi | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/scripts/dev/templates/agent/Dockerfile.ubi b/scripts/dev/templates/agent/Dockerfile.ubi index a70db5ef2..379447a0a 100644 --- a/scripts/dev/templates/agent/Dockerfile.ubi +++ b/scripts/dev/templates/agent/Dockerfile.ubi @@ -1,10 +1,12 @@ {% extends "Dockerfile.template" %} -{% set base_image = "registry.access.redhat.com/ubi7/ubi" %} +{% set base_image = "registry.access.redhat.com/ubi8/ubi-minimal" %} {% block packages -%} -RUN yum install -y --disableplugin=subscription-manager -q curl \ - hostname nss_wrapper --exclude perl-IO-Socket-SSL procps \ - && yum upgrade -y -q \ +RUN microdnf install -y --disableplugin=subscription-manager curl \ + hostname nss_wrapper tar gzip procps\ + && microdnf upgrade -y \ && rm -rf /var/lib/apt/lists/* + +RUN microdnf remove perl-IO-Socket-SSL {% endblock -%} From 67a5515c6e7e8223cf7352ce94bcc0b9bc269081 Mon Sep 17 00:00:00 2001 From: mircea-cosbuc Date: Fri, 14 Jan 2022 16:27:39 +0100 Subject: [PATCH 452/790] Add sample for specifying additional mongod config (#852) --- ...community_additional_mongod_config_cr.yaml | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 config/samples/mongodb.com_v1_mongodbcommunity_additional_mongod_config_cr.yaml diff --git a/config/samples/mongodb.com_v1_mongodbcommunity_additional_mongod_config_cr.yaml b/config/samples/mongodb.com_v1_mongodbcommunity_additional_mongod_config_cr.yaml new file mode 100644 index 000000000..bf4b12b67 --- /dev/null +++ b/config/samples/mongodb.com_v1_mongodbcommunity_additional_mongod_config_cr.yaml @@ -0,0 +1,41 @@ +--- +apiVersion: mongodbcommunity.mongodb.com/v1 +kind: MongoDBCommunity +metadata: + name: example-mongodb +spec: + members: 3 + type: ReplicaSet + version: "5.0.3" + security: + authentication: + modes: ["SCRAM"] + users: + - name: my-user + db: admin + passwordSecretRef: # a reference to the secret that will be used to generate the user's password + name: my-user-password + roles: + - name: clusterAdmin + db: admin + - name: userAdminAnyDatabase + db: admin + scramCredentialsSecretName: my-scram + additionalMongodConfig: + # the additional config passed to the mongod process can be specified + # either in nested or dot notation + storage.wiredTiger.engineConfig.journalCompressor: zlib + net: + port: 40333 + + +# the user credentials will be generated from this secret +# once the credentials are generated, this secret is no longer required +--- +apiVersion: v1 +kind: Secret +metadata: + name: my-user-password +type: Opaque +stringData: + password: From 1d799769e3225b240e94ffcfe4c1b4bdc1736020 Mon Sep 17 00:00:00 2001 From: Choubani Amir Date: Mon, 17 Jan 2022 14:51:28 +0100 Subject: [PATCH 453/790] fix names from role to cluster_role and from role_binding to cluster_role_binding (#793) --- deploy/clusterwide/{role.yaml => cluster_role.yaml} | 0 .../clusterwide/{role_binding.yaml => cluster_role_binding.yaml} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename deploy/clusterwide/{role.yaml => cluster_role.yaml} (100%) rename deploy/clusterwide/{role_binding.yaml => cluster_role_binding.yaml} (100%) diff --git a/deploy/clusterwide/role.yaml b/deploy/clusterwide/cluster_role.yaml similarity index 100% rename from deploy/clusterwide/role.yaml rename to deploy/clusterwide/cluster_role.yaml diff --git a/deploy/clusterwide/role_binding.yaml b/deploy/clusterwide/cluster_role_binding.yaml similarity index 100% rename from deploy/clusterwide/role_binding.yaml rename to deploy/clusterwide/cluster_role_binding.yaml From c3ccf8a9db492bee404876fe9feb8ce6c2a15381 Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Mon, 17 Jan 2022 15:50:38 +0000 Subject: [PATCH 454/790] Update CODEOWNERS (#855) --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 7935763f8..d4168078f 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1 @@ -* @bznein @chatton @irajdeep @rodrigovalin @tibulca @priyolahiri +* @chatton @irajdeep @rodrigovalin @tibulca @priyolahiri @mircea-cosbuc @igor-karpukhin From a7a3098d4ed54fab23525adfa437dba098a2a5ac Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Tue, 18 Jan 2022 08:50:34 +0000 Subject: [PATCH 455/790] Removed Uneeded Action (#854) --- .action_templates/e2e-fork-template.yaml | 1 - .action_templates/e2e-pr-template.yaml | 1 - .action_templates/jobs/e2e-success.yaml | 8 -------- .github/workflows/e2e-fork.yml | 9 --------- .github/workflows/e2e.yml | 9 --------- 5 files changed, 28 deletions(-) delete mode 100644 .action_templates/jobs/e2e-success.yaml diff --git a/.action_templates/e2e-fork-template.yaml b/.action_templates/e2e-fork-template.yaml index df3fde82e..40070737f 100644 --- a/.action_templates/e2e-fork-template.yaml +++ b/.action_templates/e2e-fork-template.yaml @@ -21,7 +21,6 @@ jobs: - template: run-test-matrix - template: save-run-status - template: dump-and-upload-diagnostics - - template: e2e-success events: - template: pull-request-target diff --git a/.action_templates/e2e-pr-template.yaml b/.action_templates/e2e-pr-template.yaml index 35c6efac7..ac3fce7b3 100644 --- a/.action_templates/e2e-pr-template.yaml +++ b/.action_templates/e2e-pr-template.yaml @@ -21,7 +21,6 @@ jobs: - template: run-test-matrix - template: save-run-status - template: dump-and-upload-diagnostics - - template: e2e-success events: - template: on-pull-request-master diff --git a/.action_templates/jobs/e2e-success.yaml b/.action_templates/jobs/e2e-success.yaml deleted file mode 100644 index d4aac0eab..000000000 --- a/.action_templates/jobs/e2e-success.yaml +++ /dev/null @@ -1,8 +0,0 @@ -e2e-success: - if: always() - needs: [tests] - runs-on: ubuntu-latest - steps: - - name: Check E2E Result - if: needs.tests.result != 'success' - run: exit 1 diff --git a/.github/workflows/e2e-fork.yml b/.github/workflows/e2e-fork.yml index b88a335ff..e64442a5c 100644 --- a/.github/workflows/e2e-fork.yml +++ b/.github/workflows/e2e-fork.yml @@ -217,12 +217,3 @@ jobs: with: name: ${{ matrix.test-name }}-${{ matrix.distro }}-diagnostics path: ${{ github.workspace }}/diagnostics - # template: .action_templates/jobs/e2e-success.yaml - e2e-success: - if: always() - needs: [tests] - runs-on: ubuntu-latest - steps: - - name: Check E2E Result - if: needs.tests.result != 'success' - run: exit 1 diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index fc82c4acf..348604d1b 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -221,12 +221,3 @@ jobs: with: name: ${{ matrix.test-name }}-${{ matrix.distro }}-diagnostics path: ${{ github.workspace }}/diagnostics - # template: .action_templates/jobs/e2e-success.yaml - e2e-success: - if: always() - needs: [tests] - runs-on: ubuntu-latest - steps: - - name: Check E2E Result - if: needs.tests.result != 'success' - run: exit 1 From f806387bd79a094451ebc309bb48413bdd72b0c6 Mon Sep 17 00:00:00 2001 From: corryroot <72401712+corryroot@users.noreply.github.com> Date: Wed, 19 Jan 2022 08:50:26 -0500 Subject: [PATCH 456/790] (DOCSP-16543): Changed MongoDB resource to MongoDBCommunity resource. (#859) * (DOCSP-16543): Changed MongoDB resource to MongoDBCommunity resource. * (DOCSP-16543): Updated links per JW's feedback. --- docs/README.md | 4 ++-- docs/architecture.md | 4 ++-- docs/contributing.md | 2 +- docs/deploy-configure.md | 28 ++++++++++++++-------------- docs/install-upgrade.md | 6 +++--- docs/release-notes-template.md | 2 +- docs/secure.md | 24 ++++++++++++------------ docs/users.md | 12 ++++++------ 8 files changed, 41 insertions(+), 41 deletions(-) diff --git a/docs/README.md b/docs/README.md index 58e3d56d5..be9716b66 100644 --- a/docs/README.md +++ b/docs/README.md @@ -5,6 +5,6 @@ - [Contribute to the MongoDB Kubernetes Operator](/docs/contributing.md) - [MongoDB Community Kubernetes Operator Architecture](/docs/architecture.md) - [Install and Upgrade the Community Kubernetes Operator](/docs/install-upgrade.md) -- [Deploy and Configure MongoDB Resources](/docs/deploy-configure.md) +- [Deploy and Configure MongoDBCommunity Resources](/docs/deploy-configure.md) - [Create Database Users](/docs/users.md) -- [Secure MongoDB Resources](/docs/secure.md) +- [Secure MongoDBCommunity Resources](/docs/secure.md) diff --git a/docs/architecture.md b/docs/architecture.md index 4b66c1fe0..97e4f0a26 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -10,7 +10,7 @@ The MongoDB Community Kubernetes Operator is a [Custom Resource Definition](http ## Cluster Configuration -You create and update MongoDB resources by defining a MongoDB resource definition. When you apply the MongoDB resource definition to your Kubernetes environment, the Operator: +You create and update MongoDBCommunity resources by defining a MongoDBCommunity resource definition. When you apply the MongoDBCommunity resource definition to your Kubernetes environment, the Operator: 1. Creates a [StatefulSet](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/) that contains one [pod](https://kubernetes.io/docs/concepts/workloads/pods/pod-overview/) for each [replica set](https://docs.mongodb.com/manual/replication/) member. 1. Writes the Automation configuration as a [Secret](https://kubernetes.io/docs/concepts/configuration/secret/) and mounts it to each pod. @@ -28,7 +28,7 @@ You create and update MongoDB resources by defining a MongoDB resource definitio - `automation-config` which is mounted from the previously generated `Secret` to both the server and agent. Only lives as long as the pod. - `healthstatus` which contains the agent's current status. This is shared with the `mongod` container where it's used by the pre-stop hook. Only lives as long as the pod. -1. Initiates the MongoDB Agent, which in turn creates the database configuration and launches the `mongod` process according to your [MongoDB resource definition](../deploy/crds/mongodb.com_v1_mongodbcommunity_cr.yaml). +1. Initiates the MongoDB Agent, which in turn creates the database configuration and launches the `mongod` process according to your [MongoDBCommunity resource definition](../deploy/crds/mongodb.com_v1_mongodbcommunity_cr.yaml). + ### All Submissions: * [ ] Have you opened an Issue before filing this PR? From 74d13f189566574b862e5670b366b61ec5b65923 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Sierant?= Date: Mon, 15 Apr 2024 19:29:01 +0200 Subject: [PATCH 725/790] Bump k8s apis to 1.27, refactor to support contexts (#1523) --- .gitignore | 1 + cmd/readiness/main.go | 12 +- cmd/readiness/readiness_test.go | 16 +- cmd/versionhook/main.go | 7 +- ...ommunity.mongodb.com_mongodbcommunity.yaml | 19 +- controllers/mongodb_cleanup.go | 9 +- controllers/mongodb_cleanup_test.go | 12 +- controllers/mongodb_tls.go | 67 +-- controllers/mongodb_tls_test.go | 172 +++--- controllers/mongodb_users.go | 17 +- controllers/prometheus.go | 7 +- controllers/replica_set_controller.go | 258 ++++----- controllers/replicaset_controller_test.go | 500 +++++++++--------- controllers/watch/watch.go | 14 +- controllers/watch/watch_test.go | 45 +- go.mod | 50 +- go.sum | 489 +++-------------- pkg/agent/agent_readiness.go | 9 +- pkg/agent/agent_readiness_test.go | 12 +- pkg/authentication/authentication.go | 7 +- pkg/authentication/authentication_test.go | 12 +- pkg/authentication/mocks/mocks.go | 13 +- pkg/authentication/scram/scram.go | 47 +- pkg/authentication/scram/scram_test.go | 43 +- pkg/authentication/x509/x509.go | 11 +- pkg/authentication/x509/x509_test.go | 15 +- .../automation_config_secret.go | 17 +- .../automation_config_secret_test.go | 20 +- pkg/kube/annotations/annotations.go | 10 +- pkg/kube/client/client.go | 76 +-- pkg/kube/client/client_test.go | 23 +- pkg/kube/client/mocked_client.go | 9 + pkg/kube/client/mocked_client_test.go | 9 +- pkg/kube/client/mocked_manager.go | 21 +- pkg/kube/configmap/configmap.go | 39 +- pkg/kube/configmap/configmap_test.go | 33 +- pkg/kube/pod/pod.go | 3 +- pkg/kube/secret/secret.go | 59 ++- pkg/kube/secret/secret_test.go | 56 +- pkg/kube/service/service.go | 9 +- pkg/kube/statefulset/statefulset.go | 27 +- pkg/readiness/headless/headless.go | 7 +- pkg/readiness/headless/headless_test.go | 5 +- pkg/readiness/pod/podannotation.go | 6 +- pkg/readiness/pod/podannotation_test.go | 14 +- pkg/readiness/pod/podpatcher.go | 4 +- pkg/readiness/secret/automationconfig.go | 5 +- pkg/readiness/secret/secretreader.go | 4 +- pkg/util/status/status.go | 4 +- test/e2e/client.go | 20 +- test/e2e/e2eutil.go | 22 +- .../feature_compatibility_version_test.go | 32 +- test/e2e/mongodbtests/mongodbtests.go | 267 +++++----- test/e2e/prometheus/prometheus_test.go | 34 +- test/e2e/replica_set/replica_set_test.go | 28 +- .../replica_set_arbiter_test.go | 32 +- .../replica_set_authentication_test.go | 32 +- .../replica_set_change_version_test.go | 40 +- ...lica_set_connection_string_options_test.go | 62 +-- ...replica_set_cross_namespace_deploy_test.go | 37 +- .../replica_set_custom_annotations_test.go | 24 +- ...plica_set_custom_persistent_volume_test.go | 33 +- .../replica_set_custom_role_test.go | 22 +- .../replica_set_enterprise_upgrade.go | 27 +- ...replica_set_enterprise_upgrade_4_5_test.go | 4 +- ...replica_set_enterprise_upgrade_5_6_test.go | 4 +- ...replica_set_enterprise_upgrade_5_6_test.go | 4 +- .../replica_set_mongod_config_test.go | 20 +- ...t_mongod_port_change_with_arbiters_test.go | 54 +- .../replica_set_mongod_readiness_test.go | 26 +- ...eplica_set_mount_connection_string_test.go | 35 +- .../replica_set_multiple_test.go | 53 +- .../replica_set_operator_upgrade_test.go | 59 ++- .../replica_set_recovery_test.go | 41 +- .../replica_set_scaling_test.go | 43 +- .../replica_set_scale_down_test.go | 43 +- .../replica_set_tls/replica_set_tls_test.go | 38 +- .../replica_set_tls_recreate_mdbc_test.go | 47 +- .../replica_set_tls_rotate_test.go | 38 +- .../replica_set_tls_rotate_delete_sts_test.go | 34 +- .../replica_set_tls_upgrade_test.go | 38 +- .../replica_set_x509/replica_set_x509_test.go | 69 +-- test/e2e/setup/setup.go | 42 +- .../statefulset_arbitrary_config_test.go | 26 +- ...tatefulset_arbitrary_config_update_test.go | 28 +- .../statefulset_delete_test.go | 35 +- test/e2e/tlstests/tlstests.go | 30 +- test/e2e/util/mongotester/mongotester.go | 72 +-- test/e2e/util/wait/wait.go | 98 ++-- 89 files changed, 1905 insertions(+), 2112 deletions(-) diff --git a/.gitignore b/.gitignore index 7836e6873..0229263df 100644 --- a/.gitignore +++ b/.gitignore @@ -100,3 +100,4 @@ diagnostics Pipfile Pipfile.lock .community-operator-dev +*.iml diff --git a/cmd/readiness/main.go b/cmd/readiness/main.go index c335c67fe..14d369b22 100644 --- a/cmd/readiness/main.go +++ b/cmd/readiness/main.go @@ -1,6 +1,7 @@ package main import ( + "context" "encoding/json" "fmt" "io" @@ -42,7 +43,7 @@ func init() { // - If MongoDB: then just the 'statuses[0].IsInGoalState` field is used to learn if the Agent has reached the goal // - if AppDB: the 'mmsStatus[0].lastGoalVersionAchieved' field is compared with the one from mounted automation config // Additionally if the previous check hasn't returned 'true' an additional check for wait steps is being performed -func isPodReady(conf config.Config) (bool, error) { +func isPodReady(ctx context.Context, conf config.Config) (bool, error) { healthStatus, err := parseHealthStatus(conf.HealthStatusReader) if err != nil { logger.Errorf("There was problem parsing health status file: %s", err) @@ -56,7 +57,7 @@ func isPodReady(conf config.Config) (bool, error) { } // If the agent has reached the goal state - inGoalState, err := isInGoalState(healthStatus, conf) + inGoalState, err := isInGoalState(ctx, healthStatus, conf) if err != nil { logger.Errorf("There was problem checking the health status: %s", err) return false, err @@ -159,9 +160,9 @@ func isWaitStep(status *health.StepStatus) bool { return false } -func isInGoalState(health health.Status, conf config.Config) (bool, error) { +func isInGoalState(ctx context.Context, health health.Status, conf config.Config) (bool, error) { if isHeadlessMode() { - return headless.PerformCheckHeadlessMode(health, conf) + return headless.PerformCheckHeadlessMode(ctx, health, conf) } return performCheckOMMode(health), nil } @@ -216,6 +217,7 @@ func initLogger(l *lumberjack.Logger) { } func main() { + ctx := context.Background() clientSet, err := kubernetesClientset() if err != nil { panic(err) @@ -238,7 +240,7 @@ func main() { panic(err) } - ready, err := isPodReady(cfg) + ready, err := isPodReady(ctx, cfg) if err != nil { panic(err) } diff --git a/cmd/readiness/readiness_test.go b/cmd/readiness/readiness_test.go index 0f4fa29b4..97db1dc11 100644 --- a/cmd/readiness/readiness_test.go +++ b/cmd/readiness/readiness_test.go @@ -22,6 +22,7 @@ import ( // TestDeadlockDetection verifies that if the agent is stuck in "WaitAllRsMembersUp" phase (started > 15 seconds ago) // then the function returns "ready" func TestDeadlockDetection(t *testing.T) { + ctx := context.Background() type TestConfig struct { conf config.Config isErrorExpected bool @@ -108,7 +109,7 @@ func TestDeadlockDetection(t *testing.T) { for testName, _ := range tests { testConfig := tests[testName] t.Run(testName, func(t *testing.T) { - ready, err := isPodReady(testConfig.conf) + ready, err := isPodReady(ctx, testConfig.conf) if testConfig.isErrorExpected { assert.Error(t, err) } else { @@ -241,8 +242,9 @@ func TestObtainingCurrentStep(t *testing.T) { // In this case, the Readiness Probe needs to return Ready and let the StatefulSet Controller to proceed // with the Pod rollout. func TestReadyWithWaitForCorrectBinaries(t *testing.T) { + ctx := context.Background() c := testConfigWithMongoUp("testdata/health-status-ok-with-WaitForCorrectBinaries.json", time.Second*30) - ready, err := isPodReady(c) + ready, err := isPodReady(ctx, c) assert.True(t, ready) assert.NoError(t, err) @@ -254,26 +256,28 @@ func TestReadyWithWaitForCorrectBinaries(t *testing.T) { // (as Agent doesn't marks all the step statuses finished when it reaches the goal) but this doesn't affect the result // as the whole plan is complete already func TestHeadlessAgentHasntReachedGoal(t *testing.T) { + ctx := context.Background() t.Setenv(headlessAgent, "true") c := testConfig("testdata/health-status-ok.json") c.ClientSet = fake.NewSimpleClientset(testdata.TestPod(c.Namespace, c.Hostname), testdata.TestSecret(c.Namespace, c.AutomationConfigSecretName, 6)) - ready, err := isPodReady(c) + ready, err := isPodReady(ctx, c) assert.False(t, ready) assert.NoError(t, err) - thePod, _ := c.ClientSet.CoreV1().Pods(c.Namespace).Get(context.TODO(), c.Hostname, metav1.GetOptions{}) + thePod, _ := c.ClientSet.CoreV1().Pods(c.Namespace).Get(ctx, c.Hostname, metav1.GetOptions{}) assert.Equal(t, map[string]string{"agent.mongodb.com/version": "5"}, thePod.Annotations) } // TestHeadlessAgentReachedGoal verifies that the probe reports "true" if the config version is equal to the // last achieved version of the Agent func TestHeadlessAgentReachedGoal(t *testing.T) { + ctx := context.Background() t.Setenv(headlessAgent, "true") c := testConfig("testdata/health-status-ok.json") c.ClientSet = fake.NewSimpleClientset(testdata.TestPod(c.Namespace, c.Hostname), testdata.TestSecret(c.Namespace, c.AutomationConfigSecretName, 5)) - ready, err := isPodReady(c) + ready, err := isPodReady(ctx, c) assert.True(t, ready) assert.NoError(t, err) - thePod, _ := c.ClientSet.CoreV1().Pods(c.Namespace).Get(context.TODO(), c.Hostname, metav1.GetOptions{}) + thePod, _ := c.ClientSet.CoreV1().Pods(c.Namespace).Get(ctx, c.Hostname, metav1.GetOptions{}) assert.Equal(t, map[string]string{"agent.mongodb.com/version": "5"}, thePod.Annotations) } diff --git a/cmd/versionhook/main.go b/cmd/versionhook/main.go index e2b551630..6e0d02f95 100644 --- a/cmd/versionhook/main.go +++ b/cmd/versionhook/main.go @@ -27,6 +27,7 @@ const ( ) func main() { + ctx := context.Background() logger := setupLogger() logger.Info("Running version change post-start hook") @@ -57,7 +58,7 @@ func main() { if shouldDelete { logger.Infof("Pod should be deleted") - if err := deletePod(); err != nil { + if err := deletePod(ctx); err != nil { // We should not raise an error if the Pod could not be deleted. It can have even // worse consequences: Pod being restarted with the same version, and the agent // killing it immediately after. @@ -182,7 +183,7 @@ func isWaitingToBeDeleted(healthStatus agent.MmsDirectorStatus) bool { } // deletePod attempts to delete the pod this mongod is running in -func deletePod() error { +func deletePod(ctx context.Context) error { thisPod, err := getThisPod() if err != nil { return fmt.Errorf("could not get pod: %s", err) @@ -192,7 +193,7 @@ func deletePod() error { return fmt.Errorf("could not get client: %s", err) } - if err := k8sClient.Delete(context.TODO(), &thisPod); err != nil { + if err := k8sClient.Delete(ctx, &thisPod); err != nil { return fmt.Errorf("could not delete pod: %s", err) } return nil diff --git a/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml b/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml index 863cdb301..ba3121647 100644 --- a/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml +++ b/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml @@ -1,15 +1,10 @@ + --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.11.3 - service.binding: path={.metadata.name}-{.spec.users[0].db}-{.spec.users[0].name},objectType=Secret - service.binding/connectionString: path={.metadata.name}-{.spec.users[0].db}-{.spec.users[0].name},objectType=Secret,sourceKey=connectionString.standardSrv - service.binding/password: path={.metadata.name}-{.spec.users[0].db}-{.spec.users[0].name},objectType=Secret,sourceKey=password - service.binding/provider: community - service.binding/type: mongodb - service.binding/username: path={.metadata.name}-{.spec.users[0].db}-{.spec.users[0].name},objectType=Secret,sourceKey=username + controller-gen.kubebuilder.io/version: v0.4.1 creationTimestamp: null name: mongodbcommunity.mongodbcommunity.mongodb.com spec: @@ -290,7 +285,6 @@ spec: TODO: Add other useful fields. apiVersion, kind, uid?' type: string type: object - x-kubernetes-map-type: atomic agentMode: description: AgentMode contains the authentication mode used by the automation agent. @@ -419,7 +413,6 @@ spec: TODO: Add other useful fields. apiVersion, kind, uid?' type: string type: object - x-kubernetes-map-type: atomic caConfigMapRef: description: CaConfigMap is a reference to a ConfigMap containing the certificate for the CA which signed the server certificates @@ -432,7 +425,6 @@ spec: TODO: Add other useful fields. apiVersion, kind, uid?' type: string type: object - x-kubernetes-map-type: atomic certificateKeySecretRef: description: CertificateKeySecret is a reference to a Secret containing a private key and certificate to use for TLS. @@ -450,7 +442,6 @@ spec: TODO: Add other useful fields. apiVersion, kind, uid?' type: string type: object - x-kubernetes-map-type: atomic enabled: type: boolean optional: @@ -602,3 +593,9 @@ spec: storage: true subresources: status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/controllers/mongodb_cleanup.go b/controllers/mongodb_cleanup.go index 59b391acf..85d720ec0 100644 --- a/controllers/mongodb_cleanup.go +++ b/controllers/mongodb_cleanup.go @@ -1,6 +1,7 @@ package controllers import ( + "context" apiErrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" @@ -9,14 +10,14 @@ import ( ) // cleanupPemSecret cleans up the old pem secret generated for the agent certificate. -func (r *ReplicaSetReconciler) cleanupPemSecret(currentMDB mdbv1.MongoDBCommunitySpec, lastAppliedMDBSpec mdbv1.MongoDBCommunitySpec, namespace string) { +func (r *ReplicaSetReconciler) cleanupPemSecret(ctx context.Context, currentMDB mdbv1.MongoDBCommunitySpec, lastAppliedMDBSpec mdbv1.MongoDBCommunitySpec, namespace string) { if currentMDB.GetAgentAuthMode() == lastAppliedMDBSpec.GetAgentAuthMode() { return } if !currentMDB.IsAgentX509() && lastAppliedMDBSpec.IsAgentX509() { agentCertSecret := lastAppliedMDBSpec.GetAgentCertificateRef() - if err := r.client.DeleteSecret(types.NamespacedName{ + if err := r.client.DeleteSecret(ctx, types.NamespacedName{ Namespace: namespace, Name: agentCertSecret + "-pem", }); err != nil { @@ -30,11 +31,11 @@ func (r *ReplicaSetReconciler) cleanupPemSecret(currentMDB mdbv1.MongoDBCommunit } // cleanupScramSecrets cleans up old scram secrets based on the last successful applied mongodb spec. -func (r *ReplicaSetReconciler) cleanupScramSecrets(currentMDB mdbv1.MongoDBCommunitySpec, lastAppliedMDBSpec mdbv1.MongoDBCommunitySpec, namespace string) { +func (r *ReplicaSetReconciler) cleanupScramSecrets(ctx context.Context, currentMDB mdbv1.MongoDBCommunitySpec, lastAppliedMDBSpec mdbv1.MongoDBCommunitySpec, namespace string) { secretsToDelete := getScramSecretsToDelete(currentMDB, lastAppliedMDBSpec) for _, s := range secretsToDelete { - if err := r.client.DeleteSecret(types.NamespacedName{ + if err := r.client.DeleteSecret(ctx, types.NamespacedName{ Name: s, Namespace: namespace, }); err != nil { diff --git a/controllers/mongodb_cleanup_test.go b/controllers/mongodb_cleanup_test.go index 896b1797c..472e77266 100644 --- a/controllers/mongodb_cleanup_test.go +++ b/controllers/mongodb_cleanup_test.go @@ -1,6 +1,7 @@ package controllers import ( + "context" mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" kubeClient "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/client" "github.com/stretchr/testify/assert" @@ -98,6 +99,7 @@ func TestReplicaSetReconcilerCleanupScramSecrets(t *testing.T) { } func TestReplicaSetReconcilerCleanupPemSecret(t *testing.T) { + ctx := context.Background() lastAppliedSpec := mdbv1.MongoDBCommunitySpec{ Security: mdbv1.Security{ Authentication: mdbv1.Authentication{ @@ -134,21 +136,21 @@ func TestReplicaSetReconcilerCleanupPemSecret(t *testing.T) { }, } - mgr := kubeClient.NewManager(&mdb) + mgr := kubeClient.NewManager(ctx, &mdb) client := kubeClient.NewClient(mgr.GetClient()) - err := createAgentCertPemSecret(client, mdb, "CERT", "KEY", "") + err := createAgentCertPemSecret(ctx, client, mdb, "CERT", "KEY", "") assert.NoError(t, err) r := NewReconciler(mgr) - secret, err := r.client.GetSecret(mdb.AgentCertificatePemSecretNamespacedName()) + secret, err := r.client.GetSecret(ctx, mdb.AgentCertificatePemSecretNamespacedName()) assert.NoError(t, err) assert.Equal(t, "CERT", string(secret.Data["tls.crt"])) assert.Equal(t, "KEY", string(secret.Data["tls.key"])) - r.cleanupPemSecret(mdb.Spec, lastAppliedSpec, "my-ns") + r.cleanupPemSecret(ctx, mdb.Spec, lastAppliedSpec, "my-ns") - _, err = r.client.GetSecret(mdb.AgentCertificatePemSecretNamespacedName()) + _, err = r.client.GetSecret(ctx, mdb.AgentCertificatePemSecretNamespacedName()) assert.Error(t, err) } diff --git a/controllers/mongodb_tls.go b/controllers/mongodb_tls.go index d5427d76e..fe872d371 100644 --- a/controllers/mongodb_tls.go +++ b/controllers/mongodb_tls.go @@ -1,6 +1,7 @@ package controllers import ( + "context" "crypto/sha256" "fmt" "strings" @@ -35,7 +36,7 @@ const ( ) // validateTLSConfig will check that the configured ConfigMap and Secret exist and that they have the correct fields. -func (r *ReplicaSetReconciler) validateTLSConfig(mdb mdbv1.MongoDBCommunity) (bool, error) { +func (r *ReplicaSetReconciler) validateTLSConfig(ctx context.Context, mdb mdbv1.MongoDBCommunity) (bool, error) { if !mdb.Spec.Security.TLS.Enabled { return true, nil } @@ -43,7 +44,7 @@ func (r *ReplicaSetReconciler) validateTLSConfig(mdb mdbv1.MongoDBCommunity) (bo r.log.Info("Ensuring TLS is correctly configured") // Ensure CA cert is configured - _, err := getCaCrt(r.client, r.client, mdb) + _, err := getCaCrt(ctx, r.client, r.client, mdb) if err != nil { if apiErrors.IsNotFound(err) { @@ -55,7 +56,7 @@ func (r *ReplicaSetReconciler) validateTLSConfig(mdb mdbv1.MongoDBCommunity) (bo } // Ensure Secret exists - _, err = secret.ReadStringData(r.client, mdb.TLSSecretNamespacedName()) + _, err = secret.ReadStringData(ctx, r.client, mdb.TLSSecretNamespacedName()) if err != nil { if apiErrors.IsNotFound(err) { r.log.Warnf(`Secret "%s" not found`, mdb.TLSSecretNamespacedName()) @@ -67,20 +68,20 @@ func (r *ReplicaSetReconciler) validateTLSConfig(mdb mdbv1.MongoDBCommunity) (bo // validate whether the secret contains "tls.crt" and "tls.key", or it contains "tls.pem" // if it contains all three, then the pem entry should be equal to the concatenation of crt and key - _, err = getPemOrConcatenatedCrtAndKey(r.client, mdb, mdb.TLSSecretNamespacedName()) + _, err = getPemOrConcatenatedCrtAndKey(ctx, r.client, mdb, mdb.TLSSecretNamespacedName()) if err != nil { r.log.Warnf(err.Error()) return false, nil } // Watch certificate-key secret to handle rotations - r.secretWatcher.Watch(mdb.TLSSecretNamespacedName(), mdb.NamespacedName()) + r.secretWatcher.Watch(ctx, mdb.TLSSecretNamespacedName(), mdb.NamespacedName()) // Watch CA certificate changes if mdb.Spec.Security.TLS.CaCertificateSecret != nil { - r.secretWatcher.Watch(mdb.TLSCaCertificateSecretNamespacedName(), mdb.NamespacedName()) + r.secretWatcher.Watch(ctx, mdb.TLSCaCertificateSecretNamespacedName(), mdb.NamespacedName()) } else { - r.configMapWatcher.Watch(mdb.TLSConfigMapNamespacedName(), mdb.NamespacedName()) + r.configMapWatcher.Watch(ctx, mdb.TLSConfigMapNamespacedName(), mdb.NamespacedName()) } r.log.Infof("Successfully validated TLS config") @@ -89,17 +90,17 @@ func (r *ReplicaSetReconciler) validateTLSConfig(mdb mdbv1.MongoDBCommunity) (bo // getTLSConfigModification creates a modification function which enables TLS in the automation config. // It will also ensure that the combined cert-key secret is created. -func getTLSConfigModification(cmGetter configmap.Getter, secretGetter secret.Getter, mdb mdbv1.MongoDBCommunity) (automationconfig.Modification, error) { +func getTLSConfigModification(ctx context.Context, cmGetter configmap.Getter, secretGetter secret.Getter, mdb mdbv1.MongoDBCommunity) (automationconfig.Modification, error) { if !mdb.Spec.Security.TLS.Enabled { return automationconfig.NOOP(), nil } - caCert, err := getCaCrt(cmGetter, secretGetter, mdb) + caCert, err := getCaCrt(ctx, cmGetter, secretGetter, mdb) if err != nil { return automationconfig.NOOP(), err } - certKey, err := getPemOrConcatenatedCrtAndKey(secretGetter, mdb, mdb.TLSSecretNamespacedName()) + certKey, err := getPemOrConcatenatedCrtAndKey(ctx, secretGetter, mdb, mdb.TLSSecretNamespacedName()) if err != nil { return automationconfig.NOOP(), err } @@ -108,13 +109,13 @@ func getTLSConfigModification(cmGetter configmap.Getter, secretGetter secret.Get } // getCertAndKey will fetch the certificate and key from the user-provided Secret. -func getCertAndKey(getter secret.Getter, mdb mdbv1.MongoDBCommunity, secretName types.NamespacedName) string { - cert, err := secret.ReadKey(getter, tlsSecretCertName, secretName) +func getCertAndKey(ctx context.Context, getter secret.Getter, mdb mdbv1.MongoDBCommunity, secretName types.NamespacedName) string { + cert, err := secret.ReadKey(ctx, getter, tlsSecretCertName, secretName) if err != nil { return "" } - key, err := secret.ReadKey(getter, tlsSecretKeyName, secretName) + key, err := secret.ReadKey(ctx, getter, tlsSecretKeyName, secretName) if err != nil { return "" } @@ -123,8 +124,8 @@ func getCertAndKey(getter secret.Getter, mdb mdbv1.MongoDBCommunity, secretName } // getPem will fetch the pem from the user-provided secret -func getPem(getter secret.Getter, mdb mdbv1.MongoDBCommunity, secretName types.NamespacedName) string { - pem, err := secret.ReadKey(getter, tlsSecretPemName, secretName) +func getPem(ctx context.Context, getter secret.Getter, mdb mdbv1.MongoDBCommunity, secretName types.NamespacedName) string { + pem, err := secret.ReadKey(ctx, getter, tlsSecretPemName, secretName) if err != nil { return "" } @@ -141,9 +142,9 @@ func combineCertificateAndKey(cert, key string) string { // This is either the tls.pem entry in the given secret, or the concatenation // of tls.crt and tls.key // It performs a basic validation on the entries. -func getPemOrConcatenatedCrtAndKey(getter secret.Getter, mdb mdbv1.MongoDBCommunity, secretName types.NamespacedName) (string, error) { - certKey := getCertAndKey(getter, mdb, secretName) - pem := getPem(getter, mdb, secretName) +func getPemOrConcatenatedCrtAndKey(ctx context.Context, getter secret.Getter, mdb mdbv1.MongoDBCommunity, secretName types.NamespacedName) (string, error) { + certKey := getCertAndKey(ctx, getter, mdb, secretName) + pem := getPem(ctx, getter, mdb, secretName) if certKey == "" && pem == "" { return "", fmt.Errorf(`neither "%s" nor the pair "%s"/"%s" were present in the TLS secret`, tlsSecretPemName, tlsSecretCertName, tlsSecretKeyName) } @@ -159,16 +160,16 @@ func getPemOrConcatenatedCrtAndKey(getter secret.Getter, mdb mdbv1.MongoDBCommun return certKey, nil } -func getCaCrt(cmGetter configmap.Getter, secretGetter secret.Getter, mdb mdbv1.MongoDBCommunity) (string, error) { +func getCaCrt(ctx context.Context, cmGetter configmap.Getter, secretGetter secret.Getter, mdb mdbv1.MongoDBCommunity) (string, error) { var caResourceName types.NamespacedName var caData map[string]string var err error if mdb.Spec.Security.TLS.CaCertificateSecret != nil { caResourceName = mdb.TLSCaCertificateSecretNamespacedName() - caData, err = secret.ReadStringData(secretGetter, caResourceName) + caData, err = secret.ReadStringData(ctx, secretGetter, caResourceName) } else if mdb.Spec.Security.TLS.CaConfigMap != nil { caResourceName = mdb.TLSConfigMapNamespacedName() - caData, err = configmap.ReadData(cmGetter, caResourceName) + caData, err = configmap.ReadData(ctx, cmGetter, caResourceName) } if err != nil { @@ -188,8 +189,8 @@ func getCaCrt(cmGetter configmap.Getter, secretGetter secret.Getter, mdb mdbv1.M // ensureCASecret will create or update the operator managed Secret containing // the CA certficate from the user provided Secret or ConfigMap. -func ensureCASecret(cmGetter configmap.Getter, secretGetter secret.Getter, getUpdateCreator secret.GetUpdateCreator, mdb mdbv1.MongoDBCommunity) error { - cert, err := getCaCrt(cmGetter, secretGetter, mdb) +func ensureCASecret(ctx context.Context, cmGetter configmap.Getter, secretGetter secret.Getter, getUpdateCreator secret.GetUpdateCreator, mdb mdbv1.MongoDBCommunity) error { + cert, err := getCaCrt(ctx, cmGetter, secretGetter, mdb) if err != nil { return err } @@ -203,13 +204,13 @@ func ensureCASecret(cmGetter configmap.Getter, secretGetter secret.Getter, getUp SetOwnerReferences(mdb.GetOwnerReferences()). Build() - return secret.CreateOrUpdate(getUpdateCreator, operatorSecret) + return secret.CreateOrUpdate(ctx, getUpdateCreator, operatorSecret) } // ensureTLSSecret will create or update the operator-managed Secret containing // the concatenated certificate and key from the user-provided Secret. -func ensureTLSSecret(getUpdateCreator secret.GetUpdateCreator, mdb mdbv1.MongoDBCommunity) error { - certKey, err := getPemOrConcatenatedCrtAndKey(getUpdateCreator, mdb, mdb.TLSSecretNamespacedName()) +func ensureTLSSecret(ctx context.Context, getUpdateCreator secret.GetUpdateCreator, mdb mdbv1.MongoDBCommunity) error { + certKey, err := getPemOrConcatenatedCrtAndKey(ctx, getUpdateCreator, mdb, mdb.TLSSecretNamespacedName()) if err != nil { return err } @@ -223,15 +224,15 @@ func ensureTLSSecret(getUpdateCreator secret.GetUpdateCreator, mdb mdbv1.MongoDB SetOwnerReferences(mdb.GetOwnerReferences()). Build() - return secret.CreateOrUpdate(getUpdateCreator, operatorSecret) + return secret.CreateOrUpdate(ctx, getUpdateCreator, operatorSecret) } -func ensureAgentCertSecret(getUpdateCreator secret.GetUpdateCreator, mdb mdbv1.MongoDBCommunity) error { +func ensureAgentCertSecret(ctx context.Context, getUpdateCreator secret.GetUpdateCreator, mdb mdbv1.MongoDBCommunity) error { if mdb.Spec.GetAgentAuthMode() != "X509" { return nil } - certKey, err := getPemOrConcatenatedCrtAndKey(getUpdateCreator, mdb, mdb.AgentCertificateSecretNamespacedName()) + certKey, err := getPemOrConcatenatedCrtAndKey(ctx, getUpdateCreator, mdb, mdb.AgentCertificateSecretNamespacedName()) if err != nil { return err } @@ -243,13 +244,13 @@ func ensureAgentCertSecret(getUpdateCreator secret.GetUpdateCreator, mdb mdbv1.M SetOwnerReferences(mdb.GetOwnerReferences()). Build() - return secret.CreateOrUpdate(getUpdateCreator, agentCertSecret) + return secret.CreateOrUpdate(ctx, getUpdateCreator, agentCertSecret) } // ensurePrometheusTLSSecret will create or update the operator-managed Secret containing // the concatenated certificate and key from the user-provided Secret. -func ensurePrometheusTLSSecret(getUpdateCreator secret.GetUpdateCreator, mdb mdbv1.MongoDBCommunity) error { - certKey, err := getPemOrConcatenatedCrtAndKey(getUpdateCreator, mdb, mdb.DeepCopy().PrometheusTLSSecretNamespacedName()) +func ensurePrometheusTLSSecret(ctx context.Context, getUpdateCreator secret.GetUpdateCreator, mdb mdbv1.MongoDBCommunity) error { + certKey, err := getPemOrConcatenatedCrtAndKey(ctx, getUpdateCreator, mdb, mdb.DeepCopy().PrometheusTLSSecretNamespacedName()) if err != nil { return err } @@ -263,7 +264,7 @@ func ensurePrometheusTLSSecret(getUpdateCreator secret.GetUpdateCreator, mdb mdb SetOwnerReferences(mdb.GetOwnerReferences()). Build() - return secret.CreateOrUpdate(getUpdateCreator, operatorSecret) + return secret.CreateOrUpdate(ctx, getUpdateCreator, operatorSecret) } // tlsOperatorSecretFileName calculates the file name to use for the mounted diff --git a/controllers/mongodb_tls_test.go b/controllers/mongodb_tls_test.go index 1e755b612..8724738a3 100644 --- a/controllers/mongodb_tls_test.go +++ b/controllers/mongodb_tls_test.go @@ -24,66 +24,68 @@ import ( ) func TestStatefulSetIsCorrectlyConfiguredWithTLS(t *testing.T) { + ctx := context.Background() mdb := newTestReplicaSetWithTLS() - mgr := kubeClient.NewManager(&mdb) + mgr := kubeClient.NewManager(ctx, &mdb) client := kubeClient.NewClient(mgr.GetClient()) - err := createTLSSecret(client, mdb, "CERT", "KEY", "") + err := createTLSSecret(ctx, client, mdb, "CERT", "KEY", "") assert.NoError(t, err) - err = createTLSConfigMap(client, mdb) + err = createTLSConfigMap(ctx, client, mdb) assert.NoError(t, err) r := NewReconciler(mgr) - res, err := r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) + res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) sts := appsv1.StatefulSet{} - err = mgr.GetClient().Get(context.TODO(), types.NamespacedName{Name: mdb.Name, Namespace: mdb.Namespace}, &sts) + err = mgr.GetClient().Get(ctx, types.NamespacedName{Name: mdb.Name, Namespace: mdb.Namespace}, &sts) assert.NoError(t, err) assertStatefulSetVolumesAndVolumeMounts(t, sts, mdb.TLSOperatorCASecretNamespacedName().Name, mdb.TLSOperatorSecretNamespacedName().Name, "", "") } func TestStatefulSetIsCorrectlyConfiguredWithTLSAndX509(t *testing.T) { + ctx := context.Background() mdb := newTestReplicaSetWithTLS() mdb.Spec.Security.Authentication.Modes = []mdbv1.AuthMode{"X509"} - mgr := kubeClient.NewManager(&mdb) + mgr := kubeClient.NewManager(ctx, &mdb) client := kubeClient.NewClient(mgr.GetClient()) - err := createTLSSecret(client, mdb, "CERT", "KEY", "") + err := createTLSSecret(ctx, client, mdb, "CERT", "KEY", "") assert.NoError(t, err) - err = createTLSConfigMap(client, mdb) + err = createTLSConfigMap(ctx, client, mdb) assert.NoError(t, err) crt, key, err := x509.CreateAgentCertificate() assert.NoError(t, err) - err = createAgentCertSecret(client, mdb, crt, key, "") + err = createAgentCertSecret(ctx, client, mdb, crt, key, "") assert.NoError(t, err) r := NewReconciler(mgr) - res, err := r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) + res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) sts := appsv1.StatefulSet{} - err = mgr.GetClient().Get(context.TODO(), types.NamespacedName{Name: mdb.Name, Namespace: mdb.Namespace}, &sts) + err = mgr.GetClient().Get(ctx, types.NamespacedName{Name: mdb.Name, Namespace: mdb.Namespace}, &sts) assert.NoError(t, err) // Check that the pem secret has been created s := corev1.Secret{} - err = mgr.GetClient().Get(context.TODO(), mdb.AgentCertificatePemSecretNamespacedName(), &s) + err = mgr.GetClient().Get(ctx, mdb.AgentCertificatePemSecretNamespacedName(), &s) assert.NoError(t, err) assertStatefulSetVolumesAndVolumeMounts(t, sts, mdb.TLSOperatorCASecretNamespacedName().Name, mdb.TLSOperatorSecretNamespacedName().Name, "", mdb.AgentCertificatePemSecretNamespacedName().Name) // If we deactivate X509 for the agent, we expect the certificates to be unmounted. mdb.Spec.Security.Authentication.Modes = []mdbv1.AuthMode{"SCRAM"} - err = mgr.GetClient().Update(context.TODO(), &mdb) + err = mgr.GetClient().Update(ctx, &mdb) assert.NoError(t, err) - res, err = r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) + res, err = r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) sts = appsv1.StatefulSet{} - err = mgr.GetClient().Get(context.TODO(), types.NamespacedName{Name: mdb.Name, Namespace: mdb.Namespace}, &sts) + err = mgr.GetClient().Get(ctx, types.NamespacedName{Name: mdb.Name, Namespace: mdb.Namespace}, &sts) assert.NoError(t, err) assertStatefulSetVolumesAndVolumeMounts(t, sts, mdb.TLSOperatorCASecretNamespacedName().Name, mdb.TLSOperatorSecretNamespacedName().Name, "", "") @@ -198,6 +200,7 @@ func assertStatefulSetVolumesAndVolumeMounts(t *testing.T, sts appsv1.StatefulSe } func TestStatefulSetIsCorrectlyConfiguredWithPrometheusTLS(t *testing.T) { + ctx := context.Background() mdb := newTestReplicaSetWithTLS() mdb.Spec.Prometheus = &mdbv1.Prometheus{ Username: "username", @@ -210,59 +213,58 @@ func TestStatefulSetIsCorrectlyConfiguredWithPrometheusTLS(t *testing.T) { }, } - mgr := kubeClient.NewManager(&mdb) + mgr := kubeClient.NewManager(ctx, &mdb) cli := kubeClient.NewClient(mgr.GetClient()) - err := secret.CreateOrUpdate(mgr.Client, - secret.Builder(). - SetName("prom-password-secret"). - SetNamespace(mdb.Namespace). - SetField("password", "my-password"). - Build(), - ) + err := secret.CreateOrUpdate(ctx, mgr.Client, secret.Builder(). + SetName("prom-password-secret"). + SetNamespace(mdb.Namespace). + SetField("password", "my-password"). + Build()) assert.NoError(t, err) - err = createTLSSecret(cli, mdb, "CERT", "KEY", "") + err = createTLSSecret(ctx, cli, mdb, "CERT", "KEY", "") assert.NoError(t, err) - err = createPrometheusTLSSecret(cli, mdb, "CERT", "KEY", "") + err = createPrometheusTLSSecret(ctx, cli, mdb, "CERT", "KEY", "") assert.NoError(t, err) - err = createTLSConfigMap(cli, mdb) + err = createTLSConfigMap(ctx, cli, mdb) assert.NoError(t, err) r := NewReconciler(mgr) - res, err := r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) + res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) sts := appsv1.StatefulSet{} - err = mgr.GetClient().Get(context.TODO(), types.NamespacedName{Name: mdb.Name, Namespace: mdb.Namespace}, &sts) + err = mgr.GetClient().Get(ctx, types.NamespacedName{Name: mdb.Name, Namespace: mdb.Namespace}, &sts) assert.NoError(t, err) assertStatefulSetVolumesAndVolumeMounts(t, sts, mdb.TLSOperatorCASecretNamespacedName().Name, mdb.TLSOperatorSecretNamespacedName().Name, mdb.PrometheusTLSOperatorSecretNamespacedName().Name, "") } func TestStatefulSetIsCorrectlyConfiguredWithTLSAfterChangingExistingVolumes(t *testing.T) { + ctx := context.Background() mdb := newTestReplicaSetWithTLS() - mgr := kubeClient.NewManager(&mdb) + mgr := kubeClient.NewManager(ctx, &mdb) cli := kubeClient.NewClient(mgr.GetClient()) - err := createTLSSecret(cli, mdb, "CERT", "KEY", "") + err := createTLSSecret(ctx, cli, mdb, "CERT", "KEY", "") assert.NoError(t, err) tlsCAVolumeSecretName := mdb.TLSOperatorCASecretNamespacedName().Name changedTLSCAVolumeSecretName := tlsCAVolumeSecretName + "-old" - err = createTLSSecretWithNamespaceAndName(cli, mdb.Namespace, changedTLSCAVolumeSecretName, "CERT", "KEY", "") + err = createTLSSecretWithNamespaceAndName(ctx, cli, mdb.Namespace, changedTLSCAVolumeSecretName, "CERT", "KEY", "") assert.NoError(t, err) - err = createTLSConfigMap(cli, mdb) + err = createTLSConfigMap(ctx, cli, mdb) assert.NoError(t, err) r := NewReconciler(mgr) - res, err := r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) + res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) sts := appsv1.StatefulSet{} - err = mgr.GetClient().Get(context.TODO(), types.NamespacedName{Name: mdb.Name, Namespace: mdb.Namespace}, &sts) + err = mgr.GetClient().Get(ctx, types.NamespacedName{Name: mdb.Name, Namespace: mdb.Namespace}, &sts) assert.NoError(t, err) assertStatefulSetVolumesAndVolumeMounts(t, sts, tlsCAVolumeSecretName, mdb.TLSOperatorSecretNamespacedName().Name, "", "") @@ -274,29 +276,30 @@ func TestStatefulSetIsCorrectlyConfiguredWithTLSAfterChangingExistingVolumes(t * } } - err = mgr.GetClient().Update(context.TODO(), &sts) + err = mgr.GetClient().Update(ctx, &sts) assert.NoError(t, err) assertStatefulSetVolumesAndVolumeMounts(t, sts, changedTLSCAVolumeSecretName, mdb.TLSOperatorSecretNamespacedName().Name, "", "") - res, err = r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) + res, err = r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) sts = appsv1.StatefulSet{} - err = mgr.GetClient().Get(context.TODO(), types.NamespacedName{Name: mdb.Name, Namespace: mdb.Namespace}, &sts) + err = mgr.GetClient().Get(ctx, types.NamespacedName{Name: mdb.Name, Namespace: mdb.Namespace}, &sts) assert.NoError(t, err) assertStatefulSetVolumesAndVolumeMounts(t, sts, tlsCAVolumeSecretName, mdb.TLSOperatorSecretNamespacedName().Name, "", "") } func TestAutomationConfigIsCorrectlyConfiguredWithTLS(t *testing.T) { + ctx := context.Background() createAC := func(mdb mdbv1.MongoDBCommunity) automationconfig.AutomationConfig { - client := kubeClient.NewClient(kubeClient.NewManager(&mdb).GetClient()) - err := createTLSSecret(client, mdb, "CERT", "KEY", "") + client := kubeClient.NewClient(kubeClient.NewManager(ctx, &mdb).GetClient()) + err := createTLSSecret(ctx, client, mdb, "CERT", "KEY", "") assert.NoError(t, err) - err = createTLSConfigMap(client, mdb) + err = createTLSConfigMap(ctx, client, mdb) assert.NoError(t, err) - tlsModification, err := getTLSConfigModification(client, client, mdb) + tlsModification, err := getTLSConfigModification(ctx, client, client, mdb) assert.NoError(t, err) ac, err := buildAutomationConfig(mdb, automationconfig.Auth{}, automationconfig.AutomationConfig{}, tlsModification) assert.NoError(t, err) @@ -370,33 +373,34 @@ func TestAutomationConfigIsCorrectlyConfiguredWithTLS(t *testing.T) { } func TestTLSOperatorSecret(t *testing.T) { + ctx := context.Background() t.Run("Secret is created if it doesn't exist", func(t *testing.T) { mdb := newTestReplicaSetWithTLS() - c := kubeClient.NewClient(kubeClient.NewManager(&mdb).GetClient()) - err := createTLSSecret(c, mdb, "CERT", "KEY", "") + c := kubeClient.NewClient(kubeClient.NewManager(ctx, &mdb).GetClient()) + err := createTLSSecret(ctx, c, mdb, "CERT", "KEY", "") assert.NoError(t, err) - err = createTLSConfigMap(c, mdb) + err = createTLSConfigMap(ctx, c, mdb) assert.NoError(t, err) r := NewReconciler(kubeClient.NewManagerWithClient(c)) - err = r.ensureTLSResources(mdb) + err = r.ensureTLSResources(ctx, mdb) assert.NoError(t, err) // Operator-managed secret should have been created and contains the // concatenated certificate and key. expectedCertificateKey := "CERT\nKEY" - certificateKey, err := secret.ReadKey(c, tlsOperatorSecretFileName(expectedCertificateKey), mdb.TLSOperatorSecretNamespacedName()) + certificateKey, err := secret.ReadKey(ctx, c, tlsOperatorSecretFileName(expectedCertificateKey), mdb.TLSOperatorSecretNamespacedName()) assert.NoError(t, err) assert.Equal(t, expectedCertificateKey, certificateKey) }) t.Run("Secret is updated if it already exists", func(t *testing.T) { mdb := newTestReplicaSetWithTLS() - k8sclient := kubeClient.NewClient(kubeClient.NewManager(&mdb).GetClient()) - err := createTLSSecret(k8sclient, mdb, "CERT", "KEY", "") + k8sclient := kubeClient.NewClient(kubeClient.NewManager(ctx, &mdb).GetClient()) + err := createTLSSecret(ctx, k8sclient, mdb, "CERT", "KEY", "") assert.NoError(t, err) - err = createTLSConfigMap(k8sclient, mdb) + err = createTLSConfigMap(ctx, k8sclient, mdb) assert.NoError(t, err) // Create operator-managed secret @@ -405,18 +409,18 @@ func TestTLSOperatorSecret(t *testing.T) { SetNamespace(mdb.TLSOperatorSecretNamespacedName().Namespace). SetField(tlsOperatorSecretFileName(""), ""). Build() - err = k8sclient.CreateSecret(s) + err = k8sclient.CreateSecret(ctx, s) assert.NoError(t, err) r := NewReconciler(kubeClient.NewManagerWithClient(k8sclient)) - err = r.ensureTLSResources(mdb) + err = r.ensureTLSResources(ctx, mdb) assert.NoError(t, err) // Operator-managed secret should have been updated with the concatenated // certificate and key. expectedCertificateKey := "CERT\nKEY" - certificateKey, err := secret.ReadKey(k8sclient, tlsOperatorSecretFileName(expectedCertificateKey), mdb.TLSOperatorSecretNamespacedName()) + certificateKey, err := secret.ReadKey(ctx, k8sclient, tlsOperatorSecretFileName(expectedCertificateKey), mdb.TLSOperatorSecretNamespacedName()) assert.NoError(t, err) assert.Equal(t, expectedCertificateKey, certificateKey) }) @@ -442,63 +446,65 @@ func TestCombineCertificateAndKey(t *testing.T) { } func TestPemSupport(t *testing.T) { + ctx := context.Background() t.Run("Success if only pem is provided", func(t *testing.T) { mdb := newTestReplicaSetWithTLS() - c := kubeClient.NewClient(kubeClient.NewManager(&mdb).GetClient()) - err := createTLSSecret(c, mdb, "", "", "CERT\nKEY") + c := kubeClient.NewClient(kubeClient.NewManager(ctx, &mdb).GetClient()) + err := createTLSSecret(ctx, c, mdb, "", "", "CERT\nKEY") assert.NoError(t, err) - err = createTLSConfigMap(c, mdb) + err = createTLSConfigMap(ctx, c, mdb) assert.NoError(t, err) r := NewReconciler(kubeClient.NewManagerWithClient(c)) - err = r.ensureTLSResources(mdb) + err = r.ensureTLSResources(ctx, mdb) assert.NoError(t, err) // Operator-managed secret should have been created and contains the // concatenated certificate and key. expectedCertificateKey := "CERT\nKEY" - certificateKey, err := secret.ReadKey(c, tlsOperatorSecretFileName(expectedCertificateKey), mdb.TLSOperatorSecretNamespacedName()) + certificateKey, err := secret.ReadKey(ctx, c, tlsOperatorSecretFileName(expectedCertificateKey), mdb.TLSOperatorSecretNamespacedName()) assert.NoError(t, err) assert.Equal(t, expectedCertificateKey, certificateKey) }) t.Run("Success if pem is equal to cert+key", func(t *testing.T) { mdb := newTestReplicaSetWithTLS() - c := kubeClient.NewClient(kubeClient.NewManager(&mdb).GetClient()) - err := createTLSSecret(c, mdb, "CERT", "KEY", "CERT\nKEY") + c := kubeClient.NewClient(kubeClient.NewManager(ctx, &mdb).GetClient()) + err := createTLSSecret(ctx, c, mdb, "CERT", "KEY", "CERT\nKEY") assert.NoError(t, err) - err = createTLSConfigMap(c, mdb) + err = createTLSConfigMap(ctx, c, mdb) assert.NoError(t, err) r := NewReconciler(kubeClient.NewManagerWithClient(c)) - err = r.ensureTLSResources(mdb) + err = r.ensureTLSResources(ctx, mdb) assert.NoError(t, err) // Operator-managed secret should have been created and contains the // concatenated certificate and key. expectedCertificateKey := "CERT\nKEY" - certificateKey, err := secret.ReadKey(c, tlsOperatorSecretFileName(expectedCertificateKey), mdb.TLSOperatorSecretNamespacedName()) + certificateKey, err := secret.ReadKey(ctx, c, tlsOperatorSecretFileName(expectedCertificateKey), mdb.TLSOperatorSecretNamespacedName()) assert.NoError(t, err) assert.Equal(t, expectedCertificateKey, certificateKey) }) t.Run("Failure if pem is different from cert+key", func(t *testing.T) { mdb := newTestReplicaSetWithTLS() - c := kubeClient.NewClient(kubeClient.NewManager(&mdb).GetClient()) - err := createTLSSecret(c, mdb, "CERT1", "KEY1", "CERT\nKEY") + c := kubeClient.NewClient(kubeClient.NewManager(ctx, &mdb).GetClient()) + err := createTLSSecret(ctx, c, mdb, "CERT1", "KEY1", "CERT\nKEY") assert.NoError(t, err) - err = createTLSConfigMap(c, mdb) + err = createTLSConfigMap(ctx, c, mdb) assert.NoError(t, err) r := NewReconciler(kubeClient.NewManagerWithClient(c)) - err = r.ensureTLSResources(mdb) + err = r.ensureTLSResources(ctx, mdb) assert.Error(t, err) assert.Contains(t, err.Error(), `if all of "tls.crt", "tls.key" and "tls.pem" are present in the secret, the entry for "tls.pem" must be equal to the concatenation of "tls.crt" with "tls.key"`) }) } func TestTLSConfigReferencesToCACertAreValidated(t *testing.T) { + ctx := context.Background() type args struct { caConfigMap *corev1.LocalObjectReference caCertificateSecret *corev1.LocalObjectReference @@ -531,15 +537,15 @@ func TestTLSConfigReferencesToCACertAreValidated(t *testing.T) { t.Run(testName, func(t *testing.T) { mdb := newTestReplicaSetWithTLSCaCertificateReferences(tc.caConfigMap, tc.caCertificateSecret) - mgr := kubeClient.NewManager(&mdb) + mgr := kubeClient.NewManager(ctx, &mdb) cli := kubeClient.NewClient(mgr.GetClient()) - err := createTLSSecret(cli, mdb, "cert", "key", "pem") + err := createTLSSecret(ctx, cli, mdb, "cert", "key", "pem") assert.NoError(t, err) r := NewReconciler(mgr) - _, err = r.validateTLSConfig(mdb) + _, err = r.validateTLSConfig(ctx, mdb) if tc.expectedError != nil { assert.EqualError(t, err, tc.expectedError.Error()) } else { @@ -550,7 +556,7 @@ func TestTLSConfigReferencesToCACertAreValidated(t *testing.T) { } -func createTLSConfigMap(c k8sClient.Client, mdb mdbv1.MongoDBCommunity) error { +func createTLSConfigMap(ctx context.Context, c k8sClient.Client, mdb mdbv1.MongoDBCommunity) error { if !mdb.Spec.Security.TLS.Enabled { return nil } @@ -561,10 +567,10 @@ func createTLSConfigMap(c k8sClient.Client, mdb mdbv1.MongoDBCommunity) error { SetDataField("ca.crt", "CERT"). Build() - return c.Create(context.TODO(), &configMap) + return c.Create(ctx, &configMap) } -func createTLSSecretWithNamespaceAndName(c k8sClient.Client, namespace string, name string, crt string, key string, pem string) error { +func createTLSSecretWithNamespaceAndName(ctx context.Context, c k8sClient.Client, namespace string, name string, crt string, key string, pem string) error { sBuilder := secret.Builder(). SetName(name). SetNamespace(namespace). @@ -581,31 +587,31 @@ func createTLSSecretWithNamespaceAndName(c k8sClient.Client, namespace string, n } s := sBuilder.Build() - return c.Create(context.TODO(), &s) + return c.Create(ctx, &s) } -func createTLSSecret(c k8sClient.Client, mdb mdbv1.MongoDBCommunity, crt string, key string, pem string) error { - return createTLSSecretWithNamespaceAndName(c, mdb.Namespace, mdb.Spec.Security.TLS.CertificateKeySecret.Name, crt, key, pem) +func createTLSSecret(ctx context.Context, c k8sClient.Client, mdb mdbv1.MongoDBCommunity, crt string, key string, pem string) error { + return createTLSSecretWithNamespaceAndName(ctx, c, mdb.Namespace, mdb.Spec.Security.TLS.CertificateKeySecret.Name, crt, key, pem) } -func createAgentCertSecret(c k8sClient.Client, mdb mdbv1.MongoDBCommunity, crt string, key string, pem string) error { - return createTLSSecretWithNamespaceAndName(c, mdb.Namespace, mdb.AgentCertificateSecretNamespacedName().Name, crt, key, pem) +func createAgentCertSecret(ctx context.Context, c k8sClient.Client, mdb mdbv1.MongoDBCommunity, crt string, key string, pem string) error { + return createTLSSecretWithNamespaceAndName(ctx, c, mdb.Namespace, mdb.AgentCertificateSecretNamespacedName().Name, crt, key, pem) } -func createAgentCertPemSecret(c k8sClient.Client, mdb mdbv1.MongoDBCommunity, crt string, key string, pem string) error { - return createTLSSecretWithNamespaceAndName(c, mdb.Namespace, mdb.AgentCertificatePemSecretNamespacedName().Name, crt, key, pem) +func createAgentCertPemSecret(ctx context.Context, c k8sClient.Client, mdb mdbv1.MongoDBCommunity, crt string, key string, pem string) error { + return createTLSSecretWithNamespaceAndName(ctx, c, mdb.Namespace, mdb.AgentCertificatePemSecretNamespacedName().Name, crt, key, pem) } -func createPrometheusTLSSecret(c k8sClient.Client, mdb mdbv1.MongoDBCommunity, crt string, key string, pem string) error { - return createTLSSecretWithNamespaceAndName(c, mdb.Namespace, mdb.Spec.Prometheus.TLSSecretRef.Name, crt, key, pem) +func createPrometheusTLSSecret(ctx context.Context, c k8sClient.Client, mdb mdbv1.MongoDBCommunity, crt string, key string, pem string) error { + return createTLSSecretWithNamespaceAndName(ctx, c, mdb.Namespace, mdb.Spec.Prometheus.TLSSecretRef.Name, crt, key, pem) } -func createUserPasswordSecret(c k8sClient.Client, mdb mdbv1.MongoDBCommunity, userPasswordSecretName string, password string) error { +func createUserPasswordSecret(ctx context.Context, c k8sClient.Client, mdb mdbv1.MongoDBCommunity, userPasswordSecretName string, password string) error { sBuilder := secret.Builder(). SetName(userPasswordSecretName). SetNamespace(mdb.Namespace). SetField("password", password) s := sBuilder.Build() - return c.Create(context.TODO(), &s) + return c.Create(ctx, &s) } diff --git a/controllers/mongodb_users.go b/controllers/mongodb_users.go index 824dff2e7..2af312d2c 100644 --- a/controllers/mongodb_users.go +++ b/controllers/mongodb_users.go @@ -1,6 +1,7 @@ package controllers import ( + "context" "fmt" mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" @@ -12,15 +13,15 @@ import ( // ensureUserResources will check that the configured user password secrets can be found // and will start monitor them so that the reconcile process is triggered every time these secrets are updated -func (r ReplicaSetReconciler) ensureUserResources(mdb mdbv1.MongoDBCommunity) error { +func (r ReplicaSetReconciler) ensureUserResources(ctx context.Context, mdb mdbv1.MongoDBCommunity) error { for _, user := range mdb.GetAuthUsers() { if user.Database != constants.ExternalDB { secretNamespacedName := types.NamespacedName{Name: user.PasswordSecretName, Namespace: mdb.Namespace} - if _, err := secret.ReadKey(r.client, user.PasswordSecretKey, secretNamespacedName); err != nil { + if _, err := secret.ReadKey(ctx, r.client, user.PasswordSecretKey, secretNamespacedName); err != nil { if apiErrors.IsNotFound(err) { // check for SCRAM secret as well scramSecretName := types.NamespacedName{Name: user.ScramCredentialsSecretName, Namespace: mdb.Namespace} - _, err = r.client.GetSecret(scramSecretName) + _, err = r.client.GetSecret(ctx, scramSecretName) if apiErrors.IsNotFound(err) { return fmt.Errorf(`user password secret: %s and scram secret: %s not found`, secretNamespacedName, scramSecretName) } @@ -29,7 +30,7 @@ func (r ReplicaSetReconciler) ensureUserResources(mdb mdbv1.MongoDBCommunity) er } return err } - r.secretWatcher.Watch(secretNamespacedName, mdb.NamespacedName()) + r.secretWatcher.Watch(ctx, secretNamespacedName, mdb.NamespacedName()) } } @@ -38,7 +39,7 @@ func (r ReplicaSetReconciler) ensureUserResources(mdb mdbv1.MongoDBCommunity) er // updateConnectionStringSecrets updates secrets where user specific connection strings are stored. // The client applications can mount these secrets and connect to the mongodb cluster -func (r ReplicaSetReconciler) updateConnectionStringSecrets(mdb mdbv1.MongoDBCommunity, clusterDomain string) error { +func (r ReplicaSetReconciler) updateConnectionStringSecrets(ctx context.Context, mdb mdbv1.MongoDBCommunity, clusterDomain string) error { for _, user := range mdb.GetAuthUsers() { secretName := user.ConnectionStringSecretName @@ -47,7 +48,7 @@ func (r ReplicaSetReconciler) updateConnectionStringSecrets(mdb mdbv1.MongoDBCom secretNamespace = user.ConnectionStringSecretNamespace } - existingSecret, err := r.client.GetSecret(types.NamespacedName{ + existingSecret, err := r.client.GetSecret(ctx, types.NamespacedName{ Name: secretName, Namespace: secretNamespace, }) @@ -62,7 +63,7 @@ func (r ReplicaSetReconciler) updateConnectionStringSecrets(mdb mdbv1.MongoDBCom if user.Database != constants.ExternalDB { secretNamespacedName := types.NamespacedName{Name: user.PasswordSecretName, Namespace: secretNamespace} - pwd, err = secret.ReadKey(r.client, user.PasswordSecretKey, secretNamespacedName) + pwd, err = secret.ReadKey(ctx, r.client, user.PasswordSecretKey, secretNamespacedName) if err != nil { return err } @@ -78,7 +79,7 @@ func (r ReplicaSetReconciler) updateConnectionStringSecrets(mdb mdbv1.MongoDBCom SetOwnerReferences(mdb.GetOwnerReferences()). Build() - if err := secret.CreateOrUpdate(r.client, connectionStringSecret); err != nil { + if err := secret.CreateOrUpdate(ctx, r.client, connectionStringSecret); err != nil { return err } } diff --git a/controllers/prometheus.go b/controllers/prometheus.go index 8e6151908..165fe72a0 100644 --- a/controllers/prometheus.go +++ b/controllers/prometheus.go @@ -1,6 +1,7 @@ package controllers import ( + "context" "fmt" corev1 "k8s.io/api/core/v1" @@ -19,13 +20,13 @@ const ( ) // PrometheusModification adds Prometheus configuration to AutomationConfig. -func getPrometheusModification(getUpdateCreator secret.GetUpdateCreator, mdb mdbv1.MongoDBCommunity) (automationconfig.Modification, error) { +func getPrometheusModification(ctx context.Context, getUpdateCreator secret.GetUpdateCreator, mdb mdbv1.MongoDBCommunity) (automationconfig.Modification, error) { if mdb.Spec.Prometheus == nil { return automationconfig.NOOP(), nil } secretNamespacedName := types.NamespacedName{Name: mdb.Spec.Prometheus.PasswordSecretRef.Name, Namespace: mdb.Namespace} - password, err := secret.ReadKey(getUpdateCreator, mdb.Spec.Prometheus.GetPasswordKey(), secretNamespacedName) + password, err := secret.ReadKey(ctx, getUpdateCreator, mdb.Spec.Prometheus.GetPasswordKey(), secretNamespacedName) if err != nil { return automationconfig.NOOP(), fmt.Errorf("could not configure Prometheus modification: %s", err) } @@ -35,7 +36,7 @@ func getPrometheusModification(getUpdateCreator secret.GetUpdateCreator, mdb mdb var scheme string if mdb.Spec.Prometheus.TLSSecretRef.Name != "" { - certKey, err = getPemOrConcatenatedCrtAndKey(getUpdateCreator, mdb, mdb.PrometheusTLSSecretNamespacedName()) + certKey, err = getPemOrConcatenatedCrtAndKey(ctx, getUpdateCreator, mdb, mdb.PrometheusTLSSecretNamespacedName()) if err != nil { return automationconfig.NOOP(), err } diff --git a/controllers/replica_set_controller.go b/controllers/replica_set_controller.go index 992f5175e..2d9355a57 100644 --- a/controllers/replica_set_controller.go +++ b/controllers/replica_set_controller.go @@ -8,22 +8,6 @@ import ( "strconv" "strings" - "go.uber.org/zap" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - apiErrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/builder" - k8sClient "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - "sigs.k8s.io/controller-runtime/pkg/manager" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/controller-runtime/pkg/source" - "github.com/imdario/mergo" mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" "github.com/mongodb/mongodb-kubernetes-operator/controllers/construct" @@ -45,6 +29,20 @@ import ( "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/scale" "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/status" "github.com/stretchr/objx" + "go.uber.org/zap" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apiErrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + k8sClient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" ) const ( @@ -80,8 +78,8 @@ func (r *ReplicaSetReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). WithOptions(controller.Options{MaxConcurrentReconciles: 3}). For(&mdbv1.MongoDBCommunity{}, builder.WithPredicates(predicates.OnlyOnSpecChange())). - Watches(&source.Kind{Type: &corev1.Secret{}}, r.secretWatcher). - Watches(&source.Kind{Type: &corev1.ConfigMap{}}, r.configMapWatcher). + Watches(&corev1.Secret{}, r.secretWatcher). + Watches(&corev1.ConfigMap{}, r.configMapWatcher). Owns(&appsv1.StatefulSet{}). Complete(r) } @@ -113,7 +111,7 @@ func (r ReplicaSetReconciler) Reconcile(ctx context.Context, request reconcile.R // TODO: generalize preparation for resource // Fetch the MongoDB instance mdb := mdbv1.MongoDBCommunity{} - err := r.client.Get(context.TODO(), request.NamespacedName, &mdb) + err := r.client.Get(ctx, request.NamespacedName, &mdb) if err != nil { if apiErrors.IsNotFound(err) { // Request object not found, could have been deleted after reconcile request. @@ -132,127 +130,104 @@ func (r ReplicaSetReconciler) Reconcile(ctx context.Context, request reconcile.R r.log.Debug("Validating MongoDB.Spec") err, lastAppliedSpec := r.validateSpec(mdb) if err != nil { - return status.Update(r.client.Status(), &mdb, - statusOptions(). - withMessage(Error, fmt.Sprintf("error validating new Spec: %s", err)). - withFailedPhase(), - ) + return status.Update(ctx, r.client.Status(), &mdb, statusOptions(). + withMessage(Error, fmt.Sprintf("error validating new Spec: %s", err)). + withFailedPhase()) } r.log.Debug("Ensuring the service exists") - if err := r.ensureService(mdb); err != nil { - return status.Update(r.client.Status(), &mdb, - statusOptions(). - withMessage(Error, fmt.Sprintf("Error ensuring the service (members) exists: %s", err)). - withFailedPhase(), - ) + if err := r.ensureService(ctx, mdb); err != nil { + return status.Update(ctx, r.client.Status(), &mdb, statusOptions(). + withMessage(Error, fmt.Sprintf("Error ensuring the service (members) exists: %s", err)). + withFailedPhase()) } - isTLSValid, err := r.validateTLSConfig(mdb) + isTLSValid, err := r.validateTLSConfig(ctx, mdb) if err != nil { - return status.Update(r.client.Status(), &mdb, - statusOptions(). - withMessage(Error, fmt.Sprintf("Error validating TLS config: %s", err)). - withFailedPhase(), - ) + return status.Update(ctx, r.client.Status(), &mdb, statusOptions(). + withMessage(Error, fmt.Sprintf("Error validating TLS config: %s", err)). + withFailedPhase()) } if !isTLSValid { - return status.Update(r.client.Status(), &mdb, - statusOptions(). - withMessage(Info, "TLS config is not yet valid, retrying in 10 seconds"). - withPendingPhase(10), - ) + return status.Update(ctx, r.client.Status(), &mdb, statusOptions(). + withMessage(Info, "TLS config is not yet valid, retrying in 10 seconds"). + withPendingPhase(10)) } - if err := r.ensureTLSResources(mdb); err != nil { - return status.Update(r.client.Status(), &mdb, - statusOptions(). - withMessage(Error, fmt.Sprintf("Error ensuring TLS resources: %s", err)). - withFailedPhase(), - ) + if err := r.ensureTLSResources(ctx, mdb); err != nil { + return status.Update(ctx, r.client.Status(), &mdb, statusOptions(). + withMessage(Error, fmt.Sprintf("Error ensuring TLS resources: %s", err)). + withFailedPhase()) } - if err := r.ensurePrometheusTLSResources(mdb); err != nil { - return status.Update(r.client.Status(), &mdb, - statusOptions(). - withMessage(Error, fmt.Sprintf("Error ensuring TLS resources: %s", err)). - withFailedPhase(), - ) + if err := r.ensurePrometheusTLSResources(ctx, mdb); err != nil { + return status.Update(ctx, r.client.Status(), &mdb, statusOptions(). + withMessage(Error, fmt.Sprintf("Error ensuring TLS resources: %s", err)). + withFailedPhase()) } - if err := r.ensureUserResources(mdb); err != nil { - return status.Update(r.client.Status(), &mdb, - statusOptions(). - withMessage(Error, fmt.Sprintf("Error ensuring User config: %s", err)). - withFailedPhase(), - ) + if err := r.ensureUserResources(ctx, mdb); err != nil { + return status.Update(ctx, r.client.Status(), &mdb, statusOptions(). + withMessage(Error, fmt.Sprintf("Error ensuring User config: %s", err)). + withFailedPhase()) } - ready, err := r.deployMongoDBReplicaSet(mdb) + ready, err := r.deployMongoDBReplicaSet(ctx, mdb) if err != nil { - return status.Update(r.client.Status(), &mdb, - statusOptions(). - withMessage(Error, fmt.Sprintf("Error deploying MongoDB ReplicaSet: %s", err)). - withFailedPhase(), - ) + return status.Update(ctx, r.client.Status(), &mdb, statusOptions(). + withMessage(Error, fmt.Sprintf("Error deploying MongoDB ReplicaSet: %s", err)). + withFailedPhase()) } if !ready { - return status.Update(r.client.Status(), &mdb, - statusOptions(). - withMessage(Info, "ReplicaSet is not yet ready, retrying in 10 seconds"). - withPendingPhase(10), - ) + return status.Update(ctx, r.client.Status(), &mdb, statusOptions(). + withMessage(Info, "ReplicaSet is not yet ready, retrying in 10 seconds"). + withPendingPhase(10)) } r.log.Debug("Resetting StatefulSet UpdateStrategy to RollingUpdate") - if err := statefulset.ResetUpdateStrategy(&mdb, r.client); err != nil { - return status.Update(r.client.Status(), &mdb, - statusOptions(). - withMessage(Error, fmt.Sprintf("Error resetting StatefulSet UpdateStrategyType: %s", err)). - withFailedPhase(), - ) + if err := statefulset.ResetUpdateStrategy(ctx, &mdb, r.client); err != nil { + return status.Update(ctx, r.client.Status(), &mdb, statusOptions(). + withMessage(Error, fmt.Sprintf("Error resetting StatefulSet UpdateStrategyType: %s", err)). + withFailedPhase()) } if mdb.IsStillScaling() { - return status.Update(r.client.Status(), &mdb, statusOptions(). + return status.Update(ctx, r.client.Status(), &mdb, statusOptions(). withMongoDBMembers(mdb.AutomationConfigMembersThisReconciliation()). withMessage(Info, fmt.Sprintf("Performing scaling operation, currentMembers=%d, desiredMembers=%d", mdb.CurrentReplicas(), mdb.DesiredReplicas())). withStatefulSetReplicas(mdb.StatefulSetReplicasThisReconciliation()). withStatefulSetArbiters(mdb.StatefulSetArbitersThisReconciliation()). withMongoDBArbiters(mdb.AutomationConfigArbitersThisReconciliation()). - withPendingPhase(10), - ) - } - - res, err := status.Update(r.client.Status(), &mdb, - statusOptions(). - withMongoURI(mdb.MongoURI(os.Getenv(clusterDomain))). - withMongoDBMembers(mdb.AutomationConfigMembersThisReconciliation()). - withStatefulSetReplicas(mdb.StatefulSetReplicasThisReconciliation()). - withStatefulSetArbiters(mdb.StatefulSetArbitersThisReconciliation()). - withMongoDBArbiters(mdb.AutomationConfigArbitersThisReconciliation()). - withMessage(None, ""). - withRunningPhase(). - withVersion(mdb.GetMongoDBVersion(nil)), - ) + withPendingPhase(10)) + } + + res, err := status.Update(ctx, r.client.Status(), &mdb, statusOptions(). + withMongoURI(mdb.MongoURI(os.Getenv(clusterDomain))). + withMongoDBMembers(mdb.AutomationConfigMembersThisReconciliation()). + withStatefulSetReplicas(mdb.StatefulSetReplicasThisReconciliation()). + withStatefulSetArbiters(mdb.StatefulSetArbitersThisReconciliation()). + withMongoDBArbiters(mdb.AutomationConfigArbitersThisReconciliation()). + withMessage(None, ""). + withRunningPhase(). + withVersion(mdb.GetMongoDBVersion(nil))) if err != nil { r.log.Errorf("Error updating the status of the MongoDB resource: %s", err) return res, err } - if err := r.updateConnectionStringSecrets(mdb, os.Getenv(clusterDomain)); err != nil { + if err := r.updateConnectionStringSecrets(ctx, mdb, os.Getenv(clusterDomain)); err != nil { r.log.Errorf("Could not update connection string secrets: %s", err) } if lastAppliedSpec != nil { - r.cleanupScramSecrets(mdb.Spec, *lastAppliedSpec, mdb.Namespace) - r.cleanupPemSecret(mdb.Spec, *lastAppliedSpec, mdb.Namespace) + r.cleanupScramSecrets(ctx, mdb.Spec, *lastAppliedSpec, mdb.Namespace) + r.cleanupPemSecret(ctx, mdb.Spec, *lastAppliedSpec, mdb.Namespace) } - if err := r.updateLastSuccessfulConfiguration(mdb); err != nil { + if err := r.updateLastSuccessfulConfiguration(ctx, mdb); err != nil { r.log.Errorf("Could not save current spec as an annotation: %s", err) } @@ -266,7 +241,7 @@ func (r ReplicaSetReconciler) Reconcile(ctx context.Context, request reconcile.R } // updateLastSuccessfulConfiguration annotates the MongoDBCommunity resource with the latest configuration -func (r *ReplicaSetReconciler) updateLastSuccessfulConfiguration(mdb mdbv1.MongoDBCommunity) error { +func (r *ReplicaSetReconciler) updateLastSuccessfulConfiguration(ctx context.Context, mdb mdbv1.MongoDBCommunity) error { currentSpec, err := json.Marshal(mdb.Spec) if err != nil { return err @@ -278,12 +253,12 @@ func (r *ReplicaSetReconciler) updateLastSuccessfulConfiguration(mdb mdbv1.Mongo // This is needed to reuse the update strategy logic in enterprise lastAppliedMongoDBVersion: mdb.Spec.Version, } - return annotations.SetAnnotations(&mdb, specAnnotations, r.client) + return annotations.SetAnnotations(ctx, &mdb, specAnnotations, r.client) } // ensureTLSResources creates any required TLS resources that the MongoDBCommunity // requires for TLS configuration. -func (r *ReplicaSetReconciler) ensureTLSResources(mdb mdbv1.MongoDBCommunity) error { +func (r *ReplicaSetReconciler) ensureTLSResources(ctx context.Context, mdb mdbv1.MongoDBCommunity) error { if !mdb.Spec.Security.TLS.Enabled { return nil } @@ -291,16 +266,16 @@ func (r *ReplicaSetReconciler) ensureTLSResources(mdb mdbv1.MongoDBCommunity) er // require the contents. if mdb.Spec.Security.TLS.Enabled { r.log.Infof("TLS is enabled, creating/updating CA secret") - if err := ensureCASecret(r.client, r.client, r.client, mdb); err != nil { + if err := ensureCASecret(ctx, r.client, r.client, r.client, mdb); err != nil { return fmt.Errorf("could not ensure CA secret: %s", err) } r.log.Infof("TLS is enabled, creating/updating TLS secret") - if err := ensureTLSSecret(r.client, mdb); err != nil { + if err := ensureTLSSecret(ctx, r.client, mdb); err != nil { return fmt.Errorf("could not ensure TLS secret: %s", err) } if mdb.Spec.IsAgentX509() { r.log.Infof("Agent X509 authentication is enabled, creating/updating agent certificate secret") - if err := ensureAgentCertSecret(r.client, mdb); err != nil { + if err := ensureAgentCertSecret(ctx, r.client, mdb); err != nil { return fmt.Errorf("could not ensure Agent Certificate secret: %s", err) } } @@ -310,7 +285,7 @@ func (r *ReplicaSetReconciler) ensureTLSResources(mdb mdbv1.MongoDBCommunity) er // ensurePrometheusTLSResources creates any required TLS resources that the MongoDBCommunity // requires for TLS configuration. -func (r *ReplicaSetReconciler) ensurePrometheusTLSResources(mdb mdbv1.MongoDBCommunity) error { +func (r *ReplicaSetReconciler) ensurePrometheusTLSResources(ctx context.Context, mdb mdbv1.MongoDBCommunity) error { if mdb.Spec.Prometheus == nil || mdb.Spec.Prometheus.TLSSecretRef.Name == "" { return nil } @@ -318,7 +293,7 @@ func (r *ReplicaSetReconciler) ensurePrometheusTLSResources(mdb mdbv1.MongoDBCom // the TLS secret needs to be created beforehand, as both the StatefulSet and AutomationConfig // require the contents. r.log.Infof("Prometheus TLS is enabled, creating/updating TLS secret") - if err := ensurePrometheusTLSSecret(r.client, mdb); err != nil { + if err := ensurePrometheusTLSSecret(ctx, r.client, mdb); err != nil { return fmt.Errorf("could not ensure TLS secret: %s", err) } @@ -331,18 +306,18 @@ func (r *ReplicaSetReconciler) ensurePrometheusTLSResources(mdb mdbv1.MongoDBCom // of Pods corresponding to the amount of expected arbiters. // // The returned boolean indicates that the StatefulSet is ready. -func (r *ReplicaSetReconciler) deployStatefulSet(mdb mdbv1.MongoDBCommunity) (bool, error) { +func (r *ReplicaSetReconciler) deployStatefulSet(ctx context.Context, mdb mdbv1.MongoDBCommunity) (bool, error) { r.log.Info("Creating/Updating StatefulSet") - if err := r.createOrUpdateStatefulSet(mdb, false); err != nil { + if err := r.createOrUpdateStatefulSet(ctx, mdb, false); err != nil { return false, fmt.Errorf("error creating/updating StatefulSet: %s", err) } r.log.Info("Creating/Updating StatefulSet for Arbiters") - if err := r.createOrUpdateStatefulSet(mdb, true); err != nil { + if err := r.createOrUpdateStatefulSet(ctx, mdb, true); err != nil { return false, fmt.Errorf("error creating/updating StatefulSet: %s", err) } - currentSts, err := r.client.GetStatefulSet(mdb.NamespacedName()) + currentSts, err := r.client.GetStatefulSet(ctx, mdb.NamespacedName()) if err != nil { return false, fmt.Errorf("error getting StatefulSet: %s", err) } @@ -356,15 +331,15 @@ func (r *ReplicaSetReconciler) deployStatefulSet(mdb mdbv1.MongoDBCommunity) (bo // deployAutomationConfig deploys the AutomationConfig for the MongoDBCommunity resource. // The returned boolean indicates whether or not that Agents have all reached goal state. -func (r *ReplicaSetReconciler) deployAutomationConfig(mdb mdbv1.MongoDBCommunity) (bool, error) { +func (r *ReplicaSetReconciler) deployAutomationConfig(ctx context.Context, mdb mdbv1.MongoDBCommunity) (bool, error) { r.log.Infof("Creating/Updating AutomationConfig") - sts, err := r.client.GetStatefulSet(mdb.NamespacedName()) + sts, err := r.client.GetStatefulSet(ctx, mdb.NamespacedName()) if err != nil && !apiErrors.IsNotFound(err) { return false, fmt.Errorf("failed to get StatefulSet: %s", err) } - ac, err := r.ensureAutomationConfig(mdb) + ac, err := r.ensureAutomationConfig(mdb, ctx) if err != nil { return false, fmt.Errorf("failed to ensure AutomationConfig: %s", err) } @@ -383,7 +358,7 @@ func (r *ReplicaSetReconciler) deployAutomationConfig(mdb mdbv1.MongoDBCommunity r.log.Debugf("Waiting for agents to reach version %d", ac.Version) // Note: we pass in the expected number of replicas this reconciliation as we scale members one at a time. If we were // to pass in the final member count, we would be waiting for agents that do not exist yet to be ready. - ready, err := agent.AllReachedGoalState(sts, r.client, mdb.StatefulSetReplicasThisReconciliation(), ac.Version, r.log) + ready, err := agent.AllReachedGoalState(ctx, sts, r.client, mdb.StatefulSetReplicasThisReconciliation(), ac.Version, r.log) if err != nil { return false, fmt.Errorf("failed to ensure agents have reached goal state: %s", err) } @@ -393,9 +368,9 @@ func (r *ReplicaSetReconciler) deployAutomationConfig(mdb mdbv1.MongoDBCommunity // shouldRunInOrder returns true if the order of execution of the AutomationConfig & StatefulSet // functions should be sequential or not. A value of false indicates they will run in reversed order. -func (r *ReplicaSetReconciler) shouldRunInOrder(mdb mdbv1.MongoDBCommunity) bool { +func (r *ReplicaSetReconciler) shouldRunInOrder(ctx context.Context, mdb mdbv1.MongoDBCommunity) bool { // The only case when we push the StatefulSet first is when we are ensuring TLS for the already existing ReplicaSet - sts, err := r.client.GetStatefulSet(mdb.NamespacedName()) + sts, err := r.client.GetStatefulSet(ctx, mdb.NamespacedName()) if !statefulset.IsReady(sts, mdb.StatefulSetReplicasThisReconciliation()) && mdb.Spec.Security.TLS.Enabled { r.log.Debug("Enabling TLS on a deployment with a StatefulSet that is not Ready, the Automation Config must be updated first") return true @@ -433,13 +408,13 @@ func (r *ReplicaSetReconciler) shouldRunInOrder(mdb mdbv1.MongoDBCommunity) bool // deployMongoDBReplicaSet will ensure that both the AutomationConfig secret and backing StatefulSet // have been successfully created. A boolean is returned indicating if the process is complete // and an error if there was one. -func (r *ReplicaSetReconciler) deployMongoDBReplicaSet(mdb mdbv1.MongoDBCommunity) (bool, error) { - return functions.RunSequentially(r.shouldRunInOrder(mdb), +func (r *ReplicaSetReconciler) deployMongoDBReplicaSet(ctx context.Context, mdb mdbv1.MongoDBCommunity) (bool, error) { + return functions.RunSequentially(r.shouldRunInOrder(ctx, mdb), func() (bool, error) { - return r.deployAutomationConfig(mdb) + return r.deployAutomationConfig(ctx, mdb) }, func() (bool, error) { - return r.deployStatefulSet(mdb) + return r.deployStatefulSet(ctx, mdb) }) } @@ -447,14 +422,14 @@ func (r *ReplicaSetReconciler) deployMongoDBReplicaSet(mdb mdbv1.MongoDBCommunit // // The Service definition is built from the `mdb` resource. If `isArbiter` is set to true, the Service // will be created for the arbiters Statefulset. -func (r *ReplicaSetReconciler) ensureService(mdb mdbv1.MongoDBCommunity) error { - processPortManager, err := r.createProcessPortManager(mdb) +func (r *ReplicaSetReconciler) ensureService(ctx context.Context, mdb mdbv1.MongoDBCommunity) error { + processPortManager, err := r.createProcessPortManager(ctx, mdb) if err != nil { return err } svc := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: mdb.ServiceName(), Namespace: mdb.Namespace}} - op, err := controllerutil.CreateOrUpdate(context.TODO(), r.client, svc, func() error { + op, err := controllerutil.CreateOrUpdate(ctx, r.client, svc, func() error { resourceVersion := svc.ResourceVersion // Save resourceVersion for later *svc = r.buildService(mdb, processPortManager) svc.ResourceVersion = resourceVersion @@ -473,13 +448,13 @@ func (r *ReplicaSetReconciler) ensureService(mdb mdbv1.MongoDBCommunity) error { // createProcessPortManager is a helper method for creating new ReplicaSetPortManager. // ReplicaSetPortManager needs current automation config and current pod state and the code for getting them // was extracted here as it is used in ensureService and buildAutomationConfig. -func (r *ReplicaSetReconciler) createProcessPortManager(mdb mdbv1.MongoDBCommunity) (*agent.ReplicaSetPortManager, error) { - currentAC, err := automationconfig.ReadFromSecret(r.client, types.NamespacedName{Name: mdb.AutomationConfigSecretName(), Namespace: mdb.Namespace}) +func (r *ReplicaSetReconciler) createProcessPortManager(ctx context.Context, mdb mdbv1.MongoDBCommunity) (*agent.ReplicaSetPortManager, error) { + currentAC, err := automationconfig.ReadFromSecret(ctx, r.client, types.NamespacedName{Name: mdb.AutomationConfigSecretName(), Namespace: mdb.Namespace}) if err != nil { return nil, fmt.Errorf("could not read existing automation config: %s", err) } - currentPodStates, err := agent.GetAllDesiredMembersAndArbitersPodState(mdb.NamespacedName(), r.client, mdb.StatefulSetReplicasThisReconciliation(), mdb.StatefulSetArbitersThisReconciliation(), currentAC.Version, r.log) + currentPodStates, err := agent.GetAllDesiredMembersAndArbitersPodState(ctx, mdb.NamespacedName(), r.client, mdb.StatefulSetReplicasThisReconciliation(), mdb.StatefulSetArbitersThisReconciliation(), currentAC.Version, r.log) if err != nil { return nil, fmt.Errorf("cannot get all pods goal state: %w", err) } @@ -487,7 +462,7 @@ func (r *ReplicaSetReconciler) createProcessPortManager(mdb mdbv1.MongoDBCommuni return agent.NewReplicaSetPortManager(r.log, mdb.Spec.AdditionalMongodConfig.GetDBPort(), currentPodStates, currentAC.Processes), nil } -func (r *ReplicaSetReconciler) createOrUpdateStatefulSet(mdb mdbv1.MongoDBCommunity, isArbiter bool) error { +func (r *ReplicaSetReconciler) createOrUpdateStatefulSet(ctx context.Context, mdb mdbv1.MongoDBCommunity, isArbiter bool) error { set := appsv1.StatefulSet{} name := mdb.NamespacedName() @@ -495,7 +470,7 @@ func (r *ReplicaSetReconciler) createOrUpdateStatefulSet(mdb mdbv1.MongoDBCommun name = mdb.ArbiterNamespacedName() } - err := r.client.Get(context.TODO(), name, &set) + err := r.client.Get(ctx, name, &set) err = k8sClient.IgnoreNotFound(err) if err != nil { return fmt.Errorf("error getting StatefulSet: %s", err) @@ -506,7 +481,7 @@ func (r *ReplicaSetReconciler) createOrUpdateStatefulSet(mdb mdbv1.MongoDBCommun buildArbitersModificationFunction(mdb)(&set) } - if _, err = statefulset.CreateOrUpdate(r.client, set); err != nil { + if _, err = statefulset.CreateOrUpdate(ctx, r.client, set); err != nil { return fmt.Errorf("error creating/updating StatefulSet: %s", err) } return nil @@ -514,18 +489,13 @@ func (r *ReplicaSetReconciler) createOrUpdateStatefulSet(mdb mdbv1.MongoDBCommun // ensureAutomationConfig makes sure the AutomationConfig secret has been successfully created. The automation config // that was updated/created is returned. -func (r ReplicaSetReconciler) ensureAutomationConfig(mdb mdbv1.MongoDBCommunity) (automationconfig.AutomationConfig, error) { - ac, err := r.buildAutomationConfig(mdb) +func (r ReplicaSetReconciler) ensureAutomationConfig(mdb mdbv1.MongoDBCommunity, ctx context.Context) (automationconfig.AutomationConfig, error) { + ac, err := r.buildAutomationConfig(ctx, mdb) if err != nil { return automationconfig.AutomationConfig{}, fmt.Errorf("could not build automation config: %s", err) } - return automationconfig.EnsureSecret( - r.client, - types.NamespacedName{Name: mdb.AutomationConfigSecretName(), Namespace: mdb.Namespace}, - mdb.GetOwnerReferences(), - ac, - ) + return automationconfig.EnsureSecret(ctx, r.client, types.NamespacedName{Name: mdb.AutomationConfigSecretName(), Namespace: mdb.Namespace}, mdb.GetOwnerReferences(), ac) } func buildAutomationConfig(mdb mdbv1.MongoDBCommunity, auth automationconfig.Auth, currentAc automationconfig.AutomationConfig, modifications ...automationconfig.Modification) (automationconfig.AutomationConfig, error) { @@ -652,8 +622,8 @@ func getCustomRolesModification(mdb mdbv1.MongoDBCommunity) (automationconfig.Mo }, nil } -func (r ReplicaSetReconciler) buildAutomationConfig(mdb mdbv1.MongoDBCommunity) (automationconfig.AutomationConfig, error) { - tlsModification, err := getTLSConfigModification(r.client, r.client, mdb) +func (r ReplicaSetReconciler) buildAutomationConfig(ctx context.Context, mdb mdbv1.MongoDBCommunity) (automationconfig.AutomationConfig, error) { + tlsModification, err := getTLSConfigModification(ctx, r.client, r.client, mdb) if err != nil { return automationconfig.AutomationConfig{}, fmt.Errorf("could not configure TLS modification: %s", err) } @@ -663,33 +633,33 @@ func (r ReplicaSetReconciler) buildAutomationConfig(mdb mdbv1.MongoDBCommunity) return automationconfig.AutomationConfig{}, fmt.Errorf("could not configure custom roles: %s", err) } - currentAC, err := automationconfig.ReadFromSecret(r.client, types.NamespacedName{Name: mdb.AutomationConfigSecretName(), Namespace: mdb.Namespace}) + currentAC, err := automationconfig.ReadFromSecret(ctx, r.client, types.NamespacedName{Name: mdb.AutomationConfigSecretName(), Namespace: mdb.Namespace}) if err != nil { return automationconfig.AutomationConfig{}, fmt.Errorf("could not read existing automation config: %s", err) } auth := automationconfig.Auth{} - if err := authentication.Enable(&auth, r.client, &mdb, mdb.AgentCertificateSecretNamespacedName()); err != nil { + if err := authentication.Enable(ctx, &auth, r.client, &mdb, mdb.AgentCertificateSecretNamespacedName()); err != nil { return automationconfig.AutomationConfig{}, err } prometheusModification := automationconfig.NOOP() if mdb.Spec.Prometheus != nil { secretNamespacedName := types.NamespacedName{Name: mdb.Spec.Prometheus.PasswordSecretRef.Name, Namespace: mdb.Namespace} - r.secretWatcher.Watch(secretNamespacedName, mdb.NamespacedName()) + r.secretWatcher.Watch(ctx, secretNamespacedName, mdb.NamespacedName()) - prometheusModification, err = getPrometheusModification(r.client, mdb) + prometheusModification, err = getPrometheusModification(ctx, r.client, mdb) if err != nil { return automationconfig.AutomationConfig{}, fmt.Errorf("could not enable TLS on Prometheus endpoint: %s", err) } } if mdb.Spec.IsAgentX509() { - r.secretWatcher.Watch(mdb.AgentCertificateSecretNamespacedName(), mdb.NamespacedName()) - r.secretWatcher.Watch(mdb.AgentCertificatePemSecretNamespacedName(), mdb.NamespacedName()) + r.secretWatcher.Watch(ctx, mdb.AgentCertificateSecretNamespacedName(), mdb.NamespacedName()) + r.secretWatcher.Watch(ctx, mdb.AgentCertificatePemSecretNamespacedName(), mdb.NamespacedName()) } - processPortManager, err := r.createProcessPortManager(mdb) + processPortManager, err := r.createProcessPortManager(ctx, mdb) if err != nil { return automationconfig.AutomationConfig{}, err } diff --git a/controllers/replicaset_controller_test.go b/controllers/replicaset_controller_test.go index aac79b07b..91094c936 100644 --- a/controllers/replicaset_controller_test.go +++ b/controllers/replicaset_controller_test.go @@ -154,17 +154,18 @@ func newTestReplicaSetWithTLSCaCertificateReferences(caConfigMap, caCertificateS } func TestKubernetesResources_AreCreated(t *testing.T) { + ctx := context.Background() // TODO: Create builder/yaml fixture of some type to construct MDB objects for unit tests mdb := newTestReplicaSet() - mgr := client.NewManager(&mdb) + mgr := client.NewManager(ctx, &mdb) r := NewReconciler(mgr) - res, err := r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) + res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) s := corev1.Secret{} - err = mgr.GetClient().Get(context.TODO(), types.NamespacedName{Name: mdb.AutomationConfigSecretName(), Namespace: mdb.Namespace}, &s) + err = mgr.GetClient().Get(ctx, types.NamespacedName{Name: mdb.AutomationConfigSecretName(), Namespace: mdb.Namespace}, &s) assert.NoError(t, err) assert.Equal(t, mdb.Namespace, s.Namespace) assert.Equal(t, mdb.AutomationConfigSecretName(), s.Name) @@ -173,17 +174,18 @@ func TestKubernetesResources_AreCreated(t *testing.T) { } func TestStatefulSet_IsCorrectlyConfigured(t *testing.T) { + ctx := context.Background() t.Setenv(construct.MongodbRepoUrl, "docker.io/mongodb") t.Setenv(construct.MongodbImageEnv, "mongodb-community-server") mdb := newTestReplicaSet() - mgr := client.NewManager(&mdb) + mgr := client.NewManager(ctx, &mdb) r := NewReconciler(mgr) - res, err := r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) + res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) sts := appsv1.StatefulSet{} - err = mgr.GetClient().Get(context.TODO(), types.NamespacedName{Name: mdb.Name, Namespace: mdb.Namespace}, &sts) + err = mgr.GetClient().Get(ctx, types.NamespacedName{Name: mdb.Name, Namespace: mdb.Namespace}, &sts) assert.NoError(t, err) assert.Len(t, sts.Spec.Template.Spec.Containers, 2) @@ -340,39 +342,40 @@ func getVolumeByName(sts appsv1.StatefulSet, volumeName string) (corev1.Volume, } func TestChangingVersion_ResultsInRollingUpdateStrategyType(t *testing.T) { + ctx := context.Background() mdb := newTestReplicaSet() - mgr := client.NewManager(&mdb) + mgr := client.NewManager(ctx, &mdb) mgrClient := mgr.GetClient() r := NewReconciler(mgr) - res, err := r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: mdb.NamespacedName()}) + res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: mdb.NamespacedName()}) assertReconciliationSuccessful(t, res, err) // fetch updated resource after first reconciliation - _ = mgrClient.Get(context.TODO(), mdb.NamespacedName(), &mdb) + _ = mgrClient.Get(ctx, mdb.NamespacedName(), &mdb) sts := appsv1.StatefulSet{} - err = mgrClient.Get(context.TODO(), types.NamespacedName{Name: mdb.Name, Namespace: mdb.Namespace}, &sts) + err = mgrClient.Get(ctx, types.NamespacedName{Name: mdb.Name, Namespace: mdb.Namespace}, &sts) assert.NoError(t, err) assert.Equal(t, appsv1.RollingUpdateStatefulSetStrategyType, sts.Spec.UpdateStrategy.Type) mdbRef := &mdb mdbRef.Spec.Version = "4.2.3" - _ = mgrClient.Update(context.TODO(), &mdb) + _ = mgrClient.Update(ctx, &mdb) // agents start the upgrade, they are not all ready sts.Status.UpdatedReplicas = 1 sts.Status.ReadyReplicas = 2 - err = mgrClient.Update(context.TODO(), &sts) + err = mgrClient.Update(ctx, &sts) assert.NoError(t, err) - _ = mgrClient.Get(context.TODO(), mdb.NamespacedName(), &sts) + _ = mgrClient.Get(ctx, mdb.NamespacedName(), &sts) // reconcilliation is successful - res, err = r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) + res, err = r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) sts = appsv1.StatefulSet{} - err = mgrClient.Get(context.TODO(), types.NamespacedName{Name: mdb.Name, Namespace: mdb.Namespace}, &sts) + err = mgrClient.Get(ctx, types.NamespacedName{Name: mdb.Name, Namespace: mdb.Namespace}, &sts) assert.NoError(t, err) assert.Equal(t, appsv1.RollingUpdateStatefulSetStrategyType, sts.Spec.UpdateStrategy.Type, @@ -417,59 +420,61 @@ func TestBuildStatefulSet_ConfiguresUpdateStrategyCorrectly(t *testing.T) { } func TestService_isCorrectlyCreatedAndUpdated(t *testing.T) { + ctx := context.Background() mdb := newTestReplicaSet() - mgr := client.NewManager(&mdb) + mgr := client.NewManager(ctx, &mdb) r := NewReconciler(mgr) - res, err := r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) + res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) svc := corev1.Service{} - err = mgr.GetClient().Get(context.TODO(), types.NamespacedName{Name: mdb.ServiceName(), Namespace: mdb.Namespace}, &svc) + err = mgr.GetClient().Get(ctx, types.NamespacedName{Name: mdb.ServiceName(), Namespace: mdb.Namespace}, &svc) assert.NoError(t, err) assert.Equal(t, svc.Spec.Type, corev1.ServiceTypeClusterIP) assert.Equal(t, svc.Spec.Selector["app"], mdb.ServiceName()) assert.Len(t, svc.Spec.Ports, 1) assert.Equal(t, svc.Spec.Ports[0], corev1.ServicePort{Port: 27017, Name: "mongodb"}) - res, err = r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) + res, err = r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) } func TestService_usesCustomMongodPortWhenSpecified(t *testing.T) { + ctx := context.Background() mdb := newTestReplicaSet() mongodConfig := objx.New(map[string]interface{}{}) mongodConfig.Set("net.port", 1000.) mdb.Spec.AdditionalMongodConfig.Object = mongodConfig - mgr := client.NewManager(&mdb) + mgr := client.NewManager(ctx, &mdb) r := NewReconciler(mgr) - res, err := r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) + res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) svc := corev1.Service{} - err = mgr.GetClient().Get(context.TODO(), types.NamespacedName{Name: mdb.ServiceName(), Namespace: mdb.Namespace}, &svc) + err = mgr.GetClient().Get(ctx, types.NamespacedName{Name: mdb.ServiceName(), Namespace: mdb.Namespace}, &svc) assert.NoError(t, err) assert.Equal(t, svc.Spec.Type, corev1.ServiceTypeClusterIP) assert.Equal(t, svc.Spec.Selector["app"], mdb.ServiceName()) assert.Len(t, svc.Spec.Ports, 1) assert.Equal(t, svc.Spec.Ports[0], corev1.ServicePort{Port: 1000, Name: "mongodb"}) - res, err = r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) + res, err = r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) } -func createOrUpdatePodsWithVersions(t *testing.T, c k8sClient.Client, name types.NamespacedName, versions []string) { +func createOrUpdatePodsWithVersions(ctx context.Context, t *testing.T, c k8sClient.Client, name types.NamespacedName, versions []string) { for i, version := range versions { - createPodWithAgentAnnotation(t, c, types.NamespacedName{ + createPodWithAgentAnnotation(ctx, t, c, types.NamespacedName{ Namespace: name.Namespace, Name: fmt.Sprintf("%s-%d", name.Name, i), }, version) } } -func createPodWithAgentAnnotation(t *testing.T, c k8sClient.Client, name types.NamespacedName, versionStr string) { +func createPodWithAgentAnnotation(ctx context.Context, t *testing.T, c k8sClient.Client, name types.NamespacedName, versionStr string) { pod := corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: name.Name, @@ -480,10 +485,10 @@ func createPodWithAgentAnnotation(t *testing.T, c k8sClient.Client, name types.N }, } - err := c.Create(context.TODO(), &pod) + err := c.Create(ctx, &pod) if err != nil && apiErrors.IsAlreadyExists(err) { - err = c.Update(context.TODO(), &pod) + err = c.Update(ctx, &pod) assert.NoError(t, err) } @@ -491,6 +496,7 @@ func createPodWithAgentAnnotation(t *testing.T, c k8sClient.Client, name types.N } func TestService_changesMongodPortOnRunningClusterWithArbiters(t *testing.T) { + ctx := context.Background() mdb := newScramReplicaSet(mdbv1.MongoDBUser{ Name: "testuser", PasswordSecretRef: mdbv1.SecretKeyReference{ @@ -505,53 +511,53 @@ func TestService_changesMongodPortOnRunningClusterWithArbiters(t *testing.T) { const oldPort = automationconfig.DefaultDBPort const newPort = 8000 - mgr := client.NewManager(&mdb) + mgr := client.NewManager(ctx, &mdb) r := NewReconciler(mgr) t.Run("Prepare cluster with arbiters and change port", func(t *testing.T) { - err := createUserPasswordSecret(mgr.Client, mdb, "password-secret-name", "pass") + err := createUserPasswordSecret(ctx, mgr.Client, mdb, "password-secret-name", "pass") assert.NoError(t, err) mdb.Spec.Arbiters = 1 - res, err := r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: namespacedName}) + res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: namespacedName}) assertReconciliationSuccessful(t, res, err) - assertServicePorts(t, mgr.Client, mdb, map[int]string{ + assertServicePorts(ctx, t, mgr.Client, mdb, map[int]string{ oldPort: "mongodb", }) - _ = assertAutomationConfigVersion(t, mgr.Client, mdb, 1) + _ = assertAutomationConfigVersion(ctx, t, mgr.Client, mdb, 1) - setStatefulSetReadyReplicas(t, mgr.GetClient(), mdb, 3) - setArbiterStatefulSetReadyReplicas(t, mgr.GetClient(), mdb, 1) - createOrUpdatePodsWithVersions(t, mgr.GetClient(), namespacedName, []string{"1", "1", "1"}) - createOrUpdatePodsWithVersions(t, mgr.GetClient(), arbiterNamespacedName, []string{"1"}) + setStatefulSetReadyReplicas(ctx, t, mgr.GetClient(), mdb, 3) + setArbiterStatefulSetReadyReplicas(ctx, t, mgr.GetClient(), mdb, 1) + createOrUpdatePodsWithVersions(ctx, t, mgr.GetClient(), namespacedName, []string{"1", "1", "1"}) + createOrUpdatePodsWithVersions(ctx, t, mgr.GetClient(), arbiterNamespacedName, []string{"1"}) - res, err = r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: namespacedName}) + res, err = r.Reconcile(ctx, reconcile.Request{NamespacedName: namespacedName}) assertReconciliationSuccessful(t, res, err) - assertServicePorts(t, mgr.Client, mdb, map[int]string{ + assertServicePorts(ctx, t, mgr.Client, mdb, map[int]string{ oldPort: "mongodb", }) - _ = assertAutomationConfigVersion(t, mgr.Client, mdb, 1) - assertStatefulsetReady(t, mgr, namespacedName, 3) - assertStatefulsetReady(t, mgr, arbiterNamespacedName, 1) + _ = assertAutomationConfigVersion(ctx, t, mgr.Client, mdb, 1) + assertStatefulsetReady(ctx, t, mgr, namespacedName, 3) + assertStatefulsetReady(ctx, t, mgr, arbiterNamespacedName, 1) mdb.Spec.AdditionalMongodConfig = mdbv1.NewMongodConfiguration() mdb.Spec.AdditionalMongodConfig.SetDBPort(newPort) - err = mgr.GetClient().Update(context.TODO(), &mdb) + err = mgr.GetClient().Update(ctx, &mdb) assert.NoError(t, err) - assertConnectionStringSecretPorts(t, mgr.GetClient(), mdb, oldPort, newPort) + assertConnectionStringSecretPorts(ctx, t, mgr.GetClient(), mdb, oldPort, newPort) }) t.Run("Port should be changed only in the process #0", func(t *testing.T) { // port changes should be performed one at a time // should set port #0 to new one - res, err := r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: namespacedName}) + res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: namespacedName}) require.NoError(t, err) assert.True(t, res.Requeue) - currentAc := assertAutomationConfigVersion(t, mgr.Client, mdb, 2) + currentAc := assertAutomationConfigVersion(ctx, t, mgr.Client, mdb, 2) require.Len(t, currentAc.Processes, 4) assert.Equal(t, newPort, currentAc.Processes[0].GetPort()) assert.Equal(t, oldPort, currentAc.Processes[1].GetPort()) @@ -559,24 +565,24 @@ func TestService_changesMongodPortOnRunningClusterWithArbiters(t *testing.T) { assert.Equal(t, oldPort, currentAc.Processes[3].GetPort()) // not all ports are changed, so there are still two ports in the service - assertServicePorts(t, mgr.Client, mdb, map[int]string{ + assertServicePorts(ctx, t, mgr.Client, mdb, map[int]string{ oldPort: "mongodb", newPort: "mongodb-new", }) - assertConnectionStringSecretPorts(t, mgr.GetClient(), mdb, oldPort, newPort) + assertConnectionStringSecretPorts(ctx, t, mgr.GetClient(), mdb, oldPort, newPort) }) t.Run("Ports should be changed in processes #0,#1", func(t *testing.T) { - setStatefulSetReadyReplicas(t, mgr.GetClient(), mdb, 3) - setArbiterStatefulSetReadyReplicas(t, mgr.GetClient(), mdb, 1) - createOrUpdatePodsWithVersions(t, mgr.GetClient(), namespacedName, []string{"2", "2", "2"}) - createOrUpdatePodsWithVersions(t, mgr.GetClient(), arbiterNamespacedName, []string{"2"}) + setStatefulSetReadyReplicas(ctx, t, mgr.GetClient(), mdb, 3) + setArbiterStatefulSetReadyReplicas(ctx, t, mgr.GetClient(), mdb, 1) + createOrUpdatePodsWithVersions(ctx, t, mgr.GetClient(), namespacedName, []string{"2", "2", "2"}) + createOrUpdatePodsWithVersions(ctx, t, mgr.GetClient(), arbiterNamespacedName, []string{"2"}) - res, err := r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: namespacedName}) + res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: namespacedName}) require.NoError(t, err) assert.True(t, res.Requeue) - currentAc := assertAutomationConfigVersion(t, mgr.Client, mdb, 3) + currentAc := assertAutomationConfigVersion(ctx, t, mgr.Client, mdb, 3) require.Len(t, currentAc.Processes, 4) assert.Equal(t, newPort, currentAc.Processes[0].GetPort()) assert.Equal(t, newPort, currentAc.Processes[1].GetPort()) @@ -584,24 +590,24 @@ func TestService_changesMongodPortOnRunningClusterWithArbiters(t *testing.T) { assert.Equal(t, oldPort, currentAc.Processes[3].GetPort()) // not all ports are changed, so there are still two ports in the service - assertServicePorts(t, mgr.Client, mdb, map[int]string{ + assertServicePorts(ctx, t, mgr.Client, mdb, map[int]string{ oldPort: "mongodb", newPort: "mongodb-new", }) - assertConnectionStringSecretPorts(t, mgr.GetClient(), mdb, oldPort, newPort) + assertConnectionStringSecretPorts(ctx, t, mgr.GetClient(), mdb, oldPort, newPort) }) t.Run("Ports should be changed in processes #0,#1,#2", func(t *testing.T) { - setStatefulSetReadyReplicas(t, mgr.GetClient(), mdb, 3) - setArbiterStatefulSetReadyReplicas(t, mgr.GetClient(), mdb, 1) - createOrUpdatePodsWithVersions(t, mgr.GetClient(), namespacedName, []string{"3", "3", "3"}) - createOrUpdatePodsWithVersions(t, mgr.GetClient(), arbiterNamespacedName, []string{"3"}) + setStatefulSetReadyReplicas(ctx, t, mgr.GetClient(), mdb, 3) + setArbiterStatefulSetReadyReplicas(ctx, t, mgr.GetClient(), mdb, 1) + createOrUpdatePodsWithVersions(ctx, t, mgr.GetClient(), namespacedName, []string{"3", "3", "3"}) + createOrUpdatePodsWithVersions(ctx, t, mgr.GetClient(), arbiterNamespacedName, []string{"3"}) - res, err := r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: namespacedName}) + res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: namespacedName}) require.NoError(t, err) assert.True(t, res.Requeue) - currentAc := assertAutomationConfigVersion(t, mgr.Client, mdb, 4) + currentAc := assertAutomationConfigVersion(ctx, t, mgr.Client, mdb, 4) require.Len(t, currentAc.Processes, 4) assert.Equal(t, newPort, currentAc.Processes[0].GetPort()) assert.Equal(t, newPort, currentAc.Processes[1].GetPort()) @@ -609,24 +615,24 @@ func TestService_changesMongodPortOnRunningClusterWithArbiters(t *testing.T) { assert.Equal(t, oldPort, currentAc.Processes[3].GetPort()) // not all ports are changed, so there are still two ports in the service - assertServicePorts(t, mgr.Client, mdb, map[int]string{ + assertServicePorts(ctx, t, mgr.Client, mdb, map[int]string{ oldPort: "mongodb", newPort: "mongodb-new", }) - assertConnectionStringSecretPorts(t, mgr.GetClient(), mdb, oldPort, newPort) + assertConnectionStringSecretPorts(ctx, t, mgr.GetClient(), mdb, oldPort, newPort) }) t.Run("Ports should be changed in all processes", func(t *testing.T) { - setStatefulSetReadyReplicas(t, mgr.GetClient(), mdb, 3) - setArbiterStatefulSetReadyReplicas(t, mgr.GetClient(), mdb, 1) - createOrUpdatePodsWithVersions(t, mgr.GetClient(), namespacedName, []string{"4", "4", "4"}) - createOrUpdatePodsWithVersions(t, mgr.GetClient(), arbiterNamespacedName, []string{"4"}) + setStatefulSetReadyReplicas(ctx, t, mgr.GetClient(), mdb, 3) + setArbiterStatefulSetReadyReplicas(ctx, t, mgr.GetClient(), mdb, 1) + createOrUpdatePodsWithVersions(ctx, t, mgr.GetClient(), namespacedName, []string{"4", "4", "4"}) + createOrUpdatePodsWithVersions(ctx, t, mgr.GetClient(), arbiterNamespacedName, []string{"4"}) - res, err := r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) + res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assert.NoError(t, err) assert.True(t, res.Requeue) - currentAc := assertAutomationConfigVersion(t, mgr.Client, mdb, 5) + currentAc := assertAutomationConfigVersion(ctx, t, mgr.Client, mdb, 5) require.Len(t, currentAc.Processes, 4) assert.Equal(t, newPort, currentAc.Processes[0].GetPort()) assert.Equal(t, newPort, currentAc.Processes[1].GetPort()) @@ -634,58 +640,58 @@ func TestService_changesMongodPortOnRunningClusterWithArbiters(t *testing.T) { assert.Equal(t, newPort, currentAc.Processes[3].GetPort()) // all the ports are changed but there are still two service ports for old and new port until the next reconcile - assertServicePorts(t, mgr.Client, mdb, map[int]string{ + assertServicePorts(ctx, t, mgr.Client, mdb, map[int]string{ oldPort: "mongodb", newPort: "mongodb-new", }) - assertConnectionStringSecretPorts(t, mgr.GetClient(), mdb, oldPort, newPort) + assertConnectionStringSecretPorts(ctx, t, mgr.GetClient(), mdb, oldPort, newPort) }) t.Run("At the end there should be only new port in the service", func(t *testing.T) { - setStatefulSetReadyReplicas(t, mgr.GetClient(), mdb, 3) - setArbiterStatefulSetReadyReplicas(t, mgr.GetClient(), mdb, 1) - createOrUpdatePodsWithVersions(t, mgr.GetClient(), namespacedName, []string{"5", "5", "5"}) - createOrUpdatePodsWithVersions(t, mgr.GetClient(), arbiterNamespacedName, []string{"5"}) + setStatefulSetReadyReplicas(ctx, t, mgr.GetClient(), mdb, 3) + setArbiterStatefulSetReadyReplicas(ctx, t, mgr.GetClient(), mdb, 1) + createOrUpdatePodsWithVersions(ctx, t, mgr.GetClient(), namespacedName, []string{"5", "5", "5"}) + createOrUpdatePodsWithVersions(ctx, t, mgr.GetClient(), arbiterNamespacedName, []string{"5"}) - res, err := r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: namespacedName}) + res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: namespacedName}) assert.NoError(t, err) // no need to requeue, port change is finished assert.False(t, res.Requeue) // there should not be any changes in config anymore - currentAc := assertAutomationConfigVersion(t, mgr.Client, mdb, 5) + currentAc := assertAutomationConfigVersion(ctx, t, mgr.Client, mdb, 5) require.Len(t, currentAc.Processes, 4) assert.Equal(t, newPort, currentAc.Processes[0].GetPort()) assert.Equal(t, newPort, currentAc.Processes[1].GetPort()) assert.Equal(t, newPort, currentAc.Processes[2].GetPort()) assert.Equal(t, newPort, currentAc.Processes[3].GetPort()) - assertServicePorts(t, mgr.Client, mdb, map[int]string{ + assertServicePorts(ctx, t, mgr.Client, mdb, map[int]string{ newPort: "mongodb", }) // only at the end, when all pods are ready we have updated connection strings - assertConnectionStringSecretPorts(t, mgr.GetClient(), mdb, newPort, oldPort) + assertConnectionStringSecretPorts(ctx, t, mgr.GetClient(), mdb, newPort, oldPort) }) } // assertConnectionStringSecretPorts checks that connection string secret has expectedPort and does not have notExpectedPort. -func assertConnectionStringSecretPorts(t *testing.T, c k8sClient.Client, mdb mdbv1.MongoDBCommunity, expectedPort int, notExpectedPort int) { +func assertConnectionStringSecretPorts(ctx context.Context, t *testing.T, c k8sClient.Client, mdb mdbv1.MongoDBCommunity, expectedPort int, notExpectedPort int) { connectionStringSecret := corev1.Secret{} scramUsers := mdb.GetAuthUsers() require.Len(t, scramUsers, 1) secretNamespacedName := types.NamespacedName{Name: scramUsers[0].ConnectionStringSecretName, Namespace: scramUsers[0].ConnectionStringSecretNamespace} - err := c.Get(context.TODO(), secretNamespacedName, &connectionStringSecret) + err := c.Get(ctx, secretNamespacedName, &connectionStringSecret) require.NoError(t, err) require.Contains(t, connectionStringSecret.Data, "connectionString.standard") assert.Contains(t, string(connectionStringSecret.Data["connectionString.standard"]), fmt.Sprintf("%d", expectedPort)) assert.NotContains(t, string(connectionStringSecret.Data["connectionString.standard"]), fmt.Sprintf("%d", notExpectedPort)) } -func assertServicePorts(t *testing.T, c k8sClient.Client, mdb mdbv1.MongoDBCommunity, expectedServicePorts map[int]string) { +func assertServicePorts(ctx context.Context, t *testing.T, c k8sClient.Client, mdb mdbv1.MongoDBCommunity, expectedServicePorts map[int]string) { svc := corev1.Service{} - err := c.Get(context.TODO(), types.NamespacedName{Name: mdb.ServiceName(), Namespace: mdb.Namespace}, &svc) + err := c.Get(ctx, types.NamespacedName{Name: mdb.ServiceName(), Namespace: mdb.Namespace}, &svc) require.NoError(t, err) assert.Equal(t, corev1.ServiceTypeClusterIP, svc.Spec.Type) assert.Equal(t, mdb.ServiceName(), svc.Spec.Selector["app"]) @@ -699,21 +705,22 @@ func assertServicePorts(t *testing.T, c k8sClient.Client, mdb mdbv1.MongoDBCommu assert.Equal(t, expectedServicePorts, actualServicePorts) } -func assertAutomationConfigVersion(t *testing.T, c client.Client, mdb mdbv1.MongoDBCommunity, expectedVersion int) automationconfig.AutomationConfig { - ac, err := automationconfig.ReadFromSecret(c, types.NamespacedName{Name: mdb.AutomationConfigSecretName(), Namespace: mdb.Namespace}) +func assertAutomationConfigVersion(ctx context.Context, t *testing.T, c client.Client, mdb mdbv1.MongoDBCommunity, expectedVersion int) automationconfig.AutomationConfig { + ac, err := automationconfig.ReadFromSecret(ctx, c, types.NamespacedName{Name: mdb.AutomationConfigSecretName(), Namespace: mdb.Namespace}) require.NoError(t, err) assert.Equal(t, expectedVersion, ac.Version) return ac } -func assertStatefulsetReady(t *testing.T, mgr manager.Manager, name types.NamespacedName, expectedReplicas int) { +func assertStatefulsetReady(ctx context.Context, t *testing.T, mgr manager.Manager, name types.NamespacedName, expectedReplicas int) { sts := appsv1.StatefulSet{} - err := mgr.GetClient().Get(context.TODO(), name, &sts) + err := mgr.GetClient().Get(ctx, name, &sts) require.NoError(t, err) assert.True(t, statefulset.IsReady(sts, expectedReplicas)) } func TestService_configuresPrometheusCustomPorts(t *testing.T) { + ctx := context.Background() mdb := newTestReplicaSet() mdb.Spec.Prometheus = &mdbv1.Prometheus{ Username: "username", @@ -727,22 +734,20 @@ func TestService_configuresPrometheusCustomPorts(t *testing.T) { mongodConfig.Set("net.port", 1000.) mdb.Spec.AdditionalMongodConfig.Object = mongodConfig - mgr := client.NewManager(&mdb) - err := secret.CreateOrUpdate(mgr.Client, - secret.Builder(). - SetName("secret"). - SetNamespace(mdb.Namespace). - SetField("password", "my-password"). - Build(), - ) + mgr := client.NewManager(ctx, &mdb) + err := secret.CreateOrUpdate(ctx, mgr.Client, secret.Builder(). + SetName("secret"). + SetNamespace(mdb.Namespace). + SetField("password", "my-password"). + Build()) assert.NoError(t, err) r := NewReconciler(mgr) - res, err := r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) + res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) svc := corev1.Service{} - err = mgr.GetClient().Get(context.TODO(), types.NamespacedName{Name: mdb.ServiceName(), Namespace: mdb.Namespace}, &svc) + err = mgr.GetClient().Get(ctx, types.NamespacedName{Name: mdb.ServiceName(), Namespace: mdb.Namespace}, &svc) assert.NoError(t, err) assert.Equal(t, svc.Spec.Type, corev1.ServiceTypeClusterIP) assert.Equal(t, svc.Spec.Selector["app"], mdb.ServiceName()) @@ -752,11 +757,12 @@ func TestService_configuresPrometheusCustomPorts(t *testing.T) { assert.Equal(t, svc.Labels["app"], mdb.ServiceName()) - res, err = r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) + res, err = r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) } func TestService_configuresPrometheus(t *testing.T) { + ctx := context.Background() mdb := newTestReplicaSet() mdb.Spec.Prometheus = &mdbv1.Prometheus{ Username: "username", @@ -765,22 +771,20 @@ func TestService_configuresPrometheus(t *testing.T) { }, } - mgr := client.NewManager(&mdb) - err := secret.CreateOrUpdate(mgr.Client, - secret.Builder(). - SetName("secret"). - SetNamespace(mdb.Namespace). - SetField("password", "my-password"). - Build(), - ) + mgr := client.NewManager(ctx, &mdb) + err := secret.CreateOrUpdate(ctx, mgr.Client, secret.Builder(). + SetName("secret"). + SetNamespace(mdb.Namespace). + SetField("password", "my-password"). + Build()) assert.NoError(t, err) r := NewReconciler(mgr) - res, err := r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) + res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) svc := corev1.Service{} - err = mgr.GetClient().Get(context.TODO(), types.NamespacedName{Name: mdb.ServiceName(), Namespace: mdb.Namespace}, &svc) + err = mgr.GetClient().Get(ctx, types.NamespacedName{Name: mdb.ServiceName(), Namespace: mdb.Namespace}, &svc) assert.NoError(t, err) assert.Len(t, svc.Spec.Ports, 2) @@ -789,65 +793,69 @@ func TestService_configuresPrometheus(t *testing.T) { } func TestCustomNetPort_Configuration(t *testing.T) { - svc, _ := performReconciliationAndGetService(t, "specify_net_port.yaml") + ctx := context.Background() + svc, _ := performReconciliationAndGetService(ctx, t, "specify_net_port.yaml") assert.Equal(t, corev1.ServiceTypeClusterIP, svc.Spec.Type) assert.Len(t, svc.Spec.Ports, 1) assert.Equal(t, corev1.ServicePort{Port: 40333, Name: "mongodb"}, svc.Spec.Ports[0]) } func TestAutomationConfig_versionIsBumpedOnChange(t *testing.T) { + ctx := context.Background() mdb := newTestReplicaSet() - mgr := client.NewManager(&mdb) + mgr := client.NewManager(ctx, &mdb) r := NewReconciler(mgr) - res, err := r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) + res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) - currentAc, err := automationconfig.ReadFromSecret(mgr.Client, types.NamespacedName{Name: mdb.AutomationConfigSecretName(), Namespace: mdb.Namespace}) + currentAc, err := automationconfig.ReadFromSecret(ctx, mgr.Client, types.NamespacedName{Name: mdb.AutomationConfigSecretName(), Namespace: mdb.Namespace}) assert.NoError(t, err) assert.Equal(t, 1, currentAc.Version) mdb.Spec.Members++ - makeStatefulSetReady(t, mgr.GetClient(), mdb) + makeStatefulSetReady(ctx, t, mgr.GetClient(), mdb) - _ = mgr.GetClient().Update(context.TODO(), &mdb) - res, err = r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) + _ = mgr.GetClient().Update(ctx, &mdb) + res, err = r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) - currentAc, err = automationconfig.ReadFromSecret(mgr.Client, types.NamespacedName{Name: mdb.AutomationConfigSecretName(), Namespace: mdb.Namespace}) + currentAc, err = automationconfig.ReadFromSecret(ctx, mgr.Client, types.NamespacedName{Name: mdb.AutomationConfigSecretName(), Namespace: mdb.Namespace}) assert.NoError(t, err) assert.Equal(t, 2, currentAc.Version) } func TestAutomationConfig_versionIsNotBumpedWithNoChanges(t *testing.T) { + ctx := context.Background() mdb := newTestReplicaSet() - mgr := client.NewManager(&mdb) + mgr := client.NewManager(ctx, &mdb) r := NewReconciler(mgr) - res, err := r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) + res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) - currentAc, err := automationconfig.ReadFromSecret(mgr.Client, types.NamespacedName{Name: mdb.AutomationConfigSecretName(), Namespace: mdb.Namespace}) + currentAc, err := automationconfig.ReadFromSecret(ctx, mgr.Client, types.NamespacedName{Name: mdb.AutomationConfigSecretName(), Namespace: mdb.Namespace}) assert.NoError(t, err) assert.Equal(t, currentAc.Version, 1) - res, err = r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) + res, err = r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) - currentAc, err = automationconfig.ReadFromSecret(mgr.Client, types.NamespacedName{Name: mdb.AutomationConfigSecretName(), Namespace: mdb.Namespace}) + currentAc, err = automationconfig.ReadFromSecret(ctx, mgr.Client, types.NamespacedName{Name: mdb.AutomationConfigSecretName(), Namespace: mdb.Namespace}) assert.NoError(t, err) assert.Equal(t, currentAc.Version, 1) } func TestAutomationConfigFCVIsNotIncreasedWhenUpgradingMinorVersion(t *testing.T) { + ctx := context.Background() mdb := newTestReplicaSet() mdb.Spec.Version = "4.2.2" - mgr := client.NewManager(&mdb) + mgr := client.NewManager(ctx, &mdb) r := NewReconciler(mgr) - res, err := r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) + res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) - currentAc, err := automationconfig.ReadFromSecret(mgr.Client, types.NamespacedName{Name: mdb.AutomationConfigSecretName(), Namespace: mdb.Namespace}) + currentAc, err := automationconfig.ReadFromSecret(ctx, mgr.Client, types.NamespacedName{Name: mdb.AutomationConfigSecretName(), Namespace: mdb.Namespace}) assert.NoError(t, err) assert.Len(t, currentAc.Processes, 3) assert.Equal(t, currentAc.Processes[0].FeatureCompatibilityVersion, "4.2") @@ -855,11 +863,11 @@ func TestAutomationConfigFCVIsNotIncreasedWhenUpgradingMinorVersion(t *testing.T // Upgrading minor version does not change the FCV on the automationConfig mdbRef := &mdb mdbRef.Spec.Version = "4.4.0" - _ = mgr.Client.Update(context.TODO(), mdbRef) - res, err = r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) + _ = mgr.Client.Update(ctx, mdbRef) + res, err = r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) - currentAc, err = automationconfig.ReadFromSecret(mgr.Client, types.NamespacedName{Name: mdb.AutomationConfigSecretName(), Namespace: mdb.Namespace}) + currentAc, err = automationconfig.ReadFromSecret(ctx, mgr.Client, types.NamespacedName{Name: mdb.AutomationConfigSecretName(), Namespace: mdb.Namespace}) assert.NoError(t, err) assert.Len(t, currentAc.Processes, 3) assert.Equal(t, currentAc.Processes[0].FeatureCompatibilityVersion, "4.2") @@ -867,6 +875,7 @@ func TestAutomationConfigFCVIsNotIncreasedWhenUpgradingMinorVersion(t *testing.T } func TestAutomationConfig_CustomMongodConfig(t *testing.T) { + ctx := context.Background() mdb := newTestReplicaSet() mongodConfig := objx.New(map[string]interface{}{}) @@ -875,12 +884,12 @@ func TestAutomationConfig_CustomMongodConfig(t *testing.T) { mongodConfig.Set("arbitrary.config.path", "value") mdb.Spec.AdditionalMongodConfig.Object = mongodConfig - mgr := client.NewManager(&mdb) + mgr := client.NewManager(ctx, &mdb) r := NewReconciler(mgr) - res, err := r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) + res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) - currentAc, err := automationconfig.ReadFromSecret(mgr.Client, types.NamespacedName{Name: mdb.AutomationConfigSecretName(), Namespace: mdb.Namespace}) + currentAc, err := automationconfig.ReadFromSecret(ctx, mgr.Client, types.NamespacedName{Name: mdb.AutomationConfigSecretName(), Namespace: mdb.Namespace}) assert.NoError(t, err) for _, p := range currentAc.Processes { @@ -898,36 +907,33 @@ func TestAutomationConfig_CustomMongodConfig(t *testing.T) { } func TestExistingPasswordAndKeyfile_AreUsedWhenTheSecretExists(t *testing.T) { + ctx := context.Background() mdb := newScramReplicaSet() - mgr := client.NewManager(&mdb) + mgr := client.NewManager(ctx, &mdb) c := mgr.Client keyFileNsName := mdb.GetAgentKeyfileSecretNamespacedName() - err := secret.CreateOrUpdate(c, - secret.Builder(). - SetName(keyFileNsName.Name). - SetNamespace(keyFileNsName.Namespace). - SetField(constants.AgentKeyfileKey, "my-keyfile"). - Build(), - ) + err := secret.CreateOrUpdate(ctx, c, secret.Builder(). + SetName(keyFileNsName.Name). + SetNamespace(keyFileNsName.Namespace). + SetField(constants.AgentKeyfileKey, "my-keyfile"). + Build()) assert.NoError(t, err) passwordNsName := mdb.GetAgentPasswordSecretNamespacedName() - err = secret.CreateOrUpdate(c, - secret.Builder(). - SetName(passwordNsName.Name). - SetNamespace(passwordNsName.Namespace). - SetField(constants.AgentPasswordKey, "my-pass"). - Build(), - ) + err = secret.CreateOrUpdate(ctx, c, secret.Builder(). + SetName(passwordNsName.Name). + SetNamespace(passwordNsName.Namespace). + SetField(constants.AgentPasswordKey, "my-pass"). + Build()) assert.NoError(t, err) r := NewReconciler(mgr) - res, err := r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) + res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) - currentAc, err := automationconfig.ReadFromSecret(mgr.Client, types.NamespacedName{Name: mdb.AutomationConfigSecretName(), Namespace: mdb.Namespace}) + currentAc, err := automationconfig.ReadFromSecret(ctx, mgr.Client, types.NamespacedName{Name: mdb.AutomationConfigSecretName(), Namespace: mdb.Namespace}) assert.NoError(t, err) assert.NotEmpty(t, currentAc.Auth.KeyFileWindows) assert.False(t, currentAc.Auth.Disabled) @@ -939,23 +945,26 @@ func TestExistingPasswordAndKeyfile_AreUsedWhenTheSecretExists(t *testing.T) { } func TestScramIsConfigured(t *testing.T) { - assertReplicaSetIsConfiguredWithScram(t, newScramReplicaSet()) + ctx := context.Background() + assertReplicaSetIsConfiguredWithScram(ctx, t, newScramReplicaSet()) } func TestScramIsConfiguredWhenNotSpecified(t *testing.T) { - assertReplicaSetIsConfiguredWithScram(t, newTestReplicaSet()) + ctx := context.Background() + assertReplicaSetIsConfiguredWithScram(ctx, t, newTestReplicaSet()) } func TestReplicaSet_IsScaledDown_OneMember_AtATime_WhenItAlreadyExists(t *testing.T) { + ctx := context.Background() mdb := newTestReplicaSet() mdb.Spec.Members = 5 - mgr := client.NewManager(&mdb) + mgr := client.NewManager(ctx, &mdb) r := NewReconciler(mgr) - res, err := r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) + res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) - err = mgr.GetClient().Get(context.TODO(), mdb.NamespacedName(), &mdb) + err = mgr.GetClient().Get(ctx, mdb.NamespacedName(), &mdb) assert.NoError(t, err) assert.Equal(t, 5, mdb.Status.CurrentMongoDBMembers) @@ -963,73 +972,74 @@ func TestReplicaSet_IsScaledDown_OneMember_AtATime_WhenItAlreadyExists(t *testin // scale members from five to three mdb.Spec.Members = 3 - err = mgr.GetClient().Update(context.TODO(), &mdb) + err = mgr.GetClient().Update(ctx, &mdb) assert.NoError(t, err) - makeStatefulSetReady(t, mgr.GetClient(), mdb) + makeStatefulSetReady(ctx, t, mgr.GetClient(), mdb) - res, err = r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: mdb.NamespacedName()}) + res, err = r.Reconcile(ctx, reconcile.Request{NamespacedName: mdb.NamespacedName()}) - makeStatefulSetReady(t, mgr.GetClient(), mdb) + makeStatefulSetReady(ctx, t, mgr.GetClient(), mdb) assert.NoError(t, err) - err = mgr.GetClient().Get(context.TODO(), mdb.NamespacedName(), &mdb) + err = mgr.GetClient().Get(ctx, mdb.NamespacedName(), &mdb) assert.NoError(t, err) assert.Equal(t, true, res.Requeue) assert.Equal(t, 4, mdb.Status.CurrentMongoDBMembers) - makeStatefulSetReady(t, mgr.GetClient(), mdb) + makeStatefulSetReady(ctx, t, mgr.GetClient(), mdb) - res, err = r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: mdb.NamespacedName()}) + res, err = r.Reconcile(ctx, reconcile.Request{NamespacedName: mdb.NamespacedName()}) assert.NoError(t, err) - err = mgr.GetClient().Get(context.TODO(), mdb.NamespacedName(), &mdb) + err = mgr.GetClient().Get(ctx, mdb.NamespacedName(), &mdb) assert.NoError(t, err) assert.Equal(t, false, res.Requeue) assert.Equal(t, 3, mdb.Status.CurrentMongoDBMembers) } func TestReplicaSet_IsScaledUp_OneMember_AtATime_WhenItAlreadyExists(t *testing.T) { + ctx := context.Background() mdb := newTestReplicaSet() - mgr := client.NewManager(&mdb) + mgr := client.NewManager(ctx, &mdb) r := NewReconciler(mgr) - res, err := r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) + res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) - err = mgr.GetClient().Get(context.TODO(), mdb.NamespacedName(), &mdb) + err = mgr.GetClient().Get(ctx, mdb.NamespacedName(), &mdb) assert.NoError(t, err) assert.Equal(t, 3, mdb.Status.CurrentMongoDBMembers) // scale members from three to five mdb.Spec.Members = 5 - err = mgr.GetClient().Update(context.TODO(), &mdb) + err = mgr.GetClient().Update(ctx, &mdb) assert.NoError(t, err) - makeStatefulSetReady(t, mgr.GetClient(), mdb) + makeStatefulSetReady(ctx, t, mgr.GetClient(), mdb) - res, err = r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: mdb.NamespacedName()}) + res, err = r.Reconcile(ctx, reconcile.Request{NamespacedName: mdb.NamespacedName()}) assert.NoError(t, err) - err = mgr.GetClient().Get(context.TODO(), mdb.NamespacedName(), &mdb) + err = mgr.GetClient().Get(ctx, mdb.NamespacedName(), &mdb) assert.NoError(t, err) assert.Equal(t, true, res.Requeue) assert.Equal(t, 4, mdb.Status.CurrentMongoDBMembers) - makeStatefulSetReady(t, mgr.GetClient(), mdb) + makeStatefulSetReady(ctx, t, mgr.GetClient(), mdb) - makeStatefulSetReady(t, mgr.GetClient(), mdb) + makeStatefulSetReady(ctx, t, mgr.GetClient(), mdb) - res, err = r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: mdb.NamespacedName()}) + res, err = r.Reconcile(ctx, reconcile.Request{NamespacedName: mdb.NamespacedName()}) assert.NoError(t, err) - err = mgr.GetClient().Get(context.TODO(), mdb.NamespacedName(), &mdb) + err = mgr.GetClient().Get(ctx, mdb.NamespacedName(), &mdb) assert.NoError(t, err) assert.Equal(t, false, res.Requeue) @@ -1037,37 +1047,39 @@ func TestReplicaSet_IsScaledUp_OneMember_AtATime_WhenItAlreadyExists(t *testing. } func TestIgnoreUnknownUsers(t *testing.T) { + ctx := context.Background() t.Run("Ignore Unkown Users set to true", func(t *testing.T) { mdb := newTestReplicaSet() ignoreUnknownUsers := true mdb.Spec.Security.Authentication.IgnoreUnknownUsers = &ignoreUnknownUsers - assertAuthoritativeSet(t, mdb, false) + assertAuthoritativeSet(ctx, t, mdb, false) }) t.Run("IgnoreUnknownUsers is not set", func(t *testing.T) { mdb := newTestReplicaSet() mdb.Spec.Security.Authentication.IgnoreUnknownUsers = nil - assertAuthoritativeSet(t, mdb, false) + assertAuthoritativeSet(ctx, t, mdb, false) }) t.Run("IgnoreUnknownUsers set to false", func(t *testing.T) { mdb := newTestReplicaSet() ignoreUnknownUsers := false mdb.Spec.Security.Authentication.IgnoreUnknownUsers = &ignoreUnknownUsers - assertAuthoritativeSet(t, mdb, true) + assertAuthoritativeSet(ctx, t, mdb, true) }) } func TestAnnotationsAreAppliedToResource(t *testing.T) { + ctx := context.Background() mdb := newTestReplicaSet() - mgr := client.NewManager(&mdb) + mgr := client.NewManager(ctx, &mdb) r := NewReconciler(mgr) - res, err := r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) + res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) - err = mgr.GetClient().Get(context.TODO(), mdb.NamespacedName(), &mdb) + err = mgr.GetClient().Get(ctx, mdb.NamespacedName(), &mdb) assert.NoError(t, err) assert.NotNil(t, mdb.Annotations) @@ -1077,13 +1089,13 @@ func TestAnnotationsAreAppliedToResource(t *testing.T) { // assertAuthoritativeSet asserts that a reconciliation of the given MongoDBCommunity resource // results in the AuthoritativeSet of the created AutomationConfig to have the expectedValue provided. -func assertAuthoritativeSet(t *testing.T, mdb mdbv1.MongoDBCommunity, expectedValue bool) { - mgr := client.NewManager(&mdb) +func assertAuthoritativeSet(ctx context.Context, t *testing.T, mdb mdbv1.MongoDBCommunity, expectedValue bool) { + mgr := client.NewManager(ctx, &mdb) r := NewReconciler(mgr) - res, err := r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) + res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) - s, err := mgr.Client.GetSecret(types.NamespacedName{Name: mdb.AutomationConfigSecretName(), Namespace: mdb.Namespace}) + s, err := mgr.Client.GetSecret(ctx, types.NamespacedName{Name: mdb.AutomationConfigSecretName(), Namespace: mdb.Namespace}) assert.NoError(t, err) bytes := s.Data[automationconfig.ConfigKey] @@ -1093,13 +1105,13 @@ func assertAuthoritativeSet(t *testing.T, mdb mdbv1.MongoDBCommunity, expectedVa assert.Equal(t, expectedValue, ac.Auth.AuthoritativeSet) } -func assertReplicaSetIsConfiguredWithScram(t *testing.T, mdb mdbv1.MongoDBCommunity) { - mgr := client.NewManager(&mdb) +func assertReplicaSetIsConfiguredWithScram(ctx context.Context, t *testing.T, mdb mdbv1.MongoDBCommunity) { + mgr := client.NewManager(ctx, &mdb) r := NewReconciler(mgr) - res, err := r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) + res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) - currentAc, err := automationconfig.ReadFromSecret(mgr.Client, types.NamespacedName{Name: mdb.AutomationConfigSecretName(), Namespace: mdb.Namespace}) + currentAc, err := automationconfig.ReadFromSecret(ctx, mgr.Client, types.NamespacedName{Name: mdb.AutomationConfigSecretName(), Namespace: mdb.Namespace}) t.Run("Automation Config is configured with SCRAM", func(t *testing.T) { assert.NotEmpty(t, currentAc.Auth.Key) assert.NoError(t, err) @@ -1109,31 +1121,31 @@ func assertReplicaSetIsConfiguredWithScram(t *testing.T, mdb mdbv1.MongoDBCommun }) t.Run("Secret with password was created", func(t *testing.T) { secretNsName := mdb.GetAgentPasswordSecretNamespacedName() - s, err := mgr.Client.GetSecret(secretNsName) + s, err := mgr.Client.GetSecret(ctx, secretNsName) assert.NoError(t, err) assert.Equal(t, s.Data[constants.AgentPasswordKey], []byte(currentAc.Auth.AutoPwd)) }) t.Run("Secret with keyfile was created", func(t *testing.T) { secretNsName := mdb.GetAgentKeyfileSecretNamespacedName() - s, err := mgr.Client.GetSecret(secretNsName) + s, err := mgr.Client.GetSecret(ctx, secretNsName) assert.NoError(t, err) assert.Equal(t, s.Data[constants.AgentKeyfileKey], []byte(currentAc.Auth.Key)) }) } -func assertReplicaSetIsConfiguredWithScramTLS(t *testing.T, mdb mdbv1.MongoDBCommunity) { - mgr := client.NewManager(&mdb) +func assertReplicaSetIsConfiguredWithScramTLS(ctx context.Context, t *testing.T, mdb mdbv1.MongoDBCommunity) { + mgr := client.NewManager(ctx, &mdb) newClient := client.NewClient(mgr.GetClient()) - err := createTLSSecret(newClient, mdb, "CERT", "KEY", "") + err := createTLSSecret(ctx, newClient, mdb, "CERT", "KEY", "") assert.NoError(t, err) - err = createTLSConfigMap(newClient, mdb) + err = createTLSConfigMap(ctx, newClient, mdb) assert.NoError(t, err) r := NewReconciler(mgr) - res, err := r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) + res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) - currentAc, err := automationconfig.ReadFromSecret(mgr.Client, types.NamespacedName{Name: mdb.AutomationConfigSecretName(), Namespace: mdb.Namespace}) + currentAc, err := automationconfig.ReadFromSecret(ctx, mgr.Client, types.NamespacedName{Name: mdb.AutomationConfigSecretName(), Namespace: mdb.Namespace}) t.Run("Automation Config is configured with SCRAM", func(t *testing.T) { assert.Empty(t, currentAc.TLSConfig.AutoPEMKeyFilePath) assert.NotEmpty(t, currentAc.Auth.Key) @@ -1144,36 +1156,36 @@ func assertReplicaSetIsConfiguredWithScramTLS(t *testing.T, mdb mdbv1.MongoDBCom }) t.Run("Secret with password was created", func(t *testing.T) { secretNsName := mdb.GetAgentPasswordSecretNamespacedName() - s, err := mgr.Client.GetSecret(secretNsName) + s, err := mgr.Client.GetSecret(ctx, secretNsName) assert.NoError(t, err) assert.Equal(t, s.Data[constants.AgentPasswordKey], []byte(currentAc.Auth.AutoPwd)) }) t.Run("Secret with keyfile was created", func(t *testing.T) { secretNsName := mdb.GetAgentKeyfileSecretNamespacedName() - s, err := mgr.Client.GetSecret(secretNsName) + s, err := mgr.Client.GetSecret(ctx, secretNsName) assert.NoError(t, err) assert.Equal(t, s.Data[constants.AgentKeyfileKey], []byte(currentAc.Auth.Key)) }) } -func assertReplicaSetIsConfiguredWithX509(t *testing.T, mdb mdbv1.MongoDBCommunity) { - mgr := client.NewManager(&mdb) +func assertReplicaSetIsConfiguredWithX509(ctx context.Context, t *testing.T, mdb mdbv1.MongoDBCommunity) { + mgr := client.NewManager(ctx, &mdb) newClient := client.NewClient(mgr.GetClient()) - err := createTLSSecret(newClient, mdb, "CERT", "KEY", "") + err := createTLSSecret(ctx, newClient, mdb, "CERT", "KEY", "") assert.NoError(t, err) - err = createTLSConfigMap(newClient, mdb) + err = createTLSConfigMap(ctx, newClient, mdb) assert.NoError(t, err) crt, key, err := x509.CreateAgentCertificate() assert.NoError(t, err) - err = createAgentCertSecret(newClient, mdb, crt, key, "") + err = createAgentCertSecret(ctx, newClient, mdb, crt, key, "") assert.NoError(t, err) r := NewReconciler(mgr) - res, err := r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) + res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) - currentAc, err := automationconfig.ReadFromSecret(mgr.Client, types.NamespacedName{Name: mdb.AutomationConfigSecretName(), Namespace: mdb.Namespace}) + currentAc, err := automationconfig.ReadFromSecret(ctx, mgr.Client, types.NamespacedName{Name: mdb.AutomationConfigSecretName(), Namespace: mdb.Namespace}) t.Run("Automation Config is configured with X509", func(t *testing.T) { assert.NotEmpty(t, currentAc.TLSConfig.AutoPEMKeyFilePath) @@ -1187,56 +1199,61 @@ func assertReplicaSetIsConfiguredWithX509(t *testing.T, mdb mdbv1.MongoDBCommuni }) t.Run("Secret with password was not created", func(t *testing.T) { secretNsName := mdb.GetAgentPasswordSecretNamespacedName() - _, err := mgr.Client.GetSecret(secretNsName) + _, err := mgr.Client.GetSecret(ctx, secretNsName) assert.Error(t, err) }) t.Run("Secret with keyfile was created", func(t *testing.T) { secretNsName := mdb.GetAgentKeyfileSecretNamespacedName() - s, err := mgr.Client.GetSecret(secretNsName) + s, err := mgr.Client.GetSecret(ctx, secretNsName) assert.NoError(t, err) assert.Equal(t, s.Data[constants.AgentKeyfileKey], []byte(currentAc.Auth.Key)) }) } func TestX509andSCRAMIsConfiguredWithX509Agent(t *testing.T) { + ctx := context.Background() mdb := newTestReplicaSetWithTLS() mdb.Spec.Security.Authentication.Modes = []mdbv1.AuthMode{"X509", "SCRAM"} mdb.Spec.Security.Authentication.AgentMode = "X509" - assertReplicaSetIsConfiguredWithX509(t, mdb) + assertReplicaSetIsConfiguredWithX509(ctx, t, mdb) } func TestX509andSCRAMIsConfiguredWithSCRAMAgent(t *testing.T) { + ctx := context.Background() mdb := newTestReplicaSetWithTLS() mdb.Spec.Security.Authentication.Modes = []mdbv1.AuthMode{"X509", "SCRAM"} mdb.Spec.Security.Authentication.AgentMode = "SCRAM" - assertReplicaSetIsConfiguredWithScramTLS(t, mdb) + assertReplicaSetIsConfiguredWithScramTLS(ctx, t, mdb) } func TestX509IsConfigured(t *testing.T) { + ctx := context.Background() mdb := newTestReplicaSetWithTLS() mdb.Spec.Security.Authentication.Modes = []mdbv1.AuthMode{"X509"} - assertReplicaSetIsConfiguredWithX509(t, mdb) + assertReplicaSetIsConfiguredWithX509(ctx, t, mdb) } func TestReplicaSet_IsScaledUpToDesiredMembers_WhenFirstCreated(t *testing.T) { + ctx := context.Background() mdb := newTestReplicaSet() - mgr := client.NewManager(&mdb) + mgr := client.NewManager(ctx, &mdb) r := NewReconciler(mgr) - res, err := r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) + res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) - err = mgr.GetClient().Get(context.TODO(), mdb.NamespacedName(), &mdb) + err = mgr.GetClient().Get(ctx, mdb.NamespacedName(), &mdb) assert.NoError(t, err) assert.Equal(t, 3, mdb.Status.CurrentMongoDBMembers) } func TestVolumeClaimTemplates_Configuration(t *testing.T) { - sts, _ := performReconciliationAndGetStatefulSet(t, "volume_claim_templates_mdb.yaml") + ctx := context.Background() + sts, _ := performReconciliationAndGetStatefulSet(ctx, t, "volume_claim_templates_mdb.yaml") assert.Len(t, sts.Spec.VolumeClaimTemplates, 3) @@ -1251,7 +1268,8 @@ func TestVolumeClaimTemplates_Configuration(t *testing.T) { } func TestChangeDataVolume_Configuration(t *testing.T) { - sts, _ := performReconciliationAndGetStatefulSet(t, "change_data_volume.yaml") + ctx := context.Background() + sts, _ := performReconciliationAndGetStatefulSet(ctx, t, "change_data_volume.yaml") assert.Len(t, sts.Spec.VolumeClaimTemplates, 2) dataVolume := sts.Spec.VolumeClaimTemplates[0] @@ -1264,7 +1282,8 @@ func TestChangeDataVolume_Configuration(t *testing.T) { } func TestCustomStorageClass_Configuration(t *testing.T) { - sts, _ := performReconciliationAndGetStatefulSet(t, "custom_storage_class.yaml") + ctx := context.Background() + sts, _ := performReconciliationAndGetStatefulSet(ctx, t, "custom_storage_class.yaml") dataVolume := sts.Spec.VolumeClaimTemplates[0] @@ -1280,7 +1299,8 @@ func TestCustomStorageClass_Configuration(t *testing.T) { } func TestCustomTaintsAndTolerations_Configuration(t *testing.T) { - sts, _ := performReconciliationAndGetStatefulSet(t, "tolerations_example.yaml") + ctx := context.Background() + sts, _ := performReconciliationAndGetStatefulSet(ctx, t, "tolerations_example.yaml") assert.Len(t, sts.Spec.Template.Spec.Tolerations, 2) assert.Equal(t, "example-key", sts.Spec.Template.Spec.Tolerations[0].Key) @@ -1293,7 +1313,8 @@ func TestCustomTaintsAndTolerations_Configuration(t *testing.T) { } func TestCustomDataDir_Configuration(t *testing.T) { - sts, c := performReconciliationAndGetStatefulSet(t, "specify_data_dir.yaml") + ctx := context.Background() + sts, c := performReconciliationAndGetStatefulSet(ctx, t, "specify_data_dir.yaml") agentContainer := container.GetByName("mongodb-agent", sts.Spec.Template.Spec.Containers) assert.NotNil(t, agentContainer) @@ -1305,7 +1326,7 @@ func TestCustomDataDir_Configuration(t *testing.T) { lastCommand := mongoContainer.Command[len(agentContainer.Command)-1] assert.Contains(t, lastCommand, "/some/path/db", "startup command should be using the newly specified path") - ac, err := automationconfig.ReadFromSecret(c, types.NamespacedName{Name: "example-mongodb-config", Namespace: "test-ns"}) + ac, err := automationconfig.ReadFromSecret(ctx, c, types.NamespacedName{Name: "example-mongodb-config", Namespace: "test-ns"}) assert.NoError(t, err) for _, p := range ac.Processes { @@ -1315,15 +1336,16 @@ func TestCustomDataDir_Configuration(t *testing.T) { } func TestInconsistentReplicas(t *testing.T) { + ctx := context.Background() mdb := newTestReplicaSet() stsReplicas := new(int32) *stsReplicas = 3 mdb.Spec.StatefulSetConfiguration.SpecWrapper.Spec.Replicas = stsReplicas mdb.Spec.Members = 4 - mgr := client.NewManager(&mdb) + mgr := client.NewManager(ctx, &mdb) r := NewReconciler(mgr) - _, err := r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) + _, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assert.NoError(t, err) } @@ -1337,34 +1359,34 @@ func assertVolumeMountPath(t *testing.T, mounts []corev1.VolumeMount, name, path t.Fatalf("volume with name %s was not present!", name) } -func performReconciliationAndGetStatefulSet(t *testing.T, filePath string) (appsv1.StatefulSet, client.Client) { +func performReconciliationAndGetStatefulSet(ctx context.Context, t *testing.T, filePath string) (appsv1.StatefulSet, client.Client) { mdb, err := loadTestFixture(filePath) assert.NoError(t, err) - mgr := client.NewManager(&mdb) - assert.NoError(t, generatePasswordsForAllUsers(mdb, mgr.Client)) + mgr := client.NewManager(ctx, &mdb) + assert.NoError(t, generatePasswordsForAllUsers(ctx, mdb, mgr.Client)) r := NewReconciler(mgr) - res, err := r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: mdb.NamespacedName()}) + res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: mdb.NamespacedName()}) assertReconciliationSuccessful(t, res, err) - sts, err := mgr.Client.GetStatefulSet(mdb.NamespacedName()) + sts, err := mgr.Client.GetStatefulSet(ctx, mdb.NamespacedName()) assert.NoError(t, err) return sts, mgr.Client } -func performReconciliationAndGetService(t *testing.T, filePath string) (corev1.Service, client.Client) { +func performReconciliationAndGetService(ctx context.Context, t *testing.T, filePath string) (corev1.Service, client.Client) { mdb, err := loadTestFixture(filePath) assert.NoError(t, err) - mgr := client.NewManager(&mdb) - assert.NoError(t, generatePasswordsForAllUsers(mdb, mgr.Client)) + mgr := client.NewManager(ctx, &mdb) + assert.NoError(t, generatePasswordsForAllUsers(ctx, mdb, mgr.Client)) r := NewReconciler(mgr) - res, err := r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: mdb.NamespacedName()}) + res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: mdb.NamespacedName()}) assertReconciliationSuccessful(t, res, err) - svc, err := mgr.Client.GetService(types.NamespacedName{Name: mdb.ServiceName(), Namespace: mdb.Namespace}) + svc, err := mgr.Client.GetService(ctx, types.NamespacedName{Name: mdb.ServiceName(), Namespace: mdb.Namespace}) assert.NoError(t, err) return svc, mgr.Client } -func generatePasswordsForAllUsers(mdb mdbv1.MongoDBCommunity, c client.Client) error { +func generatePasswordsForAllUsers(ctx context.Context, mdb mdbv1.MongoDBCommunity, c client.Client) error { for _, user := range mdb.Spec.Users { key := "password" @@ -1378,7 +1400,7 @@ func generatePasswordsForAllUsers(mdb mdbv1.MongoDBCommunity, c client.Client) e SetField(key, "GAGTQK2ccRRaxJFudI5y"). Build() - if err := c.CreateSecret(passwordSecret); err != nil { + if err := c.CreateSecret(ctx, passwordSecret); err != nil { return err } } @@ -1394,27 +1416,27 @@ func assertReconciliationSuccessful(t *testing.T, result reconcile.Result, err e // makeStatefulSetReady updates the StatefulSet corresponding to the // provided MongoDB resource to mark it as ready for the case of `statefulset.IsReady` -func makeStatefulSetReady(t *testing.T, c k8sClient.Client, mdb mdbv1.MongoDBCommunity) { - setStatefulSetReadyReplicas(t, c, mdb, mdb.StatefulSetReplicasThisReconciliation()) +func makeStatefulSetReady(ctx context.Context, t *testing.T, c k8sClient.Client, mdb mdbv1.MongoDBCommunity) { + setStatefulSetReadyReplicas(ctx, t, c, mdb, mdb.StatefulSetReplicasThisReconciliation()) } -func setStatefulSetReadyReplicas(t *testing.T, c k8sClient.Client, mdb mdbv1.MongoDBCommunity, readyReplicas int) { +func setStatefulSetReadyReplicas(ctx context.Context, t *testing.T, c k8sClient.Client, mdb mdbv1.MongoDBCommunity, readyReplicas int) { sts := appsv1.StatefulSet{} - err := c.Get(context.TODO(), mdb.NamespacedName(), &sts) + err := c.Get(ctx, mdb.NamespacedName(), &sts) assert.NoError(t, err) sts.Status.ReadyReplicas = int32(readyReplicas) sts.Status.UpdatedReplicas = int32(mdb.StatefulSetReplicasThisReconciliation()) - err = c.Update(context.TODO(), &sts) + err = c.Update(ctx, &sts) assert.NoError(t, err) } -func setArbiterStatefulSetReadyReplicas(t *testing.T, c k8sClient.Client, mdb mdbv1.MongoDBCommunity, readyReplicas int) { +func setArbiterStatefulSetReadyReplicas(ctx context.Context, t *testing.T, c k8sClient.Client, mdb mdbv1.MongoDBCommunity, readyReplicas int) { sts := appsv1.StatefulSet{} - err := c.Get(context.TODO(), mdb.ArbiterNamespacedName(), &sts) + err := c.Get(ctx, mdb.ArbiterNamespacedName(), &sts) assert.NoError(t, err) sts.Status.ReadyReplicas = int32(readyReplicas) sts.Status.UpdatedReplicas = int32(mdb.StatefulSetArbitersThisReconciliation()) - err = c.Update(context.TODO(), &sts) + err = c.Update(ctx, &sts) assert.NoError(t, err) } diff --git a/controllers/watch/watch.go b/controllers/watch/watch.go index 376bbfcf2..9522c53c3 100644 --- a/controllers/watch/watch.go +++ b/controllers/watch/watch.go @@ -1,11 +1,13 @@ package watch import ( + "context" "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/contains" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/util/workqueue" "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) @@ -16,6 +18,8 @@ type ResourceWatcher struct { watched map[types.NamespacedName][]types.NamespacedName } +var _ handler.EventHandler = &ResourceWatcher{} + // New will create a new ResourceWatcher with no watched objects. func New() ResourceWatcher { return ResourceWatcher{ @@ -24,7 +28,7 @@ func New() ResourceWatcher { } // Watch will add a new object to watch. -func (w ResourceWatcher) Watch(watchedName, dependentName types.NamespacedName) { +func (w ResourceWatcher) Watch(ctx context.Context, watchedName, dependentName types.NamespacedName) { existing, hasExisting := w.watched[watchedName] if !hasExisting { existing = []types.NamespacedName{} @@ -38,19 +42,19 @@ func (w ResourceWatcher) Watch(watchedName, dependentName types.NamespacedName) w.watched[watchedName] = append(existing, dependentName) } -func (w ResourceWatcher) Create(event event.CreateEvent, queue workqueue.RateLimitingInterface) { +func (w ResourceWatcher) Create(ctx context.Context, event event.CreateEvent, queue workqueue.RateLimitingInterface) { w.handleEvent(event.Object, queue) } -func (w ResourceWatcher) Update(event event.UpdateEvent, queue workqueue.RateLimitingInterface) { +func (w ResourceWatcher) Update(ctx context.Context, event event.UpdateEvent, queue workqueue.RateLimitingInterface) { w.handleEvent(event.ObjectOld, queue) } -func (w ResourceWatcher) Delete(event event.DeleteEvent, queue workqueue.RateLimitingInterface) { +func (w ResourceWatcher) Delete(ctx context.Context, event event.DeleteEvent, queue workqueue.RateLimitingInterface) { w.handleEvent(event.Object, queue) } -func (w ResourceWatcher) Generic(event event.GenericEvent, queue workqueue.RateLimitingInterface) { +func (w ResourceWatcher) Generic(ctx context.Context, event event.GenericEvent, queue workqueue.RateLimitingInterface) { w.handleEvent(event.Object, queue) } diff --git a/controllers/watch/watch_test.go b/controllers/watch/watch_test.go index 027c1b78d..ab8c522be 100644 --- a/controllers/watch/watch_test.go +++ b/controllers/watch/watch_test.go @@ -1,6 +1,7 @@ package watch import ( + "context" "testing" "k8s.io/apimachinery/pkg/types" @@ -19,6 +20,7 @@ import ( ) func TestWatcher(t *testing.T) { + ctx := context.Background() obj := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "pod", @@ -45,9 +47,9 @@ func TestWatcher(t *testing.T) { watcher := New() queue := controllertest.Queue{Interface: workqueue.New()} - watcher.Create(event.CreateEvent{ + watcher.Create(ctx, event.CreateEvent{ Object: obj, - }, queue) + }, &queue) // Ensure no reconciliation is queued if object is not watched. assert.Equal(t, 0, queue.Len()) @@ -56,12 +58,12 @@ func TestWatcher(t *testing.T) { t.Run("Multiple objects to reconcile", func(t *testing.T) { watcher := New() queue := controllertest.Queue{Interface: workqueue.New()} - watcher.Watch(objNsName, mdb1.NamespacedName()) - watcher.Watch(objNsName, mdb2.NamespacedName()) + watcher.Watch(ctx, objNsName, mdb1.NamespacedName()) + watcher.Watch(ctx, objNsName, mdb2.NamespacedName()) - watcher.Create(event.CreateEvent{ + watcher.Create(ctx, event.CreateEvent{ Object: obj, - }, queue) + }, &queue) // Ensure multiple reconciliations are enqueued. assert.Equal(t, 2, queue.Len()) @@ -70,11 +72,11 @@ func TestWatcher(t *testing.T) { t.Run("Create event", func(t *testing.T) { watcher := New() queue := controllertest.Queue{Interface: workqueue.New()} - watcher.Watch(objNsName, mdb1.NamespacedName()) + watcher.Watch(ctx, objNsName, mdb1.NamespacedName()) - watcher.Create(event.CreateEvent{ + watcher.Create(ctx, event.CreateEvent{ Object: obj, - }, queue) + }, &queue) assert.Equal(t, 1, queue.Len()) }) @@ -82,12 +84,12 @@ func TestWatcher(t *testing.T) { t.Run("Update event", func(t *testing.T) { watcher := New() queue := controllertest.Queue{Interface: workqueue.New()} - watcher.Watch(objNsName, mdb1.NamespacedName()) + watcher.Watch(ctx, objNsName, mdb1.NamespacedName()) - watcher.Update(event.UpdateEvent{ + watcher.Update(ctx, event.UpdateEvent{ ObjectOld: obj, ObjectNew: obj, - }, queue) + }, &queue) assert.Equal(t, 1, queue.Len()) }) @@ -95,11 +97,11 @@ func TestWatcher(t *testing.T) { t.Run("Delete event", func(t *testing.T) { watcher := New() queue := controllertest.Queue{Interface: workqueue.New()} - watcher.Watch(objNsName, mdb1.NamespacedName()) + watcher.Watch(ctx, objNsName, mdb1.NamespacedName()) - watcher.Delete(event.DeleteEvent{ + watcher.Delete(ctx, event.DeleteEvent{ Object: obj, - }, queue) + }, &queue) assert.Equal(t, 1, queue.Len()) }) @@ -107,17 +109,18 @@ func TestWatcher(t *testing.T) { t.Run("Generic event", func(t *testing.T) { watcher := New() queue := controllertest.Queue{Interface: workqueue.New()} - watcher.Watch(objNsName, mdb1.NamespacedName()) + watcher.Watch(ctx, objNsName, mdb1.NamespacedName()) - watcher.Generic(event.GenericEvent{ + watcher.Generic(ctx, event.GenericEvent{ Object: obj, - }, queue) + }, &queue) assert.Equal(t, 1, queue.Len()) }) } func TestWatcherAdd(t *testing.T) { + ctx := context.Background() watcher := New() assert.Empty(t, watcher.watched) @@ -137,17 +140,17 @@ func TestWatcherAdd(t *testing.T) { } // Ensure single object can be added to empty watchlist. - watcher.Watch(watchedName, mdb1.NamespacedName()) + watcher.Watch(ctx, watchedName, mdb1.NamespacedName()) assert.Len(t, watcher.watched, 1) assert.Equal(t, []types.NamespacedName{mdb1.NamespacedName()}, watcher.watched[watchedName]) // Ensure object can only be watched once. - watcher.Watch(watchedName, mdb1.NamespacedName()) + watcher.Watch(ctx, watchedName, mdb1.NamespacedName()) assert.Len(t, watcher.watched, 1) assert.Equal(t, []types.NamespacedName{mdb1.NamespacedName()}, watcher.watched[watchedName]) // Ensure a single object can be watched for multiple reconciliations. - watcher.Watch(watchedName, mdb2.NamespacedName()) + watcher.Watch(ctx, watchedName, mdb2.NamespacedName()) assert.Len(t, watcher.watched, 1) assert.Equal(t, []types.NamespacedName{ mdb1.NamespacedName(), diff --git a/go.mod b/go.mod index 2c0e18dd0..678e785eb 100644 --- a/go.mod +++ b/go.mod @@ -14,10 +14,10 @@ require ( go.mongodb.org/mongo-driver v1.14.0 go.uber.org/zap v1.27.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 - k8s.io/api v0.26.10 - k8s.io/apimachinery v0.26.10 - k8s.io/client-go v0.26.10 - sigs.k8s.io/controller-runtime v0.14.7 + k8s.io/api v0.27.12 + k8s.io/apimachinery v0.27.12 + k8s.io/client-go v0.27.12 + sigs.k8s.io/controller-runtime v0.15.3 sigs.k8s.io/yaml v1.4.0 ) @@ -25,18 +25,18 @@ require google.golang.org/protobuf v1.33.0 // indirect require ( github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.9.0 // indirect github.com/evanphx/json-patch v4.12.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect - github.com/go-openapi/jsonpointer v0.19.5 // indirect - github.com/go-openapi/jsonreference v0.20.0 // indirect - github.com/go-openapi/swag v0.19.14 // indirect + github.com/go-openapi/jsonpointer v0.19.6 // indirect + github.com/go-openapi/jsonreference v0.20.1 // indirect + github.com/go-openapi/swag v0.22.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.3 // indirect github.com/google/gnostic v0.5.7-v3refs // indirect github.com/google/go-cmp v0.5.9 // indirect @@ -46,8 +46,8 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.13.6 // indirect - github.com/mailru/easyjson v0.7.6 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/moby/spdystream v0.2.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect @@ -55,10 +55,10 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_golang v1.14.0 // indirect - github.com/prometheus/client_model v0.3.0 // indirect - github.com/prometheus/common v0.37.0 // indirect - github.com/prometheus/procfs v0.8.0 // indirect + github.com/prometheus/client_golang v1.15.1 // indirect + github.com/prometheus/client_model v0.4.0 // indirect + github.com/prometheus/common v0.42.0 // indirect + github.com/prometheus/procfs v0.9.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.2 // indirect @@ -66,23 +66,23 @@ require ( github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect go.uber.org/multierr v1.10.0 // indirect golang.org/x/crypto v0.17.0 // indirect - golang.org/x/net v0.17.0 // indirect - golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b // indirect - golang.org/x/sync v0.1.0 // indirect + golang.org/x/net v0.19.0 // indirect + golang.org/x/oauth2 v0.7.0 // indirect + golang.org/x/sync v0.5.0 // indirect golang.org/x/sys v0.15.0 // indirect golang.org/x/term v0.15.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.3.0 // indirect - gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect + gomodules.xyz/jsonpatch/v2 v2.3.0 // indirect google.golang.org/appengine v1.6.7 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.26.10 // indirect - k8s.io/component-base v0.26.10 // indirect - k8s.io/klog/v2 v2.80.1 // indirect - k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect - k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 // indirect - sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect + k8s.io/apiextensions-apiserver v0.27.12 // indirect + k8s.io/component-base v0.27.12 // indirect + k8s.io/klog/v2 v2.90.1 // indirect + k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect + k8s.io/utils v0.0.0-20230209194617-a36077c30491 // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect ) diff --git a/go.sum b/go.sum index 7625ea472..61801ad94 100644 --- a/go.sum +++ b/go.sum @@ -1,74 +1,24 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= -github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= -github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE= github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= @@ -77,216 +27,130 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-logr/zapr v1.2.3 h1:a9vnzlIBPQBBkeaR9IuMUfmVOrQlkoC4YfPoFkX3T7A= -github.com/go-logr/zapr v1.2.3/go.mod h1:eIauM6P8qSvTw5o2ez6UEAfGjQKrxQTl5EoK+Qa2oG4= -github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= -github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonreference v0.20.0 h1:MYlu0sBgChmCfJxxUKZ8g1cPWFOB37YSZqewK7OKeyA= -github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= -github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng= -github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= +github.com/go-logr/zapr v1.2.4/go.mod h1:FyHWQIzQORZ0QVE1BtVHv3cKtNLuXsbNLtpuhNapBOA= +github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonreference v0.20.1 h1:FBLnyygC4/IZZr893oiomc9XaghoveYTrLC1F86HID8= +github.com/go-openapi/jsonreference v0.20.1/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54= github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM= github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= -github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.2 h1:hAHbPm5IJGijwng3PWk09JkG9WeqChjprR5s9bBZ+OM= -github.com/matttproud/golang_protobuf_extensions v1.0.2/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/onsi/ginkgo/v2 v2.6.0 h1:9t9b9vRUbFq3C4qKFCGkVuq/fIHji802N1nrtkh1mNc= -github.com/onsi/ginkgo/v2 v2.6.0/go.mod h1:63DOGlLAH8+REH8jUGdL3YpCpu7JODesutUjdENfUAc= -github.com/onsi/gomega v1.24.1 h1:KORJXNNTzJXzu4ScJWssJfJMnJ+2QJqhoQSRwNlze9E= -github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q= +github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k= +github.com/onsi/gomega v1.27.7 h1:fVih9JD6ogIiHUN6ePK7HJidyEDpWGVB5mzM7cWNXoU= +github.com/onsi/gomega v1.27.7/go.mod h1:1p8OOlwo2iUUDsHnOrjE5UKYJ+e3W8eQ3qSlRahPmr4= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= -github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI= +github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= -github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= -github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= -github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= -github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= +github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= +github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= +github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= +github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= +github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= @@ -299,160 +163,61 @@ github.com/xdg/stringprep v1.0.3 h1:cmL5Enob4W83ti/ZHuZLuKD/xqJfus4fVPwE+/BDm+4= github.com/xdg/stringprep v1.0.3/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.mongodb.org/mongo-driver v1.14.0 h1:P98w8egYRjYe3XDjxhYJagTokP/H6HzlsnojRgZRd80= go.mongodb.org/mongo-driver v1.14.0/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b h1:clP8eMhB30EHdc0bd2Twtq6kgU7yl5ub2cQLSdrv1Dg= -golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g= +golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= +golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -462,167 +227,65 @@ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9sn golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.16.1 h1:TLyB3WofjdOEepBHAU20JdNC1Zbg87elYofWYAY5oZA= +golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY= -gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +gomodules.xyz/jsonpatch/v2 v2.3.0 h1:8NFhfS6gzxNqjLIYnZxg319wZ5Qjnx4m/CcX+Klzazc= +gomodules.xyz/jsonpatch/v2 v2.3.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -630,35 +293,27 @@ gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.26.10 h1:skTnrDR0r8dg4MMLf6YZIzugxNM0BjFsWKPkNc5kOvk= -k8s.io/api v0.26.10/go.mod h1:ou/H3yviqrHtP/DSPVTfsc7qNfmU06OhajytJfYXkXw= -k8s.io/apiextensions-apiserver v0.26.10 h1:wAriTUc6l7gUqJKOxhmXnYo/VNJzk4oh4QLCUR4Uq+k= -k8s.io/apiextensions-apiserver v0.26.10/go.mod h1:N2qhlxkhJLSoC4f0M1/1lNG627b45SYqnOPEVFoQXw4= -k8s.io/apimachinery v0.26.10 h1:aE+J2KIbjctFqPp3Y0q4Wh2PD+l1p2g3Zp4UYjSvtGU= -k8s.io/apimachinery v0.26.10/go.mod h1:iT1ZP4JBP34wwM+ZQ8ByPEQ81u043iqAcsJYftX9amM= -k8s.io/client-go v0.26.10 h1:4mDzl+1IrfRxh4Ro0s65JRGJp14w77gSMUTjACYWVRo= -k8s.io/client-go v0.26.10/go.mod h1:sh74ig838gCckU4ElYclWb24lTesPdEDPnlyg5vcbkA= -k8s.io/component-base v0.26.10 h1:vl3Gfe5aC09mNxfnQtTng7u3rnBVrShOK3MAkqEleb0= -k8s.io/component-base v0.26.10/go.mod h1:/IDdENUHG5uGxqcofZajovYXE9KSPzJ4yQbkYQt7oN0= -k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4= -k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 h1:+70TFaan3hfJzs+7VK2o+OGxg8HsuBr/5f6tVAjDu6E= -k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4= -k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 h1:KTgPnR10d5zhztWptI952TNtt/4u5h3IzDXkdIMuo2Y= -k8s.io/utils v0.0.0-20221128185143-99ec85e7a448/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/controller-runtime v0.14.7 h1:Vrnm2vk9ZFlRkXATHz0W0wXcqNl7kPat8q2JyxVy0Q8= -sigs.k8s.io/controller-runtime v0.14.7/go.mod h1:ErTs3SJCOujNUnTz4AS+uh8hp6DHMo1gj6fFndJT1X8= -sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k= -sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +k8s.io/api v0.27.12 h1:Qprj/nuFj4xjbsAuJ05F1sCHs1d0x33n/Ni0oAVEFDo= +k8s.io/api v0.27.12/go.mod h1:PNRL63V26JzKe2++ho6W/YRp3k9XG7nirN4J7WRy5gY= +k8s.io/apiextensions-apiserver v0.27.12 h1:1A1+0rlOrqKi+LbCTf3MeFbYmBFGoQ9rFHxBgFnhOpE= +k8s.io/apiextensions-apiserver v0.27.12/go.mod h1:KkUHCjJ/Awhly9NnVsYySZtQSxuWU+8IL4p2ffE4JQ8= +k8s.io/apimachinery v0.27.12 h1:Nt20vwaAHcZsM4WdkOtLaDeBJHg9QJW8JyOOEn6xzRA= +k8s.io/apimachinery v0.27.12/go.mod h1:5/SjQaDYQgZOv8kuzNMzmNGrqh4/iyknC5yWjxU9ll8= +k8s.io/client-go v0.27.12 h1:ouIB3ZitBjmBWh/9auP4erVl8AXkheqcmbH7FSFa7DI= +k8s.io/client-go v0.27.12/go.mod h1:h3X7RGr5s9Wm4NtI06Bzt3am4Kj6aXuZQcP7OD+48Sk= +k8s.io/component-base v0.27.12 h1:2/ooM9/gNxS2fZuRnLj3TWWg7KnEYmRj321du10waZA= +k8s.io/component-base v0.27.12/go.mod h1:SPA6ABqK2HDRuzMjoEEkj7ciDvK5bUpdHtvZQwdi/xM= +k8s.io/klog/v2 v2.90.1 h1:m4bYOKall2MmOiRaR1J+We67Do7vm9KiQVlT96lnHUw= +k8s.io/klog/v2 v2.90.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f h1:2kWPakN3i/k81b0gvD5C5FJ2kxm1WrQFanWchyKuqGg= +k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f/go.mod h1:byini6yhqGC14c3ebc/QwanvYwhuMWF6yz2F8uwW8eg= +k8s.io/utils v0.0.0-20230209194617-a36077c30491 h1:r0BAOLElQnnFhE/ApUsg3iHdVYYPBjNSSOMowRZxxsY= +k8s.io/utils v0.0.0-20230209194617-a36077c30491/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/controller-runtime v0.15.3 h1:L+t5heIaI3zeejoIyyvLQs5vTVu/67IU2FfisVzFlBc= +sigs.k8s.io/controller-runtime v0.15.3/go.mod h1:kp4jckA4vTx281S/0Yk2LFEEQe67mjg+ev/yknv47Ds= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= diff --git a/pkg/agent/agent_readiness.go b/pkg/agent/agent_readiness.go index 290697f3d..eefe3a49d 100644 --- a/pkg/agent/agent_readiness.go +++ b/pkg/agent/agent_readiness.go @@ -1,6 +1,7 @@ package agent import ( + "context" "fmt" "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/pod" @@ -27,9 +28,9 @@ type PodState struct { // AllReachedGoalState returns whether the agents associated with a given StatefulSet have reached goal state. // it achieves this by reading the Pod annotations and checking to see if they have reached the expected config versions. -func AllReachedGoalState(sts appsv1.StatefulSet, podGetter pod.Getter, desiredMemberCount, targetConfigVersion int, log *zap.SugaredLogger) (bool, error) { +func AllReachedGoalState(ctx context.Context, sts appsv1.StatefulSet, podGetter pod.Getter, desiredMemberCount, targetConfigVersion int, log *zap.SugaredLogger) (bool, error) { // AllReachedGoalState does not use desiredArbitersCount for backwards compatibility - podStates, err := GetAllDesiredMembersAndArbitersPodState(types.NamespacedName{ + podStates, err := GetAllDesiredMembersAndArbitersPodState(ctx, types.NamespacedName{ Namespace: sts.Namespace, Name: sts.Name, }, podGetter, desiredMemberCount, 0, targetConfigVersion, log) @@ -63,7 +64,7 @@ func AllReachedGoalState(sts appsv1.StatefulSet, podGetter pod.Getter, desiredMe // GetAllDesiredMembersAndArbitersPodState returns states of all desired pods in a replica set. // Pod names to search for are calculated using desiredMemberCount and desiredArbitersCount. Each pod is then checked if it exists // or if it reached goal state vs targetConfigVersion. -func GetAllDesiredMembersAndArbitersPodState(namespacedName types.NamespacedName, podGetter pod.Getter, desiredMembersCount, desiredArbitersCount, targetConfigVersion int, log *zap.SugaredLogger) ([]PodState, error) { +func GetAllDesiredMembersAndArbitersPodState(ctx context.Context, namespacedName types.NamespacedName, podGetter pod.Getter, desiredMembersCount, desiredArbitersCount, targetConfigVersion int, log *zap.SugaredLogger) ([]PodState, error) { podStates := make([]PodState, desiredMembersCount+desiredArbitersCount) membersPodNames := statefulSetPodNames(namespacedName.Name, desiredMembersCount) @@ -78,7 +79,7 @@ func GetAllDesiredMembersAndArbitersPodState(namespacedName types.NamespacedName IsArbiter: i >= len(membersPodNames), } - p, err := podGetter.GetPod(podNamespacedName) + p, err := podGetter.GetPod(ctx, podNamespacedName) if err != nil { if apiErrors.IsNotFound(err) { // we can skip below iteration and check for our goal state since the pod is not available yet diff --git a/pkg/agent/agent_readiness_test.go b/pkg/agent/agent_readiness_test.go index 685413322..2f898ad9d 100644 --- a/pkg/agent/agent_readiness_test.go +++ b/pkg/agent/agent_readiness_test.go @@ -1,6 +1,7 @@ package agent import ( + "context" "os" "testing" @@ -22,17 +23,18 @@ func init() { } func TestAllReachedGoalState(t *testing.T) { + ctx := context.Background() sts, err := statefulset.NewBuilder().SetName("sts").SetNamespace("test-ns").Build() assert.NoError(t, err) t.Run("Returns true if all pods are not found", func(t *testing.T) { - ready, err := AllReachedGoalState(sts, mockPodGetter{}, 3, 3, zap.S()) + ready, err := AllReachedGoalState(ctx, sts, mockPodGetter{}, 3, 3, zap.S()) assert.NoError(t, err) assert.True(t, ready) }) t.Run("Returns true if all pods are ready", func(t *testing.T) { - ready, err := AllReachedGoalState(sts, mockPodGetter{pods: []corev1.Pod{ + ready, err := AllReachedGoalState(ctx, sts, mockPodGetter{pods: []corev1.Pod{ createPodWithAgentAnnotation("3"), createPodWithAgentAnnotation("3"), createPodWithAgentAnnotation("3"), @@ -42,7 +44,7 @@ func TestAllReachedGoalState(t *testing.T) { }) t.Run("Returns false if one pod is not ready", func(t *testing.T) { - ready, err := AllReachedGoalState(sts, mockPodGetter{pods: []corev1.Pod{ + ready, err := AllReachedGoalState(ctx, sts, mockPodGetter{pods: []corev1.Pod{ createPodWithAgentAnnotation("2"), createPodWithAgentAnnotation("3"), createPodWithAgentAnnotation("3"), @@ -52,7 +54,7 @@ func TestAllReachedGoalState(t *testing.T) { }) t.Run("Returns true when the pods are not found", func(t *testing.T) { - ready, err := AllReachedGoalState(sts, mockPodGetter{shouldReturnNotFoundError: true}, 3, 3, zap.S()) + ready, err := AllReachedGoalState(ctx, sts, mockPodGetter{shouldReturnNotFoundError: true}, 3, 3, zap.S()) assert.NoError(t, err) assert.True(t, ready) }) @@ -92,7 +94,7 @@ type mockPodGetter struct { shouldReturnNotFoundError bool } -func (m mockPodGetter) GetPod(client.ObjectKey) (corev1.Pod, error) { +func (m mockPodGetter) GetPod(context.Context, client.ObjectKey) (corev1.Pod, error) { if m.shouldReturnNotFoundError || m.currPodIndex >= len(m.pods) { return corev1.Pod{}, notFoundError() } diff --git a/pkg/authentication/authentication.go b/pkg/authentication/authentication.go index 00bcfd83c..b73c97898 100644 --- a/pkg/authentication/authentication.go +++ b/pkg/authentication/authentication.go @@ -1,6 +1,7 @@ package authentication import ( + "context" "fmt" "k8s.io/apimachinery/pkg/types" @@ -13,19 +14,19 @@ import ( "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/constants" ) -func Enable(auth *automationconfig.Auth, secretGetUpdateCreateDeleter secret.GetUpdateCreateDeleter, mdb authtypes.Configurable, agentCertSecret types.NamespacedName) error { +func Enable(ctx context.Context, auth *automationconfig.Auth, secretGetUpdateCreateDeleter secret.GetUpdateCreateDeleter, mdb authtypes.Configurable, agentCertSecret types.NamespacedName) error { scramEnabled := false for _, authMode := range mdb.GetAuthOptions().AuthMechanisms { switch authMode { case constants.Sha1, constants.Sha256: if !scramEnabled { - if err := scram.Enable(auth, secretGetUpdateCreateDeleter, mdb); err != nil { + if err := scram.Enable(ctx, auth, secretGetUpdateCreateDeleter, mdb); err != nil { return fmt.Errorf("could not configure scram authentication: %s", err) } scramEnabled = true } case constants.X509: - if err := x509.Enable(auth, secretGetUpdateCreateDeleter, mdb, agentCertSecret); err != nil { + if err := x509.Enable(ctx, auth, secretGetUpdateCreateDeleter, mdb, agentCertSecret); err != nil { return fmt.Errorf("could not configure x509 authentication: %s", err) } } diff --git a/pkg/authentication/authentication_test.go b/pkg/authentication/authentication_test.go index e97b7327d..8521cbf1c 100644 --- a/pkg/authentication/authentication_test.go +++ b/pkg/authentication/authentication_test.go @@ -1,6 +1,7 @@ package authentication import ( + "context" "testing" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -16,6 +17,7 @@ import ( ) func TestEnable(t *testing.T) { + ctx := context.Background() t.Run("SCRAM only", func(t *testing.T) { auth := automationconfig.Auth{} user := mocks.BuildScramMongoDBUser("my-user") @@ -27,7 +29,7 @@ func TestEnable(t *testing.T) { Build() secrets := mocks.NewMockedSecretGetUpdateCreateDeleter(passwordSecret) - err := Enable(&auth, secrets, mdb, mdb.AgentCertificateSecretNamespacedName()) + err := Enable(ctx, &auth, secrets, mdb, mdb.AgentCertificateSecretNamespacedName()) assert.NoError(t, err) assert.Equal(t, false, auth.Disabled) @@ -49,7 +51,7 @@ func TestEnable(t *testing.T) { Build() secrets := mocks.NewMockedSecretGetUpdateCreateDeleter(passwordSecret) - err := Enable(&auth, secrets, mdb, mdb.AgentCertificateSecretNamespacedName()) + err := Enable(ctx, &auth, secrets, mdb, mdb.AgentCertificateSecretNamespacedName()) assert.NoError(t, err) assert.Equal(t, false, auth.Disabled) @@ -67,7 +69,7 @@ func TestEnable(t *testing.T) { agentSecret := x509.CreateAgentCertificateSecret("tls.crt", false, mdb.AgentCertificateSecretNamespacedName()) secrets := mocks.NewMockedSecretGetUpdateCreateDeleter(agentSecret) - err := Enable(&auth, secrets, mdb, mdb.AgentCertificateSecretNamespacedName()) + err := Enable(ctx, &auth, secrets, mdb, mdb.AgentCertificateSecretNamespacedName()) assert.NoError(t, err) assert.Equal(t, false, auth.Disabled) @@ -90,7 +92,7 @@ func TestEnable(t *testing.T) { Build() secrets := mocks.NewMockedSecretGetUpdateCreateDeleter(passwordSecret) - err := Enable(&auth, secrets, mdb, mdb.AgentCertificateSecretNamespacedName()) + err := Enable(ctx, &auth, secrets, mdb, mdb.AgentCertificateSecretNamespacedName()) assert.NoError(t, err) assert.Equal(t, false, auth.Disabled) @@ -115,7 +117,7 @@ func TestEnable(t *testing.T) { agentSecret := x509.CreateAgentCertificateSecret("tls.crt", false, mdb.AgentCertificateSecretNamespacedName()) secrets := mocks.NewMockedSecretGetUpdateCreateDeleter(passwordSecret, agentSecret) - err := Enable(&auth, secrets, mdb, mdb.AgentCertificateSecretNamespacedName()) + err := Enable(ctx, &auth, secrets, mdb, mdb.AgentCertificateSecretNamespacedName()) assert.NoError(t, err) assert.Equal(t, false, auth.Disabled) diff --git a/pkg/authentication/mocks/mocks.go b/pkg/authentication/mocks/mocks.go index c89e9d8e4..627a105be 100644 --- a/pkg/authentication/mocks/mocks.go +++ b/pkg/authentication/mocks/mocks.go @@ -1,6 +1,7 @@ package mocks import ( + "context" "fmt" corev1 "k8s.io/api/core/v1" @@ -26,21 +27,21 @@ func NewMockedSecretGetUpdateCreateDeleter(secrets ...corev1.Secret) secret.GetU return mockSecretGetUpdateCreateDeleter } -func (c MockSecretGetUpdateCreateDeleter) DeleteSecret(objectKey client.ObjectKey) error { - delete(c.secrets, objectKey) +func (c MockSecretGetUpdateCreateDeleter) DeleteSecret(_ context.Context, key client.ObjectKey) error { + delete(c.secrets, key) return nil } -func (c MockSecretGetUpdateCreateDeleter) UpdateSecret(s corev1.Secret) error { +func (c MockSecretGetUpdateCreateDeleter) UpdateSecret(_ context.Context, s corev1.Secret) error { c.secrets[types.NamespacedName{Name: s.Name, Namespace: s.Namespace}] = s return nil } -func (c MockSecretGetUpdateCreateDeleter) CreateSecret(secret corev1.Secret) error { - return c.UpdateSecret(secret) +func (c MockSecretGetUpdateCreateDeleter) CreateSecret(ctx context.Context, secret corev1.Secret) error { + return c.UpdateSecret(ctx, secret) } -func (c MockSecretGetUpdateCreateDeleter) GetSecret(objectKey client.ObjectKey) (corev1.Secret, error) { +func (c MockSecretGetUpdateCreateDeleter) GetSecret(_ context.Context, objectKey client.ObjectKey) (corev1.Secret, error) { if s, ok := c.secrets[objectKey]; !ok { return corev1.Secret{}, &errors.StatusError{ErrStatus: metav1.Status{Reason: metav1.StatusReasonNotFound}} } else { diff --git a/pkg/authentication/scram/scram.go b/pkg/authentication/scram/scram.go index 564596185..c21e185f1 100644 --- a/pkg/authentication/scram/scram.go +++ b/pkg/authentication/scram/scram.go @@ -1,6 +1,7 @@ package scram import ( + "context" "encoding/base64" "fmt" @@ -31,16 +32,16 @@ const ( // Enable will configure all of the required Kubernetes resources for SCRAM-SHA to be enabled. // The agent password and keyfile contents will be configured and stored in a secret. // the user credentials will be generated if not present, or existing credentials will be read. -func Enable(auth *automationconfig.Auth, secretGetUpdateCreateDeleter secret.GetUpdateCreateDeleter, mdb authtypes.Configurable) error { +func Enable(ctx context.Context, auth *automationconfig.Auth, secretGetUpdateCreateDeleter secret.GetUpdateCreateDeleter, mdb authtypes.Configurable) error { opts := mdb.GetAuthOptions() - desiredUsers, err := convertMongoDBResourceUsersToAutomationConfigUsers(secretGetUpdateCreateDeleter, mdb) + desiredUsers, err := convertMongoDBResourceUsersToAutomationConfigUsers(ctx, secretGetUpdateCreateDeleter, mdb) if err != nil { return fmt.Errorf("could not convert users to Automation Config users: %s", err) } if opts.AutoAuthMechanism == constants.Sha256 || opts.AutoAuthMechanism == constants.Sha1 { - if err := ensureAgent(auth, secretGetUpdateCreateDeleter, mdb); err != nil { + if err := ensureAgent(ctx, auth, secretGetUpdateCreateDeleter, mdb); err != nil { return err } } @@ -48,7 +49,7 @@ func Enable(auth *automationconfig.Auth, secretGetUpdateCreateDeleter secret.Get return enableClientAuthentication(auth, opts, desiredUsers) } -func ensureAgent(auth *automationconfig.Auth, secretGetUpdateCreateDeleter secret.GetUpdateCreateDeleter, mdb authtypes.Configurable) error { +func ensureAgent(ctx context.Context, auth *automationconfig.Auth, secretGetUpdateCreateDeleter secret.GetUpdateCreateDeleter, mdb authtypes.Configurable) error { generatedPassword, err := generate.RandomFixedLengthStringOfSize(20) if err != nil { return fmt.Errorf("could not generate password: %s", err) @@ -60,13 +61,13 @@ func ensureAgent(auth *automationconfig.Auth, secretGetUpdateCreateDeleter secre } // ensure that the agent password secret exists or read existing password. - agentPassword, err := secret.EnsureSecretWithKey(secretGetUpdateCreateDeleter, mdb.GetAgentPasswordSecretNamespacedName(), mdb.GetOwnerReferences(), constants.AgentPasswordKey, generatedPassword) + agentPassword, err := secret.EnsureSecretWithKey(ctx, secretGetUpdateCreateDeleter, mdb.GetAgentPasswordSecretNamespacedName(), mdb.GetOwnerReferences(), constants.AgentPasswordKey, generatedPassword) if err != nil { return err } // ensure that the agent keyfile secret exists or read existing keyfile. - agentKeyFile, err := secret.EnsureSecretWithKey(secretGetUpdateCreateDeleter, mdb.GetAgentKeyfileSecretNamespacedName(), mdb.GetOwnerReferences(), constants.AgentKeyfileKey, generatedContents) + agentKeyFile, err := secret.EnsureSecretWithKey(ctx, secretGetUpdateCreateDeleter, mdb.GetAgentKeyfileSecretNamespacedName(), mdb.GetOwnerReferences(), constants.AgentKeyfileKey, generatedContents) if err != nil { return err } @@ -76,14 +77,14 @@ func ensureAgent(auth *automationconfig.Auth, secretGetUpdateCreateDeleter secre // ensureScramCredentials will ensure that the ScramSha1 & ScramSha256 credentials exist and are stored in the credentials // secret corresponding to user of the given MongoDB deployment. -func ensureScramCredentials(getUpdateCreator secret.GetUpdateCreator, user authtypes.User, mdbNamespacedName types.NamespacedName, ownerRef []metav1.OwnerReference) (scramcredentials.ScramCreds, scramcredentials.ScramCreds, error) { +func ensureScramCredentials(ctx context.Context, getUpdateCreator secret.GetUpdateCreator, user authtypes.User, mdbNamespacedName types.NamespacedName, ownerRef []metav1.OwnerReference) (scramcredentials.ScramCreds, scramcredentials.ScramCreds, error) { - password, err := secret.ReadKey(getUpdateCreator, user.PasswordSecretKey, types.NamespacedName{Name: user.PasswordSecretName, Namespace: mdbNamespacedName.Namespace}) + password, err := secret.ReadKey(ctx, getUpdateCreator, user.PasswordSecretKey, types.NamespacedName{Name: user.PasswordSecretName, Namespace: mdbNamespacedName.Namespace}) if err != nil { // if the password is deleted, that's fine we can read from the stored credentials that were previously generated if secret.SecretNotExist(err) { zap.S().Debugf("password secret was not found, reading from credentials from secret/%s", user.ScramCredentialsSecretName) - return readExistingCredentials(getUpdateCreator, mdbNamespacedName, user.ScramCredentialsSecretName) + return readExistingCredentials(ctx, getUpdateCreator, mdbNamespacedName, user.ScramCredentialsSecretName) } return scramcredentials.ScramCreds{}, scramcredentials.ScramCreds{}, fmt.Errorf("could not read secret key: %s", err) } @@ -91,7 +92,7 @@ func ensureScramCredentials(getUpdateCreator secret.GetUpdateCreator, user autht // we should only need to generate new credentials in two situations. // 1. We are creating the credentials for the first time // 2. We are changing the password - shouldGenerateNewCredentials, err := needToGenerateNewCredentials(getUpdateCreator, user.Username, user.ScramCredentialsSecretName, mdbNamespacedName, password) + shouldGenerateNewCredentials, err := needToGenerateNewCredentials(ctx, getUpdateCreator, user.Username, user.ScramCredentialsSecretName, mdbNamespacedName, password) if err != nil { return scramcredentials.ScramCreds{}, scramcredentials.ScramCreds{}, fmt.Errorf("could not determine if new credentials need to be generated: %s", err) } @@ -99,7 +100,7 @@ func ensureScramCredentials(getUpdateCreator secret.GetUpdateCreator, user autht // there are no changes required, we can re-use the same credentials. if !shouldGenerateNewCredentials { zap.S().Debugf("Credentials have not changed, using credentials stored in: secret/%s", user.ScramCredentialsSecretName) - return readExistingCredentials(getUpdateCreator, mdbNamespacedName, user.ScramCredentialsSecretName) + return readExistingCredentials(ctx, getUpdateCreator, mdbNamespacedName, user.ScramCredentialsSecretName) } // the password has changed, or we are generating it for the first time @@ -110,7 +111,7 @@ func ensureScramCredentials(getUpdateCreator secret.GetUpdateCreator, user autht } // create or update our credentials secret for this user - if err := createScramCredentialsSecret(getUpdateCreator, mdbNamespacedName, ownerRef, user.ScramCredentialsSecretName, sha1Creds, sha256Creds); err != nil { + if err := createScramCredentialsSecret(ctx, getUpdateCreator, mdbNamespacedName, ownerRef, user.ScramCredentialsSecretName, sha1Creds, sha256Creds); err != nil { return scramcredentials.ScramCreds{}, scramcredentials.ScramCreds{}, fmt.Errorf("faild to create scram credentials secret %s: %s", user.ScramCredentialsSecretName, err) } @@ -120,8 +121,8 @@ func ensureScramCredentials(getUpdateCreator secret.GetUpdateCreator, user autht // needToGenerateNewCredentials determines if it is required to generate new credentials or not. // this will be the case if we are either changing password, or are generating credentials for the first time. -func needToGenerateNewCredentials(secretGetter secret.Getter, username, scramCredentialsSecretName string, mdbNamespacedName types.NamespacedName, password string) (bool, error) { - s, err := secretGetter.GetSecret(types.NamespacedName{Name: scramCredentialsSecretName, Namespace: mdbNamespacedName.Namespace}) +func needToGenerateNewCredentials(ctx context.Context, secretGetter secret.Getter, username, scramCredentialsSecretName string, mdbNamespacedName types.NamespacedName, password string) (bool, error) { + s, err := secretGetter.GetSecret(ctx, types.NamespacedName{Name: scramCredentialsSecretName, Namespace: mdbNamespacedName.Namespace}) if err != nil { // haven't generated credentials yet, so we are changing password if secret.SecretNotExist(err) { @@ -151,7 +152,7 @@ func needToGenerateNewCredentials(secretGetter secret.Getter, username, scramCre return false, err } - existingSha1Creds, existingSha256Creds, err := readExistingCredentials(secretGetter, mdbNamespacedName, scramCredentialsSecretName) + existingSha1Creds, existingSha256Creds, err := readExistingCredentials(ctx, secretGetter, mdbNamespacedName, scramCredentialsSecretName) if err != nil { return false, err } @@ -194,7 +195,7 @@ func computeScramShaCredentials(username, password string, sha1Salt, sha256Salt // createScramCredentialsSecret will create a Secret that contains all of the fields required to read these credentials // back in the future. -func createScramCredentialsSecret(getUpdateCreator secret.GetUpdateCreator, mdbObjectKey types.NamespacedName, ref []metav1.OwnerReference, scramCredentialsSecretName string, sha1Creds, sha256Creds scramcredentials.ScramCreds) error { +func createScramCredentialsSecret(ctx context.Context, getUpdateCreator secret.GetUpdateCreator, mdbObjectKey types.NamespacedName, ref []metav1.OwnerReference, scramCredentialsSecretName string, sha1Creds, sha256Creds scramcredentials.ScramCreds) error { scramCredsSecret := secret.Builder(). SetName(scramCredentialsSecretName). SetNamespace(mdbObjectKey.Namespace). @@ -206,12 +207,12 @@ func createScramCredentialsSecret(getUpdateCreator secret.GetUpdateCreator, mdbO SetField(sha256ServerKeyKey, sha256Creds.ServerKey). SetOwnerReferences(ref). Build() - return secret.CreateOrUpdate(getUpdateCreator, scramCredsSecret) + return secret.CreateOrUpdate(ctx, getUpdateCreator, scramCredsSecret) } // readExistingCredentials reads the existing set of credentials for both ScramSha 1 & 256 -func readExistingCredentials(secretGetter secret.Getter, mdbObjectKey types.NamespacedName, scramCredentialsSecretName string) (scramcredentials.ScramCreds, scramcredentials.ScramCreds, error) { - credentialsSecret, err := secretGetter.GetSecret(types.NamespacedName{Name: scramCredentialsSecretName, Namespace: mdbObjectKey.Namespace}) +func readExistingCredentials(ctx context.Context, secretGetter secret.Getter, mdbObjectKey types.NamespacedName, scramCredentialsSecretName string) (scramcredentials.ScramCreds, scramcredentials.ScramCreds, error) { + credentialsSecret, err := secretGetter.GetSecret(ctx, types.NamespacedName{Name: scramCredentialsSecretName, Namespace: mdbObjectKey.Namespace}) if err != nil { return scramcredentials.ScramCreds{}, scramcredentials.ScramCreds{}, fmt.Errorf("could not get secret %s/%s: %s", mdbObjectKey.Namespace, scramCredentialsSecretName, err) } @@ -239,11 +240,11 @@ func readExistingCredentials(secretGetter secret.Getter, mdbObjectKey types.Name } // convertMongoDBResourceUsersToAutomationConfigUsers returns a list of users that are able to be set in the AutomationConfig -func convertMongoDBResourceUsersToAutomationConfigUsers(secretGetUpdateCreateDeleter secret.GetUpdateCreateDeleter, mdb authtypes.Configurable) ([]automationconfig.MongoDBUser, error) { +func convertMongoDBResourceUsersToAutomationConfigUsers(ctx context.Context, secretGetUpdateCreateDeleter secret.GetUpdateCreateDeleter, mdb authtypes.Configurable) ([]automationconfig.MongoDBUser, error) { var usersWanted []automationconfig.MongoDBUser for _, u := range mdb.GetAuthUsers() { if u.Database != constants.ExternalDB { - acUser, err := convertMongoDBUserToAutomationConfigUser(secretGetUpdateCreateDeleter, mdb.NamespacedName(), mdb.GetOwnerReferences(), u) + acUser, err := convertMongoDBUserToAutomationConfigUser(ctx, secretGetUpdateCreateDeleter, mdb.NamespacedName(), mdb.GetOwnerReferences(), u) if err != nil { return nil, fmt.Errorf("failed to convert scram user %s to Automation Config user: %s", u.Username, err) } @@ -255,7 +256,7 @@ func convertMongoDBResourceUsersToAutomationConfigUsers(secretGetUpdateCreateDel // convertMongoDBUserToAutomationConfigUser converts a single user configured in the MongoDB resource and converts it to a user // that can be added directly to the AutomationConfig. -func convertMongoDBUserToAutomationConfigUser(secretGetUpdateCreateDeleter secret.GetUpdateCreateDeleter, mdbNsName types.NamespacedName, ownerRef []metav1.OwnerReference, user authtypes.User) (automationconfig.MongoDBUser, error) { +func convertMongoDBUserToAutomationConfigUser(ctx context.Context, secretGetUpdateCreateDeleter secret.GetUpdateCreateDeleter, mdbNsName types.NamespacedName, ownerRef []metav1.OwnerReference, user authtypes.User) (automationconfig.MongoDBUser, error) { acUser := automationconfig.MongoDBUser{ Username: user.Username, Database: user.Database, @@ -266,7 +267,7 @@ func convertMongoDBUserToAutomationConfigUser(secretGetUpdateCreateDeleter secre Database: role.Database, }) } - sha1Creds, sha256Creds, err := ensureScramCredentials(secretGetUpdateCreateDeleter, user, mdbNsName, ownerRef) + sha1Creds, sha256Creds, err := ensureScramCredentials(ctx, secretGetUpdateCreateDeleter, user, mdbNsName, ownerRef) if err != nil { return automationconfig.MongoDBUser{}, fmt.Errorf("could not ensure scram credentials: %s", err) } diff --git a/pkg/authentication/scram/scram_test.go b/pkg/authentication/scram/scram_test.go index f6b9aedaf..dd43ffc9c 100644 --- a/pkg/authentication/scram/scram_test.go +++ b/pkg/authentication/scram/scram_test.go @@ -1,6 +1,7 @@ package scram import ( + "context" "os" "reflect" "testing" @@ -39,23 +40,24 @@ const ( ) func TestReadExistingCredentials(t *testing.T) { + ctx := context.Background() mdbObjectKey := types.NamespacedName{Name: "mdb-0", Namespace: "default"} user := mocks.BuildScramMongoDBUser("mdbuser-0") t.Run("credentials are successfully generated when all fields are present", func(t *testing.T) { scramCredsSecret := validScramCredentialsSecret(mdbObjectKey, user.ScramCredentialsSecretName) - scram1Creds, scram256Creds, err := readExistingCredentials(mocks.NewMockedSecretGetUpdateCreateDeleter(scramCredsSecret), mdbObjectKey, user.ScramCredentialsSecretName) + scram1Creds, scram256Creds, err := readExistingCredentials(ctx, mocks.NewMockedSecretGetUpdateCreateDeleter(scramCredsSecret), mdbObjectKey, user.ScramCredentialsSecretName) assert.NoError(t, err) assertScramCredsCredentialsValidity(t, scram1Creds, scram256Creds) }) t.Run("credentials are not generated if a field is missing", func(t *testing.T) { scramCredsSecret := invalidSecret(mdbObjectKey, user.ScramCredentialsSecretName) - _, _, err := readExistingCredentials(mocks.NewMockedSecretGetUpdateCreateDeleter(scramCredsSecret), mdbObjectKey, user.ScramCredentialsSecretName) + _, _, err := readExistingCredentials(ctx, mocks.NewMockedSecretGetUpdateCreateDeleter(scramCredsSecret), mdbObjectKey, user.ScramCredentialsSecretName) assert.Error(t, err) }) t.Run("credentials are not generated if the secret does not exist", func(t *testing.T) { scramCredsSecret := validScramCredentialsSecret(mdbObjectKey, user.ScramCredentialsSecretName) - _, _, err := readExistingCredentials(mocks.NewMockedSecretGetUpdateCreateDeleter(scramCredsSecret), mdbObjectKey, "different-username") + _, _, err := readExistingCredentials(ctx, mocks.NewMockedSecretGetUpdateCreateDeleter(scramCredsSecret), mdbObjectKey, "different-username") assert.Error(t, err) }) } @@ -79,14 +81,15 @@ func TestComputeScramCredentials_ComputesSameStoredAndServerKey_WithSameSalt(t * } func TestEnsureScramCredentials(t *testing.T) { + ctx := context.Background() mdb, user := buildConfigurableAndUser("mdb-0") t.Run("Fails when there is no password secret, and no credentials secret", func(t *testing.T) { - _, _, err := ensureScramCredentials(mocks.NewMockedSecretGetUpdateCreateDeleter(), user, mdb.NamespacedName(), nil) + _, _, err := ensureScramCredentials(ctx, mocks.NewMockedSecretGetUpdateCreateDeleter(), user, mdb.NamespacedName(), nil) assert.Error(t, err) }) t.Run("Existing credentials are used when password does not exist, but credentials secret has been created", func(t *testing.T) { scramCredentialsSecret := validScramCredentialsSecret(mdb.NamespacedName(), user.ScramCredentialsSecretName) - scram1Creds, scram256Creds, err := ensureScramCredentials(mocks.NewMockedSecretGetUpdateCreateDeleter(scramCredentialsSecret), user, mdb.NamespacedName(), nil) + scram1Creds, scram256Creds, err := ensureScramCredentials(ctx, mocks.NewMockedSecretGetUpdateCreateDeleter(scramCredentialsSecret), user, mdb.NamespacedName(), nil) assert.NoError(t, err) assertScramCredsCredentialsValidity(t, scram1Creds, scram256Creds) }) @@ -101,7 +104,7 @@ func TestEnsureScramCredentials(t *testing.T) { Build() scramCredentialsSecret := validScramCredentialsSecret(mdb.NamespacedName(), user.ScramCredentialsSecretName) - scram1Creds, scram256Creds, err := ensureScramCredentials(mocks.NewMockedSecretGetUpdateCreateDeleter(scramCredentialsSecret, differentPasswordSecret), user, mdb.NamespacedName(), nil) + scram1Creds, scram256Creds, err := ensureScramCredentials(ctx, mocks.NewMockedSecretGetUpdateCreateDeleter(scramCredentialsSecret, differentPasswordSecret), user, mdb.NamespacedName(), nil) assert.NoError(t, err) assert.NotEqual(t, testSha1Salt, scram1Creds.Salt) assert.NotEmpty(t, scram1Creds.Salt) @@ -122,6 +125,7 @@ func TestEnsureScramCredentials(t *testing.T) { } func TestConvertMongoDBUserToAutomationConfigUser(t *testing.T) { + ctx := context.Background() mdb, user := buildConfigurableAndUser("mdb-0") t.Run("When password exists, the user is created in the automation config", func(t *testing.T) { @@ -131,7 +135,7 @@ func TestConvertMongoDBUserToAutomationConfigUser(t *testing.T) { SetField(user.PasswordSecretKey, "TDg_DESiScDrJV6"). Build() - acUser, err := convertMongoDBUserToAutomationConfigUser(mocks.NewMockedSecretGetUpdateCreateDeleter(passwordSecret), mdb.NamespacedName(), nil, user) + acUser, err := convertMongoDBUserToAutomationConfigUser(ctx, mocks.NewMockedSecretGetUpdateCreateDeleter(passwordSecret), mdb.NamespacedName(), nil, user) assert.NoError(t, err) assert.Equal(t, user.Username, acUser.Username) @@ -146,18 +150,19 @@ func TestConvertMongoDBUserToAutomationConfigUser(t *testing.T) { }) t.Run("If there is no password secret, the creation fails", func(t *testing.T) { - _, err := convertMongoDBUserToAutomationConfigUser(mocks.NewMockedSecretGetUpdateCreateDeleter(), mdb.NamespacedName(), nil, user) + _, err := convertMongoDBUserToAutomationConfigUser(ctx, mocks.NewMockedSecretGetUpdateCreateDeleter(), mdb.NamespacedName(), nil, user) assert.Error(t, err) }) } func TestConfigureScram(t *testing.T) { + ctx := context.Background() t.Run("Should fail if there is no password present for the user", func(t *testing.T) { mdb, _ := buildConfigurableAndUser("mdb-0") s := mocks.NewMockedSecretGetUpdateCreateDeleter() auth := automationconfig.Auth{} - err := Enable(&auth, s, mdb) + err := Enable(ctx, &auth, s, mdb) assert.Error(t, err) }) @@ -165,15 +170,15 @@ func TestConfigureScram(t *testing.T) { mdb := buildConfigurable("mdb-0") s := mocks.NewMockedSecretGetUpdateCreateDeleter() auth := automationconfig.Auth{} - err := Enable(&auth, s, mdb) + err := Enable(ctx, &auth, s, mdb) assert.NoError(t, err) - passwordSecret, err := s.GetSecret(mdb.GetAgentPasswordSecretNamespacedName()) + passwordSecret, err := s.GetSecret(ctx, mdb.GetAgentPasswordSecretNamespacedName()) assert.NoError(t, err) assert.True(t, secret.HasAllKeys(passwordSecret, constants.AgentPasswordKey)) assert.NotEmpty(t, passwordSecret.Data[constants.AgentPasswordKey]) - keyfileSecret, err := s.GetSecret(mdb.GetAgentKeyfileSecretNamespacedName()) + keyfileSecret, err := s.GetSecret(ctx, mdb.GetAgentKeyfileSecretNamespacedName()) assert.NoError(t, err) assert.True(t, secret.HasAllKeys(keyfileSecret, constants.AgentKeyfileKey)) assert.NotEmpty(t, keyfileSecret.Data[constants.AgentKeyfileKey]) @@ -183,10 +188,10 @@ func TestConfigureScram(t *testing.T) { mdb := buildConfigurable("mdb-0") s := mocks.NewMockedSecretGetUpdateCreateDeleter() auth := automationconfig.Auth{} - err := Enable(&auth, s, mdb) + err := Enable(ctx, &auth, s, mdb) assert.NoError(t, err) - passwordSecret, err := s.GetSecret(mdb.GetAgentPasswordSecretNamespacedName()) + passwordSecret, err := s.GetSecret(ctx, mdb.GetAgentPasswordSecretNamespacedName()) assert.NoError(t, err) actualRef := passwordSecret.GetOwnerReferences() @@ -209,10 +214,10 @@ func TestConfigureScram(t *testing.T) { s := mocks.NewMockedSecretGetUpdateCreateDeleter(agentPasswordSecret) auth := automationconfig.Auth{} - err := Enable(&auth, s, mdb) + err := Enable(ctx, &auth, s, mdb) assert.NoError(t, err) - ps, err := s.GetSecret(mdb.GetAgentPasswordSecretNamespacedName()) + ps, err := s.GetSecret(ctx, mdb.GetAgentPasswordSecretNamespacedName()) assert.NoError(t, err) assert.True(t, secret.HasAllKeys(ps, constants.AgentPasswordKey)) assert.NotEmpty(t, ps.Data[constants.AgentPasswordKey]) @@ -231,10 +236,10 @@ func TestConfigureScram(t *testing.T) { s := mocks.NewMockedSecretGetUpdateCreateDeleter(keyfileSecret) auth := automationconfig.Auth{} - err := Enable(&auth, s, mdb) + err := Enable(ctx, &auth, s, mdb) assert.NoError(t, err) - ks, err := s.GetSecret(mdb.GetAgentKeyfileSecretNamespacedName()) + ks, err := s.GetSecret(ctx, mdb.GetAgentKeyfileSecretNamespacedName()) assert.NoError(t, err) assert.True(t, secret.HasAllKeys(ks, constants.AgentKeyfileKey)) assert.Equal(t, "RuPeMaIe2g0SNTTa", string(ks.Data[constants.AgentKeyfileKey])) @@ -245,7 +250,7 @@ func TestConfigureScram(t *testing.T) { mdb := buildConfigurable("mdb-0") s := mocks.NewMockedSecretGetUpdateCreateDeleter() auth := automationconfig.Auth{} - err := Enable(&auth, s, mdb) + err := Enable(ctx, &auth, s, mdb) assert.NoError(t, err) }) } diff --git a/pkg/authentication/x509/x509.go b/pkg/authentication/x509/x509.go index 1506471ae..20297e35f 100644 --- a/pkg/authentication/x509/x509.go +++ b/pkg/authentication/x509/x509.go @@ -2,6 +2,7 @@ package x509 import ( "bytes" + "context" "crypto/ecdsa" "crypto/elliptic" "crypto/rand" @@ -27,13 +28,13 @@ import ( // Enable will configure all of the required Kubernetes resources for X509 to be enabled. // The agent password and keyfile contents will be configured and stored in a secret. // the user credentials will be generated if not present, or existing credentials will be read. -func Enable(auth *automationconfig.Auth, secretGetUpdateCreateDeleter secret.GetUpdateCreateDeleter, mdb authtypes.Configurable, agentCertSecret types.NamespacedName) error { +func Enable(ctx context.Context, auth *automationconfig.Auth, secretGetUpdateCreateDeleter secret.GetUpdateCreateDeleter, mdb authtypes.Configurable, agentCertSecret types.NamespacedName) error { opts := mdb.GetAuthOptions() desiredUsers := convertMongoDBResourceUsersToAutomationConfigUsers(mdb) if opts.AutoAuthMechanism == constants.X509 { - if err := ensureAgent(auth, secretGetUpdateCreateDeleter, mdb, agentCertSecret); err != nil { + if err := ensureAgent(ctx, auth, secretGetUpdateCreateDeleter, mdb, agentCertSecret); err != nil { return err } } @@ -41,19 +42,19 @@ func Enable(auth *automationconfig.Auth, secretGetUpdateCreateDeleter secret.Get return enableClientAuthentication(auth, opts, desiredUsers) } -func ensureAgent(auth *automationconfig.Auth, secretGetUpdateCreateDeleter secret.GetUpdateCreateDeleter, mdb authtypes.Configurable, agentCertSecret types.NamespacedName) error { +func ensureAgent(ctx context.Context, auth *automationconfig.Auth, secretGetUpdateCreateDeleter secret.GetUpdateCreateDeleter, mdb authtypes.Configurable, agentCertSecret types.NamespacedName) error { generatedContents, err := generate.KeyFileContents() if err != nil { return fmt.Errorf("could not generate keyfile contents: %s", err) } // ensure that the agent keyfile secret exists or read existing keyfile. - agentKeyFile, err := secret.EnsureSecretWithKey(secretGetUpdateCreateDeleter, mdb.GetAgentKeyfileSecretNamespacedName(), mdb.GetOwnerReferences(), constants.AgentKeyfileKey, generatedContents) + agentKeyFile, err := secret.EnsureSecretWithKey(ctx, secretGetUpdateCreateDeleter, mdb.GetAgentKeyfileSecretNamespacedName(), mdb.GetOwnerReferences(), constants.AgentKeyfileKey, generatedContents) if err != nil { return err } - agentCert, err := secret.ReadKey(secretGetUpdateCreateDeleter, "tls.crt", agentCertSecret) + agentCert, err := secret.ReadKey(ctx, secretGetUpdateCreateDeleter, "tls.crt", agentCertSecret) if err != nil { return err } diff --git a/pkg/authentication/x509/x509_test.go b/pkg/authentication/x509/x509_test.go index 13a4efb95..ed4f728fc 100644 --- a/pkg/authentication/x509/x509_test.go +++ b/pkg/authentication/x509/x509_test.go @@ -1,6 +1,7 @@ package x509 import ( + "context" "reflect" "testing" @@ -16,6 +17,7 @@ import ( ) func TestEnable(t *testing.T) { + ctx := context.Background() t.Run("X509 agent", func(t *testing.T) { auth := automationconfig.Auth{} mdb := buildX509Configurable("mdb", mocks.BuildX509MongoDBUser("my-user"), mocks.BuildScramMongoDBUser("my-scram-user")) @@ -28,7 +30,7 @@ func TestEnable(t *testing.T) { Build() secrets := mocks.NewMockedSecretGetUpdateCreateDeleter(agentSecret, keyfileSecret) - err := Enable(&auth, secrets, mdb, mdb.AgentCertificateSecretNamespacedName()) + err := Enable(ctx, &auth, secrets, mdb, mdb.AgentCertificateSecretNamespacedName()) assert.NoError(t, err) expected := automationconfig.Auth{ @@ -70,7 +72,7 @@ func TestEnable(t *testing.T) { secrets := mocks.NewMockedSecretGetUpdateCreateDeleter() - err := Enable(&auth, secrets, mdb, mdb.AgentCertificateSecretNamespacedName()) + err := Enable(ctx, &auth, secrets, mdb, mdb.AgentCertificateSecretNamespacedName()) assert.NoError(t, err) expected := automationconfig.Auth{ @@ -100,18 +102,19 @@ func TestEnable(t *testing.T) { } func Test_ensureAgent(t *testing.T) { + ctx := context.Background() auth := automationconfig.Auth{} mdb := buildX509Configurable("mdb") secrets := mocks.NewMockedSecretGetUpdateCreateDeleter() - err := ensureAgent(&auth, secrets, mdb, mdb.AgentCertificateSecretNamespacedName()) + err := ensureAgent(ctx, &auth, secrets, mdb, mdb.AgentCertificateSecretNamespacedName()) assert.Error(t, err) auth = automationconfig.Auth{} agentSecret := CreateAgentCertificateSecret("tls.pem", false, mdb.AgentCertificateSecretNamespacedName()) secrets = mocks.NewMockedSecretGetUpdateCreateDeleter(agentSecret) - err = ensureAgent(&auth, secrets, mdb, mdb.AgentCertificateSecretNamespacedName()) + err = ensureAgent(ctx, &auth, secrets, mdb, mdb.AgentCertificateSecretNamespacedName()) assert.Error(t, err) assert.ErrorContains(t, err, "key \"tls.crt\" not present in the Secret") @@ -119,7 +122,7 @@ func Test_ensureAgent(t *testing.T) { agentSecret = CreateAgentCertificateSecret("tls.crt", true, mdb.AgentCertificateSecretNamespacedName()) secrets = mocks.NewMockedSecretGetUpdateCreateDeleter(agentSecret) - err = ensureAgent(&auth, secrets, mdb, mdb.AgentCertificateSecretNamespacedName()) + err = ensureAgent(ctx, &auth, secrets, mdb, mdb.AgentCertificateSecretNamespacedName()) assert.Error(t, err) assert.ErrorContains(t, err, "x509: malformed certificate") @@ -127,7 +130,7 @@ func Test_ensureAgent(t *testing.T) { agentSecret = CreateAgentCertificateSecret("tls.crt", false, mdb.AgentCertificateSecretNamespacedName()) secrets = mocks.NewMockedSecretGetUpdateCreateDeleter(agentSecret) - err = ensureAgent(&auth, secrets, mdb, mdb.AgentCertificateSecretNamespacedName()) + err = ensureAgent(ctx, &auth, secrets, mdb, mdb.AgentCertificateSecretNamespacedName()) assert.NoError(t, err) } diff --git a/pkg/automationconfig/automation_config_secret.go b/pkg/automationconfig/automation_config_secret.go index c27aa766e..9ca6ed469 100644 --- a/pkg/automationconfig/automation_config_secret.go +++ b/pkg/automationconfig/automation_config_secret.go @@ -1,6 +1,7 @@ package automationconfig import ( + "context" "encoding/json" "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/secret" @@ -12,8 +13,8 @@ const ConfigKey = "cluster-config.json" // ReadFromSecret returns the AutomationConfig present in the given Secret. If the Secret is not // found, it is not considered an error and an empty AutomationConfig is returned. -func ReadFromSecret(secretGetter secret.Getter, secretNsName types.NamespacedName) (AutomationConfig, error) { - acSecret, err := secretGetter.GetSecret(secretNsName) +func ReadFromSecret(ctx context.Context, secretGetter secret.Getter, secretNsName types.NamespacedName) (AutomationConfig, error) { + acSecret, err := secretGetter.GetSecret(ctx, secretNsName) if err != nil { if secret.SecretNotExist(err) { err = nil @@ -27,11 +28,11 @@ func ReadFromSecret(secretGetter secret.Getter, secretNsName types.NamespacedNam // if the desired config is the same as the current contents, no change is made. // The most recent AutomationConfig is returned. If no change is made, it will return the existing one, if there // is a change, the new AutomationConfig is returned. -func EnsureSecret(secretGetUpdateCreator secret.GetUpdateCreator, secretNsName types.NamespacedName, owner []metav1.OwnerReference, desiredAutomationConfig AutomationConfig) (AutomationConfig, error) { - existingSecret, err := secretGetUpdateCreator.GetSecret(secretNsName) +func EnsureSecret(ctx context.Context, secretGetUpdateCreator secret.GetUpdateCreator, secretNsName types.NamespacedName, owner []metav1.OwnerReference, desiredAutomationConfig AutomationConfig) (AutomationConfig, error) { + existingSecret, err := secretGetUpdateCreator.GetSecret(ctx, secretNsName) if err != nil { if secret.SecretNotExist(err) { - return createNewAutomationConfigSecret(secretGetUpdateCreator, secretNsName, owner, desiredAutomationConfig) + return createNewAutomationConfigSecret(ctx, secretGetUpdateCreator, secretNsName, owner, desiredAutomationConfig) } return AutomationConfig{}, err } @@ -62,10 +63,10 @@ func EnsureSecret(secretGetUpdateCreator secret.GetUpdateCreator, secretNsName t existingSecret.Name = secretNsName.Name existingSecret.Namespace = secretNsName.Namespace - return desiredAutomationConfig, secretGetUpdateCreator.UpdateSecret(existingSecret) + return desiredAutomationConfig, secretGetUpdateCreator.UpdateSecret(ctx, existingSecret) } -func createNewAutomationConfigSecret(secretGetUpdateCreator secret.GetUpdateCreator, secretNsName types.NamespacedName, owner []metav1.OwnerReference, desiredAutomation AutomationConfig) (AutomationConfig, error) { +func createNewAutomationConfigSecret(ctx context.Context, secretGetUpdateCreator secret.GetUpdateCreator, secretNsName types.NamespacedName, owner []metav1.OwnerReference, desiredAutomation AutomationConfig) (AutomationConfig, error) { acBytes, err := json.Marshal(desiredAutomation) if err != nil { return AutomationConfig{}, err @@ -78,7 +79,7 @@ func createNewAutomationConfigSecret(secretGetUpdateCreator secret.GetUpdateCrea SetOwnerReferences(owner). Build() - if err := secretGetUpdateCreator.CreateSecret(newSecret); err != nil { + if err := secretGetUpdateCreator.CreateSecret(ctx, newSecret); err != nil { return AutomationConfig{}, err } return desiredAutomation, nil diff --git a/pkg/automationconfig/automation_config_secret_test.go b/pkg/automationconfig/automation_config_secret_test.go index 0eb85d809..e6d601e0e 100644 --- a/pkg/automationconfig/automation_config_secret_test.go +++ b/pkg/automationconfig/automation_config_secret_test.go @@ -1,6 +1,7 @@ package automationconfig import ( + "context" "encoding/json" "testing" @@ -14,6 +15,7 @@ import ( ) func TestEnsureSecret(t *testing.T) { + ctx := context.Background() secretNsName := types.NamespacedName{Name: "ac-secret", Namespace: "test-namespace"} desiredAutomationConfig, err := newAutomationConfig() assert.NoError(t, err) @@ -27,11 +29,11 @@ func TestEnsureSecret(t *testing.T) { secretGetUpdateCreator := &mockSecretGetUpdateCreator{secret: &s} - ac, err := EnsureSecret(secretGetUpdateCreator, secretNsName, []metav1.OwnerReference{}, desiredAutomationConfig) + ac, err := EnsureSecret(ctx, secretGetUpdateCreator, secretNsName, []metav1.OwnerReference{}, desiredAutomationConfig) assert.NoError(t, err) assert.Equal(t, desiredAutomationConfig, ac, "The config should be returned if there is not one currently.") - acSecret, err := secretGetUpdateCreator.GetSecret(secretNsName) + acSecret, err := secretGetUpdateCreator.GetSecret(ctx, secretNsName) assert.NoError(t, err) assert.Contains(t, acSecret.Data, ConfigKey, "The secret of the given name should have been updated with the config.") @@ -39,6 +41,7 @@ func TestEnsureSecret(t *testing.T) { }) t.Run("test LogRotate marshal and unmarshal", func(t *testing.T) { + ctx := context.Background() desiredAutomationConfig, err = NewBuilder().SetMembers(3).AddProcessModification(func(i_ int, p *Process) { p.SetLogRotate(&CrdLogRotate{ @@ -61,7 +64,7 @@ func TestEnsureSecret(t *testing.T) { secretGetUpdateCreator := &mockSecretGetUpdateCreator{secret: &s} - ac, err := EnsureSecret(secretGetUpdateCreator, secretNsName, []metav1.OwnerReference{}, desiredAutomationConfig) + ac, err := EnsureSecret(ctx, secretGetUpdateCreator, secretNsName, []metav1.OwnerReference{}, desiredAutomationConfig) assert.NoError(t, err) assert.Equal(t, desiredAutomationConfig, ac, "The config should be returned if there is not one currently.") @@ -73,6 +76,7 @@ func TestEnsureSecret(t *testing.T) { }) t.Run("test LogRotate marshal and unmarshal if not set", func(t *testing.T) { + ctx := context.Background() desiredAutomationConfig, err = NewBuilder().SetMembers(3).AddProcessModification(func(i_ int, p *Process) {}).Build() assert.NoError(t, err) @@ -84,7 +88,7 @@ func TestEnsureSecret(t *testing.T) { secretGetUpdateCreator := &mockSecretGetUpdateCreator{secret: &s} - ac, err := EnsureSecret(secretGetUpdateCreator, secretNsName, []metav1.OwnerReference{}, desiredAutomationConfig) + ac, err := EnsureSecret(ctx, secretGetUpdateCreator, secretNsName, []metav1.OwnerReference{}, desiredAutomationConfig) assert.NoError(t, err) assert.Equal(t, desiredAutomationConfig, ac, "The config should be returned if there is not one currently.") @@ -107,7 +111,7 @@ func TestEnsureSecret(t *testing.T) { newAc, err := newAutomationConfigBuilder().SetDomain("different-domain").Build() assert.NoError(t, err) - res, err := EnsureSecret(secretGetUpdateCreator, secretNsName, []metav1.OwnerReference{}, newAc) + res, err := EnsureSecret(ctx, secretGetUpdateCreator, secretNsName, []metav1.OwnerReference{}, newAc) assert.NoError(t, err) assert.Equal(t, newAc, res) @@ -140,7 +144,7 @@ type mockSecretGetUpdateCreator struct { secret *corev1.Secret } -func (m *mockSecretGetUpdateCreator) GetSecret(objectKey client.ObjectKey) (corev1.Secret, error) { +func (m *mockSecretGetUpdateCreator) GetSecret(ctx context.Context, objectKey client.ObjectKey) (corev1.Secret, error) { if m.secret != nil { if objectKey.Name == m.secret.Name && objectKey.Namespace == m.secret.Namespace { return *m.secret, nil @@ -149,12 +153,12 @@ func (m *mockSecretGetUpdateCreator) GetSecret(objectKey client.ObjectKey) (core return corev1.Secret{}, notFoundError() } -func (m *mockSecretGetUpdateCreator) UpdateSecret(secret corev1.Secret) error { +func (m *mockSecretGetUpdateCreator) UpdateSecret(ctx context.Context, secret corev1.Secret) error { m.secret = &secret return nil } -func (m *mockSecretGetUpdateCreator) CreateSecret(secret corev1.Secret) error { +func (m *mockSecretGetUpdateCreator) CreateSecret(ctx context.Context, secret corev1.Secret) error { if m.secret == nil { m.secret = &secret return nil diff --git a/pkg/kube/annotations/annotations.go b/pkg/kube/annotations/annotations.go index 9f508b990..44f5e9695 100644 --- a/pkg/kube/annotations/annotations.go +++ b/pkg/kube/annotations/annotations.go @@ -35,9 +35,9 @@ func GetAnnotation(object client.Object, key string) string { } // SetAnnotations updates the objects.Annotation with the supplied annotation and does the same with the object backed in kubernetes. -func SetAnnotations(object client.Object, annotations map[string]string, kubeClient client.Client) error { +func SetAnnotations(ctx context.Context, object client.Object, annotations map[string]string, kubeClient client.Client) error { currentObject := object.DeepCopyObject().(client.Object) - err := kubeClient.Get(context.TODO(), types.NamespacedName{Name: object.GetName(), Namespace: object.GetNamespace()}, currentObject) + err := kubeClient.Get(ctx, types.NamespacedName{Name: object.GetName(), Namespace: object.GetNamespace()}, currentObject) if err != nil { return err } @@ -68,17 +68,17 @@ func SetAnnotations(object client.Object, annotations map[string]string, kubeCli } patch := client.RawPatch(types.JSONPatchType, data) - if err = kubeClient.Patch(context.TODO(), currentObject, patch); err != nil { + if err = kubeClient.Patch(ctx, currentObject, patch); err != nil { return err } object.SetAnnotations(currentObject.GetAnnotations()) return nil } -func UpdateLastAppliedMongoDBVersion(mdb Versioned, kubeClient client.Client) error { +func UpdateLastAppliedMongoDBVersion(ctx context.Context, mdb Versioned, kubeClient client.Client) error { annotations := map[string]string{ LastAppliedMongoDBVersion: mdb.GetMongoDBVersionForAnnotation(), } - return SetAnnotations(mdb, annotations, kubeClient) + return SetAnnotations(ctx, mdb, annotations, kubeClient) } diff --git a/pkg/kube/client/client.go b/pkg/kube/client/client.go index f70048226..640e23373 100644 --- a/pkg/kube/client/client.go +++ b/pkg/kube/client/client.go @@ -27,7 +27,7 @@ type Client interface { k8sClient.Client KubernetesSecretClient // TODO: remove this function, add mongodb package which has GetAndUpdate function - GetAndUpdate(nsName types.NamespacedName, obj k8sClient.Object, updateFunc func()) error + GetAndUpdate(ctx context.Context, nsName types.NamespacedName, obj k8sClient.Object, updateFunc func()) error configmap.GetUpdateCreateDeleter service.GetUpdateCreateDeleter statefulset.GetUpdateCreateDeleter @@ -45,119 +45,119 @@ type client struct { // GetAndUpdate fetches the most recent version of the runtime.Object with the provided // nsName and applies the update function. The update function should update "obj" from // an outer scope -func (c client) GetAndUpdate(nsName types.NamespacedName, obj k8sClient.Object, updateFunc func()) error { - err := c.Get(context.TODO(), nsName, obj) +func (c client) GetAndUpdate(ctx context.Context, nsName types.NamespacedName, obj k8sClient.Object, updateFunc func()) error { + err := c.Get(ctx, nsName, obj) if err != nil { return err } // apply the function on the most recent version of the resource updateFunc() - return c.Update(context.TODO(), obj) + return c.Update(ctx, obj) } // GetConfigMap provides a thin wrapper and client.client to access corev1.ConfigMap types -func (c client) GetConfigMap(objectKey k8sClient.ObjectKey) (corev1.ConfigMap, error) { +func (c client) GetConfigMap(ctx context.Context, objectKey k8sClient.ObjectKey) (corev1.ConfigMap, error) { cm := corev1.ConfigMap{} - if err := c.Get(context.TODO(), objectKey, &cm); err != nil { + if err := c.Get(ctx, objectKey, &cm); err != nil { return corev1.ConfigMap{}, err } return cm, nil } // UpdateConfigMap provides a thin wrapper and client.Client to update corev1.ConfigMap types -func (c client) UpdateConfigMap(cm corev1.ConfigMap) error { - return c.Update(context.TODO(), &cm) +func (c client) UpdateConfigMap(ctx context.Context, cm corev1.ConfigMap) error { + return c.Update(ctx, &cm) } // CreateConfigMap provides a thin wrapper and client.Client to create corev1.ConfigMap types -func (c client) CreateConfigMap(cm corev1.ConfigMap) error { - return c.Create(context.TODO(), &cm) +func (c client) CreateConfigMap(ctx context.Context, cm corev1.ConfigMap) error { + return c.Create(ctx, &cm) } // DeleteConfigMap deletes the configmap of the given object key -func (c client) DeleteConfigMap(key k8sClient.ObjectKey) error { +func (c client) DeleteConfigMap(ctx context.Context, key k8sClient.ObjectKey) error { cm := corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: key.Name, Namespace: key.Namespace, }, } - return c.Delete(context.TODO(), &cm) + return c.Delete(ctx, &cm) } // GetPod provides a thin wrapper and client.client to access corev1.Pod types. -func (c client) GetPod(objectKey k8sClient.ObjectKey) (corev1.Pod, error) { +func (c client) GetPod(ctx context.Context, objectKey k8sClient.ObjectKey) (corev1.Pod, error) { p := corev1.Pod{} - if err := c.Get(context.TODO(), objectKey, &p); err != nil { + if err := c.Get(ctx, objectKey, &p); err != nil { return corev1.Pod{}, err } return p, nil } // GetSecret provides a thin wrapper and client.Client to access corev1.Secret types -func (c client) GetSecret(objectKey k8sClient.ObjectKey) (corev1.Secret, error) { +func (c client) GetSecret(ctx context.Context, objectKey k8sClient.ObjectKey) (corev1.Secret, error) { s := corev1.Secret{} - if err := c.Get(context.TODO(), objectKey, &s); err != nil { + if err := c.Get(ctx, objectKey, &s); err != nil { return corev1.Secret{}, err } return s, nil } // UpdateSecret provides a thin wrapper and client.Client to update corev1.Secret types -func (c client) UpdateSecret(secret corev1.Secret) error { - return c.Update(context.TODO(), &secret) +func (c client) UpdateSecret(ctx context.Context, secret corev1.Secret) error { + return c.Update(ctx, &secret) } // CreateSecret provides a thin wrapper and client.Client to create corev1.Secret types -func (c client) CreateSecret(secret corev1.Secret) error { - return c.Create(context.TODO(), &secret) +func (c client) CreateSecret(ctx context.Context, secret corev1.Secret) error { + return c.Create(ctx, &secret) } // DeleteSecret provides a thin wrapper and client.Client to delete corev1.Secret types -func (c client) DeleteSecret(key k8sClient.ObjectKey) error { +func (c client) DeleteSecret(ctx context.Context, key k8sClient.ObjectKey) error { s := corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: key.Name, Namespace: key.Namespace, }, } - return c.Delete(context.TODO(), &s) + return c.Delete(ctx, &s) } // GetService provides a thin wrapper and client.Client to access corev1.Service types -func (c client) GetService(objectKey k8sClient.ObjectKey) (corev1.Service, error) { +func (c client) GetService(ctx context.Context, objectKey k8sClient.ObjectKey) (corev1.Service, error) { s := corev1.Service{} - if err := c.Get(context.TODO(), objectKey, &s); err != nil { + if err := c.Get(ctx, objectKey, &s); err != nil { return corev1.Service{}, err } return s, nil } // UpdateService provides a thin wrapper and client.Client to update corev1.Service types -func (c client) UpdateService(service corev1.Service) error { - return c.Update(context.TODO(), &service) +func (c client) UpdateService(ctx context.Context, service corev1.Service) error { + return c.Update(ctx, &service) } // CreateService provides a thin wrapper and client.Client to create corev1.Service types -func (c client) CreateService(service corev1.Service) error { - return c.Create(context.TODO(), &service) +func (c client) CreateService(ctx context.Context, service corev1.Service) error { + return c.Create(ctx, &service) } // DeleteService provides a thin wrapper around client.Client to delete corev1.Service types -func (c client) DeleteService(objectKey k8sClient.ObjectKey) error { +func (c client) DeleteService(ctx context.Context, objectKey k8sClient.ObjectKey) error { svc := corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: objectKey.Name, Namespace: objectKey.Namespace, }, } - return c.Delete(context.TODO(), &svc) + return c.Delete(ctx, &svc) } // GetStatefulSet provides a thin wrapper and client.Client to access appsv1.StatefulSet types -func (c client) GetStatefulSet(objectKey k8sClient.ObjectKey) (appsv1.StatefulSet, error) { +func (c client) GetStatefulSet(ctx context.Context, objectKey k8sClient.ObjectKey) (appsv1.StatefulSet, error) { sts := appsv1.StatefulSet{} - if err := c.Get(context.TODO(), objectKey, &sts); err != nil { + if err := c.Get(ctx, objectKey, &sts); err != nil { return appsv1.StatefulSet{}, err } return sts, nil @@ -165,24 +165,24 @@ func (c client) GetStatefulSet(objectKey k8sClient.ObjectKey) (appsv1.StatefulSe // UpdateStatefulSet provides a thin wrapper and client.Client to update appsv1.StatefulSet types // the updated StatefulSet is returned -func (c client) UpdateStatefulSet(sts appsv1.StatefulSet) (appsv1.StatefulSet, error) { +func (c client) UpdateStatefulSet(ctx context.Context, sts appsv1.StatefulSet) (appsv1.StatefulSet, error) { stsToUpdate := &sts - err := c.Update(context.TODO(), stsToUpdate) + err := c.Update(ctx, stsToUpdate) return *stsToUpdate, err } // CreateStatefulSet provides a thin wrapper and client.Client to create appsv1.StatefulSet types -func (c client) CreateStatefulSet(sts appsv1.StatefulSet) error { - return c.Create(context.TODO(), &sts) +func (c client) CreateStatefulSet(ctx context.Context, sts appsv1.StatefulSet) error { + return c.Create(ctx, &sts) } // DeleteStatefulSet provides a thin wrapper and client.Client to delete appsv1.StatefulSet types -func (c client) DeleteStatefulSet(objectKey k8sClient.ObjectKey) error { +func (c client) DeleteStatefulSet(ctx context.Context, objectKey k8sClient.ObjectKey) error { sts := appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ Name: objectKey.Name, Namespace: objectKey.Namespace, }, } - return c.Delete(context.TODO(), &sts) + return c.Delete(ctx, &sts) } diff --git a/pkg/kube/client/client_test.go b/pkg/kube/client/client_test.go index fcaeccd31..083df075b 100644 --- a/pkg/kube/client/client_test.go +++ b/pkg/kube/client/client_test.go @@ -13,20 +13,21 @@ import ( ) func TestChangingName_CreatesNewObject(t *testing.T) { + ctx := context.Background() cm := configmap.Builder(). SetName("some-name"). SetNamespace("some-namespace"). Build() client := NewClient(NewMockedClient()) - err := configmap.CreateOrUpdate(client, cm) + err := configmap.CreateOrUpdate(ctx, client, cm) assert.NoError(t, err) newCm := corev1.ConfigMap{} objectKey := k8sClient.ObjectKeyFromObject(&cm) assert.NoError(t, err) - err = client.Get(context.TODO(), objectKey, &newCm) + err = client.Get(ctx, objectKey, &newCm) assert.NoError(t, err) assert.Equal(t, newCm.Name, "some-name") @@ -35,49 +36,51 @@ func TestChangingName_CreatesNewObject(t *testing.T) { newCm.Name = "new-name" objectKey = k8sClient.ObjectKeyFromObject(&newCm) - _ = configmap.CreateOrUpdate(client, newCm) + _ = configmap.CreateOrUpdate(ctx, client, newCm) - _ = client.Get(context.TODO(), objectKey, &newCm) + _ = client.Get(ctx, objectKey, &newCm) assert.Equal(t, newCm.Name, "new-name") assert.Equal(t, newCm.Namespace, "some-namespace") } func TestAddingDataField_ModifiesExistingObject(t *testing.T) { + ctx := context.Background() cm := configmap.Builder(). SetName("some-name"). SetNamespace("some-namespace"). Build() client := NewClient(NewMockedClient()) - err := configmap.CreateOrUpdate(client, cm) + err := configmap.CreateOrUpdate(ctx, client, cm) assert.NoError(t, err) cm.Data["new-field"] = "value" - _ = configmap.CreateOrUpdate(client, cm) + _ = configmap.CreateOrUpdate(ctx, client, cm) newCm := corev1.ConfigMap{} objectKey := k8sClient.ObjectKeyFromObject(&newCm) assert.NoError(t, err) - _ = client.Get(context.TODO(), objectKey, &newCm) + _ = client.Get(ctx, objectKey, &newCm) assert.Contains(t, cm.Data, "new-field") assert.Equal(t, cm.Data["new-field"], "value") } func TestDeleteConfigMap(t *testing.T) { + ctx := context.Background() cm := configmap.Builder(). SetName("config-map"). SetNamespace("default"). Build() client := NewClient(NewMockedClient()) - err := client.CreateConfigMap(cm) + err := client.CreateConfigMap(ctx, cm) assert.NoError(t, err) - err = client.DeleteConfigMap(types.NamespacedName{Name: "config-map", Namespace: "default"}) + err = client.DeleteConfigMap(ctx, types.NamespacedName{Name: "config-map", Namespace: "default"}) assert.NoError(t, err) - _, err = client.GetConfigMap(types.NamespacedName{Name: "config-map", Namespace: "default"}) + _, err = client.GetConfigMap(ctx, types.NamespacedName{Name: "config-map", Namespace: "default"}) assert.Equal(t, err, notFoundError()) } diff --git a/pkg/kube/client/mocked_client.go b/pkg/kube/client/mocked_client.go index 124f948e2..f9b67e2b1 100644 --- a/pkg/kube/client/mocked_client.go +++ b/pkg/kube/client/mocked_client.go @@ -9,6 +9,7 @@ import ( "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "reflect" k8sClient "sigs.k8s.io/controller-runtime/pkg/client" @@ -31,6 +32,14 @@ type mockedClient struct { backingMap map[reflect.Type]map[k8sClient.ObjectKey]k8sClient.Object } +func (m mockedClient) GroupVersionKindFor(obj runtime.Object) (schema.GroupVersionKind, error) { + panic("not implemented") +} + +func (m mockedClient) IsObjectNamespaced(obj runtime.Object) (bool, error) { + panic("not implemented") +} + func (m mockedClient) Create(_ context.Context, obj k8sClient.Object, _ ...k8sClient.CreateOption) error { relevantMap := m.ensureMapFor(obj) objKey := k8sClient.ObjectKeyFromObject(obj) diff --git a/pkg/kube/client/mocked_client_test.go b/pkg/kube/client/mocked_client_test.go index a27d00518..870b85380 100644 --- a/pkg/kube/client/mocked_client_test.go +++ b/pkg/kube/client/mocked_client_test.go @@ -12,6 +12,7 @@ import ( ) func TestMockedClient(t *testing.T) { + ctx := context.Background() mockedClient := NewMockedClient() cm := configmap.Builder(). @@ -21,11 +22,11 @@ func TestMockedClient(t *testing.T) { SetData(map[string]string{"key-2": "field-2"}). Build() - err := mockedClient.Create(context.TODO(), &cm) + err := mockedClient.Create(ctx, &cm) assert.NoError(t, err) newCm := corev1.ConfigMap{} - err = mockedClient.Get(context.TODO(), types.NamespacedName{Name: "cm-name", Namespace: "cm-namespace"}, &newCm) + err = mockedClient.Get(ctx, types.NamespacedName{Name: "cm-name", Namespace: "cm-namespace"}, &newCm) assert.NoError(t, err) assert.Equal(t, "cm-namespace", newCm.Namespace) assert.Equal(t, "cm-name", newCm.Name) @@ -37,11 +38,11 @@ func TestMockedClient(t *testing.T) { SetServiceType("service-type"). Build() - err = mockedClient.Create(context.TODO(), &svc) + err = mockedClient.Create(ctx, &svc) assert.NoError(t, err) newSvc := corev1.Service{} - err = mockedClient.Get(context.TODO(), types.NamespacedName{Name: "svc-name", Namespace: "svc-namespace"}, &newSvc) + err = mockedClient.Get(ctx, types.NamespacedName{Name: "svc-name", Namespace: "svc-namespace"}, &newSvc) assert.NoError(t, err) assert.Equal(t, "svc-namespace", newSvc.Namespace) assert.Equal(t, "svc-name", newSvc.Name) diff --git a/pkg/kube/client/mocked_manager.go b/pkg/kube/client/mocked_manager.go index 66facfa40..6adaf2609 100644 --- a/pkg/kube/client/mocked_manager.go +++ b/pkg/kube/client/mocked_manager.go @@ -3,6 +3,7 @@ package client import ( "context" "net/http" + "sigs.k8s.io/controller-runtime/pkg/config" "time" "github.com/go-logr/logr" @@ -12,7 +13,6 @@ import ( "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/cache" k8sClient "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/webhook" @@ -24,10 +24,10 @@ type MockedManager struct { Client Client } -func NewManager(obj k8sClient.Object) *MockedManager { +func NewManager(ctx context.Context, obj k8sClient.Object) *MockedManager { c := NewMockedClient() if obj != nil { - _ = c.Create(context.TODO(), obj) + _ = c.Create(ctx, obj) } return &MockedManager{Client: NewClient(c)} } @@ -36,6 +36,10 @@ func NewManagerWithClient(c k8sClient.Client) *MockedManager { return &MockedManager{Client: NewClient(c)} } +func (m *MockedManager) GetHTTPClient() *http.Client { + panic("implement me") +} + func (m *MockedManager) Add(_ manager.Runnable) error { return nil } @@ -69,8 +73,7 @@ func (m *MockedManager) GetScheme() *runtime.Scheme { // GetAdmissionDecoder returns the runtime.Decoder based on the scheme. func (m *MockedManager) GetAdmissionDecoder() admission.Decoder { // just returning nothing - d, _ := admission.NewDecoder(runtime.NewScheme()) - return *d + return *admission.NewDecoder(runtime.NewScheme()) } // GetAPIReader returns the client reader @@ -107,7 +110,7 @@ func (m *MockedManager) GetRESTMapper() meta.RESTMapper { return nil } -func (m *MockedManager) GetWebhookServer() *webhook.Server { +func (m *MockedManager) GetWebhookServer() webhook.Server { return nil } @@ -129,9 +132,9 @@ func (m *MockedManager) GetLogger() logr.Logger { return logr.Logger{} } -func (m *MockedManager) GetControllerOptions() v1alpha1.ControllerConfigurationSpec { +func (m *MockedManager) GetControllerOptions() config.Controller { var duration = time.Duration(0) - return v1alpha1.ControllerConfigurationSpec{ - CacheSyncTimeout: &duration, + return config.Controller{ + CacheSyncTimeout: duration, } } diff --git a/pkg/kube/configmap/configmap.go b/pkg/kube/configmap/configmap.go index 5a6abbd03..a94c31c1f 100644 --- a/pkg/kube/configmap/configmap.go +++ b/pkg/kube/configmap/configmap.go @@ -1,6 +1,7 @@ package configmap import ( + "context" "fmt" "strings" @@ -11,19 +12,19 @@ import ( ) type Getter interface { - GetConfigMap(objectKey client.ObjectKey) (corev1.ConfigMap, error) + GetConfigMap(ctx context.Context, objectKey client.ObjectKey) (corev1.ConfigMap, error) } type Updater interface { - UpdateConfigMap(cm corev1.ConfigMap) error + UpdateConfigMap(ctx context.Context, cm corev1.ConfigMap) error } type Creator interface { - CreateConfigMap(cm corev1.ConfigMap) error + CreateConfigMap(ctx context.Context, cm corev1.ConfigMap) error } type Deleter interface { - DeleteConfigMap(key client.ObjectKey) error + DeleteConfigMap(ctx context.Context, key client.ObjectKey) error } type GetUpdater interface { @@ -51,8 +52,8 @@ const ( // ReadKey accepts a ConfigMap Getter, the object of the ConfigMap to get, and the key within // the config map to read. It returns the string value, and an error if one occurred. -func ReadKey(getter Getter, key string, objectKey client.ObjectKey) (string, error) { - data, err := ReadData(getter, objectKey) +func ReadKey(ctx context.Context, getter Getter, key string, objectKey client.ObjectKey) (string, error) { + data, err := ReadData(ctx, getter, objectKey) if err != nil { return "", err } @@ -63,8 +64,8 @@ func ReadKey(getter Getter, key string, objectKey client.ObjectKey) (string, err } // ReadData extracts the contents of the Data field in a given config map -func ReadData(getter Getter, key client.ObjectKey) (map[string]string, error) { - cm, err := getter.GetConfigMap(key) +func ReadData(ctx context.Context, getter Getter, key client.ObjectKey) (map[string]string, error) { + cm, err := getter.GetConfigMap(ctx, key) if err != nil { return nil, err } @@ -72,26 +73,26 @@ func ReadData(getter Getter, key client.ObjectKey) (map[string]string, error) { } // UpdateField updates the sets "key" to the given "value" -func UpdateField(getUpdater GetUpdater, objectKey client.ObjectKey, key, value string) error { - cm, err := getUpdater.GetConfigMap(objectKey) +func UpdateField(ctx context.Context, getUpdater GetUpdater, objectKey client.ObjectKey, key, value string) error { + cm, err := getUpdater.GetConfigMap(ctx, objectKey) if err != nil { return err } cm.Data[key] = value - return getUpdater.UpdateConfigMap(cm) + return getUpdater.UpdateConfigMap(ctx, cm) } // CreateOrUpdate creates the given ConfigMap if it doesn't exist, // or updates it if it does. -func CreateOrUpdate(getUpdateCreator GetUpdateCreator, cm corev1.ConfigMap) error { - _, err := getUpdateCreator.GetConfigMap(types.NamespacedName{Name: cm.Name, Namespace: cm.Namespace}) +func CreateOrUpdate(ctx context.Context, getUpdateCreator GetUpdateCreator, cm corev1.ConfigMap) error { + _, err := getUpdateCreator.GetConfigMap(ctx, types.NamespacedName{Name: cm.Name, Namespace: cm.Namespace}) if err != nil { if apiErrors.IsNotFound(err) { - return getUpdateCreator.CreateConfigMap(cm) + return getUpdateCreator.CreateConfigMap(ctx, cm) } return err } - return getUpdateCreator.UpdateConfigMap(cm) + return getUpdateCreator.UpdateConfigMap(ctx, cm) } // filelikePropertiesToMap converts a file-like field in a ConfigMap to a map[string]string. @@ -109,8 +110,8 @@ func filelikePropertiesToMap(s string) (map[string]string, error) { } // ReadFileLikeField reads a ConfigMap with file-like properties and returns the value inside one of the fields. -func ReadFileLikeField(getter Getter, objectKey client.ObjectKey, externalKey string, internalKey string) (string, error) { - cmData, err := ReadData(getter, objectKey) +func ReadFileLikeField(ctx context.Context, getter Getter, objectKey client.ObjectKey, externalKey string, internalKey string) (string, error) { + cmData, err := ReadData(ctx, getter, objectKey) if err != nil { return "", err } @@ -130,8 +131,8 @@ func ReadFileLikeField(getter Getter, objectKey client.ObjectKey, externalKey st } // Exists return whether a configmap with the given namespaced name exists -func Exists(cmGetter Getter, nsName types.NamespacedName) (bool, error) { - _, err := cmGetter.GetConfigMap(nsName) +func Exists(ctx context.Context, cmGetter Getter, nsName types.NamespacedName) (bool, error) { + _, err := cmGetter.GetConfigMap(ctx, nsName) if err != nil { if apiErrors.IsNotFound(err) { diff --git a/pkg/kube/configmap/configmap_test.go b/pkg/kube/configmap/configmap_test.go index b685180ad..1d731573a 100644 --- a/pkg/kube/configmap/configmap_test.go +++ b/pkg/kube/configmap/configmap_test.go @@ -1,6 +1,7 @@ package configmap import ( + "context" "testing" "github.com/stretchr/testify/assert" @@ -15,7 +16,7 @@ type configMapGetter struct { cm corev1.ConfigMap } -func (c configMapGetter) GetConfigMap(objectKey client.ObjectKey) (corev1.ConfigMap, error) { +func (c configMapGetter) GetConfigMap(ctx context.Context, objectKey client.ObjectKey) (corev1.ConfigMap, error) { if c.cm.Name == objectKey.Name && c.cm.Namespace == objectKey.Namespace { return c.cm, nil } @@ -29,6 +30,7 @@ func newGetter(cm corev1.ConfigMap) Getter { } func TestReadKey(t *testing.T) { + ctx := context.Background() getter := newGetter( Builder(). SetName("name"). @@ -38,19 +40,20 @@ func TestReadKey(t *testing.T) { Build(), ) - value, err := ReadKey(getter, "key1", nsName("namespace", "name")) + value, err := ReadKey(ctx, getter, "key1", nsName("namespace", "name")) assert.Equal(t, "value1", value) assert.NoError(t, err) - value, err = ReadKey(getter, "key2", nsName("namespace", "name")) + value, err = ReadKey(ctx, getter, "key2", nsName("namespace", "name")) assert.Equal(t, "value2", value) assert.NoError(t, err) - _, err = ReadKey(getter, "key3", nsName("namespace", "name")) + _, err = ReadKey(ctx, getter, "key3", nsName("namespace", "name")) assert.Error(t, err) } func TestReadData(t *testing.T) { + ctx := context.Background() getter := newGetter( Builder(). SetName("name"). @@ -60,7 +63,7 @@ func TestReadData(t *testing.T) { Build(), ) - data, err := ReadData(getter, nsName("namespace", "name")) + data, err := ReadData(ctx, getter, nsName("namespace", "name")) assert.NoError(t, err) assert.Contains(t, data, "key1") @@ -71,6 +74,7 @@ func TestReadData(t *testing.T) { } func TestReadFileLikeField(t *testing.T) { + ctx := context.Background() getter := newGetter( Builder(). SetName("name"). @@ -79,13 +83,14 @@ func TestReadFileLikeField(t *testing.T) { Build(), ) - data, err := ReadFileLikeField(getter, nsName("namespace", "name"), "key1", "value1") + data, err := ReadFileLikeField(ctx, getter, nsName("namespace", "name"), "key1", "value1") assert.NoError(t, err) assert.Equal(t, "1", data) } func TestReadFileLikeField_InvalidExternalKey(t *testing.T) { + ctx := context.Background() getter := newGetter( Builder(). SetName("name"). @@ -94,12 +99,13 @@ func TestReadFileLikeField_InvalidExternalKey(t *testing.T) { Build(), ) - _, err := ReadFileLikeField(getter, nsName("namespace", "name"), "key2", "value1") + _, err := ReadFileLikeField(ctx, getter, nsName("namespace", "name"), "key2", "value1") assert.Error(t, err) assert.Equal(t, "key key2 is not present in ConfigMap namespace/name", err.Error()) } func TestReadFileLikeField_InvalidInternalKey(t *testing.T) { + ctx := context.Background() getter := newGetter( Builder(). SetName("name"). @@ -108,7 +114,7 @@ func TestReadFileLikeField_InvalidInternalKey(t *testing.T) { Build(), ) - _, err := ReadFileLikeField(getter, nsName("namespace", "name"), "key1", "value3") + _, err := ReadFileLikeField(ctx, getter, nsName("namespace", "name"), "key1", "value3") assert.Error(t, err) assert.Equal(t, "key value3 is not present in the key1 field of ConfigMap namespace/name", err.Error()) } @@ -117,14 +123,14 @@ type configMapGetUpdater struct { cm corev1.ConfigMap } -func (c configMapGetUpdater) GetConfigMap(objectKey client.ObjectKey) (corev1.ConfigMap, error) { +func (c configMapGetUpdater) GetConfigMap(ctx context.Context, objectKey client.ObjectKey) (corev1.ConfigMap, error) { if c.cm.Name == objectKey.Name && c.cm.Namespace == objectKey.Namespace { return c.cm, nil } return corev1.ConfigMap{}, notFoundError() } -func (c *configMapGetUpdater) UpdateConfigMap(cm corev1.ConfigMap) error { +func (c *configMapGetUpdater) UpdateConfigMap(ctx context.Context, cm corev1.ConfigMap) error { c.cm = cm return nil } @@ -136,6 +142,7 @@ func newGetUpdater(cm corev1.ConfigMap) GetUpdater { } func TestUpdateField(t *testing.T) { + ctx := context.Background() getUpdater := newGetUpdater( Builder(). SetName("name"). @@ -144,11 +151,11 @@ func TestUpdateField(t *testing.T) { SetDataField("field2", "value2"). Build(), ) - err := UpdateField(getUpdater, nsName("namespace", "name"), "field1", "newValue") + err := UpdateField(ctx, getUpdater, nsName("namespace", "name"), "field1", "newValue") assert.NoError(t, err) - val, _ := ReadKey(getUpdater, "field1", nsName("namespace", "name")) + val, _ := ReadKey(ctx, getUpdater, "field1", nsName("namespace", "name")) assert.Equal(t, "newValue", val) - val2, _ := ReadKey(getUpdater, "field2", nsName("namespace", "name")) + val2, _ := ReadKey(ctx, getUpdater, "field2", nsName("namespace", "name")) assert.Equal(t, "value2", val2) } diff --git a/pkg/kube/pod/pod.go b/pkg/kube/pod/pod.go index 434f62b6a..7b991a694 100644 --- a/pkg/kube/pod/pod.go +++ b/pkg/kube/pod/pod.go @@ -1,10 +1,11 @@ package pod import ( + "context" corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) type Getter interface { - GetPod(objectKey client.ObjectKey) (corev1.Pod, error) + GetPod(ctx context.Context, objectKey client.ObjectKey) (corev1.Pod, error) } diff --git a/pkg/kube/secret/secret.go b/pkg/kube/secret/secret.go index a8c4774df..fa89193db 100644 --- a/pkg/kube/secret/secret.go +++ b/pkg/kube/secret/secret.go @@ -1,6 +1,7 @@ package secret import ( + "context" "fmt" "reflect" "strings" @@ -15,19 +16,19 @@ import ( ) type Getter interface { - GetSecret(objectKey client.ObjectKey) (corev1.Secret, error) + GetSecret(ctx context.Context, objectKey client.ObjectKey) (corev1.Secret, error) } type Updater interface { - UpdateSecret(secret corev1.Secret) error + UpdateSecret(ctx context.Context, secret corev1.Secret) error } type Creator interface { - CreateSecret(secret corev1.Secret) error + CreateSecret(ctx context.Context, secret corev1.Secret) error } type Deleter interface { - DeleteSecret(objectKey client.ObjectKey) error + DeleteSecret(ctx context.Context, key client.ObjectKey) error } type GetUpdater interface { @@ -48,8 +49,8 @@ type GetUpdateCreateDeleter interface { Deleter } -func ReadKey(getter Getter, key string, objectKey client.ObjectKey) (string, error) { - data, err := ReadStringData(getter, objectKey) +func ReadKey(ctx context.Context, getter Getter, key string, objectKey client.ObjectKey) (string, error) { + data, err := ReadStringData(ctx, getter, objectKey) if err != nil { return "", err } @@ -60,8 +61,8 @@ func ReadKey(getter Getter, key string, objectKey client.ObjectKey) (string, err } // ReadByteData reads the Data field of the secret with the given objectKey -func ReadByteData(getter Getter, objectKey client.ObjectKey) (map[string][]byte, error) { - secret, err := getter.GetSecret(objectKey) +func ReadByteData(ctx context.Context, getter Getter, objectKey client.ObjectKey) (map[string][]byte, error) { + secret, err := getter.GetSecret(ctx, objectKey) if err != nil { return nil, err } @@ -69,8 +70,8 @@ func ReadByteData(getter Getter, objectKey client.ObjectKey) (map[string][]byte, } // ReadStringData reads the StringData field of the secret with the given objectKey -func ReadStringData(getter Getter, key client.ObjectKey) (map[string]string, error) { - secret, err := getter.GetSecret(key) +func ReadStringData(ctx context.Context, getter Getter, key client.ObjectKey) (map[string]string, error) { + secret, err := getter.GetSecret(ctx, key) if err != nil { return nil, err } @@ -87,25 +88,25 @@ func dataToStringData(data map[string][]byte) map[string]string { } // UpdateField updates a single field in the secret with the provided objectKey -func UpdateField(getUpdater GetUpdater, objectKey client.ObjectKey, key, value string) error { - secret, err := getUpdater.GetSecret(objectKey) +func UpdateField(ctx context.Context, getUpdater GetUpdater, objectKey client.ObjectKey, key, value string) error { + secret, err := getUpdater.GetSecret(ctx, objectKey) if err != nil { return err } secret.Data[key] = []byte(value) - return getUpdater.UpdateSecret(secret) + return getUpdater.UpdateSecret(ctx, secret) } // CreateOrUpdate creates the Secret if it doesn't exist, other wise it updates it -func CreateOrUpdate(getUpdateCreator GetUpdateCreator, secret corev1.Secret) error { - _, err := getUpdateCreator.GetSecret(types.NamespacedName{Name: secret.Name, Namespace: secret.Namespace}) +func CreateOrUpdate(ctx context.Context, getUpdateCreator GetUpdateCreator, secret corev1.Secret) error { + _, err := getUpdateCreator.GetSecret(ctx, types.NamespacedName{Name: secret.Name, Namespace: secret.Namespace}) if err != nil { if SecretNotExist(err) { - return getUpdateCreator.CreateSecret(secret) + return getUpdateCreator.CreateSecret(ctx, secret) } return err } - return getUpdateCreator.UpdateSecret(secret) + return getUpdateCreator.UpdateSecret(ctx, secret) } // HasAllKeys returns true if the provided secret contains an element for every @@ -121,8 +122,8 @@ func HasAllKeys(secret corev1.Secret, keys ...string) bool { // EnsureSecretWithKey makes sure the Secret with the given name has a key with the given value if the key is not already present. // if the key is present, it will return the existing value associated with this key. -func EnsureSecretWithKey(secretGetUpdateCreateDeleter GetUpdateCreateDeleter, nsName types.NamespacedName, ownerReferences []metav1.OwnerReference, key, value string) (string, error) { - existingSecret, err0 := secretGetUpdateCreateDeleter.GetSecret(nsName) +func EnsureSecretWithKey(ctx context.Context, secretGetUpdateCreateDeleter GetUpdateCreateDeleter, nsName types.NamespacedName, ownerReferences []metav1.OwnerReference, key, value string) (string, error) { + existingSecret, err0 := secretGetUpdateCreateDeleter.GetSecret(ctx, nsName) if err0 != nil { if SecretNotExist(err0) { s := Builder(). @@ -132,7 +133,7 @@ func EnsureSecretWithKey(secretGetUpdateCreateDeleter GetUpdateCreateDeleter, ns SetOwnerReferences(ownerReferences). Build() - if err1 := secretGetUpdateCreateDeleter.CreateSecret(s); err1 != nil { + if err1 := secretGetUpdateCreateDeleter.CreateSecret(ctx, s); err1 != nil { return "", err1 } return value, nil @@ -143,8 +144,8 @@ func EnsureSecretWithKey(secretGetUpdateCreateDeleter GetUpdateCreateDeleter, ns } // CopySecret copies secret object(data) from one cluster client to another, the from and to cluster-client can belong to the same or different clusters -func CopySecret(fromClient Getter, toClient GetUpdateCreator, sourceSecretNsName, destNsName types.NamespacedName) error { - s, err := fromClient.GetSecret(sourceSecretNsName) +func CopySecret(ctx context.Context, fromClient Getter, toClient GetUpdateCreator, sourceSecretNsName, destNsName types.NamespacedName) error { + s, err := fromClient.GetSecret(ctx, sourceSecretNsName) if err != nil { return err } @@ -156,12 +157,12 @@ func CopySecret(fromClient Getter, toClient GetUpdateCreator, sourceSecretNsName SetDataType(s.Type). Build() - return CreateOrUpdate(toClient, secretCopy) + return CreateOrUpdate(ctx, toClient, secretCopy) } // Exists return whether a secret with the given namespaced name exists -func Exists(secretGetter Getter, nsName types.NamespacedName) (bool, error) { - _, err := secretGetter.GetSecret(nsName) +func Exists(ctx context.Context, secretGetter Getter, nsName types.NamespacedName) (bool, error) { + _, err := secretGetter.GetSecret(ctx, nsName) if err != nil { if apiErrors.IsNotFound(err) { @@ -184,12 +185,12 @@ func HasOwnerReferences(secret corev1.Secret, ownerRefs []metav1.OwnerReference) } // CreateOrUpdateIfNeeded creates a secret if it doesn't exist, or updates it if needed. -func CreateOrUpdateIfNeeded(getUpdateCreator GetUpdateCreator, secret corev1.Secret) error { +func CreateOrUpdateIfNeeded(ctx context.Context, getUpdateCreator GetUpdateCreator, secret corev1.Secret) error { // Check if the secret exists - oldSecret, err := getUpdateCreator.GetSecret(types.NamespacedName{Name: secret.Name, Namespace: secret.Namespace}) + oldSecret, err := getUpdateCreator.GetSecret(ctx, types.NamespacedName{Name: secret.Name, Namespace: secret.Namespace}) if err != nil { if apiErrors.IsNotFound(err) { - return getUpdateCreator.CreateSecret(secret) + return getUpdateCreator.CreateSecret(ctx, secret) } return err } @@ -200,7 +201,7 @@ func CreateOrUpdateIfNeeded(getUpdateCreator GetUpdateCreator, secret corev1.Sec } // They are different so we need to update it - return getUpdateCreator.UpdateSecret(secret) + return getUpdateCreator.UpdateSecret(ctx, secret) } func SecretNotExist(err error) bool { diff --git a/pkg/kube/secret/secret_test.go b/pkg/kube/secret/secret_test.go index a157533cf..71810e32d 100644 --- a/pkg/kube/secret/secret_test.go +++ b/pkg/kube/secret/secret_test.go @@ -1,6 +1,7 @@ package secret import ( + "context" "testing" "github.com/stretchr/testify/assert" @@ -15,7 +16,7 @@ type secretGetter struct { secret corev1.Secret } -func (c secretGetter) GetSecret(objectKey client.ObjectKey) (corev1.Secret, error) { +func (c secretGetter) GetSecret(ctx context.Context, objectKey client.ObjectKey) (corev1.Secret, error) { if c.secret.Name == objectKey.Name && c.secret.Namespace == objectKey.Namespace { return c.secret, nil } @@ -29,6 +30,7 @@ func newGetter(s corev1.Secret) Getter { } func TestReadKey(t *testing.T) { + ctx := context.Background() getter := newGetter( Builder(). SetName("name"). @@ -38,15 +40,15 @@ func TestReadKey(t *testing.T) { Build(), ) - value, err := ReadKey(getter, "key1", nsName("namespace", "name")) + value, err := ReadKey(ctx, getter, "key1", nsName("namespace", "name")) assert.Equal(t, "value1", value) assert.NoError(t, err) - value, err = ReadKey(getter, "key2", nsName("namespace", "name")) + value, err = ReadKey(ctx, getter, "key2", nsName("namespace", "name")) assert.Equal(t, "value2", value) assert.NoError(t, err) - _, err = ReadKey(getter, "key3", nsName("namespace", "name")) + _, err = ReadKey(ctx, getter, "key3", nsName("namespace", "name")) assert.Error(t, err) } @@ -60,7 +62,8 @@ func TestReadData(t *testing.T) { Build(), ) t.Run("ReadStringData", func(t *testing.T) { - stringData, err := ReadStringData(getter, nsName("namespace", "name")) + ctx := context.Background() + stringData, err := ReadStringData(ctx, getter, nsName("namespace", "name")) assert.NoError(t, err) assert.Contains(t, stringData, "key1") @@ -71,7 +74,8 @@ func TestReadData(t *testing.T) { }) t.Run("ReadByteData", func(t *testing.T) { - data, err := ReadByteData(getter, nsName("namespace", "name")) + ctx := context.Background() + data, err := ReadByteData(ctx, getter, nsName("namespace", "name")) assert.NoError(t, err) assert.Contains(t, data, "key1") @@ -95,15 +99,15 @@ type secretGetUpdater struct { secret corev1.Secret } -func (c secretGetUpdater) GetSecret(objectKey client.ObjectKey) (corev1.Secret, error) { +func (c secretGetUpdater) GetSecret(ctx context.Context, objectKey client.ObjectKey) (corev1.Secret, error) { if c.secret.Name == objectKey.Name && c.secret.Namespace == objectKey.Namespace { return c.secret, nil } return corev1.Secret{}, notFoundError() } -func (c *secretGetUpdater) UpdateSecret(s corev1.Secret) error { - c.secret = s +func (c *secretGetUpdater) UpdateSecret(ctx context.Context, secret corev1.Secret) error { + c.secret = secret return nil } @@ -114,6 +118,7 @@ func newGetUpdater(s corev1.Secret) GetUpdater { } func TestUpdateField(t *testing.T) { + ctx := context.Background() getUpdater := newGetUpdater( Builder(). SetName("name"). @@ -122,11 +127,11 @@ func TestUpdateField(t *testing.T) { SetField("field2", "value2"). Build(), ) - err := UpdateField(getUpdater, nsName("namespace", "name"), "field1", "newValue") + err := UpdateField(ctx, getUpdater, nsName("namespace", "name"), "field1", "newValue") assert.NoError(t, err) - val, _ := ReadKey(getUpdater, "field1", nsName("namespace", "name")) + val, _ := ReadKey(ctx, getUpdater, "field1", nsName("namespace", "name")) assert.Equal(t, "newValue", val) - val2, _ := ReadKey(getUpdater, "field2", nsName("namespace", "name")) + val2, _ := ReadKey(ctx, getUpdater, "field2", nsName("namespace", "name")) assert.Equal(t, "value2", val2) } @@ -135,23 +140,23 @@ type mockSecretGetUpdateCreateDeleter struct { apiCalls int } -func (c *mockSecretGetUpdateCreateDeleter) DeleteSecret(objectKey client.ObjectKey) error { - delete(c.secrets, objectKey) +func (c *mockSecretGetUpdateCreateDeleter) DeleteSecret(ctx context.Context, key client.ObjectKey) error { + delete(c.secrets, key) c.apiCalls += 1 return nil } -func (c *mockSecretGetUpdateCreateDeleter) UpdateSecret(s corev1.Secret) error { - c.secrets[types.NamespacedName{Name: s.Name, Namespace: s.Namespace}] = s +func (c *mockSecretGetUpdateCreateDeleter) UpdateSecret(ctx context.Context, secret corev1.Secret) error { + c.secrets[types.NamespacedName{Name: secret.Name, Namespace: secret.Namespace}] = secret c.apiCalls += 1 return nil } -func (c *mockSecretGetUpdateCreateDeleter) CreateSecret(secret corev1.Secret) error { - return c.UpdateSecret(secret) +func (c *mockSecretGetUpdateCreateDeleter) CreateSecret(ctx context.Context, secret corev1.Secret) error { + return c.UpdateSecret(ctx, secret) } -func (c *mockSecretGetUpdateCreateDeleter) GetSecret(objectKey client.ObjectKey) (corev1.Secret, error) { +func (c *mockSecretGetUpdateCreateDeleter) GetSecret(ctx context.Context, objectKey client.ObjectKey) (corev1.Secret, error) { c.apiCalls += 1 if s, ok := c.secrets[objectKey]; !ok { return corev1.Secret{}, notFoundError() @@ -161,6 +166,7 @@ func (c *mockSecretGetUpdateCreateDeleter) GetSecret(objectKey client.ObjectKey) } func TestCreateOrUpdateIfNeededCreate(t *testing.T) { + ctx := context.Background() mock := &mockSecretGetUpdateCreateDeleter{ secrets: map[client.ObjectKey]corev1.Secret{}, apiCalls: 0, @@ -169,12 +175,13 @@ func TestCreateOrUpdateIfNeededCreate(t *testing.T) { secret := getDefaultSecret() // first time it does not exist, we create it - err := CreateOrUpdateIfNeeded(mock, secret) + err := CreateOrUpdateIfNeeded(ctx, mock, secret) assert.NoError(t, err) assert.Equal(t, 2, mock.apiCalls) // 2 calls -> get + creation } func TestCreateOrUpdateIfNeededUpdate(t *testing.T) { + ctx := context.Background() mock := &mockSecretGetUpdateCreateDeleter{ secrets: map[client.ObjectKey]corev1.Secret{}, apiCalls: 0, @@ -182,7 +189,7 @@ func TestCreateOrUpdateIfNeededUpdate(t *testing.T) { secret := getDefaultSecret() { - err := mock.CreateSecret(secret) + err := mock.CreateSecret(ctx, secret) assert.NoError(t, err) mock.apiCalls = 0 } @@ -190,13 +197,14 @@ func TestCreateOrUpdateIfNeededUpdate(t *testing.T) { { secret.Data = map[string][]byte{"test": {1, 2, 3}} // secret differs -> we update - err := CreateOrUpdateIfNeeded(mock, secret) + err := CreateOrUpdateIfNeeded(ctx, mock, secret) assert.NoError(t, err) assert.Equal(t, 2, mock.apiCalls) // 2 calls -> get + update } } func TestCreateOrUpdateIfNeededEqual(t *testing.T) { + ctx := context.Background() mock := &mockSecretGetUpdateCreateDeleter{ secrets: map[client.ObjectKey]corev1.Secret{}, apiCalls: 0, @@ -204,14 +212,14 @@ func TestCreateOrUpdateIfNeededEqual(t *testing.T) { secret := getDefaultSecret() { - err := mock.CreateSecret(secret) + err := mock.CreateSecret(ctx, secret) assert.NoError(t, err) mock.apiCalls = 0 } { // the secret already exists, so we only call get - err := CreateOrUpdateIfNeeded(mock, secret) + err := CreateOrUpdateIfNeeded(ctx, mock, secret) assert.NoError(t, err) assert.Equal(t, 1, mock.apiCalls) // 1 call -> get } diff --git a/pkg/kube/service/service.go b/pkg/kube/service/service.go index a2792065e..abb749acf 100644 --- a/pkg/kube/service/service.go +++ b/pkg/kube/service/service.go @@ -1,24 +1,25 @@ package service import ( + "context" corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) type Getter interface { - GetService(objectKey client.ObjectKey) (corev1.Service, error) + GetService(ctx context.Context, objectKey client.ObjectKey) (corev1.Service, error) } type Updater interface { - UpdateService(service corev1.Service) error + UpdateService(ctx context.Context, service corev1.Service) error } type Creator interface { - CreateService(service corev1.Service) error + CreateService(ctx context.Context, service corev1.Service) error } type Deleter interface { - DeleteService(objectKey client.ObjectKey) error + DeleteService(ctx context.Context, objectKey client.ObjectKey) error } type GetDeleter interface { diff --git a/pkg/kube/statefulset/statefulset.go b/pkg/kube/statefulset/statefulset.go index 28ddd1f4f..a6c5c35ec 100644 --- a/pkg/kube/statefulset/statefulset.go +++ b/pkg/kube/statefulset/statefulset.go @@ -1,6 +1,7 @@ package statefulset import ( + "context" "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/annotations" "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/merge" apiErrors "k8s.io/apimachinery/pkg/api/errors" @@ -17,19 +18,19 @@ const ( ) type Getter interface { - GetStatefulSet(objectKey client.ObjectKey) (appsv1.StatefulSet, error) + GetStatefulSet(ctx context.Context, objectKey client.ObjectKey) (appsv1.StatefulSet, error) } type Updater interface { - UpdateStatefulSet(sts appsv1.StatefulSet) (appsv1.StatefulSet, error) + UpdateStatefulSet(ctx context.Context, sts appsv1.StatefulSet) (appsv1.StatefulSet, error) } type Creator interface { - CreateStatefulSet(sts appsv1.StatefulSet) error + CreateStatefulSet(ctx context.Context, sts appsv1.StatefulSet) error } type Deleter interface { - DeleteStatefulSet(objectKey client.ObjectKey) error + DeleteStatefulSet(ctx context.Context, objectKey client.ObjectKey) error } type GetUpdater interface { @@ -52,26 +53,26 @@ type GetUpdateCreateDeleter interface { // CreateOrUpdate creates the given StatefulSet if it doesn't exist, // or updates it if it does. -func CreateOrUpdate(getUpdateCreator GetUpdateCreator, sts appsv1.StatefulSet) (appsv1.StatefulSet, error) { - _, err := getUpdateCreator.GetStatefulSet(types.NamespacedName{Name: sts.Name, Namespace: sts.Namespace}) +func CreateOrUpdate(ctx context.Context, getUpdateCreator GetUpdateCreator, sts appsv1.StatefulSet) (appsv1.StatefulSet, error) { + _, err := getUpdateCreator.GetStatefulSet(ctx, types.NamespacedName{Name: sts.Name, Namespace: sts.Namespace}) if err != nil { if apiErrors.IsNotFound(err) { - return appsv1.StatefulSet{}, getUpdateCreator.CreateStatefulSet(sts) + return appsv1.StatefulSet{}, getUpdateCreator.CreateStatefulSet(ctx, sts) } return appsv1.StatefulSet{}, err } - return getUpdateCreator.UpdateStatefulSet(sts) + return getUpdateCreator.UpdateStatefulSet(ctx, sts) } // GetAndUpdate applies the provided function to the most recent version of the object -func GetAndUpdate(getUpdater GetUpdater, nsName types.NamespacedName, updateFunc func(*appsv1.StatefulSet)) (appsv1.StatefulSet, error) { - sts, err := getUpdater.GetStatefulSet(nsName) +func GetAndUpdate(ctx context.Context, getUpdater GetUpdater, nsName types.NamespacedName, updateFunc func(*appsv1.StatefulSet)) (appsv1.StatefulSet, error) { + sts, err := getUpdater.GetStatefulSet(ctx, nsName) if err != nil { return appsv1.StatefulSet{}, err } // apply the function on the most recent version of the resource updateFunc(&sts) - return getUpdater.UpdateStatefulSet(sts) + return getUpdater.UpdateStatefulSet(ctx, sts) } // VolumeMountData contains values required for the MountVolume function @@ -333,13 +334,13 @@ func VolumeMountWithNameExists(mounts []corev1.VolumeMount, volumeName string) b // ResetUpdateStrategy resets the statefulset update strategy to RollingUpdate. // If a version change is in progress, it doesn't do anything. -func ResetUpdateStrategy(mdb annotations.Versioned, kubeClient GetUpdater) error { +func ResetUpdateStrategy(ctx context.Context, mdb annotations.Versioned, kubeClient GetUpdater) error { if !mdb.IsChangingVersion() { return nil } // if we changed the version, we need to reset the UpdatePolicy back to OnUpdate - _, err := GetAndUpdate(kubeClient, mdb.NamespacedName(), func(sts *appsv1.StatefulSet) { + _, err := GetAndUpdate(ctx, kubeClient, mdb.NamespacedName(), func(sts *appsv1.StatefulSet) { sts.Spec.UpdateStrategy.Type = appsv1.RollingUpdateStatefulSetStrategyType }) return err diff --git a/pkg/readiness/headless/headless.go b/pkg/readiness/headless/headless.go index 9f4fc7d6d..18c28e23f 100644 --- a/pkg/readiness/headless/headless.go +++ b/pkg/readiness/headless/headless.go @@ -1,6 +1,7 @@ package headless import ( + "context" "fmt" "io" "os" @@ -23,11 +24,11 @@ const ( // /var/run/secrets/kubernetes.io/serviceaccount/namespace file (see // https://kubernetes.io/docs/tasks/access-application-cluster/access-cluster/#accessing-the-api-from-a-pod) // though passing the namespace as an environment variable makes the code simpler for testing and saves an IO operation -func PerformCheckHeadlessMode(health health.Status, conf config.Config) (bool, error) { +func PerformCheckHeadlessMode(ctx context.Context, health health.Status, conf config.Config) (bool, error) { var targetVersion int64 var err error - targetVersion, err = secret.ReadAutomationConfigVersionFromSecret(conf.Namespace, conf.ClientSet, conf.AutomationConfigSecretName) + targetVersion, err = secret.ReadAutomationConfigVersionFromSecret(ctx, conf.Namespace, conf.ClientSet, conf.AutomationConfigSecretName) if err != nil { // this file is expected to be present in case of AppDB, there is no point trying to access it in // community, it masks the underlying error @@ -54,7 +55,7 @@ func PerformCheckHeadlessMode(health health.Status, conf config.Config) (bool, e currentAgentVersion := readCurrentAgentInfo(health, targetVersion) - if err = pod.PatchPodAnnotation(conf.Namespace, currentAgentVersion, conf.Hostname, conf.ClientSet); err != nil { + if err = pod.PatchPodAnnotation(ctx, conf.Namespace, currentAgentVersion, conf.Hostname, conf.ClientSet); err != nil { return false, err } diff --git a/pkg/readiness/headless/headless_test.go b/pkg/readiness/headless/headless_test.go index afff7c506..d6f2f293c 100644 --- a/pkg/readiness/headless/headless_test.go +++ b/pkg/readiness/headless/headless_test.go @@ -15,6 +15,7 @@ import ( ) func TestPerformCheckHeadlessMode(t *testing.T) { + ctx := context.Background() c := testConfig() c.ClientSet = fake.NewSimpleClientset(testdata.TestPod(c.Namespace, c.Hostname), testdata.TestSecret(c.Namespace, c.AutomationConfigSecretName, 11)) @@ -24,12 +25,12 @@ func TestPerformCheckHeadlessMode(t *testing.T) { }}, } - achieved, err := PerformCheckHeadlessMode(status, c) + achieved, err := PerformCheckHeadlessMode(ctx, status, c) require.NoError(t, err) assert.False(t, achieved) - thePod, _ := c.ClientSet.CoreV1().Pods(c.Namespace).Get(context.TODO(), c.Hostname, metav1.GetOptions{}) + thePod, _ := c.ClientSet.CoreV1().Pods(c.Namespace).Get(ctx, c.Hostname, metav1.GetOptions{}) assert.Equal(t, map[string]string{"agent.mongodb.com/version": "10"}, thePod.Annotations) } diff --git a/pkg/readiness/pod/podannotation.go b/pkg/readiness/pod/podannotation.go index 211991822..d36bda37f 100644 --- a/pkg/readiness/pod/podannotation.go +++ b/pkg/readiness/pod/podannotation.go @@ -13,8 +13,8 @@ import ( const mongodbAgentVersionAnnotation = "agent.mongodb.com/version" -func PatchPodAnnotation(podNamespace string, lastVersionAchieved int64, memberName string, clientSet kubernetes.Interface) error { - pod, err := clientSet.CoreV1().Pods(podNamespace).Get(context.Background(), memberName, metav1.GetOptions{}) +func PatchPodAnnotation(ctx context.Context, podNamespace string, lastVersionAchieved int64, memberName string, clientSet kubernetes.Interface) error { + pod, err := clientSet.CoreV1().Pods(podNamespace).Get(ctx, memberName, metav1.GetOptions{}) if err != nil { return err } @@ -36,7 +36,7 @@ func PatchPodAnnotation(podNamespace string, lastVersionAchieved int64, memberNa }) patcher := NewKubernetesPodPatcher(clientSet) - updatedPod, err := patcher.patchPod(podNamespace, memberName, payload) + updatedPod, err := patcher.patchPod(ctx, podNamespace, memberName, payload) if updatedPod != nil { zap.S().Debugf("Updated Pod annotation: %v (%s)", pod.Annotations, memberName) } diff --git a/pkg/readiness/pod/podannotation_test.go b/pkg/readiness/pod/podannotation_test.go index d9ab9b0fd..b75382421 100644 --- a/pkg/readiness/pod/podannotation_test.go +++ b/pkg/readiness/pod/podannotation_test.go @@ -17,6 +17,7 @@ import ( // TestPatchPodAnnotation verifies that patching of the pod works correctly func TestPatchPodAnnotation(t *testing.T) { + ctx := context.Background() clientset := fake.NewSimpleClientset(&v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "my-replica-set-0", @@ -27,20 +28,21 @@ func TestPatchPodAnnotation(t *testing.T) { }, }) - pod, _ := clientset.CoreV1().Pods("test-ns").Get(context.TODO(), "my-replica-set-0", metav1.GetOptions{}) + pod, _ := clientset.CoreV1().Pods("test-ns").Get(ctx, "my-replica-set-0", metav1.GetOptions{}) assert.Empty(t, pod.Annotations[mongodbAgentVersionAnnotation]) // adding the annotations - assert.NoError(t, PatchPodAnnotation("test-ns", 1, "my-replica-set-0", clientset)) - pod, _ = clientset.CoreV1().Pods("test-ns").Get(context.TODO(), "my-replica-set-0", metav1.GetOptions{}) + assert.NoError(t, PatchPodAnnotation(ctx, "test-ns", 1, "my-replica-set-0", clientset)) + pod, _ = clientset.CoreV1().Pods("test-ns").Get(ctx, "my-replica-set-0", metav1.GetOptions{}) assert.Equal(t, map[string]string{"agent.mongodb.com/version": "1"}, pod.Annotations) // changing the annotations - no new annotations were added - assert.NoError(t, PatchPodAnnotation("test-ns", 2, "my-replica-set-0", clientset)) - pod, _ = clientset.CoreV1().Pods("test-ns").Get(context.TODO(), "my-replica-set-0", metav1.GetOptions{}) + assert.NoError(t, PatchPodAnnotation(ctx, "test-ns", 2, "my-replica-set-0", clientset)) + pod, _ = clientset.CoreV1().Pods("test-ns").Get(ctx, "my-replica-set-0", metav1.GetOptions{}) assert.Equal(t, map[string]string{"agent.mongodb.com/version": "2"}, pod.Annotations) } func TestUpdatePodAnnotationPodNotFound(t *testing.T) { - assert.True(t, apiErrors.IsNotFound(PatchPodAnnotation("wrong-ns", 1, "my-replica-set-0", fake.NewSimpleClientset()))) + ctx := context.Background() + assert.True(t, apiErrors.IsNotFound(PatchPodAnnotation(ctx, "wrong-ns", 1, "my-replica-set-0", fake.NewSimpleClientset()))) } diff --git a/pkg/readiness/pod/podpatcher.go b/pkg/readiness/pod/podpatcher.go index 8d39d627a..5bea91f33 100644 --- a/pkg/readiness/pod/podpatcher.go +++ b/pkg/readiness/pod/podpatcher.go @@ -24,10 +24,10 @@ func NewKubernetesPodPatcher(clientSet kubernetes.Interface) Patcher { return Patcher{clientset: clientSet} } -func (p Patcher) patchPod(namespace, podName string, payload []patchValue) (*v1.Pod, error) { +func (p Patcher) patchPod(ctx context.Context, namespace, podName string, payload []patchValue) (*v1.Pod, error) { data, err := json.Marshal(payload) if err != nil { return nil, err } - return p.clientset.CoreV1().Pods(namespace).Patch(context.TODO(), podName, types.JSONPatchType, data, metav1.PatchOptions{}) + return p.clientset.CoreV1().Pods(namespace).Patch(ctx, podName, types.JSONPatchType, data, metav1.PatchOptions{}) } diff --git a/pkg/readiness/secret/automationconfig.go b/pkg/readiness/secret/automationconfig.go index dc007b2ed..b08ebded0 100644 --- a/pkg/readiness/secret/automationconfig.go +++ b/pkg/readiness/secret/automationconfig.go @@ -1,6 +1,7 @@ package secret import ( + "context" "encoding/json" "github.com/spf13/cast" @@ -11,9 +12,9 @@ const ( automationConfigKey = "cluster-config.json" ) -func ReadAutomationConfigVersionFromSecret(namespace string, clientSet kubernetes.Interface, automationConfigMap string) (int64, error) { +func ReadAutomationConfigVersionFromSecret(ctx context.Context, namespace string, clientSet kubernetes.Interface, automationConfigMap string) (int64, error) { secretReader := newKubernetesSecretReader(clientSet) - theSecret, err := secretReader.ReadSecret(namespace, automationConfigMap) + theSecret, err := secretReader.ReadSecret(ctx, namespace, automationConfigMap) if err != nil { return -1, err } diff --git a/pkg/readiness/secret/secretreader.go b/pkg/readiness/secret/secretreader.go index a33161fce..aecb845e0 100644 --- a/pkg/readiness/secret/secretreader.go +++ b/pkg/readiness/secret/secretreader.go @@ -16,6 +16,6 @@ func newKubernetesSecretReader(clientSet kubernetes.Interface) *reader { return &reader{clientset: clientSet} } -func (r *reader) ReadSecret(namespace, secretName string) (*corev1.Secret, error) { - return r.clientset.CoreV1().Secrets(namespace).Get(context.TODO(), secretName, metav1.GetOptions{}) +func (r *reader) ReadSecret(ctx context.Context, namespace, secretName string) (*corev1.Secret, error) { + return r.clientset.CoreV1().Secrets(namespace).Get(ctx, secretName, metav1.GetOptions{}) } diff --git a/pkg/util/status/status.go b/pkg/util/status/status.go index e9dc0c9cb..21aebfc62 100644 --- a/pkg/util/status/status.go +++ b/pkg/util/status/status.go @@ -19,13 +19,13 @@ type OptionBuilder interface { } // Update takes the options provided by the given option builder, applies them all and then updates the resource -func Update(statusWriter client.StatusWriter, mdb *mdbv1.MongoDBCommunity, optionBuilder OptionBuilder) (reconcile.Result, error) { +func Update(ctx context.Context, statusWriter client.StatusWriter, mdb *mdbv1.MongoDBCommunity, optionBuilder OptionBuilder) (reconcile.Result, error) { options := optionBuilder.GetOptions() for _, opt := range options { opt.ApplyOption(mdb) } - if err := statusWriter.Update(context.TODO(), mdb); err != nil { + if err := statusWriter.Update(ctx, mdb); err != nil { return reconcile.Result{}, err } diff --git a/test/e2e/client.go b/test/e2e/client.go index 205cc7413..291ba4473 100644 --- a/test/e2e/client.go +++ b/test/e2e/client.go @@ -34,14 +34,16 @@ var OperatorNamespace string // CleanupOptions are a way to register cleanup functions on object creation using the test client. type CleanupOptions struct { - TestContext *Context + TestContext *TestContext } // ApplyToCreate is a required method for CleanupOptions passed to the Create api. func (*CleanupOptions) ApplyToCreate(*client.CreateOptions) {} -// Context tracks cleanup functions to be called at the end of a test. -type Context struct { +// TestContext tracks cleanup functions to be called at the end of a test. +type TestContext struct { + Ctx context.Context + // shouldPerformCleanup indicates whether or not cleanup should happen after this test shouldPerformCleanup bool @@ -57,17 +59,17 @@ type Context struct { } // NewContext creates a context. -func NewContext(t *testing.T, performCleanup bool) (*Context, error) { +func NewContext(ctx context.Context, t *testing.T, performCleanup bool) (*TestContext, error) { testId, err := generate.RandomValidDNS1123Label(10) if err != nil { return nil, err } - return &Context{t: t, ExecutionId: testId, shouldPerformCleanup: performCleanup}, nil + return &TestContext{Ctx: ctx, t: t, ExecutionId: testId, shouldPerformCleanup: performCleanup}, nil } // Teardown is called at the end of a test. -func (ctx *Context) Teardown() { +func (ctx *TestContext) Teardown() { if !ctx.shouldPerformCleanup { return } @@ -80,7 +82,7 @@ func (ctx *Context) Teardown() { } // AddCleanupFunc adds a cleanup function to the context to be called at the end of a test. -func (ctx *Context) AddCleanupFunc(fn func() error) { +func (ctx *TestContext) AddCleanupFunc(fn func() error) { ctx.cleanupFuncs = append(ctx.cleanupFuncs, fn) } @@ -148,7 +150,7 @@ func (c *E2ETestClient) Get(ctx context.Context, key types.NamespacedName, obj c return c.Client.Get(ctx, key, obj) } -func (c *E2ETestClient) Execute(pod corev1.Pod, containerName, command string) (string, error) { +func (c *E2ETestClient) Execute(ctx context.Context, pod corev1.Pod, containerName, command string) (string, error) { req := c.CoreV1Client.RESTClient(). Post(). Namespace(pod.Namespace). @@ -170,7 +172,7 @@ func (c *E2ETestClient) Execute(pod corev1.Pod, containerName, command string) ( if err != nil { return "", err } - err = exec.StreamWithContext(context.Background(), remotecommand.StreamOptions{ + err = exec.StreamWithContext(ctx, remotecommand.StreamOptions{ Stdout: buf, Stderr: errBuf, }) diff --git a/test/e2e/e2eutil.go b/test/e2e/e2eutil.go index 10a8eb9a2..5b78df60a 100644 --- a/test/e2e/e2eutil.go +++ b/test/e2e/e2eutil.go @@ -45,18 +45,18 @@ func TlsTestDataDir() string { // UpdateMongoDBResource applies the provided function to the most recent version of the MongoDB resource // and retries when there are conflicts -func UpdateMongoDBResource(original *mdbv1.MongoDBCommunity, updateFunc func(*mdbv1.MongoDBCommunity)) error { - err := TestClient.Get(context.TODO(), types.NamespacedName{Name: original.Name, Namespace: original.Namespace}, original) +func UpdateMongoDBResource(ctx context.Context, original *mdbv1.MongoDBCommunity, updateFunc func(*mdbv1.MongoDBCommunity)) error { + err := TestClient.Get(ctx, types.NamespacedName{Name: original.Name, Namespace: original.Namespace}, original) if err != nil { return err } updateFunc(original) - return TestClient.Update(context.TODO(), original) + return TestClient.Update(ctx, original) } -func NewTestMongoDB(ctx *Context, name string, namespace string) (mdbv1.MongoDBCommunity, mdbv1.MongoDBUser) { +func NewTestMongoDB(ctx *TestContext, name string, namespace string) (mdbv1.MongoDBCommunity, mdbv1.MongoDBUser) { mongodbNamespace := namespace if mongodbNamespace == "" { mongodbNamespace = OperatorNamespace @@ -166,13 +166,13 @@ func NewTestTLSConfig(optional bool) mdbv1.TLS { } } -func NewPrometheusConfig(namespace string) *mdbv1.Prometheus { +func NewPrometheusConfig(ctx context.Context, namespace string) *mdbv1.Prometheus { sec := secret.Builder(). SetName("prom-secret"). SetNamespace(namespace). SetField("password", "prom-password"). Build() - err := TestClient.Create(context.TODO(), &sec, &CleanupOptions{}) + err := TestClient.Create(ctx, &sec, &CleanupOptions{}) if err != nil { if !apiErrors.IsAlreadyExists(err) { panic(fmt.Sprintf("Error trying to create secret: %s", err)) @@ -187,22 +187,22 @@ func NewPrometheusConfig(namespace string) *mdbv1.Prometheus { } } -func ensureObject(ctx *Context, obj k8sClient.Object) error { +func ensureObject(ctx *TestContext, obj k8sClient.Object) error { key := k8sClient.ObjectKeyFromObject(obj) obj.SetLabels(TestLabels()) - err := TestClient.Get(context.TODO(), key, obj) + err := TestClient.Get(ctx.Ctx, key, obj) if err != nil { if !apiErrors.IsNotFound(err) { return err } - err = TestClient.Create(context.TODO(), obj, &CleanupOptions{TestContext: ctx}) + err = TestClient.Create(ctx.Ctx, obj, &CleanupOptions{TestContext: ctx}) if err != nil { return err } } else { fmt.Printf("%s %s/%s already exists!\n", reflect.TypeOf(obj), key.Namespace, key.Name) - err = TestClient.Update(context.TODO(), obj) + err = TestClient.Update(ctx.Ctx, obj) if err != nil { return err } @@ -211,7 +211,7 @@ func ensureObject(ctx *Context, obj k8sClient.Object) error { } // EnsureNamespace checks that the given namespace exists and creates it if not. -func EnsureNamespace(ctx *Context, namespace string) error { +func EnsureNamespace(ctx *TestContext, namespace string) error { return ensureObject(ctx, &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ Name: namespace, diff --git a/test/e2e/feature_compatibility_version/feature_compatibility_version_test.go b/test/e2e/feature_compatibility_version/feature_compatibility_version_test.go index 64321860d..2cc2db6d9 100644 --- a/test/e2e/feature_compatibility_version/feature_compatibility_version_test.go +++ b/test/e2e/feature_compatibility_version/feature_compatibility_version_test.go @@ -1,6 +1,7 @@ package feature_compatibility_version import ( + "context" "fmt" mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" . "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/util/mongotester" @@ -11,7 +12,7 @@ import ( e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/mongodbtests" - setup "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" + "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" ) func TestMain(m *testing.M) { @@ -27,8 +28,9 @@ func TestMain(m *testing.M) { // format remains the same. Versions 5 and 6 are one way upgrade only. // See: https://www.mongodb.com/docs/manual/reference/command/setFeatureCompatibilityVersion/ func TestFeatureCompatibilityVersion(t *testing.T) { - ctx := setup.Setup(t) - defer ctx.Teardown() + ctx := context.Background() + testCtx := setup.Setup(ctx, t) + defer testCtx.Teardown() // This is the lowest version available for the official images const lowestMDBVersion = "4.4.16" @@ -36,30 +38,30 @@ func TestFeatureCompatibilityVersion(t *testing.T) { const featureCompatibility = "4.2" const upgradedFeatureCompatibility = "4.4" - mdb, user := e2eutil.NewTestMongoDB(ctx, "mdb0", "") + mdb, user := e2eutil.NewTestMongoDB(testCtx, "mdb0", "") mdb.Spec.Version = lowestMDBVersion mdb.Spec.FeatureCompatibilityVersion = featureCompatibility - _, err := setup.GeneratePasswordForUser(ctx, user, "") + _, err := setup.GeneratePasswordForUser(testCtx, user, "") if err != nil { t.Fatal(err) } - tester, err := FromResource(t, mdb) + tester, err := FromResource(ctx, t, mdb) if err != nil { t.Fatal(err) } - t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) - t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb)) + t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, testCtx)) + t.Run("Basic tests", mongodbtests.BasicFunctionality(ctx, &mdb)) t.Run("Ensure Authentication", tester.EnsureAuthenticationIsConfigured(3)) t.Run(fmt.Sprintf("Test FeatureCompatibilityVersion is %s", featureCompatibility), tester.HasFCV(featureCompatibility, 3)) // Upgrade while keeping the Feature Compatibility intact t.Run("MongoDB is reachable while version is upgraded", func(t *testing.T) { defer tester.StartBackgroundConnectivityTest(t, time.Second*20)() - t.Run("Test Version can be upgraded", mongodbtests.ChangeVersion(&mdb, highestMDBVersion)) - t.Run("Stateful Set Reaches Ready State, after Upgrading", mongodbtests.StatefulSetBecomesReady(&mdb)) + t.Run("Test Version can be upgraded", mongodbtests.ChangeVersion(ctx, &mdb, highestMDBVersion)) + t.Run("Stateful Set Reaches Ready State, after Upgrading", mongodbtests.StatefulSetBecomesReady(ctx, &mdb)) }) t.Run("Test Basic Connectivity after upgrade has completed", tester.ConnectivitySucceeds()) @@ -68,20 +70,20 @@ func TestFeatureCompatibilityVersion(t *testing.T) { // Downgrade while keeping the Feature Compatibility intact t.Run("MongoDB is reachable while version is downgraded", func(t *testing.T) { defer tester.StartBackgroundConnectivityTest(t, time.Second*10)() - t.Run("Test Version can be downgraded", mongodbtests.ChangeVersion(&mdb, lowestMDBVersion)) - t.Run("Stateful Set Reaches Ready State, after Upgrading", mongodbtests.StatefulSetBecomesReady(&mdb)) + t.Run("Test Version can be downgraded", mongodbtests.ChangeVersion(ctx, &mdb, lowestMDBVersion)) + t.Run("Stateful Set Reaches Ready State, after Upgrading", mongodbtests.StatefulSetBecomesReady(ctx, &mdb)) }) t.Run(fmt.Sprintf("Test FeatureCompatibilityVersion, after downgrade, is %s", featureCompatibility), tester.HasFCV(featureCompatibility, 3)) // Upgrade the Feature Compatibility keeping the MongoDB version the same t.Run("Test FeatureCompatibilityVersion can be upgraded", func(t *testing.T) { - err := e2eutil.UpdateMongoDBResource(&mdb, func(db *mdbv1.MongoDBCommunity) { + err := e2eutil.UpdateMongoDBResource(ctx, &mdb, func(db *mdbv1.MongoDBCommunity) { db.Spec.FeatureCompatibilityVersion = upgradedFeatureCompatibility }) assert.NoError(t, err) - t.Run("Stateful Set Reaches Ready State, after Upgrading FeatureCompatibilityVersion", mongodbtests.StatefulSetBecomesReady(&mdb)) - t.Run("MongoDB Reaches Running Phase, after Upgrading FeatureCompatibilityVersion", mongodbtests.MongoDBReachesRunningPhase(&mdb)) + t.Run("Stateful Set Reaches Ready State, after Upgrading FeatureCompatibilityVersion", mongodbtests.StatefulSetBecomesReady(ctx, &mdb)) + t.Run("MongoDB Reaches Running Phase, after Upgrading FeatureCompatibilityVersion", mongodbtests.MongoDBReachesRunningPhase(ctx, &mdb)) }) t.Run(fmt.Sprintf("Test FeatureCompatibilityVersion, after upgrading FeatureCompatibilityVersion, is %s", upgradedFeatureCompatibility), tester.HasFCV(upgradedFeatureCompatibility, 10)) diff --git a/test/e2e/mongodbtests/mongodbtests.go b/test/e2e/mongodbtests/mongodbtests.go index 5a2652347..4a9b861ed 100644 --- a/test/e2e/mongodbtests/mongodbtests.go +++ b/test/e2e/mongodbtests/mongodbtests.go @@ -43,43 +43,43 @@ func strPointer(n float32) *string { // StatefulSetBecomesReady ensures that the underlying stateful set // reaches the running state. -func StatefulSetBecomesReady(mdb *mdbv1.MongoDBCommunity, opts ...wait.Configuration) func(t *testing.T) { +func StatefulSetBecomesReady(ctx context.Context, mdb *mdbv1.MongoDBCommunity, opts ...wait.Configuration) func(t *testing.T) { defaultOpts := []wait.Configuration{ wait.RetryInterval(time.Second * 15), wait.Timeout(time.Minute * 25), } defaultOpts = append(defaultOpts, opts...) - return statefulSetIsReady(mdb, defaultOpts...) + return statefulSetIsReady(ctx, mdb, defaultOpts...) } // ArbitersStatefulSetBecomesReady ensures that the underlying stateful set // reaches the running state. -func ArbitersStatefulSetBecomesReady(mdb *mdbv1.MongoDBCommunity, opts ...wait.Configuration) func(t *testing.T) { +func ArbitersStatefulSetBecomesReady(ctx context.Context, mdb *mdbv1.MongoDBCommunity, opts ...wait.Configuration) func(t *testing.T) { defaultOpts := []wait.Configuration{ wait.RetryInterval(time.Second * 15), wait.Timeout(time.Minute * 20), } defaultOpts = append(defaultOpts, opts...) - return arbitersStatefulSetIsReady(mdb, defaultOpts...) + return arbitersStatefulSetIsReady(ctx, mdb, defaultOpts...) } // StatefulSetBecomesUnready ensures the underlying stateful set reaches // the unready state. -func StatefulSetBecomesUnready(mdb *mdbv1.MongoDBCommunity, opts ...wait.Configuration) func(t *testing.T) { +func StatefulSetBecomesUnready(ctx context.Context, mdb *mdbv1.MongoDBCommunity, opts ...wait.Configuration) func(t *testing.T) { defaultOpts := []wait.Configuration{ wait.RetryInterval(time.Second * 15), wait.Timeout(time.Minute * 15), } defaultOpts = append(defaultOpts, opts...) - return statefulSetIsNotReady(mdb, defaultOpts...) + return statefulSetIsNotReady(ctx, mdb, defaultOpts...) } // StatefulSetIsReadyAfterScaleDown ensures that a replica set is scaled down correctly // note: scaling down takes considerably longer than scaling up due the readiness probe // failure threshold being high -func StatefulSetIsReadyAfterScaleDown(mdb *mdbv1.MongoDBCommunity) func(t *testing.T) { +func StatefulSetIsReadyAfterScaleDown(ctx context.Context, mdb *mdbv1.MongoDBCommunity) func(t *testing.T) { return func(t *testing.T) { - err := wait.ForStatefulSetToBeReadyAfterScaleDown(t, mdb, wait.RetryInterval(time.Second*60), wait.Timeout(time.Minute*45)) + err := wait.ForStatefulSetToBeReadyAfterScaleDown(ctx, t, mdb, wait.RetryInterval(time.Second*60), wait.Timeout(time.Minute*45)) if err != nil { t.Fatal(err) } @@ -89,10 +89,10 @@ func StatefulSetIsReadyAfterScaleDown(mdb *mdbv1.MongoDBCommunity) func(t *testi // statefulSetIsReady ensures that the underlying stateful set // reaches the running state. -func statefulSetIsReady(mdb *mdbv1.MongoDBCommunity, opts ...wait.Configuration) func(t *testing.T) { +func statefulSetIsReady(ctx context.Context, mdb *mdbv1.MongoDBCommunity, opts ...wait.Configuration) func(t *testing.T) { return func(t *testing.T) { start := time.Now() - err := wait.ForStatefulSetToBeReady(t, mdb, opts...) + err := wait.ForStatefulSetToBeReady(ctx, t, mdb, opts...) if err != nil { t.Fatal(err) } @@ -103,9 +103,9 @@ func statefulSetIsReady(mdb *mdbv1.MongoDBCommunity, opts ...wait.Configuration) // arbitersStatefulSetIsReady ensures that the underlying stateful set // reaches the running state. -func arbitersStatefulSetIsReady(mdb *mdbv1.MongoDBCommunity, opts ...wait.Configuration) func(t *testing.T) { +func arbitersStatefulSetIsReady(ctx context.Context, mdb *mdbv1.MongoDBCommunity, opts ...wait.Configuration) func(t *testing.T) { return func(t *testing.T) { - err := wait.ForArbitersStatefulSetToBeReady(t, mdb, opts...) + err := wait.ForArbitersStatefulSetToBeReady(ctx, t, mdb, opts...) if err != nil { t.Fatal(err) } @@ -114,9 +114,9 @@ func arbitersStatefulSetIsReady(mdb *mdbv1.MongoDBCommunity, opts ...wait.Config } // statefulSetIsNotReady ensures that the underlying stateful set reaches the unready state. -func statefulSetIsNotReady(mdb *mdbv1.MongoDBCommunity, opts ...wait.Configuration) func(t *testing.T) { +func statefulSetIsNotReady(ctx context.Context, mdb *mdbv1.MongoDBCommunity, opts ...wait.Configuration) func(t *testing.T) { return func(t *testing.T) { - err := wait.ForStatefulSetToBeUnready(t, mdb, opts...) + err := wait.ForStatefulSetToBeUnready(ctx, t, mdb, opts...) if err != nil { t.Fatal(err) } @@ -124,11 +124,11 @@ func statefulSetIsNotReady(mdb *mdbv1.MongoDBCommunity, opts ...wait.Configurati } } -func StatefulSetHasOwnerReference(mdb *mdbv1.MongoDBCommunity, expectedOwnerReference metav1.OwnerReference) func(t *testing.T) { +func StatefulSetHasOwnerReference(ctx context.Context, mdb *mdbv1.MongoDBCommunity, expectedOwnerReference metav1.OwnerReference) func(t *testing.T) { return func(t *testing.T) { stsNamespacedName := types.NamespacedName{Name: mdb.Name, Namespace: mdb.Namespace} sts := appsv1.StatefulSet{} - err := e2eutil.TestClient.Get(context.TODO(), stsNamespacedName, &sts) + err := e2eutil.TestClient.Get(ctx, stsNamespacedName, &sts) if err != nil { t.Fatal(err) @@ -138,20 +138,20 @@ func StatefulSetHasOwnerReference(mdb *mdbv1.MongoDBCommunity, expectedOwnerRefe } // StatefulSetIsDeleted ensures that the underlying stateful set is deleted -func StatefulSetIsDeleted(mdb *mdbv1.MongoDBCommunity) func(t *testing.T) { +func StatefulSetIsDeleted(ctx context.Context, mdb *mdbv1.MongoDBCommunity) func(t *testing.T) { return func(t *testing.T) { - err := wait.ForStatefulSetToBeDeleted(mdb.Name, time.Second*10, time.Minute*1, mdb.Namespace) + err := wait.ForStatefulSetToBeDeleted(ctx, mdb.Name, time.Second*10, time.Minute*1, mdb.Namespace) if err != nil { t.Fatal(err) } } } -func ServiceHasOwnerReference(mdb *mdbv1.MongoDBCommunity, expectedOwnerReference metav1.OwnerReference) func(t *testing.T) { +func ServiceHasOwnerReference(ctx context.Context, mdb *mdbv1.MongoDBCommunity, expectedOwnerReference metav1.OwnerReference) func(t *testing.T) { return func(t *testing.T) { serviceNamespacedName := types.NamespacedName{Name: mdb.ServiceName(), Namespace: mdb.Namespace} srv := corev1.Service{} - err := e2eutil.TestClient.Get(context.TODO(), serviceNamespacedName, &srv) + err := e2eutil.TestClient.Get(ctx, serviceNamespacedName, &srv) if err != nil { t.Fatal(err) } @@ -159,11 +159,11 @@ func ServiceHasOwnerReference(mdb *mdbv1.MongoDBCommunity, expectedOwnerReferenc } } -func ServiceUsesCorrectPort(mdb *mdbv1.MongoDBCommunity, expectedPort int32) func(t *testing.T) { +func ServiceUsesCorrectPort(ctx context.Context, mdb *mdbv1.MongoDBCommunity, expectedPort int32) func(t *testing.T) { return func(t *testing.T) { serviceNamespacedName := types.NamespacedName{Name: mdb.ServiceName(), Namespace: mdb.Namespace} svc := corev1.Service{} - err := e2eutil.TestClient.Get(context.TODO(), serviceNamespacedName, &svc) + err := e2eutil.TestClient.Get(ctx, serviceNamespacedName, &svc) if err != nil { t.Fatal(err) } @@ -172,22 +172,22 @@ func ServiceUsesCorrectPort(mdb *mdbv1.MongoDBCommunity, expectedPort int32) fun } } -func AgentX509SecretsExists(mdb *mdbv1.MongoDBCommunity) func(t *testing.T) { +func AgentX509SecretsExists(ctx context.Context, mdb *mdbv1.MongoDBCommunity) func(t *testing.T) { return func(t *testing.T) { agentCertSecret := corev1.Secret{} - err := e2eutil.TestClient.Get(context.TODO(), mdb.AgentCertificateSecretNamespacedName(), &agentCertSecret) + err := e2eutil.TestClient.Get(ctx, mdb.AgentCertificateSecretNamespacedName(), &agentCertSecret) assert.NoError(t, err) agentCertPemSecret := corev1.Secret{} - err = e2eutil.TestClient.Get(context.TODO(), mdb.AgentCertificatePemSecretNamespacedName(), &agentCertPemSecret) + err = e2eutil.TestClient.Get(ctx, mdb.AgentCertificatePemSecretNamespacedName(), &agentCertPemSecret) assert.NoError(t, err) } } -func AgentSecretsHaveOwnerReference(mdb *mdbv1.MongoDBCommunity, expectedOwnerReference metav1.OwnerReference) func(t *testing.T) { +func AgentSecretsHaveOwnerReference(ctx context.Context, mdb *mdbv1.MongoDBCommunity, expectedOwnerReference metav1.OwnerReference) func(t *testing.T) { checkSecret := func(t *testing.T, resourceNamespacedName types.NamespacedName) { secret := corev1.Secret{} - err := e2eutil.TestClient.Get(context.TODO(), resourceNamespacedName, &secret) + err := e2eutil.TestClient.Get(ctx, resourceNamespacedName, &secret) assert.NoError(t, err) assertEqualOwnerReference(t, "Secret", resourceNamespacedName, secret.GetOwnerReferences(), expectedOwnerReference) @@ -201,12 +201,12 @@ func AgentSecretsHaveOwnerReference(mdb *mdbv1.MongoDBCommunity, expectedOwnerRe // ConnectionStringSecretsAreConfigured verifies that secrets storing the connection string were generated for all scram users // and that they have the expected owner reference -func ConnectionStringSecretsAreConfigured(mdb *mdbv1.MongoDBCommunity, expectedOwnerReference metav1.OwnerReference) func(t *testing.T) { +func ConnectionStringSecretsAreConfigured(ctx context.Context, mdb *mdbv1.MongoDBCommunity, expectedOwnerReference metav1.OwnerReference) func(t *testing.T) { return func(t *testing.T) { for _, user := range mdb.GetAuthUsers() { secret := corev1.Secret{} secretNamespacedName := types.NamespacedName{Name: user.ConnectionStringSecretName, Namespace: mdb.Namespace} - err := e2eutil.TestClient.Get(context.TODO(), secretNamespacedName, &secret) + err := e2eutil.TestClient.Get(ctx, secretNamespacedName, &secret) assert.NoError(t, err) assertEqualOwnerReference(t, "Secret", secretNamespacedName, secret.GetOwnerReferences(), expectedOwnerReference) @@ -216,9 +216,9 @@ func ConnectionStringSecretsAreConfigured(mdb *mdbv1.MongoDBCommunity, expectedO // StatefulSetHasUpdateStrategy verifies that the StatefulSet holding this MongoDB // resource has the correct Update Strategy -func StatefulSetHasUpdateStrategy(mdb *mdbv1.MongoDBCommunity, strategy appsv1.StatefulSetUpdateStrategyType) func(t *testing.T) { +func StatefulSetHasUpdateStrategy(ctx context.Context, mdb *mdbv1.MongoDBCommunity, strategy appsv1.StatefulSetUpdateStrategyType) func(t *testing.T) { return func(t *testing.T) { - err := wait.ForStatefulSetToHaveUpdateStrategy(t, mdb, strategy, wait.RetryInterval(time.Second*15), wait.Timeout(time.Minute*8)) + err := wait.ForStatefulSetToHaveUpdateStrategy(ctx, t, mdb, strategy, wait.RetryInterval(time.Second*15), wait.Timeout(time.Minute*8)) if err != nil { t.Fatal(err) } @@ -227,8 +227,8 @@ func StatefulSetHasUpdateStrategy(mdb *mdbv1.MongoDBCommunity, strategy appsv1.S } // GetPersistentVolumes returns all persistent volumes on the cluster -func getPersistentVolumesList() (*corev1.PersistentVolumeList, error) { - return e2eutil.TestClient.CoreV1Client.PersistentVolumes().List(context.TODO(), metav1.ListOptions{}) +func getPersistentVolumesList(ctx context.Context) (*corev1.PersistentVolumeList, error) { + return e2eutil.TestClient.CoreV1Client.PersistentVolumes().List(ctx, metav1.ListOptions{}) } func containsVolume(volumes []corev1.PersistentVolume, volumeName string) bool { @@ -240,9 +240,9 @@ func containsVolume(volumes []corev1.PersistentVolume, volumeName string) bool { return false } -func HasExpectedPersistentVolumes(volumes []corev1.PersistentVolume) func(t *testing.T) { +func HasExpectedPersistentVolumes(ctx context.Context, volumes []corev1.PersistentVolume) func(t *testing.T) { return func(t *testing.T) { - volumeList, err := getPersistentVolumesList() + volumeList, err := getPersistentVolumesList(ctx) actualVolumes := volumeList.Items assert.NoError(t, err) assert.Len(t, actualVolumes, len(volumes), @@ -253,12 +253,12 @@ func HasExpectedPersistentVolumes(volumes []corev1.PersistentVolume) func(t *tes } } } -func HasExpectedMetadata(mdb *mdbv1.MongoDBCommunity, expectedLabels map[string]string, expectedAnnotations map[string]string) func(t *testing.T) { +func HasExpectedMetadata(ctx context.Context, mdb *mdbv1.MongoDBCommunity, expectedLabels map[string]string, expectedAnnotations map[string]string) func(t *testing.T) { return func(t *testing.T) { namespace := mdb.Namespace statefulSetList := appsv1.StatefulSetList{} - err := e2eutil.TestClient.Client.List(context.TODO(), &statefulSetList, client.InNamespace(namespace)) + err := e2eutil.TestClient.Client.List(ctx, &statefulSetList, client.InNamespace(namespace)) assert.NoError(t, err) assert.NotEmpty(t, statefulSetList.Items) for _, s := range statefulSetList.Items { @@ -266,7 +266,7 @@ func HasExpectedMetadata(mdb *mdbv1.MongoDBCommunity, expectedLabels map[string] } volumeList := corev1.PersistentVolumeList{} - err = e2eutil.TestClient.Client.List(context.TODO(), &volumeList, client.InNamespace(namespace)) + err = e2eutil.TestClient.Client.List(ctx, &volumeList, client.InNamespace(namespace)) assert.NoError(t, err) assert.NotEmpty(t, volumeList.Items) for _, s := range volumeList.Items { @@ -277,7 +277,7 @@ func HasExpectedMetadata(mdb *mdbv1.MongoDBCommunity, expectedLabels map[string] } podList := corev1.PodList{} - err = e2eutil.TestClient.Client.List(context.TODO(), &podList, client.InNamespace(namespace)) + err = e2eutil.TestClient.Client.List(ctx, &podList, client.InNamespace(namespace)) assert.NoError(t, err) assert.NotEmpty(t, podList.Items) @@ -326,9 +326,9 @@ func containsMetadata(t *testing.T, metadata *metav1.ObjectMeta, expectedLabels } // MongoDBReachesPendingPhase ensures the MongoDB resources gets to the Pending phase -func MongoDBReachesPendingPhase(mdb *mdbv1.MongoDBCommunity) func(t *testing.T) { +func MongoDBReachesPendingPhase(ctx context.Context, mdb *mdbv1.MongoDBCommunity) func(t *testing.T) { return func(t *testing.T) { - err := wait.ForMongoDBToReachPhase(t, mdb, mdbv1.Pending, time.Second*15, time.Minute*2) + err := wait.ForMongoDBToReachPhase(ctx, t, mdb, mdbv1.Pending, time.Second*15, time.Minute*2) if err != nil { t.Fatal(err) } @@ -337,9 +337,9 @@ func MongoDBReachesPendingPhase(mdb *mdbv1.MongoDBCommunity) func(t *testing.T) } // MongoDBReachesRunningPhase ensure the MongoDB resource reaches the Running phase -func MongoDBReachesRunningPhase(mdb *mdbv1.MongoDBCommunity) func(t *testing.T) { +func MongoDBReachesRunningPhase(ctx context.Context, mdb *mdbv1.MongoDBCommunity) func(t *testing.T) { return func(t *testing.T) { - err := wait.ForMongoDBToReachPhase(t, mdb, mdbv1.Running, time.Second*15, time.Minute*12) + err := wait.ForMongoDBToReachPhase(ctx, t, mdb, mdbv1.Running, time.Second*15, time.Minute*12) if err != nil { t.Fatal(err) } @@ -348,9 +348,9 @@ func MongoDBReachesRunningPhase(mdb *mdbv1.MongoDBCommunity) func(t *testing.T) } // MongoDBReachesFailedPhase ensure the MongoDB resource reaches the Failed phase. -func MongoDBReachesFailedPhase(mdb *mdbv1.MongoDBCommunity) func(t *testing.T) { +func MongoDBReachesFailedPhase(ctx context.Context, mdb *mdbv1.MongoDBCommunity) func(t *testing.T) { return func(t *testing.T) { - err := wait.ForMongoDBToReachPhase(t, mdb, mdbv1.Failed, time.Second*15, time.Minute*5) + err := wait.ForMongoDBToReachPhase(ctx, t, mdb, mdbv1.Failed, time.Second*15, time.Minute*5) if err != nil { t.Fatal(err) } @@ -358,9 +358,9 @@ func MongoDBReachesFailedPhase(mdb *mdbv1.MongoDBCommunity) func(t *testing.T) { } } -func AutomationConfigSecretExists(mdb *mdbv1.MongoDBCommunity) func(t *testing.T) { +func AutomationConfigSecretExists(ctx context.Context, mdb *mdbv1.MongoDBCommunity) func(t *testing.T) { return func(t *testing.T) { - s, err := wait.ForSecretToExist(mdb.AutomationConfigSecretName(), time.Second*5, time.Minute*1, mdb.Namespace) + s, err := wait.ForSecretToExist(ctx, mdb.AutomationConfigSecretName(), time.Second*5, time.Minute*1, mdb.Namespace) assert.NoError(t, err) t.Logf("Secret %s/%s was successfully created", mdb.Namespace, mdb.AutomationConfigSecretName()) @@ -370,10 +370,10 @@ func AutomationConfigSecretExists(mdb *mdbv1.MongoDBCommunity) func(t *testing.T } } -func getAutomationConfig(t *testing.T, mdb *mdbv1.MongoDBCommunity) automationconfig.AutomationConfig { +func getAutomationConfig(ctx context.Context, t *testing.T, mdb *mdbv1.MongoDBCommunity) automationconfig.AutomationConfig { currentSecret := corev1.Secret{} currentAc := automationconfig.AutomationConfig{} - err := e2eutil.TestClient.Get(context.TODO(), types.NamespacedName{Name: mdb.AutomationConfigSecretName(), Namespace: mdb.Namespace}, ¤tSecret) + err := e2eutil.TestClient.Get(ctx, types.NamespacedName{Name: mdb.AutomationConfigSecretName(), Namespace: mdb.Namespace}, ¤tSecret) assert.NoError(t, err) err = json.Unmarshal(currentSecret.Data[automationconfig.ConfigKey], ¤tAc) assert.NoError(t, err) @@ -381,34 +381,34 @@ func getAutomationConfig(t *testing.T, mdb *mdbv1.MongoDBCommunity) automationco } // AutomationConfigVersionHasTheExpectedVersion verifies that the automation config has the expected version. -func AutomationConfigVersionHasTheExpectedVersion(mdb *mdbv1.MongoDBCommunity, expectedVersion int) func(t *testing.T) { +func AutomationConfigVersionHasTheExpectedVersion(ctx context.Context, mdb *mdbv1.MongoDBCommunity, expectedVersion int) func(t *testing.T) { return func(t *testing.T) { - currentAc := getAutomationConfig(t, mdb) + currentAc := getAutomationConfig(ctx, t, mdb) assert.Equal(t, expectedVersion, currentAc.Version) } } // AutomationConfigHasLogRotationConfig verifies that the automation config contains the given logRotate config. -func AutomationConfigHasLogRotationConfig(mdb *mdbv1.MongoDBCommunity, lrc *automationconfig.CrdLogRotate) func(t *testing.T) { +func AutomationConfigHasLogRotationConfig(ctx context.Context, mdb *mdbv1.MongoDBCommunity, lrc *automationconfig.CrdLogRotate) func(t *testing.T) { return func(t *testing.T) { - currentAc := getAutomationConfig(t, mdb) + currentAc := getAutomationConfig(ctx, t, mdb) for _, p := range currentAc.Processes { assert.Equal(t, automationconfig.ConvertCrdLogRotateToAC(lrc), p.LogRotate) } } } -func AutomationConfigHasSettings(mdb *mdbv1.MongoDBCommunity, settings map[string]interface{}) func(t *testing.T) { +func AutomationConfigHasSettings(ctx context.Context, mdb *mdbv1.MongoDBCommunity, settings map[string]interface{}) func(t *testing.T) { return func(t *testing.T) { - currentAc := getAutomationConfig(t, mdb) + currentAc := getAutomationConfig(ctx, t, mdb) assert.Equal(t, currentAc.ReplicaSets[0].Settings, settings) } } // AutomationConfigReplicaSetsHaveExpectedArbiters verifies that the automation config has the expected version. -func AutomationConfigReplicaSetsHaveExpectedArbiters(mdb *mdbv1.MongoDBCommunity, expectedArbiters int) func(t *testing.T) { +func AutomationConfigReplicaSetsHaveExpectedArbiters(ctx context.Context, mdb *mdbv1.MongoDBCommunity, expectedArbiters int) func(t *testing.T) { return func(t *testing.T) { - currentAc := getAutomationConfig(t, mdb) + currentAc := getAutomationConfig(ctx, t, mdb) lsRs := currentAc.ReplicaSets for _, rs := range lsRs { arbiters := 0 @@ -423,18 +423,18 @@ func AutomationConfigReplicaSetsHaveExpectedArbiters(mdb *mdbv1.MongoDBCommunity } // AutomationConfigHasTheExpectedCustomRoles verifies that the automation config has the expected custom roles. -func AutomationConfigHasTheExpectedCustomRoles(mdb *mdbv1.MongoDBCommunity, roles []automationconfig.CustomRole) func(t *testing.T) { +func AutomationConfigHasTheExpectedCustomRoles(ctx context.Context, mdb *mdbv1.MongoDBCommunity, roles []automationconfig.CustomRole) func(t *testing.T) { return func(t *testing.T) { - currentAc := getAutomationConfig(t, mdb) + currentAc := getAutomationConfig(ctx, t, mdb) assert.ElementsMatch(t, roles, currentAc.Roles) } } -func AutomationConfigHasVoteTagPriorityConfigured(mdb *mdbv1.MongoDBCommunity, memberOptions []automationconfig.MemberOptions) func(t *testing.T) { +func AutomationConfigHasVoteTagPriorityConfigured(ctx context.Context, mdb *mdbv1.MongoDBCommunity, memberOptions []automationconfig.MemberOptions) func(t *testing.T) { acMemberOptions := make([]automationconfig.MemberOptions, 0) return func(t *testing.T) { - currentAc := getAutomationConfig(t, mdb) + currentAc := getAutomationConfig(ctx, t, mdb) rsMemebers := currentAc.ReplicaSets for _, m := range rsMemebers[0].Members { @@ -445,9 +445,9 @@ func AutomationConfigHasVoteTagPriorityConfigured(mdb *mdbv1.MongoDBCommunity, m } // CreateMongoDBResource creates the MongoDB resource -func CreateMongoDBResource(mdb *mdbv1.MongoDBCommunity, ctx *e2eutil.Context) func(*testing.T) { +func CreateMongoDBResource(mdb *mdbv1.MongoDBCommunity, textCtx *e2eutil.TestContext) func(*testing.T) { return func(t *testing.T) { - if err := e2eutil.TestClient.Create(context.TODO(), mdb, &e2eutil.CleanupOptions{TestContext: ctx}); err != nil { + if err := e2eutil.TestClient.Create(textCtx.Ctx, mdb, &e2eutil.CleanupOptions{TestContext: textCtx}); err != nil { t.Fatal(err) } t.Logf("Created MongoDB resource %s/%s", mdb.Name, mdb.Namespace) @@ -455,9 +455,9 @@ func CreateMongoDBResource(mdb *mdbv1.MongoDBCommunity, ctx *e2eutil.Context) fu } // DeleteMongoDBResource deletes the MongoDB resource -func DeleteMongoDBResource(mdb *mdbv1.MongoDBCommunity, ctx *e2eutil.Context) func(*testing.T) { +func DeleteMongoDBResource(mdb *mdbv1.MongoDBCommunity, testCtx *e2eutil.TestContext) func(*testing.T) { return func(t *testing.T) { - if err := e2eutil.TestClient.Delete(context.TODO(), mdb); err != nil { + if err := e2eutil.TestClient.Delete(testCtx.Ctx, mdb); err != nil { t.Fatal(err) } t.Logf("Deleted MongoDB resource %s/%s", mdb.Name, mdb.Namespace) @@ -465,21 +465,21 @@ func DeleteMongoDBResource(mdb *mdbv1.MongoDBCommunity, ctx *e2eutil.Context) fu } // GetConnectionStringSecret returnes the secret generated by the operator that is storing the connection string for a specific user -func GetConnectionStringSecret(mdb mdbv1.MongoDBCommunity, user authtypes.User) corev1.Secret { +func GetConnectionStringSecret(ctx context.Context, mdb mdbv1.MongoDBCommunity, user authtypes.User) corev1.Secret { secret := corev1.Secret{} secretNamespacedName := types.NamespacedName{Name: user.ConnectionStringSecretName, Namespace: mdb.Namespace} - _ = e2eutil.TestClient.Get(context.TODO(), secretNamespacedName, &secret) + _ = e2eutil.TestClient.Get(ctx, secretNamespacedName, &secret) return secret } // GetConnectionStringForUser returns the mongodb standard connection string for a user -func GetConnectionStringForUser(mdb mdbv1.MongoDBCommunity, user authtypes.User) string { - return string(GetConnectionStringSecret(mdb, user).Data["connectionString.standard"]) +func GetConnectionStringForUser(ctx context.Context, mdb mdbv1.MongoDBCommunity, user authtypes.User) string { + return string(GetConnectionStringSecret(ctx, mdb, user).Data["connectionString.standard"]) } // GetSrvConnectionStringForUser returns the mongodb service connection string for a user -func GetSrvConnectionStringForUser(mdb mdbv1.MongoDBCommunity, user authtypes.User) string { - return string(GetConnectionStringSecret(mdb, user).Data["connectionString.standardSrv"]) +func GetSrvConnectionStringForUser(ctx context.Context, mdb mdbv1.MongoDBCommunity, user authtypes.User) string { + return string(GetConnectionStringSecret(ctx, mdb, user).Data["connectionString.standardSrv"]) } func getOwnerReference(mdb *mdbv1.MongoDBCommunity) metav1.OwnerReference { @@ -490,48 +490,47 @@ func getOwnerReference(mdb *mdbv1.MongoDBCommunity) metav1.OwnerReference { }) } -func BasicFunctionality(mdb *mdbv1.MongoDBCommunity, skipStatusCheck ...bool) func(*testing.T) { +func BasicFunctionality(ctx context.Context, mdb *mdbv1.MongoDBCommunity, skipStatusCheck ...bool) func(*testing.T) { return func(t *testing.T) { mdbOwnerReference := getOwnerReference(mdb) - t.Run("Secret Was Correctly Created", AutomationConfigSecretExists(mdb)) - t.Run("Stateful Set Reaches Ready State", StatefulSetBecomesReady(mdb)) - t.Run("MongoDB Reaches Running Phase", MongoDBReachesRunningPhase(mdb)) - t.Run("Stateful Set Has OwnerReference", StatefulSetHasOwnerReference(mdb, mdbOwnerReference)) - t.Run("Service Set Has OwnerReference", ServiceHasOwnerReference(mdb, mdbOwnerReference)) - t.Run("Agent Secrets Have OwnerReference", AgentSecretsHaveOwnerReference(mdb, mdbOwnerReference)) - t.Run("Connection string secrets are configured", ConnectionStringSecretsAreConfigured(mdb, mdbOwnerReference)) + t.Run("Secret Was Correctly Created", AutomationConfigSecretExists(ctx, mdb)) + t.Run("Stateful Set Reaches Ready State", StatefulSetBecomesReady(ctx, mdb)) + t.Run("MongoDB Reaches Running Phase", MongoDBReachesRunningPhase(ctx, mdb)) + t.Run("Stateful Set Has OwnerReference", StatefulSetHasOwnerReference(ctx, mdb, mdbOwnerReference)) + t.Run("Service Set Has OwnerReference", ServiceHasOwnerReference(ctx, mdb, mdbOwnerReference)) + t.Run("Agent Secrets Have OwnerReference", AgentSecretsHaveOwnerReference(ctx, mdb, mdbOwnerReference)) + t.Run("Connection string secrets are configured", ConnectionStringSecretsAreConfigured(ctx, mdb, mdbOwnerReference)) // TODO: this is temporary, remove the need for skipStatuscheck after 0.7.4 operator release if len(skipStatusCheck) > 0 && !skipStatusCheck[0] { - t.Run("Test Status Was Updated", Status(mdb, - mdbv1.MongoDBCommunityStatus{ - MongoURI: mdb.MongoURI(""), - Phase: mdbv1.Running, - Version: mdb.GetMongoDBVersion(nil), - CurrentMongoDBMembers: mdb.Spec.Members, - CurrentStatefulSetReplicas: mdb.Spec.Members, - })) + t.Run("Test Status Was Updated", Status(ctx, mdb, mdbv1.MongoDBCommunityStatus{ + MongoURI: mdb.MongoURI(""), + Phase: mdbv1.Running, + Version: mdb.GetMongoDBVersion(nil), + CurrentMongoDBMembers: mdb.Spec.Members, + CurrentStatefulSetReplicas: mdb.Spec.Members, + })) } } } -func BasicFunctionalityX509(mdb *mdbv1.MongoDBCommunity) func(t *testing.T) { +func BasicFunctionalityX509(ctx context.Context, mdb *mdbv1.MongoDBCommunity) func(t *testing.T) { return func(t *testing.T) { mdbOwnerReference := getOwnerReference(mdb) - t.Run("Secret Was Correctly Created", AutomationConfigSecretExists(mdb)) - t.Run("Stateful Set Reaches Ready State", StatefulSetBecomesReady(mdb)) - t.Run("MongoDB Reaches Running Phase", MongoDBReachesRunningPhase(mdb)) - t.Run("Stateful Set Has OwnerReference", StatefulSetHasOwnerReference(mdb, mdbOwnerReference)) - t.Run("Service Set Has OwnerReference", ServiceHasOwnerReference(mdb, mdbOwnerReference)) - t.Run("Connection string secrets are configured", ConnectionStringSecretsAreConfigured(mdb, mdbOwnerReference)) + t.Run("Secret Was Correctly Created", AutomationConfigSecretExists(ctx, mdb)) + t.Run("Stateful Set Reaches Ready State", StatefulSetBecomesReady(ctx, mdb)) + t.Run("MongoDB Reaches Running Phase", MongoDBReachesRunningPhase(ctx, mdb)) + t.Run("Stateful Set Has OwnerReference", StatefulSetHasOwnerReference(ctx, mdb, mdbOwnerReference)) + t.Run("Service Set Has OwnerReference", ServiceHasOwnerReference(ctx, mdb, mdbOwnerReference)) + t.Run("Connection string secrets are configured", ConnectionStringSecretsAreConfigured(ctx, mdb, mdbOwnerReference)) } } // ServiceWithNameExists checks whether a service with the name serviceName exists -func ServiceWithNameExists(serviceName string, namespace string) func(t *testing.T) { +func ServiceWithNameExists(ctx context.Context, serviceName string, namespace string) func(t *testing.T) { return func(t *testing.T) { serviceNamespacedName := types.NamespacedName{Name: serviceName, Namespace: namespace} srv := corev1.Service{} - err := e2eutil.TestClient.Get(context.TODO(), serviceNamespacedName, &srv) + err := e2eutil.TestClient.Get(ctx, serviceNamespacedName, &srv) if err != nil { t.Fatal(err) } @@ -540,10 +539,10 @@ func ServiceWithNameExists(serviceName string, namespace string) func(t *testing } // DeletePod will delete a pod that belongs to this MongoDB resource's StatefulSet -func DeletePod(mdb *mdbv1.MongoDBCommunity, podNum int) func(*testing.T) { +func DeletePod(ctx context.Context, mdb *mdbv1.MongoDBCommunity, podNum int) func(*testing.T) { return func(t *testing.T) { pod := podFromMongoDBCommunity(mdb, podNum) - if err := e2eutil.TestClient.Delete(context.TODO(), &pod); err != nil { + if err := e2eutil.TestClient.Delete(ctx, &pod); err != nil { t.Fatal(err) } @@ -552,7 +551,7 @@ func DeletePod(mdb *mdbv1.MongoDBCommunity, podNum int) func(*testing.T) { } // DeleteStatefulSet provides a wrapper to delete appsv1.StatefulSet types -func DeleteStatefulSet(mdb *mdbv1.MongoDBCommunity) func(*testing.T) { +func DeleteStatefulSet(ctx context.Context, mdb *mdbv1.MongoDBCommunity) func(*testing.T) { return func(t *testing.T) { sts := appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ @@ -560,7 +559,7 @@ func DeleteStatefulSet(mdb *mdbv1.MongoDBCommunity) func(*testing.T) { Namespace: mdb.Namespace, }, } - if err := e2eutil.TestClient.Delete(context.TODO(), &sts); err != nil { + if err := e2eutil.TestClient.Delete(ctx, &sts); err != nil { t.Fatal(err) } @@ -569,9 +568,9 @@ func DeleteStatefulSet(mdb *mdbv1.MongoDBCommunity) func(*testing.T) { } // Status compares the given status to the actual status of the MongoDB resource -func Status(mdb *mdbv1.MongoDBCommunity, expectedStatus mdbv1.MongoDBCommunityStatus) func(t *testing.T) { +func Status(ctx context.Context, mdb *mdbv1.MongoDBCommunity, expectedStatus mdbv1.MongoDBCommunityStatus) func(t *testing.T) { return func(t *testing.T) { - if err := e2eutil.TestClient.Get(context.TODO(), types.NamespacedName{Name: mdb.Name, Namespace: mdb.Namespace}, mdb); err != nil { + if err := e2eutil.TestClient.Get(ctx, types.NamespacedName{Name: mdb.Name, Namespace: mdb.Namespace}, mdb); err != nil { t.Fatalf("error getting MongoDB resource: %s", err) } assert.Equal(t, expectedStatus, mdb.Status) @@ -579,10 +578,10 @@ func Status(mdb *mdbv1.MongoDBCommunity, expectedStatus mdbv1.MongoDBCommunitySt } // Scale update the MongoDB with a new number of members and updates the resource. -func Scale(mdb *mdbv1.MongoDBCommunity, newMembers int) func(*testing.T) { +func Scale(ctx context.Context, mdb *mdbv1.MongoDBCommunity, newMembers int) func(*testing.T) { return func(t *testing.T) { t.Logf("Scaling Mongodb %s, to %d members", mdb.Name, newMembers) - err := e2eutil.UpdateMongoDBResource(mdb, func(db *mdbv1.MongoDBCommunity) { + err := e2eutil.UpdateMongoDBResource(ctx, mdb, func(db *mdbv1.MongoDBCommunity) { db.Spec.Members = newMembers }) if err != nil { @@ -592,10 +591,10 @@ func Scale(mdb *mdbv1.MongoDBCommunity, newMembers int) func(*testing.T) { } // ScaleArbiters update the MongoDB with a new number of arbiters and updates the resource. -func ScaleArbiters(mdb *mdbv1.MongoDBCommunity, newArbiters int) func(*testing.T) { +func ScaleArbiters(ctx context.Context, mdb *mdbv1.MongoDBCommunity, newArbiters int) func(*testing.T) { return func(t *testing.T) { t.Logf("Scaling Mongodb %s, to %d members", mdb.Name, newArbiters) - err := e2eutil.UpdateMongoDBResource(mdb, func(db *mdbv1.MongoDBCommunity) { + err := e2eutil.UpdateMongoDBResource(ctx, mdb, func(db *mdbv1.MongoDBCommunity) { db.Spec.Arbiters = newArbiters }) if err != nil { @@ -605,20 +604,20 @@ func ScaleArbiters(mdb *mdbv1.MongoDBCommunity, newArbiters int) func(*testing.T } // DisableTLS changes the tls.enabled attribute to false. -func DisableTLS(mdb *mdbv1.MongoDBCommunity) func(*testing.T) { - return tls(mdb, false) +func DisableTLS(ctx context.Context, mdb *mdbv1.MongoDBCommunity) func(*testing.T) { + return tls(ctx, mdb, false) } // EnableTLS changes the tls.enabled attribute to true. -func EnableTLS(mdb *mdbv1.MongoDBCommunity) func(*testing.T) { - return tls(mdb, true) +func EnableTLS(ctx context.Context, mdb *mdbv1.MongoDBCommunity) func(*testing.T) { + return tls(ctx, mdb, true) } // tls function configures the security.tls.enabled attribute. -func tls(mdb *mdbv1.MongoDBCommunity, enabled bool) func(*testing.T) { +func tls(ctx context.Context, mdb *mdbv1.MongoDBCommunity, enabled bool) func(*testing.T) { return func(t *testing.T) { t.Logf("Setting security.tls.enabled to %t", enabled) - err := e2eutil.UpdateMongoDBResource(mdb, func(db *mdbv1.MongoDBCommunity) { + err := e2eutil.UpdateMongoDBResource(ctx, mdb, func(db *mdbv1.MongoDBCommunity) { db.Spec.Security.TLS.Enabled = enabled }) if err != nil { @@ -627,10 +626,10 @@ func tls(mdb *mdbv1.MongoDBCommunity, enabled bool) func(*testing.T) { } } -func ChangeVersion(mdb *mdbv1.MongoDBCommunity, newVersion string) func(*testing.T) { +func ChangeVersion(ctx context.Context, mdb *mdbv1.MongoDBCommunity, newVersion string) func(*testing.T) { return func(t *testing.T) { t.Logf("Changing versions from: %s to %s", mdb.Spec.Version, newVersion) - err := e2eutil.UpdateMongoDBResource(mdb, func(db *mdbv1.MongoDBCommunity) { + err := e2eutil.UpdateMongoDBResource(ctx, mdb, func(db *mdbv1.MongoDBCommunity) { db.Spec.Version = newVersion }) if err != nil { @@ -639,10 +638,10 @@ func ChangeVersion(mdb *mdbv1.MongoDBCommunity, newVersion string) func(*testing } } -func ChangePort(mdb *mdbv1.MongoDBCommunity, newPort int) func(*testing.T) { +func ChangePort(ctx context.Context, mdb *mdbv1.MongoDBCommunity, newPort int) func(*testing.T) { return func(t *testing.T) { t.Logf("Changing port from: %d to %d", mdb.GetMongodConfiguration().GetDBPort(), newPort) - err := e2eutil.UpdateMongoDBResource(mdb, func(db *mdbv1.MongoDBCommunity) { + err := e2eutil.UpdateMongoDBResource(ctx, mdb, func(db *mdbv1.MongoDBCommunity) { db.Spec.AdditionalMongodConfig.SetDBPort(newPort) }) if err != nil { @@ -651,10 +650,10 @@ func ChangePort(mdb *mdbv1.MongoDBCommunity, newPort int) func(*testing.T) { } } -func AddConnectionStringOption(mdb *mdbv1.MongoDBCommunity, key string, value interface{}) func(t *testing.T) { +func AddConnectionStringOption(ctx context.Context, mdb *mdbv1.MongoDBCommunity, key string, value interface{}) func(t *testing.T) { return func(t *testing.T) { t.Logf("Adding %s:%v to connection string", key, value) - err := e2eutil.UpdateMongoDBResource(mdb, func(db *mdbv1.MongoDBCommunity) { + err := e2eutil.UpdateMongoDBResource(ctx, mdb, func(db *mdbv1.MongoDBCommunity) { db.Spec.AdditionalConnectionStringConfig.SetOption(key, value) }) if err != nil { @@ -663,9 +662,9 @@ func AddConnectionStringOption(mdb *mdbv1.MongoDBCommunity, key string, value in } } -func ResetConnectionStringOptions(mdb *mdbv1.MongoDBCommunity) func(t *testing.T) { +func ResetConnectionStringOptions(ctx context.Context, mdb *mdbv1.MongoDBCommunity) func(t *testing.T) { return func(t *testing.T) { - err := e2eutil.UpdateMongoDBResource(mdb, func(db *mdbv1.MongoDBCommunity) { + err := e2eutil.UpdateMongoDBResource(ctx, mdb, func(db *mdbv1.MongoDBCommunity) { db.Spec.AdditionalConnectionStringConfig = mdbv1.NewMapWrapper() db.Spec.Users[0].AdditionalConnectionStringConfig = mdbv1.NewMapWrapper() }) @@ -675,10 +674,10 @@ func ResetConnectionStringOptions(mdb *mdbv1.MongoDBCommunity) func(t *testing.T } } -func AddConnectionStringOptionToUser(mdb *mdbv1.MongoDBCommunity, key string, value interface{}) func(t *testing.T) { +func AddConnectionStringOptionToUser(ctx context.Context, mdb *mdbv1.MongoDBCommunity, key string, value interface{}) func(t *testing.T) { return func(t *testing.T) { t.Logf("Adding %s:%v to connection string to first user", key, value) - err := e2eutil.UpdateMongoDBResource(mdb, func(db *mdbv1.MongoDBCommunity) { + err := e2eutil.UpdateMongoDBResource(ctx, mdb, func(db *mdbv1.MongoDBCommunity) { db.Spec.Users[0].AdditionalConnectionStringConfig.SetOption(key, value) }) if err != nil { @@ -687,10 +686,10 @@ func AddConnectionStringOptionToUser(mdb *mdbv1.MongoDBCommunity, key string, va } } -func StatefulSetContainerConditionIsTrue(mdb *mdbv1.MongoDBCommunity, containerName string, condition func(c corev1.Container) bool) func(*testing.T) { +func StatefulSetContainerConditionIsTrue(ctx context.Context, mdb *mdbv1.MongoDBCommunity, containerName string, condition func(c corev1.Container) bool) func(*testing.T) { return func(t *testing.T) { sts := appsv1.StatefulSet{} - err := e2eutil.TestClient.Get(context.TODO(), types.NamespacedName{Name: mdb.Name, Namespace: mdb.Namespace}, &sts) + err := e2eutil.TestClient.Get(ctx, types.NamespacedName{Name: mdb.Name, Namespace: mdb.Namespace}, &sts) if err != nil { t.Fatal(err) } @@ -706,10 +705,10 @@ func StatefulSetContainerConditionIsTrue(mdb *mdbv1.MongoDBCommunity, containerN } } -func StatefulSetConditionIsTrue(mdb *mdbv1.MongoDBCommunity, condition func(s appsv1.StatefulSet) bool) func(*testing.T) { +func StatefulSetConditionIsTrue(ctx context.Context, mdb *mdbv1.MongoDBCommunity, condition func(s appsv1.StatefulSet) bool) func(*testing.T) { return func(t *testing.T) { sts := appsv1.StatefulSet{} - err := e2eutil.TestClient.Get(context.TODO(), types.NamespacedName{Name: mdb.Name, Namespace: mdb.Namespace}, &sts) + err := e2eutil.TestClient.Get(ctx, types.NamespacedName{Name: mdb.Name, Namespace: mdb.Namespace}, &sts) if err != nil { t.Fatal(err) } @@ -721,33 +720,33 @@ func StatefulSetConditionIsTrue(mdb *mdbv1.MongoDBCommunity, condition func(s ap } // PodContainerBecomesNotReady waits until the container with 'containerName' in the pod #podNum becomes not ready. -func PodContainerBecomesNotReady(mdb *mdbv1.MongoDBCommunity, podNum int, containerName string) func(*testing.T) { +func PodContainerBecomesNotReady(ctx context.Context, mdb *mdbv1.MongoDBCommunity, podNum int, containerName string) func(*testing.T) { return func(t *testing.T) { pod := podFromMongoDBCommunity(mdb, podNum) - assert.NoError(t, wait.ForPodReadiness(t, false, containerName, time.Minute*10, pod)) + assert.NoError(t, wait.ForPodReadiness(ctx, t, false, containerName, time.Minute*10, pod)) } } // PodContainerBecomesReady waits until the container with 'containerName' in the pod #podNum becomes ready. -func PodContainerBecomesReady(mdb *mdbv1.MongoDBCommunity, podNum int, containerName string) func(*testing.T) { +func PodContainerBecomesReady(ctx context.Context, mdb *mdbv1.MongoDBCommunity, podNum int, containerName string) func(*testing.T) { return func(t *testing.T) { pod := podFromMongoDBCommunity(mdb, podNum) - assert.NoError(t, wait.ForPodReadiness(t, true, containerName, time.Minute*3, pod)) + assert.NoError(t, wait.ForPodReadiness(ctx, t, true, containerName, time.Minute*3, pod)) } } -func ExecInContainer(mdb *mdbv1.MongoDBCommunity, podNum int, containerName, command string) func(*testing.T) { +func ExecInContainer(ctx context.Context, mdb *mdbv1.MongoDBCommunity, podNum int, containerName, command string) func(*testing.T) { return func(t *testing.T) { pod := podFromMongoDBCommunity(mdb, podNum) - _, err := e2eutil.TestClient.Execute(pod, containerName, command) + _, err := e2eutil.TestClient.Execute(ctx, pod, containerName, command) assert.NoError(t, err) } } // StatefulSetMessageIsReceived waits (up to 5 minutes) to get desiredMessageStatus as a mongodb message status or returns a fatal error. -func StatefulSetMessageIsReceived(mdb *mdbv1.MongoDBCommunity, ctx *e2eutil.Context, desiredMessageStatus string) func(t *testing.T) { +func StatefulSetMessageIsReceived(mdb *mdbv1.MongoDBCommunity, testCtx *e2eutil.TestContext, desiredMessageStatus string) func(t *testing.T) { return func(t *testing.T) { - err := wait.ForMongoDBMessageStatus(t, mdb, time.Second*15, time.Minute*5, desiredMessageStatus) + err := wait.ForMongoDBMessageStatus(testCtx.Ctx, t, mdb, time.Second*15, time.Minute*5, desiredMessageStatus) if err != nil { t.Fatal(err) } diff --git a/test/e2e/prometheus/prometheus_test.go b/test/e2e/prometheus/prometheus_test.go index 39e355019..809b9ca9c 100644 --- a/test/e2e/prometheus/prometheus_test.go +++ b/test/e2e/prometheus/prometheus_test.go @@ -1,6 +1,7 @@ package prometheus import ( + "context" "fmt" "os" "testing" @@ -10,7 +11,7 @@ import ( e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/mongodbtests" - setup "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" + "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" "github.com/stretchr/testify/assert" ) @@ -23,44 +24,45 @@ func TestMain(m *testing.M) { } func TestPrometheus(t *testing.T) { + ctx := context.Background() resourceName := "mdb0" - ctx, testConfig := setup.SetupWithTLS(t, resourceName) - defer ctx.Teardown() + testCtx, testConfig := setup.SetupWithTLS(ctx, t, resourceName) + defer testCtx.Teardown() - mdb, user := e2eutil.NewTestMongoDB(ctx, resourceName, testConfig.Namespace) + mdb, user := e2eutil.NewTestMongoDB(testCtx, resourceName, testConfig.Namespace) mdb.Spec.Security.TLS = e2eutil.NewTestTLSConfig(false) - mdb.Spec.Prometheus = e2eutil.NewPrometheusConfig(mdb.Namespace) + mdb.Spec.Prometheus = e2eutil.NewPrometheusConfig(ctx, mdb.Namespace) - _, err := setup.GeneratePasswordForUser(ctx, user, testConfig.Namespace) + _, err := setup.GeneratePasswordForUser(testCtx, user, testConfig.Namespace) if err != nil { t.Fatal(err) } - tester, err := FromResource(t, mdb) + tester, err := FromResource(ctx, t, mdb) if err != nil { t.Fatal(err) } - t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) - t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb)) + t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, testCtx)) + t.Run("Basic tests", mongodbtests.BasicFunctionality(ctx, &mdb)) mongodbtests.SkipTestIfLocal(t, "Ensure MongoDB with Prometheus configuration", func(t *testing.T) { - t.Run("Resource has TLS Mode", tester.HasTlsMode("requireSSL", 60, WithTls(mdb))) - t.Run("Test Basic Connectivity", tester.ConnectivitySucceeds(WithTls(mdb))) + t.Run("Resource has TLS Mode", tester.HasTlsMode("requireSSL", 60, WithTls(ctx, mdb))) + t.Run("Test Basic Connectivity", tester.ConnectivitySucceeds(WithTls(ctx, mdb))) t.Run("Test Prometheus endpoint is active", tester.PrometheusEndpointIsReachable("prom-user", "prom-password", false)) - t.Run("Ensure Authentication", tester.EnsureAuthenticationIsConfigured(3, WithTls(mdb))) - t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 1)) + t.Run("Ensure Authentication", tester.EnsureAuthenticationIsConfigured(3, WithTls(ctx, mdb))) + t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(ctx, &mdb, 1)) t.Run("Enabling HTTPS on the Prometheus endpoint", func(t *testing.T) { - err = e2eutil.UpdateMongoDBResource(&mdb, func(mdb *v1.MongoDBCommunity) { + err = e2eutil.UpdateMongoDBResource(ctx, &mdb, func(mdb *v1.MongoDBCommunity) { mdb.Spec.Prometheus.TLSSecretRef.Name = "tls-certificate" }) assert.NoError(t, err) - t.Run("MongoDB Reaches Running Phase", mongodbtests.MongoDBReachesRunningPhase(&mdb)) + t.Run("MongoDB Reaches Running Phase", mongodbtests.MongoDBReachesRunningPhase(ctx, &mdb)) t.Run("Test Prometheus HTTPS endpoint is active", tester.PrometheusEndpointIsReachable("prom-user", "prom-password", true)) - t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 2)) + t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(ctx, &mdb, 2)) }) }) } diff --git a/test/e2e/replica_set/replica_set_test.go b/test/e2e/replica_set/replica_set_test.go index c353a6a07..6fa62c5ff 100644 --- a/test/e2e/replica_set/replica_set_test.go +++ b/test/e2e/replica_set/replica_set_test.go @@ -1,6 +1,7 @@ package replica_set import ( + "context" "fmt" "os" "testing" @@ -25,13 +26,14 @@ func intPtr(x int) *int { return &x } func strPtr(s string) *string { return &s } func TestReplicaSet(t *testing.T) { - ctx := setup.Setup(t) - defer ctx.Teardown() + ctx := context.Background() + testCtx := setup.Setup(ctx, t) + defer testCtx.Teardown() - mdb, user := e2eutil.NewTestMongoDB(ctx, "mdb0", "") + mdb, user := e2eutil.NewTestMongoDB(testCtx, "mdb0", "") scramUser := mdb.GetAuthUsers()[0] - _, err := setup.GeneratePasswordForUser(ctx, user, "") + _, err := setup.GeneratePasswordForUser(testCtx, user, "") if err != nil { t.Fatal(err) } @@ -85,23 +87,23 @@ func TestReplicaSet(t *testing.T) { ReplicaSet: v1.OverrideReplicaSet{Settings: v1.MapWrapper{Object: settings}}, } - tester, err := FromResource(t, mdb) + tester, err := FromResource(ctx, t, mdb) if err != nil { t.Fatal(err) } - t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) - t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb)) + t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, testCtx)) + t.Run("Basic tests", mongodbtests.BasicFunctionality(ctx, &mdb)) t.Run("Keyfile authentication is configured", tester.HasKeyfileAuth(3)) - t.Run("AutomationConfig has the correct logRotateConfig", mongodbtests.AutomationConfigHasLogRotationConfig(&mdb, &lcr)) + t.Run("AutomationConfig has the correct logRotateConfig", mongodbtests.AutomationConfigHasLogRotationConfig(ctx, &mdb, &lcr)) t.Run("Test Basic Connectivity", tester.ConnectivitySucceeds()) t.Run("Test SRV Connectivity", tester.ConnectivitySucceeds(WithURI(mdb.MongoSRVURI("")), WithoutTls(), WithReplicaSet(mdb.Name))) t.Run("Test Basic Connectivity with generated connection string secret", - tester.ConnectivitySucceeds(WithURI(mongodbtests.GetConnectionStringForUser(mdb, scramUser)))) + tester.ConnectivitySucceeds(WithURI(mongodbtests.GetConnectionStringForUser(ctx, mdb, scramUser)))) t.Run("Test SRV Connectivity with generated connection string secret", - tester.ConnectivitySucceeds(WithURI(mongodbtests.GetSrvConnectionStringForUser(mdb, scramUser)))) + tester.ConnectivitySucceeds(WithURI(mongodbtests.GetSrvConnectionStringForUser(ctx, mdb, scramUser)))) t.Run("Ensure Authentication", tester.EnsureAuthenticationIsConfigured(3)) - t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 1)) - t.Run("AutomationConfig has correct member options", mongodbtests.AutomationConfigHasVoteTagPriorityConfigured(&mdb, memberOptions)) - t.Run("AutomationConfig has correct settings", mongodbtests.AutomationConfigHasSettings(&mdb, settings)) + t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(ctx, &mdb, 1)) + t.Run("AutomationConfig has correct member options", mongodbtests.AutomationConfigHasVoteTagPriorityConfigured(ctx, &mdb, memberOptions)) + t.Run("AutomationConfig has correct settings", mongodbtests.AutomationConfigHasSettings(ctx, &mdb, settings)) } diff --git a/test/e2e/replica_set_arbiter/replica_set_arbiter_test.go b/test/e2e/replica_set_arbiter/replica_set_arbiter_test.go index d8d0057c3..9d682acdc 100644 --- a/test/e2e/replica_set_arbiter/replica_set_arbiter_test.go +++ b/test/e2e/replica_set_arbiter/replica_set_arbiter_test.go @@ -1,13 +1,14 @@ package replica_set import ( + "context" "fmt" "os" "testing" e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/mongodbtests" - setup "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" + "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/util/mongotester" "github.com/stretchr/testify/assert" ) @@ -25,8 +26,9 @@ func Test(t *testing.T) { } func TestReplicaSetArbiter(t *testing.T) { - ctx := setup.Setup(t) - defer ctx.Teardown() + ctx := context.Background() + testCtx := setup.Setup(ctx, t) + defer testCtx.Teardown() type args struct { numberOfArbiters int @@ -70,41 +72,41 @@ func TestReplicaSetArbiter(t *testing.T) { for testName, _ := range tests { t.Run(testName, func(t *testing.T) { testConfig, _ := tests[testName] - mdb, user := e2eutil.NewTestMongoDB(ctx, testConfig.resourceName, "") + mdb, user := e2eutil.NewTestMongoDB(testCtx, testConfig.resourceName, "") mdb.Spec.Arbiters = testConfig.numberOfArbiters mdb.Spec.Members = testConfig.numberOfMembers // FIXME: This behavior has been changed in 6.x timeline and now the arbiter (nor the RS) can't reach the goal state. mdb.Spec.Version = "4.4.19" - pwd, err := setup.GeneratePasswordForUser(ctx, user, "") + pwd, err := setup.GeneratePasswordForUser(testCtx, user, "") if err != nil { t.Fatal(err) } - t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) + t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, testCtx)) if len(testConfig.expectedErrorMessage) > 0 { - t.Run("Check status", mongodbtests.StatefulSetMessageIsReceived(&mdb, ctx, testConfig.expectedErrorMessage)) + t.Run("Check status", mongodbtests.StatefulSetMessageIsReceived(&mdb, testCtx, testConfig.expectedErrorMessage)) } else { - t.Run("Check that the stateful set becomes ready", mongodbtests.StatefulSetBecomesReady(&mdb)) - t.Run("Check the number of arbiters", mongodbtests.AutomationConfigReplicaSetsHaveExpectedArbiters(&mdb, testConfig.numberOfArbiters)) + t.Run("Check that the stateful set becomes ready", mongodbtests.StatefulSetBecomesReady(ctx, &mdb)) + t.Run("Check the number of arbiters", mongodbtests.AutomationConfigReplicaSetsHaveExpectedArbiters(ctx, &mdb, testConfig.numberOfArbiters)) if testConfig.numberOfArbiters != testConfig.scaleArbitersTo { - t.Run(fmt.Sprintf("Scale Arbiters to %v", testConfig.scaleArbitersTo), mongodbtests.ScaleArbiters(&mdb, testConfig.scaleArbitersTo)) - t.Run("Arbiters Stateful Set Scaled Correctly", mongodbtests.ArbitersStatefulSetBecomesReady(&mdb)) + t.Run(fmt.Sprintf("Scale Arbiters to %v", testConfig.scaleArbitersTo), mongodbtests.ScaleArbiters(ctx, &mdb, testConfig.scaleArbitersTo)) + t.Run("Arbiters Stateful Set Scaled Correctly", mongodbtests.ArbitersStatefulSetBecomesReady(ctx, &mdb)) } - t.Run("MongoDB Reaches Running Phase", mongodbtests.MongoDBReachesRunningPhase(&mdb)) + t.Run("MongoDB Reaches Running Phase", mongodbtests.MongoDBReachesRunningPhase(ctx, &mdb)) t.Run("Test SRV Connectivity with generated connection string secret", func(t *testing.T) { - tester, err := mongotester.FromResource(t, mdb) + tester, err := mongotester.FromResource(ctx, t, mdb) if err != nil { t.Fatal(err) } scramUser := mdb.GetAuthUsers()[0] expectedCnxStr := fmt.Sprintf("mongodb+srv://%s-user:%s@%s-svc.%s.svc.cluster.local/admin?replicaSet=%s&ssl=false", mdb.Name, pwd, mdb.Name, mdb.Namespace, mdb.Name) - cnxStrSrv := mongodbtests.GetSrvConnectionStringForUser(mdb, scramUser) + cnxStrSrv := mongodbtests.GetSrvConnectionStringForUser(ctx, mdb, scramUser) assert.Equal(t, expectedCnxStr, cnxStrSrv) tester.ConnectivitySucceeds(mongotester.WithURI(cnxStrSrv)) }) } - t.Run("Delete MongoDB Resource", mongodbtests.DeleteMongoDBResource(&mdb, ctx)) + t.Run("Delete MongoDB Resource", mongodbtests.DeleteMongoDBResource(&mdb, testCtx)) }) } } diff --git a/test/e2e/replica_set_authentication/replica_set_authentication_test.go b/test/e2e/replica_set_authentication/replica_set_authentication_test.go index 3f20a0c93..38dbcd962 100644 --- a/test/e2e/replica_set_authentication/replica_set_authentication_test.go +++ b/test/e2e/replica_set_authentication/replica_set_authentication_test.go @@ -1,6 +1,7 @@ package replica_set_authentication import ( + "context" "fmt" "os" "testing" @@ -11,7 +12,7 @@ import ( e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/mongodbtests" - setup "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" + "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" ) func TestMain(m *testing.M) { @@ -23,22 +24,23 @@ func TestMain(m *testing.M) { } func TestReplicaSetAuthentication(t *testing.T) { - ctx := setup.Setup(t) - defer ctx.Teardown() + ctx := context.Background() + testCtx := setup.Setup(ctx, t) + defer testCtx.Teardown() - mdb, user := e2eutil.NewTestMongoDB(ctx, "mdb0", "") - pw, err := setup.GeneratePasswordForUser(ctx, user, "") + mdb, user := e2eutil.NewTestMongoDB(testCtx, "mdb0", "") + pw, err := setup.GeneratePasswordForUser(testCtx, user, "") if err != nil { t.Fatal(err) } - t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) + t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, testCtx)) // Run all the possible configuration using sha256 or sha1 - t.Run("Auth test with SHA-256", testConfigAuthentication(mdb, user, pw)) - t.Run("Auth test with SHA-256 and SHA-1", testConfigAuthentication(mdb, user, pw, withSha1())) - t.Run("Auth test with SHA-256 (using label)", testConfigAuthentication(mdb, user, pw, withLabeledSha256())) - t.Run("Auth test with SHA-256 (using label) and SHA-1", testConfigAuthentication(mdb, user, pw, withSha1(), withLabeledSha256())) - t.Run("Auth test with SHA-1", testConfigAuthentication(mdb, user, pw, withSha1(), withoutSha256())) + t.Run("Auth test with SHA-256", testConfigAuthentication(ctx, mdb, user, pw)) + t.Run("Auth test with SHA-256 and SHA-1", testConfigAuthentication(ctx, mdb, user, pw, withSha1())) + t.Run("Auth test with SHA-256 (using label)", testConfigAuthentication(ctx, mdb, user, pw, withLabeledSha256())) + t.Run("Auth test with SHA-256 (using label) and SHA-1", testConfigAuthentication(ctx, mdb, user, pw, withSha1(), withLabeledSha256())) + t.Run("Auth test with SHA-1", testConfigAuthentication(ctx, mdb, user, pw, withSha1(), withoutSha256())) } type authOptions struct { @@ -63,7 +65,7 @@ func withSha1() func(*authOptions) { } // testConfigAuthentication run the tests using the auth options to update mdb and then checks that the resources are correctly configured -func testConfigAuthentication(mdb mdbv1.MongoDBCommunity, user mdbv1.MongoDBUser, pw string, allOptions ...func(*authOptions)) func(t *testing.T) { +func testConfigAuthentication(ctx context.Context, mdb mdbv1.MongoDBCommunity, user mdbv1.MongoDBUser, pw string, allOptions ...func(*authOptions)) func(t *testing.T) { return func(t *testing.T) { pickedOpts := authOptions{ @@ -92,19 +94,19 @@ func testConfigAuthentication(mdb mdbv1.MongoDBCommunity, user mdbv1.MongoDBUser } } - err := e2eutil.UpdateMongoDBResource(&mdb, func(db *mdbv1.MongoDBCommunity) { + err := e2eutil.UpdateMongoDBResource(ctx, &mdb, func(db *mdbv1.MongoDBCommunity) { db.Spec.Security.Authentication.Modes = acceptedModes }) if err != nil { t.Fatal(err) } - tester, err := FromResource(t, mdb) + tester, err := FromResource(ctx, t, mdb) if err != nil { t.Fatal(err) } - t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb)) + t.Run("Basic tests", mongodbtests.BasicFunctionality(ctx, &mdb)) if pickedOpts.sha256 { t.Run("Test Basic Connectivity with accepted auth", tester.ConnectivitySucceeds(WithScramWithAuth(user.Name, pw, "SCRAM-SHA-256"))) } else { diff --git a/test/e2e/replica_set_change_version/replica_set_change_version_test.go b/test/e2e/replica_set_change_version/replica_set_change_version_test.go index 0016ec4e0..4d022f9d7 100644 --- a/test/e2e/replica_set_change_version/replica_set_change_version_test.go +++ b/test/e2e/replica_set_change_version/replica_set_change_version_test.go @@ -1,6 +1,7 @@ package replica_set import ( + "context" "fmt" "os" "testing" @@ -10,7 +11,7 @@ import ( e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/mongodbtests" - setup "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" + "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" appsv1 "k8s.io/api/apps/v1" ) @@ -24,50 +25,51 @@ func TestMain(m *testing.M) { } func TestReplicaSetUpgradeVersion(t *testing.T) { - ctx := setup.Setup(t) - defer ctx.Teardown() + ctx := context.Background() + testCtx := setup.Setup(ctx, t) + defer testCtx.Teardown() const initialMDBVersion = "4.4.18" const upgradedMDBVersion = "5.0.12" const upgradedWithIncreasedPatchMDBVersion = "5.0.15" - mdb, user := e2eutil.NewTestMongoDB(ctx, "mdb0", "") + mdb, user := e2eutil.NewTestMongoDB(testCtx, "mdb0", "") mdb.Spec.Version = initialMDBVersion - _, err := setup.GeneratePasswordForUser(ctx, user, "") + _, err := setup.GeneratePasswordForUser(testCtx, user, "") if err != nil { t.Fatal(err) } - tester, err := FromResource(t, mdb) + tester, err := FromResource(ctx, t, mdb) if err != nil { t.Fatal(err) } - t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) - t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb)) + t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, testCtx)) + t.Run("Basic tests", mongodbtests.BasicFunctionality(ctx, &mdb)) t.Run("Test Basic Connectivity", tester.ConnectivitySucceeds()) - t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 1)) + t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(ctx, &mdb, 1)) t.Run("Ensure Authentication", tester.EnsureAuthenticationIsConfigured(3)) // Upgrade minor version to upgradedMDBVersion t.Run("MongoDB is reachable while minor version is upgraded", func(t *testing.T) { defer tester.StartBackgroundConnectivityTest(t, time.Second*10)() - t.Run("Test Minor Version can be upgraded", mongodbtests.ChangeVersion(&mdb, upgradedMDBVersion)) - t.Run("StatefulSet has OnDelete update strategy", mongodbtests.StatefulSetHasUpdateStrategy(&mdb, appsv1.OnDeleteStatefulSetStrategyType)) - t.Run("Stateful Set Reaches Ready State, after Upgrading", mongodbtests.StatefulSetBecomesReady(&mdb)) - t.Run("AutomationConfig's version has been increased", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 2)) + t.Run("Test Minor Version can be upgraded", mongodbtests.ChangeVersion(ctx, &mdb, upgradedMDBVersion)) + t.Run("StatefulSet has OnDelete update strategy", mongodbtests.StatefulSetHasUpdateStrategy(ctx, &mdb, appsv1.OnDeleteStatefulSetStrategyType)) + t.Run("Stateful Set Reaches Ready State, after Upgrading", mongodbtests.StatefulSetBecomesReady(ctx, &mdb)) + t.Run("AutomationConfig's version has been increased", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(ctx, &mdb, 2)) }) - t.Run("StatefulSet has RollingUpgrade restart strategy", mongodbtests.StatefulSetHasUpdateStrategy(&mdb, appsv1.RollingUpdateStatefulSetStrategyType)) + t.Run("StatefulSet has RollingUpgrade restart strategy", mongodbtests.StatefulSetHasUpdateStrategy(ctx, &mdb, appsv1.RollingUpdateStatefulSetStrategyType)) // Upgrade patch version to upgradedWithIncreasedPatchMDBVersion t.Run("MongoDB is reachable while patch version is upgraded", func(t *testing.T) { defer tester.StartBackgroundConnectivityTest(t, time.Second*10)() - t.Run("Test Patch Version can be upgraded", mongodbtests.ChangeVersion(&mdb, upgradedWithIncreasedPatchMDBVersion)) - t.Run("StatefulSet has OnDelete restart strategy", mongodbtests.StatefulSetHasUpdateStrategy(&mdb, appsv1.OnDeleteStatefulSetStrategyType)) - t.Run("Stateful Set Reaches Ready State, after upgrading", mongodbtests.StatefulSetBecomesReady(&mdb)) - t.Run("AutomationConfig's version has been increased", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 3)) + t.Run("Test Patch Version can be upgraded", mongodbtests.ChangeVersion(ctx, &mdb, upgradedWithIncreasedPatchMDBVersion)) + t.Run("StatefulSet has OnDelete restart strategy", mongodbtests.StatefulSetHasUpdateStrategy(ctx, &mdb, appsv1.OnDeleteStatefulSetStrategyType)) + t.Run("Stateful Set Reaches Ready State, after upgrading", mongodbtests.StatefulSetBecomesReady(ctx, &mdb)) + t.Run("AutomationConfig's version has been increased", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(ctx, &mdb, 3)) }) - t.Run("StatefulSet has RollingUpgrade restart strategy", mongodbtests.StatefulSetHasUpdateStrategy(&mdb, appsv1.RollingUpdateStatefulSetStrategyType)) + t.Run("StatefulSet has RollingUpgrade restart strategy", mongodbtests.StatefulSetHasUpdateStrategy(ctx, &mdb, appsv1.RollingUpdateStatefulSetStrategyType)) } diff --git a/test/e2e/replica_set_connection_string_options/replica_set_connection_string_options_test.go b/test/e2e/replica_set_connection_string_options/replica_set_connection_string_options_test.go index 878509f81..6358f9d3a 100644 --- a/test/e2e/replica_set_connection_string_options/replica_set_connection_string_options_test.go +++ b/test/e2e/replica_set_connection_string_options/replica_set_connection_string_options_test.go @@ -1,6 +1,7 @@ package replica_set_connection_string_options import ( + "context" "fmt" "os" "testing" @@ -9,7 +10,7 @@ import ( e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/mongodbtests" - setup "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" + "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" ) func TestMain(m *testing.M) { @@ -21,89 +22,90 @@ func TestMain(m *testing.M) { } func TestReplicaSetWithConnectionString(t *testing.T) { - ctx := setup.Setup(t) - defer ctx.Teardown() + ctx := context.Background() + testCtx := setup.Setup(ctx, t) + defer testCtx.Teardown() - mdb, user := e2eutil.NewTestMongoDB(ctx, "mdb0", "") + mdb, user := e2eutil.NewTestMongoDB(testCtx, "mdb0", "") scramUser := mdb.GetAuthUsers()[0] - _, err := setup.GeneratePasswordForUser(ctx, user, "") + _, err := setup.GeneratePasswordForUser(testCtx, user, "") if err != nil { t.Fatal(err) } - tester, err := FromResource(t, mdb) + tester, err := FromResource(ctx, t, mdb) if err != nil { t.Fatal(err) } - t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) - t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb)) + t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, testCtx)) + t.Run("Basic tests", mongodbtests.BasicFunctionality(ctx, &mdb)) t.Run("Keyfile authentication is configured", tester.HasKeyfileAuth(3)) t.Run("Ensure Authentication", tester.EnsureAuthenticationIsConfigured(3)) - t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 1)) + t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(ctx, &mdb, 1)) /** User options only. */ t.Run("Connection String With User Options Only", func(t *testing.T) { - t.Run("Test Add New Connection String Option to User", mongodbtests.AddConnectionStringOptionToUser(&mdb, "readPreference", "primary")) - t.Run("Test Secrets Are Updated", mongodbtests.MongoDBReachesRunningPhase(&mdb)) + t.Run("Test Add New Connection String Option to User", mongodbtests.AddConnectionStringOptionToUser(ctx, &mdb, "readPreference", "primary")) + t.Run("Test Secrets Are Updated", mongodbtests.MongoDBReachesRunningPhase(ctx, &mdb)) scramUser = mdb.GetAuthUsers()[0] t.Run("Test Basic Connectivity With User Options", tester.ConnectivitySucceeds()) t.Run("Test SRV Connectivity With User Options", tester.ConnectivitySucceeds(WithURI(mdb.MongoSRVURI("")), WithoutTls(), WithReplicaSet(mdb.Name))) t.Run("Test Basic Connectivity with generated connection string secret with user options", - tester.ConnectivitySucceeds(WithURI(mongodbtests.GetConnectionStringForUser(mdb, scramUser)))) + tester.ConnectivitySucceeds(WithURI(mongodbtests.GetConnectionStringForUser(ctx, mdb, scramUser)))) t.Run("Test SRV Connectivity with generated connection string secret with user options", - tester.ConnectivitySucceeds(WithURI(mongodbtests.GetSrvConnectionStringForUser(mdb, scramUser)))) + tester.ConnectivitySucceeds(WithURI(mongodbtests.GetSrvConnectionStringForUser(ctx, mdb, scramUser)))) }) /** General options only. */ t.Run("Connection String With General Options Only", func(t *testing.T) { - t.Run("Resetting Connection String Options", mongodbtests.ResetConnectionStringOptions(&mdb)) - t.Run("Test Add New Connection String Option to Resource", mongodbtests.AddConnectionStringOption(&mdb, "readPreference", "primary")) - t.Run("Test Secrets Are Updated", mongodbtests.MongoDBReachesRunningPhase(&mdb)) + t.Run("Resetting Connection String Options", mongodbtests.ResetConnectionStringOptions(ctx, &mdb)) + t.Run("Test Add New Connection String Option to Resource", mongodbtests.AddConnectionStringOption(ctx, &mdb, "readPreference", "primary")) + t.Run("Test Secrets Are Updated", mongodbtests.MongoDBReachesRunningPhase(ctx, &mdb)) scramUser = mdb.GetAuthUsers()[0] t.Run("Test Basic Connectivity With Resource Options", tester.ConnectivitySucceeds()) t.Run("Test SRV Connectivity With Resource Options", tester.ConnectivitySucceeds(WithURI(mdb.MongoSRVURI("")), WithoutTls(), WithReplicaSet(mdb.Name))) t.Run("Test Basic Connectivity with generated connection string secret with resource options", - tester.ConnectivitySucceeds(WithURI(mongodbtests.GetConnectionStringForUser(mdb, scramUser)))) + tester.ConnectivitySucceeds(WithURI(mongodbtests.GetConnectionStringForUser(ctx, mdb, scramUser)))) t.Run("Test SRV Connectivity with generated connection string secret with resource options", - tester.ConnectivitySucceeds(WithURI(mongodbtests.GetSrvConnectionStringForUser(mdb, scramUser)))) + tester.ConnectivitySucceeds(WithURI(mongodbtests.GetSrvConnectionStringForUser(ctx, mdb, scramUser)))) }) /** Overwritten options. */ t.Run("Connection String With Overwritten Options", func(t *testing.T) { - t.Run("Test Add New Connection String Option to Resource", mongodbtests.AddConnectionStringOption(&mdb, "readPreference", "primary")) - t.Run("Test Add New Connection String Option to User", mongodbtests.AddConnectionStringOptionToUser(&mdb, "readPreference", "secondary")) - t.Run("Test Secrets Are Updated", mongodbtests.MongoDBReachesRunningPhase(&mdb)) + t.Run("Test Add New Connection String Option to Resource", mongodbtests.AddConnectionStringOption(ctx, &mdb, "readPreference", "primary")) + t.Run("Test Add New Connection String Option to User", mongodbtests.AddConnectionStringOptionToUser(ctx, &mdb, "readPreference", "secondary")) + t.Run("Test Secrets Are Updated", mongodbtests.MongoDBReachesRunningPhase(ctx, &mdb)) scramUser = mdb.GetAuthUsers()[0] t.Run("Test Basic Connectivity With Overwritten Options", tester.ConnectivitySucceeds()) t.Run("Test SRV Connectivity With Overwritten Options", tester.ConnectivitySucceeds(WithURI(mdb.MongoSRVURI("")), WithoutTls(), WithReplicaSet(mdb.Name))) t.Run("Test Basic Connectivity with generated connection string secret with overwritten options", - tester.ConnectivitySucceeds(WithURI(mongodbtests.GetConnectionStringForUser(mdb, scramUser)))) + tester.ConnectivitySucceeds(WithURI(mongodbtests.GetConnectionStringForUser(ctx, mdb, scramUser)))) t.Run("Test SRV Connectivity with generated connection string secret with overwritten options", - tester.ConnectivitySucceeds(WithURI(mongodbtests.GetSrvConnectionStringForUser(mdb, scramUser)))) + tester.ConnectivitySucceeds(WithURI(mongodbtests.GetSrvConnectionStringForUser(ctx, mdb, scramUser)))) }) /** Wrong options. */ t.Run("Connection String With Wrong Options", func(t *testing.T) { - t.Run("Resetting Connection String Options", mongodbtests.ResetConnectionStringOptions(&mdb)) - t.Run("Test Add New Connection String Option to Resource", mongodbtests.AddConnectionStringOption(&mdb, "readPreference", "wrong")) - t.Run("Test Secrets Are Updated", mongodbtests.MongoDBReachesRunningPhase(&mdb)) + t.Run("Resetting Connection String Options", mongodbtests.ResetConnectionStringOptions(ctx, &mdb)) + t.Run("Test Add New Connection String Option to Resource", mongodbtests.AddConnectionStringOption(ctx, &mdb, "readPreference", "wrong")) + t.Run("Test Secrets Are Updated", mongodbtests.MongoDBReachesRunningPhase(ctx, &mdb)) scramUser = mdb.GetAuthUsers()[0] - t.Run("Test Basic Connectivity", tester.ConnectivityRejected(WithURI(mdb.MongoURI("")), WithoutTls(), WithReplicaSet(mdb.Name))) - t.Run("Test SRV Connectivity", tester.ConnectivityRejected(WithURI(mdb.MongoSRVURI("")), WithoutTls(), WithReplicaSet(mdb.Name))) + t.Run("Test Basic Connectivity", tester.ConnectivityRejected(ctx, WithURI(mdb.MongoURI("")), WithoutTls(), WithReplicaSet(mdb.Name))) + t.Run("Test SRV Connectivity", tester.ConnectivityRejected(ctx, WithURI(mdb.MongoSRVURI("")), WithoutTls(), WithReplicaSet(mdb.Name))) t.Run("Test Basic Connectivity with generated connection string secret", - tester.ConnectivityRejected(WithURI(mongodbtests.GetConnectionStringForUser(mdb, scramUser)))) + tester.ConnectivityRejected(ctx, WithURI(mongodbtests.GetConnectionStringForUser(ctx, mdb, scramUser)))) t.Run("Test SRV Connectivity with generated connection string secret", - tester.ConnectivityRejected(WithURI(mongodbtests.GetSrvConnectionStringForUser(mdb, scramUser)))) + tester.ConnectivityRejected(ctx, WithURI(mongodbtests.GetSrvConnectionStringForUser(ctx, mdb, scramUser)))) }) } diff --git a/test/e2e/replica_set_cross_namespace_deploy/replica_set_cross_namespace_deploy_test.go b/test/e2e/replica_set_cross_namespace_deploy/replica_set_cross_namespace_deploy_test.go index afd961ee1..2bd41ed86 100644 --- a/test/e2e/replica_set_cross_namespace_deploy/replica_set_cross_namespace_deploy_test.go +++ b/test/e2e/replica_set_cross_namespace_deploy/replica_set_cross_namespace_deploy_test.go @@ -15,7 +15,7 @@ import ( e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/mongodbtests" - setup "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" + "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" ) func TestMain(m *testing.M) { @@ -27,8 +27,9 @@ func TestMain(m *testing.M) { } func TestCrossNamespaceDeploy(t *testing.T) { - ctx := setup.Setup(t) - defer ctx.Teardown() + ctx := context.Background() + testCtx := setup.Setup(ctx, t) + defer testCtx.Teardown() postfix, err := generate.RandomValidDNS1123Label(5) if err != nil { @@ -36,40 +37,40 @@ func TestCrossNamespaceDeploy(t *testing.T) { } namespace := fmt.Sprintf("clusterwide-test-%s", postfix) - err = e2eutil.EnsureNamespace(ctx, namespace) + err = e2eutil.EnsureNamespace(testCtx, namespace) if err != nil { t.Fatal(err) } - if err := createDatabaseServiceAccountRoleAndRoleBinding(t, namespace); err != nil { + if err := createDatabaseServiceAccountRoleAndRoleBinding(ctx, t, namespace); err != nil { t.Fatal(err) } - mdb, user := e2eutil.NewTestMongoDB(ctx, "mdb0", namespace) + mdb, user := e2eutil.NewTestMongoDB(testCtx, "mdb0", namespace) - _, err = setup.GeneratePasswordForUser(ctx, user, namespace) + _, err = setup.GeneratePasswordForUser(testCtx, user, namespace) if err != nil { t.Fatal(err) } - tester, err := FromResource(t, mdb) + tester, err := FromResource(ctx, t, mdb) if err != nil { t.Fatal(err) } - t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) - t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb)) + t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, testCtx)) + t.Run("Basic tests", mongodbtests.BasicFunctionality(ctx, &mdb)) t.Run("Keyfile authentication is configured", tester.HasKeyfileAuth(3)) t.Run("Test Basic Connectivity", tester.ConnectivitySucceeds()) t.Run("Ensure Authentication", tester.EnsureAuthenticationIsConfigured(3)) - t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 1)) + t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(ctx, &mdb, 1)) } // createDatabaseServiceAccountRoleAndRoleBinding creates the ServiceAccount, Role and RoleBinding required // for the database StatefulSet in the other namespace. -func createDatabaseServiceAccountRoleAndRoleBinding(t *testing.T, namespace string) error { +func createDatabaseServiceAccountRoleAndRoleBinding(ctx context.Context, t *testing.T, namespace string) error { sa := corev1.ServiceAccount{} - err := e2eutil.TestClient.Get(context.TODO(), types.NamespacedName{Name: "mongodb-database", Namespace: e2eutil.OperatorNamespace}, &sa) + err := e2eutil.TestClient.Get(ctx, types.NamespacedName{Name: "mongodb-database", Namespace: e2eutil.OperatorNamespace}, &sa) if err != nil { t.Fatal(err) } @@ -77,13 +78,13 @@ func createDatabaseServiceAccountRoleAndRoleBinding(t *testing.T, namespace stri sa.Namespace = namespace sa.ObjectMeta.ResourceVersion = "" - err = e2eutil.TestClient.Create(context.TODO(), &sa, &e2eutil.CleanupOptions{}) + err = e2eutil.TestClient.Create(ctx, &sa, &e2eutil.CleanupOptions{}) if err != nil { t.Fatal(err) } role := rbacv1.Role{} - err = e2eutil.TestClient.Get(context.TODO(), types.NamespacedName{Name: "mongodb-database", Namespace: e2eutil.OperatorNamespace}, &role) + err = e2eutil.TestClient.Get(ctx, types.NamespacedName{Name: "mongodb-database", Namespace: e2eutil.OperatorNamespace}, &role) if err != nil { t.Fatal(err) } @@ -91,13 +92,13 @@ func createDatabaseServiceAccountRoleAndRoleBinding(t *testing.T, namespace stri role.Namespace = namespace role.ObjectMeta.ResourceVersion = "" - err = e2eutil.TestClient.Create(context.TODO(), &role, &e2eutil.CleanupOptions{}) + err = e2eutil.TestClient.Create(ctx, &role, &e2eutil.CleanupOptions{}) if err != nil { t.Fatal(err) } rolebinding := rbacv1.RoleBinding{} - err = e2eutil.TestClient.Get(context.TODO(), types.NamespacedName{Name: "mongodb-database", Namespace: e2eutil.OperatorNamespace}, &rolebinding) + err = e2eutil.TestClient.Get(ctx, types.NamespacedName{Name: "mongodb-database", Namespace: e2eutil.OperatorNamespace}, &rolebinding) if err != nil { t.Fatal(err) } @@ -105,7 +106,7 @@ func createDatabaseServiceAccountRoleAndRoleBinding(t *testing.T, namespace stri rolebinding.Namespace = namespace rolebinding.ObjectMeta.ResourceVersion = "" - err = e2eutil.TestClient.Create(context.TODO(), &rolebinding, &e2eutil.CleanupOptions{}) + err = e2eutil.TestClient.Create(ctx, &rolebinding, &e2eutil.CleanupOptions{}) if err != nil { t.Fatal(err) } diff --git a/test/e2e/replica_set_custom_annotations_test/replica_set_custom_annotations_test.go b/test/e2e/replica_set_custom_annotations_test/replica_set_custom_annotations_test.go index 5f5f5dc84..d92d5db1b 100644 --- a/test/e2e/replica_set_custom_annotations_test/replica_set_custom_annotations_test.go +++ b/test/e2e/replica_set_custom_annotations_test/replica_set_custom_annotations_test.go @@ -1,6 +1,7 @@ package replica_set_custom_annotations_test import ( + "context" "fmt" v1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" @@ -22,10 +23,11 @@ func TestMain(m *testing.M) { } func TestReplicaSetCustomAnnotations(t *testing.T) { - ctx := setup.Setup(t) - defer ctx.Teardown() + ctx := context.Background() + testCtx := setup.Setup(ctx, t) + defer testCtx.Teardown() - mdb, user := e2eutil.NewTestMongoDB(ctx, "mdb0", "") + mdb, user := e2eutil.NewTestMongoDB(testCtx, "mdb0", "") mdb.Spec.StatefulSetConfiguration.SpecWrapper.Spec.Template.ObjectMeta = metav1.ObjectMeta{ Labels: e2eutil.TestLabels(), Annotations: e2eutil.TestAnnotations(), @@ -52,26 +54,26 @@ func TestReplicaSetCustomAnnotations(t *testing.T) { } scramUser := mdb.GetAuthUsers()[0] - _, err := setup.GeneratePasswordForUser(ctx, user, "") + _, err := setup.GeneratePasswordForUser(testCtx, user, "") if err != nil { t.Fatal(err) } - tester, err := FromResource(t, mdb) + tester, err := FromResource(ctx, t, mdb) if err != nil { t.Fatal(err) } - t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) - t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb)) + t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, testCtx)) + t.Run("Basic tests", mongodbtests.BasicFunctionality(ctx, &mdb)) t.Run("Keyfile authentication is configured", tester.HasKeyfileAuth(3)) t.Run("Test Basic Connectivity", tester.ConnectivitySucceeds()) t.Run("Test SRV Connectivity", tester.ConnectivitySucceeds(WithURI(mdb.MongoSRVURI("")), WithoutTls(), WithReplicaSet((mdb.Name)))) t.Run("Test Basic Connectivity with generated connection string secret", - tester.ConnectivitySucceeds(WithURI(mongodbtests.GetConnectionStringForUser(mdb, scramUser)))) + tester.ConnectivitySucceeds(WithURI(mongodbtests.GetConnectionStringForUser(ctx, mdb, scramUser)))) t.Run("Test SRV Connectivity with generated connection string secret", - tester.ConnectivitySucceeds(WithURI(mongodbtests.GetSrvConnectionStringForUser(mdb, scramUser)))) + tester.ConnectivitySucceeds(WithURI(mongodbtests.GetSrvConnectionStringForUser(ctx, mdb, scramUser)))) t.Run("Ensure Authentication", tester.EnsureAuthenticationIsConfigured(3)) - t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 1)) - t.Run("Cluster has the expected labels and annotations", mongodbtests.HasExpectedMetadata(&mdb, e2eutil.TestLabels(), e2eutil.TestAnnotations())) + t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(ctx, &mdb, 1)) + t.Run("Cluster has the expected labels and annotations", mongodbtests.HasExpectedMetadata(ctx, &mdb, e2eutil.TestLabels(), e2eutil.TestAnnotations())) } diff --git a/test/e2e/replica_set_custom_persistent_volume/replica_set_custom_persistent_volume_test.go b/test/e2e/replica_set_custom_persistent_volume/replica_set_custom_persistent_volume_test.go index a814dd29f..752486361 100644 --- a/test/e2e/replica_set_custom_persistent_volume/replica_set_custom_persistent_volume_test.go +++ b/test/e2e/replica_set_custom_persistent_volume/replica_set_custom_persistent_volume_test.go @@ -12,7 +12,7 @@ import ( "github.com/stretchr/testify/assert" e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" - setup "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" + "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -65,7 +65,7 @@ func getPersistentVolumeLocal(name string, localPath string, label string) corev // getVolumes returns two persistentVolumes for each of the `members` pod. // one volume will be for the `data` claim and the other will be for the `logs` claim -func getVolumes(ctx *e2eutil.Context, volumeType string, members int) []corev1.PersistentVolume { +func getVolumes(ctx *e2eutil.TestContext, volumeType string, members int) []corev1.PersistentVolume { volumes := make([]corev1.PersistentVolume, members) for i := 0; i < members; i++ { volumes[i] = getPersistentVolumeLocal( @@ -103,43 +103,44 @@ func getPvc(pvcType string, mdb v1.MongoDBCommunity) corev1.PersistentVolumeClai } func TestReplicaSetCustomPersistentVolumes(t *testing.T) { - ctx := setup.Setup(t) - defer ctx.Teardown() + ctx := context.Background() + testCtx := setup.Setup(ctx, t) + defer testCtx.Teardown() - mdb, user := e2eutil.NewTestMongoDB(ctx, "mdb0", "") + mdb, user := e2eutil.NewTestMongoDB(testCtx, "mdb0", "") mdb.Spec.StatefulSetConfiguration.SpecWrapper.Spec.VolumeClaimTemplates = []corev1.PersistentVolumeClaim{ getPvc("data", mdb), getPvc("logs", mdb), } - volumesToCreate := getVolumes(ctx, "data", mdb.Spec.Members) - volumesToCreate = append(volumesToCreate, getVolumes(ctx, "logs", mdb.Spec.Members)...) + volumesToCreate := getVolumes(testCtx, "data", mdb.Spec.Members) + volumesToCreate = append(volumesToCreate, getVolumes(testCtx, "logs", mdb.Spec.Members)...) for i := range volumesToCreate { - err := e2eutil.TestClient.Create(context.TODO(), &volumesToCreate[i], &e2eutil.CleanupOptions{TestContext: ctx}) + err := e2eutil.TestClient.Create(ctx, &volumesToCreate[i], &e2eutil.CleanupOptions{TestContext: testCtx}) assert.NoError(t, err) } scramUser := mdb.GetAuthUsers()[0] - _, err := setup.GeneratePasswordForUser(ctx, user, "") + _, err := setup.GeneratePasswordForUser(testCtx, user, "") if err != nil { t.Fatal(err) } - tester, err := FromResource(t, mdb) + tester, err := FromResource(ctx, t, mdb) if err != nil { t.Fatal(err) } - t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) - t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb)) + t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, testCtx)) + t.Run("Basic tests", mongodbtests.BasicFunctionality(ctx, &mdb)) t.Run("Keyfile authentication is configured", tester.HasKeyfileAuth(3)) t.Run("Test Basic Connectivity", tester.ConnectivitySucceeds()) t.Run("Test SRV Connectivity", tester.ConnectivitySucceeds(WithURI(mdb.MongoSRVURI("")), WithoutTls(), WithReplicaSet((mdb.Name)))) t.Run("Test Basic Connectivity with generated connection string secret", - tester.ConnectivitySucceeds(WithURI(mongodbtests.GetConnectionStringForUser(mdb, scramUser)))) + tester.ConnectivitySucceeds(WithURI(mongodbtests.GetConnectionStringForUser(ctx, mdb, scramUser)))) t.Run("Test SRV Connectivity with generated connection string secret", - tester.ConnectivitySucceeds(WithURI(mongodbtests.GetSrvConnectionStringForUser(mdb, scramUser)))) + tester.ConnectivitySucceeds(WithURI(mongodbtests.GetSrvConnectionStringForUser(ctx, mdb, scramUser)))) t.Run("Ensure Authentication", tester.EnsureAuthenticationIsConfigured(3)) - t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 1)) - t.Run("Cluster has the expected persistent volumes", mongodbtests.HasExpectedPersistentVolumes(volumesToCreate)) + t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(ctx, &mdb, 1)) + t.Run("Cluster has the expected persistent volumes", mongodbtests.HasExpectedPersistentVolumes(ctx, volumesToCreate)) } diff --git a/test/e2e/replica_set_custom_role/replica_set_custom_role_test.go b/test/e2e/replica_set_custom_role/replica_set_custom_role_test.go index db120ba4c..0b019d299 100644 --- a/test/e2e/replica_set_custom_role/replica_set_custom_role_test.go +++ b/test/e2e/replica_set_custom_role/replica_set_custom_role_test.go @@ -1,6 +1,7 @@ package replica_set_custom_role import ( + "context" "fmt" "os" "testing" @@ -10,7 +11,7 @@ import ( e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/mongodbtests" - setup "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" + "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" ) func TestMain(m *testing.M) { @@ -22,15 +23,16 @@ func TestMain(m *testing.M) { } func TestReplicaSetCustomRole(t *testing.T) { - ctx := setup.Setup(t) - defer ctx.Teardown() + ctx := context.Background() + testCtx := setup.Setup(ctx, t) + defer testCtx.Teardown() someDB := "test" someCollection := "foo" anyDB := "" anyCollection := "" - mdb, user := e2eutil.NewTestMongoDB(ctx, "mdb0", "") + mdb, user := e2eutil.NewTestMongoDB(testCtx, "mdb0", "") mdb.Spec.Security.Roles = []mdbv1.CustomRole{ { Role: "testRole", @@ -71,26 +73,26 @@ func TestReplicaSetCustomRole(t *testing.T) { }, } - _, err := setup.GeneratePasswordForUser(ctx, user, "") + _, err := setup.GeneratePasswordForUser(testCtx, user, "") if err != nil { t.Fatal(err) } - tester, err := FromResource(t, mdb) + tester, err := FromResource(ctx, t, mdb) if err != nil { t.Fatal(err) } - t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) - t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb)) + t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, testCtx)) + t.Run("Basic tests", mongodbtests.BasicFunctionality(ctx, &mdb)) t.Run("Keyfile authentication is configured", tester.HasKeyfileAuth(3)) t.Run("Test Basic Connectivity", tester.ConnectivitySucceeds()) t.Run("Ensure Authentication", tester.EnsureAuthenticationIsConfigured(3)) - t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 1)) + t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(ctx, &mdb, 1)) // Verify automation config roles and roles created in admin database. roles := mdbv1.ConvertCustomRolesToAutomationConfigCustomRole(mdb.Spec.Security.Roles) - t.Run("AutomationConfig has the correct custom role", mongodbtests.AutomationConfigHasTheExpectedCustomRoles(&mdb, roles)) + t.Run("AutomationConfig has the correct custom role", mongodbtests.AutomationConfigHasTheExpectedCustomRoles(ctx, &mdb, roles)) t.Run("Custom Role was created ", tester.VerifyRoles(roles, 1)) } diff --git a/test/e2e/replica_set_enterprise_upgrade/replica_set_enterprise_upgrade.go b/test/e2e/replica_set_enterprise_upgrade/replica_set_enterprise_upgrade.go index b3c6602e1..d8c0bfb23 100644 --- a/test/e2e/replica_set_enterprise_upgrade/replica_set_enterprise_upgrade.go +++ b/test/e2e/replica_set_enterprise_upgrade/replica_set_enterprise_upgrade.go @@ -1,6 +1,7 @@ package replica_set_enterprise_upgrade import ( + "context" "fmt" "testing" "time" @@ -12,37 +13,37 @@ import ( "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/util/mongotester" ) -func DeployEnterpriseAndUpgradeTest(t *testing.T, versionsToBeTested []string) { +func DeployEnterpriseAndUpgradeTest(ctx context.Context, t *testing.T, versionsToBeTested []string) { t.Setenv(construct.MongodbRepoUrl, "docker.io/mongodb") t.Setenv(construct.MongodbImageEnv, "mongodb-enterprise-server") - ctx := setup.Setup(t) - defer ctx.Teardown() + testCtx := setup.Setup(ctx, t) + defer testCtx.Teardown() - mdb, user := e2eutil.NewTestMongoDB(ctx, "mdb0", "") + mdb, user := e2eutil.NewTestMongoDB(testCtx, "mdb0", "") mdb.Spec.Version = versionsToBeTested[0] - _, err := setup.GeneratePasswordForUser(ctx, user, "") + _, err := setup.GeneratePasswordForUser(testCtx, user, "") if err != nil { t.Fatal(err) } - tester, err := mongotester.FromResource(t, mdb) + tester, err := mongotester.FromResource(ctx, t, mdb) if err != nil { t.Fatal(err) } - t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) - t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb)) + t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, testCtx)) + t.Run("Basic tests", mongodbtests.BasicFunctionality(ctx, &mdb)) t.Run("Test Basic Connectivity", tester.ConnectivitySucceeds()) - t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 1)) + t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(ctx, &mdb, 1)) for i := 1; i < len(versionsToBeTested); i++ { t.Run(fmt.Sprintf("Testing upgrade from %s to %s", versionsToBeTested[i-1], versionsToBeTested[i]), func(t *testing.T) { defer tester.StartBackgroundConnectivityTest(t, time.Second*10)() - t.Run(fmt.Sprintf("Upgrading to %s", versionsToBeTested[i]), mongodbtests.ChangeVersion(&mdb, versionsToBeTested[i])) - t.Run("Stateful Set Reaches Ready State, after Upgrading", mongodbtests.StatefulSetBecomesReady(&mdb)) - t.Run("MongoDB Reaches Running Phase", mongodbtests.MongoDBReachesRunningPhase(&mdb)) - t.Run("AutomationConfig's version has been increased", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, i+1)) + t.Run(fmt.Sprintf("Upgrading to %s", versionsToBeTested[i]), mongodbtests.ChangeVersion(ctx, &mdb, versionsToBeTested[i])) + t.Run("Stateful Set Reaches Ready State, after Upgrading", mongodbtests.StatefulSetBecomesReady(ctx, &mdb)) + t.Run("MongoDB Reaches Running Phase", mongodbtests.MongoDBReachesRunningPhase(ctx, &mdb)) + t.Run("AutomationConfig's version has been increased", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(ctx, &mdb, i+1)) }) } } diff --git a/test/e2e/replica_set_enterprise_upgrade_4_5/replica_set_enterprise_upgrade_4_5_test.go b/test/e2e/replica_set_enterprise_upgrade_4_5/replica_set_enterprise_upgrade_4_5_test.go index ff7b19dc7..298829059 100644 --- a/test/e2e/replica_set_enterprise_upgrade_4_5/replica_set_enterprise_upgrade_4_5_test.go +++ b/test/e2e/replica_set_enterprise_upgrade_4_5/replica_set_enterprise_upgrade_4_5_test.go @@ -1,6 +1,7 @@ package replica_set import ( + "context" "fmt" "os" "testing" @@ -23,5 +24,6 @@ func TestMain(m *testing.M) { } func TestReplicaSet(t *testing.T) { - replica_set_enterprise_upgrade.DeployEnterpriseAndUpgradeTest(t, versionsForUpgrades) + ctx := context.Background() + replica_set_enterprise_upgrade.DeployEnterpriseAndUpgradeTest(ctx, t, versionsForUpgrades) } diff --git a/test/e2e/replica_set_enterprise_upgrade_5_6/replica_set_enterprise_upgrade_5_6_test.go b/test/e2e/replica_set_enterprise_upgrade_5_6/replica_set_enterprise_upgrade_5_6_test.go index 5c3ecf509..0e0eedef5 100644 --- a/test/e2e/replica_set_enterprise_upgrade_5_6/replica_set_enterprise_upgrade_5_6_test.go +++ b/test/e2e/replica_set_enterprise_upgrade_5_6/replica_set_enterprise_upgrade_5_6_test.go @@ -1,6 +1,7 @@ package replica_set import ( + "context" "fmt" "os" "testing" @@ -22,5 +23,6 @@ func TestMain(m *testing.M) { } func TestReplicaSet(t *testing.T) { - replica_set_enterprise_upgrade.DeployEnterpriseAndUpgradeTest(t, versionsForUpgrades) + ctx := context.Background() + replica_set_enterprise_upgrade.DeployEnterpriseAndUpgradeTest(ctx, t, versionsForUpgrades) } diff --git a/test/e2e/replica_set_enterprise_upgrade_6_7/replica_set_enterprise_upgrade_5_6_test.go b/test/e2e/replica_set_enterprise_upgrade_6_7/replica_set_enterprise_upgrade_5_6_test.go index e9fdb3f29..c447ca6c6 100644 --- a/test/e2e/replica_set_enterprise_upgrade_6_7/replica_set_enterprise_upgrade_5_6_test.go +++ b/test/e2e/replica_set_enterprise_upgrade_6_7/replica_set_enterprise_upgrade_5_6_test.go @@ -1,6 +1,7 @@ package replica_set import ( + "context" "fmt" "os" "testing" @@ -22,5 +23,6 @@ func TestMain(m *testing.M) { } func TestReplicaSet(t *testing.T) { - replica_set_enterprise_upgrade.DeployEnterpriseAndUpgradeTest(t, versionsForUpgrades) + ctx := context.Background() + replica_set_enterprise_upgrade.DeployEnterpriseAndUpgradeTest(ctx, t, versionsForUpgrades) } diff --git a/test/e2e/replica_set_mongod_config/replica_set_mongod_config_test.go b/test/e2e/replica_set_mongod_config/replica_set_mongod_config_test.go index 6954e85a4..1a009c812 100644 --- a/test/e2e/replica_set_mongod_config/replica_set_mongod_config_test.go +++ b/test/e2e/replica_set_mongod_config/replica_set_mongod_config_test.go @@ -1,6 +1,7 @@ package replica_set_mongod_config import ( + "context" "fmt" "os" "testing" @@ -22,12 +23,13 @@ func TestMain(m *testing.M) { } func TestReplicaSet(t *testing.T) { - ctx := setup.Setup(t) - defer ctx.Teardown() + ctx := context.Background() + testCtx := setup.Setup(ctx, t) + defer testCtx.Teardown() - mdb, user := e2eutil.NewTestMongoDB(ctx, "mdb0", "") + mdb, user := e2eutil.NewTestMongoDB(testCtx, "mdb0", "") - _, err := setup.GeneratePasswordForUser(ctx, user, "") + _, err := setup.GeneratePasswordForUser(testCtx, user, "") if err != nil { t.Fatal(err) } @@ -53,19 +55,19 @@ func TestReplicaSet(t *testing.T) { mdb.Spec.AdditionalMongodConfig.Object = mongodConfig - tester, err := FromResource(t, mdb) + tester, err := FromResource(ctx, t, mdb) if err != nil { t.Fatal(err) } - t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) - t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb)) + t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, testCtx)) + t.Run("Basic tests", mongodbtests.BasicFunctionality(ctx, &mdb)) t.Run("Ensure Authentication", tester.EnsureAuthenticationIsConfigured(3)) t.Run("Test Basic Connectivity", tester.ConnectivitySucceeds()) - t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 1)) + t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(ctx, &mdb, 1)) for i := range settings { t.Run(fmt.Sprintf("Mongod setting %s has been set", settings[i]), tester.EnsureMongodConfig(settings[i], values[i])) } t.Run("Mongod setting net.port has been set", tester.EnsureMongodConfig("net.port", int32(40333))) - t.Run("Service has the correct port", mongodbtests.ServiceUsesCorrectPort(&mdb, 40333)) + t.Run("Service has the correct port", mongodbtests.ServiceUsesCorrectPort(ctx, &mdb, 40333)) } diff --git a/test/e2e/replica_set_mongod_port_change_with_arbiters/replica_set_mongod_port_change_with_arbiters_test.go b/test/e2e/replica_set_mongod_port_change_with_arbiters/replica_set_mongod_port_change_with_arbiters_test.go index 91d6f72de..f398e36fc 100644 --- a/test/e2e/replica_set_mongod_port_change_with_arbiters/replica_set_mongod_port_change_with_arbiters_test.go +++ b/test/e2e/replica_set_mongod_port_change_with_arbiters/replica_set_mongod_port_change_with_arbiters_test.go @@ -1,6 +1,7 @@ package replica_set_mongod_config import ( + "context" "fmt" "os" "testing" @@ -11,7 +12,7 @@ import ( e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/mongodbtests" - setup "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" + "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" ) func TestMain(m *testing.M) { @@ -23,28 +24,29 @@ func TestMain(m *testing.M) { } func TestReplicaSetMongodPortChangeWithArbiters(t *testing.T) { - ctx := setup.Setup(t) - defer ctx.Teardown() + ctx := context.Background() + testCtx := setup.Setup(ctx, t) + defer testCtx.Teardown() - mdb, user := e2eutil.NewTestMongoDB(ctx, "mdb0", "") + mdb, user := e2eutil.NewTestMongoDB(testCtx, "mdb0", "") // FIXME: This behavior has been changed in 6.x timeline and now the arbiter (nor the RS) can't reach the goal state. mdb.Spec.Version = "4.4.19" scramUser := mdb.GetAuthUsers()[0] - _, err := setup.GeneratePasswordForUser(ctx, user, "") + _, err := setup.GeneratePasswordForUser(testCtx, user, "") if err != nil { t.Fatal(err) } - tester, err := FromResource(t, mdb) + tester, err := FromResource(ctx, t, mdb) if err != nil { t.Fatal(err) } connectivityTests := func(t *testing.T) { - fmt.Printf("connectionStringForUser: %s\n", mongodbtests.GetConnectionStringForUser(mdb, scramUser)) + fmt.Printf("connectionStringForUser: %s\n", mongodbtests.GetConnectionStringForUser(ctx, mdb, scramUser)) t.Run("Test Basic Connectivity with generated connection string secret", - tester.ConnectivitySucceeds(WithURI(mongodbtests.GetConnectionStringForUser(mdb, scramUser)))) + tester.ConnectivitySucceeds(WithURI(mongodbtests.GetConnectionStringForUser(ctx, mdb, scramUser)))) // FIXME after port change in the service mongodb+srv connection stopped working! //t.Run("Test SRV Connectivity", tester.ConnectivitySucceeds(WithURI(mdb.MongoSRVURI("")), WithoutTls(), WithReplicaSet(mdb.Name))) @@ -52,30 +54,30 @@ func TestReplicaSetMongodPortChangeWithArbiters(t *testing.T) { // tester.ConnectivitySucceeds(WithURI(mongodbtests.GetSrvConnectionStringForUser(mdb, scramUser)))) } - t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) - t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb)) + t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, testCtx)) + t.Run("Basic tests", mongodbtests.BasicFunctionality(ctx, &mdb)) t.Run("Ensure Authentication", tester.EnsureAuthenticationIsConfigured(3)) - t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 1)) + t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(ctx, &mdb, 1)) t.Run("Mongod setting net.port has been set", tester.EnsureMongodConfig("net.port", int32(automationconfig.DefaultDBPort))) - t.Run("Service has the correct port", mongodbtests.ServiceUsesCorrectPort(&mdb, int32(automationconfig.DefaultDBPort))) - t.Run("Stateful Set becomes ready", mongodbtests.StatefulSetBecomesReady(&mdb)) - t.Run("Wait for MongoDB to finish setup cluster", mongodbtests.MongoDBReachesRunningPhase(&mdb)) + t.Run("Service has the correct port", mongodbtests.ServiceUsesCorrectPort(ctx, &mdb, int32(automationconfig.DefaultDBPort))) + t.Run("Stateful Set becomes ready", mongodbtests.StatefulSetBecomesReady(ctx, &mdb)) + t.Run("Wait for MongoDB to finish setup cluster", mongodbtests.MongoDBReachesRunningPhase(ctx, &mdb)) t.Run("Connectivity tests", connectivityTests) - t.Run("Scale to 1 Arbiter", mongodbtests.ScaleArbiters(&mdb, 1)) - t.Run("Wait for MongoDB to start scaling arbiters", mongodbtests.MongoDBReachesPendingPhase(&mdb)) - t.Run("Wait for MongoDB to finish scaling arbiters", mongodbtests.MongoDBReachesRunningPhase(&mdb)) - t.Run("Automation config has expecter arbiter", mongodbtests.AutomationConfigReplicaSetsHaveExpectedArbiters(&mdb, 1)) - t.Run("Stateful Set becomes ready", mongodbtests.StatefulSetBecomesReady(&mdb)) - t.Run("Arbiters Stateful Set becomes ready", mongodbtests.ArbitersStatefulSetBecomesReady(&mdb)) + t.Run("Scale to 1 Arbiter", mongodbtests.ScaleArbiters(ctx, &mdb, 1)) + t.Run("Wait for MongoDB to start scaling arbiters", mongodbtests.MongoDBReachesPendingPhase(ctx, &mdb)) + t.Run("Wait for MongoDB to finish scaling arbiters", mongodbtests.MongoDBReachesRunningPhase(ctx, &mdb)) + t.Run("Automation config has expecter arbiter", mongodbtests.AutomationConfigReplicaSetsHaveExpectedArbiters(ctx, &mdb, 1)) + t.Run("Stateful Set becomes ready", mongodbtests.StatefulSetBecomesReady(ctx, &mdb)) + t.Run("Arbiters Stateful Set becomes ready", mongodbtests.ArbitersStatefulSetBecomesReady(ctx, &mdb)) t.Run("Connectivity tests", connectivityTests) - t.Run("Change port of running cluster", mongodbtests.ChangePort(&mdb, 40333)) - t.Run("Wait for MongoDB to start changing port", mongodbtests.MongoDBReachesPendingPhase(&mdb)) - t.Run("Wait for MongoDB to finish changing port", mongodbtests.MongoDBReachesRunningPhase(&mdb)) - t.Run("Stateful Set becomes ready", mongodbtests.StatefulSetBecomesReady(&mdb)) - t.Run("Arbiters Stateful Set becomes ready", mongodbtests.ArbitersStatefulSetBecomesReady(&mdb)) + t.Run("Change port of running cluster", mongodbtests.ChangePort(ctx, &mdb, 40333)) + t.Run("Wait for MongoDB to start changing port", mongodbtests.MongoDBReachesPendingPhase(ctx, &mdb)) + t.Run("Wait for MongoDB to finish changing port", mongodbtests.MongoDBReachesRunningPhase(ctx, &mdb)) + t.Run("Stateful Set becomes ready", mongodbtests.StatefulSetBecomesReady(ctx, &mdb)) + t.Run("Arbiters Stateful Set becomes ready", mongodbtests.ArbitersStatefulSetBecomesReady(ctx, &mdb)) t.Run("Mongod setting net.port has been set", tester.EnsureMongodConfig("net.port", int32(40333))) - t.Run("Service has the correct port", mongodbtests.ServiceUsesCorrectPort(&mdb, int32(40333))) + t.Run("Service has the correct port", mongodbtests.ServiceUsesCorrectPort(ctx, &mdb, int32(40333))) t.Run("Connectivity tests", connectivityTests) } diff --git a/test/e2e/replica_set_mongod_readiness/replica_set_mongod_readiness_test.go b/test/e2e/replica_set_mongod_readiness/replica_set_mongod_readiness_test.go index be8652690..d82837fb9 100644 --- a/test/e2e/replica_set_mongod_readiness/replica_set_mongod_readiness_test.go +++ b/test/e2e/replica_set_mongod_readiness/replica_set_mongod_readiness_test.go @@ -1,13 +1,14 @@ package replica_set_mongod_readiness import ( + "context" "fmt" "os" "testing" e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/mongodbtests" - setup "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" + "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" ) func TestMain(m *testing.M) { @@ -19,32 +20,33 @@ func TestMain(m *testing.M) { } func TestReplicaSet(t *testing.T) { - ctx := setup.Setup(t) - defer ctx.Teardown() + ctx := context.Background() + testCtx := setup.Setup(ctx, t) + defer testCtx.Teardown() - mdb, user := e2eutil.NewTestMongoDB(ctx, "mdb0", "") + mdb, user := e2eutil.NewTestMongoDB(testCtx, "mdb0", "") - _, err := setup.GeneratePasswordForUser(ctx, user, "") + _, err := setup.GeneratePasswordForUser(testCtx, user, "") if err != nil { t.Fatal(err) } - t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) - t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb)) + t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, testCtx)) + t.Run("Basic tests", mongodbtests.BasicFunctionality(ctx, &mdb)) t.Run("Ensure Agent container is marked as non-ready", func(t *testing.T) { - t.Run("Break mongod data files", mongodbtests.ExecInContainer(&mdb, 0, "mongod", "mkdir /data/tmp; mv /data/WiredTiger.wt /data/tmp")) + t.Run("Break mongod data files", mongodbtests.ExecInContainer(ctx, &mdb, 0, "mongod", "mkdir /data/tmp; mv /data/WiredTiger.wt /data/tmp")) // Just moving the file doesn't fail the mongod until any data is written - the easiest way is to kill the mongod // and in this case it won't restart - t.Run("Kill mongod process", mongodbtests.ExecInContainer(&mdb, 0, "mongod", "kill 1")) + t.Run("Kill mongod process", mongodbtests.ExecInContainer(ctx, &mdb, 0, "mongod", "kill 1")) // CLOUDP-89260: mongod uptime 1 minute and readiness probe failureThreshold 40 (40 * 5 -> 200 seconds) // note, that this may take much longer on evergreen than locally - t.Run("Pod agent container becomes not-ready", mongodbtests.PodContainerBecomesNotReady(&mdb, 0, "mongodb-agent")) + t.Run("Pod agent container becomes not-ready", mongodbtests.PodContainerBecomesNotReady(ctx, &mdb, 0, "mongodb-agent")) }) t.Run("Ensure Agent container gets fixed", func(t *testing.T) { // Note, that we call this command on the 'mongodb-agent' container as the 'mongod' container is down and we cannot // execute shell there. But both containers share the same /data directory so we can do it from any of them. - t.Run("Fix mongod data files", mongodbtests.ExecInContainer(&mdb, 0, "mongodb-agent", "mv /data/tmp/WiredTiger.wt /data/")) + t.Run("Fix mongod data files", mongodbtests.ExecInContainer(ctx, &mdb, 0, "mongodb-agent", "mv /data/tmp/WiredTiger.wt /data/")) // Eventually the agent will start mongod again - t.Run("Pod agent container becomes ready", mongodbtests.PodContainerBecomesReady(&mdb, 0, "mongodb-agent")) + t.Run("Pod agent container becomes ready", mongodbtests.PodContainerBecomesReady(ctx, &mdb, 0, "mongodb-agent")) }) } diff --git a/test/e2e/replica_set_mount_connection_string/replica_set_mount_connection_string_test.go b/test/e2e/replica_set_mount_connection_string/replica_set_mount_connection_string_test.go index 5f5c3bf51..b4a03cbec 100644 --- a/test/e2e/replica_set_mount_connection_string/replica_set_mount_connection_string_test.go +++ b/test/e2e/replica_set_mount_connection_string/replica_set_mount_connection_string_test.go @@ -16,7 +16,7 @@ import ( e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/mongodbtests" - setup "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" + "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" ) func TestMain(m *testing.M) { @@ -63,49 +63,50 @@ func createPythonTestPod(idx int, namespace, secretName, secretKey string) corev } func TestMountConnectionString(t *testing.T) { - ctx := setup.Setup(t) - defer ctx.Teardown() + ctx := context.Background() + testCtx := setup.Setup(ctx, t) + defer testCtx.Teardown() - mdb, user := e2eutil.NewTestMongoDB(ctx, "mdb0", "") + mdb, user := e2eutil.NewTestMongoDB(testCtx, "mdb0", "") scramUser := mdb.GetAuthUsers()[0] - _, err := setup.GeneratePasswordForUser(ctx, user, "") + _, err := setup.GeneratePasswordForUser(testCtx, user, "") if err != nil { t.Fatal(err) } - tester, err := FromResource(t, mdb) + tester, err := FromResource(ctx, t, mdb) if err != nil { t.Fatal(err) } - t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) - t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb)) + t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, testCtx)) + t.Run("Basic tests", mongodbtests.BasicFunctionality(ctx, &mdb)) t.Run("Keyfile authentication is configured", tester.HasKeyfileAuth(3)) t.Run("Test Basic Connectivity", tester.ConnectivitySucceeds()) t.Run("Test SRV Connectivity", tester.ConnectivitySucceeds(WithURI(mdb.MongoSRVURI("")), WithoutTls(), WithReplicaSet((mdb.Name)))) t.Run("Test Basic Connectivity with generated connection string secret", - tester.ConnectivitySucceeds(WithURI(mongodbtests.GetConnectionStringForUser(mdb, scramUser)))) + tester.ConnectivitySucceeds(WithURI(mongodbtests.GetConnectionStringForUser(ctx, mdb, scramUser)))) t.Run("Test SRV Connectivity with generated connection string secret", - tester.ConnectivitySucceeds(WithURI(mongodbtests.GetSrvConnectionStringForUser(mdb, scramUser)))) + tester.ConnectivitySucceeds(WithURI(mongodbtests.GetSrvConnectionStringForUser(ctx, mdb, scramUser)))) t.Run("Ensure Authentication", tester.EnsureAuthenticationIsConfigured(3)) - t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 1)) + t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(ctx, &mdb, 1)) t.Run("Application Pod can connect to MongoDB using the generated standard connection string.", func(t *testing.T) { testPod := createPythonTestPod(0, mdb.Namespace, fmt.Sprintf("%s-admin-%s", mdb.Name, user.Name), "connectionString.standard") - err := e2eutil.TestClient.Create(context.TODO(), &testPod, &e2eutil.CleanupOptions{ - TestContext: ctx, + err := e2eutil.TestClient.Create(ctx, &testPod, &e2eutil.CleanupOptions{ + TestContext: testCtx, }) assert.NoError(t, err) - assert.NoError(t, wait.ForPodPhase(t, time.Minute*5, testPod, corev1.PodSucceeded)) + assert.NoError(t, wait.ForPodPhase(ctx, t, time.Minute*5, testPod, corev1.PodSucceeded)) }) t.Run("Application Pod can connect to MongoDB using the generated secret SRV connection string", func(t *testing.T) { testPod := createPythonTestPod(1, mdb.Namespace, fmt.Sprintf("%s-admin-%s", mdb.Name, user.Name), "connectionString.standardSrv") - err := e2eutil.TestClient.Create(context.TODO(), &testPod, &e2eutil.CleanupOptions{ - TestContext: ctx, + err := e2eutil.TestClient.Create(ctx, &testPod, &e2eutil.CleanupOptions{ + TestContext: testCtx, }) assert.NoError(t, err) - assert.NoError(t, wait.ForPodPhase(t, time.Minute*5, testPod, corev1.PodSucceeded)) + assert.NoError(t, wait.ForPodPhase(ctx, t, time.Minute*5, testPod, corev1.PodSucceeded)) }) } diff --git a/test/e2e/replica_set_multiple/replica_set_multiple_test.go b/test/e2e/replica_set_multiple/replica_set_multiple_test.go index b6c173197..a38786eb0 100644 --- a/test/e2e/replica_set_multiple/replica_set_multiple_test.go +++ b/test/e2e/replica_set_multiple/replica_set_multiple_test.go @@ -1,6 +1,7 @@ package replica_set_multiple import ( + "context" "fmt" "os" "testing" @@ -11,7 +12,7 @@ import ( mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/mongodbtests" - setup "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" + "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" ) func TestMain(m *testing.M) { @@ -25,60 +26,60 @@ func TestMain(m *testing.M) { // TestReplicaSetMultiple creates two MongoDB resources that are handled by the Operator at the // same time. One of them is scaled to 5 and then back to 3 func TestReplicaSetMultiple(t *testing.T) { + ctx := context.Background() - ctx := setup.Setup(t) - defer ctx.Teardown() + testCtx := setup.Setup(ctx, t) + defer testCtx.Teardown() - mdb0, user0 := e2eutil.NewTestMongoDB(ctx, "mdb0", "") - mdb1, user1 := e2eutil.NewTestMongoDB(ctx, "mdb1", "") + mdb0, user0 := e2eutil.NewTestMongoDB(testCtx, "mdb0", "") + mdb1, user1 := e2eutil.NewTestMongoDB(testCtx, "mdb1", "") - _, err := setup.GeneratePasswordForUser(ctx, user0, "") + _, err := setup.GeneratePasswordForUser(testCtx, user0, "") if err != nil { t.Fatal(err) } - _, err = setup.GeneratePasswordForUser(ctx, user1, "") + _, err = setup.GeneratePasswordForUser(testCtx, user1, "") if err != nil { t.Fatal(err) } - tester0, err := mongotester.FromResource(t, mdb0) + tester0, err := mongotester.FromResource(ctx, t, mdb0) if err != nil { t.Fatal(err) } - tester1, err := mongotester.FromResource(t, mdb1) + tester1, err := mongotester.FromResource(ctx, t, mdb1) if err != nil { t.Fatal(err) } - t.Run("Create MongoDB Resource mdb0", mongodbtests.CreateMongoDBResource(&mdb0, ctx)) - t.Run("Create MongoDB Resource mdb1", mongodbtests.CreateMongoDBResource(&mdb1, ctx)) + t.Run("Create MongoDB Resource mdb0", mongodbtests.CreateMongoDBResource(&mdb0, testCtx)) + t.Run("Create MongoDB Resource mdb1", mongodbtests.CreateMongoDBResource(&mdb1, testCtx)) - t.Run("mdb0: Basic tests", mongodbtests.BasicFunctionality(&mdb0)) - t.Run("mdb1: Basic tests", mongodbtests.BasicFunctionality(&mdb1)) + t.Run("mdb0: Basic tests", mongodbtests.BasicFunctionality(ctx, &mdb0)) + t.Run("mdb1: Basic tests", mongodbtests.BasicFunctionality(ctx, &mdb1)) t.Run("mdb0: Test Basic Connectivity", tester0.ConnectivitySucceeds()) t.Run("mdb1: Test Basic Connectivity", tester1.ConnectivitySucceeds()) - t.Run("mdb0: AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb0, 1)) - t.Run("mdb1: AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb1, 1)) + t.Run("mdb0: AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(ctx, &mdb0, 1)) + t.Run("mdb1: AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(ctx, &mdb1, 1)) t.Run("mdb0: Ensure Authentication", tester0.EnsureAuthenticationIsConfigured(3)) t.Run("mdb1: Ensure Authentication", tester1.EnsureAuthenticationIsConfigured(3)) t.Run("MongoDB is reachable while being scaled up", func(t *testing.T) { defer tester0.StartBackgroundConnectivityTest(t, time.Second*10)() - t.Run("Scale MongoDB Resource Up", mongodbtests.Scale(&mdb0, 5)) - t.Run("Stateful Set Scaled Up Correctly", mongodbtests.StatefulSetBecomesReady(&mdb0)) - t.Run("MongoDB Reaches Running Phase", mongodbtests.MongoDBReachesRunningPhase(&mdb0)) - t.Run("AutomationConfig's version has been increased", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb0, 3)) - t.Run("Test Status Was Updated", mongodbtests.Status(&mdb0, - mdbv1.MongoDBCommunityStatus{ - MongoURI: mdb0.MongoURI(""), - Phase: mdbv1.Running, - CurrentMongoDBMembers: 5, - CurrentStatefulSetReplicas: 5, - })) + t.Run("Scale MongoDB Resource Up", mongodbtests.Scale(ctx, &mdb0, 5)) + t.Run("Stateful Set Scaled Up Correctly", mongodbtests.StatefulSetBecomesReady(ctx, &mdb0)) + t.Run("MongoDB Reaches Running Phase", mongodbtests.MongoDBReachesRunningPhase(ctx, &mdb0)) + t.Run("AutomationConfig's version has been increased", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(ctx, &mdb0, 3)) + t.Run("Test Status Was Updated", mongodbtests.Status(ctx, &mdb0, mdbv1.MongoDBCommunityStatus{ + MongoURI: mdb0.MongoURI(""), + Phase: mdbv1.Running, + CurrentMongoDBMembers: 5, + CurrentStatefulSetReplicas: 5, + })) // TODO: Currently the scale down process takes too long to reasonably include this in the test //t.Run("Scale MongoDB Resource Down", mongodbtests.Scale(&mdb0, 3)) diff --git a/test/e2e/replica_set_operator_upgrade/replica_set_operator_upgrade_test.go b/test/e2e/replica_set_operator_upgrade/replica_set_operator_upgrade_test.go index 1b99153d4..be45239e7 100644 --- a/test/e2e/replica_set_operator_upgrade/replica_set_operator_upgrade_test.go +++ b/test/e2e/replica_set_operator_upgrade/replica_set_operator_upgrade_test.go @@ -1,6 +1,7 @@ package replica_set_operator_upgrade import ( + "context" "fmt" "os" "testing" @@ -21,12 +22,13 @@ func TestMain(m *testing.M) { } func TestReplicaSetOperatorUpgrade(t *testing.T) { + ctx := context.Background() resourceName := "mdb0" testConfig := setup.LoadTestConfigFromEnv() - ctx := setup.SetupWithTestConfig(t, testConfig, true, true, resourceName) - defer ctx.Teardown() + testCtx := setup.SetupWithTestConfig(ctx, t, testConfig, true, true, resourceName) + defer testCtx.Teardown() - mdb, user := e2eutil.NewTestMongoDB(ctx, resourceName, testConfig.Namespace) + mdb, user := e2eutil.NewTestMongoDB(testCtx, resourceName, testConfig.Namespace) // Prior operator versions did not support MDB7 mdb.Spec.Version = "6.0.5" scramUser := mdb.GetAuthUsers()[0] @@ -34,43 +36,44 @@ func TestReplicaSetOperatorUpgrade(t *testing.T) { mdb.Spec.Arbiters = 1 mdb.Spec.Members = 2 - _, err := setup.GeneratePasswordForUser(ctx, user, testConfig.Namespace) + _, err := setup.GeneratePasswordForUser(testCtx, user, testConfig.Namespace) if err != nil { t.Fatal(err) } - tester, err := FromResource(t, mdb) + tester, err := FromResource(ctx, t, mdb) if err != nil { t.Fatal(err) } - t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) - t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb, true)) - t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 1)) + t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, testCtx)) + t.Run("Basic tests", mongodbtests.BasicFunctionality(ctx, &mdb, true)) + t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(ctx, &mdb, 1)) mongodbtests.SkipTestIfLocal(t, "Ensure MongoDB TLS Configuration", func(t *testing.T) { - t.Run("Has TLS Mode", tester.HasTlsMode("requireSSL", 60, WithTls(mdb))) - t.Run("Basic Connectivity Succeeds", tester.ConnectivitySucceeds(WithTls(mdb))) - t.Run("SRV Connectivity Succeeds", tester.ConnectivitySucceeds(WithURI(mdb.MongoSRVURI("")), WithTls(mdb))) + t.Run("Has TLS Mode", tester.HasTlsMode("requireSSL", 60, WithTls(ctx, mdb))) + t.Run("Basic Connectivity Succeeds", tester.ConnectivitySucceeds(WithTls(ctx, mdb))) + t.Run("SRV Connectivity Succeeds", tester.ConnectivitySucceeds(WithURI(mdb.MongoSRVURI("")), WithTls(ctx, mdb))) t.Run("Basic Connectivity With Generated Connection String Secret Succeeds", - tester.ConnectivitySucceeds(WithURI(mongodbtests.GetConnectionStringForUser(mdb, scramUser)), WithTls(mdb))) + tester.ConnectivitySucceeds(WithURI(mongodbtests.GetConnectionStringForUser(ctx, mdb, scramUser)), WithTls(ctx, mdb))) t.Run("SRV Connectivity With Generated Connection String Secret Succeeds", - tester.ConnectivitySucceeds(WithURI(mongodbtests.GetSrvConnectionStringForUser(mdb, scramUser)), WithTls(mdb))) + tester.ConnectivitySucceeds(WithURI(mongodbtests.GetSrvConnectionStringForUser(ctx, mdb, scramUser)), WithTls(ctx, mdb))) t.Run("Connectivity Fails", tester.ConnectivityFails(WithoutTls())) - t.Run("Ensure authentication is configured", tester.EnsureAuthenticationIsConfigured(3, WithTls(mdb))) + t.Run("Ensure authentication is configured", tester.EnsureAuthenticationIsConfigured(3, WithTls(ctx, mdb))) }) // upgrade the operator to master config := setup.LoadTestConfigFromEnv() - err = setup.DeployOperator(config, resourceName, true, false) + err = setup.DeployOperator(ctx, config, resourceName, true, false) assert.NoError(t, err) // Perform the basic tests - t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb, true)) + t.Run("Basic tests", mongodbtests.BasicFunctionality(ctx, &mdb, true)) } // TestReplicaSetOperatorUpgradeFrom0_7_2 is intended to be run locally not in CI. // It simulates deploying cluster using community operator 0.7.2 and then upgrading it using newer version. func TestReplicaSetOperatorUpgradeFrom0_7_2(t *testing.T) { + ctx := context.Background() t.Skip("Supporting this test in CI requires installing also CRDs from release v0.7.2") resourceName := "mdb-upg" testConfig := setup.LoadTestConfigFromEnv() @@ -81,35 +84,35 @@ func TestReplicaSetOperatorUpgradeFrom0_7_2(t *testing.T) { testConfig.ReadinessProbeImage = "quay.io/mongodb/mongodb-kubernetes-readinessprobe:1.0.6" testConfig.AgentImage = "quay.io/mongodb/mongodb-agent:11.0.5.6963-1" - ctx := setup.SetupWithTestConfig(t, testConfig, true, false, resourceName) - defer ctx.Teardown() + testCtx := setup.SetupWithTestConfig(ctx, t, testConfig, true, false, resourceName) + defer testCtx.Teardown() - mdb, user := e2eutil.NewTestMongoDB(ctx, resourceName, "") + mdb, user := e2eutil.NewTestMongoDB(testCtx, resourceName, "") scramUser := mdb.GetAuthUsers()[0] mdb.Spec.Security.TLS = e2eutil.NewTestTLSConfig(false) - _, err := setup.GeneratePasswordForUser(ctx, user, "") + _, err := setup.GeneratePasswordForUser(testCtx, user, "") if err != nil { t.Fatal(err) } - tester, err := FromResource(t, mdb) + tester, err := FromResource(ctx, t, mdb) if err != nil { t.Fatal(err) } runTests := func(t *testing.T) { - t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) - t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb, true)) - t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 1)) + t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, testCtx)) + t.Run("Basic tests", mongodbtests.BasicFunctionality(ctx, &mdb, true)) + t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(ctx, &mdb, 1)) t.Run("Keyfile authentication is configured", tester.HasKeyfileAuth(3)) - t.Run("Has TLS Mode", tester.HasTlsMode("requireSSL", 60, WithTls(mdb))) + t.Run("Has TLS Mode", tester.HasTlsMode("requireSSL", 60, WithTls(ctx, mdb))) t.Run("Test Basic Connectivity", tester.ConnectivitySucceeds()) t.Run("Test SRV Connectivity", tester.ConnectivitySucceeds(WithURI(mdb.MongoSRVURI("")), WithoutTls(), WithReplicaSet(mdb.Name))) t.Run("Test Basic Connectivity with generated connection string secret", - tester.ConnectivitySucceeds(WithURI(mongodbtests.GetConnectionStringForUser(mdb, scramUser)))) + tester.ConnectivitySucceeds(WithURI(mongodbtests.GetConnectionStringForUser(ctx, mdb, scramUser)))) t.Run("Test SRV Connectivity with generated connection string secret", - tester.ConnectivitySucceeds(WithURI(mongodbtests.GetSrvConnectionStringForUser(mdb, scramUser)))) + tester.ConnectivitySucceeds(WithURI(mongodbtests.GetSrvConnectionStringForUser(ctx, mdb, scramUser)))) t.Run("Ensure Authentication", tester.EnsureAuthenticationIsConfigured(3)) } @@ -119,7 +122,7 @@ func TestReplicaSetOperatorUpgradeFrom0_7_2(t *testing.T) { // rescale helm operator deployment to zero and run local operator then. testConfig = setup.LoadTestConfigFromEnv() - err = setup.DeployOperator(testConfig, resourceName, true, false) + err = setup.DeployOperator(ctx, testConfig, resourceName, true, false) assert.NoError(t, err) runTests(t) diff --git a/test/e2e/replica_set_recovery/replica_set_recovery_test.go b/test/e2e/replica_set_recovery/replica_set_recovery_test.go index 0ac3d4168..a6ab6f7a0 100644 --- a/test/e2e/replica_set_recovery/replica_set_recovery_test.go +++ b/test/e2e/replica_set_recovery/replica_set_recovery_test.go @@ -1,6 +1,7 @@ package replica_set_recovery import ( + "context" "crypto/rand" "fmt" "math/big" @@ -13,7 +14,7 @@ import ( mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/mongodbtests" - setup "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" + "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" ) func TestMain(m *testing.M) { @@ -25,24 +26,25 @@ func TestMain(m *testing.M) { } func TestReplicaSetRecovery(t *testing.T) { - ctx := setup.Setup(t) - defer ctx.Teardown() + ctx := context.Background() + testCtx := setup.Setup(ctx, t) + defer testCtx.Teardown() - mdb, user := e2eutil.NewTestMongoDB(ctx, "mdb0", "") - _, err := setup.GeneratePasswordForUser(ctx, user, "") + mdb, user := e2eutil.NewTestMongoDB(testCtx, "mdb0", "") + _, err := setup.GeneratePasswordForUser(testCtx, user, "") if err != nil { t.Fatal(err) } - tester, err := mongotester.FromResource(t, mdb) + tester, err := mongotester.FromResource(ctx, t, mdb) if err != nil { t.Fatal(err) } - t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) - t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb)) + t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, testCtx)) + t.Run("Basic tests", mongodbtests.BasicFunctionality(ctx, &mdb)) t.Run("Test Basic Connectivity", tester.ConnectivitySucceeds()) - t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 1)) + t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(ctx, &mdb, 1)) t.Run("Ensure Authentication", tester.EnsureAuthenticationIsConfigured(3)) t.Run("MongoDB is reachable", func(t *testing.T) { @@ -51,17 +53,16 @@ func TestReplicaSetRecovery(t *testing.T) { if err != nil { t.Fatal(err) } - t.Run("Delete Random Pod", mongodbtests.DeletePod(&mdb, int(n.Int64()))) - t.Run("Test Replica Set Recovers", mongodbtests.StatefulSetBecomesReady(&mdb)) - t.Run("MongoDB Reaches Running Phase", mongodbtests.MongoDBReachesRunningPhase(&mdb)) - t.Run("Test Status Was Updated", mongodbtests.Status(&mdb, - mdbv1.MongoDBCommunityStatus{ - MongoURI: mdb.MongoURI(""), - Phase: mdbv1.Running, - Version: mdb.GetMongoDBVersion(nil), - CurrentMongoDBMembers: 3, - CurrentStatefulSetReplicas: 3, - })) + t.Run("Delete Random Pod", mongodbtests.DeletePod(ctx, &mdb, int(n.Int64()))) + t.Run("Test Replica Set Recovers", mongodbtests.StatefulSetBecomesReady(ctx, &mdb)) + t.Run("MongoDB Reaches Running Phase", mongodbtests.MongoDBReachesRunningPhase(ctx, &mdb)) + t.Run("Test Status Was Updated", mongodbtests.Status(ctx, &mdb, mdbv1.MongoDBCommunityStatus{ + MongoURI: mdb.MongoURI(""), + Phase: mdbv1.Running, + Version: mdb.GetMongoDBVersion(nil), + CurrentMongoDBMembers: 3, + CurrentStatefulSetReplicas: 3, + })) }) } diff --git a/test/e2e/replica_set_scale/replica_set_scaling_test.go b/test/e2e/replica_set_scale/replica_set_scaling_test.go index 7d2557715..ee0fe2f19 100644 --- a/test/e2e/replica_set_scale/replica_set_scaling_test.go +++ b/test/e2e/replica_set_scale/replica_set_scaling_test.go @@ -1,6 +1,7 @@ package replica_set_scale_up import ( + "context" "fmt" "os" "testing" @@ -9,7 +10,7 @@ import ( mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/mongodbtests" - setup "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" + "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/util/mongotester" ) @@ -22,41 +23,41 @@ func TestMain(m *testing.M) { } func TestReplicaSetScaleUp(t *testing.T) { + ctx := context.Background() - ctx := setup.Setup(t) - defer ctx.Teardown() + testCtx := setup.Setup(ctx, t) + defer testCtx.Teardown() - mdb, user := e2eutil.NewTestMongoDB(ctx, "mdb0", "") + mdb, user := e2eutil.NewTestMongoDB(testCtx, "mdb0", "") - _, err := setup.GeneratePasswordForUser(ctx, user, "") + _, err := setup.GeneratePasswordForUser(testCtx, user, "") if err != nil { t.Fatal(err) } - tester, err := mongotester.FromResource(t, mdb) + tester, err := mongotester.FromResource(ctx, t, mdb) if err != nil { t.Fatal(err) } - t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) - t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb)) + t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, testCtx)) + t.Run("Basic tests", mongodbtests.BasicFunctionality(ctx, &mdb)) t.Run("Test Basic Connectivity", tester.ConnectivitySucceeds()) t.Run("Ensure Authentication", tester.EnsureAuthenticationIsConfigured(3)) - t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 1)) + t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(ctx, &mdb, 1)) t.Run("MongoDB is reachable", func(t *testing.T) { defer tester.StartBackgroundConnectivityTest(t, time.Second*10)() - t.Run("Scale MongoDB Resource Up", mongodbtests.Scale(&mdb, 5)) - t.Run("Stateful Set Scaled Up Correctly", mongodbtests.StatefulSetBecomesReady(&mdb)) - t.Run("MongoDB Reaches Running Phase", mongodbtests.MongoDBReachesRunningPhase(&mdb)) - t.Run("AutomationConfig's version has been increased", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 3)) - t.Run("Test Status Was Updated", mongodbtests.Status(&mdb, - mdbv1.MongoDBCommunityStatus{ - MongoURI: mdb.MongoURI(""), - Phase: mdbv1.Running, - Version: mdb.GetMongoDBVersion(nil), - CurrentMongoDBMembers: 5, - CurrentStatefulSetReplicas: 5, - })) + t.Run("Scale MongoDB Resource Up", mongodbtests.Scale(ctx, &mdb, 5)) + t.Run("Stateful Set Scaled Up Correctly", mongodbtests.StatefulSetBecomesReady(ctx, &mdb)) + t.Run("MongoDB Reaches Running Phase", mongodbtests.MongoDBReachesRunningPhase(ctx, &mdb)) + t.Run("AutomationConfig's version has been increased", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(ctx, &mdb, 3)) + t.Run("Test Status Was Updated", mongodbtests.Status(ctx, &mdb, mdbv1.MongoDBCommunityStatus{ + MongoURI: mdb.MongoURI(""), + Phase: mdbv1.Running, + Version: mdb.GetMongoDBVersion(nil), + CurrentMongoDBMembers: 5, + CurrentStatefulSetReplicas: 5, + })) // TODO: Currently the scale down process takes too long to reasonably include this in the test //t.Run("Scale MongoDB Resource Down", mongodbtests.Scale(&mdb, 3)) diff --git a/test/e2e/replica_set_scale_down/replica_set_scale_down_test.go b/test/e2e/replica_set_scale_down/replica_set_scale_down_test.go index 287b91bc8..529738712 100644 --- a/test/e2e/replica_set_scale_down/replica_set_scale_down_test.go +++ b/test/e2e/replica_set_scale_down/replica_set_scale_down_test.go @@ -1,6 +1,7 @@ package replica_set_scale_down import ( + "context" "fmt" "os" "testing" @@ -12,7 +13,7 @@ import ( e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/mongodbtests" - setup "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" + "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" ) func TestMain(m *testing.M) { @@ -24,41 +25,41 @@ func TestMain(m *testing.M) { } func TestReplicaSetScaleDown(t *testing.T) { - ctx := setup.Setup(t) - defer ctx.Teardown() + ctx := context.Background() + testCtx := setup.Setup(ctx, t) + defer testCtx.Teardown() - mdb, user := e2eutil.NewTestMongoDB(ctx, "replica-set-scale-down", "") + mdb, user := e2eutil.NewTestMongoDB(testCtx, "replica-set-scale-down", "") - _, err := setup.GeneratePasswordForUser(ctx, user, "") + _, err := setup.GeneratePasswordForUser(testCtx, user, "") if err != nil { t.Fatal(err) } - tester, err := FromResource(t, mdb) + tester, err := FromResource(ctx, t, mdb) if err != nil { t.Fatal(err) } - t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) - t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb)) + t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, testCtx)) + t.Run("Basic tests", mongodbtests.BasicFunctionality(ctx, &mdb)) t.Run("Keyfile authentication is configured", tester.HasKeyfileAuth(3)) t.Run("Test Basic Connectivity", tester.ConnectivitySucceeds()) t.Run("Ensure Authentication", tester.EnsureAuthenticationIsConfigured(3)) - t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 1)) + t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(ctx, &mdb, 1)) t.Run("MongoDB is reachable", func(t *testing.T) { defer tester.StartBackgroundConnectivityTest(t, time.Second*10)() - t.Run("Scale MongoDB Resource Down", mongodbtests.Scale(&mdb, 1)) - t.Run("Stateful Set Scaled Down Correctly", mongodbtests.StatefulSetIsReadyAfterScaleDown(&mdb)) - t.Run("MongoDB Reaches Running Phase", mongodbtests.MongoDBReachesRunningPhase(&mdb)) - t.Run("AutomationConfig's version has been increased", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 3)) - t.Run("Test Status Was Updated", mongodbtests.Status(&mdb, - mdbv1.MongoDBCommunityStatus{ - MongoURI: mdb.MongoURI(""), - Phase: mdbv1.Running, - Version: mdb.GetMongoDBVersion(nil), - CurrentMongoDBMembers: 1, - CurrentStatefulSetReplicas: 1, - })) + t.Run("Scale MongoDB Resource Down", mongodbtests.Scale(ctx, &mdb, 1)) + t.Run("Stateful Set Scaled Down Correctly", mongodbtests.StatefulSetIsReadyAfterScaleDown(ctx, &mdb)) + t.Run("MongoDB Reaches Running Phase", mongodbtests.MongoDBReachesRunningPhase(ctx, &mdb)) + t.Run("AutomationConfig's version has been increased", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(ctx, &mdb, 3)) + t.Run("Test Status Was Updated", mongodbtests.Status(ctx, &mdb, mdbv1.MongoDBCommunityStatus{ + MongoURI: mdb.MongoURI(""), + Phase: mdbv1.Running, + Version: mdb.GetMongoDBVersion(nil), + CurrentMongoDBMembers: 1, + CurrentStatefulSetReplicas: 1, + })) }) } diff --git a/test/e2e/replica_set_tls/replica_set_tls_test.go b/test/e2e/replica_set_tls/replica_set_tls_test.go index ea86db71a..719bcdc8f 100644 --- a/test/e2e/replica_set_tls/replica_set_tls_test.go +++ b/test/e2e/replica_set_tls/replica_set_tls_test.go @@ -1,6 +1,7 @@ package replica_set_tls import ( + "context" "fmt" "os" "testing" @@ -9,7 +10,7 @@ import ( e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/mongodbtests" - setup "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" + "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" ) func TestMain(m *testing.M) { @@ -21,40 +22,41 @@ func TestMain(m *testing.M) { } func TestReplicaSetTLS(t *testing.T) { + ctx := context.Background() resourceName := "mdb-tls" - ctx, testConfig := setup.SetupWithTLS(t, resourceName) - defer ctx.Teardown() + testCtx, testConfig := setup.SetupWithTLS(ctx, t, resourceName) + defer testCtx.Teardown() - mdb, user := e2eutil.NewTestMongoDB(ctx, resourceName, testConfig.Namespace) + mdb, user := e2eutil.NewTestMongoDB(testCtx, resourceName, testConfig.Namespace) scramUser := mdb.GetAuthUsers()[0] mdb.Spec.Security.TLS = e2eutil.NewTestTLSConfig(false) - _, err := setup.GeneratePasswordForUser(ctx, user, testConfig.Namespace) + _, err := setup.GeneratePasswordForUser(testCtx, user, testConfig.Namespace) if err != nil { t.Fatal(err) } - tester, err := FromResource(t, mdb) + tester, err := FromResource(ctx, t, mdb) if err != nil { t.Fatal(err) } - t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) - t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb)) + t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, testCtx)) + t.Run("Basic tests", mongodbtests.BasicFunctionality(ctx, &mdb)) mongodbtests.SkipTestIfLocal(t, "Ensure MongoDB TLS Configuration", func(t *testing.T) { - t.Run("Has TLS Mode", tester.HasTlsMode("requireSSL", 60, WithTls(mdb))) - t.Run("Basic Connectivity Succeeds", tester.ConnectivitySucceeds(WithTls(mdb))) - t.Run("SRV Connectivity Succeeds", tester.ConnectivitySucceeds(WithURI(mdb.MongoSRVURI("")), WithTls(mdb))) + t.Run("Has TLS Mode", tester.HasTlsMode("requireSSL", 60, WithTls(ctx, mdb))) + t.Run("Basic Connectivity Succeeds", tester.ConnectivitySucceeds(WithTls(ctx, mdb))) + t.Run("SRV Connectivity Succeeds", tester.ConnectivitySucceeds(WithURI(mdb.MongoSRVURI("")), WithTls(ctx, mdb))) t.Run("Basic Connectivity With Generated Connection String Secret Succeeds", - tester.ConnectivitySucceeds(WithURI(mongodbtests.GetConnectionStringForUser(mdb, scramUser)), WithTls(mdb))) + tester.ConnectivitySucceeds(WithURI(mongodbtests.GetConnectionStringForUser(ctx, mdb, scramUser)), WithTls(ctx, mdb))) t.Run("SRV Connectivity With Generated Connection String Secret Succeeds", - tester.ConnectivitySucceeds(WithURI(mongodbtests.GetSrvConnectionStringForUser(mdb, scramUser)), WithTls(mdb))) + tester.ConnectivitySucceeds(WithURI(mongodbtests.GetSrvConnectionStringForUser(ctx, mdb, scramUser)), WithTls(ctx, mdb))) t.Run("Connectivity Fails", tester.ConnectivityFails(WithoutTls())) - t.Run("Ensure authentication is configured", tester.EnsureAuthenticationIsConfigured(3, WithTls(mdb))) + t.Run("Ensure authentication is configured", tester.EnsureAuthenticationIsConfigured(3, WithTls(ctx, mdb))) }) - t.Run("TLS is disabled", mongodbtests.DisableTLS(&mdb)) - t.Run("MongoDB Reaches Failed Phase", mongodbtests.MongoDBReachesFailedPhase(&mdb)) - t.Run("TLS is enabled", mongodbtests.EnableTLS(&mdb)) - t.Run("MongoDB Reaches Running Phase", mongodbtests.MongoDBReachesRunningPhase(&mdb)) + t.Run("TLS is disabled", mongodbtests.DisableTLS(ctx, &mdb)) + t.Run("MongoDB Reaches Failed Phase", mongodbtests.MongoDBReachesFailedPhase(ctx, &mdb)) + t.Run("TLS is enabled", mongodbtests.EnableTLS(ctx, &mdb)) + t.Run("MongoDB Reaches Running Phase", mongodbtests.MongoDBReachesRunningPhase(ctx, &mdb)) } diff --git a/test/e2e/replica_set_tls_recreate_mdbc/replica_set_tls_recreate_mdbc_test.go b/test/e2e/replica_set_tls_recreate_mdbc/replica_set_tls_recreate_mdbc_test.go index 8207b6739..751a048c4 100644 --- a/test/e2e/replica_set_tls_recreate_mdbc/replica_set_tls_recreate_mdbc_test.go +++ b/test/e2e/replica_set_tls_recreate_mdbc/replica_set_tls_recreate_mdbc_test.go @@ -10,7 +10,7 @@ import ( e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/mongodbtests" - setup "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" + "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" ) func TestMain(m *testing.M) { @@ -22,50 +22,51 @@ func TestMain(m *testing.M) { } func TestReplicaSetTLSRecreateMdbc(t *testing.T) { + ctx := context.Background() resourceName := "mdb-tls" - ctx, testConfig := setup.SetupWithTLS(t, resourceName) - defer ctx.Teardown() + testCtx, testConfig := setup.SetupWithTLS(ctx, t, resourceName) + defer testCtx.Teardown() - mdb1, user := e2eutil.NewTestMongoDB(ctx, resourceName, testConfig.Namespace) + mdb1, user := e2eutil.NewTestMongoDB(testCtx, resourceName, testConfig.Namespace) scramUser := mdb1.GetAuthUsers()[0] mdb1.Spec.Security.TLS = e2eutil.NewTestTLSConfig(false) - _, err := setup.GeneratePasswordForUser(ctx, user, testConfig.Namespace) + _, err := setup.GeneratePasswordForUser(testCtx, user, testConfig.Namespace) if err != nil { t.Fatal(err) } - t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb1, ctx)) - t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb1)) + t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb1, testCtx)) + t.Run("Basic tests", mongodbtests.BasicFunctionality(ctx, &mdb1)) - if err := e2eutil.TestClient.Delete(context.TODO(), &mdb1); err != nil { + if err := e2eutil.TestClient.Delete(ctx, &mdb1); err != nil { t.Fatalf("Failed to delete first test MongoDB: %s", err) } - t.Run("Stateful Set Is Deleted", mongodbtests.StatefulSetIsDeleted(&mdb1)) + t.Run("Stateful Set Is Deleted", mongodbtests.StatefulSetIsDeleted(ctx, &mdb1)) - mdb2, _ := e2eutil.NewTestMongoDB(ctx, resourceName, testConfig.Namespace) + mdb2, _ := e2eutil.NewTestMongoDB(testCtx, resourceName, testConfig.Namespace) mdb2.Spec.Security.TLS = e2eutil.NewTestTLSConfig(false) - tester1, err := FromResource(t, mdb2) + tester1, err := FromResource(ctx, t, mdb2) if err != nil { t.Fatal(err) } - t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb2, ctx)) - t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb2)) + t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb2, testCtx)) + t.Run("Basic tests", mongodbtests.BasicFunctionality(ctx, &mdb2)) mongodbtests.SkipTestIfLocal(t, "Ensure MongoDB TLS Configuration", func(t *testing.T) { - t.Run("Has TLS Mode", tester1.HasTlsMode("requireSSL", 60, WithTls(mdb2))) - t.Run("Basic Connectivity Succeeds", tester1.ConnectivitySucceeds(WithTls(mdb2))) - t.Run("SRV Connectivity Succeeds", tester1.ConnectivitySucceeds(WithURI(mdb2.MongoSRVURI("")), WithTls(mdb2))) + t.Run("Has TLS Mode", tester1.HasTlsMode("requireSSL", 60, WithTls(ctx, mdb2))) + t.Run("Basic Connectivity Succeeds", tester1.ConnectivitySucceeds(WithTls(ctx, mdb2))) + t.Run("SRV Connectivity Succeeds", tester1.ConnectivitySucceeds(WithURI(mdb2.MongoSRVURI("")), WithTls(ctx, mdb2))) t.Run("Basic Connectivity With Generated Connection String Secret Succeeds", - tester1.ConnectivitySucceeds(WithURI(mongodbtests.GetConnectionStringForUser(mdb2, scramUser)), WithTls(mdb2))) + tester1.ConnectivitySucceeds(WithURI(mongodbtests.GetConnectionStringForUser(ctx, mdb2, scramUser)), WithTls(ctx, mdb2))) t.Run("SRV Connectivity With Generated Connection String Secret Succeeds", - tester1.ConnectivitySucceeds(WithURI(mongodbtests.GetSrvConnectionStringForUser(mdb2, scramUser)), WithTls(mdb2))) + tester1.ConnectivitySucceeds(WithURI(mongodbtests.GetSrvConnectionStringForUser(ctx, mdb2, scramUser)), WithTls(ctx, mdb2))) t.Run("Connectivity Fails", tester1.ConnectivityFails(WithoutTls())) - t.Run("Ensure authentication is configured", tester1.EnsureAuthenticationIsConfigured(3, WithTls(mdb2))) + t.Run("Ensure authentication is configured", tester1.EnsureAuthenticationIsConfigured(3, WithTls(ctx, mdb2))) }) - t.Run("TLS is disabled", mongodbtests.DisableTLS(&mdb2)) - t.Run("MongoDB Reaches Failed Phase", mongodbtests.MongoDBReachesFailedPhase(&mdb2)) - t.Run("TLS is enabled", mongodbtests.EnableTLS(&mdb2)) - t.Run("MongoDB Reaches Running Phase", mongodbtests.MongoDBReachesRunningPhase(&mdb2)) + t.Run("TLS is disabled", mongodbtests.DisableTLS(ctx, &mdb2)) + t.Run("MongoDB Reaches Failed Phase", mongodbtests.MongoDBReachesFailedPhase(ctx, &mdb2)) + t.Run("TLS is enabled", mongodbtests.EnableTLS(ctx, &mdb2)) + t.Run("MongoDB Reaches Running Phase", mongodbtests.MongoDBReachesRunningPhase(ctx, &mdb2)) } diff --git a/test/e2e/replica_set_tls_rotate/replica_set_tls_rotate_test.go b/test/e2e/replica_set_tls_rotate/replica_set_tls_rotate_test.go index 6eb51f50c..86c1b6614 100644 --- a/test/e2e/replica_set_tls_rotate/replica_set_tls_rotate_test.go +++ b/test/e2e/replica_set_tls_rotate/replica_set_tls_rotate_test.go @@ -1,6 +1,7 @@ package replica_set_tls import ( + "context" "fmt" "os" "testing" @@ -11,7 +12,7 @@ import ( e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/mongodbtests" - setup "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" + "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" ) func TestMain(m *testing.M) { @@ -23,44 +24,45 @@ func TestMain(m *testing.M) { } func TestReplicaSetTLSRotate(t *testing.T) { + ctx := context.Background() resourceName := "mdb-tls" - ctx, testConfig := setup.SetupWithTLS(t, resourceName) - defer ctx.Teardown() + testCtx, testConfig := setup.SetupWithTLS(ctx, t, resourceName) + defer testCtx.Teardown() - mdb, user := e2eutil.NewTestMongoDB(ctx, resourceName, testConfig.Namespace) + mdb, user := e2eutil.NewTestMongoDB(testCtx, resourceName, testConfig.Namespace) mdb.Spec.Security.TLS = e2eutil.NewTestTLSConfig(false) - _, err := setup.GeneratePasswordForUser(ctx, user, testConfig.Namespace) + _, err := setup.GeneratePasswordForUser(testCtx, user, testConfig.Namespace) if err != nil { t.Fatal(err) } - tester, err := FromResource(t, mdb) + tester, err := FromResource(ctx, t, mdb) if err != nil { t.Fatal(err) } - clientCert, err := GetClientCert(mdb) + clientCert, err := GetClientCert(ctx, mdb) if err != nil { t.Fatal(err) } initialCertSerialNumber := clientCert.SerialNumber - t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) - t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb)) - t.Run("Wait for TLS to be enabled", tester.HasTlsMode("requireSSL", 60, WithTls(mdb))) - t.Run("Test Basic TLS Connectivity", tester.ConnectivitySucceeds(WithTls(mdb))) - t.Run("Ensure Authentication", tester.EnsureAuthenticationIsConfigured(3, WithTls(mdb))) + t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, testCtx)) + t.Run("Basic tests", mongodbtests.BasicFunctionality(ctx, &mdb)) + t.Run("Wait for TLS to be enabled", tester.HasTlsMode("requireSSL", 60, WithTls(ctx, mdb))) + t.Run("Test Basic TLS Connectivity", tester.ConnectivitySucceeds(WithTls(ctx, mdb))) + t.Run("Ensure Authentication", tester.EnsureAuthenticationIsConfigured(3, WithTls(ctx, mdb))) t.Run("Test TLS required", tester.ConnectivityFails(WithoutTls())) t.Run("MongoDB is reachable while certificate is rotated", func(t *testing.T) { - defer tester.StartBackgroundConnectivityTest(t, time.Second*10, WithTls(mdb))() - t.Run("Update certificate secret", tlstests.RotateCertificate(&mdb)) + defer tester.StartBackgroundConnectivityTest(t, time.Second*10, WithTls(ctx, mdb))() + t.Run("Update certificate secret", tlstests.RotateCertificate(ctx, &mdb)) t.Run("Wait for certificate to be rotated", tester.WaitForRotatedCertificate(mdb, initialCertSerialNumber)) - t.Run("Wait for MongoDB to reach Running Phase after rotating server cert", mongodbtests.MongoDBReachesRunningPhase(&mdb)) - t.Run("Extend CA certificate validity", tlstests.ExtendCACertificate(&mdb)) - t.Run("Wait for MongoDB to start reconciling after extending CA", mongodbtests.MongoDBReachesPendingPhase(&mdb)) - t.Run("Wait for MongoDB to reach Running Phase after extending CA", mongodbtests.MongoDBReachesRunningPhase(&mdb)) + t.Run("Wait for MongoDB to reach Running Phase after rotating server cert", mongodbtests.MongoDBReachesRunningPhase(ctx, &mdb)) + t.Run("Extend CA certificate validity", tlstests.ExtendCACertificate(ctx, &mdb)) + t.Run("Wait for MongoDB to start reconciling after extending CA", mongodbtests.MongoDBReachesPendingPhase(ctx, &mdb)) + t.Run("Wait for MongoDB to reach Running Phase after extending CA", mongodbtests.MongoDBReachesRunningPhase(ctx, &mdb)) }) } diff --git a/test/e2e/replica_set_tls_rotate_delete_sts/replica_set_tls_rotate_delete_sts_test.go b/test/e2e/replica_set_tls_rotate_delete_sts/replica_set_tls_rotate_delete_sts_test.go index 14cda1006..0bc0448bd 100644 --- a/test/e2e/replica_set_tls_rotate_delete_sts/replica_set_tls_rotate_delete_sts_test.go +++ b/test/e2e/replica_set_tls_rotate_delete_sts/replica_set_tls_rotate_delete_sts_test.go @@ -1,6 +1,7 @@ package replica_set_tls_rotate_delete_sts import ( + "context" "os" "testing" @@ -23,43 +24,44 @@ func TestMain(m *testing.M) { } func TestReplicaSetTLSRotateDeleteSts(t *testing.T) { + ctx := context.Background() resourceName := "mdb-tls" - ctx, testConfig := setup.SetupWithTLS(t, resourceName) - defer ctx.Teardown() + testCtx, testConfig := setup.SetupWithTLS(ctx, t, resourceName) + defer testCtx.Teardown() - mdb, user := e2eutil.NewTestMongoDB(ctx, resourceName, testConfig.Namespace) + mdb, user := e2eutil.NewTestMongoDB(testCtx, resourceName, testConfig.Namespace) mdb.Spec.Security.TLS = e2eutil.NewTestTLSConfig(false) - _, err := setup.GeneratePasswordForUser(ctx, user, testConfig.Namespace) + _, err := setup.GeneratePasswordForUser(testCtx, user, testConfig.Namespace) if err != nil { t.Fatal(err) } - tester, err := FromResource(t, mdb) + tester, err := FromResource(ctx, t, mdb) if err != nil { t.Fatal(err) } - clientCert, err := GetClientCert(mdb) + clientCert, err := GetClientCert(ctx, mdb) if err != nil { t.Fatal(err) } initialCertSerialNumber := clientCert.SerialNumber - t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) - t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb)) - t.Run("Wait for TLS to be enabled", tester.HasTlsMode("requireSSL", 60, WithTls(mdb))) - t.Run("Test Basic TLS Connectivity", tester.ConnectivitySucceeds(WithTls(mdb))) - t.Run("Ensure Authentication", tester.EnsureAuthenticationIsConfigured(3, WithTls(mdb))) + t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, testCtx)) + t.Run("Basic tests", mongodbtests.BasicFunctionality(ctx, &mdb)) + t.Run("Wait for TLS to be enabled", tester.HasTlsMode("requireSSL", 60, WithTls(ctx, mdb))) + t.Run("Test Basic TLS Connectivity", tester.ConnectivitySucceeds(WithTls(ctx, mdb))) + t.Run("Ensure Authentication", tester.EnsureAuthenticationIsConfigured(3, WithTls(ctx, mdb))) t.Run("Test TLS required", tester.ConnectivityFails(WithoutTls())) t.Run("MongoDB is reachable while certificate is rotated", func(t *testing.T) { - t.Run("Delete Statefulset", mongodbtests.DeleteStatefulSet(&mdb)) - t.Run("Update certificate secret", tlstests.RotateCertificate(&mdb)) + t.Run("Delete Statefulset", mongodbtests.DeleteStatefulSet(ctx, &mdb)) + t.Run("Update certificate secret", tlstests.RotateCertificate(ctx, &mdb)) t.Run("Wait for certificate to be rotated", tester.WaitForRotatedCertificate(mdb, initialCertSerialNumber)) - t.Run("Test Replica Set Recovers", mongodbtests.StatefulSetBecomesReady(&mdb)) - t.Run("Wait for MongoDB to reach Running Phase", mongodbtests.MongoDBReachesRunningPhase(&mdb)) - t.Run("Test Basic TLS Connectivity", tester.ConnectivitySucceeds(WithTls(mdb))) + t.Run("Test Replica Set Recovers", mongodbtests.StatefulSetBecomesReady(ctx, &mdb)) + t.Run("Wait for MongoDB to reach Running Phase", mongodbtests.MongoDBReachesRunningPhase(ctx, &mdb)) + t.Run("Test Basic TLS Connectivity", tester.ConnectivitySucceeds(WithTls(ctx, mdb))) }) } diff --git a/test/e2e/replica_set_tls_upgrade/replica_set_tls_upgrade_test.go b/test/e2e/replica_set_tls_upgrade/replica_set_tls_upgrade_test.go index 8ac787079..eb85477f3 100644 --- a/test/e2e/replica_set_tls_upgrade/replica_set_tls_upgrade_test.go +++ b/test/e2e/replica_set_tls_upgrade/replica_set_tls_upgrade_test.go @@ -1,6 +1,7 @@ package replica_set_tls import ( + "context" "fmt" "os" "testing" @@ -12,7 +13,7 @@ import ( e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/mongodbtests" - setup "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" + "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" ) func TestMain(m *testing.M) { @@ -24,50 +25,51 @@ func TestMain(m *testing.M) { } func TestReplicaSetTLSUpgrade(t *testing.T) { + ctx := context.Background() resourceName := "mdb-tls" - ctx, testConfig := setup.SetupWithTLS(t, resourceName) - defer ctx.Teardown() + testCtx, testConfig := setup.SetupWithTLS(ctx, t, resourceName) + defer testCtx.Teardown() - mdb, user := e2eutil.NewTestMongoDB(ctx, resourceName, testConfig.Namespace) - _, err := setup.GeneratePasswordForUser(ctx, user, testConfig.Namespace) + mdb, user := e2eutil.NewTestMongoDB(testCtx, resourceName, testConfig.Namespace) + _, err := setup.GeneratePasswordForUser(testCtx, user, testConfig.Namespace) if err != nil { t.Fatal(err) } - tester, err := FromResource(t, mdb) + tester, err := FromResource(ctx, t, mdb) if err != nil { t.Fatal(err) } - t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) - t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb)) + t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, testCtx)) + t.Run("Basic tests", mongodbtests.BasicFunctionality(ctx, &mdb)) t.Run("Test Basic Connectivity", tester.ConnectivitySucceeds(WithoutTls())) t.Run("Ensure Authentication", tester.EnsureAuthenticationIsConfigured(3)) // Enable TLS as optional t.Run("MongoDB is reachable while TLS is being enabled", func(t *testing.T) { defer tester.StartBackgroundConnectivityTest(t, time.Second*15, WithoutTls())() - t.Run("Upgrade to TLS", tlstests.EnableTLS(&mdb, true)) - t.Run("Stateful Set Leaves Ready State, after setting TLS to preferSSL", mongodbtests.StatefulSetBecomesUnready(&mdb)) - t.Run("Stateful Set Reaches Ready State, after setting TLS to preferSSL", mongodbtests.StatefulSetBecomesReady(&mdb)) + t.Run("Upgrade to TLS", tlstests.EnableTLS(ctx, &mdb, true)) + t.Run("Stateful Set Leaves Ready State, after setting TLS to preferSSL", mongodbtests.StatefulSetBecomesUnready(ctx, &mdb)) + t.Run("Stateful Set Reaches Ready State, after setting TLS to preferSSL", mongodbtests.StatefulSetBecomesReady(ctx, &mdb)) t.Run("Wait for TLS to be enabled", tester.HasTlsMode("preferSSL", 60, WithoutTls())) }) // Ensure MongoDB is reachable both with and without TLS t.Run("Test Basic Connectivity", tester.ConnectivitySucceeds(WithoutTls())) - t.Run("Test Basic TLS Connectivity", tester.ConnectivitySucceeds(WithTls(mdb))) - t.Run("Internal cluster keyfile authentication is enabled", tester.HasKeyfileAuth(3, WithTls(mdb))) + t.Run("Test Basic TLS Connectivity", tester.ConnectivitySucceeds(WithTls(ctx, mdb))) + t.Run("Internal cluster keyfile authentication is enabled", tester.HasKeyfileAuth(3, WithTls(ctx, mdb))) // Make TLS required t.Run("MongoDB is reachable over TLS while making TLS required", func(t *testing.T) { - defer tester.StartBackgroundConnectivityTest(t, time.Second*10, WithTls(mdb))() - t.Run("Make TLS required", tlstests.EnableTLS(&mdb, false)) - t.Run("Stateful Set Reaches Ready State, after setting TLS to requireSSL", mongodbtests.StatefulSetBecomesReady(&mdb)) - t.Run("Wait for TLS to be required", tester.HasTlsMode("requireSSL", 120, WithTls(mdb))) + defer tester.StartBackgroundConnectivityTest(t, time.Second*10, WithTls(ctx, mdb))() + t.Run("Make TLS required", tlstests.EnableTLS(ctx, &mdb, false)) + t.Run("Stateful Set Reaches Ready State, after setting TLS to requireSSL", mongodbtests.StatefulSetBecomesReady(ctx, &mdb)) + t.Run("Wait for TLS to be required", tester.HasTlsMode("requireSSL", 120, WithTls(ctx, mdb))) }) // Ensure MongoDB is reachable only over TLS - t.Run("Test Basic TLS Connectivity", tester.ConnectivitySucceeds(WithTls(mdb))) + t.Run("Test Basic TLS Connectivity", tester.ConnectivitySucceeds(WithTls(ctx, mdb))) t.Run("Test TLS Required For Connectivity", tester.ConnectivityFails(WithoutTls())) } diff --git a/test/e2e/replica_set_x509/replica_set_x509_test.go b/test/e2e/replica_set_x509/replica_set_x509_test.go index ec45e34b8..a7ed3503c 100644 --- a/test/e2e/replica_set_x509/replica_set_x509_test.go +++ b/test/e2e/replica_set_x509/replica_set_x509_test.go @@ -28,19 +28,20 @@ func TestMain(m *testing.M) { } func TestReplicaSetX509(t *testing.T) { + ctx := context.Background() resourceName := "mdb-tls" helmArgs := []setup.HelmArg{ {Name: "resource.tls.useX509", Value: "true"}, {Name: "resource.tls.sampleX509User", Value: "true"}, } - ctx, testConfig := setup.SetupWithTLS(t, resourceName, helmArgs...) - defer ctx.Teardown() + testCtx, testConfig := setup.SetupWithTLS(ctx, t, resourceName, helmArgs...) + defer testCtx.Teardown() - mdb, _ := e2eutil.NewTestMongoDB(ctx, resourceName, testConfig.Namespace) + mdb, _ := e2eutil.NewTestMongoDB(testCtx, resourceName, testConfig.Namespace) mdb.Spec.Security.Authentication.Modes = []v1.AuthMode{"X509"} mdb.Spec.Security.TLS = e2eutil.NewTestTLSConfig(false) - tester, err := FromX509Resource(t, mdb) + tester, err := FromX509Resource(ctx, t, mdb) if err != nil { t.Fatal(err) } @@ -51,55 +52,55 @@ func TestReplicaSetX509(t *testing.T) { } users := mdb.GetAuthUsers() - t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) - t.Run("Basic tests", mongodbtests.BasicFunctionalityX509(&mdb)) - t.Run("Agent certificate secrets configured", mongodbtests.AgentX509SecretsExists(&mdb)) + t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, testCtx)) + t.Run("Basic tests", mongodbtests.BasicFunctionalityX509(ctx, &mdb)) + t.Run("Agent certificate secrets configured", mongodbtests.AgentX509SecretsExists(ctx, &mdb)) - cert, root, dir := createCerts(t, &mdb) + cert, root, dir := createCerts(ctx, t, &mdb) defer os.RemoveAll(dir) - t.Run("Connectivity Fails without certs", tester.ConnectivityFails(WithURI(mongodbtests.GetConnectionStringForUser(mdb, users[0])), WithTls(mdb))) - t.Run("Connectivity Fails with invalid certs", tester.ConnectivityFails(WithURI(fmt.Sprintf("%s&tlsCAFile=%s&tlsCertificateKeyFile=%s", mongodbtests.GetConnectionStringForUser(mdb, users[0]), root, cert)))) + t.Run("Connectivity Fails without certs", tester.ConnectivityFails(WithURI(mongodbtests.GetConnectionStringForUser(ctx, mdb, users[0])), WithTls(ctx, mdb))) + t.Run("Connectivity Fails with invalid certs", tester.ConnectivityFails(WithURI(fmt.Sprintf("%s&tlsCAFile=%s&tlsCertificateKeyFile=%s", mongodbtests.GetConnectionStringForUser(ctx, mdb, users[0]), root, cert)))) }) t.Run("Connection with valid certificate", func(t *testing.T) { t.Run("Update MongoDB Resource", func(t *testing.T) { - err := e2eutil.UpdateMongoDBResource(&mdb, func(m *v1.MongoDBCommunity) { + err := e2eutil.UpdateMongoDBResource(ctx, &mdb, func(m *v1.MongoDBCommunity) { m.Spec.Users = []v1.MongoDBUser{getValidUser()} }) assert.NoError(t, err) }) - cert, root, dir := createCerts(t, &mdb) + cert, root, dir := createCerts(ctx, t, &mdb) defer os.RemoveAll(dir) users := mdb.GetAuthUsers() - t.Run("Basic tests", mongodbtests.BasicFunctionalityX509(&mdb)) - t.Run("Agent certificate secrets configured", mongodbtests.AgentX509SecretsExists(&mdb)) - t.Run("Connectivity Succeeds", tester.ConnectivitySucceeds(WithURI(fmt.Sprintf("%s&tlsCAFile=%s&tlsCertificateKeyFile=%s", mongodbtests.GetConnectionStringForUser(mdb, users[0]), root, cert)))) + t.Run("Basic tests", mongodbtests.BasicFunctionalityX509(ctx, &mdb)) + t.Run("Agent certificate secrets configured", mongodbtests.AgentX509SecretsExists(ctx, &mdb)) + t.Run("Connectivity Succeeds", tester.ConnectivitySucceeds(WithURI(fmt.Sprintf("%s&tlsCAFile=%s&tlsCertificateKeyFile=%s", mongodbtests.GetConnectionStringForUser(ctx, mdb, users[0]), root, cert)))) }) t.Run("Rotate agent certificate", func(t *testing.T) { - agentCert, err := GetAgentCert(mdb) + agentCert, err := GetAgentCert(ctx, mdb) if err != nil { t.Fatal(err) } initialCertSerialNumber := agentCert.SerialNumber initialAgentPem := &corev1.Secret{} - err = e2eutil.TestClient.Get(context.TODO(), mdb.AgentCertificatePemSecretNamespacedName(), initialAgentPem) + err = e2eutil.TestClient.Get(ctx, mdb.AgentCertificatePemSecretNamespacedName(), initialAgentPem) assert.NoError(t, err) - cert, root, dir := createCerts(t, &mdb) + cert, root, dir := createCerts(ctx, t, &mdb) defer os.RemoveAll(dir) users := mdb.GetAuthUsers() - t.Run("Update certificate secret", tlstests.RotateAgentCertificate(&mdb)) - t.Run("Wait for MongoDB to reach Running Phase after rotating agent cert", mongodbtests.MongoDBReachesRunningPhase(&mdb)) + t.Run("Update certificate secret", tlstests.RotateAgentCertificate(ctx, &mdb)) + t.Run("Wait for MongoDB to reach Running Phase after rotating agent cert", mongodbtests.MongoDBReachesRunningPhase(ctx, &mdb)) - agentCert, err = GetAgentCert(mdb) + agentCert, err = GetAgentCert(ctx, mdb) if err != nil { t.Fatal(err) } @@ -108,48 +109,48 @@ func TestReplicaSetX509(t *testing.T) { assert.NotEqual(t, finalCertSerialNumber, initialCertSerialNumber) finalAgentPem := &corev1.Secret{} - err = e2eutil.TestClient.Get(context.TODO(), mdb.AgentCertificatePemSecretNamespacedName(), finalAgentPem) + err = e2eutil.TestClient.Get(ctx, mdb.AgentCertificatePemSecretNamespacedName(), finalAgentPem) assert.NoError(t, err) assert.NotEqual(t, finalAgentPem.Data, initialAgentPem.Data) - t.Run("Connectivity Succeeds", tester.ConnectivitySucceeds(WithURI(fmt.Sprintf("%s&tlsCAFile=%s&tlsCertificateKeyFile=%s", mongodbtests.GetConnectionStringForUser(mdb, users[0]), root, cert)))) + t.Run("Connectivity Succeeds", tester.ConnectivitySucceeds(WithURI(fmt.Sprintf("%s&tlsCAFile=%s&tlsCertificateKeyFile=%s", mongodbtests.GetConnectionStringForUser(ctx, mdb, users[0]), root, cert)))) }) t.Run("Transition to also allow SCRAM", func(t *testing.T) { t.Run("Update MongoDB Resource", func(t *testing.T) { - err := e2eutil.UpdateMongoDBResource(&mdb, func(m *v1.MongoDBCommunity) { + err := e2eutil.UpdateMongoDBResource(ctx, &mdb, func(m *v1.MongoDBCommunity) { m.Spec.Security.Authentication.Modes = []v1.AuthMode{"X509", "SCRAM"} m.Spec.Security.Authentication.AgentMode = "X509" }) assert.NoError(t, err) }) - cert, root, dir := createCerts(t, &mdb) + cert, root, dir := createCerts(ctx, t, &mdb) defer os.RemoveAll(dir) users := mdb.GetAuthUsers() - t.Run("Basic tests", mongodbtests.BasicFunctionalityX509(&mdb)) - t.Run("Agent certificate secrets configured", mongodbtests.AgentX509SecretsExists(&mdb)) - t.Run("Connectivity Succeeds", tester.ConnectivitySucceeds(WithURI(fmt.Sprintf("%s&tlsCAFile=%s&tlsCertificateKeyFile=%s", mongodbtests.GetConnectionStringForUser(mdb, users[0]), root, cert)))) + t.Run("Basic tests", mongodbtests.BasicFunctionalityX509(ctx, &mdb)) + t.Run("Agent certificate secrets configured", mongodbtests.AgentX509SecretsExists(ctx, &mdb)) + t.Run("Connectivity Succeeds", tester.ConnectivitySucceeds(WithURI(fmt.Sprintf("%s&tlsCAFile=%s&tlsCertificateKeyFile=%s", mongodbtests.GetConnectionStringForUser(ctx, mdb, users[0]), root, cert)))) }) t.Run("Transition to SCRAM agent", func(t *testing.T) { t.Run("Update MongoDB Resource", func(t *testing.T) { - err := e2eutil.UpdateMongoDBResource(&mdb, func(m *v1.MongoDBCommunity) { + err := e2eutil.UpdateMongoDBResource(ctx, &mdb, func(m *v1.MongoDBCommunity) { m.Spec.Security.Authentication.AgentMode = "SCRAM" }) assert.NoError(t, err) }) - cert, root, dir := createCerts(t, &mdb) + cert, root, dir := createCerts(ctx, t, &mdb) defer os.RemoveAll(dir) users := mdb.GetAuthUsers() - t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb)) - t.Run("Connectivity Succeeds", tester.ConnectivitySucceeds(WithURI(fmt.Sprintf("%s&tlsCAFile=%s&tlsCertificateKeyFile=%s", mongodbtests.GetConnectionStringForUser(mdb, users[0]), root, cert)))) + t.Run("Basic tests", mongodbtests.BasicFunctionality(ctx, &mdb)) + t.Run("Connectivity Succeeds", tester.ConnectivitySucceeds(WithURI(fmt.Sprintf("%s&tlsCAFile=%s&tlsCertificateKeyFile=%s", mongodbtests.GetConnectionStringForUser(ctx, mdb, users[0]), root, cert)))) }) } @@ -196,13 +197,13 @@ func getInvalidUser() v1.MongoDBUser { } } -func createCerts(t *testing.T, mdb *v1.MongoDBCommunity) (string, string, string) { +func createCerts(ctx context.Context, t *testing.T, mdb *v1.MongoDBCommunity) (string, string, string) { dir, _ := os.MkdirTemp("", "certdir") t.Logf("Creating client certificate pem file") cert, _ := os.CreateTemp(dir, "pem") clientCertSecret := corev1.Secret{} - err := e2eutil.TestClient.Get(context.TODO(), types.NamespacedName{ + err := e2eutil.TestClient.Get(ctx, types.NamespacedName{ Namespace: mdb.Namespace, Name: "my-x509-user-cert", }, &clientCertSecret) diff --git a/test/e2e/setup/setup.go b/test/e2e/setup/setup.go index 6c42e9888..22e87f253 100644 --- a/test/e2e/setup/setup.go +++ b/test/e2e/setup/setup.go @@ -41,23 +41,23 @@ const ( Pem tlsSecretType = "PEM" ) -func Setup(t *testing.T) *e2eutil.Context { - ctx, err := e2eutil.NewContext(t, envvar.ReadBool(performCleanupEnv)) +func Setup(ctx context.Context, t *testing.T) *e2eutil.TestContext { + testCtx, err := e2eutil.NewContext(ctx, t, envvar.ReadBool(performCleanupEnv)) if err != nil { t.Fatal(err) } config := LoadTestConfigFromEnv() - if err := DeployOperator(config, "mdb", false, false); err != nil { + if err := DeployOperator(ctx, config, "mdb", false, false); err != nil { t.Fatal(err) } - return ctx + return testCtx } -func SetupWithTLS(t *testing.T, resourceName string, additionalHelmArgs ...HelmArg) (*e2eutil.Context, TestConfig) { - ctx, err := e2eutil.NewContext(t, envvar.ReadBool(performCleanupEnv)) +func SetupWithTLS(ctx context.Context, t *testing.T, resourceName string, additionalHelmArgs ...HelmArg) (*e2eutil.TestContext, TestConfig) { + textCtx, err := e2eutil.NewContext(ctx, t, envvar.ReadBool(performCleanupEnv)) if err != nil { t.Fatal(err) @@ -68,15 +68,15 @@ func SetupWithTLS(t *testing.T, resourceName string, additionalHelmArgs ...HelmA t.Fatal(err) } - if err := DeployOperator(config, resourceName, true, false, additionalHelmArgs...); err != nil { + if err := DeployOperator(ctx, config, resourceName, true, false, additionalHelmArgs...); err != nil { t.Fatal(err) } - return ctx, config + return textCtx, config } -func SetupWithTestConfig(t *testing.T, testConfig TestConfig, withTLS, defaultOperator bool, resourceName string) *e2eutil.Context { - ctx, err := e2eutil.NewContext(t, envvar.ReadBool(performCleanupEnv)) +func SetupWithTestConfig(ctx context.Context, t *testing.T, testConfig TestConfig, withTLS, defaultOperator bool, resourceName string) *e2eutil.TestContext { + testCtx, err := e2eutil.NewContext(ctx, t, envvar.ReadBool(performCleanupEnv)) if err != nil { t.Fatal(err) @@ -88,15 +88,15 @@ func SetupWithTestConfig(t *testing.T, testConfig TestConfig, withTLS, defaultOp } } - if err := DeployOperator(testConfig, resourceName, withTLS, defaultOperator); err != nil { + if err := DeployOperator(ctx, testConfig, resourceName, withTLS, defaultOperator); err != nil { t.Fatal(err) } - return ctx + return testCtx } // GeneratePasswordForUser will create a secret with a password for the given user -func GeneratePasswordForUser(ctx *e2eutil.Context, mdbu mdbv1.MongoDBUser, namespace string) (string, error) { +func GeneratePasswordForUser(testCtx *e2eutil.TestContext, mdbu mdbv1.MongoDBUser, namespace string) (string, error) { passwordKey := mdbu.PasswordSecretRef.Key if passwordKey == "" { passwordKey = "password" @@ -119,7 +119,7 @@ func GeneratePasswordForUser(ctx *e2eutil.Context, mdbu mdbv1.MongoDBUser, names SetLabels(e2eutil.TestLabels()). Build() - return password, e2eutil.TestClient.Create(context.TODO(), &passwordSecret, &e2eutil.CleanupOptions{TestContext: ctx}) + return password, e2eutil.TestClient.Create(testCtx.Ctx, &passwordSecret, &e2eutil.CleanupOptions{TestContext: testCtx}) } // extractRegistryNameAndVersion splits a full image string and returns the individual components. @@ -186,7 +186,7 @@ func getHelmArgs(testConfig TestConfig, watchNamespace string, resourceName stri } // DeployOperator installs all resources required by the operator using helm. -func DeployOperator(config TestConfig, resourceName string, withTLS bool, defaultOperator bool, additionalHelmArgs ...HelmArg) error { +func DeployOperator(ctx context.Context, config TestConfig, resourceName string, withTLS bool, defaultOperator bool, additionalHelmArgs ...HelmArg) error { e2eutil.OperatorNamespace = config.Namespace fmt.Printf("Setting operator namespace to %s\n", e2eutil.OperatorNamespace) watchNamespace := config.Namespace @@ -218,7 +218,7 @@ func DeployOperator(config TestConfig, resourceName string, withTLS bool, defaul return err } - dep, err := waite2e.ForDeploymentToExist("mongodb-kubernetes-operator", time.Second*10, time.Minute*1, e2eutil.OperatorNamespace) + dep, err := waite2e.ForDeploymentToExist(ctx, "mongodb-kubernetes-operator", time.Second*10, time.Minute*1, e2eutil.OperatorNamespace) if err != nil { return err } @@ -232,12 +232,12 @@ func DeployOperator(config TestConfig, resourceName string, withTLS bool, defaul cont.Resources.Requests["cpu"] = quantityCPU } - err = e2eutil.TestClient.Update(context.TODO(), &dep) + err = e2eutil.TestClient.Update(ctx, &dep) if err != nil { return err } - if err := wait.PollImmediate(time.Second, 60*time.Second, hasDeploymentRequiredReplicas(&dep)); err != nil { + if err := wait.PollUntilContextTimeout(ctx, time.Second, 60*time.Second, true, hasDeploymentRequiredReplicas(&dep)); err != nil { return errors.New("error building operator deployment: the deployment does not have the required replicas") } fmt.Println("Successfully installed the operator deployment") @@ -265,9 +265,9 @@ func deployCertManager(config TestConfig) error { // hasDeploymentRequiredReplicas returns a condition function that indicates whether the given deployment // currently has the required amount of replicas in the ready state as specified in spec.replicas -func hasDeploymentRequiredReplicas(dep *appsv1.Deployment) wait.ConditionFunc { - return func() (bool, error) { - err := e2eutil.TestClient.Get(context.TODO(), +func hasDeploymentRequiredReplicas(dep *appsv1.Deployment) wait.ConditionWithContextFunc { + return func(ctx context.Context) (bool, error) { + err := e2eutil.TestClient.Get(ctx, types.NamespacedName{Name: dep.Name, Namespace: e2eutil.OperatorNamespace}, dep) diff --git a/test/e2e/statefulset_arbitrary_config/statefulset_arbitrary_config_test.go b/test/e2e/statefulset_arbitrary_config/statefulset_arbitrary_config_test.go index 3a4f48e21..d622cc68d 100644 --- a/test/e2e/statefulset_arbitrary_config/statefulset_arbitrary_config_test.go +++ b/test/e2e/statefulset_arbitrary_config/statefulset_arbitrary_config_test.go @@ -1,6 +1,7 @@ package statefulset_arbitrary_config_update import ( + "context" "fmt" "os" "reflect" @@ -10,7 +11,7 @@ import ( e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/mongodbtests" - setup "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" + "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" ) @@ -24,12 +25,13 @@ func TestMain(m *testing.M) { } func TestStatefulSetArbitraryConfig(t *testing.T) { - ctx := setup.Setup(t) - defer ctx.Teardown() + ctx := context.Background() + testCtx := setup.Setup(ctx, t) + defer testCtx.Teardown() - mdb, user := e2eutil.NewTestMongoDB(ctx, "mdb0", "") + mdb, user := e2eutil.NewTestMongoDB(testCtx, "mdb0", "") - _, err := setup.GeneratePasswordForUser(ctx, user, "") + _, err := setup.GeneratePasswordForUser(testCtx, user, "") if err != nil { t.Fatal(err) } @@ -54,20 +56,20 @@ func TestStatefulSetArbitraryConfig(t *testing.T) { customServiceName := "database" mdb.Spec.StatefulSetConfiguration.SpecWrapper.Spec.ServiceName = customServiceName - tester, err := mongotester.FromResource(t, mdb) + tester, err := mongotester.FromResource(ctx, t, mdb) if err != nil { t.Fatal(err) } - t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) - t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb)) - t.Run("Test setting Service Name", mongodbtests.ServiceWithNameExists(customServiceName, mdb.Namespace)) + t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, testCtx)) + t.Run("Basic tests", mongodbtests.BasicFunctionality(ctx, &mdb)) + t.Run("Test setting Service Name", mongodbtests.ServiceWithNameExists(ctx, customServiceName, mdb.Namespace)) t.Run("Test Basic Connectivity", tester.ConnectivitySucceeds()) - t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 1)) - t.Run("Container has been merged by name", mongodbtests.StatefulSetContainerConditionIsTrue(&mdb, "mongodb-agent", func(container corev1.Container) bool { + t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(ctx, &mdb, 1)) + t.Run("Container has been merged by name", mongodbtests.StatefulSetContainerConditionIsTrue(ctx, &mdb, "mongodb-agent", func(container corev1.Container) bool { return container.ReadinessProbe.TimeoutSeconds == 100 })) - t.Run("Tolerations have been added correctly", mongodbtests.StatefulSetConditionIsTrue(&mdb, func(sts appsv1.StatefulSet) bool { + t.Run("Tolerations have been added correctly", mongodbtests.StatefulSetConditionIsTrue(ctx, &mdb, func(sts appsv1.StatefulSet) bool { return reflect.DeepEqual(overrideTolerations, sts.Spec.Template.Spec.Tolerations) })) } diff --git a/test/e2e/statefulset_arbitrary_config_update/statefulset_arbitrary_config_update_test.go b/test/e2e/statefulset_arbitrary_config_update/statefulset_arbitrary_config_update_test.go index 11b97d806..051189946 100644 --- a/test/e2e/statefulset_arbitrary_config_update/statefulset_arbitrary_config_update_test.go +++ b/test/e2e/statefulset_arbitrary_config_update/statefulset_arbitrary_config_update_test.go @@ -1,6 +1,7 @@ package statefulset_arbitrary_config import ( + "context" "fmt" "os" "reflect" @@ -11,7 +12,7 @@ import ( mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/mongodbtests" - setup "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" + "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" "github.com/stretchr/testify/assert" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -26,25 +27,26 @@ func TestMain(m *testing.M) { } func TestStatefulSetArbitraryConfig(t *testing.T) { - ctx := setup.Setup(t) - defer ctx.Teardown() + ctx := context.Background() + testCtx := setup.Setup(ctx, t) + defer testCtx.Teardown() - mdb, user := e2eutil.NewTestMongoDB(ctx, "mdb0", "") + mdb, user := e2eutil.NewTestMongoDB(testCtx, "mdb0", "") - _, err := setup.GeneratePasswordForUser(ctx, user, "") + _, err := setup.GeneratePasswordForUser(testCtx, user, "") if err != nil { t.Fatal(err) } - tester, err := mongotester.FromResource(t, mdb) + tester, err := mongotester.FromResource(ctx, t, mdb) if err != nil { t.Fatal(err) } - t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) - t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb)) + t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, testCtx)) + t.Run("Basic tests", mongodbtests.BasicFunctionality(ctx, &mdb)) t.Run("Test basic connectivity", tester.ConnectivitySucceeds()) - t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 1)) + t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(ctx, &mdb, 1)) overrideTolerations := []corev1.Toleration{ { @@ -64,16 +66,16 @@ func TestStatefulSetArbitraryConfig(t *testing.T) { overrideSpec.SpecWrapper.Spec.Template.Spec.Containers[1].ReadinessProbe = &corev1.Probe{TimeoutSeconds: 100} overrideSpec.SpecWrapper.Spec.Template.Spec.Tolerations = overrideTolerations - err = e2eutil.UpdateMongoDBResource(&mdb, func(mdb *mdbv1.MongoDBCommunity) { mdb.Spec.StatefulSetConfiguration = overrideSpec }) + err = e2eutil.UpdateMongoDBResource(ctx, &mdb, func(mdb *mdbv1.MongoDBCommunity) { mdb.Spec.StatefulSetConfiguration = overrideSpec }) assert.NoError(t, err) - t.Run("Basic tests after update", mongodbtests.BasicFunctionality(&mdb)) + t.Run("Basic tests after update", mongodbtests.BasicFunctionality(ctx, &mdb)) t.Run("Test basic connectivity after update", tester.ConnectivitySucceeds()) - t.Run("Container has been merged by name", mongodbtests.StatefulSetContainerConditionIsTrue(&mdb, "mongodb-agent", func(container corev1.Container) bool { + t.Run("Container has been merged by name", mongodbtests.StatefulSetContainerConditionIsTrue(ctx, &mdb, "mongodb-agent", func(container corev1.Container) bool { return container.ReadinessProbe.TimeoutSeconds == 100 })) - t.Run("Tolerations have been added correctly", mongodbtests.StatefulSetConditionIsTrue(&mdb, func(sts appsv1.StatefulSet) bool { + t.Run("Tolerations have been added correctly", mongodbtests.StatefulSetConditionIsTrue(ctx, &mdb, func(sts appsv1.StatefulSet) bool { return reflect.DeepEqual(overrideTolerations, sts.Spec.Template.Spec.Tolerations) })) } diff --git a/test/e2e/statefulset_delete/statefulset_delete_test.go b/test/e2e/statefulset_delete/statefulset_delete_test.go index ad2a73f50..686d50071 100644 --- a/test/e2e/statefulset_delete/statefulset_delete_test.go +++ b/test/e2e/statefulset_delete/statefulset_delete_test.go @@ -1,6 +1,7 @@ package statefulset_delete import ( + "context" "fmt" "os" "testing" @@ -8,7 +9,7 @@ import ( mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/mongodbtests" - setup "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" + "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" ) func TestMain(m *testing.M) { @@ -20,29 +21,29 @@ func TestMain(m *testing.M) { } func TestStatefulSetDelete(t *testing.T) { - ctx := setup.Setup(t) - defer ctx.Teardown() + ctx := context.Background() + testCtx := setup.Setup(ctx, t) + defer testCtx.Teardown() - mdb, user := e2eutil.NewTestMongoDB(ctx, "mdb0", "") + mdb, user := e2eutil.NewTestMongoDB(testCtx, "mdb0", "") - _, err := setup.GeneratePasswordForUser(ctx, user, "") + _, err := setup.GeneratePasswordForUser(testCtx, user, "") if err != nil { t.Fatal(err) } - t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) - t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb)) + t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, testCtx)) + t.Run("Basic tests", mongodbtests.BasicFunctionality(ctx, &mdb)) t.Run("Operator recreates StatefulSet", func(t *testing.T) { - t.Run("Delete Statefulset", mongodbtests.DeleteStatefulSet(&mdb)) - t.Run("Test Replica Set Recovers", mongodbtests.StatefulSetBecomesReady(&mdb)) - t.Run("MongoDB Reaches Running Phase", mongodbtests.MongoDBReachesRunningPhase(&mdb)) - t.Run("Test Status Was Updated", mongodbtests.Status(&mdb, - mdbv1.MongoDBCommunityStatus{ - MongoURI: mdb.MongoURI(""), - Phase: mdbv1.Running, - Version: mdb.GetMongoDBVersion(nil), - CurrentMongoDBMembers: mdb.DesiredReplicas(), - })) + t.Run("Delete Statefulset", mongodbtests.DeleteStatefulSet(ctx, &mdb)) + t.Run("Test Replica Set Recovers", mongodbtests.StatefulSetBecomesReady(ctx, &mdb)) + t.Run("MongoDB Reaches Running Phase", mongodbtests.MongoDBReachesRunningPhase(ctx, &mdb)) + t.Run("Test Status Was Updated", mongodbtests.Status(ctx, &mdb, mdbv1.MongoDBCommunityStatus{ + MongoURI: mdb.MongoURI(""), + Phase: mdbv1.Running, + Version: mdb.GetMongoDBVersion(nil), + CurrentMongoDBMembers: mdb.DesiredReplicas(), + })) }) } diff --git a/test/e2e/tlstests/tlstests.go b/test/e2e/tlstests/tlstests.go index ac6156a17..6d327ec0d 100644 --- a/test/e2e/tlstests/tlstests.go +++ b/test/e2e/tlstests/tlstests.go @@ -18,9 +18,9 @@ import ( ) // EnableTLS will upgrade an existing TLS cluster to use TLS. -func EnableTLS(mdb *mdbv1.MongoDBCommunity, optional bool) func(*testing.T) { +func EnableTLS(ctx context.Context, mdb *mdbv1.MongoDBCommunity, optional bool) func(*testing.T) { return func(t *testing.T) { - err := e2eutil.UpdateMongoDBResource(mdb, func(db *mdbv1.MongoDBCommunity) { + err := e2eutil.UpdateMongoDBResource(ctx, mdb, func(db *mdbv1.MongoDBCommunity) { db.Spec.Security.TLS = e2eutil.NewTestTLSConfig(optional) }) if err != nil { @@ -29,7 +29,7 @@ func EnableTLS(mdb *mdbv1.MongoDBCommunity, optional bool) func(*testing.T) { } } -func ExtendCACertificate(mdb *mdbv1.MongoDBCommunity) func(*testing.T) { +func ExtendCACertificate(ctx context.Context, mdb *mdbv1.MongoDBCommunity) func(*testing.T) { return func(t *testing.T) { certGVR := schema.GroupVersionResource{ Group: "cert-manager.io", @@ -56,44 +56,44 @@ func ExtendCACertificate(mdb *mdbv1.MongoDBCommunity) func(*testing.T) { } payload, err := json.Marshal(patch) assert.NoError(t, err) - _, err = caCertificateClient.Patch(context.TODO(), "tls-selfsigned-ca", types.JSONPatchType, payload, metav1.PatchOptions{}) + _, err = caCertificateClient.Patch(ctx, "tls-selfsigned-ca", types.JSONPatchType, payload, metav1.PatchOptions{}) assert.NoError(t, err) } } -func RotateCertificate(mdb *mdbv1.MongoDBCommunity) func(*testing.T) { +func RotateCertificate(ctx context.Context, mdb *mdbv1.MongoDBCommunity) func(*testing.T) { return func(t *testing.T) { certKeySecretName := mdb.TLSSecretNamespacedName() - rotateCertManagerSecret(certKeySecretName, t) + rotateCertManagerSecret(ctx, certKeySecretName, t) } } -func RotateAgentCertificate(mdb *mdbv1.MongoDBCommunity) func(*testing.T) { +func RotateAgentCertificate(ctx context.Context, mdb *mdbv1.MongoDBCommunity) func(*testing.T) { return func(t *testing.T) { agentCertSecretName := mdb.AgentCertificateSecretNamespacedName() - rotateCertManagerSecret(agentCertSecretName, t) + rotateCertManagerSecret(ctx, agentCertSecretName, t) } } -func RotateCACertificate(mdb *mdbv1.MongoDBCommunity) func(*testing.T) { +func RotateCACertificate(ctx context.Context, mdb *mdbv1.MongoDBCommunity) func(*testing.T) { return func(t *testing.T) { caCertSecretName := mdb.TLSCaCertificateSecretNamespacedName() - rotateCertManagerSecret(caCertSecretName, t) + rotateCertManagerSecret(ctx, caCertSecretName, t) } } -func rotateCertManagerSecret(secretName types.NamespacedName, t *testing.T) { +func rotateCertManagerSecret(ctx context.Context, secretName types.NamespacedName, t *testing.T) { currentSecret := corev1.Secret{} - err := e2eutil.TestClient.Get(context.TODO(), secretName, ¤tSecret) + err := e2eutil.TestClient.Get(ctx, secretName, ¤tSecret) assert.NoError(t, err) // delete current cert secret, cert-manager should generate a new one - err = e2eutil.TestClient.Delete(context.TODO(), ¤tSecret) + err = e2eutil.TestClient.Delete(ctx, ¤tSecret) assert.NoError(t, err) newSecret := corev1.Secret{} - err = wait.Poll(5*time.Second, 1*time.Minute, func() (done bool, err error) { - if err := e2eutil.TestClient.Get(context.TODO(), secretName, &newSecret); err != nil { + err = wait.PollUntilContextTimeout(ctx, 5*time.Second, 1*time.Minute, false, func(ctx context.Context) (done bool, err error) { + if err := e2eutil.TestClient.Get(ctx, secretName, &newSecret); err != nil { return false, nil } return true, nil diff --git a/test/e2e/util/mongotester/mongotester.go b/test/e2e/util/mongotester/mongotester.go index d5e2e526e..58ad54181 100644 --- a/test/e2e/util/mongotester/mongotester.go +++ b/test/e2e/util/mongotester/mongotester.go @@ -29,13 +29,15 @@ import ( ) type Tester struct { + ctx context.Context mongoClient *mongo.Client clientOpts []*options.ClientOptions resource *mdbv1.MongoDBCommunity } -func newTester(mdb *mdbv1.MongoDBCommunity, opts ...*options.ClientOptions) *Tester { +func newTester(ctx context.Context, mdb *mdbv1.MongoDBCommunity, opts ...*options.ClientOptions) *Tester { t := &Tester{ + ctx: ctx, resource: mdb, } t.clientOpts = append(t.clientOpts, opts...) @@ -51,7 +53,7 @@ type OptionApplier interface { // FromResource returns a Tester instance from a MongoDB resource. It infers SCRAM username/password // and the hosts from the resource. -func FromResource(t *testing.T, mdb mdbv1.MongoDBCommunity, opts ...OptionApplier) (*Tester, error) { +func FromResource(ctx context.Context, t *testing.T, mdb mdbv1.MongoDBCommunity, opts ...OptionApplier) (*Tester, error) { var clientOpts []*options.ClientOptions clientOpts = WithHosts(mdb.Hosts("")).ApplyOption(clientOpts...) @@ -62,7 +64,7 @@ func FromResource(t *testing.T, mdb mdbv1.MongoDBCommunity, opts ...OptionApplie if len(users) == 1 { user := users[0] passwordSecret := corev1.Secret{} - err := e2eutil.TestClient.Get(context.TODO(), types.NamespacedName{Name: user.PasswordSecretRef.Name, Namespace: mdb.Namespace}, &passwordSecret) + err := e2eutil.TestClient.Get(ctx, types.NamespacedName{Name: user.PasswordSecretRef.Name, Namespace: mdb.Namespace}, &passwordSecret) if err != nil { return nil, err } @@ -75,10 +77,10 @@ func FromResource(t *testing.T, mdb mdbv1.MongoDBCommunity, opts ...OptionApplie clientOpts = opt.ApplyOption(clientOpts...) } - return newTester(&mdb, clientOpts...), nil + return newTester(ctx, &mdb, clientOpts...), nil } -func FromX509Resource(t *testing.T, mdb mdbv1.MongoDBCommunity, opts ...OptionApplier) (*Tester, error) { +func FromX509Resource(ctx context.Context, t *testing.T, mdb mdbv1.MongoDBCommunity, opts ...OptionApplier) (*Tester, error) { var clientOpts []*options.ClientOptions clientOpts = WithHosts(mdb.Hosts("")).ApplyOption(clientOpts...) @@ -95,7 +97,7 @@ func FromX509Resource(t *testing.T, mdb mdbv1.MongoDBCommunity, opts ...OptionAp clientOpts = opt.ApplyOption(clientOpts...) } - return newTester(&mdb, clientOpts...), nil + return newTester(ctx, &mdb, clientOpts...), nil } // ConnectivitySucceeds performs a basic check that ensures that it is possible @@ -110,7 +112,7 @@ func (m *Tester) ConnectivityFails(opts ...OptionApplier) func(t *testing.T) { return m.connectivityCheck(false, opts...) } -func (m *Tester) ConnectivityRejected(opts ...OptionApplier) func(t *testing.T) { +func (m *Tester) ConnectivityRejected(ctx context.Context, opts ...OptionApplier) func(t *testing.T) { clientOpts := make([]*options.ClientOptions, 0) for _, optApplier := range opts { clientOpts = optApplier.ApplyOption(clientOpts...) @@ -122,7 +124,7 @@ func (m *Tester) ConnectivityRejected(opts ...OptionApplier) func(t *testing.T) t.Skip() } - if err := m.ensureClient(clientOpts...); err == nil { + if err := m.ensureClient(ctx, clientOpts...); err == nil { t.Fatalf("No error, but it should have failed") } } @@ -171,7 +173,7 @@ func (m *Tester) VerifyRoles(expectedRoles []automationconfig.CustomRole, tries return m.hasAdminCommandResult(func(t *testing.T) bool { var result CustomRolesResult err := m.mongoClient.Database("admin"). - RunCommand(context.TODO(), + RunCommand(m.ctx, bson.D{ {Key: "rolesInfo", Value: 1}, {Key: "showPrivileges", Value: true}, @@ -195,7 +197,7 @@ func (m *Tester) hasAdminCommandResult(verify verifyAdminResultFunc, tries int, } return func(t *testing.T) { - if err := m.ensureClient(clientOpts...); err != nil { + if err := m.ensureClient(m.ctx, clientOpts...); err != nil { t.Fatal(err) } @@ -216,7 +218,7 @@ func (m *Tester) hasAdminParameter(key string, expectedValue interface{}, tries return m.hasAdminCommandResult(func(t *testing.T) bool { var result map[string]interface{} err := m.mongoClient.Database("admin"). - RunCommand(context.TODO(), bson.D{{Key: "getParameter", Value: 1}, {Key: key, Value: 1}}). + RunCommand(m.ctx, bson.D{{Key: "getParameter", Value: 1}, {Key: key, Value: 1}}). Decode(&result) if err != nil { t.Logf("Unable to get admin setting %s with error : %s", key, err) @@ -244,16 +246,16 @@ func (m *Tester) connectivityCheck(shouldSucceed bool, opts ...OptionApplier) fu t.Skip() } - ctx, cancel := context.WithTimeout(context.Background(), connectivityOpts.ContextTimeout) + ctx, cancel := context.WithTimeout(m.ctx, connectivityOpts.ContextTimeout) defer cancel() - if err := m.ensureClient(clientOpts...); err != nil { + if err := m.ensureClient(ctx, clientOpts...); err != nil { t.Fatal(err) } attempts := 0 // There can be a short time before the user can auth as the user - err := wait.Poll(connectivityOpts.IntervalTime, connectivityOpts.TimeoutTime, func() (done bool, err error) { + err := wait.PollUntilContextTimeout(ctx, connectivityOpts.IntervalTime, connectivityOpts.TimeoutTime, false, func(ctx context.Context) (done bool, err error) { attempts++ collection := m.mongoClient.Database(connectivityOpts.Database).Collection(connectivityOpts.Collection) _, err = collection.InsertOne(ctx, bson.M{"name": "pi", "value": 3.14159}) @@ -280,7 +282,7 @@ func (m *Tester) connectivityCheck(shouldSucceed bool, opts ...OptionApplier) fu func (m *Tester) WaitForRotatedCertificate(mdb mdbv1.MongoDBCommunity, initialCertSerialNumber *big.Int) func(*testing.T) { return func(t *testing.T) { - tls, err := getClientTLSConfig(mdb) + tls, err := getClientTLSConfig(m.ctx, mdb) assert.NoError(t, err) // Reject all server certificates that don't have the expected serial number @@ -292,13 +294,13 @@ func (m *Tester) WaitForRotatedCertificate(mdb mdbv1.MongoDBCommunity, initialCe return nil } - if err := m.ensureClient(&options.ClientOptions{TLSConfig: tls}); err != nil { + if err := m.ensureClient(m.ctx, &options.ClientOptions{TLSConfig: tls}); err != nil { t.Fatal(err) } // Ping the cluster until it succeeds. The ping will only succeed with the right certificate. - err = wait.Poll(5*time.Second, 5*time.Minute, func() (done bool, err error) { - if err := m.mongoClient.Ping(context.TODO(), nil); err != nil { + err = wait.PollUntilContextTimeout(m.ctx, 5*time.Second, 5*time.Minute, false, func(ctx context.Context) (done bool, err error) { + if err := m.mongoClient.Ping(m.ctx, nil); err != nil { return false, nil } return true, nil @@ -314,7 +316,7 @@ func (m *Tester) WaitForRotatedCertificate(mdb mdbv1.MongoDBCommunity, initialCe func (m *Tester) EnsureMongodConfig(selector string, expected interface{}) func(*testing.T) { return func(t *testing.T) { connectivityOpts := defaults() - err := wait.Poll(connectivityOpts.IntervalTime, connectivityOpts.TimeoutTime, func() (done bool, err error) { + err := wait.PollUntilContextTimeout(m.ctx, connectivityOpts.IntervalTime, connectivityOpts.TimeoutTime, false, func(ctx context.Context) (done bool, err error) { opts, err := m.getCommandLineOptions() assert.NoError(t, err) @@ -334,7 +336,7 @@ func (m *Tester) getCommandLineOptions() (bson.M, error) { var result bson.M err := m.mongoClient. Database("admin"). - RunCommand(context.TODO(), bson.D{primitive.E{Key: "getCmdLineOpts", Value: 1}}). + RunCommand(m.ctx, bson.D{primitive.E{Key: "getCmdLineOpts", Value: 1}}). Decode(&result) return result, err @@ -357,7 +359,7 @@ func bsonToMap(m bson.M) map[string]interface{} { // StartBackgroundConnectivityTest starts periodically checking connectivity to the MongoDB deployment // with the defined interval. A cancel function is returned, which can be called to stop testing connectivity. func (m *Tester) StartBackgroundConnectivityTest(t *testing.T, interval time.Duration, opts ...OptionApplier) func() { - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(m.ctx) t.Logf("Starting background connectivity test") // start a go routine which will periodically check basic MongoDB connectivity @@ -375,17 +377,17 @@ func (m *Tester) StartBackgroundConnectivityTest(t *testing.T, interval time.Dur return func() { cancel() if t != nil { - t.Log("Context cancelled, no longer checking connectivity") + t.Log("TestContext cancelled, no longer checking connectivity") } } } // ensureClient establishes a mongo client connection applying any addition // client options on top of what were provided at construction. -func (t *Tester) ensureClient(opts ...*options.ClientOptions) error { +func (t *Tester) ensureClient(ctx context.Context, opts ...*options.ClientOptions) error { allOpts := t.clientOpts allOpts = append(allOpts, opts...) - mongoClient, err := mongo.Connect(context.TODO(), allOpts...) + mongoClient, err := mongo.Connect(ctx, allOpts...) if err != nil { return err } @@ -406,7 +408,7 @@ func (m *Tester) PrometheusEndpointIsReachable(username, password string, useTls client := &http.Client{Transport: customTransport} return func(t *testing.T) { - _ = wait.Poll(5*time.Second, 60*time.Second, func() (bool, error) { + _ = wait.PollUntilContextTimeout(m.ctx, 5*time.Second, 60*time.Second, false, func(ctx context.Context) (bool, error) { var idx int // Verify that the Prometheus port is enabled and responding with 200 @@ -507,8 +509,8 @@ func WithHosts(hosts []string) OptionApplier { } // WithTls configures the client to use tls -func WithTls(mdb mdbv1.MongoDBCommunity) OptionApplier { - tlsConfig, err := getClientTLSConfig(mdb) +func WithTls(ctx context.Context, mdb mdbv1.MongoDBCommunity) OptionApplier { + tlsConfig, err := getClientTLSConfig(ctx, mdb) if err != nil { panic(fmt.Errorf("could not retrieve TLS config: %s", err)) } @@ -550,10 +552,10 @@ func WithReplicaSet(rsname string) OptionApplier { } // getClientTLSConfig reads in the tls fixtures -func getClientTLSConfig(mdb mdbv1.MongoDBCommunity) (*tls.Config, error) { +func getClientTLSConfig(ctx context.Context, mdb mdbv1.MongoDBCommunity) (*tls.Config, error) { caSecret := corev1.Secret{} caSecretName := types.NamespacedName{Name: mdb.Spec.Security.TLS.CaCertificateSecret.Name, Namespace: mdb.Namespace} - if err := e2eutil.TestClient.Get(context.TODO(), caSecretName, &caSecret); err != nil { + if err := e2eutil.TestClient.Get(ctx, caSecretName, &caSecret); err != nil { return nil, err } caPEM := caSecret.Data["ca.crt"] @@ -566,10 +568,10 @@ func getClientTLSConfig(mdb mdbv1.MongoDBCommunity) (*tls.Config, error) { } // GetAgentCert reads the agent key certificate -func GetAgentCert(mdb mdbv1.MongoDBCommunity) (*x509.Certificate, error) { +func GetAgentCert(ctx context.Context, mdb mdbv1.MongoDBCommunity) (*x509.Certificate, error) { certSecret := corev1.Secret{} certSecretName := mdb.AgentCertificateSecretNamespacedName() - if err := e2eutil.TestClient.Get(context.TODO(), certSecretName, &certSecret); err != nil { + if err := e2eutil.TestClient.Get(ctx, certSecretName, &certSecret); err != nil { return nil, err } block, _ := pem.Decode(certSecret.Data["tls.crt"]) @@ -580,10 +582,10 @@ func GetAgentCert(mdb mdbv1.MongoDBCommunity) (*x509.Certificate, error) { } // GetClientCert reads the client key certificate -func GetClientCert(mdb mdbv1.MongoDBCommunity) (*x509.Certificate, error) { +func GetClientCert(ctx context.Context, mdb mdbv1.MongoDBCommunity) (*x509.Certificate, error) { certSecret := corev1.Secret{} certSecretName := types.NamespacedName{Name: mdb.Spec.Security.TLS.CertificateKeySecret.Name, Namespace: mdb.Namespace} - if err := e2eutil.TestClient.Get(context.TODO(), certSecretName, &certSecret); err != nil { + if err := e2eutil.TestClient.Get(ctx, certSecretName, &certSecret); err != nil { return nil, err } block, _ := pem.Decode(certSecret.Data["tls.crt"]) @@ -593,10 +595,10 @@ func GetClientCert(mdb mdbv1.MongoDBCommunity) (*x509.Certificate, error) { return x509.ParseCertificate(block.Bytes) } -func GetUserCert(mdb mdbv1.MongoDBCommunity, userCertSecret string) (string, error) { +func GetUserCert(ctx context.Context, mdb mdbv1.MongoDBCommunity, userCertSecret string) (string, error) { certSecret := corev1.Secret{} certSecretName := types.NamespacedName{Name: userCertSecret, Namespace: mdb.Namespace} - if err := e2eutil.TestClient.Get(context.TODO(), certSecretName, &certSecret); err != nil { + if err := e2eutil.TestClient.Get(ctx, certSecretName, &certSecret); err != nil { return "", err } crt, _ := pem.Decode(certSecret.Data["tls.crt"]) diff --git a/test/e2e/util/wait/wait.go b/test/e2e/util/wait/wait.go index 19ffc2b57..e599e935c 100644 --- a/test/e2e/util/wait/wait.go +++ b/test/e2e/util/wait/wait.go @@ -26,39 +26,39 @@ const ( // ForConfigMapToExist waits until a ConfigMap of the given name exists // using the provided retryInterval and timeout -func ForConfigMapToExist(cmName string, retryInterval, timeout time.Duration) (corev1.ConfigMap, error) { +func ForConfigMapToExist(ctx context.Context, cmName string, retryInterval, timeout time.Duration) (corev1.ConfigMap, error) { cm := corev1.ConfigMap{} - return cm, waitForRuntimeObjectToExist(cmName, retryInterval, timeout, &cm, e2eutil.OperatorNamespace) + return cm, waitForRuntimeObjectToExist(ctx, cmName, retryInterval, timeout, &cm, e2eutil.OperatorNamespace) } // ForSecretToExist waits until a Secret of the given name exists // using the provided retryInterval and timeout -func ForSecretToExist(cmName string, retryInterval, timeout time.Duration, namespace string) (corev1.Secret, error) { +func ForSecretToExist(ctx context.Context, cmName string, retryInterval, timeout time.Duration, namespace string) (corev1.Secret, error) { s := corev1.Secret{} - return s, waitForRuntimeObjectToExist(cmName, retryInterval, timeout, &s, namespace) + return s, waitForRuntimeObjectToExist(ctx, cmName, retryInterval, timeout, &s, namespace) } // ForMongoDBToReachPhase waits until the given MongoDB resource reaches the expected phase -func ForMongoDBToReachPhase(t *testing.T, mdb *mdbv1.MongoDBCommunity, phase mdbv1.Phase, retryInterval, timeout time.Duration) error { - return waitForMongoDBCondition(mdb, retryInterval, timeout, func(db mdbv1.MongoDBCommunity) bool { +func ForMongoDBToReachPhase(ctx context.Context, t *testing.T, mdb *mdbv1.MongoDBCommunity, phase mdbv1.Phase, retryInterval, timeout time.Duration) error { + return waitForMongoDBCondition(ctx, mdb, retryInterval, timeout, func(db mdbv1.MongoDBCommunity) bool { t.Logf("current phase: %s, waiting for phase: %s", db.Status.Phase, phase) return db.Status.Phase == phase }) } // ForMongoDBMessageStatus waits until the given MongoDB resource gets the expected message status -func ForMongoDBMessageStatus(t *testing.T, mdb *mdbv1.MongoDBCommunity, retryInterval, timeout time.Duration, message string) error { - return waitForMongoDBCondition(mdb, retryInterval, timeout, func(db mdbv1.MongoDBCommunity) bool { +func ForMongoDBMessageStatus(ctx context.Context, t *testing.T, mdb *mdbv1.MongoDBCommunity, retryInterval, timeout time.Duration, message string) error { + return waitForMongoDBCondition(ctx, mdb, retryInterval, timeout, func(db mdbv1.MongoDBCommunity) bool { t.Logf("current message: %s, waiting for message: %s", db.Status.Message, message) return db.Status.Message == message }) } // waitForMongoDBCondition polls and waits for a given condition to be true -func waitForMongoDBCondition(mdb *mdbv1.MongoDBCommunity, retryInterval, timeout time.Duration, condition func(mdbv1.MongoDBCommunity) bool) error { +func waitForMongoDBCondition(ctx context.Context, mdb *mdbv1.MongoDBCommunity, retryInterval, timeout time.Duration, condition func(mdbv1.MongoDBCommunity) bool) error { mdbNew := mdbv1.MongoDBCommunity{} - return wait.Poll(retryInterval, timeout, func() (done bool, err error) { - err = e2eutil.TestClient.Get(context.TODO(), mdb.NamespacedName(), &mdbNew) + return wait.PollUntilContextTimeout(ctx, retryInterval, timeout, false, func(ctx context.Context) (done bool, err error) { + err = e2eutil.TestClient.Get(ctx, mdb.NamespacedName(), &mdbNew) if err != nil { return false, err } @@ -69,72 +69,72 @@ func waitForMongoDBCondition(mdb *mdbv1.MongoDBCommunity, retryInterval, timeout // ForDeploymentToExist waits until a Deployment of the given name exists // using the provided retryInterval and timeout -func ForDeploymentToExist(deployName string, retryInterval, timeout time.Duration, namespace string) (appsv1.Deployment, error) { +func ForDeploymentToExist(ctx context.Context, deployName string, retryInterval, timeout time.Duration, namespace string) (appsv1.Deployment, error) { deploy := appsv1.Deployment{} - return deploy, waitForRuntimeObjectToExist(deployName, retryInterval, timeout, &deploy, namespace) + return deploy, waitForRuntimeObjectToExist(ctx, deployName, retryInterval, timeout, &deploy, namespace) } // ForStatefulSetToExist waits until a StatefulSet of the given name exists // using the provided retryInterval and timeout -func ForStatefulSetToExist(stsName string, retryInterval, timeout time.Duration, namespace string) (appsv1.StatefulSet, error) { +func ForStatefulSetToExist(ctx context.Context, stsName string, retryInterval, timeout time.Duration, namespace string) (appsv1.StatefulSet, error) { sts := appsv1.StatefulSet{} - return sts, waitForRuntimeObjectToExist(stsName, retryInterval, timeout, &sts, namespace) + return sts, waitForRuntimeObjectToExist(ctx, stsName, retryInterval, timeout, &sts, namespace) } // ForStatefulSetToBeDeleted waits until a StatefulSet of the given name is deleted // using the provided retryInterval and timeout -func ForStatefulSetToBeDeleted(stsName string, retryInterval, timeout time.Duration, namespace string) error { +func ForStatefulSetToBeDeleted(ctx context.Context, stsName string, retryInterval, timeout time.Duration, namespace string) error { sts := appsv1.StatefulSet{} - return waitForRuntimeObjectToBeDeleted(stsName, retryInterval, timeout, &sts, namespace) + return waitForRuntimeObjectToBeDeleted(ctx, stsName, retryInterval, timeout, &sts, namespace) } // ForStatefulSetToHaveUpdateStrategy waits until all replicas of the StatefulSet with the given name // have reached the ready status -func ForStatefulSetToHaveUpdateStrategy(t *testing.T, mdb *mdbv1.MongoDBCommunity, strategy appsv1.StatefulSetUpdateStrategyType, opts ...Configuration) error { +func ForStatefulSetToHaveUpdateStrategy(ctx context.Context, t *testing.T, mdb *mdbv1.MongoDBCommunity, strategy appsv1.StatefulSetUpdateStrategyType, opts ...Configuration) error { options := newOptions(opts...) - return waitForStatefulSetCondition(t, mdb, options, func(sts appsv1.StatefulSet) bool { + return waitForStatefulSetCondition(ctx, t, mdb, options, func(sts appsv1.StatefulSet) bool { return sts.Spec.UpdateStrategy.Type == strategy }) } // ForStatefulSetToBeReady waits until all replicas of the StatefulSet with the given name // have reached the ready status -func ForStatefulSetToBeReady(t *testing.T, mdb *mdbv1.MongoDBCommunity, opts ...Configuration) error { +func ForStatefulSetToBeReady(ctx context.Context, t *testing.T, mdb *mdbv1.MongoDBCommunity, opts ...Configuration) error { options := newOptions(opts...) - return waitForStatefulSetCondition(t, mdb, options, func(sts appsv1.StatefulSet) bool { + return waitForStatefulSetCondition(ctx, t, mdb, options, func(sts appsv1.StatefulSet) bool { return statefulset.IsReady(sts, mdb.Spec.Members) }) } // ForStatefulSetToBeUnready waits until all replicas of the StatefulSet with the given name // is not ready. -func ForStatefulSetToBeUnready(t *testing.T, mdb *mdbv1.MongoDBCommunity, opts ...Configuration) error { +func ForStatefulSetToBeUnready(ctx context.Context, t *testing.T, mdb *mdbv1.MongoDBCommunity, opts ...Configuration) error { options := newOptions(opts...) - return waitForStatefulSetCondition(t, mdb, options, func(sts appsv1.StatefulSet) bool { + return waitForStatefulSetCondition(ctx, t, mdb, options, func(sts appsv1.StatefulSet) bool { return !statefulset.IsReady(sts, mdb.Spec.Members) }) } // ForArbitersStatefulSetToBeReady waits until all replicas of the StatefulSet with the given name // have reached the ready status. -func ForArbitersStatefulSetToBeReady(t *testing.T, mdb *mdbv1.MongoDBCommunity, opts ...Configuration) error { +func ForArbitersStatefulSetToBeReady(ctx context.Context, t *testing.T, mdb *mdbv1.MongoDBCommunity, opts ...Configuration) error { options := newOptions(opts...) - return waitForStatefulSetConditionWithSpecificSts(t, mdb, ArbitersStatefulSet, options, func(sts appsv1.StatefulSet) bool { + return waitForStatefulSetConditionWithSpecificSts(ctx, t, mdb, ArbitersStatefulSet, options, func(sts appsv1.StatefulSet) bool { return statefulset.IsReady(sts, mdb.Spec.Arbiters) }) } // ForStatefulSetToBeReadyAfterScaleDown waits for just the ready replicas to be correct // and does not account for the updated replicas -func ForStatefulSetToBeReadyAfterScaleDown(t *testing.T, mdb *mdbv1.MongoDBCommunity, opts ...Configuration) error { +func ForStatefulSetToBeReadyAfterScaleDown(ctx context.Context, t *testing.T, mdb *mdbv1.MongoDBCommunity, opts ...Configuration) error { options := newOptions(opts...) - return waitForStatefulSetCondition(t, mdb, options, func(sts appsv1.StatefulSet) bool { + return waitForStatefulSetCondition(ctx, t, mdb, options, func(sts appsv1.StatefulSet) bool { return int32(mdb.Spec.Members) == sts.Status.ReadyReplicas }) } -func waitForStatefulSetConditionWithSpecificSts(t *testing.T, mdb *mdbv1.MongoDBCommunity, statefulSetType StatefulSetType, waitOpts Options, condition func(set appsv1.StatefulSet) bool) error { - _, err := ForStatefulSetToExist(mdb.Name, waitOpts.RetryInterval, waitOpts.Timeout, mdb.Namespace) +func waitForStatefulSetConditionWithSpecificSts(ctx context.Context, t *testing.T, mdb *mdbv1.MongoDBCommunity, statefulSetType StatefulSetType, waitOpts Options, condition func(set appsv1.StatefulSet) bool) error { + _, err := ForStatefulSetToExist(ctx, mdb.Name, waitOpts.RetryInterval, waitOpts.Timeout, mdb.Namespace) if err != nil { return fmt.Errorf("error waiting for stateful set to be created: %s", err) } @@ -144,8 +144,8 @@ func waitForStatefulSetConditionWithSpecificSts(t *testing.T, mdb *mdbv1.MongoDB if statefulSetType == ArbitersStatefulSet { name = mdb.ArbiterNamespacedName() } - return wait.Poll(waitOpts.RetryInterval, waitOpts.Timeout, func() (done bool, err error) { - err = e2eutil.TestClient.Get(context.TODO(), name, &sts) + return wait.PollUntilContextTimeout(ctx, waitOpts.RetryInterval, waitOpts.Timeout, false, func(ctx context.Context) (done bool, err error) { + err = e2eutil.TestClient.Get(ctx, name, &sts) if err != nil { return false, err } @@ -156,19 +156,19 @@ func waitForStatefulSetConditionWithSpecificSts(t *testing.T, mdb *mdbv1.MongoDB }) } -func waitForStatefulSetCondition(t *testing.T, mdb *mdbv1.MongoDBCommunity, waitOpts Options, condition func(set appsv1.StatefulSet) bool) error { +func waitForStatefulSetCondition(ctx context.Context, t *testing.T, mdb *mdbv1.MongoDBCommunity, waitOpts Options, condition func(set appsv1.StatefulSet) bool) error { // uses members statefulset - return waitForStatefulSetConditionWithSpecificSts(t, mdb, MembersStatefulSet, waitOpts, condition) + return waitForStatefulSetConditionWithSpecificSts(ctx, t, mdb, MembersStatefulSet, waitOpts, condition) } -func waitForStatefulSetConditionArbiters(t *testing.T, mdb *mdbv1.MongoDBCommunity, waitOpts Options, condition func(set appsv1.StatefulSet) bool) error { +func waitForStatefulSetConditionArbiters(ctx context.Context, t *testing.T, mdb *mdbv1.MongoDBCommunity, waitOpts Options, condition func(set appsv1.StatefulSet) bool) error { // uses members statefulset - return waitForStatefulSetConditionWithSpecificSts(t, mdb, ArbitersStatefulSet, waitOpts, condition) + return waitForStatefulSetConditionWithSpecificSts(ctx, t, mdb, ArbitersStatefulSet, waitOpts, condition) } -func ForPodReadiness(t *testing.T, isReady bool, containerName string, timeout time.Duration, pod corev1.Pod) error { - return wait.Poll(time.Second*3, timeout, func() (done bool, err error) { - err = e2eutil.TestClient.Get(context.TODO(), types.NamespacedName{Name: pod.Name, Namespace: pod.Namespace}, &pod) +func ForPodReadiness(ctx context.Context, t *testing.T, isReady bool, containerName string, timeout time.Duration, pod corev1.Pod) error { + return wait.PollUntilContextTimeout(ctx, time.Second*3, timeout, false, func(ctx context.Context) (done bool, err error) { + err = e2eutil.TestClient.Get(ctx, types.NamespacedName{Name: pod.Name, Namespace: pod.Namespace}, &pod) if err != nil { return false, err } @@ -182,9 +182,9 @@ func ForPodReadiness(t *testing.T, isReady bool, containerName string, timeout t }) } -func ForPodPhase(t *testing.T, timeout time.Duration, pod corev1.Pod, podPhase corev1.PodPhase) error { - return wait.Poll(time.Second*3, timeout, func() (done bool, err error) { - err = e2eutil.TestClient.Get(context.TODO(), types.NamespacedName{Name: pod.Name, Namespace: pod.Namespace}, &pod) +func ForPodPhase(ctx context.Context, t *testing.T, timeout time.Duration, pod corev1.Pod, podPhase corev1.PodPhase) error { + return wait.PollUntilContextTimeout(ctx, time.Second*3, timeout, false, func(ctx context.Context) (done bool, err error) { + err = e2eutil.TestClient.Get(ctx, types.NamespacedName{Name: pod.Name, Namespace: pod.Namespace}, &pod) if err != nil { return false, err } @@ -195,24 +195,24 @@ func ForPodPhase(t *testing.T, timeout time.Duration, pod corev1.Pod, podPhase c // waitForRuntimeObjectToExist waits until a runtime.Object of the given name exists // using the provided retryInterval and timeout provided. -func waitForRuntimeObjectToExist(name string, retryInterval, timeout time.Duration, obj client.Object, namespace string) error { - return wait.Poll(retryInterval, timeout, func() (done bool, err error) { - return runtimeObjectExists(name, obj, namespace) +func waitForRuntimeObjectToExist(ctx context.Context, name string, retryInterval, timeout time.Duration, obj client.Object, namespace string) error { + return wait.PollUntilContextTimeout(ctx, retryInterval, timeout, false, func(ctx context.Context) (done bool, err error) { + return runtimeObjectExists(ctx, name, obj, namespace) }) } // waitForRuntimeObjectToBeDeleted waits until a runtime.Object of the given name is deleted // using the provided retryInterval and timeout provided. -func waitForRuntimeObjectToBeDeleted(name string, retryInterval, timeout time.Duration, obj client.Object, namespace string) error { - return wait.Poll(retryInterval, timeout, func() (done bool, err error) { - exists, err := runtimeObjectExists(name, obj, namespace) +func waitForRuntimeObjectToBeDeleted(ctx context.Context, name string, retryInterval, timeout time.Duration, obj client.Object, namespace string) error { + return wait.PollUntilContextTimeout(ctx, retryInterval, timeout, false, func(ctx context.Context) (done bool, err error) { + exists, err := runtimeObjectExists(ctx, name, obj, namespace) return !exists, err }) } // runtimeObjectExists checks if a runtime.Object of the given name exists -func runtimeObjectExists(name string, obj client.Object, namespace string) (bool, error) { - err := e2eutil.TestClient.Get(context.TODO(), types.NamespacedName{Name: name, Namespace: namespace}, obj) +func runtimeObjectExists(ctx context.Context, name string, obj client.Object, namespace string) (bool, error) { + err := e2eutil.TestClient.Get(ctx, types.NamespacedName{Name: name, Namespace: namespace}, obj) if err != nil { return false, client.IgnoreNotFound(err) } From ea008d7675fc0fe1809ec0f137a2e2ea3819265f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 19 Apr 2024 16:19:08 +0200 Subject: [PATCH 726/790] Bump golang.org/x/net from 0.19.0 to 0.23.0 (#1529) Bumps [golang.org/x/net](https://github.com/golang/net) from 0.19.0 to 0.23.0. - [Commits](https://github.com/golang/net/compare/v0.19.0...v0.23.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 8 ++++---- go.sum | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 678e785eb..d1c859b5c 100644 --- a/go.mod +++ b/go.mod @@ -65,12 +65,12 @@ require ( github.com/xdg-go/stringprep v1.0.4 // indirect github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect go.uber.org/multierr v1.10.0 // indirect - golang.org/x/crypto v0.17.0 // indirect - golang.org/x/net v0.19.0 // indirect + golang.org/x/crypto v0.21.0 // indirect + golang.org/x/net v0.23.0 // indirect golang.org/x/oauth2 v0.7.0 // indirect golang.org/x/sync v0.5.0 // indirect - golang.org/x/sys v0.15.0 // indirect - golang.org/x/term v0.15.0 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/term v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.3.0 // indirect gomodules.xyz/jsonpatch/v2 v2.3.0 // indirect diff --git a/go.sum b/go.sum index 61801ad94..9faef68f0 100644 --- a/go.sum +++ b/go.sum @@ -178,8 +178,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -198,8 +198,8 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= -golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g= golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= @@ -221,12 +221,12 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= -golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= +golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= From 7cfc2f75f543a0da97e0712acdd5045b7484490b Mon Sep 17 00:00:00 2001 From: Julien-Ben <33035980+Julien-Ben@users.noreply.github.com> Date: Mon, 6 May 2024 13:26:50 +0200 Subject: [PATCH 727/790] Sign images (#1533) * Bump Sonar * Refactor inventories * Refactor agent inventory * Use logger * Sign released images * Add --sign flag to CLI * Update release notes * Use Actions env. variables * Remove the if for testing * Change AWS default region * Artifactory login * Debug env variable access * Fix logging * Try echo variable * Update release-single-image.yml * Replace env with var * Putting back the conditional release * Remove dot after ubi * Fix pipeline arguments * Refactor e2e inventory --- .action_templates/jobs/setup.yaml | 2 +- .action_templates/jobs/tests.yaml | 2 +- .github/workflows/release-images.yml | 8 +- .github/workflows/release-single-image.yml | 7 +- docs/RELEASE_NOTES.md | 8 +- inventories/e2e-inventory.yaml | 43 +-- inventories/operator-inventory.yaml | 133 +------- inventory.yaml | 372 ++------------------- pipeline.py | 93 ++++-- requirements.txt | 4 +- scripts/ci/base_logger.py | 21 ++ scripts/ci/images_signing.py | 208 ++++++++++++ 12 files changed, 367 insertions(+), 534 deletions(-) create mode 100644 scripts/ci/base_logger.py create mode 100644 scripts/ci/images_signing.py diff --git a/.action_templates/jobs/setup.yaml b/.action_templates/jobs/setup.yaml index 66c766062..ad46dc26d 100644 --- a/.action_templates/jobs/setup.yaml +++ b/.action_templates/jobs/setup.yaml @@ -4,7 +4,7 @@ setup: fail-fast: false matrix: include: - - pipeline-argument: mongodb-kubernetes-operator + - pipeline-argument: operator - pipeline-argument: version-upgrade-hook - pipeline-argument: readiness-probe - pipeline-argument: agent diff --git a/.action_templates/jobs/tests.yaml b/.action_templates/jobs/tests.yaml index 229e08117..6128df9ed 100644 --- a/.action_templates/jobs/tests.yaml +++ b/.action_templates/jobs/tests.yaml @@ -10,7 +10,7 @@ tests: - test-name: replica_set_enterprise_upgrade_4_5 distro: ubi - test-name: replica_set_enterprise_upgrade_5_6 - distro: ubi. + distro: ubi - test-name: replica_set_enterprise_upgrade_6_7 distro: ubi - test-name: replica_set_recovery diff --git a/.github/workflows/release-images.yml b/.github/workflows/release-images.yml index f190796b0..af38b3181 100644 --- a/.github/workflows/release-images.yml +++ b/.github/workflows/release-images.yml @@ -56,11 +56,17 @@ jobs: - name: Publish Image To Quay if: steps.release_status.outputs.OUTPUT == 'unreleased' - run: python pipeline.py --image-name ${{ matrix.pipeline-argument }} --release + run: python pipeline.py --image-name ${{ matrix.pipeline-argument }} --release --sign env: MONGODB_COMMUNITY_CONFIG: "${{ github.workspace }}/scripts/ci/config.json" AWS_ACCESS_KEY_ID: "${{ secrets.AWS_ACCESS_KEY_ID }}" AWS_SECRET_ACCESS_KEY: "${{ secrets.AWS_SECRET_ACCESS_KEY }}" + GRS_USERNAME: "${{ vars.GRS_USERNAME }}" + GRS_PASSWORD: "${{ secrets.GRS_PASSWORD }}" + PKCS11_URI: "${{ vars.PKCS11_URI }}" + ARTIFACTORY_USERNAME: "${{ vars.ARTIFACTORY_USERNAME }}" + ARTIFACTORY_PASSWORD: "${{ secrets.ARTIFACTORY_PASSWORD }}" + AWS_DEFAULT_REGION: "${{ vars.AWS_DEFAULT_REGION }}" create-draft-release: runs-on: ubuntu-latest diff --git a/.github/workflows/release-single-image.yml b/.github/workflows/release-single-image.yml index 8641245cd..11bd5683d 100644 --- a/.github/workflows/release-single-image.yml +++ b/.github/workflows/release-single-image.yml @@ -46,8 +46,13 @@ jobs: - name: Publish Image To Quay if: steps.release_status.outputs.OUTPUT == 'unreleased' - run: python pipeline.py --image-name ${{ github.event.inputs.pipeline-argument }} --release + run: python pipeline.py --image-name ${{ github.event.inputs.pipeline-argument }} --release --sign env: MONGODB_COMMUNITY_CONFIG: "${{ github.workspace }}/scripts/ci/config.json" AWS_ACCESS_KEY_ID: "${{ secrets.AWS_ACCESS_KEY_ID }}" AWS_SECRET_ACCESS_KEY: "${{ secrets.AWS_SECRET_ACCESS_KEY }}" + GRS_USERNAME: "${{ vars.GRS_USERNAME }}" + GRS_PASSWORD: "${{ secrets.GRS_PASSWORD }}" + PKCS11_URI: "${{ vars.PKCS11_URI }}" + ARTIFACTORY_USERNAME: "${{ vars.ARTIFACTORY_USERNAME }}" + ARTIFACTORY_PASSWORD: "${{ secrets.ARTIFACTORY_PASSWORD }}" diff --git a/docs/RELEASE_NOTES.md b/docs/RELEASE_NOTES.md index 1470fc1df..f0dbc1519 100644 --- a/docs/RELEASE_NOTES.md +++ b/docs/RELEASE_NOTES.md @@ -1,7 +1,5 @@ -# MongoDB Kubernetes Operator 0.9.0 +# MongoDB Kubernetes Operator 0.10.0 -## MongoDBCommunity Resource +## Released images signed -- Changes - - Introduced support for [Mongodb7](https://www.mongodb.com/docs/manual/release-notes/7.0/) - - Upgrading Kubernetes client APIs to 1.26 \ No newline at end of file +All container images published for the community operator are signed with our private key. This is visible on our Quay registry. Signature can be verified using our public key, which is available at [this address](https://cosign.mongodb.com/mongodb-enterprise-kubernetes-operator.pem). \ No newline at end of file diff --git a/inventories/e2e-inventory.yaml b/inventories/e2e-inventory.yaml index f9bbcfec2..c2247dff4 100644 --- a/inventories/e2e-inventory.yaml +++ b/inventories/e2e-inventory.yaml @@ -1,14 +1,15 @@ vars: registry: + architecture: amd64 images: - - name: e2e-arm64 + - name: e2e vars: context: . template_context: scripts/dev/templates inputs: - image - platform: linux/arm64 + platform: linux/$(inputs.params.architecture) stages: - name: e2e-template task_type: dockerfile_template @@ -31,40 +32,6 @@ images: output: - registry: $(inputs.params.registry)/$(inputs.params.image) - tag: $(inputs.params.version_id)-arm64 + tag: $(inputs.params.version_id)-$(inputs.params.architecture) - registry: $(inputs.params.registry)/$(inputs.params.image) - tag: latest-arm64 - - - name: e2e-amd64 - vars: - context: . - template_context: scripts/dev/templates - inputs: - - image - platform: linux/amd64 - stages: - - name: e2e-template - task_type: dockerfile_template - distro: e2e - - inputs: - - builder - - base_image - - output: - - dockerfile: scripts/dev/templates/Dockerfile.ubi-$(inputs.params.version_id) - - - name: e2e-build - task_type: docker_build - - dockerfile: scripts/dev/templates/Dockerfile.ubi-$(inputs.params.version_id) - - labels: - quay.expires-after: 48h - - output: - - registry: $(inputs.params.registry)/$(inputs.params.image) - tag: $(inputs.params.version_id)-amd64 - - registry: $(inputs.params.registry)/$(inputs.params.image) - tag: latest-amd64 - + tag: latest-$(inputs.params.architecture) diff --git a/inventories/operator-inventory.yaml b/inventories/operator-inventory.yaml index adb009e86..ab08796a8 100644 --- a/inventories/operator-inventory.yaml +++ b/inventories/operator-inventory.yaml @@ -1,8 +1,9 @@ vars: registry: + architecture: amd64 images: - - name: operator-amd64 + - name: operator vars: context: . template_context: scripts/dev/templates/operator @@ -11,7 +12,7 @@ images: - image - image_dev - platform: linux/amd64 + platform: linux/$(inputs.params.architecture) stages: # @@ -30,7 +31,7 @@ images: output: - registry: $(inputs.params.registry)/$(inputs.params.image_dev) - tag: $(inputs.params.version_id)-context-amd64 + tag: $(inputs.params.version_id)-context-$(inputs.params.architecture) - name: operator-template-dev task_type: dockerfile_template @@ -51,16 +52,16 @@ images: - version_id buildargs: - imagebase: $(inputs.params.registry)/$(inputs.params.image_dev):$(inputs.params.version_id)-context-amd64 + imagebase: $(inputs.params.registry)/$(inputs.params.image_dev):$(inputs.params.version_id)-context-$(inputs.params.architecture) labels: quay.expires-after: 48h output: - registry: $(inputs.params.registry)/$(inputs.params.image_dev) - tag: $(inputs.params.version_id)-amd64 + tag: $(inputs.params.version_id)-$(inputs.params.architecture) - registry: $(inputs.params.registry)/$(inputs.params.image_dev) - tag: latest-amd64 + tag: latest-$(inputs.params.architecture) # # Release build stages @@ -83,7 +84,7 @@ images: output: - registry: $(inputs.params.registry)/$(inputs.params.image) - tag: $(inputs.params.release_version)-context-amd64 + tag: $(inputs.params.release_version)-context-$(inputs.params.architecture) - name: operator-template-release task_type: dockerfile_template @@ -107,125 +108,11 @@ images: dockerfile: scripts/dev/templates/operator/Dockerfile.operator-$(inputs.params.release_version) buildargs: - imagebase: $(inputs.params.registry)/$(inputs.params.image):$(inputs.params.release_version)-context-amd64 + imagebase: $(inputs.params.registry)/$(inputs.params.image):$(inputs.params.release_version)-context-$(inputs.params.architecture) labels: quay.expires-after: Never output: - registry: $(inputs.params.registry)/$(inputs.params.image) - tag: $(inputs.params.release_version)-amd64 - - - name: operator-arm64 - vars: - context: . - template_context: scripts/dev/templates/operator - - inputs: - - image - - image_dev - - platform: linux/arm64 - - stages: - # - # Dev build stages - # - - name: operator-builder-dev - task_type: docker_build - tags: [ "ubi" ] - dockerfile: scripts/dev/templates/operator/Dockerfile.builder - - buildargs: - builder_image: $(inputs.params.builder_image) - - labels: - quay.expires-after: 48h - - output: - - registry: $(inputs.params.registry)/$(inputs.params.image_dev) - tag: $(inputs.params.version_id)-context-arm64 - - - name: operator-template-dev - task_type: dockerfile_template - tags: [ "ubi" ] - template_file_extension: operator - inputs: - - base_image - - output: - - dockerfile: scripts/dev/templates/operator/Dockerfile.operator-$(inputs.params.version_id) - - - name: operator-build-dev - task_type: docker_build - tags: [ "ubi" ] - dockerfile: scripts/dev/templates/operator/Dockerfile.operator-$(inputs.params.version_id) - - inputs: - - version_id - - buildargs: - imagebase: $(inputs.params.registry)/$(inputs.params.image_dev):$(inputs.params.version_id)-context-arm64 - - labels: - quay.expires-after: 48h - - output: - - registry: $(inputs.params.registry)/$(inputs.params.image_dev) - tag: $(inputs.params.version_id)-arm64 - - registry: $(inputs.params.registry)/$(inputs.params.image_dev) - tag: latest-arm64 - - # - # Release build stages - # - - name: operator-builder-release - task_type: docker_build - tags: [ "ubi", "release" ] - - inputs: - - builder_image - - release_version - - dockerfile: scripts/dev/templates/operator/Dockerfile.builder - - labels: - quay.expires-after: Never - - buildargs: - builder_image: $(inputs.params.builder_image) - - output: - - registry: $(inputs.params.registry)/$(inputs.params.image) - tag: $(inputs.params.release_version)-context-arm64 - - - name: operator-template-release - task_type: dockerfile_template - tags: [ "ubi", "release" ] - template_file_extension: operator - inputs: - - base_image - - release_version - - output: - - dockerfile: scripts/dev/templates/operator/Dockerfile.operator-$(inputs.params.release_version) - - dockerfile: $(inputs.params.s3_bucket)/mongodb-kubernetes-operator/$(inputs.params.release_version)/ubi/Dockerfile - - - name: operator-build-release - task_type: docker_build - tags: [ "ubi", "release" ] - - inputs: - - release_version - - dockerfile: scripts/dev/templates/operator/Dockerfile.operator-$(inputs.params.release_version) - - buildargs: - imagebase: $(inputs.params.registry)/$(inputs.params.image):$(inputs.params.release_version)-context-arm64 - - labels: - quay.expires-after: Never - - output: - - registry: $(inputs.params.registry)/$(inputs.params.image) - tag: $(inputs.params.release_version)-arm64 + tag: $(inputs.params.release_version)-$(inputs.params.architecture) diff --git a/inventory.yaml b/inventory.yaml index 69e177805..28d475740 100644 --- a/inventory.yaml +++ b/inventory.yaml @@ -1,12 +1,17 @@ vars: registry: + # Default value but overwritten in pipeline.py + architecture: amd64 images: - - name: agent-amd64 + - name: agent vars: context: . template_context: scripts/dev/templates/agent + # Default values but overwritten in pipeline.py + agent_distro: rhel7_x86_64 + tools_distro: rhel70-x86_64 inputs: - release_version @@ -14,7 +19,7 @@ images: - image - image_dev - platform: linux/amd64 + platform: linux/$(inputs.params.architecture) stages: - name: mongodb-agent-context task_type: docker_build @@ -23,15 +28,15 @@ images: buildargs: agent_version: $(inputs.params.release_version) tools_version: $(inputs.params.tools_version) - agent_distro: rhel7_x86_64 - tools_distro: rhel70-x86_64 + agent_distro: $(inputs.params.agent_distro) + tools_distro: $(inputs.params.tools_distro) labels: quay.expires-after: 48h output: - registry: $(inputs.params.registry)/$(inputs.params.image_dev) - tag: $(inputs.params.version_id)-context-amd64 + tag: $(inputs.params.version_id)-context-$(inputs.params.architecture) - name: agent-template-ubi task_type: dockerfile_template @@ -48,7 +53,7 @@ images: dockerfile: scripts/dev/templates/agent/Dockerfile.ubi-$(inputs.params.version_id) buildargs: - imagebase: $(inputs.params.registry)/$(inputs.params.image_dev):$(inputs.params.version_id)-context-amd64 + imagebase: $(inputs.params.registry)/$(inputs.params.image_dev):$(inputs.params.version_id)-context-$(inputs.params.architecture) agent_version: $(inputs.params.release_version) labels: @@ -56,9 +61,9 @@ images: output: - registry: $(inputs.params.registry)/$(inputs.params.image_dev) - tag: $(inputs.params.version_id)-amd64 + tag: $(inputs.params.version_id)-$(inputs.params.architecture) - registry: $(inputs.params.registry)/$(inputs.params.image_dev) - tag: latest-amd64 + tag: latest-$(inputs.params.architecture) - name: agent-template-ubi-s3 task_type: dockerfile_template @@ -86,108 +91,7 @@ images: output: - registry: $(inputs.params.registry)/$(inputs.params.image) - tag: $(inputs.params.release_version)-context-amd64 - - - name: mongodb-agent-release - task_type: docker_build - tags: [ "ubi", "release" ] - dockerfile: scripts/dev/templates/agent/Dockerfile.ubi-$(inputs.params.version_id) - - buildargs: - imagebase: $(inputs.params.registry)/$(inputs.params.image):$(inputs.params.release_version)-context-amd64 - agent_version: $(inputs.params.release_version) - - labels: - quay.expires-after: Never - - output: - - registry: $(inputs.params.registry)/$(inputs.params.image) - tag: $(inputs.params.release_version)-amd64 - - - name: agent-arm64 - vars: - context: . - template_context: scripts/dev/templates/agent - - inputs: - - release_version - - tools_version - - image - - image_dev - - platform: linux/arm64 - stages: - - name: mongodb-agent-context - task_type: docker_build - dockerfile: scripts/dev/templates/agent/Dockerfile.builder - tags: [ "ubi" ] - buildargs: - agent_version: $(inputs.params.release_version) - tools_version: $(inputs.params.tools_version) - agent_distro: amzn2_aarch64 - tools_distro: rhel82-aarch64 - - labels: - quay.expires-after: 48h - - output: - - registry: $(inputs.params.registry)/$(inputs.params.image_dev) - tag: $(inputs.params.version_id)-context-arm64 - - - name: agent-template-ubi - task_type: dockerfile_template - distro: ubi - tags: [ "ubi" ] - - output: - - dockerfile: scripts/dev/templates/agent/Dockerfile.ubi-$(inputs.params.version_id) - - - name: mongodb-agent-build - task_type: docker_build - tags: [ "ubi" ] - - dockerfile: scripts/dev/templates/agent/Dockerfile.ubi-$(inputs.params.version_id) - - buildargs: - imagebase: $(inputs.params.registry)/$(inputs.params.image_dev):$(inputs.params.version_id)-context-arm64 - agent_version: $(inputs.params.release_version) - - labels: - quay.expires-after: 48h - - output: - - registry: $(inputs.params.registry)/$(inputs.params.image_dev) - tag: $(inputs.params.version_id)-arm64 - - registry: $(inputs.params.registry)/$(inputs.params.image_dev) - tag: latest-arm64 - - - name: agent-template-ubi-s3 - task_type: dockerfile_template - tags: [ "ubi", "release" ] - distro: ubi - - inputs: - - release_version - - output: - - dockerfile: $(inputs.params.s3_bucket)/mongodb-agent/$(inputs.params.release_version)/ubi/Dockerfile - - - name: agent-context-ubi-release - task_type: docker_build - dockerfile: scripts/dev/templates/agent/Dockerfile.builder - tags: [ "ubi", "release" ] - buildargs: - agent_version: $(inputs.params.release_version) - tools_version: $(inputs.params.tools_version) - agent_distro: amzn2_aarch64 - tools_distro: rhel82-aarch64 - - labels: - quay.expires-after: Never - - output: - - registry: $(inputs.params.registry)/$(inputs.params.image) - tag: $(inputs.params.release_version)-context-arm64 + tag: $(inputs.params.release_version)-context-$(inputs.params.architecture) - name: mongodb-agent-release task_type: docker_build @@ -195,7 +99,7 @@ images: dockerfile: scripts/dev/templates/agent/Dockerfile.ubi-$(inputs.params.version_id) buildargs: - imagebase: $(inputs.params.registry)/$(inputs.params.image):$(inputs.params.release_version)-context-arm64 + imagebase: $(inputs.params.registry)/$(inputs.params.image):$(inputs.params.release_version)-context-$(inputs.params.architecture) agent_version: $(inputs.params.release_version) labels: @@ -203,113 +107,9 @@ images: output: - registry: $(inputs.params.registry)/$(inputs.params.image) - tag: $(inputs.params.release_version)-arm64 - - - name: readiness-probe-amd64 - vars: - context: . - template_context: scripts/dev/templates/readiness - - inputs: - - image - - image_dev - - platform: linux/amd64 - stages: - - name: readiness-init-context-build - task_type: docker_build - dockerfile: scripts/dev/templates/readiness/Dockerfile.builder - tags: [ "readiness-probe", "ubi" ] - labels: - quay.expires-after: 48h - - buildargs: - builder_image: $(inputs.params.builder_image) - - output: - - registry: $(inputs.params.registry)/$(inputs.params.image_dev) - tag: $(inputs.params.version_id)-context-amd64 - - - name: readiness-template-ubi - task_type: dockerfile_template - tags: [ "ubi" ] - template_file_extension: readiness - - inputs: - - base_image - - output: - - dockerfile: scripts/dev/templates/readiness/Dockerfile.readiness-$(inputs.params.version_id) - - - name: readiness-init-build - task_type: docker_build - tags: [ "readiness-probe", "ubi" ] - dockerfile: scripts/dev/templates/readiness/Dockerfile.readiness-$(inputs.params.version_id) - - buildargs: - imagebase: $(inputs.params.registry)/$(inputs.params.image_dev):$(inputs.params.version_id)-context-amd64 - - - labels: - quay.expires-after: 48h - - output: - - registry: $(inputs.params.registry)/$(inputs.params.image_dev) - tag: $(inputs.params.version_id)-amd64 - - registry: $(inputs.params.registry)/$(inputs.params.image_dev) - tag: latest-amd64 - - - name: readiness-init-context-release - task_type: docker_build - dockerfile: scripts/dev/templates/readiness/Dockerfile.builder - tags: [ "readiness-probe", "release" , "ubi" ] - - labels: - quay.expires-after: Never - - buildargs: - builder_image: $(inputs.params.builder_image) - - inputs: - - release_version - - builder_image - - output: - - registry: $(inputs.params.registry)/$(inputs.params.image) - tag: $(inputs.params.release_version)-context-amd64 - - - name: readiness-template-release - task_type: dockerfile_template - tags: [ "readiness-probe", "release", "ubi" ] - template_file_extension: readiness - inputs: - - base_image - - release_version - - output: - - dockerfile: scripts/dev/templates/readiness/Dockerfile.readiness-$(inputs.params.release_version) - - dockerfile: $(inputs.params.s3_bucket)/mongodb-kubernetes-readinessprobe/$(inputs.params.release_version)/ubi/Dockerfile - - - name: readiness-init-build-release - task_type: docker_build - dockerfile: scripts/dev/templates/readiness/Dockerfile.readiness-$(inputs.params.release_version) - tags: [ "readiness-probe", "release" , "ubi" ] - - buildargs: - imagebase: $(inputs.params.registry)/$(inputs.params.image):$(inputs.params.release_version)-context-amd64 - - labels: - quay.expires-after: Never - - inputs: - - base_image - - release_version - - output: - - registry: $(inputs.params.registry)/$(inputs.params.image) - tag: $(inputs.params.release_version)-amd64 + tag: $(inputs.params.release_version)-$(inputs.params.architecture) - - name: readiness-probe-arm64 + - name: readiness-probe vars: context: . template_context: scripts/dev/templates/readiness @@ -318,7 +118,7 @@ images: - image - image_dev - platform: linux/arm64 + platform: linux/$(inputs.params.architecture) stages: - name: readiness-init-context-build task_type: docker_build @@ -332,7 +132,7 @@ images: output: - registry: $(inputs.params.registry)/$(inputs.params.image_dev) - tag: $(inputs.params.version_id)-context-arm64 + tag: $(inputs.params.version_id)-context-$(inputs.params.architecture) - name: readiness-template-ubi task_type: dockerfile_template @@ -351,7 +151,7 @@ images: dockerfile: scripts/dev/templates/readiness/Dockerfile.readiness-$(inputs.params.version_id) buildargs: - imagebase: $(inputs.params.registry)/$(inputs.params.image_dev):$(inputs.params.version_id)-context-arm64 + imagebase: $(inputs.params.registry)/$(inputs.params.image_dev):$(inputs.params.version_id)-context-$(inputs.params.architecture) labels: @@ -359,9 +159,9 @@ images: output: - registry: $(inputs.params.registry)/$(inputs.params.image_dev) - tag: $(inputs.params.version_id)-arm64 + tag: $(inputs.params.version_id)-$(inputs.params.architecture) - registry: $(inputs.params.registry)/$(inputs.params.image_dev) - tag: latest-arm64 + tag: latest-$(inputs.params.architecture) - name: readiness-init-context-release task_type: docker_build @@ -380,7 +180,7 @@ images: output: - registry: $(inputs.params.registry)/$(inputs.params.image) - tag: $(inputs.params.release_version)-context-arm64 + tag: $(inputs.params.release_version)-context-$(inputs.params.architecture) - name: readiness-template-release task_type: dockerfile_template @@ -400,111 +200,7 @@ images: tags: [ "readiness-probe", "release" , "ubi" ] buildargs: - imagebase: $(inputs.params.registry)/$(inputs.params.image):$(inputs.params.release_version)-context-arm64 - - labels: - quay.expires-after: Never - - inputs: - - base_image - - release_version - - output: - - registry: $(inputs.params.registry)/$(inputs.params.image) - tag: $(inputs.params.release_version)-arm64 - - - name: version-upgrade-hook-amd64 - vars: - context: . - template_context: scripts/dev/templates/versionhook - - inputs: - - image - - image_dev - - platform: linux/amd64 - stages: - - name: version-upgrade-hook-context-build - task_type: docker_build - dockerfile: scripts/dev/templates/versionhook/Dockerfile.builder - tags: [ "post-start-hook", "ubi" ] - - buildargs: - builder_image: $(inputs.params.builder_image) - - labels: - quay.expires-after: 48h - - output: - - registry: $(inputs.params.registry)/$(inputs.params.image_dev) - tag: $(inputs.params.version_id)-context-amd64 - - - name: version-post-start-hook-template-ubi - task_type: dockerfile_template - tags: [ "ubi" ] - template_file_extension: versionhook - - inputs: - - base_image - - output: - - dockerfile: scripts/dev/templates/versionhook/Dockerfile.versionhook-$(inputs.params.version_id) - - - name: version-upgrade-hook-build - task_type: docker_build - dockerfile: scripts/dev/templates/versionhook/Dockerfile.versionhook-$(inputs.params.version_id) - tags: [ "post-start-hook", "ubi" ] - - buildargs: - imagebase: $(inputs.params.registry)/$(inputs.params.image_dev):$(inputs.params.version_id)-context-amd64 - - labels: - quay.expires-after: 48h - - output: - - registry: $(inputs.params.registry)/$(inputs.params.image_dev) - tag: $(inputs.params.version_id)-amd64 - - registry: $(inputs.params.registry)/$(inputs.params.image_dev) - tag: latest-amd64 - - - name: version-upgrade-hook-context-release - task_type: docker_build - dockerfile: scripts/dev/templates/versionhook/Dockerfile.builder - tags: [ "release", "post-start-hook", "ubi", ] - - labels: - quay.expires-after: Never - - buildargs: - builder_image: $(inputs.params.builder_image) - - inputs: - - release_version - - builder_image - - output: - - registry: $(inputs.params.registry)/$(inputs.params.image) - tag: $(inputs.params.release_version)-context-amd64 - - - name: versionhook-template-release - task_type: dockerfile_template - tags: [ "post-start-hook", "release", "ubi" ] - template_file_extension: versionhook - inputs: - - base_image - - release_version - - output: - - dockerfile: scripts/dev/templates/versionhook/Dockerfile.versionhook-$(inputs.params.release_version) - - dockerfile: $(inputs.params.s3_bucket)/mongodb-kubernetes-operator-version-upgrade-post-start-hook/$(inputs.params.release_version)/ubi/Dockerfile - - - name: version-upgrade-hook-build-release - task_type: docker_build - dockerfile: scripts/dev/templates/versionhook/Dockerfile.versionhook-$(inputs.params.release_version) - tags: [ "release", "post-start-hook", "ubi" ] - - buildargs: - imagebase: $(inputs.params.registry)/$(inputs.params.image):$(inputs.params.release_version)-context-amd64 + imagebase: $(inputs.params.registry)/$(inputs.params.image):$(inputs.params.release_version)-context-$(inputs.params.architecture) labels: quay.expires-after: Never @@ -515,9 +211,9 @@ images: output: - registry: $(inputs.params.registry)/$(inputs.params.image) - tag: $(inputs.params.release_version)-amd64 + tag: $(inputs.params.release_version)-$(inputs.params.architecture) - - name: version-upgrade-hook-arm64 + - name: version-upgrade-hook vars: context: . template_context: scripts/dev/templates/versionhook @@ -526,7 +222,7 @@ images: - image - image_dev - platform: linux/arm64 + platform: linux/$(inputs.params.architecture) stages: - name: version-upgrade-hook-context-build task_type: docker_build @@ -541,7 +237,7 @@ images: output: - registry: $(inputs.params.registry)/$(inputs.params.image_dev) - tag: $(inputs.params.version_id)-context-arm64 + tag: $(inputs.params.version_id)-context-$(inputs.params.architecture) - name: version-post-start-hook-template-ubi task_type: dockerfile_template @@ -560,16 +256,16 @@ images: tags: [ "post-start-hook", "ubi" ] buildargs: - imagebase: $(inputs.params.registry)/$(inputs.params.image_dev):$(inputs.params.version_id)-context-arm64 + imagebase: $(inputs.params.registry)/$(inputs.params.image_dev):$(inputs.params.version_id)-context-$(inputs.params.architecture) labels: quay.expires-after: 48h output: - registry: $(inputs.params.registry)/$(inputs.params.image_dev) - tag: $(inputs.params.version_id)-arm64 + tag: $(inputs.params.version_id)-$(inputs.params.architecture) - registry: $(inputs.params.registry)/$(inputs.params.image_dev) - tag: latest-arm64 + tag: latest-$(inputs.params.architecture) - name: version-upgrade-hook-context-release task_type: docker_build @@ -588,7 +284,7 @@ images: output: - registry: $(inputs.params.registry)/$(inputs.params.image) - tag: $(inputs.params.release_version)-context-arm64 + tag: $(inputs.params.release_version)-context-$(inputs.params.architecture) - name: versionhook-template-release task_type: dockerfile_template @@ -608,7 +304,7 @@ images: tags: [ "release", "post-start-hook", "ubi" ] buildargs: - imagebase: $(inputs.params.registry)/$(inputs.params.image):$(inputs.params.release_version)-context-arm64 + imagebase: $(inputs.params.registry)/$(inputs.params.image):$(inputs.params.release_version)-context-$(inputs.params.architecture) labels: quay.expires-after: Never @@ -619,4 +315,4 @@ images: output: - registry: $(inputs.params.registry)/$(inputs.params.image) - tag: $(inputs.params.release_version)-arm64 \ No newline at end of file + tag: $(inputs.params.release_version)-$(inputs.params.architecture) \ No newline at end of file diff --git a/pipeline.py b/pipeline.py index 9731fcbdd..078a096eb 100644 --- a/pipeline.py +++ b/pipeline.py @@ -3,6 +3,12 @@ import subprocess import sys from typing import Dict, List, Set +from scripts.ci.base_logger import logger +from scripts.ci.images_signing import ( + sign_image, + verify_signature, + mongodb_artifactory_login, +) from scripts.dev.dev_config import load_config, DevConfig from sonar.sonar import process_image @@ -16,6 +22,14 @@ "e2e", } +AGENT_DISTRO_KEY = "agent_distro" +TOOLS_DISTRO_KEY = "tools_distro" + +AGENT_DISTROS_PER_ARCH = { + "amd64": {AGENT_DISTRO_KEY: "rhel7_x86_64", TOOLS_DISTRO_KEY: "rhel70-x86_64"}, + "arm64": {AGENT_DISTRO_KEY: "amzn2_aarch64", TOOLS_DISTRO_KEY: "rhel82-aarch64"}, +} + def load_release() -> Dict: with open("release.json") as f: @@ -61,15 +75,27 @@ def build_image_args(config: DevConfig, image_name: str) -> Dict[str, str]: return arguments +def sign_and_verify(registry: str, tag: str) -> None: + sign_image(registry, tag) + verify_signature(registry, tag) + + def build_and_push_image( image_name: str, config: DevConfig, args: Dict[str, str], architectures: Set[str], release: bool, -): + sign: bool, +) -> None: + if sign: + mongodb_artifactory_login() for arch in architectures: - image_tag = f"{image_name}-{arch}" + image_tag = f"{image_name}" + args["architecture"] = arch + if image_name == "agent": + args[AGENT_DISTRO_KEY] = AGENT_DISTROS_PER_ARCH[arch][AGENT_DISTRO_KEY] + args[TOOLS_DISTRO_KEY] = AGENT_DISTROS_PER_ARCH[arch][TOOLS_DISTRO_KEY] process_image( image_tag, build_args=args, @@ -77,6 +103,13 @@ def build_and_push_image( skip_tags=args["skip_tags"], include_tags=args["include_tags"], ) + if release: + registry = args["registry"] + "/" + args["image"] + context_tag = args["release_version"] + "-context-" + arch + release_tag = args["release_version"] + "-" + arch + if sign: + sign_and_verify(registry, context_tag) + sign_and_verify(registry, release_tag) if args["image_dev"]: image_to_push = args["image_dev"] @@ -92,10 +125,13 @@ def build_and_push_image( push_manifest(config, architectures, image_to_push, config.gh_run_id) if release: + registry = args["registry"] + "/" + args["image"] + context_tag = args["release_version"] + "-context" push_manifest(config, architectures, args["image"], args["release_version"]) - push_manifest( - config, architectures, args["image"], args["release_version"] + "-context" - ) + push_manifest(config, architectures, args["image"], context_tag) + if sign: + sign_and_verify(registry, args["release_version"]) + sign_and_verify(registry, context_tag) """ @@ -114,11 +150,11 @@ def push_manifest( architectures: Set[str], image_name: str, image_tag: str = "latest", -): - print(f"Pushing manifest for {image_tag}") +) -> None: + logger.info(f"Pushing manifest for {image_tag}") final_manifest = "{0}/{1}:{2}".format(config.repo_url, image_name, image_tag) remove_args = ["docker", "manifest", "rm", final_manifest] - print("Removing existing manifest") + logger.info("Removing existing manifest") run_cli_command(remove_args, fail_on_error=False) create_args = [ @@ -131,18 +167,18 @@ def push_manifest( for arch in architectures: create_args.extend(["--amend", final_manifest + "-" + arch]) - print("Creating new manifest") + logger.info("Creating new manifest") run_cli_command(create_args) push_args = ["docker", "manifest", "push", final_manifest] - print("Pushing new manifest") + logger.info("Pushing new manifest") run_cli_command(push_args) # Raises exceptions by default -def run_cli_command(args: List[str], fail_on_error: bool = True): +def run_cli_command(args: List[str], fail_on_error: bool = True) -> None: command = " ".join(args) - print(f"Running: {command}") + logger.debug(f"Running: {command}") try: cp = subprocess.run( command, @@ -152,23 +188,23 @@ def run_cli_command(args: List[str], fail_on_error: bool = True): check=False, ) except Exception as e: - print(f" Command raised the following exception: {e}") + logger.error(f" Command raised the following exception: {e}") if fail_on_error: raise Exception else: - print("Continuing...") + logger.warning("Continuing...") return if cp.returncode != 0: error_msg = cp.stderr.decode().strip() stdout = cp.stdout.decode().strip() - print(f"Error running command") - print(f"stdout:\n{stdout}") - print(f"stderr:\n{error_msg}") + logger.error(f"Error running command") + logger.error(f"stdout:\n{stdout}") + logger.error(f"stderr:\n{error_msg}") if fail_on_error: raise Exception else: - print("Continuing...") + logger.warning("Continuing...") return @@ -183,6 +219,7 @@ def _parse_args() -> argparse.Namespace: help="for daily builds only, specify the list of architectures to build for images", ) parser.add_argument("--tag", type=str) + parser.add_argument("--sign", action="store_true", default=False) return parser.parse_args() @@ -191,9 +228,10 @@ def _parse_args() -> argparse.Namespace: --image-name : The name of the image to build, must be one of VALID_IMAGE_NAMES --release : We push the image to the registry only if this flag is set --architecture : List of architectures to build for the image +--sign : Sign images with our private key if sign is set (only for release) Run with --help for more information -Example usage : `python pipeline.py --image-name agent --release` +Example usage : `python pipeline.py --image-name agent --release --sign` Builds and push the docker image to the registry Many parameters are defined in the dev configuration, default path is : ~/.community-operator-dev/config.json @@ -205,7 +243,7 @@ def main() -> int: image_name = args.image_name if image_name not in VALID_IMAGE_NAMES: - print( + logger.error( f"Invalid image name: {image_name}. Valid options are: {VALID_IMAGE_NAMES}" ) return 1 @@ -216,24 +254,31 @@ def main() -> int: # Warn user if trying to release E2E tests if args.release and image_name == "e2e": - print( + logger.warning( "Warning : releasing E2E test will fail because E2E image has no release version" ) # Skipping release tasks by default if not args.release: config.ensure_skip_tag("release") + if args.sign: + logger.warning("--sign flag has no effect without --release") if args.arch: arch_set = set(args.arch) else: # Default is multi-arch - arch_set = ["amd64", "arm64"] - print("Building for architectures:", ", ".join(map(str, arch_set))) + arch_set = {"amd64", "arm64"} + logger.info(f"Building for architectures: {','.join(arch_set)}") + + if not args.sign: + logger.warning("--sign flag not provided, images won't be signed") image_args = build_image_args(config, image_name) - build_and_push_image(image_name, config, image_args, arch_set, args.release) + build_and_push_image( + image_name, config, image_args, arch_set, args.release, args.sign + ) return 0 diff --git a/requirements.txt b/requirements.txt index 65ffaa286..d7f4616f3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -git+https://github.com/mongodb/sonar@0c21097a55a4426c8824f74cdcfbceb11b27bb68 +git+https://github.com/mongodb/sonar@bc7bf7732851425421f3cfe2a19cf50b0460e633 github-action-templates==0.0.4 docker==4.3.1 kubernetes==26.1.0 @@ -17,4 +17,4 @@ semver==2.13.0 rsa>=4.7 # not directly required, pinned by Snyk to avoid a vulnerability setuptools>=65.5.1 # not directly required, pinned by Snyk to avoid a vulnerability certifi>=2022.12.7 # not directly required, pinned by Snyk to avoid a vulnerability -urllib3>=1.26.5 # not directly required, pinned by Snyk to avoid a vulnerability +urllib3<2 # not directly required, pinned by Snyk to avoid a vulnerability diff --git a/scripts/ci/base_logger.py b/scripts/ci/base_logger.py new file mode 100644 index 000000000..571c10aa0 --- /dev/null +++ b/scripts/ci/base_logger.py @@ -0,0 +1,21 @@ +import logging +import os +import sys + +LOGLEVEL = os.environ.get("LOGLEVEL", "DEBUG").upper() +logger = logging.getLogger("pipeline") +logger.setLevel(LOGLEVEL) +logger.propagate = False + +# Output Debug and Info logs to stdout, and above to stderr +stdout_handler = logging.StreamHandler(sys.stdout) +stdout_handler.setLevel(logging.DEBUG) +stdout_handler.addFilter(lambda record: record.levelno <= logging.INFO) +stderr_handler = logging.StreamHandler(sys.stderr) +stderr_handler.setLevel(logging.WARNING) + +formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s") +stdout_handler.setFormatter(formatter) +stderr_handler.setFormatter(formatter) +logger.addHandler(stdout_handler) +logger.addHandler(stderr_handler) diff --git a/scripts/ci/images_signing.py b/scripts/ci/images_signing.py new file mode 100644 index 000000000..e2fb4a94e --- /dev/null +++ b/scripts/ci/images_signing.py @@ -0,0 +1,208 @@ +import os +import subprocess +import sys +from typing import List, Optional + +import requests + +from scripts.ci.base_logger import logger + +SIGNING_IMAGE_URI = os.environ.get( + "SIGNING_IMAGE_URI", + "artifactory.corp.mongodb.com/release-tools-container-registry-local/garasign-cosign", +) + + +def mongodb_artifactory_login() -> None: + command = [ + "docker", + "login", + "--password-stdin", + "--username", + os.environ["ARTIFACTORY_USERNAME"], + "artifactory.corp.mongodb.com/release-tools-container-registry-local/garasign-cosign", + ] + try: + subprocess.run( + command, + input=os.environ["ARTIFACTORY_PASSWORD"].encode("utf-8"), + check=True, + ) + except subprocess.CalledProcessError as e: + logger.error(f"Authentication to MongoDB Artifactory failed : {e.returncode}") + logger.error(f"Output: {e.stderr}") + + +def get_ecr_login_password(region: str) -> Optional[str]: + """ + Retrieves the login password from aws CLI, the secrets need to be stored in ~/.aws/credentials or equivalent. + :param region: Registry's AWS region + :return: The password as a string + """ + try: + result = subprocess.run( + ["aws", "ecr", "get-login-password", "--region", region], + capture_output=True, + text=True, + check=True, + ) + return result.stdout.strip() + except subprocess.CalledProcessError as e: + logger.error(f"Failed to get ECR login password: {e.stderr}") + return None + + +def is_ecr_registry(image_name: str) -> bool: + return "amazonaws.com" in image_name + + +def get_image_digest(image_name: str) -> Optional[str]: + """ + Retrieves the digest of an image from its tag. Uses the skopeo container to be able to retrieve manifests tags as well. + :param image_name: The full image name with its tag. + :return: the image digest, or None in case of failure. + """ + + transport_protocol = "docker://" + # Get digest + digest_command = [ + "docker", + "run", + "--rm", + f"--volume={os.path.expanduser('~')}/.aws:/root/.aws:ro", + "quay.io/skopeo/stable:latest", + "inspect", + "--format={{.Digest}}", + ] + + # Specify ECR credentials if necessary + if is_ecr_registry(image_name): + aws_region = os.environ.get("AWS_DEFAULT_REGION", "eu-west-1") + ecr_password = get_ecr_login_password(aws_region) + digest_command.append(f"--creds=AWS:{ecr_password}") + + digest_command.append(f"{transport_protocol}{image_name}") + + try: + result = subprocess.run( + digest_command, capture_output=True, text=True, check=True + ) + digest = result.stdout.strip() + return digest + except subprocess.CalledProcessError as e: + logger.error(f"Failed to get digest for {image_name}: {e.stderr}") + sys.exit(1) + + +def build_cosign_docker_command( + additional_args: List[str], cosign_command: List[str] +) -> List[str]: + """ + Common logic to build a cosign command with the garasign cosign image provided by DevProd. + :param additional_args: additional arguments passed to the docker container, e.g mounted volume or env + :param cosign_command: actual command executed with cosign such as `sign` or `verify` + :return: the full command as a List of strings + """ + home_dir = os.path.expanduser("~") + base_command = [ + "docker", + "run", + "--platform", + "linux/amd64", + "--rm", + f"--volume={home_dir}/.docker/config.json:/root/.docker/config.json:ro", + ] + return ( + base_command + additional_args + [SIGNING_IMAGE_URI, "cosign"] + cosign_command + ) + + +def sign_image(repository: str, tag: str) -> None: + image = repository + ":" + tag + logger.debug(f"Signing image {image}") + + working_directory = os.getcwd() + container_working_directory = "/usr/local/kubernetes" + + # Referring to the image via its tag is deprecated in cosign + # We fetch the digest from the registry + digest = get_image_digest(image) + if digest is None: + logger.error("Impossible to get image digest, exiting...") + sys.exit(1) + image_ref = f"{repository}@{digest}" + + # Read secrets from environment and put them in env file for container + grs_username = os.environ["GRS_USERNAME"] + grs_password = os.environ["GRS_PASSWORD"] + pkcs11_uri = os.environ["PKCS11_URI"] + env_file_lines = [ + f"GRS_CONFIG_USER1_USERNAME={grs_username}", + f"GRS_CONFIG_USER1_PASSWORD={grs_password}", + f"COSIGN_REPOSITORY={repository}", + ] + env_file_content = "\n".join(env_file_lines) + temp_file = "./env-file" + with open(temp_file, "w") as f: + f.write(env_file_content) + + additional_args = [ + f"--env-file={temp_file}", + f"--volume={working_directory}:{container_working_directory}", + f"--workdir={container_working_directory}", + ] + cosign_command = [ + "sign", + f"--key={pkcs11_uri}", + f"--sign-container-identity={image}", + f"--tlog-upload=false", + image_ref, + ] + command = build_cosign_docker_command(additional_args, cosign_command) + + try: + subprocess.run(command, check=True) + except subprocess.CalledProcessError as e: + # Fail the pipeline if signing fails + logger.error(f"Failed to sign image {image}: {e.stderr}") + raise + logger.debug("Signing successful") + + +def verify_signature(repository: str, tag: str) -> bool: + image = repository + ":" + tag + logger.debug(f"Verifying signature of {image}") + public_key_url = os.environ.get( + "SIGNING_PUBLIC_KEY_URL", + "https://cosign.mongodb.com/mongodb-enterprise-kubernetes-operator.pem", + ) + r = requests.get(public_key_url) + # Ensure the request was successful + if r.status_code == 200: + # Access the content of the file + kubernetes_operator_public_key = r.text + else: + logger.error(f"Failed to retrieve the public key: Status code {r.status_code}") + return False + + public_key_var_name = "OPERATOR_PUBLIC_KEY" + additional_args = [ + "--env", + f"{public_key_var_name}={kubernetes_operator_public_key}", + ] + cosign_command = [ + "verify", + "--insecure-ignore-tlog", + f"--key=env://{public_key_var_name}", + image, + ] + command = build_cosign_docker_command(additional_args, cosign_command) + + try: + subprocess.run(command, capture_output=True, text=True, check=True) + except subprocess.CalledProcessError as e: + # Fail the pipeline if verification fails + logger.error(f"Failed to verify signature for image {image}: {e.stderr}") + raise + logger.debug("Successful verification") + return True From 9430f5da7f790128154454644c101a18eb4ebeae Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 May 2024 14:26:07 +0200 Subject: [PATCH 728/790] Bump go.mongodb.org/mongo-driver from 1.14.0 to 1.15.0 (#1526) Bumps [go.mongodb.org/mongo-driver](https://github.com/mongodb/mongo-go-driver) from 1.14.0 to 1.15.0. - [Release notes](https://github.com/mongodb/mongo-go-driver/releases) - [Commits](https://github.com/mongodb/mongo-go-driver/compare/v1.14.0...v1.15.0) --- updated-dependencies: - dependency-name: go.mongodb.org/mongo-driver dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index d1c859b5c..75f2301fe 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/stretchr/objx v0.5.2 github.com/stretchr/testify v1.8.4 github.com/xdg/stringprep v1.0.3 - go.mongodb.org/mongo-driver v1.14.0 + go.mongodb.org/mongo-driver v1.15.0 go.uber.org/zap v1.27.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 k8s.io/api v0.27.12 diff --git a/go.sum b/go.sum index 9faef68f0..26940b018 100644 --- a/go.sum +++ b/go.sum @@ -166,8 +166,8 @@ github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7Jul github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.mongodb.org/mongo-driver v1.14.0 h1:P98w8egYRjYe3XDjxhYJagTokP/H6HzlsnojRgZRd80= -go.mongodb.org/mongo-driver v1.14.0/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c= +go.mongodb.org/mongo-driver v1.15.0 h1:rJCKC8eEliewXjZGf0ddURtl7tTVy1TK3bfl0gkUSLc= +go.mongodb.org/mongo-driver v1.15.0/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= From c9b079611f64ba764c128ab78b67a78f33cfee97 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 May 2024 14:36:14 +0200 Subject: [PATCH 729/790] Bump github.com/stretchr/testify from 1.8.4 to 1.9.0 (#1502) Bumps [github.com/stretchr/testify](https://github.com/stretchr/testify) from 1.8.4 to 1.9.0. - [Release notes](https://github.com/stretchr/testify/releases) - [Commits](https://github.com/stretchr/testify/compare/v1.8.4...v1.9.0) --- updated-dependencies: - dependency-name: github.com/stretchr/testify dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 75f2301fe..f5ff8ad4f 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/imdario/mergo v0.3.15 github.com/spf13/cast v1.6.0 github.com/stretchr/objx v0.5.2 - github.com/stretchr/testify v1.8.4 + github.com/stretchr/testify v1.9.0 github.com/xdg/stringprep v1.0.3 go.mongodb.org/mongo-driver v1.15.0 go.uber.org/zap v1.27.0 diff --git a/go.sum b/go.sum index 26940b018..20c428f26 100644 --- a/go.sum +++ b/go.sum @@ -151,8 +151,8 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= From 771c8cd4e81b46d9e4ebdf1b7002112a8a88cbd4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 May 2024 15:54:10 +0200 Subject: [PATCH 730/790] Bump pymongo from 4.6.2 to 4.6.3 (#1520) Bumps [pymongo](https://github.com/mongodb/mongo-python-driver) from 4.6.2 to 4.6.3. - [Release notes](https://github.com/mongodb/mongo-python-driver/releases) - [Changelog](https://github.com/mongodb/mongo-python-driver/blob/master/doc/changelog.rst) - [Commits](https://github.com/mongodb/mongo-python-driver/compare/4.6.2...4.6.3) --- updated-dependencies: - dependency-name: pymongo dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d7f4616f3..11b7619d8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,7 @@ black==24.3.0 mypy==0.961 tqdm==v4.49.0 boto3==1.16.21 -pymongo==4.6.2 +pymongo==4.6.3 dnspython==2.6.1 requests==2.31.0 ruamel.yaml==0.17.9 From 212f82586f7a74f854813b3860cd1f02ec92f309 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 7 May 2024 14:22:40 +0200 Subject: [PATCH 731/790] Bump jinja2 from 3.1.3 to 3.1.4 (#1540) Bumps [jinja2](https://github.com/pallets/jinja) from 3.1.3 to 3.1.4. - [Release notes](https://github.com/pallets/jinja/releases) - [Changelog](https://github.com/pallets/jinja/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/jinja/compare/3.1.3...3.1.4) --- updated-dependencies: - dependency-name: jinja2 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 11b7619d8..e0f85c47c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ git+https://github.com/mongodb/sonar@bc7bf7732851425421f3cfe2a19cf50b0460e633 github-action-templates==0.0.4 docker==4.3.1 kubernetes==26.1.0 -jinja2==3.1.3 +jinja2==3.1.4 MarkupSafe==2.0.1 PyYAML==6.0.1 black==24.3.0 From a8a30ff0fe81f9c585c069c7b8c130e507a3c017 Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Tue, 7 May 2024 14:44:28 +0200 Subject: [PATCH 732/790] bump tqdm (#1537) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e0f85c47c..681aecb44 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ MarkupSafe==2.0.1 PyYAML==6.0.1 black==24.3.0 mypy==0.961 -tqdm==v4.49.0 +tqdm==v4.66.3 boto3==1.16.21 pymongo==4.6.3 dnspython==2.6.1 From 31a46752a99c1de75858b856a0d3665f814bf985 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20=C5=81askawiec?= Date: Tue, 21 May 2024 17:03:43 +0200 Subject: [PATCH 733/790] CLOUDP-212060 Document linting rules (#1550) --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index 95bf400aa..ecbdcb079 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,16 @@ Please file issues before filing PRs. For PRs to be accepted, contributors must Reviewers, please ensure that the CLA has been signed by referring to [the contributors tool](https://contributors.corp.mongodb.com/) (internal link). +## Linting + +This project uses the following linters upon every Pull Request: + +* `Black` is a tool that verifies if Python code is properly formatted +* `MyPy` is a Static Type Checker for Python +* `Kube-linter` is a tool that verified if all Kubernetes YAML manifests are formatted correctly +* `Go vet` A built-in Go static checker +* `Snyk` The vulnerability scanner + ## License Please see the [LICENSE](LICENSE.md) file. From 8f69586a33d8d854a24000f0e5d518eac85b0d25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20=C5=81askawiec?= Date: Wed, 5 Jun 2024 18:41:45 +0200 Subject: [PATCH 734/790] CLOUDP-250100 CVE-2024-35195 fix (#1556) * CLOUDP-250100 CVE-2024-35195 fix * Lib updates --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 681aecb44..b73c33e40 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ git+https://github.com/mongodb/sonar@bc7bf7732851425421f3cfe2a19cf50b0460e633 github-action-templates==0.0.4 -docker==4.3.1 +docker==7.1.0 kubernetes==26.1.0 jinja2==3.1.4 MarkupSafe==2.0.1 @@ -11,7 +11,7 @@ tqdm==v4.66.3 boto3==1.16.21 pymongo==4.6.3 dnspython==2.6.1 -requests==2.31.0 +requests==2.32.3 ruamel.yaml==0.17.9 semver==2.13.0 rsa>=4.7 # not directly required, pinned by Snyk to avoid a vulnerability From e4b4e7b80190053df88acad4250833877af26200 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20=C5=81askawiec?= Date: Thu, 6 Jun 2024 14:25:55 +0200 Subject: [PATCH 735/790] CLOUDP-252271 Use golangci-lint and gosec (#1558) --- .github/workflows/code-health.yml | 12 +++++++++- .github/workflows/main.yaml | 22 ------------------- .golangci.yml | 5 +---- README.md | 1 + cmd/readiness/readiness_test.go | 4 ++-- controllers/construct/mongodbstatefulset.go | 2 +- test/e2e/mongodbtests/mongodbtests.go | 8 +++---- .../replica_set_arbiter_test.go | 4 ++-- .../replica_set_operator_upgrade_test.go | 2 +- test/e2e/util/wait/wait.go | 5 ----- 10 files changed, 23 insertions(+), 42 deletions(-) diff --git a/.github/workflows/code-health.yml b/.github/workflows/code-health.yml index 3522142a5..2f217b968 100644 --- a/.github/workflows/code-health.yml +++ b/.github/workflows/code-health.yml @@ -7,7 +7,7 @@ jobs: Black: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Black Check uses: jpetrucciani/black-check@7f5b2ad20fa5484f1884f07c1937e032ed8cd939 @@ -21,3 +21,13 @@ jobs: uses: jpetrucciani/mypy-check@179fdad632bf3ccf4cabb7ee4307ef25e51d2f96 with: path: scripts/*/*.py + + Golangci-lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: stable + - name: golangci-lint + uses: golangci/golangci-lint-action@v6 \ No newline at end of file diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index b922a90d8..75e310255 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -51,25 +51,3 @@ jobs: - name: Move the dependencies run: mv .venv /home/runner/work/_temp/_github_workflow - - # This part is not needed until we can add GO linting - # - name : Install Operator SDK - # run: | - # curl -s https://api.github.com/repos/operator-framework/operator-sdk/releases/latest | grep browser_download_url | grep x86_64-linux-gnu | cut -d '"' -f 4 | wget -i - - # sudo mv operator-sdk-*-x86_64-linux-gnu /usr/local/bin/operator-sdk - # sudo chmod 777 /usr/local/bin/operator-sdk - # - name: Generate DeepCopy - # Run: operator-sdk generate k8s - - - name: Lint Code Base - uses: docker://github/super-linter:v4 - env: - VALIDATE_ALL_CODEBASE: true - # Now we set the PYTHONPATH to the path of the dependencies *inside* the container - PYTHONPATH: "/github/workspace/:\ - /github/workflow/.venv/lib/python3.6/site-packages" - VALIDATE_YAML: true - VALIDATE_PYTHON: true - VALIDATE_BASH: true - FILTER_REGEX_EXCLUDE: "/helm-charts/charts/community-operator/templates/*" - # VALIDATE_GO: true This is currently broken: https://github.com/github/super-linter/issues/143 diff --git a/.golangci.yml b/.golangci.yml index 7e0ad5f76..3782ca8c7 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -23,17 +23,14 @@ linters: - staticcheck - unused - gosimple - - structcheck - - varcheck - ineffassign - - deadcode - typecheck - rowserrcheck - gosec - unconvert run: - modules-download-mode: + modules-download-mode: mod # timeout for analysis, e.g. 30s, 5m, default is 1m timeout: 5m # default concurrency is a available CPU number diff --git a/README.md b/README.md index ecbdcb079..b25eb2618 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,7 @@ Reviewers, please ensure that the CLA has been signed by referring to [the contr This project uses the following linters upon every Pull Request: +* `gosec` is a tool that find security problems in the code * `Black` is a tool that verifies if Python code is properly formatted * `MyPy` is a Static Type Checker for Python * `Kube-linter` is a tool that verified if all Kubernetes YAML manifests are formatted correctly diff --git a/cmd/readiness/readiness_test.go b/cmd/readiness/readiness_test.go index 97db1dc11..11222effa 100644 --- a/cmd/readiness/readiness_test.go +++ b/cmd/readiness/readiness_test.go @@ -106,7 +106,7 @@ func TestDeadlockDetection(t *testing.T) { isReadyExpected: false, }, } - for testName, _ := range tests { + for testName := range tests { testConfig := tests[testName] t.Run(testName, func(t *testing.T) { ready, err := isPodReady(ctx, testConfig.conf) @@ -225,7 +225,7 @@ func TestObtainingCurrentStep(t *testing.T) { expectedStep: "test", }, } - for testName, _ := range tests { + for testName := range tests { testConfig := tests[testName] t.Run(testName, func(t *testing.T) { step := findCurrentStep(testConfig.processStatuses) diff --git a/controllers/construct/mongodbstatefulset.go b/controllers/construct/mongodbstatefulset.go index 8f5a5fa81..a798dc27a 100644 --- a/controllers/construct/mongodbstatefulset.go +++ b/controllers/construct/mongodbstatefulset.go @@ -68,8 +68,8 @@ export NSS_WRAPPER_PASSWD=/tmp/passwd export LD_PRELOAD=libnss_wrapper.so export NSS_WRAPPER_GROUP=/etc/group fi - ` + //nolint:gosec //The credentials path is hardcoded in the container. MongodbUserCommandWithAPIKeyExport = `current_uid=$(id -u) AGENT_API_KEY="$(cat /mongodb-automation/agent-api-key/agentApiKey)" declare -r current_uid diff --git a/test/e2e/mongodbtests/mongodbtests.go b/test/e2e/mongodbtests/mongodbtests.go index 4a9b861ed..d4bfdd010 100644 --- a/test/e2e/mongodbtests/mongodbtests.go +++ b/test/e2e/mongodbtests/mongodbtests.go @@ -262,7 +262,7 @@ func HasExpectedMetadata(ctx context.Context, mdb *mdbv1.MongoDBCommunity, expec assert.NoError(t, err) assert.NotEmpty(t, statefulSetList.Items) for _, s := range statefulSetList.Items { - containsMetadata(t, &s.ObjectMeta, expectedLabels, expectedAnnotations, "statefulset "+s.Name) + containsMetadata(t, s.ObjectMeta, expectedLabels, expectedAnnotations, "statefulset "+s.Name) } volumeList := corev1.PersistentVolumeList{} @@ -272,7 +272,7 @@ func HasExpectedMetadata(ctx context.Context, mdb *mdbv1.MongoDBCommunity, expec for _, s := range volumeList.Items { volName := s.Name if strings.HasPrefix(volName, "data-volume-") || strings.HasPrefix(volName, "logs-volume-") { - containsMetadata(t, &s.ObjectMeta, expectedLabels, expectedAnnotations, "volume "+volName) + containsMetadata(t, s.ObjectMeta, expectedLabels, expectedAnnotations, "volume "+volName) } } @@ -304,12 +304,12 @@ func HasExpectedMetadata(ctx context.Context, mdb *mdbv1.MongoDBCommunity, expec continue } - containsMetadata(t, &s.ObjectMeta, expectedLabels, expectedAnnotations, "pod "+s.Name) + containsMetadata(t, s.ObjectMeta, expectedLabels, expectedAnnotations, "pod "+s.Name) } } } -func containsMetadata(t *testing.T, metadata *metav1.ObjectMeta, expectedLabels map[string]string, expectedAnnotations map[string]string, msg string) { +func containsMetadata(t *testing.T, metadata metav1.ObjectMeta, expectedLabels map[string]string, expectedAnnotations map[string]string, msg string) { labels := metadata.Labels for k, v := range expectedLabels { assert.Contains(t, labels, k, msg+" has label "+k) diff --git a/test/e2e/replica_set_arbiter/replica_set_arbiter_test.go b/test/e2e/replica_set_arbiter/replica_set_arbiter_test.go index 9d682acdc..0906dd900 100644 --- a/test/e2e/replica_set_arbiter/replica_set_arbiter_test.go +++ b/test/e2e/replica_set_arbiter/replica_set_arbiter_test.go @@ -69,9 +69,9 @@ func TestReplicaSetArbiter(t *testing.T) { resourceName: "mdb4", }, } - for testName, _ := range tests { + for testName := range tests { t.Run(testName, func(t *testing.T) { - testConfig, _ := tests[testName] + testConfig := tests[testName] mdb, user := e2eutil.NewTestMongoDB(testCtx, testConfig.resourceName, "") mdb.Spec.Arbiters = testConfig.numberOfArbiters mdb.Spec.Members = testConfig.numberOfMembers diff --git a/test/e2e/replica_set_operator_upgrade/replica_set_operator_upgrade_test.go b/test/e2e/replica_set_operator_upgrade/replica_set_operator_upgrade_test.go index be45239e7..9fe7278a9 100644 --- a/test/e2e/replica_set_operator_upgrade/replica_set_operator_upgrade_test.go +++ b/test/e2e/replica_set_operator_upgrade/replica_set_operator_upgrade_test.go @@ -73,7 +73,7 @@ func TestReplicaSetOperatorUpgrade(t *testing.T) { // TestReplicaSetOperatorUpgradeFrom0_7_2 is intended to be run locally not in CI. // It simulates deploying cluster using community operator 0.7.2 and then upgrading it using newer version. func TestReplicaSetOperatorUpgradeFrom0_7_2(t *testing.T) { - ctx := context.Background() + ctx := context.Background() //nolint t.Skip("Supporting this test in CI requires installing also CRDs from release v0.7.2") resourceName := "mdb-upg" testConfig := setup.LoadTestConfigFromEnv() diff --git a/test/e2e/util/wait/wait.go b/test/e2e/util/wait/wait.go index e599e935c..54798860e 100644 --- a/test/e2e/util/wait/wait.go +++ b/test/e2e/util/wait/wait.go @@ -161,11 +161,6 @@ func waitForStatefulSetCondition(ctx context.Context, t *testing.T, mdb *mdbv1.M return waitForStatefulSetConditionWithSpecificSts(ctx, t, mdb, MembersStatefulSet, waitOpts, condition) } -func waitForStatefulSetConditionArbiters(ctx context.Context, t *testing.T, mdb *mdbv1.MongoDBCommunity, waitOpts Options, condition func(set appsv1.StatefulSet) bool) error { - // uses members statefulset - return waitForStatefulSetConditionWithSpecificSts(ctx, t, mdb, ArbitersStatefulSet, waitOpts, condition) -} - func ForPodReadiness(ctx context.Context, t *testing.T, isReady bool, containerName string, timeout time.Duration, pod corev1.Pod) error { return wait.PollUntilContextTimeout(ctx, time.Second*3, timeout, false, func(ctx context.Context) (done bool, err error) { err = e2eutil.TestClient.Get(ctx, types.NamespacedName{Name: pod.Name, Namespace: pod.Namespace}, &pod) From 90e9374533f1630b3a9598277d1a7f34f1a2775d Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Mon, 24 Jun 2024 16:36:57 +0200 Subject: [PATCH 736/790] CLOUDP-253916 - readinessProbe: send to stdout as well, give option for compression and reduce default size (#1561) --- cmd/readiness/main.go | 21 +++++++++++++++++---- pkg/readiness/config/config.go | 27 ++++++++++++++++++--------- 2 files changed, 35 insertions(+), 13 deletions(-) diff --git a/cmd/readiness/main.go b/cmd/readiness/main.go index 14d369b22..ca3bfffb1 100644 --- a/cmd/readiness/main.go +++ b/cmd/readiness/main.go @@ -208,12 +208,25 @@ func parseHealthStatus(reader io.Reader) (health.Status, error) { } func initLogger(l *lumberjack.Logger) { - log := zap.New(zapcore.NewCore( + consoleCore := zapcore.NewCore( zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig()), - zapcore.AddSync(l), - zap.DebugLevel, - ), zap.Development()) + zapcore.AddSync(os.Stdout), + zap.DebugLevel) + + cores := []zapcore.Core{consoleCore} + if config.ReadBoolWitDefault(config.WithAgentFileLogging, "true") { + fileCore := zapcore.NewCore( + zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig()), + zapcore.AddSync(l), + zap.DebugLevel) + cores = append(cores, fileCore) + } + + core := zapcore.NewTee(cores...) + log := zap.New(core, zap.Development()) logger = log.Sugar() + + logger.Infof("logging configuration: %+v", l) } func main() { diff --git a/pkg/readiness/config/config.go b/pkg/readiness/config/config.go index d520faf7e..aa0357c33 100644 --- a/pkg/readiness/config/config.go +++ b/pkg/readiness/config/config.go @@ -15,15 +15,17 @@ import ( const ( DefaultAgentHealthStatusFilePath = "/var/log/mongodb-mms-automation/agent-health-status.json" AgentHealthStatusFilePathEnv = "AGENT_STATUS_FILEPATH" + WithAgentFileLogging = "MDB_WITH_AGENT_FILE_LOGGING" - defaultLogPath = "/var/log/mongodb-mms-automation/readiness.log" - podNamespaceEnv = "POD_NAMESPACE" - automationConfigSecretEnv = "AUTOMATION_CONFIG_MAP" //nolint - logPathEnv = "LOG_FILE_PATH" - hostNameEnv = "HOSTNAME" - readinessProbeLoggerBackups = "READINESS_PROBE_LOGGER_BACKUPS" - readinessProbeLoggerMaxSize = "READINESS_PROBE_LOGGER_MAX_SIZE" - readinessProbeLoggerMaxAge = "READINESS_PROBE_LOGGER_MAX_AGE" + defaultLogPath = "/var/log/mongodb-mms-automation/readiness.log" + podNamespaceEnv = "POD_NAMESPACE" + automationConfigSecretEnv = "AUTOMATION_CONFIG_MAP" //nolint + logPathEnv = "LOG_FILE_PATH" + hostNameEnv = "HOSTNAME" + readinessProbeLoggerBackups = "READINESS_PROBE_LOGGER_BACKUPS" + readinessProbeLoggerMaxSize = "READINESS_PROBE_LOGGER_MAX_SIZE" + readinessProbeLoggerMaxAge = "READINESS_PROBE_LOGGER_MAX_AGE" + readinessProbeLoggerCompress = "READINESS_PROBE_LOGGER_COMPRESS" ) type Config struct { @@ -71,8 +73,9 @@ func GetLogger() *lumberjack.Logger { logger := &lumberjack.Logger{ Filename: readinessProbeLogFilePath(), MaxBackups: readIntOrDefault(readinessProbeLoggerBackups, 5), - MaxSize: readInt(readinessProbeLoggerMaxSize), + MaxSize: readIntOrDefault(readinessProbeLoggerMaxSize, 5), MaxAge: readInt(readinessProbeLoggerMaxAge), + Compress: ReadBoolWitDefault(readinessProbeLoggerCompress, "false"), } return logger } @@ -105,3 +108,9 @@ func readIntOrDefault(envVarName string, defaultValue int) int { } return intValue } + +// ReadBoolWitDefault returns the boolean value of an envvar of the given name. +func ReadBoolWitDefault(envVarName string, defaultValue string) bool { + envVar := GetEnvOrDefault(envVarName, defaultValue) + return strings.TrimSpace(strings.ToLower(envVar)) == "true" +} From 0c330a21f96a4d1271e14d6f351d350cf5566f0a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 25 Jun 2024 10:37:05 +0200 Subject: [PATCH 737/790] Bump github.com/go-logr/logr from 1.4.1 to 1.4.2 (#1555) Bumps [github.com/go-logr/logr](https://github.com/go-logr/logr) from 1.4.1 to 1.4.2. - [Release notes](https://github.com/go-logr/logr/releases) - [Changelog](https://github.com/go-logr/logr/blob/master/CHANGELOG.md) - [Commits](https://github.com/go-logr/logr/compare/v1.4.1...v1.4.2) --- updated-dependencies: - dependency-name: github.com/go-logr/logr dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index f5ff8ad4f..97f4e340e 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.21 require ( github.com/blang/semver v3.5.1+incompatible - github.com/go-logr/logr v1.4.1 + github.com/go-logr/logr v1.4.2 github.com/hashicorp/go-multierror v1.1.1 github.com/imdario/mergo v0.3.15 github.com/spf13/cast v1.6.0 diff --git a/go.sum b/go.sum index 20c428f26..92af21d92 100644 --- a/go.sum +++ b/go.sum @@ -28,8 +28,8 @@ github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7z github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= github.com/go-logr/zapr v1.2.4/go.mod h1:FyHWQIzQORZ0QVE1BtVHv3cKtNLuXsbNLtpuhNapBOA= github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= From 6c9f5a0b10b2d4aa161cf5664983bab817c2d935 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 25 Jun 2024 10:37:16 +0200 Subject: [PATCH 738/790] Bump dnspython from 2.2.0 to 2.6.1 in /test/test-app (#1524) Bumps [dnspython](https://github.com/rthalley/dnspython) from 2.2.0 to 2.6.1. - [Release notes](https://github.com/rthalley/dnspython/releases) - [Changelog](https://github.com/rthalley/dnspython/blob/main/doc/whatsnew.rst) - [Commits](https://github.com/rthalley/dnspython/compare/v2.2.0...v2.6.1) --- updated-dependencies: - dependency-name: dnspython dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- test/test-app/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test-app/requirements.txt b/test/test-app/requirements.txt index cbd93a746..8d60e41a3 100644 --- a/test/test-app/requirements.txt +++ b/test/test-app/requirements.txt @@ -1,2 +1,2 @@ PyMongo==4.0.1 -dnspython==2.2.0 +dnspython==2.6.1 From 08428a48f7ca1d1139657bbda32235c52d094593 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 25 Jun 2024 11:29:44 +0200 Subject: [PATCH 739/790] Bump pymongo from 4.0.1 to 4.6.3 in /test/test-app (#1519) Bumps [pymongo](https://github.com/mongodb/mongo-python-driver) from 4.0.1 to 4.6.3. - [Release notes](https://github.com/mongodb/mongo-python-driver/releases) - [Changelog](https://github.com/mongodb/mongo-python-driver/blob/master/doc/changelog.rst) - [Commits](https://github.com/mongodb/mongo-python-driver/compare/4.0.1...4.6.3) --- updated-dependencies: - dependency-name: pymongo dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Nam Nguyen --- test/test-app/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test-app/requirements.txt b/test/test-app/requirements.txt index 8d60e41a3..c4165c3f4 100644 --- a/test/test-app/requirements.txt +++ b/test/test-app/requirements.txt @@ -1,2 +1,2 @@ -PyMongo==4.0.1 +PyMongo==4.6.3 dnspython==2.6.1 From 83a7e14887949ae9c7f9fe3d892e144b78c55854 Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Tue, 25 Jun 2024 13:00:06 +0200 Subject: [PATCH 740/790] CLOUDP-242559 - expose readinessProbe settings, add documentation (#1566) * expose readinessProbe settings, add documentation * update documentation --- README.md | 1 + controllers/construct/mongodbstatefulset.go | 41 ++++++-- .../construct/mongodbstatefulset_test.go | 97 +++++++++++++++++++ docs/README.md | 1 + docs/contributing.md | 2 +- docs/logging.md | 33 +++++++ pkg/readiness/config/config.go | 16 +-- 7 files changed, 175 insertions(+), 16 deletions(-) create mode 100644 controllers/construct/mongodbstatefulset_test.go create mode 100644 docs/logging.md diff --git a/README.md b/README.md index b25eb2618..8af30d276 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,7 @@ See the [documentation](docs) to learn how to: 1. [Install or upgrade](docs/install-upgrade.md) the Operator. 1. [Deploy and configure](docs/deploy-configure.md) MongoDB resources. +1. [Configure Logging](docs/logging.md) of the MongoDB resource components. 1. [Create a database user](docs/users.md) with SCRAM authentication. 1. [Secure MongoDB resource connections](docs/secure.md) using TLS. diff --git a/controllers/construct/mongodbstatefulset.go b/controllers/construct/mongodbstatefulset.go index a798dc27a..4f556f231 100644 --- a/controllers/construct/mongodbstatefulset.go +++ b/controllers/construct/mongodbstatefulset.go @@ -2,6 +2,7 @@ package construct import ( "fmt" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/readiness/config" "os" "strconv" "strings" @@ -262,10 +263,10 @@ func AutomationAgentCommand(withAgentAPIKeyExport bool, logLevel string, logFile // To keep consistent with old behavior not setting the logFile in the config does not log to stdout but keeps // the default logFile as defined by DefaultAgentLogFile. Setting the logFile explictly to "/dev/stdout" will log to stdout. agentLogOptions := "" - if logFile != "/dev/stdout" { - agentLogOptions += " -logFile " + logFile + " -logLevel " + logLevel + " -maxLogFileDurationHrs " + strconv.Itoa(maxLogFileDurationHours) - } else { + if logFile == "/dev/stdout" { agentLogOptions += " -logLevel " + logLevel + } else { + agentLogOptions += " -logFile " + logFile + " -logLevel " + logLevel + " -maxLogFileDurationHrs " + strconv.Itoa(maxLogFileDurationHours) } if withAgentAPIKeyExport { @@ -415,11 +416,37 @@ exec mongod -f %s; container.WithArgs([]string{""}), containerSecurityContext, container.WithEnvs( - corev1.EnvVar{ - Name: agentHealthStatusFilePathEnv, - Value: "/healthstatus/agent-health-status.json", - }, + collectEnvVars()..., ), container.WithVolumeMounts(volumeMounts), ) } + +// Function to collect and return the environment variables to be used in the +// MongoDB container. +func collectEnvVars() []corev1.EnvVar { + var envVars []corev1.EnvVar + + envVars = append(envVars, corev1.EnvVar{ + Name: agentHealthStatusFilePathEnv, + Value: "/healthstatus/agent-health-status.json", + }) + + addEnvVarIfSet := func(name string) { + value := os.Getenv(name) + if value != "" { + envVars = append(envVars, corev1.EnvVar{ + Name: name, + Value: value, + }) + } + } + + addEnvVarIfSet(config.ReadinessProbeLoggerBackups) + addEnvVarIfSet(config.ReadinessProbeLoggerMaxSize) + addEnvVarIfSet(config.ReadinessProbeLoggerMaxAge) + addEnvVarIfSet(config.ReadinessProbeLoggerCompress) + addEnvVarIfSet(config.WithAgentFileLogging) + + return envVars +} diff --git a/controllers/construct/mongodbstatefulset_test.go b/controllers/construct/mongodbstatefulset_test.go new file mode 100644 index 000000000..67d78174b --- /dev/null +++ b/controllers/construct/mongodbstatefulset_test.go @@ -0,0 +1,97 @@ +package construct + +import ( + "github.com/mongodb/mongodb-kubernetes-operator/pkg/readiness/config" + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + "testing" +) + +func TestCollectEnvVars(t *testing.T) { + tests := []struct { + name string + envSetup map[string]string + expectedEnv []corev1.EnvVar + }{ + { + name: "Basic env vars set", + envSetup: map[string]string{ + config.ReadinessProbeLoggerBackups: "3", + config.ReadinessProbeLoggerMaxSize: "10M", + config.ReadinessProbeLoggerMaxAge: "7", + config.WithAgentFileLogging: "enabled", + }, + expectedEnv: []corev1.EnvVar{ + { + Name: config.AgentHealthStatusFilePathEnv, + Value: "/healthstatus/agent-health-status.json", + }, + { + Name: config.ReadinessProbeLoggerBackups, + Value: "3", + }, + { + Name: config.ReadinessProbeLoggerMaxSize, + Value: "10M", + }, + { + Name: config.ReadinessProbeLoggerMaxAge, + Value: "7", + }, + { + Name: config.WithAgentFileLogging, + Value: "enabled", + }, + }, + }, + { + name: "Additional env var set", + envSetup: map[string]string{ + config.ReadinessProbeLoggerBackups: "3", + config.ReadinessProbeLoggerMaxSize: "10M", + config.ReadinessProbeLoggerMaxAge: "7", + config.ReadinessProbeLoggerCompress: "true", + config.WithAgentFileLogging: "enabled", + }, + expectedEnv: []corev1.EnvVar{ + { + Name: config.AgentHealthStatusFilePathEnv, + Value: "/healthstatus/agent-health-status.json", + }, + { + Name: config.ReadinessProbeLoggerBackups, + Value: "3", + }, + { + Name: config.ReadinessProbeLoggerMaxSize, + Value: "10M", + }, + { + Name: config.ReadinessProbeLoggerMaxAge, + Value: "7", + }, + { + Name: config.ReadinessProbeLoggerCompress, + Value: "true", + }, + { + Name: config.WithAgentFileLogging, + Value: "enabled", + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Setup environment variables + for key, value := range tt.envSetup { + t.Setenv(key, value) + } + + actualEnvVars := collectEnvVars() + + assert.EqualValues(t, tt.expectedEnv, actualEnvVars) + }) + } +} diff --git a/docs/README.md b/docs/README.md index 2d49ef706..7475a0d10 100644 --- a/docs/README.md +++ b/docs/README.md @@ -6,5 +6,6 @@ - [MongoDB Community Kubernetes Operator Architecture](architecture.md) - [Install and Upgrade the Community Kubernetes Operator](install-upgrade.md) - [Deploy and Configure MongoDBCommunity Resources](deploy-configure.md) +- [Configure Logging of the MongoDB components](logging.md) - [Create Database Users](users.md) - [Secure MongoDBCommunity Resources](secure.md) diff --git a/docs/contributing.md b/docs/contributing.md index f4bd4d85e..d1414eb86 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -140,7 +140,7 @@ kubectl get pods #### (Optional) Create a MongoDBCommunity Resource -Follow the steps outlined [here](./deploy-configure.md) to deploy some resource. +Follow the steps outlined [here](./deploy-configure.md) to deploy some resources. #### Cleanup To remove the operator and any created resources you can run diff --git a/docs/logging.md b/docs/logging.md new file mode 100644 index 000000000..021ae48ed --- /dev/null +++ b/docs/logging.md @@ -0,0 +1,33 @@ +# Configure Logging in MongoDB Community + +This section describes the components which are logging either to a file or stdout, +how to configure them and what their defaults are. + +## MongoDB Processes +### Configuration +The exposed CRD options can be seen [in the crd yaml](https://github.com/mongodb/mongodb-kubernetes-operator/blob/74d13f189566574b862e5670b366b61ec5b65923/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml#L105-L117). +Additionally, more information regarding configuring systemLog can be found [in the official documentation of systemLog](https://www.mongodb.com/docs/manual/reference/configuration-options/#core-options)]. +`spec.agent.systemLog.destination` configures the logging destination of the mongod process. +### Default Values +By default, MongoDB sends all log output to standard output. + +## MongoDB Agent +### Configuration +`spec.agent.logFile` can be used to configure the output file of the mongoDB agent logging. +The agent will log to standard output with the following setting: `/dev/stdout`. +### Default Values +By default, the MongoDB agent logs to `/var/log/mongodb-mms-automation/automation-agent.log` + +## ReadinessProbe +### Configuration & Default Values +The readinessProbe can be configured via Environment variables. +Below is a table with each environment variable, its explanation and its default value. + +| Environment Variable | Explanation | Default Value | +|---------------------------------|-------------------------------------------------------------------------|-----------------------------------------------| +| READINESS_PROBE_LOGGER_BACKUPS | maximum number of old log files to retain | 5 | +| READINESS_PROBE_LOGGER_MAX_SIZE | maximum size in megabytes | 5 | +| READINESS_PROBE_LOGGER_MAX_AGE | maximum number of days to retain old log files | none | +| READINESS_PROBE_LOGGER_COMPRESS | if the rotated log files should be compressed | false | +| MDB_WITH_AGENT_FILE_LOGGING | whether we should also log to stdout (which shows in kubectl describe) | true | +| LOG_FILE_PATH | path of the logfile of the readinessProbe. | /var/log/mongodb-mms-automation/readiness.log | \ No newline at end of file diff --git a/pkg/readiness/config/config.go b/pkg/readiness/config/config.go index aa0357c33..09b8ca50e 100644 --- a/pkg/readiness/config/config.go +++ b/pkg/readiness/config/config.go @@ -22,10 +22,10 @@ const ( automationConfigSecretEnv = "AUTOMATION_CONFIG_MAP" //nolint logPathEnv = "LOG_FILE_PATH" hostNameEnv = "HOSTNAME" - readinessProbeLoggerBackups = "READINESS_PROBE_LOGGER_BACKUPS" - readinessProbeLoggerMaxSize = "READINESS_PROBE_LOGGER_MAX_SIZE" - readinessProbeLoggerMaxAge = "READINESS_PROBE_LOGGER_MAX_AGE" - readinessProbeLoggerCompress = "READINESS_PROBE_LOGGER_COMPRESS" + ReadinessProbeLoggerBackups = "READINESS_PROBE_LOGGER_BACKUPS" + ReadinessProbeLoggerMaxSize = "READINESS_PROBE_LOGGER_MAX_SIZE" + ReadinessProbeLoggerMaxAge = "READINESS_PROBE_LOGGER_MAX_AGE" + ReadinessProbeLoggerCompress = "READINESS_PROBE_LOGGER_COMPRESS" ) type Config struct { @@ -72,10 +72,10 @@ func BuildFromEnvVariables(clientSet kubernetes.Interface, isHeadless bool, file func GetLogger() *lumberjack.Logger { logger := &lumberjack.Logger{ Filename: readinessProbeLogFilePath(), - MaxBackups: readIntOrDefault(readinessProbeLoggerBackups, 5), - MaxSize: readIntOrDefault(readinessProbeLoggerMaxSize, 5), - MaxAge: readInt(readinessProbeLoggerMaxAge), - Compress: ReadBoolWitDefault(readinessProbeLoggerCompress, "false"), + MaxBackups: readIntOrDefault(ReadinessProbeLoggerBackups, 5), + MaxSize: readIntOrDefault(ReadinessProbeLoggerMaxSize, 5), + MaxAge: readInt(ReadinessProbeLoggerMaxAge), + Compress: ReadBoolWitDefault(ReadinessProbeLoggerCompress, "false"), } return logger } From 9ed0d3ff4f342954bc78a7d8536885687188187b Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Wed, 26 Jun 2024 16:16:30 +0200 Subject: [PATCH 741/790] Release MongoDB Kubernetes Operator v0.10.0 (#1569) --- Makefile | 2 +- api/v1/zz_generated.deepcopy.go | 1 - ...ommunity.mongodb.com_mongodbcommunity.yaml | 225 ++++++++++-------- config/manager/manager.yaml | 6 +- deploy/openshift/operator_openshift.yaml | 6 +- docs/RELEASE_NOTES.md | 16 +- pkg/automationconfig/zz_generated.deepcopy.go | 1 - release.json | 4 +- scripts/ci/update_release.py | 2 +- 9 files changed, 146 insertions(+), 117 deletions(-) diff --git a/Makefile b/Makefile index 5626db9a9..a03dfb9bd 100644 --- a/Makefile +++ b/Makefile @@ -92,7 +92,7 @@ debug: install install-rbac ## Run the operator in debug mode with dlv CONTROLLER_GEN = $(shell pwd)/bin/controller-gen controller-gen: ## Download controller-gen locally if necessary - $(call go-install-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.11.3) + $(call go-install-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.15.0) # Try to use already installed helm from PATH ifeq (ok,$(shell test -f "$$(which helm)" && echo ok)) diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index 3ba0ad464..40a547fef 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -1,5 +1,4 @@ //go:build !ignore_autogenerated -// +build !ignore_autogenerated /* Copyright 2021. diff --git a/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml b/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml index ba3121647..1b1189970 100644 --- a/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml +++ b/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml @@ -1,11 +1,15 @@ - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.4.1 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.15.0 + service.binding: path={.metadata.name}-{.spec.users[0].db}-{.spec.users[0].name},objectType=Secret + service.binding/connectionString: path={.metadata.name}-{.spec.users[0].db}-{.spec.users[0].name},objectType=Secret,sourceKey=connectionString.standardSrv + service.binding/password: path={.metadata.name}-{.spec.users[0].db}-{.spec.users[0].name},objectType=Secret,sourceKey=password + service.binding/provider: community + service.binding/type: mongodb + service.binding/username: path={.metadata.name}-{.spec.users[0].db}-{.spec.users[0].name},objectType=Secret,sourceKey=username name: mongodbcommunity.mongodbcommunity.mongodb.com spec: group: mongodbcommunity.mongodb.com @@ -33,14 +37,19 @@ spec: description: MongoDBCommunity is the Schema for the mongodbs API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -54,9 +63,10 @@ spec: type: object x-kubernetes-preserve-unknown-fields: true additionalMongodConfig: - description: 'AdditionalMongodConfig is additional configuration that - can be passed to each data-bearing mongod at runtime. Uses the same - structure as the mongod configuration file: https://www.mongodb.com/docs/manual/reference/configuration-options/' + description: |- + AdditionalMongodConfig is additional configuration that can be passed to + each data-bearing mongod at runtime. Uses the same structure as the mongod + configuration file: https://www.mongodb.com/docs/manual/reference/configuration-options/ nullable: true type: object x-kubernetes-preserve-unknown-fields: true @@ -73,8 +83,9 @@ spec: processes. properties: includeAuditLogsWithMongoDBLogs: - description: set to 'true' to have the Automation Agent rotate - the audit files along with mongodb log files + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files type: boolean numTotal: description: maximum number of log files to have total @@ -83,14 +94,15 @@ spec: description: maximum number of log files to leave uncompressed type: integer percentOfDiskspace: - description: Maximum percentage of the total disk space these - log files should take up. The string needs to be able to - be converted to float64 + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 type: string sizeThresholdMB: - description: Maximum size for an individual log file before - rotation. The string needs to be able to be converted to - float64. Fractional values of MB are supported. + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. type: string timeThresholdHrs: description: maximum hours for an individual log file before @@ -118,14 +130,15 @@ spec: type: object type: object arbiters: - description: 'Arbiters is the number of arbiters to add to the Replica - Set. It is not recommended to have more than one arbiter per Replica - Set. More info: https://www.mongodb.com/docs/manual/tutorial/add-replica-set-arbiter/' + description: |- + Arbiters is the number of arbiters to add to the Replica Set. + It is not recommended to have more than one arbiter per Replica Set. + More info: https://www.mongodb.com/docs/manual/tutorial/add-replica-set-arbiter/ type: integer automationConfig: - description: AutomationConfigOverride is merged on top of the operator - created automation config. Processes are merged by name. Currently - Only the process.disabled field is supported. + description: |- + AutomationConfigOverride is merged on top of the operator created automation config. Processes are merged + by name. Currently Only the process.disabled field is supported. properties: processes: items: @@ -140,8 +153,9 @@ spec: as float64 properties: includeAuditLogsWithMongoDBLogs: - description: set to 'true' to have the Automation Agent - rotate the audit files along with mongodb log files + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files type: boolean numTotal: description: maximum number of log files to have total @@ -150,15 +164,15 @@ spec: description: maximum number of log files to leave uncompressed type: integer percentOfDiskspace: - description: Maximum percentage of the total disk space - these log files should take up. The string needs to - be able to be converted to float64 + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 type: string sizeThresholdMB: - description: Maximum size for an individual log file - before rotation. The string needs to be able to be - converted to float64. Fractional values of MB are - supported. + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. type: string timeThresholdHrs: description: maximum hours for an individual log file @@ -178,8 +192,9 @@ spec: replicaSet: properties: settings: - description: MapWrapper is a wrapper for a map to be used - by other structs. The CRD generator does not support map[string]interface{} + description: |- + MapWrapper is a wrapper for a map to be used by other structs. + The CRD generator does not support map[string]interface{} on the top level and hence we need to work around this with a wrapping struct. type: object @@ -187,8 +202,9 @@ spec: type: object type: object featureCompatibilityVersion: - description: FeatureCompatibilityVersion configures the feature compatibility - version that will be set for the deployment + description: |- + FeatureCompatibilityVersion configures the feature compatibility version that will + be set for the deployment type: string memberConfig: description: MemberConfig @@ -233,8 +249,9 @@ spec: to 9216. type: integer tlsSecretKeyRef: - description: Name of a Secret (type kubernetes.io/tls) holding - the certificates to use in the Prometheus endpoint. + description: |- + Name of a Secret (type kubernetes.io/tls) holding the certificates to use in the + Prometheus endpoint. properties: key: description: Key is the key in the secret storing this password. @@ -255,12 +272,13 @@ spec: - username type: object replicaSetHorizons: - description: ReplicaSetHorizons Add this parameter and values if you - need your database to be accessed outside of Kubernetes. This setting - allows you to provide different DNS settings within the Kubernetes - cluster and to the Kubernetes cluster. The Kubernetes Operator uses - split horizon DNS for replica set members. This feature allows communication - both within the Kubernetes cluster and from outside Kubernetes. + description: |- + ReplicaSetHorizons Add this parameter and values if you need your database + to be accessed outside of Kubernetes. This setting allows you to + provide different DNS settings within the Kubernetes cluster and + to the Kubernetes cluster. The Kubernetes Operator uses split horizon + DNS for replica set members. This feature allows communication both + within the Kubernetes cluster and from outside Kubernetes. items: additionalProperties: type: string @@ -273,18 +291,21 @@ spec: authentication: properties: agentCertificateSecretRef: - description: 'AgentCertificateSecret is a reference to a Secret - containing the certificate and the key for the automation - agent The secret needs to have available: - certificate - under key: "tls.crt" - private key under key: "tls.key" - If additionally, tls.pem is present, then it needs to be - equal to the concatenation of tls.crt and tls.key' + description: |- + AgentCertificateSecret is a reference to a Secret containing the certificate and the key for the automation agent + The secret needs to have available: + - certificate under key: "tls.crt" + - private key under key: "tls.key" + If additionally, tls.pem is present, then it needs to be equal to the concatenation of tls.crt and tls.key properties: name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object + x-kubernetes-map-type: atomic agentMode: description: AgentMode contains the authentication mode used by the automation agent. @@ -322,9 +343,9 @@ spec: description: The authentication restrictions the server enforces on the role. items: - description: AuthenticationRestriction specifies a list - of IP addresses and CIDR ranges users are allowed to - connect to or from. + description: |- + AuthenticationRestriction specifies a list of IP addresses and CIDR ranges users + are allowed to connect to or from. properties: clientSource: items: @@ -353,9 +374,9 @@ spec: type: string type: array resource: - description: Resource specifies specifies the resources - upon which a privilege permits actions. See https://www.mongodb.com/docs/manual/reference/resource-document - for more. + description: |- + Resource specifies specifies the resources upon which a privilege permits actions. + See https://www.mongodb.com/docs/manual/reference/resource-document for more. properties: anyResource: type: boolean @@ -403,45 +424,48 @@ spec: communication properties: caCertificateSecretRef: - description: CaCertificateSecret is a reference to a Secret - containing the certificate for the CA which signed the server - certificates The certificate is expected to be available - under the key "ca.crt" + description: |- + CaCertificateSecret is a reference to a Secret containing the certificate for the CA which signed the server certificates + The certificate is expected to be available under the key "ca.crt" properties: name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object + x-kubernetes-map-type: atomic caConfigMapRef: - description: CaConfigMap is a reference to a ConfigMap containing - the certificate for the CA which signed the server certificates - The certificate is expected to be available under the key - "ca.crt" This field is ignored when CaCertificateSecretRef - is configured + description: |- + CaConfigMap is a reference to a ConfigMap containing the certificate for the CA which signed the server certificates + The certificate is expected to be available under the key "ca.crt" + This field is ignored when CaCertificateSecretRef is configured properties: name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object + x-kubernetes-map-type: atomic certificateKeySecretRef: - description: CertificateKeySecret is a reference to a Secret - containing a private key and certificate to use for TLS. - The key and cert are expected to be PEM encoded and available - at "tls.key" and "tls.crt". This is the same format used - for the standard "kubernetes.io/tls" Secret type, but no - specific type is required. Alternatively, an entry tls.pem, - containing the concatenation of cert and key, can be provided. - If all of tls.pem, tls.crt and tls.key are present, the - tls.pem one needs to be equal to the concatenation of tls.crt - and tls.key + description: |- + CertificateKeySecret is a reference to a Secret containing a private key and certificate to use for TLS. + The key and cert are expected to be PEM encoded and available at "tls.key" and "tls.crt". + This is the same format used for the standard "kubernetes.io/tls" Secret type, but no specific type is required. + Alternatively, an entry tls.pem, containing the concatenation of cert and key, can be provided. + If all of tls.pem, tls.crt and tls.key are present, the tls.pem one needs to be equal to the concatenation of tls.crt and tls.key properties: name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object + x-kubernetes-map-type: atomic enabled: type: boolean optional: @@ -453,7 +477,8 @@ spec: type: object type: object statefulSet: - description: StatefulSetConfiguration holds the optional custom StatefulSet + description: |- + StatefulSetConfiguration holds the optional custom StatefulSet that should be merged into the operator created one. properties: metadata: @@ -487,17 +512,16 @@ spec: items: properties: additionalConnectionStringConfig: - description: Additional options to be appended to the connection - string. These options apply only to this user and will override - any existing options in the resource. + description: |- + Additional options to be appended to the connection string. + These options apply only to this user and will override any existing options in the resource. nullable: true type: object x-kubernetes-preserve-unknown-fields: true connectionStringSecretName: - description: ConnectionStringSecretName is the name of the secret - object created by the operator which exposes the connection - strings for the user. If provided, this secret must be different - for each user in a deployment. + description: |- + ConnectionStringSecretName is the name of the secret object created by the operator which exposes the connection strings for the user. + If provided, this secret must be different for each user in a deployment. type: string connectionStringSecretNamespace: description: ConnectionStringSecretNamespace is the namespace @@ -544,10 +568,9 @@ spec: type: object type: array scramCredentialsSecretName: - description: ScramCredentialsSecretName appended by string "scram-credentials" - is the name of the secret object created by the mongoDB operator - for storing SCRAM credentials These secrets names must be - different for each user in a deployment. + description: |- + ScramCredentialsSecretName appended by string "scram-credentials" is the name of the secret object created by the mongoDB operator for storing SCRAM credentials + These secrets names must be different for each user in a deployment. pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ type: string required: @@ -593,9 +616,3 @@ spec: storage: true subresources: status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 458df3826..f9b53dca4 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -45,16 +45,16 @@ spec: - name: OPERATOR_NAME value: mongodb-kubernetes-operator - name: AGENT_IMAGE - value: quay.io/mongodb/mongodb-agent:107.0.0.8465-1 + value: quay.io/mongodb/mongodb-agent:107.0.1.8507-1 - name: VERSION_UPGRADE_HOOK_IMAGE value: quay.io/mongodb/mongodb-kubernetes-operator-version-upgrade-post-start-hook:1.0.8 - name: READINESS_PROBE_IMAGE - value: quay.io/mongodb/mongodb-kubernetes-readinessprobe:1.0.17 + value: quay.io/mongodb/mongodb-kubernetes-readinessprobe:1.0.19 - name: MONGODB_IMAGE value: mongodb-community-server - name: MONGODB_REPO_URL value: quay.io/mongodb - image: quay.io/mongodb/mongodb-kubernetes-operator:0.9.0 + image: quay.io/mongodb/mongodb-kubernetes-operator:0.10.0 imagePullPolicy: Always name: mongodb-kubernetes-operator resources: diff --git a/deploy/openshift/operator_openshift.yaml b/deploy/openshift/operator_openshift.yaml index 6a9c39f0f..1d9fd43a8 100644 --- a/deploy/openshift/operator_openshift.yaml +++ b/deploy/openshift/operator_openshift.yaml @@ -47,16 +47,16 @@ spec: - name: OPERATOR_NAME value: mongodb-kubernetes-operator - name: AGENT_IMAGE - value: quay.io/mongodb/mongodb-agent:107.0.0.8465-1 + value: quay.io/mongodb/mongodb-agent:107.0.1.8507-1 - name: READINESS_PROBE_IMAGE - value: quay.io/mongodb/mongodb-kubernetes-readinessprobe:1.0.17 + value: quay.io/mongodb/mongodb-kubernetes-readinessprobe:1.0.19 - name: VERSION_UPGRADE_HOOK_IMAGE value: quay.io/mongodb/mongodb-kubernetes-operator-version-upgrade-post-start-hook:1.0.8 - name: MONGODB_IMAGE value: mongo - name: MONGODB_REPO_URL value: quay.io - image: quay.io/mongodb/mongodb-kubernetes-operator:0.9.0 + image: quay.io/mongodb/mongodb-kubernetes-operator:0.10.0 imagePullPolicy: Always name: mongodb-kubernetes-operator resources: diff --git a/docs/RELEASE_NOTES.md b/docs/RELEASE_NOTES.md index f0dbc1519..dd47234e7 100644 --- a/docs/RELEASE_NOTES.md +++ b/docs/RELEASE_NOTES.md @@ -2,4 +2,18 @@ ## Released images signed -All container images published for the community operator are signed with our private key. This is visible on our Quay registry. Signature can be verified using our public key, which is available at [this address](https://cosign.mongodb.com/mongodb-enterprise-kubernetes-operator.pem). \ No newline at end of file +All container images published for the community operator are signed with our private key. This is visible on our Quay registry. Signature can be verified using our public key, which is available at [this address](https://cosign.mongodb.com/mongodb-enterprise-kubernetes-operator.pem). + +## Logging changes +- The agent logging can be configured to stdout +- ReadinessProbe logging configuration can now be configured +- More can be found [here](logging.md). + +## Overriding Mongod settings via the CRD +- Example can be found [here](../config/samples/mongodb.com_v1_mongodbcommunity_override_ac_setting.yaml). + +## ReadinessProbe error logging +- fixed a red herring which caused the probe to panic when the health status is not available. Instead it will just log the error + +## Important Bumps +- Bumped K8S libs to 1.27 diff --git a/pkg/automationconfig/zz_generated.deepcopy.go b/pkg/automationconfig/zz_generated.deepcopy.go index 0a4e123ab..723f24ba9 100644 --- a/pkg/automationconfig/zz_generated.deepcopy.go +++ b/pkg/automationconfig/zz_generated.deepcopy.go @@ -1,5 +1,4 @@ //go:build !ignore_autogenerated -// +build !ignore_autogenerated /* Copyright 2021. diff --git a/release.json b/release.json index 217d7c1e3..e07804fc6 100644 --- a/release.json +++ b/release.json @@ -1,8 +1,8 @@ { "golang-builder-image": "golang:1.21", - "operator": "0.9.0", + "operator": "0.10.0", "version-upgrade-hook": "1.0.8", - "readiness-probe": "1.0.18", + "readiness-probe": "1.0.19", "agent": "107.0.1.8507-1", "agent-tools-version": "100.9.4" } \ No newline at end of file diff --git a/scripts/ci/update_release.py b/scripts/ci/update_release.py index 1c5f6b2c6..96c76746f 100755 --- a/scripts/ci/update_release.py +++ b/scripts/ci/update_release.py @@ -58,7 +58,7 @@ def update_operator_deployment(operator_deployment: Dict, release: Dict) -> None if env["name"] == "READINESS_PROBE_IMAGE": env["value"] = _replace_tag(env["value"], release["readiness-probe"]) if env["name"] == "AGENT_IMAGE": - env["value"] = _replace_tag(env["value"], release["agent"]["version"]) + env["value"] = _replace_tag(env["value"], release["agent"]) def update_chart_values(values: Dict, release: Dict) -> None: From 270ae2a52a50346fc157d2bb833238bb3193bdab Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Wed, 26 Jun 2024 16:32:30 +0200 Subject: [PATCH 742/790] bump helm charts (#1570) --- helm-charts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm-charts b/helm-charts index 6ddf86b1b..512a25c0e 160000 --- a/helm-charts +++ b/helm-charts @@ -1 +1 @@ -Subproject commit 6ddf86b1b00cdd807840de36fc97b91466ee6981 +Subproject commit 512a25c0eb9b582851c3240f084d49b2977c090d From efc91be7f6e747b47e00a493435c4188aed5f54b Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Thu, 27 Jun 2024 11:55:38 +0200 Subject: [PATCH 743/790] Correct argument order for AutomationAgentCommand call (#1573) --- api/v1/mongodbcommunity_types.go | 8 ++++---- controllers/construct/build_statefulset_test.go | 4 ++-- controllers/construct/mongodbstatefulset.go | 12 ++++++------ 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/api/v1/mongodbcommunity_types.go b/api/v1/mongodbcommunity_types.go index 0f7128713..8c78b8a4e 100644 --- a/api/v1/mongodbcommunity_types.go +++ b/api/v1/mongodbcommunity_types.go @@ -356,10 +356,10 @@ type LogLevel string const ( LogLevelDebug LogLevel = "DEBUG" - LogLevelInfo string = "INFO" - LogLevelWarn string = "WARN" - LogLevelError string = "ERROR" - LogLevelFatal string = "FATAL" + LogLevelInfo LogLevel = "INFO" + LogLevelWarn LogLevel = "WARN" + LogLevelError LogLevel = "ERROR" + LogLevelFatal LogLevel = "FATAL" ) type AgentConfiguration struct { diff --git a/controllers/construct/build_statefulset_test.go b/controllers/construct/build_statefulset_test.go index bb5dceeaa..56e3bcaac 100644 --- a/controllers/construct/build_statefulset_test.go +++ b/controllers/construct/build_statefulset_test.go @@ -177,14 +177,14 @@ func TestMongod_Container(t *testing.T) { } func TestMongoDBAgentCommand(t *testing.T) { - cmd := AutomationAgentCommand(false, "INFO", "testfile", 24) + cmd := AutomationAgentCommand(false, mdbv1.LogLevelInfo, "testfile", 24) baseCmd := MongodbUserCommand + BaseAgentCommand() + " -cluster=" + clusterFilePath + automationAgentOptions assert.Len(t, cmd, 3) assert.Equal(t, cmd[0], "/bin/bash") assert.Equal(t, cmd[1], "-c") assert.Equal(t, cmd[2], baseCmd+" -logFile testfile -logLevel INFO -maxLogFileDurationHrs 24") - cmd = AutomationAgentCommand(false, "INFO", "/dev/stdout", 24) + cmd = AutomationAgentCommand(false, mdbv1.LogLevelInfo, "/dev/stdout", 24) assert.Len(t, cmd, 3) assert.Equal(t, cmd[0], "/bin/bash") assert.Equal(t, cmd[1], "-c") diff --git a/controllers/construct/mongodbstatefulset.go b/controllers/construct/mongodbstatefulset.go index 4f556f231..a6a71ff9b 100644 --- a/controllers/construct/mongodbstatefulset.go +++ b/controllers/construct/mongodbstatefulset.go @@ -207,7 +207,7 @@ func BuildMongoDBReplicaSetStatefulSetModificationFunction(mdb MongoDBStatefulSe agentLogLevel := mdbv1.LogLevelInfo if mdb.GetAgentLogLevel() != "" { - agentLogLevel = string(mdb.GetAgentLogLevel()) + agentLogLevel = mdb.GetAgentLogLevel() } agentLogFile := automationconfig.DefaultAgentLogFile @@ -256,7 +256,7 @@ func BaseAgentCommand() string { // AutomationAgentCommand withAgentAPIKeyExport detects whether we want to deploy this agent with the agent api key exported // it can be used to register the agent with OM. -func AutomationAgentCommand(withAgentAPIKeyExport bool, logLevel string, logFile string, maxLogFileDurationHours int) []string { +func AutomationAgentCommand(withAgentAPIKeyExport bool, logLevel mdbv1.LogLevel, logFile string, maxLogFileDurationHours int) []string { // This is somewhat undocumented at https://www.mongodb.com/docs/ops-manager/current/reference/mongodb-agent-settings/ // Not setting the -logFile option make the mongodb-agent log to stdout. Setting -logFile /dev/stdout will result in // an error by the agent trying to open /dev/stdout-verbose and still trying to do log rotation. @@ -264,9 +264,9 @@ func AutomationAgentCommand(withAgentAPIKeyExport bool, logLevel string, logFile // the default logFile as defined by DefaultAgentLogFile. Setting the logFile explictly to "/dev/stdout" will log to stdout. agentLogOptions := "" if logFile == "/dev/stdout" { - agentLogOptions += " -logLevel " + logLevel + agentLogOptions += " -logLevel " + string(logLevel) } else { - agentLogOptions += " -logFile " + logFile + " -logLevel " + logLevel + " -maxLogFileDurationHrs " + strconv.Itoa(maxLogFileDurationHours) + agentLogOptions += " -logFile " + logFile + " -logLevel " + string(logLevel) + " -maxLogFileDurationHrs " + strconv.Itoa(maxLogFileDurationHours) } if withAgentAPIKeyExport { @@ -275,7 +275,7 @@ func AutomationAgentCommand(withAgentAPIKeyExport bool, logLevel string, logFile return []string{"/bin/bash", "-c", MongodbUserCommand + BaseAgentCommand() + " -cluster=" + clusterFilePath + automationAgentOptions + agentLogOptions} } -func mongodbAgentContainer(automationConfigSecretName string, volumeMounts []corev1.VolumeMount, logLevel string, logFile string, maxLogFileDurationHours int, agentImage string) container.Modification { +func mongodbAgentContainer(automationConfigSecretName string, volumeMounts []corev1.VolumeMount, logLevel mdbv1.LogLevel, logFile string, maxLogFileDurationHours int, agentImage string) container.Modification { _, containerSecurityContext := podtemplatespec.WithDefaultSecurityContextsModifications() return container.Apply( container.WithName(AgentName), @@ -284,7 +284,7 @@ func mongodbAgentContainer(automationConfigSecretName string, volumeMounts []cor container.WithReadinessProbe(DefaultReadiness()), container.WithResourceRequirements(resourcerequirements.Defaults()), container.WithVolumeMounts(volumeMounts), - container.WithCommand(AutomationAgentCommand(false, logFile, logLevel, maxLogFileDurationHours)), + container.WithCommand(AutomationAgentCommand(false, logLevel, logFile, maxLogFileDurationHours)), containerSecurityContext, container.WithEnvs( corev1.EnvVar{ From 3108a941c0f5b04a6853a426cf98e9a1e36ebdc1 Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Tue, 2 Jul 2024 11:25:44 +0200 Subject: [PATCH 744/790] migrate all agents to ubi (#1577) --- config/manager/manager.yaml | 2 +- deploy/openshift/operator_openshift.yaml | 2 +- scripts/dev/templates/agent/README.md | 4 ++-- .../replica_set_operator_upgrade_test.go | 2 +- test/e2e/setup/test_config.go | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index f9b53dca4..044421f42 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -45,7 +45,7 @@ spec: - name: OPERATOR_NAME value: mongodb-kubernetes-operator - name: AGENT_IMAGE - value: quay.io/mongodb/mongodb-agent:107.0.1.8507-1 + value: quay.io/mongodb/mongodb-agent-ubi:107.0.1.8507-1 - name: VERSION_UPGRADE_HOOK_IMAGE value: quay.io/mongodb/mongodb-kubernetes-operator-version-upgrade-post-start-hook:1.0.8 - name: READINESS_PROBE_IMAGE diff --git a/deploy/openshift/operator_openshift.yaml b/deploy/openshift/operator_openshift.yaml index 1d9fd43a8..5eb385528 100644 --- a/deploy/openshift/operator_openshift.yaml +++ b/deploy/openshift/operator_openshift.yaml @@ -47,7 +47,7 @@ spec: - name: OPERATOR_NAME value: mongodb-kubernetes-operator - name: AGENT_IMAGE - value: quay.io/mongodb/mongodb-agent:107.0.1.8507-1 + value: quay.io/mongodb/mongodb-agent-ubi:107.0.1.8507-1 - name: READINESS_PROBE_IMAGE value: quay.io/mongodb/mongodb-kubernetes-readinessprobe:1.0.19 - name: VERSION_UPGRADE_HOOK_IMAGE diff --git a/scripts/dev/templates/agent/README.md b/scripts/dev/templates/agent/README.md index 0b9f1f797..d7ea6554a 100644 --- a/scripts/dev/templates/agent/README.md +++ b/scripts/dev/templates/agent/README.md @@ -3,6 +3,6 @@ This folder contains the `Dockerfile` used to build [https://quay.io/repository/mongodb/mongodb-agent](https://quay.io/repository/mongodb/mongodb-agent). To build and push a new image to quay.io, use the following commands: ``` -docker build . --build-arg agent_version=${agent_version} --build-arg tools_version=${tools_version} -t "quay.io/mongodb/mongodb-agent:${agent_version}" -docker push "quay.io/mongodb/mongodb-agent:${agent_version}" +docker build . --build-arg agent_version=${agent_version} --build-arg tools_version=${tools_version} -t "quay.io/mongodb/mongodb-agent-ubi:${agent_version}" +docker push "quay.io/mongodb/mongodb-agent-ubi:${agent_version}" ``` \ No newline at end of file diff --git a/test/e2e/replica_set_operator_upgrade/replica_set_operator_upgrade_test.go b/test/e2e/replica_set_operator_upgrade/replica_set_operator_upgrade_test.go index 9fe7278a9..726c52514 100644 --- a/test/e2e/replica_set_operator_upgrade/replica_set_operator_upgrade_test.go +++ b/test/e2e/replica_set_operator_upgrade/replica_set_operator_upgrade_test.go @@ -82,7 +82,7 @@ func TestReplicaSetOperatorUpgradeFrom0_7_2(t *testing.T) { testConfig.OperatorImage = "quay.io/mongodb/mongodb-kubernetes-operator:0.7.2" testConfig.VersionUpgradeHookImage = "quay.io/mongodb/mongodb-kubernetes-operator-version-upgrade-post-start-hook:1.0.3" testConfig.ReadinessProbeImage = "quay.io/mongodb/mongodb-kubernetes-readinessprobe:1.0.6" - testConfig.AgentImage = "quay.io/mongodb/mongodb-agent:11.0.5.6963-1" + testConfig.AgentImage = "quay.io/mongodb/mongodb-agent-ubi:11.0.5.6963-1" testCtx := setup.SetupWithTestConfig(ctx, t, testConfig, true, false, resourceName) defer testCtx.Teardown() diff --git a/test/e2e/setup/test_config.go b/test/e2e/setup/test_config.go index 19e7c5d92..06f2018fe 100644 --- a/test/e2e/setup/test_config.go +++ b/test/e2e/setup/test_config.go @@ -41,7 +41,7 @@ func LoadTestConfigFromEnv() TestConfig { MongoDBImage: envvar.GetEnvOrDefault(construct.MongodbImageEnv, "mongodb-community-server"), MongoDBRepoUrl: envvar.GetEnvOrDefault(construct.MongodbRepoUrl, "quay.io/mongodb"), VersionUpgradeHookImage: envvar.GetEnvOrDefault(construct.VersionUpgradeHookImageEnv, "quay.io/mongodb/mongodb-kubernetes-operator-version-upgrade-post-start-hook:1.0.2"), - AgentImage: envvar.GetEnvOrDefault(construct.AgentImageEnv, "quay.io/mongodb/mongodb-agent:10.29.0.6830-1"), // TODO: better way to decide default agent image. + AgentImage: envvar.GetEnvOrDefault(construct.AgentImageEnv, "quay.io/mongodb/mongodb-agent-ubi:10.29.0.6830-1"), // TODO: better way to decide default agent image. ClusterWide: envvar.ReadBool(clusterWideEnvName), PerformCleanup: envvar.ReadBool(performCleanupEnvName), ReadinessProbeImage: envvar.GetEnvOrDefault(construct.ReadinessProbeImageEnv, "quay.io/mongodb/mongodb-kubernetes-readinessprobe:1.0.3"), From ec4a131d819c58fbc13c4af9aaf9b275e30662c6 Mon Sep 17 00:00:00 2001 From: Matei Grigore <43472562+MateiGrigore@users.noreply.github.com> Date: Fri, 5 Jul 2024 17:22:39 +0100 Subject: [PATCH 745/790] Modify misleading or outdated documentation (#1583) * Modify missleading or outdated documentation * Fix indentation * Removed redundant flag --- docs/secure.md | 27 ++++++++++++++------------- docs/users.md | 2 +- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/docs/secure.md b/docs/secure.md index f2b7d877a..e1a1e8631 100644 --- a/docs/secure.md +++ b/docs/secure.md @@ -36,20 +36,21 @@ To secure connections to MongoDBCommunity resources with TLS using `cert-manager helm repo update ``` -1. Install `cert-manager`: +2. Install `cert-manager`: ``` - helm install cert-manager jetstack/cert-manager --namespace cert-manager \ - --create-namespace --set installCRDs=true + helm install cert-manager jetstack/cert-manager --namespace cert-manager --create-namespace --set crds.enabled=true ``` -1. Create a TLS-secured MongoDBCommunity resource: +3. Create a TLS-secured MongoDBCommunity resource: + + This assumes you already have the operator installed in namespace `` ``` helm upgrade --install community-operator mongodb/community-operator \ - --namespace mongodb --set resource.tls.useCertManager=true \ + --namespace --set resource.tls.useCertManager=true \ --set createResource=true --set resource.tls.enabled=true \ - --set namespace=mongodb --create-namespace + --set namespace= ``` This creates a resource secured with TLS and generates the necessary @@ -72,21 +73,21 @@ To secure connections to MongoDBCommunity resources with TLS using `cert-manager 1. Test your connection over TLS by - - Connecting to a `mongod` container using `kubectl`: + - Connecting to a `mongod` container inside a pod using `kubectl`: ``` - kubectl exec -it mongodb-replica-set -c mongod -- bash + kubectl exec -it -c mongod -- bash ``` - Where `mongodb-replica-set` is the name of your MongoDBCommunity resource + Where `mongodb-replica-set-pod` is the name of a pod from your MongoDBCommunity resource - Then, use `mongosh` to connect over TLS: + For how to get the connection string look at [Deploy A Replica Set](deploy-configure.md#deploy-a-replica-set) ``` - mongosh --tls --tlsCAFile /var/lib/tls/ca/ca.crt --tlsCertificateKeyFile \ - /var/lib/tls/server/*.pem \ - --host .-svc..svc.cluster.local + mongosh "" --tls --tlsCAFile /var/lib/tls/ca/ca.crt --tlsCertificateKeyFile /var/lib/tls/server/*.pem ``` Where `mongodb-replica-set` is the name of your MongoDBCommunity - resource and `namespace` is the namespace of your deployment. \ No newline at end of file + resource, `namespace` is the namespace of your deployment + and `connection-string` is a connection string for your `-svc` service. \ No newline at end of file diff --git a/docs/users.md b/docs/users.md index a1980e4fb..96a44570a 100644 --- a/docs/users.md +++ b/docs/users.md @@ -84,6 +84,6 @@ You cannot disable SCRAM authentication. - To authenticate to your MongoDBCommunity resource, run the following command: ``` - mongo "mongodb://..svc.cluster.local:27017/?replicaSet=" --username --password --authenticationDatabase + mongosh "mongodb://-svc..svc.cluster.local:27017/?replicaSet=" --username --password --authenticationDatabase ``` - To change a user's password, create and apply a new secret resource definition with a `metadata.name` that is the same as the name specified in `passwordSecretRef.name` of the MongoDB CRD. The Operator will automatically regenerate credentials. From 76f4e23b8b193123afd171c605e8d32d5a928c11 Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Wed, 10 Jul 2024 12:56:15 +0200 Subject: [PATCH 746/790] bump community helm charts commit (#1584) --- helm-charts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm-charts b/helm-charts index 512a25c0e..b8b68108e 160000 --- a/helm-charts +++ b/helm-charts @@ -1 +1 @@ -Subproject commit 512a25c0eb9b582851c3240f084d49b2977c090d +Subproject commit b8b68108e6128f5c1eba70a3c4b5a5594671d69e From c85178f69d30237da3210fd1e0fa931ba8a06e5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20=C5=81askawiec?= Date: Wed, 10 Jul 2024 18:49:04 +0200 Subject: [PATCH 747/790] Libraries upgrade (#1585) --- go.mod | 10 +++++----- go.sum | 24 ++++++++++++------------ 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/go.mod b/go.mod index 97f4e340e..aca5f18a2 100644 --- a/go.mod +++ b/go.mod @@ -65,13 +65,13 @@ require ( github.com/xdg-go/stringprep v1.0.4 // indirect github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect go.uber.org/multierr v1.10.0 // indirect - golang.org/x/crypto v0.21.0 // indirect + golang.org/x/crypto v0.25.0 // indirect golang.org/x/net v0.23.0 // indirect golang.org/x/oauth2 v0.7.0 // indirect - golang.org/x/sync v0.5.0 // indirect - golang.org/x/sys v0.18.0 // indirect - golang.org/x/term v0.18.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/sync v0.7.0 // indirect + golang.org/x/sys v0.22.0 // indirect + golang.org/x/term v0.22.0 // indirect + golang.org/x/text v0.16.0 // indirect golang.org/x/time v0.3.0 // indirect gomodules.xyz/jsonpatch/v2 v2.3.0 // indirect google.golang.org/appengine v1.6.7 // indirect diff --git a/go.sum b/go.sum index 92af21d92..8f5f79f5f 100644 --- a/go.sum +++ b/go.sum @@ -178,8 +178,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= +golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -210,8 +210,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= -golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -221,19 +221,19 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= -golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= +golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -245,8 +245,8 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.16.1 h1:TLyB3WofjdOEepBHAU20JdNC1Zbg87elYofWYAY5oZA= -golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From 1322fd47a6a530cc3e857b0d695c99b7a8c3e1c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Lima?= Date: Fri, 12 Jul 2024 10:16:12 +0100 Subject: [PATCH 748/790] fix: use correct namespace for password secret (#1581) --- controllers/mongodb_users.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/controllers/mongodb_users.go b/controllers/mongodb_users.go index 2af312d2c..fd3311011 100644 --- a/controllers/mongodb_users.go +++ b/controllers/mongodb_users.go @@ -62,7 +62,7 @@ func (r ReplicaSetReconciler) updateConnectionStringSecrets(ctx context.Context, pwd := "" if user.Database != constants.ExternalDB { - secretNamespacedName := types.NamespacedName{Name: user.PasswordSecretName, Namespace: secretNamespace} + secretNamespacedName := types.NamespacedName{Name: user.PasswordSecretName, Namespace: mdb.Namespace} pwd, err = secret.ReadKey(ctx, r.client, user.PasswordSecretKey, secretNamespacedName) if err != nil { return err From 5f90b99157dac88385fb6772041e2a8ca79950b5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 12 Jul 2024 11:16:58 +0200 Subject: [PATCH 749/790] Bump go.mongodb.org/mongo-driver from 1.15.0 to 1.16.0 (#1576) Bumps [go.mongodb.org/mongo-driver](https://github.com/mongodb/mongo-go-driver) from 1.15.0 to 1.16.0. - [Release notes](https://github.com/mongodb/mongo-go-driver/releases) - [Commits](https://github.com/mongodb/mongo-go-driver/compare/v1.15.0...v1.16.0) --- updated-dependencies: - dependency-name: go.mongodb.org/mongo-driver dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 8 ++++---- go.sum | 15 ++++++++------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index aca5f18a2..94a3ad34a 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/stretchr/objx v0.5.2 github.com/stretchr/testify v1.9.0 github.com/xdg/stringprep v1.0.3 - go.mongodb.org/mongo-driver v1.15.0 + go.mongodb.org/mongo-driver v1.16.0 go.uber.org/zap v1.27.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 k8s.io/api v0.27.12 @@ -37,9 +37,9 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect - github.com/golang/snappy v0.0.3 // indirect + github.com/golang/snappy v0.0.4 // indirect github.com/google/gnostic v0.5.7-v3refs // indirect - github.com/google/go-cmp v0.5.9 // indirect + github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.1.0 // indirect github.com/google/uuid v1.3.0 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect @@ -51,7 +51,7 @@ require ( github.com/moby/spdystream v0.2.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect + github.com/montanaflynn/stats v0.7.1 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect diff --git a/go.sum b/go.sum index 8f5f79f5f..bfcd68e97 100644 --- a/go.sum +++ b/go.sum @@ -59,8 +59,8 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= -github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54= github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -68,8 +68,9 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -112,8 +113,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0= -github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE= +github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q= @@ -166,8 +167,8 @@ github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7Jul github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.mongodb.org/mongo-driver v1.15.0 h1:rJCKC8eEliewXjZGf0ddURtl7tTVy1TK3bfl0gkUSLc= -go.mongodb.org/mongo-driver v1.15.0/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c= +go.mongodb.org/mongo-driver v1.16.0 h1:tpRsfBJMROVHKpdGyc1BBEzzjDUWjItxbVSZ8Ls4BQ4= +go.mongodb.org/mongo-driver v1.16.0/go.mod h1:oB6AhJQvFQL4LEHyXi6aJzQJtBiTQHiAd83l0GdFaiw= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= From 604ef867822c48ba685f2b143d500f7bf8bed506 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20=C5=81askawiec?= Date: Fri, 19 Jul 2024 12:12:16 +0200 Subject: [PATCH 750/790] Bump setuptools (#1590) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b73c33e40..e86c8aa2f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,6 +15,6 @@ requests==2.32.3 ruamel.yaml==0.17.9 semver==2.13.0 rsa>=4.7 # not directly required, pinned by Snyk to avoid a vulnerability -setuptools>=65.5.1 # not directly required, pinned by Snyk to avoid a vulnerability +setuptools==71.0.3 # not directly required, pinned by Snyk to avoid a vulnerability certifi>=2022.12.7 # not directly required, pinned by Snyk to avoid a vulnerability urllib3<2 # not directly required, pinned by Snyk to avoid a vulnerability From 35e9ae1f85a686210caad80445810332f8a6226d Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Mon, 22 Jul 2024 11:55:02 +0200 Subject: [PATCH 751/790] fix logRotate and fix path naming add auditlogrotation (#1591) --- api/v1/mongodbcommunity_types.go | 3 +++ controllers/mongodb_tls_test.go | 1 + controllers/replica_set_controller.go | 2 +- controllers/replicaset_controller_test.go | 3 +++ pkg/automationconfig/automation_config.go | 12 ++++++++++-- .../automation_config_secret_test.go | 8 ++++++-- 6 files changed, 24 insertions(+), 5 deletions(-) diff --git a/api/v1/mongodbcommunity_types.go b/api/v1/mongodbcommunity_types.go index 8c78b8a4e..cd91000c1 100644 --- a/api/v1/mongodbcommunity_types.go +++ b/api/v1/mongodbcommunity_types.go @@ -373,6 +373,9 @@ type AgentConfiguration struct { // LogRotate if enabled, will enable LogRotate for all processes. LogRotate *automationconfig.CrdLogRotate `json:"logRotate,omitempty"` // +optional + // AuditLogRotate if enabled, will enable AuditLogRotate for all processes. + AuditLogRotate *automationconfig.CrdLogRotate `json:"AuditLogRotate,omitempty"` + // +optional // SystemLog configures system log of mongod SystemLog *automationconfig.SystemLog `json:"systemLog,omitempty"` } diff --git a/controllers/mongodb_tls_test.go b/controllers/mongodb_tls_test.go index 8724738a3..f6c21b6b9 100644 --- a/controllers/mongodb_tls_test.go +++ b/controllers/mongodb_tls_test.go @@ -329,6 +329,7 @@ func TestAutomationConfigIsCorrectlyConfiguredWithTLS(t *testing.T) { assert.Equal(t, "/tmp/test", process.Args26.Get("systemLog.path").String()) assert.Equal(t, "file", process.Args26.Get("systemLog.destination").String()) assert.Equal(t, process.LogRotate, automationconfig.ConvertCrdLogRotateToAC(mdb.Spec.AgentConfiguration.LogRotate)) + assert.Equal(t, process.AuditLogRotate, automationconfig.ConvertCrdLogRotateToAC(mdb.Spec.AgentConfiguration.AuditLogRotate)) } }) diff --git a/controllers/replica_set_controller.go b/controllers/replica_set_controller.go index 2d9355a57..4c2b1a0e8 100644 --- a/controllers/replica_set_controller.go +++ b/controllers/replica_set_controller.go @@ -535,7 +535,7 @@ func buildAutomationConfig(mdb mdbv1.MongoDBCommunity, auth automationconfig.Aut AddModifications(getMongodConfigModification(mdb)). AddModifications(modifications...). AddProcessModification(func(_ int, p *automationconfig.Process) { - automationconfig.ConfigureAgentConfiguration(mdb.Spec.AgentConfiguration.SystemLog, mdb.Spec.AgentConfiguration.LogRotate, p) + automationconfig.ConfigureAgentConfiguration(mdb.Spec.AgentConfiguration.SystemLog, mdb.Spec.AgentConfiguration.LogRotate, mdb.Spec.AgentConfiguration.AuditLogRotate, p) }). Build() } diff --git a/controllers/replicaset_controller_test.go b/controllers/replicaset_controller_test.go index 91094c936..da5780d62 100644 --- a/controllers/replicaset_controller_test.go +++ b/controllers/replicaset_controller_test.go @@ -88,6 +88,9 @@ func newTestReplicaSetWithSystemLogAndLogRotate() mdbv1.MongoDBCommunity { LogRotate: &automationconfig.CrdLogRotate{ SizeThresholdMB: "1", }, + AuditLogRotate: &automationconfig.CrdLogRotate{ + SizeThresholdMB: "1", + }, SystemLog: &automationconfig.SystemLog{ Destination: automationconfig.File, Path: "/tmp/test", diff --git a/pkg/automationconfig/automation_config.go b/pkg/automationconfig/automation_config.go index b3dc5a36b..c5883a921 100644 --- a/pkg/automationconfig/automation_config.go +++ b/pkg/automationconfig/automation_config.go @@ -135,7 +135,8 @@ type Process struct { ProcessType ProcessType `json:"processType"` Version string `json:"version"` AuthSchemaVersion int `json:"authSchemaVersion"` - LogRotate *AcLogRotate `json:"LogRotate,omitempty"` + LogRotate *AcLogRotate `json:"logRotate,omitempty"` + AuditLogRotate *AcLogRotate `json:"auditLogRotate,omitempty"` } func (p *Process) SetPort(port int) *Process { @@ -180,6 +181,12 @@ func (p *Process) SetLogRotate(lr *CrdLogRotate) *Process { return p } +// SetAuditLogRotate sets the acLogRotate by converting the CrdLogRotate to an acLogRotate. +func (p *Process) SetAuditLogRotate(lr *CrdLogRotate) *Process { + p.AuditLogRotate = ConvertCrdLogRotateToAC(lr) + return p +} + // ConvertCrdLogRotateToAC converts a CrdLogRotate to an AcLogRotate representation. func ConvertCrdLogRotateToAC(lr *CrdLogRotate) *AcLogRotate { if lr == nil { @@ -478,7 +485,7 @@ func FromBytes(acBytes []byte) (AutomationConfig, error) { return ac, nil } -func ConfigureAgentConfiguration(systemLog *SystemLog, logRotate *CrdLogRotate, p *Process) { +func ConfigureAgentConfiguration(systemLog *SystemLog, logRotate *CrdLogRotate, auditLR *CrdLogRotate, p *Process) { if systemLog != nil { p.SetSystemLog(*systemLog) } @@ -491,6 +498,7 @@ func ConfigureAgentConfiguration(systemLog *SystemLog, logRotate *CrdLogRotate, zap.S().Warn("Configuring LogRotate with systemLog.Destination = Syslog will not work") } p.SetLogRotate(logRotate) + p.SetAuditLogRotate(auditLR) } } diff --git a/pkg/automationconfig/automation_config_secret_test.go b/pkg/automationconfig/automation_config_secret_test.go index e6d601e0e..ed9a4af77 100644 --- a/pkg/automationconfig/automation_config_secret_test.go +++ b/pkg/automationconfig/automation_config_secret_test.go @@ -44,7 +44,7 @@ func TestEnsureSecret(t *testing.T) { ctx := context.Background() desiredAutomationConfig, err = NewBuilder().SetMembers(3).AddProcessModification(func(i_ int, p *Process) { - p.SetLogRotate(&CrdLogRotate{ + lr := &CrdLogRotate{ SizeThresholdMB: "0.001", LogRotate: LogRotate{ TimeThresholdHrs: 1, @@ -53,7 +53,9 @@ func TestEnsureSecret(t *testing.T) { IncludeAuditLogsWithMongoDBLogs: false, }, PercentOfDiskspace: "1", - }) + } + p.SetLogRotate(lr) + p.SetAuditLogRotate(lr) }).Build() assert.NoError(t, err) @@ -72,7 +74,9 @@ func TestEnsureSecret(t *testing.T) { acFromBytes, err := FromBytes(bytes) assert.NoError(t, err) assert.Equal(t, 0.001, acFromBytes.Processes[0].LogRotate.SizeThresholdMB) + assert.Equal(t, 0.001, acFromBytes.Processes[0].AuditLogRotate.SizeThresholdMB) assert.Equal(t, float64(1), acFromBytes.Processes[0].LogRotate.PercentOfDiskspace) + assert.Equal(t, float64(1), acFromBytes.Processes[0].AuditLogRotate.PercentOfDiskspace) }) t.Run("test LogRotate marshal and unmarshal if not set", func(t *testing.T) { From 31acaf75fdd65bd69aab4be4001176914dc53cc2 Mon Sep 17 00:00:00 2001 From: Matei Grigore <43472562+MateiGrigore@users.noreply.github.com> Date: Tue, 23 Jul 2024 17:50:46 +0100 Subject: [PATCH 752/790] CLOUDP-196371: Delete users removed from the resource (#1587) --- .action_templates/jobs/tests.yaml | 2 + .github/workflows/e2e-fork.yml | 2 + .github/workflows/e2e.yml | 2 + controllers/mongodb_cleanup.go | 67 +++++++-- controllers/mongodb_cleanup_test.go | 93 ++++++++++++- controllers/mongodb_users.go | 3 + controllers/replica_set_controller.go | 21 +-- pkg/authentication/authentication.go | 34 +++++ pkg/authentication/authentication_test.go | 94 +++++++++++++ pkg/automationconfig/automation_config.go | 9 ++ test/e2e/mongodbtests/mongodbtests.go | 54 ++++++++ .../replica_set_remove_user_test.go | 127 ++++++++++++++++++ 12 files changed, 486 insertions(+), 22 deletions(-) create mode 100644 test/e2e/replica_set_remove_user/replica_set_remove_user_test.go diff --git a/.action_templates/jobs/tests.yaml b/.action_templates/jobs/tests.yaml index 6128df9ed..400b47d77 100644 --- a/.action_templates/jobs/tests.yaml +++ b/.action_templates/jobs/tests.yaml @@ -62,3 +62,5 @@ tests: distro: ubi - test-name: replica_set_x509 distro: ubi + - test-name: replica_set_remove_user + distro: ubi diff --git a/.github/workflows/e2e-fork.yml b/.github/workflows/e2e-fork.yml index c5e0f4ef1..e3d465118 100644 --- a/.github/workflows/e2e-fork.yml +++ b/.github/workflows/e2e-fork.yml @@ -147,6 +147,8 @@ jobs: distro: ubi - test-name: replica_set_x509 distro: ubi + - test-name: replica_set_remove_user + distro: ubi steps: # template: .action_templates/steps/cancel-previous.yaml - name: Cancel Previous Runs diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 90e2013c6..4f43cf203 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -153,6 +153,8 @@ jobs: distro: ubi - test-name: replica_set_x509 distro: ubi + - test-name: replica_set_remove_user + distro: ubi steps: # template: .action_templates/steps/cancel-previous.yaml - name: Cancel Previous Runs diff --git a/controllers/mongodb_cleanup.go b/controllers/mongodb_cleanup.go index 85d720ec0..d13b0426d 100644 --- a/controllers/mongodb_cleanup.go +++ b/controllers/mongodb_cleanup.go @@ -10,12 +10,12 @@ import ( ) // cleanupPemSecret cleans up the old pem secret generated for the agent certificate. -func (r *ReplicaSetReconciler) cleanupPemSecret(ctx context.Context, currentMDB mdbv1.MongoDBCommunitySpec, lastAppliedMDBSpec mdbv1.MongoDBCommunitySpec, namespace string) { - if currentMDB.GetAgentAuthMode() == lastAppliedMDBSpec.GetAgentAuthMode() { +func (r *ReplicaSetReconciler) cleanupPemSecret(ctx context.Context, currentMDBSpec mdbv1.MongoDBCommunitySpec, lastAppliedMDBSpec mdbv1.MongoDBCommunitySpec, namespace string) { + if currentMDBSpec.GetAgentAuthMode() == lastAppliedMDBSpec.GetAgentAuthMode() { return } - if !currentMDB.IsAgentX509() && lastAppliedMDBSpec.IsAgentX509() { + if !currentMDBSpec.IsAgentX509() && lastAppliedMDBSpec.IsAgentX509() { agentCertSecret := lastAppliedMDBSpec.GetAgentCertificateRef() if err := r.client.DeleteSecret(ctx, types.NamespacedName{ Namespace: namespace, @@ -31,22 +31,38 @@ func (r *ReplicaSetReconciler) cleanupPemSecret(ctx context.Context, currentMDB } // cleanupScramSecrets cleans up old scram secrets based on the last successful applied mongodb spec. -func (r *ReplicaSetReconciler) cleanupScramSecrets(ctx context.Context, currentMDB mdbv1.MongoDBCommunitySpec, lastAppliedMDBSpec mdbv1.MongoDBCommunitySpec, namespace string) { - secretsToDelete := getScramSecretsToDelete(currentMDB, lastAppliedMDBSpec) +func (r *ReplicaSetReconciler) cleanupScramSecrets(ctx context.Context, currentMDBSpec mdbv1.MongoDBCommunitySpec, lastAppliedMDBSpec mdbv1.MongoDBCommunitySpec, namespace string) { + secretsToDelete := getScramSecretsToDelete(currentMDBSpec, lastAppliedMDBSpec) for _, s := range secretsToDelete { if err := r.client.DeleteSecret(ctx, types.NamespacedName{ Name: s, Namespace: namespace, }); err != nil { - r.log.Warnf("Could not cleanup old secret %s", s) + r.log.Warnf("Could not cleanup old secret %s: %s", s, err) } else { r.log.Debugf("Sucessfully cleaned up secret: %s", s) } } } -func getScramSecretsToDelete(currentMDB mdbv1.MongoDBCommunitySpec, lastAppliedMDBSpec mdbv1.MongoDBCommunitySpec) []string { +// cleanupConnectionStringSecrets cleans up old scram secrets based on the last successful applied mongodb spec. +func (r *ReplicaSetReconciler) cleanupConnectionStringSecrets(ctx context.Context, currentMDBSpec mdbv1.MongoDBCommunitySpec, lastAppliedMDBSpec mdbv1.MongoDBCommunitySpec, namespace string, resourceName string) { + secretsToDelete := getConnectionStringSecretsToDelete(currentMDBSpec, lastAppliedMDBSpec, resourceName) + + for _, s := range secretsToDelete { + if err := r.client.DeleteSecret(ctx, types.NamespacedName{ + Name: s, + Namespace: namespace, + }); err != nil { + r.log.Warnf("Could not cleanup old secret %s: %s", s, err) + } else { + r.log.Debugf("Sucessfully cleaned up secret: %s", s) + } + } +} + +func getScramSecretsToDelete(currentMDBSpec mdbv1.MongoDBCommunitySpec, lastAppliedMDBSpec mdbv1.MongoDBCommunitySpec) []string { type user struct { db string name string @@ -54,10 +70,11 @@ func getScramSecretsToDelete(currentMDB mdbv1.MongoDBCommunitySpec, lastAppliedM m := map[user]string{} var secretsToDelete []string - for _, mongoDBUser := range currentMDB.Users { - if mongoDBUser.DB != constants.ExternalDB { - m[user{db: mongoDBUser.DB, name: mongoDBUser.Name}] = mongoDBUser.GetScramCredentialsSecretName() + for _, mongoDBUser := range currentMDBSpec.Users { + if mongoDBUser.DB == constants.ExternalDB { + continue } + m[user{db: mongoDBUser.DB, name: mongoDBUser.Name}] = mongoDBUser.GetScramCredentialsSecretName() } for _, mongoDBUser := range lastAppliedMDBSpec.Users { @@ -73,3 +90,33 @@ func getScramSecretsToDelete(currentMDB mdbv1.MongoDBCommunitySpec, lastAppliedM } return secretsToDelete } + +func getConnectionStringSecretsToDelete(currentMDBSpec mdbv1.MongoDBCommunitySpec, lastAppliedMDBSpec mdbv1.MongoDBCommunitySpec, resourceName string) []string { + type user struct { + db string + name string + } + m := map[user]string{} + var secretsToDelete []string + + for _, mongoDBUser := range currentMDBSpec.Users { + if mongoDBUser.DB == constants.ExternalDB { + continue + } + m[user{db: mongoDBUser.DB, name: mongoDBUser.Name}] = mongoDBUser.GetConnectionStringSecretName(resourceName) + } + + for _, mongoDBUser := range lastAppliedMDBSpec.Users { + if mongoDBUser.DB == constants.ExternalDB { + continue + } + currentConnectionStringSecretName, ok := m[user{db: mongoDBUser.DB, name: mongoDBUser.Name}] + if !ok { // user was removed + secretsToDelete = append(secretsToDelete, mongoDBUser.GetConnectionStringSecretName(resourceName)) + } else if currentConnectionStringSecretName != mongoDBUser.GetConnectionStringSecretName(resourceName) { + // this happens when a new ConnectionStringSecretName was set for the old user + secretsToDelete = append(secretsToDelete, mongoDBUser.GetConnectionStringSecretName(resourceName)) + } + } + return secretsToDelete +} diff --git a/controllers/mongodb_cleanup_test.go b/controllers/mongodb_cleanup_test.go index 472e77266..249556061 100644 --- a/controllers/mongodb_cleanup_test.go +++ b/controllers/mongodb_cleanup_test.go @@ -22,8 +22,7 @@ func TestReplicaSetReconcilerCleanupScramSecrets(t *testing.T) { t.Run("no change same resource", func(t *testing.T) { actual := getScramSecretsToDelete(lastApplied.Spec, lastApplied.Spec) - var expected []string - assert.Equal(t, expected, actual) + assert.Equal(t, []string(nil), actual) }) t.Run("new user new secret", func(t *testing.T) { @@ -44,10 +43,9 @@ func TestReplicaSetReconcilerCleanupScramSecrets(t *testing.T) { }, ) - var expected []string actual := getScramSecretsToDelete(current.Spec, lastApplied.Spec) - assert.Equal(t, expected, actual) + assert.Equal(t, []string(nil), actual) }) t.Run("old user new secret", func(t *testing.T) { @@ -154,3 +152,90 @@ func TestReplicaSetReconcilerCleanupPemSecret(t *testing.T) { _, err = r.client.GetSecret(ctx, mdb.AgentCertificatePemSecretNamespacedName()) assert.Error(t, err) } + +func TestReplicaSetReconcilerCleanupConnectionStringSecrets(t *testing.T) { + lastApplied := newScramReplicaSet(mdbv1.MongoDBUser{ + Name: "testUser", + PasswordSecretRef: mdbv1.SecretKeyReference{ + Name: "password-secret-name", + }, + ConnectionStringSecretName: "connection-string-secret", + }) + + t.Run("no change same resource", func(t *testing.T) { + actual := getConnectionStringSecretsToDelete(lastApplied.Spec, lastApplied.Spec, "my-rs") + + assert.Equal(t, []string(nil), actual) + }) + + t.Run("new user does not require existing user cleanup", func(t *testing.T) { + current := newScramReplicaSet( + mdbv1.MongoDBUser{ + Name: "testUser", + PasswordSecretRef: mdbv1.SecretKeyReference{ + Name: "password-secret-name", + }, + ConnectionStringSecretName: "connection-string-secret", + }, + mdbv1.MongoDBUser{ + Name: "newUser", + PasswordSecretRef: mdbv1.SecretKeyReference{ + Name: "password-secret-name", + }, + ConnectionStringSecretName: "connection-string-secret-2", + }, + ) + + actual := getConnectionStringSecretsToDelete(current.Spec, lastApplied.Spec, "my-rs") + + assert.Equal(t, []string(nil), actual) + }) + + t.Run("old user new secret", func(t *testing.T) { + current := newScramReplicaSet(mdbv1.MongoDBUser{ + Name: "testUser", + PasswordSecretRef: mdbv1.SecretKeyReference{ + Name: "password-secret-name", + }, + ConnectionStringSecretName: "connection-string-secret-2", + }) + + expected := []string{"connection-string-secret"} + actual := getConnectionStringSecretsToDelete(current.Spec, lastApplied.Spec, "my-rs") + + assert.Equal(t, expected, actual) + }) + + t.Run("removed one user and changed secret of the other", func(t *testing.T) { + lastApplied = newScramReplicaSet( + mdbv1.MongoDBUser{ + Name: "testUser", + PasswordSecretRef: mdbv1.SecretKeyReference{ + Name: "password-secret-name", + }, + ConnectionStringSecretName: "connection-string-secret", + }, + mdbv1.MongoDBUser{ + Name: "anotherUser", + PasswordSecretRef: mdbv1.SecretKeyReference{ + Name: "password-secret-name", + }, + ConnectionStringSecretName: "connection-string-secret-2", + }, + ) + + current := newScramReplicaSet(mdbv1.MongoDBUser{ + Name: "testUser", + PasswordSecretRef: mdbv1.SecretKeyReference{ + Name: "password-secret-name", + }, + ConnectionStringSecretName: "connection-string-secret-1", + }) + + expected := []string{"connection-string-secret", "connection-string-secret-2"} + actual := getConnectionStringSecretsToDelete(current.Spec, lastApplied.Spec, "my-rs") + + assert.Equal(t, expected, actual) + }) + +} diff --git a/controllers/mongodb_users.go b/controllers/mongodb_users.go index fd3311011..cd99734ba 100644 --- a/controllers/mongodb_users.go +++ b/controllers/mongodb_users.go @@ -82,6 +82,9 @@ func (r ReplicaSetReconciler) updateConnectionStringSecrets(ctx context.Context, if err := secret.CreateOrUpdate(ctx, r.client, connectionStringSecret); err != nil { return err } + + secretNamespacedName := types.NamespacedName{Name: connectionStringSecret.Name, Namespace: connectionStringSecret.Namespace} + r.secretWatcher.Watch(ctx, secretNamespacedName, mdb.NamespacedName()) } return nil diff --git a/controllers/replica_set_controller.go b/controllers/replica_set_controller.go index 4c2b1a0e8..e956790ba 100644 --- a/controllers/replica_set_controller.go +++ b/controllers/replica_set_controller.go @@ -173,7 +173,7 @@ func (r ReplicaSetReconciler) Reconcile(ctx context.Context, request reconcile.R withFailedPhase()) } - ready, err := r.deployMongoDBReplicaSet(ctx, mdb) + ready, err := r.deployMongoDBReplicaSet(ctx, mdb, lastAppliedSpec) if err != nil { return status.Update(ctx, r.client.Status(), &mdb, statusOptions(). withMessage(Error, fmt.Sprintf("Error deploying MongoDB ReplicaSet: %s", err)). @@ -225,6 +225,7 @@ func (r ReplicaSetReconciler) Reconcile(ctx context.Context, request reconcile.R if lastAppliedSpec != nil { r.cleanupScramSecrets(ctx, mdb.Spec, *lastAppliedSpec, mdb.Namespace) r.cleanupPemSecret(ctx, mdb.Spec, *lastAppliedSpec, mdb.Namespace) + r.cleanupConnectionStringSecrets(ctx, mdb.Spec, *lastAppliedSpec, mdb.Namespace, mdb.Name) } if err := r.updateLastSuccessfulConfiguration(ctx, mdb); err != nil { @@ -331,7 +332,7 @@ func (r *ReplicaSetReconciler) deployStatefulSet(ctx context.Context, mdb mdbv1. // deployAutomationConfig deploys the AutomationConfig for the MongoDBCommunity resource. // The returned boolean indicates whether or not that Agents have all reached goal state. -func (r *ReplicaSetReconciler) deployAutomationConfig(ctx context.Context, mdb mdbv1.MongoDBCommunity) (bool, error) { +func (r *ReplicaSetReconciler) deployAutomationConfig(ctx context.Context, mdb mdbv1.MongoDBCommunity, lastAppliedSpec *mdbv1.MongoDBCommunitySpec) (bool, error) { r.log.Infof("Creating/Updating AutomationConfig") sts, err := r.client.GetStatefulSet(ctx, mdb.NamespacedName()) @@ -339,7 +340,7 @@ func (r *ReplicaSetReconciler) deployAutomationConfig(ctx context.Context, mdb m return false, fmt.Errorf("failed to get StatefulSet: %s", err) } - ac, err := r.ensureAutomationConfig(mdb, ctx) + ac, err := r.ensureAutomationConfig(mdb, ctx, lastAppliedSpec) if err != nil { return false, fmt.Errorf("failed to ensure AutomationConfig: %s", err) } @@ -408,10 +409,10 @@ func (r *ReplicaSetReconciler) shouldRunInOrder(ctx context.Context, mdb mdbv1.M // deployMongoDBReplicaSet will ensure that both the AutomationConfig secret and backing StatefulSet // have been successfully created. A boolean is returned indicating if the process is complete // and an error if there was one. -func (r *ReplicaSetReconciler) deployMongoDBReplicaSet(ctx context.Context, mdb mdbv1.MongoDBCommunity) (bool, error) { +func (r *ReplicaSetReconciler) deployMongoDBReplicaSet(ctx context.Context, mdb mdbv1.MongoDBCommunity, lastAppliedSpec *mdbv1.MongoDBCommunitySpec) (bool, error) { return functions.RunSequentially(r.shouldRunInOrder(ctx, mdb), func() (bool, error) { - return r.deployAutomationConfig(ctx, mdb) + return r.deployAutomationConfig(ctx, mdb, lastAppliedSpec) }, func() (bool, error) { return r.deployStatefulSet(ctx, mdb) @@ -489,8 +490,8 @@ func (r *ReplicaSetReconciler) createOrUpdateStatefulSet(ctx context.Context, md // ensureAutomationConfig makes sure the AutomationConfig secret has been successfully created. The automation config // that was updated/created is returned. -func (r ReplicaSetReconciler) ensureAutomationConfig(mdb mdbv1.MongoDBCommunity, ctx context.Context) (automationconfig.AutomationConfig, error) { - ac, err := r.buildAutomationConfig(ctx, mdb) +func (r ReplicaSetReconciler) ensureAutomationConfig(mdb mdbv1.MongoDBCommunity, ctx context.Context, lastAppliedSpec *mdbv1.MongoDBCommunitySpec) (automationconfig.AutomationConfig, error) { + ac, err := r.buildAutomationConfig(ctx, mdb, lastAppliedSpec) if err != nil { return automationconfig.AutomationConfig{}, fmt.Errorf("could not build automation config: %s", err) } @@ -622,7 +623,7 @@ func getCustomRolesModification(mdb mdbv1.MongoDBCommunity) (automationconfig.Mo }, nil } -func (r ReplicaSetReconciler) buildAutomationConfig(ctx context.Context, mdb mdbv1.MongoDBCommunity) (automationconfig.AutomationConfig, error) { +func (r ReplicaSetReconciler) buildAutomationConfig(ctx context.Context, mdb mdbv1.MongoDBCommunity, lastAppliedSpec *mdbv1.MongoDBCommunitySpec) (automationconfig.AutomationConfig, error) { tlsModification, err := getTLSConfigModification(ctx, r.client, r.client, mdb) if err != nil { return automationconfig.AutomationConfig{}, fmt.Errorf("could not configure TLS modification: %s", err) @@ -643,6 +644,10 @@ func (r ReplicaSetReconciler) buildAutomationConfig(ctx context.Context, mdb mdb return automationconfig.AutomationConfig{}, err } + if lastAppliedSpec != nil { + authentication.AddRemovedUsers(&auth, mdb, lastAppliedSpec) + } + prometheusModification := automationconfig.NOOP() if mdb.Spec.Prometheus != nil { secretNamespacedName := types.NamespacedName{Name: mdb.Spec.Prometheus.PasswordSecretRef.Name, Namespace: mdb.Namespace} diff --git a/pkg/authentication/authentication.go b/pkg/authentication/authentication.go index b73c97898..a856bda66 100644 --- a/pkg/authentication/authentication.go +++ b/pkg/authentication/authentication.go @@ -3,6 +3,7 @@ package authentication import ( "context" "fmt" + mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" "k8s.io/apimachinery/pkg/types" @@ -33,3 +34,36 @@ func Enable(ctx context.Context, auth *automationconfig.Auth, secretGetUpdateCre } return nil } + +func AddRemovedUsers(auth *automationconfig.Auth, mdb mdbv1.MongoDBCommunity, lastAppliedSpec *mdbv1.MongoDBCommunitySpec) { + deletedUsers := getRemovedUsersFromSpec(mdb.Spec, lastAppliedSpec) + + auth.UsersDeleted = append(auth.UsersDeleted, deletedUsers...) +} + +func getRemovedUsersFromSpec(currentMDB mdbv1.MongoDBCommunitySpec, lastAppliedMDBSpec *mdbv1.MongoDBCommunitySpec) []automationconfig.DeletedUser { + type user struct { + db string + name string + } + m := map[user]bool{} + var deletedUsers []automationconfig.DeletedUser + + for _, mongoDBUser := range currentMDB.Users { + if mongoDBUser.DB == constants.ExternalDB { + continue + } + m[user{db: mongoDBUser.DB, name: mongoDBUser.Name}] = true + } + + for _, mongoDBUser := range lastAppliedMDBSpec.Users { + if mongoDBUser.DB == constants.ExternalDB { + continue + } + _, ok := m[user{db: mongoDBUser.DB, name: mongoDBUser.Name}] + if !ok { + deletedUsers = append(deletedUsers, automationconfig.DeletedUser{User: mongoDBUser.Name, Dbs: []string{mongoDBUser.DB}}) + } + } + return deletedUsers +} diff --git a/pkg/authentication/authentication_test.go b/pkg/authentication/authentication_test.go index 8521cbf1c..edfef363d 100644 --- a/pkg/authentication/authentication_test.go +++ b/pkg/authentication/authentication_test.go @@ -2,6 +2,7 @@ package authentication import ( "context" + mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" "testing" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -132,6 +133,99 @@ func TestEnable(t *testing.T) { } +func TestGetDeletedUsers(t *testing.T) { + lastAppliedSpec := mdbv1.MongoDBCommunitySpec{ + Members: 3, + Type: "ReplicaSet", + Version: "7.0.2", + Arbiters: 0, + Security: mdbv1.Security{ + Authentication: mdbv1.Authentication{ + Modes: []mdbv1.AuthMode{"SCRAM"}, + }, + }, + Users: []mdbv1.MongoDBUser{ + { + Name: "testUser", + PasswordSecretRef: mdbv1.SecretKeyReference{ + Name: "password-secret-name", + }, + ConnectionStringSecretName: "connection-string-secret", + DB: "admin", + }, + }, + } + + t.Run("no change same resource", func(t *testing.T) { + actual := getRemovedUsersFromSpec(lastAppliedSpec, &lastAppliedSpec) + + var expected []automationconfig.DeletedUser + assert.Equal(t, expected, actual) + }) + + t.Run("new user", func(t *testing.T) { + current := mdbv1.MongoDBCommunitySpec{ + Members: 3, + Type: "ReplicaSet", + Version: "7.0.2", + Arbiters: 0, + Security: mdbv1.Security{ + Authentication: mdbv1.Authentication{ + Modes: []mdbv1.AuthMode{"SCRAM"}, + }, + }, + Users: []mdbv1.MongoDBUser{ + { + Name: "testUser", + PasswordSecretRef: mdbv1.SecretKeyReference{ + Name: "password-secret-name", + }, + ConnectionStringSecretName: "connection-string-secret", + DB: "admin", + }, + { + Name: "newUser", + PasswordSecretRef: mdbv1.SecretKeyReference{ + Name: "new-password-secret-name", + }, + ConnectionStringSecretName: "new-connection-string-secret", + DB: "admin", + }, + }, + } + + var expected []automationconfig.DeletedUser + actual := getRemovedUsersFromSpec(current, &lastAppliedSpec) + + assert.Equal(t, expected, actual) + }) + + t.Run("removed one user", func(t *testing.T) { + current := mdbv1.MongoDBCommunitySpec{ + Members: 3, + Type: "ReplicaSet", + Version: "7.0.2", + Arbiters: 0, + Security: mdbv1.Security{ + Authentication: mdbv1.Authentication{ + Modes: []mdbv1.AuthMode{"SCRAM"}, + }, + }, + Users: []mdbv1.MongoDBUser{}, + } + + expected := []automationconfig.DeletedUser{ + { + User: "testUser", + Dbs: []string{"admin"}, + }, + } + actual := getRemovedUsersFromSpec(current, &lastAppliedSpec) + + assert.Equal(t, expected, actual) + }) +} + func buildConfigurable(name string, auth []string, agent string, users ...authtypes.User) mocks.MockConfigurable { return mocks.NewMockConfigurable( authtypes.Options{ diff --git a/pkg/automationconfig/automation_config.go b/pkg/automationconfig/automation_config.go index c5883a921..a343aa261 100644 --- a/pkg/automationconfig/automation_config.go +++ b/pkg/automationconfig/automation_config.go @@ -334,6 +334,15 @@ type Auth struct { KeyFileWindows string `json:"keyfileWindows,omitempty"` // AutoPwd is a required field when going from `Disabled=false` to `Disabled=true` AutoPwd string `json:"autoPwd,omitempty"` + // UsersDeleted is an array of DeletedUser objects that define the authenticated users to be deleted from specified databases + UsersDeleted []DeletedUser `json:"usersDeleted,omitempty"` +} + +type DeletedUser struct { + // User is the username that should be deleted + User string `json:"user,omitempty"` + // Dbs is the array of database names from which the authenticated user should be deleted + Dbs []string `json:"dbs,omitempty"` } type Prometheus struct { diff --git a/test/e2e/mongodbtests/mongodbtests.go b/test/e2e/mongodbtests/mongodbtests.go index d4bfdd010..be14a3c30 100644 --- a/test/e2e/mongodbtests/mongodbtests.go +++ b/test/e2e/mongodbtests/mongodbtests.go @@ -771,3 +771,57 @@ func assertEqualOwnerReference(t *testing.T, resourceType string, resourceNamesp assert.Equal(t, expectedOwnerReference.Name, ownerReferences[0].Name) assert.Equal(t, expectedOwnerReference.UID, ownerReferences[0].UID) } + +func RemoveLastUserFromMongoDBCommunity(ctx context.Context, mdb *mdbv1.MongoDBCommunity) func(*testing.T) { + return func(t *testing.T) { + err := e2eutil.UpdateMongoDBResource(ctx, mdb, func(db *mdbv1.MongoDBCommunity) { + db.Spec.Users = db.Spec.Users[:len(db.Spec.Users)-1] + }) + + if err != nil { + t.Fatal(err) + } + } +} + +func EditConnectionStringSecretNameOfLastUser(ctx context.Context, mdb *mdbv1.MongoDBCommunity, newSecretName string) func(*testing.T) { + return func(t *testing.T) { + err := e2eutil.UpdateMongoDBResource(ctx, mdb, func(db *mdbv1.MongoDBCommunity) { + db.Spec.Users[len(db.Spec.Users)-1].ConnectionStringSecretName = newSecretName + }) + + if err != nil { + t.Fatal(err) + } + } +} + +func ConnectionStringSecretIsCleanedUp(ctx context.Context, mdb *mdbv1.MongoDBCommunity, removedConnectionString string) func(t *testing.T) { + return func(t *testing.T) { + connectionStringSecret := corev1.Secret{} + newErr := e2eutil.TestClient.Get(ctx, types.NamespacedName{Name: removedConnectionString, Namespace: mdb.Namespace}, &connectionStringSecret) + + assert.EqualError(t, newErr, fmt.Sprintf("secrets \"%s\" not found", removedConnectionString)) + } +} + +func AuthUsersDeletedIsUpdated(ctx context.Context, mdb *mdbv1.MongoDBCommunity, mdbUser mdbv1.MongoDBUser) func(t *testing.T) { + return func(t *testing.T) { + deletedUser := automationconfig.DeletedUser{User: mdbUser.Name, Dbs: []string{mdbUser.DB}} + + currentAc := getAutomationConfig(ctx, t, mdb) + + assert.Contains(t, currentAc.Auth.UsersDeleted, deletedUser) + } +} + +func AddUserToMongoDBCommunity(ctx context.Context, mdb *mdbv1.MongoDBCommunity, newUser mdbv1.MongoDBUser) func(t *testing.T) { + return func(t *testing.T) { + err := e2eutil.UpdateMongoDBResource(ctx, mdb, func(db *mdbv1.MongoDBCommunity) { + db.Spec.Users = append(db.Spec.Users, newUser) + }) + if err != nil { + t.Fatal(err) + } + } +} diff --git a/test/e2e/replica_set_remove_user/replica_set_remove_user_test.go b/test/e2e/replica_set_remove_user/replica_set_remove_user_test.go new file mode 100644 index 000000000..2abeb93c3 --- /dev/null +++ b/test/e2e/replica_set_remove_user/replica_set_remove_user_test.go @@ -0,0 +1,127 @@ +package replica_set_remove_user + +import ( + "context" + "fmt" + mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/automationconfig" + e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" + "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/mongodbtests" + "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" + . "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/util/mongotester" + "os" + "testing" +) + +func TestMain(m *testing.M) { + code, err := e2eutil.RunTest(m) + if err != nil { + fmt.Println(err) + } + os.Exit(code) +} + +func intPtr(x int) *int { return &x } +func strPtr(s string) *string { return &s } + +func TestCleanupUsers(t *testing.T) { + ctx := context.Background() + testCtx := setup.Setup(ctx, t) + defer testCtx.Teardown() + + mdb, user := e2eutil.NewTestMongoDB(testCtx, "mdb0", "") + + _, err := setup.GeneratePasswordForUser(testCtx, user, "") + if err != nil { + t.Fatal(err) + } + + // config member options + memberOptions := []automationconfig.MemberOptions{ + { + Votes: intPtr(1), + Tags: map[string]string{"foo1": "bar1"}, + Priority: strPtr("1.5"), + }, + { + Votes: intPtr(1), + Tags: map[string]string{"foo2": "bar2"}, + Priority: strPtr("1"), + }, + { + Votes: intPtr(1), + Tags: map[string]string{"foo3": "bar3"}, + Priority: strPtr("2.5"), + }, + } + mdb.Spec.MemberConfig = memberOptions + + settings := map[string]interface{}{ + "electionTimeoutMillis": float64(20), + } + mdb.Spec.AutomationConfigOverride = &mdbv1.AutomationConfigOverride{ + ReplicaSet: mdbv1.OverrideReplicaSet{Settings: mdbv1.MapWrapper{Object: settings}}, + } + + newUser := mdbv1.MongoDBUser{ + Name: fmt.Sprintf("%s-user-2", "mdb-0"), + PasswordSecretRef: mdbv1.SecretKeyReference{ + Key: fmt.Sprintf("%s-password-2", "mdb-0"), + Name: fmt.Sprintf("%s-%s-password-secret-2", "mdb-0", testCtx.ExecutionId), + }, + Roles: []mdbv1.Role{ + // roles on testing db for general connectivity + { + DB: "testing", + Name: "readWrite", + }, + { + DB: "testing", + Name: "clusterAdmin", + }, + // admin roles for reading FCV + { + DB: "admin", + Name: "readWrite", + }, + { + DB: "admin", + Name: "clusterAdmin", + }, + { + DB: "admin", + Name: "userAdmin", + }, + }, + ScramCredentialsSecretName: fmt.Sprintf("%s-my-scram-2", "mdb-0"), + } + + _, err = setup.GeneratePasswordForUser(testCtx, newUser, "") + if err != nil { + t.Fatal(err) + } + + tester, err := FromResource(ctx, t, mdb) + if err != nil { + t.Fatal(err) + } + + t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, testCtx)) + t.Run("Basic tests", mongodbtests.BasicFunctionality(ctx, &mdb)) + t.Run("Keyfile authentication is configured", tester.HasKeyfileAuth(3)) + t.Run("Test Basic Connectivity", tester.ConnectivitySucceeds()) + t.Run("Test SRV Connectivity", tester.ConnectivitySucceeds(WithURI(mdb.MongoSRVURI("")), WithoutTls(), WithReplicaSet(mdb.Name))) + t.Run("Add new user to MongoDB Resource", mongodbtests.AddUserToMongoDBCommunity(ctx, &mdb, newUser)) + t.Run("MongoDB reaches Running phase", mongodbtests.MongoDBReachesRunningPhase(ctx, &mdb)) + editedUser := mdb.Spec.Users[1] + t.Run("Edit connection string secret name of the added user", mongodbtests.EditConnectionStringSecretNameOfLastUser(ctx, &mdb, "other-secret-name")) + t.Run("MongoDB reaches Running phase", mongodbtests.MongoDBReachesRunningPhase(ctx, &mdb)) + t.Run("Old connection string secret is cleaned up", mongodbtests.ConnectionStringSecretIsCleanedUp(ctx, &mdb, editedUser.GetConnectionStringSecretName(mdb.Name))) + deletedUser := mdb.Spec.Users[1] + t.Run("Remove last user from MongoDB Resource", mongodbtests.RemoveLastUserFromMongoDBCommunity(ctx, &mdb)) + t.Run("MongoDB reaches Pending phase", mongodbtests.MongoDBReachesPendingPhase(ctx, &mdb)) + t.Run("Removed users are added to automation config", mongodbtests.AuthUsersDeletedIsUpdated(ctx, &mdb, deletedUser)) + t.Run("MongoDB reaches Running phase", mongodbtests.MongoDBReachesRunningPhase(ctx, &mdb)) + t.Run("Connection string secrets are cleaned up", mongodbtests.ConnectionStringSecretIsCleanedUp(ctx, &mdb, deletedUser.GetConnectionStringSecretName(mdb.Name))) + t.Run("Delete MongoDB Resource", mongodbtests.DeleteMongoDBResource(&mdb, testCtx)) +} From 57708bd06864cd9c4fefe8f2ff1c443ee0b26338 Mon Sep 17 00:00:00 2001 From: Matei Grigore <43472562+MateiGrigore@users.noreply.github.com> Date: Tue, 23 Jul 2024 17:51:16 +0100 Subject: [PATCH 753/790] Fix inventory file and upgrade documentation (#1596) --- Makefile | 15 +++++++-------- docs/build_operator_locally.md | 2 +- docs/contributing.md | 21 +++++++++++++++------ docs/run-operator-locally.md | 22 ++++++++++++++-------- inventory.yaml | 7 ++----- pipeline.py | 18 +++++++++++++----- scripts/dev/setup_kind_cluster.sh | 2 +- 7 files changed, 53 insertions(+), 34 deletions(-) diff --git a/Makefile b/Makefile index a03dfb9bd..6f1811c8f 100644 --- a/Makefile +++ b/Makefile @@ -9,8 +9,7 @@ NAMESPACE := $(shell jq -r .namespace < $(MONGODB_COMMUNITY_CONFIG)) UPGRADE_HOOK_IMG := $(shell jq -r .version_upgrade_hook_image < $(MONGODB_COMMUNITY_CONFIG)) READINESS_PROBE_IMG := $(shell jq -r .readiness_probe_image < $(MONGODB_COMMUNITY_CONFIG)) REGISTRY := $(shell jq -r .repo_url < $(MONGODB_COMMUNITY_CONFIG)) -AGENT_IMAGE_NAME := $(shell jq -r .agent_image_ubi < $(MONGODB_COMMUNITY_CONFIG)) - +AGENT_IMAGE_NAME := $(shell jq -r .agent_image < $(MONGODB_COMMUNITY_CONFIG)) HELM_CHART ?= ./helm-charts/charts/community-operator STRING_SET_VALUES := --set namespace=$(NAMESPACE),versionUpgradeHook.name=$(UPGRADE_HOOK_IMG),readinessProbe.name=$(READINESS_PROBE_IMG),registry.operator=$(REPO_URL),operator.operatorImageName=$(OPERATOR_IMAGE),operator.version=latest,registry.agent=$(REGISTRY),registry.versionUpgradeHook=$(REGISTRY),registry.readinessProbe=$(REGISTRY),registry.operator=$(REGISTRY),versionUpgradeHook.version=latest,readinessProbe.version=latest,agent.version=latest,agent.name=$(AGENT_IMAGE_NAME) @@ -134,7 +133,7 @@ install-rbac: $(HELM) template $(STRING_SET_VALUES) -s templates/operator_roles.yaml $(HELM_CHART) | kubectl apply -f - uninstall-crd: - kubectl delete crd mongodbcommunity.mongodbcommunity.mongodb.com + kubectl delete crd --ignore-not-found mongodbcommunity.mongodbcommunity.mongodb.com uninstall-chart: $(HELM) uninstall $(RELEASE_NAME_HELM) -n $(NAMESPACE) @@ -193,19 +192,19 @@ generate-env-file: ## generates a local-test.env for local testing ##@ Image operator-image: ## Build and push the operator image - python pipeline.py --image-name operator + python pipeline.py --image-name operator $(IMG_BUILD_ARGS) e2e-image: ## Build and push e2e test image - python pipeline.py --image-name e2e + python pipeline.py --image-name e2e $(IMG_BUILD_ARGS) agent-image: ## Build and push agent image - python pipeline.py --image-name agent + python pipeline.py --image-name agent $(IMG_BUILD_ARGS) readiness-probe-image: ## Build and push readiness probe image - python pipeline.py --image-name readiness-probe + python pipeline.py --image-name readiness-probe $(IMG_BUILD_ARGS) version-upgrade-post-start-hook-image: ## Build and push version upgrade post start hook image - python pipeline.py --image-name version-upgrade-hook + python pipeline.py --image-name version-upgrade-hook $(IMG_BUILD_ARGS) all-images: operator-image e2e-image agent-image readiness-probe-image version-upgrade-post-start-hook-image ## create all required images diff --git a/docs/build_operator_locally.md b/docs/build_operator_locally.md index d1219c654..33dfff340 100644 --- a/docs/build_operator_locally.md +++ b/docs/build_operator_locally.md @@ -40,7 +40,7 @@ git submodule update --init ``` -5. Build and deploy the operator: +5. Build and deploy the operator. Also add `IMG_BUILD_ARGS=--insecure` as described [here](contributing.md#deploying-the-operator) if necessary: ```sh # builds all required images and then deploys the operator diff --git a/docs/contributing.md b/docs/contributing.md index d1414eb86..139d11b71 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -58,9 +58,9 @@ to be able to run properly. Create a json file with the following content: "repo_url": "localhost:5000", "operator_image": "mongodb-kubernetes-operator", "e2e_image": "community-operator-e2e", - "version_upgrade_hook_image": "community-operator-version-upgrade-post-start-hook", + "version_upgrade_hook_image": "mongodb-kubernetes-operator-version-upgrade-post-start-hook", "agent_image": "mongodb-agent-ubi-dev", - "readiness_probe_image": "mongodb-kubernetes-readiness", + "readiness_probe_image": "mongodb-kubernetes-readinessprobe", "s3_bucket": "" } ``` @@ -132,6 +132,14 @@ make operator-image deploy This will build and deploy the operator to namespace specified in your configuration file. +If you are using a local docker registry you should run the following command. +The additional `IMG_BUILD_ARGS=--insecure` variable will add the `--insecure` flag to the command creating the manifests. +This is necessary if your local registry is not secure. Read more about the flag on the [documentatio](https://docs.docker.com/reference/cli/docker/manifest/#working-with-insecure-registries) + +```sh +IMG_BUILD_ARGS=--insecure make operator-image deploy +``` + #### See the operator deployment ```sh @@ -149,7 +157,7 @@ To remove the operator and any created resources you can run make undeploy ``` -Alternatively, you can run the operator locally with +Alternatively, you can run the operator locally. Make sure you follow the steps outlined in [run-operator-locally.md](run-operator-locally.md) ```sh make run @@ -168,7 +176,8 @@ make test ### E2E Tests If this is the first time running E2E tests, you will need to ensure that you have built and pushed -all images required by the E2E tests. You can do this by running. +all images required by the E2E tests. You can do this by running the following command, +or with the additional `IMG_BUILD_ARGS=--insecure` described above. ```sh make all-images @@ -180,7 +189,7 @@ For subsequent tests you can use make e2e-k8s test= ``` -This will only re-build the e2e test image. +This will only re-build the e2e test image. Add `IMG_BUILD_ARGS=--insecure` if necessary We have built a simple mechanism to run E2E tests on your cluster using a runner that deploys a series of Kubernetes objects, runs them, and awaits for their @@ -199,7 +208,7 @@ replica_set_scale ... ``` -The tests should run individually using the runner like this: +The tests should run individually using the runner like this, or additionally with `IMG_BUILD_ARGS=--insecure`: ```sh make e2e-k8s test=replica_set diff --git a/docs/run-operator-locally.md b/docs/run-operator-locally.md index a46ecba56..c742f54b7 100644 --- a/docs/run-operator-locally.md +++ b/docs/run-operator-locally.md @@ -18,17 +18,23 @@ Being able to run and build the binary locally can help with faster feedback-cyc - Run e2e tests locally ## Running The Operator locally -1. Use the dedicated make target which exports the needed environment variables and builds & runs the operator binary - -```sh -make run -``` +1. Use the dedicated make target which exports the needed environment variables and builds & runs the operator binary. + + Before doing that you need to add 2 more fields to the `config.json` file found in [contributing.md](contributing.md), because the python script looks for them in the file: + - `mdb_local_operator`: needs to be set to `true`, to allow for the operator to be run locally + - `kubeconfig`: needs to be set to the path of the `kubeconfig` configuration file, for example `$HOME/.kube/config` + + Then you can run the command: + + ```sh + make run + ``` 2. For debugging one can use the following make target, which uses [dlv](https://github.com/go-delve/delve): -```sh -make debug -``` + ```sh + make debug + ``` ## Running e2e tests with the local operator - Our [e2e tests](../test/e2e), contains sub-steps that will install the following helm-chart: [operator.yaml](../helm-charts/charts/community-operator/templates/operator.yaml) diff --git a/inventory.yaml b/inventory.yaml index 28d475740..e2a37214c 100644 --- a/inventory.yaml +++ b/inventory.yaml @@ -9,9 +9,6 @@ images: vars: context: . template_context: scripts/dev/templates/agent - # Default values but overwritten in pipeline.py - agent_distro: rhel7_x86_64 - tools_distro: rhel70-x86_64 inputs: - release_version @@ -83,8 +80,8 @@ images: buildargs: agent_version: $(inputs.params.release_version) tools_version: $(inputs.params.tools_version) - agent_distro: rhel7_x86_64 - tools_distro: rhel70-x86_64 + agent_distro: $(inputs.params.agent_distro) + tools_distro: $(inputs.params.tools_distro) labels: quay.expires-after: Never diff --git a/pipeline.py b/pipeline.py index 078a096eb..b8324d249 100644 --- a/pipeline.py +++ b/pipeline.py @@ -87,6 +87,7 @@ def build_and_push_image( architectures: Set[str], release: bool, sign: bool, + insecure: bool = False, ) -> None: if sign: mongodb_artifactory_login() @@ -119,16 +120,18 @@ def build_and_push_image( else: raise Exception("Dev image must be specified") - push_manifest(config, architectures, image_to_push) + push_manifest(config, architectures, image_to_push, insecure) if config.gh_run_id: - push_manifest(config, architectures, image_to_push, config.gh_run_id) + push_manifest(config, architectures, image_to_push, insecure, config.gh_run_id) if release: registry = args["registry"] + "/" + args["image"] context_tag = args["release_version"] + "-context" - push_manifest(config, architectures, args["image"], args["release_version"]) - push_manifest(config, architectures, args["image"], context_tag) + push_manifest( + config, architectures, args["image"], insecure, args["release_version"] + ) + push_manifest(config, architectures, args["image"], insecure, context_tag) if sign: sign_and_verify(registry, args["release_version"]) sign_and_verify(registry, context_tag) @@ -149,6 +152,7 @@ def push_manifest( config: DevConfig, architectures: Set[str], image_name: str, + insecure: bool = False, image_tag: str = "latest", ) -> None: logger.info(f"Pushing manifest for {image_tag}") @@ -164,6 +168,9 @@ def push_manifest( final_manifest, ] + if insecure: + create_args.append("--insecure") + for arch in architectures: create_args.extend(["--amend", final_manifest + "-" + arch]) @@ -220,6 +227,7 @@ def _parse_args() -> argparse.Namespace: ) parser.add_argument("--tag", type=str) parser.add_argument("--sign", action="store_true", default=False) + parser.add_argument("--insecure", action="store_true", default=False) return parser.parse_args() @@ -277,7 +285,7 @@ def main() -> int: image_args = build_image_args(config, image_name) build_and_push_image( - image_name, config, image_args, arch_set, args.release, args.sign + image_name, config, image_args, arch_set, args.release, args.sign, args.insecure ) return 0 diff --git a/scripts/dev/setup_kind_cluster.sh b/scripts/dev/setup_kind_cluster.sh index 7f19c0d6e..eab968b22 100755 --- a/scripts/dev/setup_kind_cluster.sh +++ b/scripts/dev/setup_kind_cluster.sh @@ -97,4 +97,4 @@ EOF if [[ "${export_kubeconfig}" == "1" ]]; then kind export kubeconfig --name "${cluster_name}" -fi \ No newline at end of file +fi From 1f36c73cdfd233b8b7e2d2446b64446286d307af Mon Sep 17 00:00:00 2001 From: Matei Grigore <43472562+MateiGrigore@users.noreply.github.com> Date: Mon, 29 Jul 2024 10:26:09 +0100 Subject: [PATCH 754/790] Fixing validation of collisions in scram secret's name (#1597) * Fix code to validate that no collisions of scramSecretName happen * Fix precommit hook * Update dependencies * Change pre commit hook comand to go install --- ...ommunity.mongodb.com_mongodbcommunity.yaml | 34 +++++++++++++++++++ controllers/validation/validation.go | 4 +-- go.mod | 3 +- go.sum | 8 ++--- scripts/git-hooks/pre-commit | 2 +- 5 files changed, 43 insertions(+), 8 deletions(-) diff --git a/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml b/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml index 1b1189970..95d8e4c32 100644 --- a/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml +++ b/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml @@ -74,6 +74,40 @@ spec: description: AgentConfiguration sets options for the MongoDB automation agent properties: + AuditLogRotate: + description: AuditLogRotate if enabled, will enable AuditLogRotate + for all processes. + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file before + rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object logFile: type: string logLevel: diff --git a/controllers/validation/validation.go b/controllers/validation/validation.go index 53c431ea2..3d84cc1c0 100644 --- a/controllers/validation/validation.go +++ b/controllers/validation/validation.go @@ -92,7 +92,7 @@ func validateUsers(mdb mdbv1.MongoDBCommunity) error { previousUser.Username, user.Username)) } else { - connectionStringSecretNameMap[connectionStringSecretName] = user + scramSecretNameMap[scramSecretName] = user } if user.Database == constants.ExternalDB { @@ -100,7 +100,7 @@ func validateUsers(mdb mdbv1.MongoDBCommunity) error { return fmt.Errorf("X.509 user %s present but X.509 is not enabled", user.Username) } if user.PasswordSecretKey != "" { - return fmt.Errorf("X509 user %s shoul not have a password secret key", user.Username) + return fmt.Errorf("X509 user %s should not have a password secret key", user.Username) } if user.PasswordSecretName != "" { return fmt.Errorf("X509 user %s should not have a password secret name", user.Username) diff --git a/go.mod b/go.mod index 94a3ad34a..970224b35 100644 --- a/go.mod +++ b/go.mod @@ -66,13 +66,14 @@ require ( github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect go.uber.org/multierr v1.10.0 // indirect golang.org/x/crypto v0.25.0 // indirect - golang.org/x/net v0.23.0 // indirect + golang.org/x/net v0.27.0 // indirect golang.org/x/oauth2 v0.7.0 // indirect golang.org/x/sync v0.7.0 // indirect golang.org/x/sys v0.22.0 // indirect golang.org/x/term v0.22.0 // indirect golang.org/x/text v0.16.0 // indirect golang.org/x/time v0.3.0 // indirect + golang.org/x/tools v0.23.0 // indirect gomodules.xyz/jsonpatch/v2 v2.3.0 // indirect google.golang.org/appengine v1.6.7 // indirect gopkg.in/inf.v0 v0.9.1 // indirect diff --git a/go.sum b/go.sum index bfcd68e97..f75ec5348 100644 --- a/go.sum +++ b/go.sum @@ -199,8 +199,8 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= +golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g= golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= @@ -246,8 +246,8 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg= +golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/scripts/git-hooks/pre-commit b/scripts/git-hooks/pre-commit index 7d2f5070c..84a7a80d0 100755 --- a/scripts/git-hooks/pre-commit +++ b/scripts/git-hooks/pre-commit @@ -5,7 +5,7 @@ function go_imports() { if ! type goimports &> /dev/null; then echo "Installing goimports" - GO111MODULE=off go get golang.org/x/tools/cmd/goimports + go install golang.org/x/tools/cmd/goimports fi # Formats each file that was changed. From 10da9cf7f8358f3b0bd527a464d1005a11b78e39 Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Wed, 31 Jul 2024 10:44:16 +0200 Subject: [PATCH 755/790] fix json naming of auditLogRotate (#1599) --- api/v1/mongodbcommunity_types.go | 2 +- api/v1/zz_generated.deepcopy.go | 5 +++++ .../bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml | 2 +- test/e2e/mongodbtests/mongodbtests.go | 2 +- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/api/v1/mongodbcommunity_types.go b/api/v1/mongodbcommunity_types.go index cd91000c1..f1e7f8710 100644 --- a/api/v1/mongodbcommunity_types.go +++ b/api/v1/mongodbcommunity_types.go @@ -374,7 +374,7 @@ type AgentConfiguration struct { LogRotate *automationconfig.CrdLogRotate `json:"logRotate,omitempty"` // +optional // AuditLogRotate if enabled, will enable AuditLogRotate for all processes. - AuditLogRotate *automationconfig.CrdLogRotate `json:"AuditLogRotate,omitempty"` + AuditLogRotate *automationconfig.CrdLogRotate `json:"auditLogRotate,omitempty"` // +optional // SystemLog configures system log of mongod SystemLog *automationconfig.SystemLog `json:"systemLog,omitempty"` diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index 40a547fef..bf58f2b77 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -34,6 +34,11 @@ func (in *AgentConfiguration) DeepCopyInto(out *AgentConfiguration) { *out = new(automationconfig.CrdLogRotate) **out = **in } + if in.AuditLogRotate != nil { + in, out := &in.AuditLogRotate, &out.AuditLogRotate + *out = new(automationconfig.CrdLogRotate) + **out = **in + } if in.SystemLog != nil { in, out := &in.SystemLog, &out.SystemLog *out = new(automationconfig.SystemLog) diff --git a/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml b/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml index 95d8e4c32..e7c3faa69 100644 --- a/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml +++ b/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml @@ -74,7 +74,7 @@ spec: description: AgentConfiguration sets options for the MongoDB automation agent properties: - AuditLogRotate: + auditLogRotate: description: AuditLogRotate if enabled, will enable AuditLogRotate for all processes. properties: diff --git a/test/e2e/mongodbtests/mongodbtests.go b/test/e2e/mongodbtests/mongodbtests.go index be14a3c30..b52e8639e 100644 --- a/test/e2e/mongodbtests/mongodbtests.go +++ b/test/e2e/mongodbtests/mongodbtests.go @@ -800,7 +800,7 @@ func ConnectionStringSecretIsCleanedUp(ctx context.Context, mdb *mdbv1.MongoDBCo return func(t *testing.T) { connectionStringSecret := corev1.Secret{} newErr := e2eutil.TestClient.Get(ctx, types.NamespacedName{Name: removedConnectionString, Namespace: mdb.Namespace}, &connectionStringSecret) - + assert.EqualError(t, newErr, fmt.Sprintf("secrets \"%s\" not found", removedConnectionString)) } } From 1947a009fb5761f4ad351a94592c22c1a716c258 Mon Sep 17 00:00:00 2001 From: Matei Grigore <43472562+MateiGrigore@users.noreply.github.com> Date: Tue, 6 Aug 2024 14:49:18 +0100 Subject: [PATCH 756/790] Improvements for local development setup process (#1589) * Add changes that improve the local setup process * Remove default values for agent_distro and tools_distro from inventory.yaml * Add suggested changes * Make explanation in the documentation more clear * force release * undo change * undo change --------- Co-authored-by: nam --- scripts/dev/setup_kind_cluster.sh | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/scripts/dev/setup_kind_cluster.sh b/scripts/dev/setup_kind_cluster.sh index eab968b22..3178f2878 100755 --- a/scripts/dev/setup_kind_cluster.sh +++ b/scripts/dev/setup_kind_cluster.sh @@ -56,7 +56,7 @@ reg_name='kind-registry' reg_port='5000' running="$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)" if [ "${running}" != 'true' ]; then - docker run -d --restart=always -p "127.0.0.1:${reg_port}:5000" --name "${reg_name}" registry:2 + docker run -d --restart=always -p "127.0.0.1:${reg_port}:5000" --network kind --name "${reg_name}" registry:2 fi if [ "${recreate}" != 0 ]; then @@ -72,10 +72,26 @@ networking: serviceSubnet: "${service_network}" containerdConfigPatches: - |- - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:${reg_port}"] - endpoint = ["http://${reg_name}:${reg_port}"] + [plugins."io.containerd.grpc.v1.cri".registry] + config_path = "/etc/containerd/certs.d" EOF +# Add the registry config to the nodes +# +# This is necessary because localhost resolves to loopback addresses that are +# network-namespace local. +# In other words: localhost in the container is not localhost on the host. +# +# We want a consistent name that works from both ends, so we tell containerd to +# alias localhost:${reg_port} to the registry container when pulling images +REGISTRY_DIR="/etc/containerd/certs.d/localhost:${reg_port}" +for node in $(kind get nodes --name "${cluster_name}"); do + docker exec "${node}" mkdir -p "${REGISTRY_DIR}" + cat < Date: Wed, 7 Aug 2024 15:24:02 +0100 Subject: [PATCH 757/790] Bump go version to 1.22 (#1600) * Bump versions of go readiness probe and version upgrade hook * Bump go version in release.json --- .github/workflows/go.yml | 2 +- go.mod | 2 +- release.json | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index d94b0c8e5..79ed8f21f 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -14,7 +14,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v4 with: - go-version: '1.21' + go-version: '1.22' - name: Test api run: go test -v ./api/... diff --git a/go.mod b/go.mod index 970224b35..0e814f0ff 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/mongodb/mongodb-kubernetes-operator -go 1.21 +go 1.22 require ( github.com/blang/semver v3.5.1+incompatible diff --git a/release.json b/release.json index e07804fc6..f80603ed3 100644 --- a/release.json +++ b/release.json @@ -1,8 +1,8 @@ { - "golang-builder-image": "golang:1.21", + "golang-builder-image": "golang:1.22", "operator": "0.10.0", - "version-upgrade-hook": "1.0.8", - "readiness-probe": "1.0.19", + "version-upgrade-hook": "1.0.9", + "readiness-probe": "1.0.20", "agent": "107.0.1.8507-1", "agent-tools-version": "100.9.4" } \ No newline at end of file From c5170eb20cec8d352c3ba66250e1c8e14f1340ee Mon Sep 17 00:00:00 2001 From: Matei Grigore <43472562+MateiGrigore@users.noreply.github.com> Date: Thu, 8 Aug 2024 13:46:54 +0100 Subject: [PATCH 758/790] Release MongoDB Kubernetes Operator v0.11.0 (#1601) * Release 0.11.0 changes * Add suggested changes * Bump agent version --- config/manager/manager.yaml | 8 ++++---- deploy/openshift/operator_openshift.yaml | 8 ++++---- docs/RELEASE_NOTES.md | 23 +++++++++++------------ release.json | 4 ++-- 4 files changed, 21 insertions(+), 22 deletions(-) diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 044421f42..8e2d9a3ce 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -45,16 +45,16 @@ spec: - name: OPERATOR_NAME value: mongodb-kubernetes-operator - name: AGENT_IMAGE - value: quay.io/mongodb/mongodb-agent-ubi:107.0.1.8507-1 + value: quay.io/mongodb/mongodb-agent-ubi:107.0.7.8596-1 - name: VERSION_UPGRADE_HOOK_IMAGE - value: quay.io/mongodb/mongodb-kubernetes-operator-version-upgrade-post-start-hook:1.0.8 + value: quay.io/mongodb/mongodb-kubernetes-operator-version-upgrade-post-start-hook:1.0.9 - name: READINESS_PROBE_IMAGE - value: quay.io/mongodb/mongodb-kubernetes-readinessprobe:1.0.19 + value: quay.io/mongodb/mongodb-kubernetes-readinessprobe:1.0.20 - name: MONGODB_IMAGE value: mongodb-community-server - name: MONGODB_REPO_URL value: quay.io/mongodb - image: quay.io/mongodb/mongodb-kubernetes-operator:0.10.0 + image: quay.io/mongodb/mongodb-kubernetes-operator:0.11.0 imagePullPolicy: Always name: mongodb-kubernetes-operator resources: diff --git a/deploy/openshift/operator_openshift.yaml b/deploy/openshift/operator_openshift.yaml index 5eb385528..8e7d47ce6 100644 --- a/deploy/openshift/operator_openshift.yaml +++ b/deploy/openshift/operator_openshift.yaml @@ -47,16 +47,16 @@ spec: - name: OPERATOR_NAME value: mongodb-kubernetes-operator - name: AGENT_IMAGE - value: quay.io/mongodb/mongodb-agent-ubi:107.0.1.8507-1 + value: quay.io/mongodb/mongodb-agent-ubi:107.0.7.8596-1 - name: READINESS_PROBE_IMAGE - value: quay.io/mongodb/mongodb-kubernetes-readinessprobe:1.0.19 + value: quay.io/mongodb/mongodb-kubernetes-readinessprobe:1.0.20 - name: VERSION_UPGRADE_HOOK_IMAGE - value: quay.io/mongodb/mongodb-kubernetes-operator-version-upgrade-post-start-hook:1.0.8 + value: quay.io/mongodb/mongodb-kubernetes-operator-version-upgrade-post-start-hook:1.0.9 - name: MONGODB_IMAGE value: mongo - name: MONGODB_REPO_URL value: quay.io - image: quay.io/mongodb/mongodb-kubernetes-operator:0.10.0 + image: quay.io/mongodb/mongodb-kubernetes-operator:0.11.0 imagePullPolicy: Always name: mongodb-kubernetes-operator resources: diff --git a/docs/RELEASE_NOTES.md b/docs/RELEASE_NOTES.md index dd47234e7..aed986aa2 100644 --- a/docs/RELEASE_NOTES.md +++ b/docs/RELEASE_NOTES.md @@ -1,19 +1,18 @@ -# MongoDB Kubernetes Operator 0.10.0 +# MongoDB Kubernetes Operator 0.11.0 -## Released images signed +## Migrating agent images to ubi +All agent images were updated to use the ubi repo -All container images published for the community operator are signed with our private key. This is visible on our Quay registry. Signature can be verified using our public key, which is available at [this address](https://cosign.mongodb.com/mongodb-enterprise-kubernetes-operator.pem). +## Documentation improvements +Improvements were made to the documentation of using the community operator as well as the one for local development. ## Logging changes -- The agent logging can be configured to stdout -- ReadinessProbe logging configuration can now be configured -- More can be found [here](logging.md). +- Added `AuditLogRotate` field to `AgentConfiguration` +- Fixed JSON key to be lower case: `logRotate` -## Overriding Mongod settings via the CRD -- Example can be found [here](../config/samples/mongodb.com_v1_mongodbcommunity_override_ac_setting.yaml). - -## ReadinessProbe error logging -- fixed a red herring which caused the probe to panic when the health status is not available. Instead it will just log the error +## Bug Fixes +- Users removed from the resource are now also deleted from the database and their connection string secrets are cleaned up +- Scram secret name collisions will now be caught when validating specs ## Important Bumps -- Bumped K8S libs to 1.27 +- Bumped go to 1.22 \ No newline at end of file diff --git a/release.json b/release.json index f80603ed3..403d3f789 100644 --- a/release.json +++ b/release.json @@ -1,8 +1,8 @@ { "golang-builder-image": "golang:1.22", - "operator": "0.10.0", + "operator": "0.11.0", "version-upgrade-hook": "1.0.9", "readiness-probe": "1.0.20", - "agent": "107.0.1.8507-1", + "agent": "107.0.7.8596-1", "agent-tools-version": "100.9.4" } \ No newline at end of file From f1d5c04c984c86919434e8bac34e383f9d6d5458 Mon Sep 17 00:00:00 2001 From: Matei Grigore <43472562+MateiGrigore@users.noreply.github.com> Date: Fri, 9 Aug 2024 12:12:47 +0100 Subject: [PATCH 759/790] Pin helm-charts to latest release (#1602) --- helm-charts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm-charts b/helm-charts index b8b68108e..f03b5c02f 160000 --- a/helm-charts +++ b/helm-charts @@ -1 +1 @@ -Subproject commit b8b68108e6128f5c1eba70a3c4b5a5594671d69e +Subproject commit f03b5c02f19a40370961652e8fb95060a443f320 From 8e1a37b82521d9c49f6862ecd8f20596b422fa5a Mon Sep 17 00:00:00 2001 From: mircea-cosbuc Date: Wed, 21 Aug 2024 19:19:07 +0200 Subject: [PATCH 760/790] Use pointer field for priority in member options (#1610) * Use pointer field for priority in member options * Ignore G115 for probes * Skip leftover overflows * Ignore G115 at golangi.yml level * Sort members by rs member id --- .golangci.yml | 5 +++- go.mod | 2 +- go.sum | 2 ++ pkg/automationconfig/automation_config.go | 6 ++--- .../automation_config_builder.go | 5 ++-- .../automation_config_test.go | 17 +++++++++++++ test/e2e/mongodbtests/mongodbtests.go | 24 ++++++++++++------- test/e2e/replica_set/replica_set_test.go | 2 +- 8 files changed, 46 insertions(+), 17 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 3782ca8c7..0fae128cd 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -28,7 +28,10 @@ linters: - rowserrcheck - gosec - unconvert - +linters-settings: + gosec: + excludes: + - G115 run: modules-download-mode: mod # timeout for analysis, e.g. 30s, 5m, default is 1m diff --git a/go.mod b/go.mod index 0e814f0ff..3c6823de3 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( k8s.io/api v0.27.12 k8s.io/apimachinery v0.27.12 k8s.io/client-go v0.27.12 + k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 sigs.k8s.io/controller-runtime v0.15.3 sigs.k8s.io/yaml v1.4.0 ) @@ -83,7 +84,6 @@ require ( k8s.io/component-base v0.27.12 // indirect k8s.io/klog/v2 v2.90.1 // indirect k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect - k8s.io/utils v0.0.0-20230209194617-a36077c30491 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect ) diff --git a/go.sum b/go.sum index f75ec5348..96e6f8a16 100644 --- a/go.sum +++ b/go.sum @@ -311,6 +311,8 @@ k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f h1:2kWPakN3i/k81b0gvD5C5F k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f/go.mod h1:byini6yhqGC14c3ebc/QwanvYwhuMWF6yz2F8uwW8eg= k8s.io/utils v0.0.0-20230209194617-a36077c30491 h1:r0BAOLElQnnFhE/ApUsg3iHdVYYPBjNSSOMowRZxxsY= k8s.io/utils v0.0.0-20230209194617-a36077c30491/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 h1:jgGTlFYnhF1PM1Ax/lAlxUPE+KfCIXHaathvJg1C3ak= +k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/controller-runtime v0.15.3 h1:L+t5heIaI3zeejoIyyvLQs5vTVu/67IU2FfisVzFlBc= sigs.k8s.io/controller-runtime v0.15.3/go.mod h1:kp4jckA4vTx281S/0Yk2LFEEQe67mjg+ev/yknv47Ds= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= diff --git a/pkg/automationconfig/automation_config.go b/pkg/automationconfig/automation_config.go index a343aa261..855985108 100644 --- a/pkg/automationconfig/automation_config.go +++ b/pkg/automationconfig/automation_config.go @@ -283,7 +283,7 @@ type ReplicaSetMember struct { // is different in AC from the CR(CR don't support float) - hence all the members are declared // separately Votes *int `json:"votes,omitempty"` - Priority float32 `json:"priority,omitempty"` + Priority *float32 `json:"priority,omitempty"` Tags map[string]string `json:"tags,omitempty"` } @@ -294,7 +294,7 @@ func newReplicaSetMember(name string, id int, horizons ReplicaSetHorizons, isArb // ensure that the number of voting members in the replica set is not more than 7 // as this is the maximum number of voting members. votes := 0 - priority := 0.0 + priority := float32(0.0) if isVotingMember { votes = 1 @@ -307,7 +307,7 @@ func newReplicaSetMember(name string, id int, horizons ReplicaSetHorizons, isArb ArbiterOnly: isArbiter, Horizons: horizons, Votes: &votes, - Priority: float32(priority), + Priority: &priority, } } diff --git a/pkg/automationconfig/automation_config_builder.go b/pkg/automationconfig/automation_config_builder.go index bc0e9222e..be69e3f29 100644 --- a/pkg/automationconfig/automation_config_builder.go +++ b/pkg/automationconfig/automation_config_builder.go @@ -6,8 +6,9 @@ import ( "strings" "github.com/blang/semver" - "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/versions" + + "k8s.io/utils/ptr" ) type Topology string @@ -351,7 +352,7 @@ func (b *Builder) Build() (AutomationConfig, error) { if len(b.memberOptions) > i { // override the member options if explicitly specified in the spec members[i].Votes = b.memberOptions[i].Votes - members[i].Priority = b.memberOptions[i].GetPriority() + members[i].Priority = ptr.To(b.memberOptions[i].GetPriority()) members[i].Tags = b.memberOptions[i].Tags } } diff --git a/pkg/automationconfig/automation_config_test.go b/pkg/automationconfig/automation_config_test.go index b6ff3b04a..2950eb4d7 100644 --- a/pkg/automationconfig/automation_config_test.go +++ b/pkg/automationconfig/automation_config_test.go @@ -477,6 +477,23 @@ func TestAreEqual(t *testing.T) { assert.NoError(t, err) assert.False(t, areEqual) }) + + t.Run("Automation Configs with nil and zero values are not equal", func(t *testing.T) { + votes := 1 + priority := "0.0" + firstBuilder := NewBuilder().SetName("name0").SetMongoDBVersion("mdbVersion0").SetOptions(Options{DownloadBase: "downloadBase0"}).SetDomain("domain0").SetMembers(2).SetAuth(Auth{Disabled: true}) + firstBuilder.SetMemberOptions([]MemberOptions{MemberOptions{Votes: &votes, Priority: &priority}}) + firstAc, _ := firstBuilder.Build() + firstAc.Version = 2 + secondBuilder := NewBuilder().SetName("name0").SetMongoDBVersion("mdbVersion0").SetOptions(Options{DownloadBase: "downloadBase0"}).SetDomain("domain0").SetMembers(2).SetAuth(Auth{Disabled: true}) + secondBuilder.SetMemberOptions([]MemberOptions{MemberOptions{Votes: &votes, Priority: nil}}) + secondAc, _ := secondBuilder.Build() + secondAc.Version = 2 + + areEqual, err := AreEqual(firstAc, secondAc) + assert.NoError(t, err) + assert.False(t, areEqual) + }) } func TestValidateFCV(t *testing.T) { diff --git a/test/e2e/mongodbtests/mongodbtests.go b/test/e2e/mongodbtests/mongodbtests.go index b52e8639e..aad73a450 100644 --- a/test/e2e/mongodbtests/mongodbtests.go +++ b/test/e2e/mongodbtests/mongodbtests.go @@ -4,7 +4,7 @@ import ( "context" "encoding/json" "fmt" - "strconv" + "sort" "strings" "testing" "time" @@ -36,11 +36,6 @@ func SkipTestIfLocal(t *testing.T, msg string, f func(t *testing.T)) { t.Run(msg, f) } -func strPointer(n float32) *string { - s := strconv.FormatFloat(float64(n), 'f', -1, 64) - return &s -} - // StatefulSetBecomesReady ensures that the underlying stateful set // reaches the running state. func StatefulSetBecomesReady(ctx context.Context, mdb *mdbv1.MongoDBCommunity, opts ...wait.Configuration) func(t *testing.T) { @@ -435,10 +430,13 @@ func AutomationConfigHasVoteTagPriorityConfigured(ctx context.Context, mdb *mdbv return func(t *testing.T) { currentAc := getAutomationConfig(ctx, t, mdb) - rsMemebers := currentAc.ReplicaSets + rsMembers := currentAc.ReplicaSets + sort.Slice(rsMembers[0].Members, func(i, j int) bool { + return rsMembers[0].Members[i].Id < rsMembers[0].Members[j].Id + }) - for _, m := range rsMemebers[0].Members { - acMemberOptions = append(acMemberOptions, automationconfig.MemberOptions{Votes: m.Votes, Priority: strPointer(m.Priority), Tags: m.Tags}) + for _, m := range rsMembers[0].Members { + acMemberOptions = append(acMemberOptions, automationconfig.MemberOptions{Votes: m.Votes, Priority: floatPtrTostringPtr(m.Priority), Tags: m.Tags}) } assert.ElementsMatch(t, memberOptions, acMemberOptions) } @@ -825,3 +823,11 @@ func AddUserToMongoDBCommunity(ctx context.Context, mdb *mdbv1.MongoDBCommunity, } } } + +func floatPtrTostringPtr(floatPtr *float32) *string { + if floatPtr != nil { + stringValue := fmt.Sprintf("%.1f", *floatPtr) + return &stringValue + } + return nil +} diff --git a/test/e2e/replica_set/replica_set_test.go b/test/e2e/replica_set/replica_set_test.go index 6fa62c5ff..4dfa5327f 100644 --- a/test/e2e/replica_set/replica_set_test.go +++ b/test/e2e/replica_set/replica_set_test.go @@ -70,7 +70,7 @@ func TestReplicaSet(t *testing.T) { { Votes: intPtr(1), Tags: map[string]string{"foo2": "bar2"}, - Priority: strPtr("1"), + Priority: strPtr("1.0"), }, { Votes: intPtr(1), From 2cf2cae57576f8c62e4e6df2306df33353c7b295 Mon Sep 17 00:00:00 2001 From: Lucian Tosa <49226451+lucian-tosa@users.noreply.github.com> Date: Thu, 19 Sep 2024 15:38:52 +0200 Subject: [PATCH 761/790] Update CI and codeowners (#1621) * Update Codeowners * Update Github actions --- .action_templates/steps/dump-and-upload-diagnostics-always.yaml | 2 +- .action_templates/steps/dump-and-upload-diagnostics.yaml | 2 +- .github/CODEOWNERS | 2 +- .github/workflows/e2e-dispatch.yml | 2 +- .github/workflows/e2e-fork.yml | 2 +- .github/workflows/e2e.yml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.action_templates/steps/dump-and-upload-diagnostics-always.yaml b/.action_templates/steps/dump-and-upload-diagnostics-always.yaml index 61cfff7db..968ecd9ce 100644 --- a/.action_templates/steps/dump-and-upload-diagnostics-always.yaml +++ b/.action_templates/steps/dump-and-upload-diagnostics-always.yaml @@ -5,7 +5,7 @@ - name: Upload Diagnostics if: always() - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 continue-on-error: true with: name: "${{ github.event.inputs.test-name }}-${{ github.event.inputs.distro }}-diagnostics" diff --git a/.action_templates/steps/dump-and-upload-diagnostics.yaml b/.action_templates/steps/dump-and-upload-diagnostics.yaml index d8c23e3dd..17f5d2688 100644 --- a/.action_templates/steps/dump-and-upload-diagnostics.yaml +++ b/.action_templates/steps/dump-and-upload-diagnostics.yaml @@ -6,7 +6,7 @@ - name: Upload Diagnostics if: always() && steps.dump_diagnostics.outcome == 'success' - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 continue-on-error: true with: name: "${{ matrix.test-name }}-${{ matrix.distro }}-diagnostics" diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 4f7b866fc..b81f41316 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1 @@ -* @mircea-cosbuc @lsierant @slaskawi @nammn @Julien-Ben +* @mircea-cosbuc @lsierant @slaskawi @nammn @Julien-Ben @MaciejKaras @lucian-tosa @fealebenpae \ No newline at end of file diff --git a/.github/workflows/e2e-dispatch.yml b/.github/workflows/e2e-dispatch.yml index 2cb9d9cc4..7ecf8e5b6 100644 --- a/.github/workflows/e2e-dispatch.yml +++ b/.github/workflows/e2e-dispatch.yml @@ -126,7 +126,7 @@ jobs: - name: Upload Diagnostics if: always() - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 continue-on-error: true with: name: ${{ github.event.inputs.test-name }}-${{ github.event.inputs.distro diff --git a/.github/workflows/e2e-fork.yml b/.github/workflows/e2e-fork.yml index e3d465118..d224b39c1 100644 --- a/.github/workflows/e2e-fork.yml +++ b/.github/workflows/e2e-fork.yml @@ -231,7 +231,7 @@ jobs: - name: Upload Diagnostics if: always() && steps.dump_diagnostics.outcome == 'success' - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 continue-on-error: true with: name: ${{ matrix.test-name }}-${{ matrix.distro }}-diagnostics diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 4f43cf203..9cb692985 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -235,7 +235,7 @@ jobs: - name: Upload Diagnostics if: always() && steps.dump_diagnostics.outcome == 'success' - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 continue-on-error: true with: name: ${{ matrix.test-name }}-${{ matrix.distro }}-diagnostics From a6075d41543400726fe42fb0dd1fb1d8edf99db5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Sierant?= Date: Mon, 30 Sep 2024 10:32:28 +0200 Subject: [PATCH 762/790] Changed the order of CreateOrUpdate functions (#1623) * Changed order or CreateOrUpdate functions * Fixed mockedclient to return NotFound on update --- pkg/kube/client/mocked_client.go | 3 +++ pkg/kube/configmap/configmap.go | 8 ++++---- pkg/kube/secret/secret.go | 8 ++++---- pkg/kube/statefulset/statefulset.go | 13 +++++++------ 4 files changed, 18 insertions(+), 14 deletions(-) diff --git a/pkg/kube/client/mocked_client.go b/pkg/kube/client/mocked_client.go index f9b67e2b1..f4a8e499e 100644 --- a/pkg/kube/client/mocked_client.go +++ b/pkg/kube/client/mocked_client.go @@ -59,6 +59,9 @@ func (m mockedClient) Create(_ context.Context, obj k8sClient.Object, _ ...k8sCl func (m mockedClient) Update(_ context.Context, obj k8sClient.Object, _ ...k8sClient.UpdateOption) error { relevantMap := m.ensureMapFor(obj) objKey := k8sClient.ObjectKeyFromObject(obj) + if _, ok := relevantMap[objKey]; !ok { + return errors.NewNotFound(schema.GroupResource{}, obj.GetName()) + } relevantMap[objKey] = obj return nil } diff --git a/pkg/kube/configmap/configmap.go b/pkg/kube/configmap/configmap.go index a94c31c1f..36f38d469 100644 --- a/pkg/kube/configmap/configmap.go +++ b/pkg/kube/configmap/configmap.go @@ -85,14 +85,14 @@ func UpdateField(ctx context.Context, getUpdater GetUpdater, objectKey client.Ob // CreateOrUpdate creates the given ConfigMap if it doesn't exist, // or updates it if it does. func CreateOrUpdate(ctx context.Context, getUpdateCreator GetUpdateCreator, cm corev1.ConfigMap) error { - _, err := getUpdateCreator.GetConfigMap(ctx, types.NamespacedName{Name: cm.Name, Namespace: cm.Namespace}) - if err != nil { + if err := getUpdateCreator.UpdateConfigMap(ctx, cm); err != nil { if apiErrors.IsNotFound(err) { return getUpdateCreator.CreateConfigMap(ctx, cm) + } else { + return err } - return err } - return getUpdateCreator.UpdateConfigMap(ctx, cm) + return nil } // filelikePropertiesToMap converts a file-like field in a ConfigMap to a map[string]string. diff --git a/pkg/kube/secret/secret.go b/pkg/kube/secret/secret.go index fa89193db..93f9b64ea 100644 --- a/pkg/kube/secret/secret.go +++ b/pkg/kube/secret/secret.go @@ -99,14 +99,14 @@ func UpdateField(ctx context.Context, getUpdater GetUpdater, objectKey client.Ob // CreateOrUpdate creates the Secret if it doesn't exist, other wise it updates it func CreateOrUpdate(ctx context.Context, getUpdateCreator GetUpdateCreator, secret corev1.Secret) error { - _, err := getUpdateCreator.GetSecret(ctx, types.NamespacedName{Name: secret.Name, Namespace: secret.Namespace}) - if err != nil { + if err := getUpdateCreator.UpdateSecret(ctx, secret); err != nil { if SecretNotExist(err) { return getUpdateCreator.CreateSecret(ctx, secret) + } else { + return err } - return err } - return getUpdateCreator.UpdateSecret(ctx, secret) + return nil } // HasAllKeys returns true if the provided secret contains an element for every diff --git a/pkg/kube/statefulset/statefulset.go b/pkg/kube/statefulset/statefulset.go index a6c5c35ec..d6e7660cb 100644 --- a/pkg/kube/statefulset/statefulset.go +++ b/pkg/kube/statefulset/statefulset.go @@ -53,15 +53,16 @@ type GetUpdateCreateDeleter interface { // CreateOrUpdate creates the given StatefulSet if it doesn't exist, // or updates it if it does. -func CreateOrUpdate(ctx context.Context, getUpdateCreator GetUpdateCreator, sts appsv1.StatefulSet) (appsv1.StatefulSet, error) { - _, err := getUpdateCreator.GetStatefulSet(ctx, types.NamespacedName{Name: sts.Name, Namespace: sts.Namespace}) - if err != nil { +func CreateOrUpdate(ctx context.Context, getUpdateCreator GetUpdateCreator, statefulSet appsv1.StatefulSet) (appsv1.StatefulSet, error) { + if sts, err := getUpdateCreator.UpdateStatefulSet(ctx, statefulSet); err != nil { if apiErrors.IsNotFound(err) { - return appsv1.StatefulSet{}, getUpdateCreator.CreateStatefulSet(ctx, sts) + return statefulSet, getUpdateCreator.CreateStatefulSet(ctx, statefulSet) + } else { + return appsv1.StatefulSet{}, err } - return appsv1.StatefulSet{}, err + } else { + return sts, nil } - return getUpdateCreator.UpdateStatefulSet(ctx, sts) } // GetAndUpdate applies the provided function to the most recent version of the object From c83d4d487e36c835f022092d516ce622321172b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20=C5=81askawiec?= Date: Fri, 4 Oct 2024 09:42:16 +0200 Subject: [PATCH 763/790] CLOUDP-276565 MongoDB 8.0.0 GA support (#1624) * CLOUDP-276565 MongoDB 8.0.0 GA support * Fixes and release notes * Fixed Agent distro --- .action_templates/jobs/tests.yaml | 2 ++ .github/workflows/e2e-fork.yml | 2 ++ .github/workflows/e2e.yml | 2 ++ docs/RELEASE_NOTES.md | 20 ++----------- pipeline.py | 4 +-- release.json | 4 +-- test/e2e/e2eutil.go | 2 +- ...replica_set_enterprise_upgrade_5_6_test.go | 28 +++++++++++++++++++ 8 files changed, 42 insertions(+), 22 deletions(-) create mode 100644 test/e2e/replica_set_enterprise_upgrade_7_8/replica_set_enterprise_upgrade_5_6_test.go diff --git a/.action_templates/jobs/tests.yaml b/.action_templates/jobs/tests.yaml index 400b47d77..f360ee3d6 100644 --- a/.action_templates/jobs/tests.yaml +++ b/.action_templates/jobs/tests.yaml @@ -13,6 +13,8 @@ tests: distro: ubi - test-name: replica_set_enterprise_upgrade_6_7 distro: ubi + - test-name: replica_set_enterprise_upgrade_7_8 + distro: ubi - test-name: replica_set_recovery distro: ubi - test-name: replica_set_mongod_readiness diff --git a/.github/workflows/e2e-fork.yml b/.github/workflows/e2e-fork.yml index d224b39c1..eede9c4ed 100644 --- a/.github/workflows/e2e-fork.yml +++ b/.github/workflows/e2e-fork.yml @@ -98,6 +98,8 @@ jobs: distro: ubi - test-name: replica_set_enterprise_upgrade_6_7 distro: ubi + - test-name: replica_set_enterprise_upgrade_7_8 + distro: ubi - test-name: replica_set_recovery distro: ubi - test-name: replica_set_mongod_readiness diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 9cb692985..6eb0a6d53 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -104,6 +104,8 @@ jobs: distro: ubi - test-name: replica_set_enterprise_upgrade_6_7 distro: ubi + - test-name: replica_set_enterprise_upgrade_7_8 + distro: ubi - test-name: replica_set_recovery distro: ubi - test-name: replica_set_mongod_readiness diff --git a/docs/RELEASE_NOTES.md b/docs/RELEASE_NOTES.md index aed986aa2..280ab59b9 100644 --- a/docs/RELEASE_NOTES.md +++ b/docs/RELEASE_NOTES.md @@ -1,18 +1,4 @@ -# MongoDB Kubernetes Operator 0.11.0 +# MongoDB Kubernetes Operator 0.12.0 -## Migrating agent images to ubi -All agent images were updated to use the ubi repo - -## Documentation improvements -Improvements were made to the documentation of using the community operator as well as the one for local development. - -## Logging changes -- Added `AuditLogRotate` field to `AgentConfiguration` -- Fixed JSON key to be lower case: `logRotate` - -## Bug Fixes -- Users removed from the resource are now also deleted from the database and their connection string secrets are cleaned up -- Scram secret name collisions will now be caught when validating specs - -## Important Bumps -- Bumped go to 1.22 \ No newline at end of file +## Added support for MongoDB 8.0.0 GA +MongoDB 8.0.0 GA is now officially supported by the Operator diff --git a/pipeline.py b/pipeline.py index b8324d249..f288d1a5b 100644 --- a/pipeline.py +++ b/pipeline.py @@ -26,8 +26,8 @@ TOOLS_DISTRO_KEY = "tools_distro" AGENT_DISTROS_PER_ARCH = { - "amd64": {AGENT_DISTRO_KEY: "rhel7_x86_64", TOOLS_DISTRO_KEY: "rhel70-x86_64"}, - "arm64": {AGENT_DISTRO_KEY: "amzn2_aarch64", TOOLS_DISTRO_KEY: "rhel82-aarch64"}, + "amd64": {AGENT_DISTRO_KEY: "rhel8_x86_64", TOOLS_DISTRO_KEY: "rhel88-x86_64"}, + "arm64": {AGENT_DISTRO_KEY: "amzn2_aarch64", TOOLS_DISTRO_KEY: "rhel88-aarch64"}, } diff --git a/release.json b/release.json index 403d3f789..be9f04b61 100644 --- a/release.json +++ b/release.json @@ -3,6 +3,6 @@ "operator": "0.11.0", "version-upgrade-hook": "1.0.9", "readiness-probe": "1.0.20", - "agent": "107.0.7.8596-1", - "agent-tools-version": "100.9.4" + "agent": "108.0.0.8694-1", + "agent-tools-version": "100.10.0" } \ No newline at end of file diff --git a/test/e2e/e2eutil.go b/test/e2e/e2eutil.go index 5b78df60a..41f64c287 100644 --- a/test/e2e/e2eutil.go +++ b/test/e2e/e2eutil.go @@ -70,7 +70,7 @@ func NewTestMongoDB(ctx *TestContext, name string, namespace string) (mdbv1.Mong Spec: mdbv1.MongoDBCommunitySpec{ Members: 3, Type: "ReplicaSet", - Version: "7.0.2", + Version: "8.0.0", Arbiters: 0, Security: mdbv1.Security{ Authentication: mdbv1.Authentication{ diff --git a/test/e2e/replica_set_enterprise_upgrade_7_8/replica_set_enterprise_upgrade_5_6_test.go b/test/e2e/replica_set_enterprise_upgrade_7_8/replica_set_enterprise_upgrade_5_6_test.go new file mode 100644 index 000000000..00cdf8f10 --- /dev/null +++ b/test/e2e/replica_set_enterprise_upgrade_7_8/replica_set_enterprise_upgrade_5_6_test.go @@ -0,0 +1,28 @@ +package replica_set + +import ( + "context" + "fmt" + "os" + "testing" + + e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" + "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/replica_set_enterprise_upgrade" +) + +var ( + versionsForUpgrades = []string{"7.0.12", "8.0.0"} +) + +func TestMain(m *testing.M) { + code, err := e2eutil.RunTest(m) + if err != nil { + fmt.Println(err) + } + os.Exit(code) +} + +func TestReplicaSet(t *testing.T) { + ctx := context.Background() + replica_set_enterprise_upgrade.DeployEnterpriseAndUpgradeTest(ctx, t, versionsForUpgrades) +} From 95f954d9b9f4800c7f3d1c39ded8b2707d28edb1 Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Tue, 15 Oct 2024 11:18:00 +0200 Subject: [PATCH 764/790] CLOUDP-274724 - readinessProbe: use human readable timestamps instead (#1630) * readinessProbe: use human readable timestamps instead * bump readinessProbe --- cmd/readiness/main.go | 7 +++++-- release.json | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/cmd/readiness/main.go b/cmd/readiness/main.go index ca3bfffb1..6cf9e7804 100644 --- a/cmd/readiness/main.go +++ b/cmd/readiness/main.go @@ -208,15 +208,18 @@ func parseHealthStatus(reader io.Reader) (health.Status, error) { } func initLogger(l *lumberjack.Logger) { + encoderConfig := zap.NewProductionEncoderConfig() + encoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder + consoleCore := zapcore.NewCore( - zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig()), + zapcore.NewJSONEncoder(encoderConfig), zapcore.AddSync(os.Stdout), zap.DebugLevel) cores := []zapcore.Core{consoleCore} if config.ReadBoolWitDefault(config.WithAgentFileLogging, "true") { fileCore := zapcore.NewCore( - zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig()), + zapcore.NewJSONEncoder(encoderConfig), zapcore.AddSync(l), zap.DebugLevel) cores = append(cores, fileCore) diff --git a/release.json b/release.json index be9f04b61..fdb1859a0 100644 --- a/release.json +++ b/release.json @@ -2,7 +2,7 @@ "golang-builder-image": "golang:1.22", "operator": "0.11.0", "version-upgrade-hook": "1.0.9", - "readiness-probe": "1.0.20", + "readiness-probe": "1.0.21", "agent": "108.0.0.8694-1", "agent-tools-version": "100.10.0" } \ No newline at end of file From 508fc47a27c2602b660c4924f65aa5e5d194d9d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Sierant?= Date: Mon, 2 Dec 2024 15:03:57 +0100 Subject: [PATCH 765/790] Bumped Go to 1.23 and kube APIs to 1.29 (#1640) * Bumped Go to 1.23 and kube APIs to 1.29 * Bumped go version in actions * Bumped go version in release.json --- .github/workflows/go.yml | 2 +- go.mod | 40 ++--- go.sum | 160 +++++------------- release.json | 4 +- ...plica_set_custom_persistent_volume_test.go | 2 +- 5 files changed, 72 insertions(+), 136 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 79ed8f21f..be273651c 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -14,7 +14,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v4 with: - go-version: '1.22' + go-version: '1.23' - name: Test api run: go test -v ./api/... diff --git a/go.mod b/go.mod index 3c6823de3..432ec19c4 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/mongodb/mongodb-kubernetes-operator -go 1.22 +go 1.23 require ( github.com/blang/semver v3.5.1+incompatible @@ -14,9 +14,9 @@ require ( go.mongodb.org/mongo-driver v1.16.0 go.uber.org/zap v1.27.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 - k8s.io/api v0.27.12 - k8s.io/apimachinery v0.27.12 - k8s.io/client-go v0.27.12 + k8s.io/api v0.29.11 + k8s.io/apimachinery v0.29.11 + k8s.io/client-go v0.29.11 k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 sigs.k8s.io/controller-runtime v0.15.3 sigs.k8s.io/yaml v1.4.0 @@ -28,21 +28,22 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/emicklei/go-restful/v3 v3.9.0 // indirect + github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/evanphx/json-patch v4.12.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect - github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect - github.com/go-openapi/jsonreference v0.20.1 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.22.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/google/gnostic v0.5.7-v3refs // indirect + github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect - github.com/google/gofuzz v1.1.0 // indirect + github.com/google/gofuzz v1.2.0 // indirect github.com/google/uuid v1.3.0 // indirect + github.com/gorilla/websocket v1.5.0 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -54,21 +55,22 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/montanaflynn/stats v0.7.1 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_golang v1.15.1 // indirect + github.com/prometheus/client_golang v1.16.0 // indirect github.com/prometheus/client_model v0.4.0 // indirect - github.com/prometheus/common v0.42.0 // indirect - github.com/prometheus/procfs v0.9.0 // indirect + github.com/prometheus/common v0.44.0 // indirect + github.com/prometheus/procfs v0.10.1 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.2 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect - go.uber.org/multierr v1.10.0 // indirect + go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.25.0 // indirect golang.org/x/net v0.27.0 // indirect - golang.org/x/oauth2 v0.7.0 // indirect + golang.org/x/oauth2 v0.10.0 // indirect golang.org/x/sync v0.7.0 // indirect golang.org/x/sys v0.22.0 // indirect golang.org/x/term v0.22.0 // indirect @@ -80,10 +82,10 @@ require ( gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.27.12 // indirect - k8s.io/component-base v0.27.12 // indirect - k8s.io/klog/v2 v2.90.1 // indirect - k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect + k8s.io/apiextensions-apiserver v0.29.11 // indirect + k8s.io/component-base v0.29.11 // indirect + k8s.io/klog/v2 v2.110.1 // indirect + k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect ) diff --git a/go.sum b/go.sum index 96e6f8a16..42c2c61f3 100644 --- a/go.sum +++ b/go.sum @@ -1,84 +1,63 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE= -github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= +github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= -github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= -github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= -github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= github.com/go-logr/zapr v1.2.4/go.mod h1:FyHWQIzQORZ0QVE1BtVHv3cKtNLuXsbNLtpuhNapBOA= github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= -github.com/go-openapi/jsonreference v0.20.1 h1:FBLnyygC4/IZZr893oiomc9XaghoveYTrLC1F86HID8= -github.com/go-openapi/jsonreference v0.20.1/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54= -github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= -github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= +github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= @@ -94,7 +73,6 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -117,38 +95,37 @@ github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8 github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q= -github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k= -github.com/onsi/gomega v1.27.7 h1:fVih9JD6ogIiHUN6ePK7HJidyEDpWGVB5mzM7cWNXoU= -github.com/onsi/gomega v1.27.7/go.mod h1:1p8OOlwo2iUUDsHnOrjE5UKYJ+e3W8eQ3qSlRahPmr4= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4= +github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o= +github.com/onsi/gomega v1.29.0 h1:KIA/t2t5UBzoirT4H9tsML45GEbo3ouUnBHsCfD2tVg= +github.com/onsi/gomega v1.29.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI= -github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= +github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= -github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= -github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= -github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= -github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= -github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= -github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= +github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= +github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= +github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= @@ -171,8 +148,8 @@ go.mongodb.org/mongo-driver v1.16.0 h1:tpRsfBJMROVHKpdGyc1BBEzzjDUWjItxbVSZ8Ls4B go.mongodb.org/mongo-driver v1.16.0/go.mod h1:oB6AhJQvFQL4LEHyXi6aJzQJtBiTQHiAd83l0GdFaiw= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= -go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -181,17 +158,9 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -201,11 +170,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g= -golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8= +golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -213,7 +179,6 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -221,7 +186,6 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -238,10 +202,6 @@ golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -254,70 +214,44 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.3.0 h1:8NFhfS6gzxNqjLIYnZxg319wZ5Qjnx4m/CcX+Klzazc= gomodules.xyz/jsonpatch/v2 v2.3.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -k8s.io/api v0.27.12 h1:Qprj/nuFj4xjbsAuJ05F1sCHs1d0x33n/Ni0oAVEFDo= -k8s.io/api v0.27.12/go.mod h1:PNRL63V26JzKe2++ho6W/YRp3k9XG7nirN4J7WRy5gY= -k8s.io/apiextensions-apiserver v0.27.12 h1:1A1+0rlOrqKi+LbCTf3MeFbYmBFGoQ9rFHxBgFnhOpE= -k8s.io/apiextensions-apiserver v0.27.12/go.mod h1:KkUHCjJ/Awhly9NnVsYySZtQSxuWU+8IL4p2ffE4JQ8= -k8s.io/apimachinery v0.27.12 h1:Nt20vwaAHcZsM4WdkOtLaDeBJHg9QJW8JyOOEn6xzRA= -k8s.io/apimachinery v0.27.12/go.mod h1:5/SjQaDYQgZOv8kuzNMzmNGrqh4/iyknC5yWjxU9ll8= -k8s.io/client-go v0.27.12 h1:ouIB3ZitBjmBWh/9auP4erVl8AXkheqcmbH7FSFa7DI= -k8s.io/client-go v0.27.12/go.mod h1:h3X7RGr5s9Wm4NtI06Bzt3am4Kj6aXuZQcP7OD+48Sk= -k8s.io/component-base v0.27.12 h1:2/ooM9/gNxS2fZuRnLj3TWWg7KnEYmRj321du10waZA= -k8s.io/component-base v0.27.12/go.mod h1:SPA6ABqK2HDRuzMjoEEkj7ciDvK5bUpdHtvZQwdi/xM= -k8s.io/klog/v2 v2.90.1 h1:m4bYOKall2MmOiRaR1J+We67Do7vm9KiQVlT96lnHUw= -k8s.io/klog/v2 v2.90.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f h1:2kWPakN3i/k81b0gvD5C5FJ2kxm1WrQFanWchyKuqGg= -k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f/go.mod h1:byini6yhqGC14c3ebc/QwanvYwhuMWF6yz2F8uwW8eg= -k8s.io/utils v0.0.0-20230209194617-a36077c30491 h1:r0BAOLElQnnFhE/ApUsg3iHdVYYPBjNSSOMowRZxxsY= -k8s.io/utils v0.0.0-20230209194617-a36077c30491/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/api v0.29.11 h1:6FwDo33f1WX5Yu0RQTX9YAd3wth8Ik0B4SXQKsoQfbk= +k8s.io/api v0.29.11/go.mod h1:3TDAW1OpFbz/Yx5r0W06b6eiAfHEwtH61VYDzpTU4Ng= +k8s.io/apiextensions-apiserver v0.29.11 h1:ytJJQ8EK0GzPa80tnPkfDoBGoNPMwqfaSWwg4FKmbEU= +k8s.io/apiextensions-apiserver v0.29.11/go.mod h1:eqKsza/nErdFNltXoVZmRt9vX99ooDLDMTcIcOG0ueg= +k8s.io/apimachinery v0.29.11 h1:55+6ue9advpA7T0sX2ZJDHCLKuiFfrAAR/39VQN9KEQ= +k8s.io/apimachinery v0.29.11/go.mod h1:i3FJVwhvSp/6n8Fl4K97PJEP8C+MM+aoDq4+ZJBf70Y= +k8s.io/client-go v0.29.11 h1:mBX7Ub0uqpLMwWz3J/AGS/xKOZsjr349qZ1vxVoL1l8= +k8s.io/client-go v0.29.11/go.mod h1:WOEoi/eLg2YEg3/yEd7YK3CNScYkM8AEScQadxUnaTE= +k8s.io/component-base v0.29.11 h1:H3GJIyDNPrscvXGP6wx+9gApcwwmrUd0YtCGp5BcHBA= +k8s.io/component-base v0.29.11/go.mod h1:0qu1WStER4wu5o8RMRndZUWPVcPH1XBy/QQiDcD6lew= +k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0= +k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo= +k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= +k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 h1:jgGTlFYnhF1PM1Ax/lAlxUPE+KfCIXHaathvJg1C3ak= k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/controller-runtime v0.15.3 h1:L+t5heIaI3zeejoIyyvLQs5vTVu/67IU2FfisVzFlBc= sigs.k8s.io/controller-runtime v0.15.3/go.mod h1:kp4jckA4vTx281S/0Yk2LFEEQe67mjg+ev/yknv47Ds= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/release.json b/release.json index fdb1859a0..4c824f66c 100644 --- a/release.json +++ b/release.json @@ -1,8 +1,8 @@ { - "golang-builder-image": "golang:1.22", + "golang-builder-image": "golang:1.23", "operator": "0.11.0", "version-upgrade-hook": "1.0.9", "readiness-probe": "1.0.21", "agent": "108.0.0.8694-1", "agent-tools-version": "100.10.0" -} \ No newline at end of file +} diff --git a/test/e2e/replica_set_custom_persistent_volume/replica_set_custom_persistent_volume_test.go b/test/e2e/replica_set_custom_persistent_volume/replica_set_custom_persistent_volume_test.go index 752486361..db16c5ebe 100644 --- a/test/e2e/replica_set_custom_persistent_volume/replica_set_custom_persistent_volume_test.go +++ b/test/e2e/replica_set_custom_persistent_volume/replica_set_custom_persistent_volume_test.go @@ -94,7 +94,7 @@ func getPvc(pvcType string, mdb v1.MongoDBCommunity) corev1.PersistentVolumeClai Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{"type": pvcType}, }, - Resources: corev1.ResourceRequirements{ + Resources: corev1.VolumeResourceRequirements{ Requests: map[corev1.ResourceName]resource.Quantity{"storage": *resource.NewScaledQuantity(int64(8), resource.Giga)}, }, StorageClassName: &defaultStorageClass, From 4818d41c25f5661990f32298dbe4f783d3407b09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Kara=C5=9B?= <6159874+MaciejKaras@users.noreply.github.com> Date: Mon, 16 Dec 2024 16:20:32 +0100 Subject: [PATCH 766/790] CLOUDP-286259 - Update package crypto to remediate multiple CVEs (#1644) --- go.mod | 10 +++++----- go.sum | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 432ec19c4..071e61542 100644 --- a/go.mod +++ b/go.mod @@ -68,13 +68,13 @@ require ( github.com/xdg-go/stringprep v1.0.4 // indirect github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.25.0 // indirect + golang.org/x/crypto v0.31.0 // indirect golang.org/x/net v0.27.0 // indirect golang.org/x/oauth2 v0.10.0 // indirect - golang.org/x/sync v0.7.0 // indirect - golang.org/x/sys v0.22.0 // indirect - golang.org/x/term v0.22.0 // indirect - golang.org/x/text v0.16.0 // indirect + golang.org/x/sync v0.10.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/term v0.27.0 // indirect + golang.org/x/text v0.21.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.23.0 // indirect gomodules.xyz/jsonpatch/v2 v2.3.0 // indirect diff --git a/go.sum b/go.sum index 42c2c61f3..cbf243274 100644 --- a/go.sum +++ b/go.sum @@ -156,8 +156,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= -golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= @@ -177,8 +177,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -186,19 +186,19 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= -golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= -golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= +golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From f3170c62c794768d123b81cf1793525e098891ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Kara=C5=9B?= <6159874+MaciejKaras@users.noreply.github.com> Date: Wed, 18 Dec 2024 11:28:31 +0100 Subject: [PATCH 767/790] Release MongoDB Kubernetes Operator v0.12.0 (#1646) --- config/manager/manager.yaml | 6 +++--- deploy/openshift/operator_openshift.yaml | 6 +++--- docs/RELEASE_NOTES.md | 7 +++++++ docs/how-to-release.md | 23 ++++++++++++++++------- release.json | 6 +++--- 5 files changed, 32 insertions(+), 16 deletions(-) diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 8e2d9a3ce..9013a8451 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -45,16 +45,16 @@ spec: - name: OPERATOR_NAME value: mongodb-kubernetes-operator - name: AGENT_IMAGE - value: quay.io/mongodb/mongodb-agent-ubi:107.0.7.8596-1 + value: quay.io/mongodb/mongodb-agent-ubi:108.0.2.8729-1 - name: VERSION_UPGRADE_HOOK_IMAGE value: quay.io/mongodb/mongodb-kubernetes-operator-version-upgrade-post-start-hook:1.0.9 - name: READINESS_PROBE_IMAGE - value: quay.io/mongodb/mongodb-kubernetes-readinessprobe:1.0.20 + value: quay.io/mongodb/mongodb-kubernetes-readinessprobe:1.0.22 - name: MONGODB_IMAGE value: mongodb-community-server - name: MONGODB_REPO_URL value: quay.io/mongodb - image: quay.io/mongodb/mongodb-kubernetes-operator:0.11.0 + image: quay.io/mongodb/mongodb-kubernetes-operator:0.12.0 imagePullPolicy: Always name: mongodb-kubernetes-operator resources: diff --git a/deploy/openshift/operator_openshift.yaml b/deploy/openshift/operator_openshift.yaml index 8e7d47ce6..c0f3cf1ce 100644 --- a/deploy/openshift/operator_openshift.yaml +++ b/deploy/openshift/operator_openshift.yaml @@ -47,16 +47,16 @@ spec: - name: OPERATOR_NAME value: mongodb-kubernetes-operator - name: AGENT_IMAGE - value: quay.io/mongodb/mongodb-agent-ubi:107.0.7.8596-1 + value: quay.io/mongodb/mongodb-agent-ubi:108.0.2.8729-1 - name: READINESS_PROBE_IMAGE - value: quay.io/mongodb/mongodb-kubernetes-readinessprobe:1.0.20 + value: quay.io/mongodb/mongodb-kubernetes-readinessprobe:1.0.22 - name: VERSION_UPGRADE_HOOK_IMAGE value: quay.io/mongodb/mongodb-kubernetes-operator-version-upgrade-post-start-hook:1.0.9 - name: MONGODB_IMAGE value: mongo - name: MONGODB_REPO_URL value: quay.io - image: quay.io/mongodb/mongodb-kubernetes-operator:0.11.0 + image: quay.io/mongodb/mongodb-kubernetes-operator:0.12.0 imagePullPolicy: Always name: mongodb-kubernetes-operator resources: diff --git a/docs/RELEASE_NOTES.md b/docs/RELEASE_NOTES.md index 280ab59b9..7a92af10c 100644 --- a/docs/RELEASE_NOTES.md +++ b/docs/RELEASE_NOTES.md @@ -2,3 +2,10 @@ ## Added support for MongoDB 8.0.0 GA MongoDB 8.0.0 GA is now officially supported by the Operator + +## Minor fixes and improvements + + - Use pointer field for priority in member options + - readinessProbe: use human-readable timestamps instead + - Bumped Go to 1.23 and kube APIs to 1.29 + - Update package crypto to remediate multiple CVEs diff --git a/docs/how-to-release.md b/docs/how-to-release.md index 499cbafb2..f92412433 100644 --- a/docs/how-to-release.md +++ b/docs/how-to-release.md @@ -1,16 +1,25 @@ ## How to Release * Prepare release PR: - * Pull the changes in the helm-charts sub module folder to get the latest main. - * `cd helm-charts && git pull origin main`. + * Pull the changes in the helm-charts submodule folder to get the latest main. + * `cd helm-charts` + * `git submodule update --init` - if submodule was not initialised before + * `git pull origin main` * Update any changing versions in [release.json](../release.json). + * `operator` - always when doing a release + * `version-upgrade-hook` - whenever we make changes in the [versionhook](../cmd/versionhook) files + * `readiness-probe` - whenever we make changes in the [readiness](../cmd/readiness) files + * `agent` - newest version available in `ops-manager` `conf-hosted.properties` file under `automation.agent.version` + * `agent-tools-version` - newest version available in `ops-manager` `conf-hosted.properties` file under `mongotools.version` * Ensure that [the release notes](./RELEASE_NOTES.md) are up to date for this release. + * all merged PRs have a covered entry in the release notes. For example, you can use `git log v0.11.0..HEAD --reverse --oneline` to get the list of commits after previous release * Run `python scripts/ci/update_release.py` to update the relevant yaml manifests. - * Copy `CRD`s to Helm Chart - - `cp config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml helm-charts/charts/community-operator-crds/templates/mongodbcommunity.mongodb.com_mongodbcommunity.yaml` - - commit changes to the [helm-charts submodule](https://github.com/mongodb/helm-charts) and create a PR against it ([similar to this one](https://github.com/mongodb/helm-charts/pull/163)). - - do not merge helm-charts PR until release PR is merged and the images are pushed to quay.io. - - do not commit the submodule change in the release pr of the community repository. + * **use venv and then `python3 -m pip install -r requirements.txt`** + * Copy ``CRD`s`` to Helm Chart + * `cp config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml helm-charts/charts/community-operator-crds/templates/mongodbcommunity.mongodb.com_mongodbcommunity.yaml` + * commit changes to the [helm-charts submodule](https://github.com/mongodb/helm-charts) and create a PR against it ([similar to this one](https://github.com/mongodb/helm-charts/pull/163)). + * do not merge helm-charts PR until release PR is merged and the images are pushed to quay.io. + * do not commit the submodule change in the release pr of the community repository. * Commit all changes (except for the submodule change) * Create a PR with the title `Release MongoDB Kubernetes Operator v` (the title must match this pattern). * Wait for the tests to pass and merge the PR. diff --git a/release.json b/release.json index 4c824f66c..6e080a120 100644 --- a/release.json +++ b/release.json @@ -1,8 +1,8 @@ { "golang-builder-image": "golang:1.23", - "operator": "0.11.0", + "operator": "0.12.0", "version-upgrade-hook": "1.0.9", - "readiness-probe": "1.0.21", - "agent": "108.0.0.8694-1", + "readiness-probe": "1.0.22", + "agent": "108.0.2.8729-1", "agent-tools-version": "100.10.0" } From 85086f749a64320a066af72c30047744d193fbb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Kara=C5=9B?= <6159874+MaciejKaras@users.noreply.github.com> Date: Wed, 18 Dec 2024 14:48:45 +0100 Subject: [PATCH 768/790] Pin helm-charts to latest release (#1647) --- helm-charts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm-charts b/helm-charts index f03b5c02f..ad091e266 160000 --- a/helm-charts +++ b/helm-charts @@ -1 +1 @@ -Subproject commit f03b5c02f19a40370961652e8fb95060a443f320 +Subproject commit ad091e2667b42cdbe240806322659525cb45663e From 097e13b2dab27f6361b773c84f984d706d891944 Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Wed, 8 Jan 2025 15:20:47 +0100 Subject: [PATCH 769/790] update cve (#1652) --- go.mod | 10 +++++----- go.sum | 16 ++++++++-------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/go.mod b/go.mod index 071e61542..428afc924 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/mongodb/mongodb-kubernetes-operator -go 1.23 +go 1.23.0 require ( github.com/blang/semver v3.5.1+incompatible @@ -68,12 +68,12 @@ require ( github.com/xdg-go/stringprep v1.0.4 // indirect github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.31.0 // indirect - golang.org/x/net v0.27.0 // indirect + golang.org/x/crypto v0.32.0 // indirect + golang.org/x/net v0.34.0 // indirect golang.org/x/oauth2 v0.10.0 // indirect golang.org/x/sync v0.10.0 // indirect - golang.org/x/sys v0.28.0 // indirect - golang.org/x/term v0.27.0 // indirect + golang.org/x/sys v0.29.0 // indirect + golang.org/x/term v0.28.0 // indirect golang.org/x/text v0.21.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.23.0 // indirect diff --git a/go.sum b/go.sum index cbf243274..9afd9ec86 100644 --- a/go.sum +++ b/go.sum @@ -156,8 +156,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= -golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= +golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= @@ -168,8 +168,8 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= -golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= +golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= +golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8= golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -186,12 +186,12 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= -golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg= +golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= From 6acec2565fbe80f172c6114432f39ac5a1080e39 Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Fri, 10 Jan 2025 16:18:45 +0100 Subject: [PATCH 770/790] fix tests - adding missing role, increase timeout (#1655) * fix test by adding the missing role * bump timeout - sometimes the operator is not starting up in time --- .../replica_set_custom_role_test.go | 13 +++++++++++++ test/e2e/setup/setup.go | 3 ++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/test/e2e/replica_set_custom_role/replica_set_custom_role_test.go b/test/e2e/replica_set_custom_role/replica_set_custom_role_test.go index 0b019d299..54075a71d 100644 --- a/test/e2e/replica_set_custom_role/replica_set_custom_role_test.go +++ b/test/e2e/replica_set_custom_role/replica_set_custom_role_test.go @@ -71,6 +71,19 @@ func TestReplicaSetCustomRole(t *testing.T) { }}, Roles: []mdbv1.Role{}, }, + { + Role: "MongodbAutomationAgentUserRole", + DB: "admin", + Privileges: []mdbv1.Privilege{ + { + Resource: mdbv1.Resource{ + AnyResource: true, + }, + Actions: []string{"bypassDefaultMaxTimeMS"}, + }, + }, + Roles: []mdbv1.Role{}, + }, } _, err := setup.GeneratePasswordForUser(testCtx, user, "") diff --git a/test/e2e/setup/setup.go b/test/e2e/setup/setup.go index 22e87f253..88769bf78 100644 --- a/test/e2e/setup/setup.go +++ b/test/e2e/setup/setup.go @@ -237,7 +237,7 @@ func DeployOperator(ctx context.Context, config TestConfig, resourceName string, return err } - if err := wait.PollUntilContextTimeout(ctx, time.Second, 60*time.Second, true, hasDeploymentRequiredReplicas(&dep)); err != nil { + if err := wait.PollUntilContextTimeout(ctx, time.Second*2, 120*time.Second, true, hasDeploymentRequiredReplicas(&dep)); err != nil { return errors.New("error building operator deployment: the deployment does not have the required replicas") } fmt.Println("Successfully installed the operator deployment") @@ -280,6 +280,7 @@ func hasDeploymentRequiredReplicas(dep *appsv1.Deployment) wait.ConditionWithCon if dep.Status.ReadyReplicas == *dep.Spec.Replicas { return true, nil } + fmt.Printf("Deployment not ready! ReadyReplicas: %d, Spec.Replicas: %d\n", dep.Status.ReadyReplicas, *dep.Spec.Replicas) return false, nil } } From e99a3ee84a969f231ebcd38da183bc3190a47921 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Kara=C5=9B?= <6159874+MaciejKaras@users.noreply.github.com> Date: Tue, 14 Jan 2025 13:22:43 +0100 Subject: [PATCH 771/790] Update dependabot and CODEOWNERS file (#1660) --- .github/CODEOWNERS | 2 +- .github/dependabot.yml | 11 +++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index b81f41316..db61cf612 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1 @@ -* @mircea-cosbuc @lsierant @slaskawi @nammn @Julien-Ben @MaciejKaras @lucian-tosa @fealebenpae \ No newline at end of file +* @mircea-cosbuc @lsierant @nammn @Julien-Ben @MaciejKaras @lucian-tosa @fealebenpae @m1kola \ No newline at end of file diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 73b9af6e4..d19a62dbe 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,9 +5,8 @@ updates: schedule: interval: weekly day: monday - reviewers: - - "slaskawi" - - "lsierant" - - "mircea-cosbuc" - - "nammn" - - "Julien-Ben" + - package-ecosystem: pip + directory: "/" + schedule: + interval: weekly + day: monday From 423b6ebbf169ec55124544160676b2e46fb1b31d Mon Sep 17 00:00:00 2001 From: Lucian Tosa <49226451+lucian-tosa@users.noreply.github.com> Date: Mon, 3 Feb 2025 18:16:30 +0100 Subject: [PATCH 772/790] Update controller runtime and kube-api (#1671) --- cmd/manager/main.go | 5 +- docs/RELEASE_NOTES.md | 10 +--- go.mod | 32 ++++++------ go.sum | 81 +++++++++++++++---------------- pkg/kube/client/mocked_manager.go | 4 +- test/e2e/client.go | 1 - 6 files changed, 62 insertions(+), 71 deletions(-) diff --git a/cmd/manager/main.go b/cmd/manager/main.go index 4a1eb6d93..a874faff6 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -3,6 +3,7 @@ package main import ( "fmt" "os" + "sigs.k8s.io/controller-runtime/pkg/cache" mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" "github.com/mongodb/mongodb-kubernetes-operator/controllers" @@ -82,7 +83,9 @@ func main() { // Create a new Cmd to provide shared dependencies and start components mgr, err := manager.New(cfg, manager.Options{ - Namespace: watchNamespace, + Cache: cache.Options{ + DefaultNamespaces: map[string]cache.Config{watchNamespace: {}}, + }, }) if err != nil { log.Sugar().Fatalf("Unable to create manager: %v", err) diff --git a/docs/RELEASE_NOTES.md b/docs/RELEASE_NOTES.md index 7a92af10c..13beb2133 100644 --- a/docs/RELEASE_NOTES.md +++ b/docs/RELEASE_NOTES.md @@ -1,11 +1,5 @@ -# MongoDB Kubernetes Operator 0.12.0 - -## Added support for MongoDB 8.0.0 GA -MongoDB 8.0.0 GA is now officially supported by the Operator +# MongoDB Kubernetes Operator 0.13.0 ## Minor fixes and improvements - - Use pointer field for priority in member options - - readinessProbe: use human-readable timestamps instead - - Bumped Go to 1.23 and kube APIs to 1.29 - - Update package crypto to remediate multiple CVEs + - Bumped kube APIs to 1.30 diff --git a/go.mod b/go.mod index 428afc924..ab7ea3254 100644 --- a/go.mod +++ b/go.mod @@ -14,11 +14,11 @@ require ( go.mongodb.org/mongo-driver v1.16.0 go.uber.org/zap v1.27.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 - k8s.io/api v0.29.11 - k8s.io/apimachinery v0.29.11 - k8s.io/client-go v0.29.11 + k8s.io/api v0.30.1 + k8s.io/apimachinery v0.30.1 + k8s.io/client-go v0.30.1 k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 - sigs.k8s.io/controller-runtime v0.15.3 + sigs.k8s.io/controller-runtime v0.18.7 sigs.k8s.io/yaml v1.4.0 ) @@ -30,7 +30,7 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/evanphx/json-patch v4.12.0+incompatible // indirect - github.com/evanphx/json-patch/v5 v5.6.0 // indirect + github.com/evanphx/json-patch/v5 v5.9.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect @@ -49,7 +49,7 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.13.6 // indirect github.com/mailru/easyjson v0.7.7 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/moby/spdystream v0.2.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect @@ -58,10 +58,10 @@ require ( github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_golang v1.16.0 // indirect - github.com/prometheus/client_model v0.4.0 // indirect - github.com/prometheus/common v0.44.0 // indirect - github.com/prometheus/procfs v0.10.1 // indirect + github.com/prometheus/client_golang v1.18.0 // indirect + github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/common v0.45.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.2 // indirect @@ -69,23 +69,23 @@ require ( github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.32.0 // indirect + golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect golang.org/x/net v0.34.0 // indirect - golang.org/x/oauth2 v0.10.0 // indirect + golang.org/x/oauth2 v0.12.0 // indirect golang.org/x/sync v0.10.0 // indirect golang.org/x/sys v0.29.0 // indirect golang.org/x/term v0.28.0 // indirect golang.org/x/text v0.21.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.23.0 // indirect - gomodules.xyz/jsonpatch/v2 v2.3.0 // indirect + gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/appengine v1.6.7 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.29.11 // indirect - k8s.io/component-base v0.29.11 // indirect - k8s.io/klog/v2 v2.110.1 // indirect - k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect + k8s.io/apiextensions-apiserver v0.30.1 // indirect + k8s.io/klog/v2 v2.120.1 // indirect + k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect ) diff --git a/go.sum b/go.sum index 9afd9ec86..6b5e2d78c 100644 --- a/go.sum +++ b/go.sum @@ -14,17 +14,16 @@ github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxER github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= -github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= +github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= +github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= -github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= -github.com/go-logr/zapr v1.2.4/go.mod h1:FyHWQIzQORZ0QVE1BtVHv3cKtNLuXsbNLtpuhNapBOA= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= @@ -37,7 +36,6 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= @@ -64,7 +62,6 @@ github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+l github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM= github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= -github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -82,8 +79,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= -github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -97,23 +94,22 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4= -github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o= -github.com/onsi/gomega v1.29.0 h1:KIA/t2t5UBzoirT4H9tsML45GEbo3ouUnBHsCfD2tVg= -github.com/onsi/gomega v1.29.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/onsi/ginkgo/v2 v2.17.1 h1:V++EzdbhI4ZV4ev0UTIj0PzhzOcReJFyJaLjtSF55M8= +github.com/onsi/ginkgo/v2 v2.17.1/go.mod h1:llBI3WDLL9Z6taip6f33H76YcWtJv+7R3HigUjbIBOs= +github.com/onsi/gomega v1.32.0 h1:JRYU78fJ1LPxlckP6Txi/EYqJvjtMrDC04/MM5XRHPk= +github.com/onsi/gomega v1.32.0/go.mod h1:a4x4gW6Pz2yK1MAmvluYme5lvYTn61afQ2ETw/8n4Lg= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= -github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= -github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= -github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= -github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= -github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= -github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= -github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= +github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= +github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= +github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= @@ -158,6 +154,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= +golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA= +golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= @@ -170,9 +168,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= -golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8= -golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4= +golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -212,8 +209,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gomodules.xyz/jsonpatch/v2 v2.3.0 h1:8NFhfS6gzxNqjLIYnZxg319wZ5Qjnx4m/CcX+Klzazc= -gomodules.xyz/jsonpatch/v2 v2.3.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= +gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= @@ -231,24 +228,22 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/api v0.29.11 h1:6FwDo33f1WX5Yu0RQTX9YAd3wth8Ik0B4SXQKsoQfbk= -k8s.io/api v0.29.11/go.mod h1:3TDAW1OpFbz/Yx5r0W06b6eiAfHEwtH61VYDzpTU4Ng= -k8s.io/apiextensions-apiserver v0.29.11 h1:ytJJQ8EK0GzPa80tnPkfDoBGoNPMwqfaSWwg4FKmbEU= -k8s.io/apiextensions-apiserver v0.29.11/go.mod h1:eqKsza/nErdFNltXoVZmRt9vX99ooDLDMTcIcOG0ueg= -k8s.io/apimachinery v0.29.11 h1:55+6ue9advpA7T0sX2ZJDHCLKuiFfrAAR/39VQN9KEQ= -k8s.io/apimachinery v0.29.11/go.mod h1:i3FJVwhvSp/6n8Fl4K97PJEP8C+MM+aoDq4+ZJBf70Y= -k8s.io/client-go v0.29.11 h1:mBX7Ub0uqpLMwWz3J/AGS/xKOZsjr349qZ1vxVoL1l8= -k8s.io/client-go v0.29.11/go.mod h1:WOEoi/eLg2YEg3/yEd7YK3CNScYkM8AEScQadxUnaTE= -k8s.io/component-base v0.29.11 h1:H3GJIyDNPrscvXGP6wx+9gApcwwmrUd0YtCGp5BcHBA= -k8s.io/component-base v0.29.11/go.mod h1:0qu1WStER4wu5o8RMRndZUWPVcPH1XBy/QQiDcD6lew= -k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0= -k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo= -k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= -k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= +k8s.io/api v0.30.1 h1:kCm/6mADMdbAxmIh0LBjS54nQBE+U4KmbCfIkF5CpJY= +k8s.io/api v0.30.1/go.mod h1:ddbN2C0+0DIiPntan/bye3SW3PdwLa11/0yqwvuRrJM= +k8s.io/apiextensions-apiserver v0.30.1 h1:4fAJZ9985BmpJG6PkoxVRpXv9vmPUOVzl614xarePws= +k8s.io/apiextensions-apiserver v0.30.1/go.mod h1:R4GuSrlhgq43oRY9sF2IToFh7PVlF1JjfWdoG3pixk4= +k8s.io/apimachinery v0.30.1 h1:ZQStsEfo4n65yAdlGTfP/uSHMQSoYzU/oeEbkmF7P2U= +k8s.io/apimachinery v0.30.1/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= +k8s.io/client-go v0.30.1 h1:uC/Ir6A3R46wdkgCV3vbLyNOYyCJ8oZnjtJGKfytl/Q= +k8s.io/client-go v0.30.1/go.mod h1:wrAqLNs2trwiCH/wxxmT/x3hKVH9PuV0GGW0oDoHVqc= +k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= +k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 h1:jgGTlFYnhF1PM1Ax/lAlxUPE+KfCIXHaathvJg1C3ak= k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -sigs.k8s.io/controller-runtime v0.15.3 h1:L+t5heIaI3zeejoIyyvLQs5vTVu/67IU2FfisVzFlBc= -sigs.k8s.io/controller-runtime v0.15.3/go.mod h1:kp4jckA4vTx281S/0Yk2LFEEQe67mjg+ev/yknv47Ds= +sigs.k8s.io/controller-runtime v0.18.7 h1:WDnx8LTRY8Fn1j/7B+S/R9MeDjWNAzpDBoaSvMSrQME= +sigs.k8s.io/controller-runtime v0.18.7/go.mod h1:L9r3fUZhID7Q9eK9mseNskpaTg2n11f/tlb8odyzJ4Y= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= diff --git a/pkg/kube/client/mocked_manager.go b/pkg/kube/client/mocked_manager.go index 6adaf2609..2f9a3d30c 100644 --- a/pkg/kube/client/mocked_manager.go +++ b/pkg/kube/client/mocked_manager.go @@ -73,7 +73,7 @@ func (m *MockedManager) GetScheme() *runtime.Scheme { // GetAdmissionDecoder returns the runtime.Decoder based on the scheme. func (m *MockedManager) GetAdmissionDecoder() admission.Decoder { // just returning nothing - return *admission.NewDecoder(runtime.NewScheme()) + return admission.NewDecoder(runtime.NewScheme()) } // GetAPIReader returns the client reader @@ -114,7 +114,7 @@ func (m *MockedManager) GetWebhookServer() webhook.Server { return nil } -func (m *MockedManager) AddMetricsExtraHandler(path string, handler http.Handler) error { +func (m *MockedManager) AddMetricsServerExtraHandler(path string, handler http.Handler) error { return nil } diff --git a/test/e2e/client.go b/test/e2e/client.go index 291ba4473..478e3b81c 100644 --- a/test/e2e/client.go +++ b/test/e2e/client.go @@ -196,7 +196,6 @@ func RunTest(m *testing.M) (int, error) { testEnv = &envtest.Environment{ UseExistingCluster: &useExistingCluster, AttachControlPlaneOutput: true, - KubeAPIServerFlags: []string{"--authorization-mode=RBAC"}, } fmt.Println("Starting test environment") From cf9809160ee517c078516772a779ed468f688a58 Mon Sep 17 00:00:00 2001 From: mircea-cosbuc Date: Tue, 25 Feb 2025 09:02:38 +0100 Subject: [PATCH 773/790] Bump golang version to 1.24 and 1.30.x k8s deps (#1678) --- .github/workflows/go.yml | 2 +- ...ommunity.mongodb.com_mongodbcommunity.yaml | 28 ++++++++++++++++--- go.mod | 8 +++--- go.sum | 12 ++++---- release.json | 2 +- 5 files changed, 36 insertions(+), 16 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index be273651c..c02456101 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -14,7 +14,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v4 with: - go-version: '1.23' + go-version: '1.24' - name: Test api run: go test -v ./api/... diff --git a/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml b/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml index e7c3faa69..42c2258d0 100644 --- a/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml +++ b/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml @@ -333,10 +333,15 @@ spec: If additionally, tls.pem is present, then it needs to be equal to the concatenation of tls.crt and tls.key properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -463,10 +468,15 @@ spec: The certificate is expected to be available under the key "ca.crt" properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -477,10 +487,15 @@ spec: This field is ignored when CaCertificateSecretRef is configured properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -493,10 +508,15 @@ spec: If all of tls.pem, tls.crt and tls.key are present, the tls.pem one needs to be equal to the concatenation of tls.crt and tls.key properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic diff --git a/go.mod b/go.mod index ab7ea3254..77ad4173c 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/mongodb/mongodb-kubernetes-operator -go 1.23.0 +go 1.24.0 require ( github.com/blang/semver v3.5.1+incompatible @@ -14,9 +14,9 @@ require ( go.mongodb.org/mongo-driver v1.16.0 go.uber.org/zap v1.27.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 - k8s.io/api v0.30.1 - k8s.io/apimachinery v0.30.1 - k8s.io/client-go v0.30.1 + k8s.io/api v0.30.10 + k8s.io/apimachinery v0.30.10 + k8s.io/client-go v0.30.10 k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 sigs.k8s.io/controller-runtime v0.18.7 sigs.k8s.io/yaml v1.4.0 diff --git a/go.sum b/go.sum index 6b5e2d78c..c85107ab4 100644 --- a/go.sum +++ b/go.sum @@ -228,14 +228,14 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/api v0.30.1 h1:kCm/6mADMdbAxmIh0LBjS54nQBE+U4KmbCfIkF5CpJY= -k8s.io/api v0.30.1/go.mod h1:ddbN2C0+0DIiPntan/bye3SW3PdwLa11/0yqwvuRrJM= +k8s.io/api v0.30.10 h1:2YvzRF/BELgCvxbQqFKaan5hnj2+y7JOuqu2WpVk3gg= +k8s.io/api v0.30.10/go.mod h1:Hyz3ZuK7jVLJBUFvwzDSGwxHuDdsrGs5RzF16wfHIn4= k8s.io/apiextensions-apiserver v0.30.1 h1:4fAJZ9985BmpJG6PkoxVRpXv9vmPUOVzl614xarePws= k8s.io/apiextensions-apiserver v0.30.1/go.mod h1:R4GuSrlhgq43oRY9sF2IToFh7PVlF1JjfWdoG3pixk4= -k8s.io/apimachinery v0.30.1 h1:ZQStsEfo4n65yAdlGTfP/uSHMQSoYzU/oeEbkmF7P2U= -k8s.io/apimachinery v0.30.1/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= -k8s.io/client-go v0.30.1 h1:uC/Ir6A3R46wdkgCV3vbLyNOYyCJ8oZnjtJGKfytl/Q= -k8s.io/client-go v0.30.1/go.mod h1:wrAqLNs2trwiCH/wxxmT/x3hKVH9PuV0GGW0oDoHVqc= +k8s.io/apimachinery v0.30.10 h1:UflKuJeSSArttm05wjYP0GwpTlvjnMbDKFn6F7rKkKU= +k8s.io/apimachinery v0.30.10/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= +k8s.io/client-go v0.30.10 h1:C0oWM82QMvosIl/IdJhWfTUb7rIxM52rNSutFBknAVY= +k8s.io/client-go v0.30.10/go.mod h1:OfTvt0yuo8VpMViOsgvYQb+tMJQLNWVBqXWkzdFXSq4= k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= diff --git a/release.json b/release.json index 6e080a120..88983b5b3 100644 --- a/release.json +++ b/release.json @@ -1,5 +1,5 @@ { - "golang-builder-image": "golang:1.23", + "golang-builder-image": "golang:1.24", "operator": "0.12.0", "version-upgrade-hook": "1.0.9", "readiness-probe": "1.0.22", From 4c8d45bfa3ca519fabe8c6f79d3d14a5a01112c6 Mon Sep 17 00:00:00 2001 From: Mikalai Radchuk <509198+m1kola@users.noreply.github.com> Date: Fri, 7 Mar 2025 15:57:05 +0100 Subject: [PATCH 774/790] Bump GitHub Actions (#1688) --- .action_templates/steps/cancel-previous.yaml | 2 +- .action_templates/steps/checkout-fork.yaml | 2 +- .action_templates/steps/checkout.yaml | 2 +- .action_templates/steps/quay-login.yaml | 2 +- .action_templates/steps/set-run-status.yaml | 2 +- .action_templates/steps/set-up-qemu.yaml | 2 +- .../steps/setup-and-install-python.yaml | 4 ++-- .github/workflows/close-stale-issues.yml | 2 +- .github/workflows/code-health.yml | 4 ++-- .github/workflows/comment-release-pr.yml | 2 +- .github/workflows/e2e-dispatch.yml | 16 +++++++------- .github/workflows/e2e-fork.yml | 22 +++++++++---------- .github/workflows/e2e.yml | 22 +++++++++---------- .github/workflows/go.yml | 4 ++-- .github/workflows/kubelinter-check.yml | 2 +- .github/workflows/main.yaml | 2 +- .github/workflows/release-images.yml | 8 +++---- .github/workflows/release-single-image.yml | 6 ++--- 18 files changed, 53 insertions(+), 53 deletions(-) diff --git a/.action_templates/steps/cancel-previous.yaml b/.action_templates/steps/cancel-previous.yaml index 59ced338b..301d5af50 100644 --- a/.action_templates/steps/cancel-previous.yaml +++ b/.action_templates/steps/cancel-previous.yaml @@ -1,4 +1,4 @@ - name: Cancel Previous Runs - uses: styfle/cancel-workflow-action@89f242ee29e10c53a841bfe71cc0ce7b2f065abc # 0.9.0 + uses: styfle/cancel-workflow-action@0.12.1 with: access_token: ${{ github.token }} diff --git a/.action_templates/steps/checkout-fork.yaml b/.action_templates/steps/checkout-fork.yaml index 81b97e54d..abd35041c 100644 --- a/.action_templates/steps/checkout-fork.yaml +++ b/.action_templates/steps/checkout-fork.yaml @@ -2,7 +2,7 @@ # Because we are using pull_request_target the Github Secrets will be passed # So code should be reviewed before labeling as "safe-to-test" - name: Checkout Code - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: ref: ${{github.event.pull_request.head.sha}} repository: ${{github.event.pull_request.head.repo.full_name}} diff --git a/.action_templates/steps/checkout.yaml b/.action_templates/steps/checkout.yaml index 8d3e07859..da02fc2f3 100644 --- a/.action_templates/steps/checkout.yaml +++ b/.action_templates/steps/checkout.yaml @@ -1,4 +1,4 @@ - name: Checkout Code - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: submodules: true diff --git a/.action_templates/steps/quay-login.yaml b/.action_templates/steps/quay-login.yaml index 85073e70b..77a8dd06f 100644 --- a/.action_templates/steps/quay-login.yaml +++ b/.action_templates/steps/quay-login.yaml @@ -1,5 +1,5 @@ - name: Login to Quay.io - uses: docker/login-action@v1 + uses: docker/login-action@v3 with: registry: quay.io username: ${{ secrets.QUAY_USERNAME }} diff --git a/.action_templates/steps/set-run-status.yaml b/.action_templates/steps/set-run-status.yaml index 40bb34079..9f4a76541 100644 --- a/.action_templates/steps/set-run-status.yaml +++ b/.action_templates/steps/set-run-status.yaml @@ -7,7 +7,7 @@ # see https://github.com/actions/runner/issues/432 - name: Restore last run status id: last_run - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: last_run_status key: ${{ github.run_id }}-${{ matrix.test-name }}-${{ matrix.distro }} diff --git a/.action_templates/steps/set-up-qemu.yaml b/.action_templates/steps/set-up-qemu.yaml index 7ae0e920a..c84384bfc 100644 --- a/.action_templates/steps/set-up-qemu.yaml +++ b/.action_templates/steps/set-up-qemu.yaml @@ -1,2 +1,2 @@ - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 diff --git a/.action_templates/steps/setup-and-install-python.yaml b/.action_templates/steps/setup-and-install-python.yaml index e97bcf462..b924e01ae 100644 --- a/.action_templates/steps/setup-and-install-python.yaml +++ b/.action_templates/steps/setup-and-install-python.yaml @@ -1,9 +1,9 @@ - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: '3.10.4' - name: Cache Dependencies - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: ~/.cache/pip key: ${{ hashFiles('requirements.txt') }} diff --git a/.github/workflows/close-stale-issues.yml b/.github/workflows/close-stale-issues.yml index 8a806294a..942020dbd 100644 --- a/.github/workflows/close-stale-issues.yml +++ b/.github/workflows/close-stale-issues.yml @@ -10,7 +10,7 @@ jobs: stale: runs-on: ubuntu-latest steps: - - uses: actions/stale@v3 + - uses: actions/stale@v9 with: repo-token: ${{ secrets.GITHUB_TOKEN }} stale-issue-message: 'This issue is being marked stale because it has been open for 60 days with no activity. Please comment if this issue is still affecting you. If there is no change, this issue will be closed in 30 days.' diff --git a/.github/workflows/code-health.yml b/.github/workflows/code-health.yml index 2f217b968..345941c18 100644 --- a/.github/workflows/code-health.yml +++ b/.github/workflows/code-health.yml @@ -15,7 +15,7 @@ jobs: Mypy: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Mypy linting uses: jpetrucciani/mypy-check@179fdad632bf3ccf4cabb7ee4307ef25e51d2f96 @@ -30,4 +30,4 @@ jobs: with: go-version: stable - name: golangci-lint - uses: golangci/golangci-lint-action@v6 \ No newline at end of file + uses: golangci/golangci-lint-action@v6 diff --git a/.github/workflows/comment-release-pr.yml b/.github/workflows/comment-release-pr.yml index 0720856d2..3944aa660 100644 --- a/.github/workflows/comment-release-pr.yml +++ b/.github/workflows/comment-release-pr.yml @@ -9,7 +9,7 @@ jobs: if: startsWith(github.event.pull_request.title, 'Release MongoDB Kubernetes Operator') runs-on: ubuntu-latest steps: - - uses: actions/github-script@v3 + - uses: actions/github-script@v7 with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | diff --git a/.github/workflows/e2e-dispatch.yml b/.github/workflows/e2e-dispatch.yml index 7ecf8e5b6..b3522124d 100644 --- a/.github/workflows/e2e-dispatch.yml +++ b/.github/workflows/e2e-dispatch.yml @@ -48,16 +48,16 @@ jobs: steps: # template: .action_templates/steps/checkout.yaml - name: Checkout Code - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: submodules: true # template: .action_templates/steps/setup-and-install-python.yaml - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: 3.10.4 - name: Cache Dependencies - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: ~/.cache/pip key: ${{ hashFiles('requirements.txt') }} @@ -65,14 +65,14 @@ jobs: run: pip install -r requirements.txt # template: .action_templates/steps/quay-login.yaml - name: Login to Quay.io - uses: docker/login-action@v1 + uses: docker/login-action@v3 with: registry: quay.io username: ${{ secrets.QUAY_USERNAME }} password: ${{ secrets.QUAY_ROBOT_TOKEN }} # template: .action_templates/steps/set-up-qemu.yaml - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 # template: .action_templates/steps/build-and-push-development-images.yaml - name: Build and Push Images run: | @@ -87,16 +87,16 @@ jobs: steps: # template: .action_templates/steps/checkout.yaml - name: Checkout Code - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: submodules: true # template: .action_templates/steps/setup-and-install-python.yaml - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: 3.10.4 - name: Cache Dependencies - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: ~/.cache/pip key: ${{ hashFiles('requirements.txt') }} diff --git a/.github/workflows/e2e-fork.yml b/.github/workflows/e2e-fork.yml index eede9c4ed..a5c3ae53e 100644 --- a/.github/workflows/e2e-fork.yml +++ b/.github/workflows/e2e-fork.yml @@ -43,23 +43,23 @@ jobs: steps: # template: .action_templates/steps/cancel-previous.yaml - name: Cancel Previous Runs - uses: styfle/cancel-workflow-action@89f242ee29e10c53a841bfe71cc0ce7b2f065abc # 0.9.0 + uses: styfle/cancel-workflow-action@0.12.1 with: access_token: ${{ github.token }} # template: .action_templates/steps/checkout-fork.yaml - name: Checkout Code - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: ref: ${{github.event.pull_request.head.sha}} repository: ${{github.event.pull_request.head.repo.full_name}} submodules: true # template: .action_templates/steps/setup-and-install-python.yaml - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: 3.10.4 - name: Cache Dependencies - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: ~/.cache/pip key: ${{ hashFiles('requirements.txt') }} @@ -67,14 +67,14 @@ jobs: run: pip install -r requirements.txt # template: .action_templates/steps/quay-login.yaml - name: Login to Quay.io - uses: docker/login-action@v1 + uses: docker/login-action@v3 with: registry: quay.io username: ${{ secrets.QUAY_USERNAME }} password: ${{ secrets.QUAY_ROBOT_TOKEN }} # template: .action_templates/steps/set-up-qemu.yaml - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 # template: .action_templates/steps/build-and-push-development-images.yaml - name: Build and Push Images run: | @@ -154,12 +154,12 @@ jobs: steps: # template: .action_templates/steps/cancel-previous.yaml - name: Cancel Previous Runs - uses: styfle/cancel-workflow-action@89f242ee29e10c53a841bfe71cc0ce7b2f065abc # 0.9.0 + uses: styfle/cancel-workflow-action@0.12.1 with: access_token: ${{ github.token }} # template: .action_templates/steps/checkout-fork.yaml - name: Checkout Code - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: ref: ${{github.event.pull_request.head.sha}} repository: ${{github.event.pull_request.head.repo.full_name}} @@ -174,7 +174,7 @@ jobs: # see https://github.com/actions/runner/issues/432 - name: Restore last run status id: last_run - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: last_run_status key: ${{ github.run_id }}-${{ matrix.test-name }}-${{ matrix.distro }} @@ -184,11 +184,11 @@ jobs: run: cat last_run_status # template: .action_templates/steps/setup-and-install-python.yaml - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: 3.10.4 - name: Cache Dependencies - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: ~/.cache/pip key: ${{ hashFiles('requirements.txt') }} diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 6eb0a6d53..8501431b6 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -51,21 +51,21 @@ jobs: steps: # template: .action_templates/steps/cancel-previous.yaml - name: Cancel Previous Runs - uses: styfle/cancel-workflow-action@89f242ee29e10c53a841bfe71cc0ce7b2f065abc # 0.9.0 + uses: styfle/cancel-workflow-action@0.12.1 with: access_token: ${{ github.token }} # template: .action_templates/steps/checkout.yaml - name: Checkout Code - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: submodules: true # template: .action_templates/steps/setup-and-install-python.yaml - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: 3.10.4 - name: Cache Dependencies - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: ~/.cache/pip key: ${{ hashFiles('requirements.txt') }} @@ -73,14 +73,14 @@ jobs: run: pip install -r requirements.txt # template: .action_templates/steps/quay-login.yaml - name: Login to Quay.io - uses: docker/login-action@v1 + uses: docker/login-action@v3 with: registry: quay.io username: ${{ secrets.QUAY_USERNAME }} password: ${{ secrets.QUAY_ROBOT_TOKEN }} # template: .action_templates/steps/set-up-qemu.yaml - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 # template: .action_templates/steps/build-and-push-development-images.yaml - name: Build and Push Images run: | @@ -160,12 +160,12 @@ jobs: steps: # template: .action_templates/steps/cancel-previous.yaml - name: Cancel Previous Runs - uses: styfle/cancel-workflow-action@89f242ee29e10c53a841bfe71cc0ce7b2f065abc # 0.9.0 + uses: styfle/cancel-workflow-action@0.12.1 with: access_token: ${{ github.token }} # template: .action_templates/steps/checkout.yaml - name: Checkout Code - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: submodules: true # template: .action_templates/steps/set-run-status.yaml @@ -178,7 +178,7 @@ jobs: # see https://github.com/actions/runner/issues/432 - name: Restore last run status id: last_run - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: last_run_status key: ${{ github.run_id }}-${{ matrix.test-name }}-${{ matrix.distro }} @@ -188,11 +188,11 @@ jobs: run: cat last_run_status # template: .action_templates/steps/setup-and-install-python.yaml - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: 3.10.4 - name: Cache Dependencies - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: ~/.cache/pip key: ${{ hashFiles('requirements.txt') }} diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index c02456101..ecce33378 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -9,10 +9,10 @@ jobs: UnitTests: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Go - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: go-version: '1.24' diff --git a/.github/workflows/kubelinter-check.yml b/.github/workflows/kubelinter-check.yml index b4ef431fd..2fcb5b725 100644 --- a/.github/workflows/kubelinter-check.yml +++ b/.github/workflows/kubelinter-check.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout Code - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Scan directory ./deploy/clusterwide/ with kube-linter uses: stackrox/kube-linter-action@v1.0.3 diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 75e310255..3442f28df 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -36,7 +36,7 @@ jobs: # Checkout the code base # ########################## - name: Checkout Code - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: # Make sure we also get the helm-charts submodule! submodules: true diff --git a/.github/workflows/release-images.yml b/.github/workflows/release-images.yml index af38b3181..b0fb77dbc 100644 --- a/.github/workflows/release-images.yml +++ b/.github/workflows/release-images.yml @@ -23,14 +23,14 @@ jobs: steps: - name: Checkout Code - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: '3.10.4' architecture: 'x64' - - uses: actions/cache@v2 + - uses: actions/cache@v4 with: path: ~/.cache/pip key: ${{ hashFiles('requirements.txt') }} @@ -73,7 +73,7 @@ jobs: needs: [release-images] steps: - name: Checkout Code - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Determine Release Tag id: release_tag run: | diff --git a/.github/workflows/release-single-image.yml b/.github/workflows/release-single-image.yml index 11bd5683d..162454391 100644 --- a/.github/workflows/release-single-image.yml +++ b/.github/workflows/release-single-image.yml @@ -13,14 +13,14 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout Code - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: '3.10.4' architecture: 'x64' - - uses: actions/cache@v2 + - uses: actions/cache@v4 with: path: ~/.cache/pip key: ${{ hashFiles('requirements.txt') }} From fb36c3e79b5c795117a8092080efa35e7018fd2b Mon Sep 17 00:00:00 2001 From: Mikalai Radchuk <509198+m1kola@users.noreply.github.com> Date: Fri, 7 Mar 2025 16:38:11 +0100 Subject: [PATCH 775/790] Clean up unused args (#1685) --- controllers/mongodb_tls.go | 20 ++++++++++---------- controllers/prometheus.go | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/controllers/mongodb_tls.go b/controllers/mongodb_tls.go index fe872d371..56c67642d 100644 --- a/controllers/mongodb_tls.go +++ b/controllers/mongodb_tls.go @@ -68,7 +68,7 @@ func (r *ReplicaSetReconciler) validateTLSConfig(ctx context.Context, mdb mdbv1. // validate whether the secret contains "tls.crt" and "tls.key", or it contains "tls.pem" // if it contains all three, then the pem entry should be equal to the concatenation of crt and key - _, err = getPemOrConcatenatedCrtAndKey(ctx, r.client, mdb, mdb.TLSSecretNamespacedName()) + _, err = getPemOrConcatenatedCrtAndKey(ctx, r.client, mdb.TLSSecretNamespacedName()) if err != nil { r.log.Warnf(err.Error()) return false, nil @@ -100,7 +100,7 @@ func getTLSConfigModification(ctx context.Context, cmGetter configmap.Getter, se return automationconfig.NOOP(), err } - certKey, err := getPemOrConcatenatedCrtAndKey(ctx, secretGetter, mdb, mdb.TLSSecretNamespacedName()) + certKey, err := getPemOrConcatenatedCrtAndKey(ctx, secretGetter, mdb.TLSSecretNamespacedName()) if err != nil { return automationconfig.NOOP(), err } @@ -109,7 +109,7 @@ func getTLSConfigModification(ctx context.Context, cmGetter configmap.Getter, se } // getCertAndKey will fetch the certificate and key from the user-provided Secret. -func getCertAndKey(ctx context.Context, getter secret.Getter, mdb mdbv1.MongoDBCommunity, secretName types.NamespacedName) string { +func getCertAndKey(ctx context.Context, getter secret.Getter, secretName types.NamespacedName) string { cert, err := secret.ReadKey(ctx, getter, tlsSecretCertName, secretName) if err != nil { return "" @@ -124,7 +124,7 @@ func getCertAndKey(ctx context.Context, getter secret.Getter, mdb mdbv1.MongoDBC } // getPem will fetch the pem from the user-provided secret -func getPem(ctx context.Context, getter secret.Getter, mdb mdbv1.MongoDBCommunity, secretName types.NamespacedName) string { +func getPem(ctx context.Context, getter secret.Getter, secretName types.NamespacedName) string { pem, err := secret.ReadKey(ctx, getter, tlsSecretPemName, secretName) if err != nil { return "" @@ -142,9 +142,9 @@ func combineCertificateAndKey(cert, key string) string { // This is either the tls.pem entry in the given secret, or the concatenation // of tls.crt and tls.key // It performs a basic validation on the entries. -func getPemOrConcatenatedCrtAndKey(ctx context.Context, getter secret.Getter, mdb mdbv1.MongoDBCommunity, secretName types.NamespacedName) (string, error) { - certKey := getCertAndKey(ctx, getter, mdb, secretName) - pem := getPem(ctx, getter, mdb, secretName) +func getPemOrConcatenatedCrtAndKey(ctx context.Context, getter secret.Getter, secretName types.NamespacedName) (string, error) { + certKey := getCertAndKey(ctx, getter, secretName) + pem := getPem(ctx, getter, secretName) if certKey == "" && pem == "" { return "", fmt.Errorf(`neither "%s" nor the pair "%s"/"%s" were present in the TLS secret`, tlsSecretPemName, tlsSecretCertName, tlsSecretKeyName) } @@ -210,7 +210,7 @@ func ensureCASecret(ctx context.Context, cmGetter configmap.Getter, secretGetter // ensureTLSSecret will create or update the operator-managed Secret containing // the concatenated certificate and key from the user-provided Secret. func ensureTLSSecret(ctx context.Context, getUpdateCreator secret.GetUpdateCreator, mdb mdbv1.MongoDBCommunity) error { - certKey, err := getPemOrConcatenatedCrtAndKey(ctx, getUpdateCreator, mdb, mdb.TLSSecretNamespacedName()) + certKey, err := getPemOrConcatenatedCrtAndKey(ctx, getUpdateCreator, mdb.TLSSecretNamespacedName()) if err != nil { return err } @@ -232,7 +232,7 @@ func ensureAgentCertSecret(ctx context.Context, getUpdateCreator secret.GetUpdat return nil } - certKey, err := getPemOrConcatenatedCrtAndKey(ctx, getUpdateCreator, mdb, mdb.AgentCertificateSecretNamespacedName()) + certKey, err := getPemOrConcatenatedCrtAndKey(ctx, getUpdateCreator, mdb.AgentCertificateSecretNamespacedName()) if err != nil { return err } @@ -250,7 +250,7 @@ func ensureAgentCertSecret(ctx context.Context, getUpdateCreator secret.GetUpdat // ensurePrometheusTLSSecret will create or update the operator-managed Secret containing // the concatenated certificate and key from the user-provided Secret. func ensurePrometheusTLSSecret(ctx context.Context, getUpdateCreator secret.GetUpdateCreator, mdb mdbv1.MongoDBCommunity) error { - certKey, err := getPemOrConcatenatedCrtAndKey(ctx, getUpdateCreator, mdb, mdb.DeepCopy().PrometheusTLSSecretNamespacedName()) + certKey, err := getPemOrConcatenatedCrtAndKey(ctx, getUpdateCreator, mdb.DeepCopy().PrometheusTLSSecretNamespacedName()) if err != nil { return err } diff --git a/controllers/prometheus.go b/controllers/prometheus.go index 165fe72a0..cebe939fe 100644 --- a/controllers/prometheus.go +++ b/controllers/prometheus.go @@ -36,7 +36,7 @@ func getPrometheusModification(ctx context.Context, getUpdateCreator secret.GetU var scheme string if mdb.Spec.Prometheus.TLSSecretRef.Name != "" { - certKey, err = getPemOrConcatenatedCrtAndKey(ctx, getUpdateCreator, mdb, mdb.PrometheusTLSSecretNamespacedName()) + certKey, err = getPemOrConcatenatedCrtAndKey(ctx, getUpdateCreator, mdb.PrometheusTLSSecretNamespacedName()) if err != nil { return automationconfig.NOOP(), err } From 7a5cd0860f5cc1e69f584e0c1b18e2bc52fe0e65 Mon Sep 17 00:00:00 2001 From: Mikalai Radchuk <509198+m1kola@users.noreply.github.com> Date: Fri, 7 Mar 2025 17:20:07 +0100 Subject: [PATCH 776/790] Update `validateSpec` to return error last (#1684) --- controllers/replica_set_controller.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/controllers/replica_set_controller.go b/controllers/replica_set_controller.go index e956790ba..325da36aa 100644 --- a/controllers/replica_set_controller.go +++ b/controllers/replica_set_controller.go @@ -128,7 +128,7 @@ func (r ReplicaSetReconciler) Reconcile(ctx context.Context, request reconcile.R r.log.Infof("Reconciling MongoDB") r.log.Debug("Validating MongoDB.Spec") - err, lastAppliedSpec := r.validateSpec(mdb) + lastAppliedSpec, err := r.validateSpec(mdb) if err != nil { return status.Update(ctx, r.client.Status(), &mdb, statusOptions(). withMessage(Error, fmt.Sprintf("error validating new Spec: %s", err)). @@ -596,20 +596,20 @@ func (r *ReplicaSetReconciler) buildService(mdb mdbv1.MongoDBCommunity, portMana // If there has not yet been a successful configuration, the function runs the initial Spec validations. Otherwise, // it checks that the attempted Spec is valid in relation to the Spec that resulted from that last successful configuration. // The validation also returns the lastSuccessFulConfiguration Spec as mdbv1.MongoDBCommunitySpec. -func (r ReplicaSetReconciler) validateSpec(mdb mdbv1.MongoDBCommunity) (error, *mdbv1.MongoDBCommunitySpec) { +func (r ReplicaSetReconciler) validateSpec(mdb mdbv1.MongoDBCommunity) (*mdbv1.MongoDBCommunitySpec, error) { lastSuccessfulConfigurationSaved, ok := mdb.Annotations[lastSuccessfulConfiguration] if !ok { // First version of Spec - return validation.ValidateInitialSpec(mdb, r.log), nil + return nil, validation.ValidateInitialSpec(mdb, r.log) } lastSpec := mdbv1.MongoDBCommunitySpec{} err := json.Unmarshal([]byte(lastSuccessfulConfigurationSaved), &lastSpec) if err != nil { - return err, &lastSpec + return &lastSpec, err } - return validation.ValidateUpdate(mdb, lastSpec, r.log), &lastSpec + return &lastSpec, validation.ValidateUpdate(mdb, lastSpec, r.log) } func getCustomRolesModification(mdb mdbv1.MongoDBCommunity) (automationconfig.Modification, error) { From 6e53d3ce9fc98fe0d7212c9f2f4dec7d3131e81c Mon Sep 17 00:00:00 2001 From: Mikalai Radchuk <509198+m1kola@users.noreply.github.com> Date: Mon, 10 Mar 2025 17:44:05 +0100 Subject: [PATCH 777/790] Refactoring: propagate env vars via parameters (#1676) --- cmd/manager/main.go | 21 +- .../construct/build_statefulset_test.go | 108 +--------- controllers/construct/mongodbstatefulset.go | 58 ++---- controllers/mongodb_cleanup_test.go | 5 +- controllers/mongodb_tls_test.go | 22 +- controllers/replica_set_controller.go | 55 +++-- controllers/replicaset_controller_test.go | 191 +++++++++++------- .../replica_set_enterprise_upgrade.go | 4 +- test/e2e/setup/test_config.go | 2 +- 9 files changed, 222 insertions(+), 244 deletions(-) diff --git a/cmd/manager/main.go b/cmd/manager/main.go index a874faff6..b8dd5d184 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -3,11 +3,13 @@ package main import ( "fmt" "os" + "sigs.k8s.io/controller-runtime/pkg/cache" mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" "github.com/mongodb/mongodb-kubernetes-operator/controllers" "github.com/mongodb/mongodb-kubernetes-operator/controllers/construct" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/envvar" "go.uber.org/zap" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" @@ -56,7 +58,14 @@ func main() { log.Sugar().Fatalf("Failed to configure logger: %v", err) } - if !hasRequiredVariables(log, construct.AgentImageEnv, construct.VersionUpgradeHookImageEnv, construct.ReadinessProbeImageEnv) { + if !hasRequiredVariables( + log, + construct.MongodbRepoUrlEnv, + construct.MongodbImageEnv, + construct.AgentImageEnv, + construct.VersionUpgradeHookImageEnv, + construct.ReadinessProbeImageEnv, + ) { os.Exit(1) } @@ -99,7 +108,15 @@ func main() { } // Setup Controller. - if err = controllers.NewReconciler(mgr).SetupWithManager(mgr); err != nil { + if err = controllers.NewReconciler( + mgr, + os.Getenv(construct.MongodbRepoUrlEnv), + os.Getenv(construct.MongodbImageEnv), + envvar.GetEnvOrDefault(construct.MongoDBImageTypeEnv, construct.DefaultImageType), + os.Getenv(construct.AgentImageEnv), + os.Getenv(construct.VersionUpgradeHookImageEnv), + os.Getenv(construct.ReadinessProbeImageEnv), + ).SetupWithManager(mgr); err != nil { log.Sugar().Fatalf("Unable to create controller: %v", err) } // +kubebuilder:scaffold:builder diff --git a/controllers/construct/build_statefulset_test.go b/controllers/construct/build_statefulset_test.go index 56e3bcaac..e02095c63 100644 --- a/controllers/construct/build_statefulset_test.go +++ b/controllers/construct/build_statefulset_test.go @@ -1,7 +1,6 @@ package construct import ( - "os" "reflect" "testing" @@ -21,10 +20,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func init() { - os.Setenv(VersionUpgradeHookImageEnv, "version-upgrade-hook-image") -} - func newTestReplicaSet() mdbv1.MongoDBCommunity { return mdbv1.MongoDBCommunity{ ObjectMeta: metav1.ObjectMeta{ @@ -40,12 +35,8 @@ func newTestReplicaSet() mdbv1.MongoDBCommunity { } func TestMultipleCalls_DoNotCauseSideEffects(t *testing.T) { - t.Setenv(MongodbRepoUrl, "docker.io/mongodb") - t.Setenv(MongodbImageEnv, "mongodb-community-server") - t.Setenv(AgentImageEnv, "agent-image") - mdb := newTestReplicaSet() - stsFunc := BuildMongoDBReplicaSetStatefulSetModificationFunction(&mdb, &mdb, os.Getenv(AgentImageEnv), true) + stsFunc := BuildMongoDBReplicaSetStatefulSetModificationFunction(&mdb, &mdb, "fake-mongodbImage", "fake-agentImage", "fake-versionUpgradeHookImage", "fake-readinessProbeImage", true) sts := &appsv1.StatefulSet{} t.Run("1st Call", func(t *testing.T) { @@ -63,13 +54,10 @@ func TestMultipleCalls_DoNotCauseSideEffects(t *testing.T) { } func TestManagedSecurityContext(t *testing.T) { - t.Setenv(MongodbRepoUrl, "docker.io/mongodb") - t.Setenv(MongodbImageEnv, "mongodb-community-server") - t.Setenv(AgentImageEnv, "agent-image") t.Setenv(podtemplatespec.ManagedSecurityContextEnv, "true") mdb := newTestReplicaSet() - stsFunc := BuildMongoDBReplicaSetStatefulSetModificationFunction(&mdb, &mdb, os.Getenv(AgentImageEnv), true) + stsFunc := BuildMongoDBReplicaSetStatefulSetModificationFunction(&mdb, &mdb, "fake-mongodbImage", "fake-agentImage", "fake-versionUpgradeHookImage", "fake-readinessProbeImage", true) sts := &appsv1.StatefulSet{} stsFunc(sts) @@ -77,89 +65,9 @@ func TestManagedSecurityContext(t *testing.T) { assertStatefulSetIsBuiltCorrectly(t, mdb, sts) } -func TestGetMongoDBImage(t *testing.T) { - type testConfig struct { - setArgs func(t *testing.T) - version string - expectedImage string - } - tests := map[string]testConfig{ - "Default UBI8 Community image": { - setArgs: func(t *testing.T) { - t.Setenv(MongodbRepoUrl, "docker.io/mongodb") - t.Setenv(MongodbImageEnv, "mongodb-community-server") - }, - version: "6.0.5", - expectedImage: "docker.io/mongodb/mongodb-community-server:6.0.5-ubi8", - }, - "Overridden UBI8 Enterprise image": { - setArgs: func(t *testing.T) { - t.Setenv(MongodbRepoUrl, "docker.io/mongodb") - t.Setenv(MongodbImageEnv, "mongodb-enterprise-server") - }, - version: "6.0.5", - expectedImage: "docker.io/mongodb/mongodb-enterprise-server:6.0.5-ubi8", - }, - "Overridden UBI8 Enterprise image from Quay": { - setArgs: func(t *testing.T) { - t.Setenv(MongodbRepoUrl, "quay.io/mongodb") - t.Setenv(MongodbImageEnv, "mongodb-enterprise-server") - }, - version: "6.0.5", - expectedImage: "quay.io/mongodb/mongodb-enterprise-server:6.0.5-ubi8", - }, - "Overridden Ubuntu Community image": { - setArgs: func(t *testing.T) { - t.Setenv(MongodbRepoUrl, "docker.io/mongodb") - t.Setenv(MongodbImageEnv, "mongodb-community-server") - t.Setenv(MongoDBImageType, "ubuntu2204") - }, - version: "6.0.5", - expectedImage: "docker.io/mongodb/mongodb-community-server:6.0.5-ubuntu2204", - }, - "Overridden UBI Community image": { - setArgs: func(t *testing.T) { - t.Setenv(MongodbRepoUrl, "docker.io/mongodb") - t.Setenv(MongodbImageEnv, "mongodb-community-server") - t.Setenv(MongoDBImageType, "ubi8") - }, - version: "6.0.5", - expectedImage: "docker.io/mongodb/mongodb-community-server:6.0.5-ubi8", - }, - "Docker Inc images": { - setArgs: func(t *testing.T) { - t.Setenv(MongodbRepoUrl, "docker.io") - t.Setenv(MongodbImageEnv, "mongo") - }, - version: "6.0.5", - expectedImage: "docker.io/mongo:6.0.5", - }, - "Deprecated AppDB images defined the old way": { - setArgs: func(t *testing.T) { - t.Setenv(MongodbRepoUrl, "quay.io") - t.Setenv(MongodbImageEnv, "mongodb/mongodb-enterprise-appdb-database-ubi") - // In this example, we intentionally don't use the suffix from the env. variable and let users - // define it in the version instead. There are some known customers who do this. - // This is a backwards compatibility case. - t.Setenv(MongoDBImageType, "will-be-ignored") - }, - - version: "5.0.14-ent", - expectedImage: "quay.io/mongodb/mongodb-enterprise-appdb-database-ubi:5.0.14-ent", - }, - } - for testName := range tests { - t.Run(testName, func(t *testing.T) { - testConfig := tests[testName] - testConfig.setArgs(t) - image := getMongoDBImage(testConfig.version) - assert.Equal(t, testConfig.expectedImage, image) - }) - } -} - func TestMongod_Container(t *testing.T) { - c := container.New(mongodbContainer("4.2", []corev1.VolumeMount{}, mdbv1.NewMongodConfiguration())) + const mongodbImageMock = "fake-mongodbImage" + c := container.New(mongodbContainer(mongodbImageMock, []corev1.VolumeMount{}, mdbv1.NewMongodConfiguration())) t.Run("Has correct Env vars", func(t *testing.T) { assert.Len(t, c.Env, 1) @@ -168,7 +76,7 @@ func TestMongod_Container(t *testing.T) { }) t.Run("Image is correct", func(t *testing.T) { - assert.Equal(t, getMongoDBImage("4.2"), c.Image) + assert.Equal(t, mongodbImageMock, c.Image) }) t.Run("Resource requirements are correct", func(t *testing.T) { @@ -210,7 +118,7 @@ func assertStatefulSetIsBuiltCorrectly(t *testing.T, mdb mdbv1.MongoDBCommunity, } agentContainer := sts.Spec.Template.Spec.Containers[0] - assert.Equal(t, "agent-image", agentContainer.Image) + assert.Equal(t, "fake-agentImage", agentContainer.Image) probe := agentContainer.ReadinessProbe assert.True(t, reflect.DeepEqual(probes.New(DefaultReadiness()), *probe)) assert.Equal(t, probes.New(DefaultReadiness()).FailureThreshold, probe.FailureThreshold) @@ -231,7 +139,7 @@ func assertStatefulSetIsBuiltCorrectly(t *testing.T, mdb mdbv1.MongoDBCommunity, assertContainsVolumeMountWithName(t, agentContainer.VolumeMounts, "my-rs-keyfile") mongodContainer := sts.Spec.Template.Spec.Containers[1] - assert.Equal(t, "docker.io/mongodb/mongodb-community-server:6.0.5-ubi8", mongodContainer.Image) + assert.Equal(t, "fake-mongodbImage", mongodContainer.Image) assert.Len(t, mongodContainer.VolumeMounts, 6) if !managedSecurityContext { assert.NotNil(t, sts.Spec.Template.Spec.Containers[1].SecurityContext) @@ -248,7 +156,7 @@ func assertStatefulSetIsBuiltCorrectly(t *testing.T, mdb mdbv1.MongoDBCommunity, initContainer := sts.Spec.Template.Spec.InitContainers[0] assert.Equal(t, versionUpgradeHookName, initContainer.Name) - assert.Equal(t, "version-upgrade-hook-image", initContainer.Image) + assert.Equal(t, "fake-versionUpgradeHookImage", initContainer.Image) assert.Len(t, initContainer.VolumeMounts, 1) if !managedSecurityContext { assert.NotNil(t, sts.Spec.Template.Spec.InitContainers[0].SecurityContext) diff --git a/controllers/construct/mongodbstatefulset.go b/controllers/construct/mongodbstatefulset.go index a6a71ff9b..cec2e5be7 100644 --- a/controllers/construct/mongodbstatefulset.go +++ b/controllers/construct/mongodbstatefulset.go @@ -2,12 +2,10 @@ package construct import ( "fmt" - "github.com/mongodb/mongodb-kubernetes-operator/pkg/readiness/config" "os" "strconv" - "strings" - "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/envvar" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/readiness/config" "github.com/mongodb/mongodb-kubernetes-operator/pkg/automationconfig" "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/container" @@ -28,6 +26,16 @@ var ( OfficialMongodbRepoUrls = []string{"docker.io/mongodb", "quay.io/mongodb"} ) +// Environment variables used to configure the MongoDB StatefulSet. +const ( + MongodbRepoUrlEnv = "MONGODB_REPO_URL" + MongodbImageEnv = "MONGODB_IMAGE" + MongoDBImageTypeEnv = "MDB_IMAGE_TYPE" + AgentImageEnv = "AGENT_IMAGE" + VersionUpgradeHookImageEnv = "VERSION_UPGRADE_HOOK_IMAGE" + ReadinessProbeImageEnv = "READINESS_PROBE_IMAGE" +) + const ( AgentName = "mongodb-agent" MongodbName = "mongod" @@ -42,18 +50,12 @@ const ( mongodbDatabaseServiceAccountName = "mongodb-database" agentHealthStatusFilePathValue = "/var/log/mongodb-mms-automation/healthstatus/agent-health-status.json" - MongodbRepoUrl = "MONGODB_REPO_URL" OfficialMongodbEnterpriseServerImageName = "mongodb-enterprise-server" headlessAgentEnv = "HEADLESS_AGENT" podNamespaceEnv = "POD_NAMESPACE" automationConfigEnv = "AUTOMATION_CONFIG_MAP" - AgentImageEnv = "AGENT_IMAGE" - MongodbImageEnv = "MONGODB_IMAGE" - MongoDBImageType = "MDB_IMAGE_TYPE" MongoDBAssumeEnterpriseEnv = "MDB_ASSUME_ENTERPRISE" - VersionUpgradeHookImageEnv = "VERSION_UPGRADE_HOOK_IMAGE" - ReadinessProbeImageEnv = "READINESS_PROBE_IMAGE" automationMongodConfFileName = "automation-mongod.conf" keyfileFilePath = "/var/lib/mongodb-mms-automation/authentication/keyfile" @@ -123,7 +125,7 @@ type MongoDBStatefulSetOwner interface { // BuildMongoDBReplicaSetStatefulSetModificationFunction builds the parts of the replica set that are common between every resource that implements // MongoDBStatefulSetOwner. // It doesn't configure TLS or additional containers/env vars that the statefulset might need. -func BuildMongoDBReplicaSetStatefulSetModificationFunction(mdb MongoDBStatefulSetOwner, scaler scale.ReplicaSetScaler, agentImage string, withInitContainers bool) statefulset.Modification { +func BuildMongoDBReplicaSetStatefulSetModificationFunction(mdb MongoDBStatefulSetOwner, scaler scale.ReplicaSetScaler, mongodbImage, agentImage, versionUpgradeHookImage, readinessProbeImage string, withInitContainers bool) statefulset.Modification { labels := map[string]string{ "app": mdb.ServiceName(), } @@ -174,8 +176,8 @@ func BuildMongoDBReplicaSetStatefulSetModificationFunction(mdb MongoDBStatefulSe scriptsVolume = statefulset.CreateVolumeFromEmptyDir("agent-scripts") scriptsVolumeMount := statefulset.CreateVolumeMount(scriptsVolume.Name, "/opt/scripts", statefulset.WithReadOnly(false)) - upgradeInitContainer = podtemplatespec.WithInitContainer(versionUpgradeHookName, versionUpgradeHookInit([]corev1.VolumeMount{hooksVolumeMount})) - readinessInitContainer = podtemplatespec.WithInitContainer(ReadinessProbeContainerName, readinessProbeInit([]corev1.VolumeMount{scriptsVolumeMount})) + upgradeInitContainer = podtemplatespec.WithInitContainer(versionUpgradeHookName, versionUpgradeHookInit([]corev1.VolumeMount{hooksVolumeMount}, versionUpgradeHookImage)) + readinessInitContainer = podtemplatespec.WithInitContainer(ReadinessProbeContainerName, readinessProbeInit([]corev1.VolumeMount{scriptsVolumeMount}, readinessProbeImage)) scriptsVolumeMod = podtemplatespec.WithVolume(scriptsVolume) hooksVolumeMod = podtemplatespec.WithVolume(hooksVolume) @@ -243,7 +245,7 @@ func BuildMongoDBReplicaSetStatefulSetModificationFunction(mdb MongoDBStatefulSe podtemplatespec.WithVolume(keyFileVolume), podtemplatespec.WithServiceAccount(mongodbDatabaseServiceAccountName), podtemplatespec.WithContainer(AgentName, mongodbAgentContainer(mdb.AutomationConfigSecretName(), mongodbAgentVolumeMounts, agentLogLevel, agentLogFile, agentMaxLogFileDurationHours, agentImage)), - podtemplatespec.WithContainer(MongodbName, mongodbContainer(mdb.GetMongoDBVersion(nil), mongodVolumeMounts, mdb.GetMongodConfiguration())), + podtemplatespec.WithContainer(MongodbName, mongodbContainer(mongodbImage, mongodVolumeMounts, mdb.GetMongodConfiguration())), upgradeInitContainer, readinessInitContainer, ), @@ -312,12 +314,12 @@ func mongodbAgentContainer(automationConfigSecretName string, volumeMounts []cor ) } -func versionUpgradeHookInit(volumeMount []corev1.VolumeMount) container.Modification { +func versionUpgradeHookInit(volumeMount []corev1.VolumeMount, versionUpgradeHookImage string) container.Modification { _, containerSecurityContext := podtemplatespec.WithDefaultSecurityContextsModifications() return container.Apply( container.WithName(versionUpgradeHookName), container.WithCommand([]string{"cp", "version-upgrade-hook", "/hooks/version-upgrade"}), - container.WithImage(os.Getenv(VersionUpgradeHookImageEnv)), + container.WithImage(versionUpgradeHookImage), container.WithResourceRequirements(resourcerequirements.Defaults()), container.WithImagePullPolicy(corev1.PullAlways), container.WithVolumeMounts(volumeMount), @@ -351,12 +353,12 @@ func logsPvc(logsVolumeName string) persistentvolumeclaim.Modification { // readinessProbeInit returns a modification function which will add the readiness probe container. // this container will copy the readiness probe binary into the /opt/scripts directory. -func readinessProbeInit(volumeMount []corev1.VolumeMount) container.Modification { +func readinessProbeInit(volumeMount []corev1.VolumeMount, readinessProbeImage string) container.Modification { _, containerSecurityContext := podtemplatespec.WithDefaultSecurityContextsModifications() return container.Apply( container.WithName(ReadinessProbeContainerName), container.WithCommand([]string{"cp", "/probes/readinessprobe", "/opt/scripts/readinessprobe"}), - container.WithImage(os.Getenv(ReadinessProbeImageEnv)), + container.WithImage(readinessProbeImage), container.WithImagePullPolicy(corev1.PullAlways), container.WithVolumeMounts(volumeMount), container.WithResourceRequirements(resourcerequirements.Defaults()), @@ -364,25 +366,7 @@ func readinessProbeInit(volumeMount []corev1.VolumeMount) container.Modification ) } -func getMongoDBImage(version string) string { - repoUrl := os.Getenv(MongodbRepoUrl) - imageType := envvar.GetEnvOrDefault(MongoDBImageType, DefaultImageType) - - if strings.HasSuffix(repoUrl, "/") { - repoUrl = strings.TrimRight(repoUrl, "/") - } - mongoImageName := os.Getenv(MongodbImageEnv) - for _, officialUrl := range OfficialMongodbRepoUrls { - if repoUrl == officialUrl { - return fmt.Sprintf("%s/%s:%s-%s", repoUrl, mongoImageName, version, imageType) - } - } - - // This is the old images backwards compatibility code path. - return fmt.Sprintf("%s/%s:%s", repoUrl, mongoImageName, version) -} - -func mongodbContainer(version string, volumeMounts []corev1.VolumeMount, additionalMongoDBConfig mdbv1.MongodConfiguration) container.Modification { +func mongodbContainer(mongodbImage string, volumeMounts []corev1.VolumeMount, additionalMongoDBConfig mdbv1.MongodConfiguration) container.Modification { filePath := additionalMongoDBConfig.GetDBDataDir() + "/" + automationMongodConfFileName mongoDbCommand := fmt.Sprintf(` if [ -e "/hooks/version-upgrade" ]; then @@ -408,7 +392,7 @@ exec mongod -f %s; return container.Apply( container.WithName(MongodbName), - container.WithImage(getMongoDBImage(version)), + container.WithImage(mongodbImage), container.WithResourceRequirements(resourcerequirements.Defaults()), container.WithCommand(containerCommand), // The official image provides both CMD and ENTRYPOINT. We're reusing the former and need to replace diff --git a/controllers/mongodb_cleanup_test.go b/controllers/mongodb_cleanup_test.go index 249556061..0123f63ee 100644 --- a/controllers/mongodb_cleanup_test.go +++ b/controllers/mongodb_cleanup_test.go @@ -2,12 +2,13 @@ package controllers import ( "context" + "testing" + mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" kubeClient "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/client" "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "testing" ) func TestReplicaSetReconcilerCleanupScramSecrets(t *testing.T) { @@ -140,7 +141,7 @@ func TestReplicaSetReconcilerCleanupPemSecret(t *testing.T) { err := createAgentCertPemSecret(ctx, client, mdb, "CERT", "KEY", "") assert.NoError(t, err) - r := NewReconciler(mgr) + r := NewReconciler(mgr, "fake-mongodbRepoUrl", "fake-mongodbImage", "ubi8", "fake-agentImage", "fake-versionUpgradeHookImage", "fake-readinessProbeImage") secret, err := r.client.GetSecret(ctx, mdb.AgentCertificatePemSecretNamespacedName()) assert.NoError(t, err) diff --git a/controllers/mongodb_tls_test.go b/controllers/mongodb_tls_test.go index f6c21b6b9..b4e832778 100644 --- a/controllers/mongodb_tls_test.go +++ b/controllers/mongodb_tls_test.go @@ -34,7 +34,7 @@ func TestStatefulSetIsCorrectlyConfiguredWithTLS(t *testing.T) { err = createTLSConfigMap(ctx, client, mdb) assert.NoError(t, err) - r := NewReconciler(mgr) + r := NewReconciler(mgr, "fake-mongodbRepoUrl", "fake-mongodbImage", "ubi8", "fake-agentImage", "fake-versionUpgradeHookImage", "fake-readinessProbeImage") res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) @@ -61,7 +61,7 @@ func TestStatefulSetIsCorrectlyConfiguredWithTLSAndX509(t *testing.T) { err = createAgentCertSecret(ctx, client, mdb, crt, key, "") assert.NoError(t, err) - r := NewReconciler(mgr) + r := NewReconciler(mgr, "fake-mongodbRepoUrl", "fake-mongodbImage", "ubi8", "fake-agentImage", "fake-versionUpgradeHookImage", "fake-readinessProbeImage") res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) @@ -230,7 +230,7 @@ func TestStatefulSetIsCorrectlyConfiguredWithPrometheusTLS(t *testing.T) { err = createTLSConfigMap(ctx, cli, mdb) assert.NoError(t, err) - r := NewReconciler(mgr) + r := NewReconciler(mgr, "fake-mongodbRepoUrl", "fake-mongodbImage", "ubi8", "fake-agentImage", "fake-versionUpgradeHookImage", "fake-readinessProbeImage") res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) @@ -259,7 +259,7 @@ func TestStatefulSetIsCorrectlyConfiguredWithTLSAfterChangingExistingVolumes(t * err = createTLSConfigMap(ctx, cli, mdb) assert.NoError(t, err) - r := NewReconciler(mgr) + r := NewReconciler(mgr, "fake-mongodbRepoUrl", "fake-mongodbImage", "ubi8", "fake-agentImage", "fake-versionUpgradeHookImage", "fake-readinessProbeImage") res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) @@ -301,7 +301,7 @@ func TestAutomationConfigIsCorrectlyConfiguredWithTLS(t *testing.T) { tlsModification, err := getTLSConfigModification(ctx, client, client, mdb) assert.NoError(t, err) - ac, err := buildAutomationConfig(mdb, automationconfig.Auth{}, automationconfig.AutomationConfig{}, tlsModification) + ac, err := buildAutomationConfig(mdb, false, automationconfig.Auth{}, automationconfig.AutomationConfig{}, tlsModification) assert.NoError(t, err) return ac @@ -383,7 +383,7 @@ func TestTLSOperatorSecret(t *testing.T) { err = createTLSConfigMap(ctx, c, mdb) assert.NoError(t, err) - r := NewReconciler(kubeClient.NewManagerWithClient(c)) + r := NewReconciler(kubeClient.NewManagerWithClient(c), "fake-mongodbRepoUrl", "fake-mongodbImage", "ubi8", "fake-agentImage", "fake-versionUpgradeHookImage", "fake-readinessProbeImage") err = r.ensureTLSResources(ctx, mdb) assert.NoError(t, err) @@ -413,7 +413,7 @@ func TestTLSOperatorSecret(t *testing.T) { err = k8sclient.CreateSecret(ctx, s) assert.NoError(t, err) - r := NewReconciler(kubeClient.NewManagerWithClient(k8sclient)) + r := NewReconciler(kubeClient.NewManagerWithClient(k8sclient), "fake-mongodbRepoUrl", "fake-mongodbImage", "ubi8", "fake-agentImage", "fake-versionUpgradeHookImage", "fake-readinessProbeImage") err = r.ensureTLSResources(ctx, mdb) assert.NoError(t, err) @@ -456,7 +456,7 @@ func TestPemSupport(t *testing.T) { err = createTLSConfigMap(ctx, c, mdb) assert.NoError(t, err) - r := NewReconciler(kubeClient.NewManagerWithClient(c)) + r := NewReconciler(kubeClient.NewManagerWithClient(c), "fake-mongodbRepoUrl", "fake-mongodbImage", "ubi8", "fake-agentImage", "fake-versionUpgradeHookImage", "fake-readinessProbeImage") err = r.ensureTLSResources(ctx, mdb) assert.NoError(t, err) @@ -476,7 +476,7 @@ func TestPemSupport(t *testing.T) { err = createTLSConfigMap(ctx, c, mdb) assert.NoError(t, err) - r := NewReconciler(kubeClient.NewManagerWithClient(c)) + r := NewReconciler(kubeClient.NewManagerWithClient(c), "fake-mongodbRepoUrl", "fake-mongodbImage", "ubi8", "fake-agentImage", "fake-versionUpgradeHookImage", "fake-readinessProbeImage") err = r.ensureTLSResources(ctx, mdb) assert.NoError(t, err) @@ -496,7 +496,7 @@ func TestPemSupport(t *testing.T) { err = createTLSConfigMap(ctx, c, mdb) assert.NoError(t, err) - r := NewReconciler(kubeClient.NewManagerWithClient(c)) + r := NewReconciler(kubeClient.NewManagerWithClient(c), "fake-mongodbRepoUrl", "fake-mongodbImage", "ubi8", "fake-agentImage", "fake-versionUpgradeHookImage", "fake-readinessProbeImage") err = r.ensureTLSResources(ctx, mdb) assert.Error(t, err) @@ -544,7 +544,7 @@ func TestTLSConfigReferencesToCACertAreValidated(t *testing.T) { assert.NoError(t, err) - r := NewReconciler(mgr) + r := NewReconciler(mgr, "fake-mongodbRepoUrl", "fake-mongodbImage", "ubi8", "fake-agentImage", "fake-versionUpgradeHookImage", "fake-readinessProbeImage") _, err = r.validateTLSConfig(ctx, mdb) if tc.expectedError != nil { diff --git a/controllers/replica_set_controller.go b/controllers/replica_set_controller.go index 325da36aa..d7f2ee65d 100644 --- a/controllers/replica_set_controller.go +++ b/controllers/replica_set_controller.go @@ -60,7 +60,7 @@ func init() { zap.ReplaceGlobals(logger) } -func NewReconciler(mgr manager.Manager) *ReplicaSetReconciler { +func NewReconciler(mgr manager.Manager, mongodbRepoUrl, mongodbImage, mongodbImageType, agentImage, versionUpgradeHookImage, readinessProbeImage string) *ReplicaSetReconciler { mgrClient := mgr.GetClient() secretWatcher := watch.New() configMapWatcher := watch.New() @@ -70,6 +70,13 @@ func NewReconciler(mgr manager.Manager) *ReplicaSetReconciler { log: zap.S(), secretWatcher: &secretWatcher, configMapWatcher: &configMapWatcher, + + mongodbRepoUrl: mongodbRepoUrl, + mongodbImage: mongodbImage, + mongodbImageType: mongodbImageType, + agentImage: agentImage, + versionUpgradeHookImage: versionUpgradeHookImage, + readinessProbeImage: readinessProbeImage, } } @@ -93,6 +100,13 @@ type ReplicaSetReconciler struct { log *zap.SugaredLogger secretWatcher *watch.ResourceWatcher configMapWatcher *watch.ResourceWatcher + + mongodbRepoUrl string + mongodbImage string + mongodbImageType string + agentImage string + versionUpgradeHookImage string + readinessProbeImage string } // +kubebuilder:rbac:groups=mongodbcommunity.mongodb.com,resources=mongodbcommunity,verbs=get;list;watch;create;update;patch;delete @@ -477,7 +491,8 @@ func (r *ReplicaSetReconciler) createOrUpdateStatefulSet(ctx context.Context, md return fmt.Errorf("error getting StatefulSet: %s", err) } - buildStatefulSetModificationFunction(mdb)(&set) + mongodbImage := getMongoDBImage(r.mongodbRepoUrl, r.mongodbImage, r.mongodbImageType, mdb.GetMongoDBVersion(nil)) + buildStatefulSetModificationFunction(mdb, mongodbImage, r.agentImage, r.versionUpgradeHookImage, r.readinessProbeImage)(&set) if isArbiter { buildArbitersModificationFunction(mdb)(&set) } @@ -499,7 +514,7 @@ func (r ReplicaSetReconciler) ensureAutomationConfig(mdb mdbv1.MongoDBCommunity, return automationconfig.EnsureSecret(ctx, r.client, types.NamespacedName{Name: mdb.AutomationConfigSecretName(), Namespace: mdb.Namespace}, mdb.GetOwnerReferences(), ac) } -func buildAutomationConfig(mdb mdbv1.MongoDBCommunity, auth automationconfig.Auth, currentAc automationconfig.AutomationConfig, modifications ...automationconfig.Modification) (automationconfig.AutomationConfig, error) { +func buildAutomationConfig(mdb mdbv1.MongoDBCommunity, isEnterprise bool, auth automationconfig.Auth, currentAc automationconfig.AutomationConfig, modifications ...automationconfig.Modification) (automationconfig.AutomationConfig, error) { domain := getDomain(mdb.ServiceName(), mdb.Namespace, os.Getenv(clusterDomain)) arbiterDomain := getDomain(mdb.ServiceName(), mdb.Namespace, os.Getenv(clusterDomain)) @@ -517,7 +532,7 @@ func buildAutomationConfig(mdb mdbv1.MongoDBCommunity, auth automationconfig.Aut } return automationconfig.NewBuilder(). - IsEnterprise(guessEnterprise(mdb)). + IsEnterprise(isEnterprise). SetTopology(automationconfig.ReplicaSetTopology). SetName(mdb.Name). SetDomain(domain). @@ -541,7 +556,7 @@ func buildAutomationConfig(mdb mdbv1.MongoDBCommunity, auth automationconfig.Aut Build() } -func guessEnterprise(mdb mdbv1.MongoDBCommunity) bool { +func guessEnterprise(mdb mdbv1.MongoDBCommunity, mongodbImage string) bool { overrideAssumption, err := strconv.ParseBool(os.Getenv(construct.MongoDBAssumeEnterpriseEnv)) if err == nil { return overrideAssumption @@ -561,7 +576,7 @@ func guessEnterprise(mdb mdbv1.MongoDBCommunity) bool { if len(overriddenImage) > 0 { return strings.Contains(overriddenImage, construct.OfficialMongodbEnterpriseServerImageName) } - return os.Getenv(construct.MongodbImageEnv) == construct.OfficialMongodbEnterpriseServerImageName + return mongodbImage == construct.OfficialMongodbEnterpriseServerImageName } // buildService creates a Service that will be used for the Replica Set StatefulSet @@ -671,6 +686,7 @@ func (r ReplicaSetReconciler) buildAutomationConfig(ctx context.Context, mdb mdb automationConfig, err := buildAutomationConfig( mdb, + guessEnterprise(mdb, r.mongodbImage), auth, currentAC, tlsModification, @@ -720,16 +736,10 @@ func getMongodConfigModification(mdb mdbv1.MongoDBCommunity) automationconfig.Mo } } -// buildStatefulSet takes a MongoDB resource and converts it into +// buildStatefulSetModificationFunction takes a MongoDB resource and converts it into // the corresponding stateful set -func buildStatefulSet(mdb mdbv1.MongoDBCommunity) (appsv1.StatefulSet, error) { - sts := appsv1.StatefulSet{} - buildStatefulSetModificationFunction(mdb)(&sts) - return sts, nil -} - -func buildStatefulSetModificationFunction(mdb mdbv1.MongoDBCommunity) statefulset.Modification { - commonModification := construct.BuildMongoDBReplicaSetStatefulSetModificationFunction(&mdb, &mdb, os.Getenv(construct.AgentImageEnv), true) +func buildStatefulSetModificationFunction(mdb mdbv1.MongoDBCommunity, mongodbImage, agentImage, versionUpgradeHookImage, readinessProbeImage string) statefulset.Modification { + commonModification := construct.BuildMongoDBReplicaSetStatefulSetModificationFunction(&mdb, &mdb, mongodbImage, agentImage, versionUpgradeHookImage, readinessProbeImage, true) return statefulset.Apply( commonModification, statefulset.WithOwnerReference(mdb.GetOwnerReferences()), @@ -769,3 +779,18 @@ func getDomain(service, namespace, clusterName string) string { func isPreReadinessInitContainerStatefulSet(sts appsv1.StatefulSet) bool { return container.GetByName(construct.ReadinessProbeContainerName, sts.Spec.Template.Spec.InitContainers) == nil } + +func getMongoDBImage(repoUrl, mongodbImage, mongodbImageType, version string) string { + if strings.HasSuffix(repoUrl, "/") { + repoUrl = strings.TrimRight(repoUrl, "/") + } + mongoImageName := mongodbImage + for _, officialUrl := range construct.OfficialMongodbRepoUrls { + if repoUrl == officialUrl { + return fmt.Sprintf("%s/%s:%s-%s", repoUrl, mongoImageName, version, mongodbImageType) + } + } + + // This is the old images backwards compatibility code path. + return fmt.Sprintf("%s/%s:%s", repoUrl, mongoImageName, version) +} diff --git a/controllers/replicaset_controller_test.go b/controllers/replicaset_controller_test.go index da5780d62..d7f2eb8da 100644 --- a/controllers/replicaset_controller_test.go +++ b/controllers/replicaset_controller_test.go @@ -46,9 +46,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" ) -func init() { - os.Setenv(construct.AgentImageEnv, "agent-image") -} +const ( + AgentImage = "fake-agentImage" +) func newTestReplicaSet() mdbv1.MongoDBCommunity { return mdbv1.MongoDBCommunity{ @@ -162,7 +162,7 @@ func TestKubernetesResources_AreCreated(t *testing.T) { mdb := newTestReplicaSet() mgr := client.NewManager(ctx, &mdb) - r := NewReconciler(mgr) + r := NewReconciler(mgr, "fake-mongodbRepoUrl", "fake-mongodbImage", "ubi8", AgentImage, "fake-versionUpgradeHookImage", "fake-readinessProbeImage") res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) @@ -178,12 +178,10 @@ func TestKubernetesResources_AreCreated(t *testing.T) { func TestStatefulSet_IsCorrectlyConfigured(t *testing.T) { ctx := context.Background() - t.Setenv(construct.MongodbRepoUrl, "docker.io/mongodb") - t.Setenv(construct.MongodbImageEnv, "mongodb-community-server") mdb := newTestReplicaSet() mgr := client.NewManager(ctx, &mdb) - r := NewReconciler(mgr) + r := NewReconciler(mgr, "docker.io/mongodb", "mongodb-community-server", "ubi8", AgentImage, "fake-versionUpgradeHookImage", "fake-readinessProbeImage") res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) @@ -195,7 +193,7 @@ func TestStatefulSet_IsCorrectlyConfigured(t *testing.T) { agentContainer := sts.Spec.Template.Spec.Containers[1] assert.Equal(t, construct.AgentName, agentContainer.Name) - assert.Equal(t, os.Getenv(construct.AgentImageEnv), agentContainer.Image) + assert.Equal(t, AgentImage, agentContainer.Image) expectedProbe := probes.New(construct.DefaultReadiness()) assert.True(t, reflect.DeepEqual(&expectedProbe, agentContainer.ReadinessProbe)) @@ -215,66 +213,42 @@ func TestGuessEnterprise(t *testing.T) { type testConfig struct { setArgs func(t *testing.T) mdb mdbv1.MongoDBCommunity + mongodbImage string expectedEnterprise bool } tests := map[string]testConfig{ "No override and Community image": { - setArgs: func(t *testing.T) { - t.Setenv(construct.MongodbRepoUrl, "docker.io/mongodb") - t.Setenv(construct.MongodbImageEnv, "mongodb-community-server") - }, + setArgs: func(t *testing.T) {}, mdb: mdbv1.MongoDBCommunity{}, + mongodbImage: "mongodb-community-server", expectedEnterprise: false, }, "No override and Enterprise image": { - setArgs: func(t *testing.T) { - t.Setenv(construct.MongodbRepoUrl, "docker.io/mongodb") - t.Setenv(construct.MongodbImageEnv, "mongodb-enterprise-server") - }, + setArgs: func(t *testing.T) {}, mdb: mdbv1.MongoDBCommunity{}, + mongodbImage: "mongodb-enterprise-server", expectedEnterprise: true, }, "Assuming enterprise manually": { setArgs: func(t *testing.T) { - t.Setenv(construct.MongodbRepoUrl, "docker.io/mongodb") - t.Setenv(construct.MongodbImageEnv, "mongodb-community-server") t.Setenv(construct.MongoDBAssumeEnterpriseEnv, "true") }, mdb: mdbv1.MongoDBCommunity{}, + mongodbImage: "mongodb-community-server", expectedEnterprise: true, }, "Assuming community manually": { setArgs: func(t *testing.T) { - t.Setenv(construct.MongodbRepoUrl, "docker.io/mongodb") - t.Setenv(construct.MongodbImageEnv, "mongodb-enterprise-server") t.Setenv(construct.MongoDBAssumeEnterpriseEnv, "false") }, mdb: mdbv1.MongoDBCommunity{}, - expectedEnterprise: false, - }, - "Enterprise with different repo": { - setArgs: func(t *testing.T) { - t.Setenv(construct.MongodbRepoUrl, "some_other_repo.com/some_other_org") - t.Setenv(construct.MongodbImageEnv, "mongodb-enterprise-server") - }, - mdb: mdbv1.MongoDBCommunity{}, - expectedEnterprise: true, - }, - "Community with different repo": { - setArgs: func(t *testing.T) { - t.Setenv(construct.MongodbRepoUrl, "some_other_repo.com/some_other_org") - t.Setenv(construct.MongodbImageEnv, "mongodb-community-server") - }, - mdb: mdbv1.MongoDBCommunity{}, + mongodbImage: "mongodb-enterprise-server", expectedEnterprise: false, }, // This one is a corner case. We don't expect users to fall here very often as there are // dedicated variables to control this type of behavior. "Enterprise with StatefulSet override": { - setArgs: func(t *testing.T) { - t.Setenv(construct.MongodbRepoUrl, "some_other_repo.com/some_other_org") - t.Setenv(construct.MongodbImageEnv, "mongodb-community-server") - }, + setArgs: func(t *testing.T) {}, mdb: mdbv1.MongoDBCommunity{ Spec: mdbv1.MongoDBCommunitySpec{ StatefulSetConfiguration: mdbv1.StatefulSetConfiguration{ @@ -295,13 +269,11 @@ func TestGuessEnterprise(t *testing.T) { }, }, }, + mongodbImage: "mongodb-community-server", expectedEnterprise: true, }, "Enterprise with StatefulSet override to Community": { - setArgs: func(t *testing.T) { - t.Setenv(construct.MongodbRepoUrl, "some_other_repo.com/some_other_org") - t.Setenv(construct.MongodbImageEnv, "mongodb-enterprise-server") - }, + setArgs: func(t *testing.T) {}, mdb: mdbv1.MongoDBCommunity{ Spec: mdbv1.MongoDBCommunitySpec{ StatefulSetConfiguration: mdbv1.StatefulSetConfiguration{ @@ -322,6 +294,7 @@ func TestGuessEnterprise(t *testing.T) { }, }, }, + mongodbImage: "mongodb-enterprise-server", expectedEnterprise: false, }, } @@ -329,7 +302,7 @@ func TestGuessEnterprise(t *testing.T) { t.Run(testName, func(t *testing.T) { testConfig := tests[testName] testConfig.setArgs(t) - calculatedEnterprise := guessEnterprise(testConfig.mdb) + calculatedEnterprise := guessEnterprise(testConfig.mdb, testConfig.mongodbImage) assert.Equal(t, testConfig.expectedEnterprise, calculatedEnterprise) }) } @@ -349,7 +322,7 @@ func TestChangingVersion_ResultsInRollingUpdateStrategyType(t *testing.T) { mdb := newTestReplicaSet() mgr := client.NewManager(ctx, &mdb) mgrClient := mgr.GetClient() - r := NewReconciler(mgr) + r := NewReconciler(mgr, "fake-mongodbRepoUrl", "fake-mongodbImage", "ubi8", AgentImage, "fake-versionUpgradeHookImage", "fake-readinessProbeImage") res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: mdb.NamespacedName()}) assertReconciliationSuccessful(t, res, err) @@ -390,16 +363,16 @@ func TestBuildStatefulSet_ConfiguresUpdateStrategyCorrectly(t *testing.T) { mdb := newTestReplicaSet() mdb.Spec.Version = "4.0.0" mdb.Annotations[annotations.LastAppliedMongoDBVersion] = "4.0.0" - sts, err := buildStatefulSet(mdb) - assert.NoError(t, err) + sts := appsv1.StatefulSet{} + buildStatefulSetModificationFunction(mdb, "fake-mongodbImage", AgentImage, "fake-versionUpgradeHookImage", "fake-readinessProbeImage")(&sts) assert.Equal(t, appsv1.RollingUpdateStatefulSetStrategyType, sts.Spec.UpdateStrategy.Type) }) t.Run("On No Version Change, First Version", func(t *testing.T) { mdb := newTestReplicaSet() mdb.Spec.Version = "4.0.0" delete(mdb.Annotations, annotations.LastAppliedMongoDBVersion) - sts, err := buildStatefulSet(mdb) - assert.NoError(t, err) + sts := appsv1.StatefulSet{} + buildStatefulSetModificationFunction(mdb, "fake-mongodbImage", AgentImage, "fake-versionUpgradeHookImage", "fake-readinessProbeImage")(&sts) assert.Equal(t, appsv1.RollingUpdateStatefulSetStrategyType, sts.Spec.UpdateStrategy.Type) }) t.Run("On Version Change", func(t *testing.T) { @@ -415,9 +388,8 @@ func TestBuildStatefulSet_ConfiguresUpdateStrategyCorrectly(t *testing.T) { assert.NoError(t, err) mdb.Annotations[annotations.LastAppliedMongoDBVersion] = string(bytes) - sts, err := buildStatefulSet(mdb) - - assert.NoError(t, err) + sts := appsv1.StatefulSet{} + buildStatefulSetModificationFunction(mdb, "fake-mongodbImage", AgentImage, "fake-versionUpgradeHookImage", "fake-readinessProbeImage")(&sts) assert.Equal(t, appsv1.OnDeleteStatefulSetStrategyType, sts.Spec.UpdateStrategy.Type) }) } @@ -427,7 +399,7 @@ func TestService_isCorrectlyCreatedAndUpdated(t *testing.T) { mdb := newTestReplicaSet() mgr := client.NewManager(ctx, &mdb) - r := NewReconciler(mgr) + r := NewReconciler(mgr, "fake-mongodbRepoUrl", "fake-mongodbImage", "ubi8", AgentImage, "fake-versionUpgradeHookImage", "fake-readinessProbeImage") res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) @@ -452,7 +424,7 @@ func TestService_usesCustomMongodPortWhenSpecified(t *testing.T) { mdb.Spec.AdditionalMongodConfig.Object = mongodConfig mgr := client.NewManager(ctx, &mdb) - r := NewReconciler(mgr) + r := NewReconciler(mgr, "fake-mongodbRepoUrl", "fake-mongodbImage", "ubi8", AgentImage, "fake-versionUpgradeHookImage", "fake-readinessProbeImage") res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) @@ -516,7 +488,7 @@ func TestService_changesMongodPortOnRunningClusterWithArbiters(t *testing.T) { mgr := client.NewManager(ctx, &mdb) - r := NewReconciler(mgr) + r := NewReconciler(mgr, "fake-mongodbRepoUrl", "fake-mongodbImage", "ubi8", AgentImage, "fake-versionUpgradeHookImage", "fake-readinessProbeImage") t.Run("Prepare cluster with arbiters and change port", func(t *testing.T) { err := createUserPasswordSecret(ctx, mgr.Client, mdb, "password-secret-name", "pass") @@ -745,7 +717,7 @@ func TestService_configuresPrometheusCustomPorts(t *testing.T) { Build()) assert.NoError(t, err) - r := NewReconciler(mgr) + r := NewReconciler(mgr, "fake-mongodbRepoUrl", "fake-mongodbImage", "ubi8", AgentImage, "fake-versionUpgradeHookImage", "fake-readinessProbeImage") res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) @@ -782,7 +754,7 @@ func TestService_configuresPrometheus(t *testing.T) { Build()) assert.NoError(t, err) - r := NewReconciler(mgr) + r := NewReconciler(mgr, "fake-mongodbRepoUrl", "fake-mongodbImage", "ubi8", AgentImage, "fake-versionUpgradeHookImage", "fake-readinessProbeImage") res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) @@ -808,7 +780,7 @@ func TestAutomationConfig_versionIsBumpedOnChange(t *testing.T) { mdb := newTestReplicaSet() mgr := client.NewManager(ctx, &mdb) - r := NewReconciler(mgr) + r := NewReconciler(mgr, "fake-mongodbRepoUrl", "fake-mongodbImage", "ubi8", AgentImage, "fake-versionUpgradeHookImage", "fake-readinessProbeImage") res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) @@ -833,7 +805,7 @@ func TestAutomationConfig_versionIsNotBumpedWithNoChanges(t *testing.T) { mdb := newTestReplicaSet() mgr := client.NewManager(ctx, &mdb) - r := NewReconciler(mgr) + r := NewReconciler(mgr, "fake-mongodbRepoUrl", "fake-mongodbImage", "ubi8", AgentImage, "fake-versionUpgradeHookImage", "fake-readinessProbeImage") res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) @@ -854,7 +826,7 @@ func TestAutomationConfigFCVIsNotIncreasedWhenUpgradingMinorVersion(t *testing.T mdb := newTestReplicaSet() mdb.Spec.Version = "4.2.2" mgr := client.NewManager(ctx, &mdb) - r := NewReconciler(mgr) + r := NewReconciler(mgr, "fake-mongodbRepoUrl", "fake-mongodbImage", "ubi8", AgentImage, "fake-versionUpgradeHookImage", "fake-readinessProbeImage") res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) @@ -888,7 +860,7 @@ func TestAutomationConfig_CustomMongodConfig(t *testing.T) { mdb.Spec.AdditionalMongodConfig.Object = mongodConfig mgr := client.NewManager(ctx, &mdb) - r := NewReconciler(mgr) + r := NewReconciler(mgr, "fake-mongodbRepoUrl", "fake-mongodbImage", "ubi8", AgentImage, "fake-versionUpgradeHookImage", "fake-readinessProbeImage") res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) @@ -932,7 +904,7 @@ func TestExistingPasswordAndKeyfile_AreUsedWhenTheSecretExists(t *testing.T) { Build()) assert.NoError(t, err) - r := NewReconciler(mgr) + r := NewReconciler(mgr, "fake-mongodbRepoUrl", "fake-mongodbImage", "ubi8", AgentImage, "fake-versionUpgradeHookImage", "fake-readinessProbeImage") res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) @@ -963,7 +935,7 @@ func TestReplicaSet_IsScaledDown_OneMember_AtATime_WhenItAlreadyExists(t *testin mdb.Spec.Members = 5 mgr := client.NewManager(ctx, &mdb) - r := NewReconciler(mgr) + r := NewReconciler(mgr, "fake-mongodbRepoUrl", "fake-mongodbImage", "ubi8", AgentImage, "fake-versionUpgradeHookImage", "fake-readinessProbeImage") res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) @@ -1008,7 +980,7 @@ func TestReplicaSet_IsScaledUp_OneMember_AtATime_WhenItAlreadyExists(t *testing. mdb := newTestReplicaSet() mgr := client.NewManager(ctx, &mdb) - r := NewReconciler(mgr) + r := NewReconciler(mgr, "fake-mongodbRepoUrl", "fake-mongodbImage", "ubi8", AgentImage, "fake-versionUpgradeHookImage", "fake-readinessProbeImage") res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) @@ -1078,7 +1050,7 @@ func TestAnnotationsAreAppliedToResource(t *testing.T) { mdb := newTestReplicaSet() mgr := client.NewManager(ctx, &mdb) - r := NewReconciler(mgr) + r := NewReconciler(mgr, "fake-mongodbRepoUrl", "fake-mongodbImage", "ubi8", AgentImage, "fake-versionUpgradeHookImage", "fake-readinessProbeImage") res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) @@ -1094,7 +1066,7 @@ func TestAnnotationsAreAppliedToResource(t *testing.T) { // results in the AuthoritativeSet of the created AutomationConfig to have the expectedValue provided. func assertAuthoritativeSet(ctx context.Context, t *testing.T, mdb mdbv1.MongoDBCommunity, expectedValue bool) { mgr := client.NewManager(ctx, &mdb) - r := NewReconciler(mgr) + r := NewReconciler(mgr, "fake-mongodbRepoUrl", "fake-mongodbImage", "ubi8", AgentImage, "fake-versionUpgradeHookImage", "fake-readinessProbeImage") res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) @@ -1110,7 +1082,7 @@ func assertAuthoritativeSet(ctx context.Context, t *testing.T, mdb mdbv1.MongoDB func assertReplicaSetIsConfiguredWithScram(ctx context.Context, t *testing.T, mdb mdbv1.MongoDBCommunity) { mgr := client.NewManager(ctx, &mdb) - r := NewReconciler(mgr) + r := NewReconciler(mgr, "fake-mongodbRepoUrl", "fake-mongodbImage", "ubi8", AgentImage, "fake-versionUpgradeHookImage", "fake-readinessProbeImage") res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) @@ -1144,7 +1116,7 @@ func assertReplicaSetIsConfiguredWithScramTLS(ctx context.Context, t *testing.T, assert.NoError(t, err) err = createTLSConfigMap(ctx, newClient, mdb) assert.NoError(t, err) - r := NewReconciler(mgr) + r := NewReconciler(mgr, "fake-mongodbRepoUrl", "fake-mongodbImage", "ubi8", AgentImage, "fake-versionUpgradeHookImage", "fake-readinessProbeImage") res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) @@ -1184,7 +1156,7 @@ func assertReplicaSetIsConfiguredWithX509(ctx context.Context, t *testing.T, mdb err = createAgentCertSecret(ctx, newClient, mdb, crt, key, "") assert.NoError(t, err) - r := NewReconciler(mgr) + r := NewReconciler(mgr, "fake-mongodbRepoUrl", "fake-mongodbImage", "ubi8", AgentImage, "fake-versionUpgradeHookImage", "fake-readinessProbeImage") res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) @@ -1244,7 +1216,7 @@ func TestReplicaSet_IsScaledUpToDesiredMembers_WhenFirstCreated(t *testing.T) { mdb := newTestReplicaSet() mgr := client.NewManager(ctx, &mdb) - r := NewReconciler(mgr) + r := NewReconciler(mgr, "fake-mongodbRepoUrl", "fake-mongodbImage", "ubi8", AgentImage, "fake-versionUpgradeHookImage", "fake-readinessProbeImage") res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) @@ -1347,7 +1319,7 @@ func TestInconsistentReplicas(t *testing.T) { mdb.Spec.Members = 4 mgr := client.NewManager(ctx, &mdb) - r := NewReconciler(mgr) + r := NewReconciler(mgr, "fake-mongodbRepoUrl", "fake-mongodbImage", "ubi8", AgentImage, "fake-versionUpgradeHookImage", "fake-readinessProbeImage") _, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assert.NoError(t, err) } @@ -1367,7 +1339,7 @@ func performReconciliationAndGetStatefulSet(ctx context.Context, t *testing.T, f assert.NoError(t, err) mgr := client.NewManager(ctx, &mdb) assert.NoError(t, generatePasswordsForAllUsers(ctx, mdb, mgr.Client)) - r := NewReconciler(mgr) + r := NewReconciler(mgr, "fake-mongodbRepoUrl", "fake-mongodbImage", "ubi8", AgentImage, "fake-versionUpgradeHookImage", "fake-readinessProbeImage") res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: mdb.NamespacedName()}) assertReconciliationSuccessful(t, res, err) @@ -1381,7 +1353,7 @@ func performReconciliationAndGetService(ctx context.Context, t *testing.T, fileP assert.NoError(t, err) mgr := client.NewManager(ctx, &mdb) assert.NoError(t, generatePasswordsForAllUsers(ctx, mdb, mgr.Client)) - r := NewReconciler(mgr) + r := NewReconciler(mgr, "fake-mongodbRepoUrl", "fake-mongodbImage", "ubi8", AgentImage, "fake-versionUpgradeHookImage", "fake-readinessProbeImage") res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: mdb.NamespacedName()}) assertReconciliationSuccessful(t, res, err) svc, err := mgr.Client.GetService(ctx, types.NamespacedName{Name: mdb.ServiceName(), Namespace: mdb.Namespace}) @@ -1468,3 +1440,74 @@ func marshalRuntimeObjectFromYAMLBytes(bytes []byte, obj runtime.Object) error { } return json.Unmarshal(jsonBytes, &obj) } + +func TestGetMongoDBImage(t *testing.T) { + type testConfig struct { + mongodbRepoUrl string + mongodbImage string + mongodbImageType string + version string + expectedImage string + } + tests := map[string]testConfig{ + "Default UBI8 Community image": { + mongodbRepoUrl: "docker.io/mongodb", + mongodbImage: "mongodb-community-server", + mongodbImageType: "ubi8", + version: "6.0.5", + expectedImage: "docker.io/mongodb/mongodb-community-server:6.0.5-ubi8", + }, + "Overridden UBI8 Enterprise image": { + mongodbRepoUrl: "docker.io/mongodb", + mongodbImage: "mongodb-enterprise-server", + mongodbImageType: "ubi8", + version: "6.0.5", + expectedImage: "docker.io/mongodb/mongodb-enterprise-server:6.0.5-ubi8", + }, + "Overridden UBI8 Enterprise image from Quay": { + mongodbRepoUrl: "quay.io/mongodb", + mongodbImage: "mongodb-enterprise-server", + mongodbImageType: "ubi8", + version: "6.0.5", + expectedImage: "quay.io/mongodb/mongodb-enterprise-server:6.0.5-ubi8", + }, + "Overridden Ubuntu Community image": { + mongodbRepoUrl: "docker.io/mongodb", + mongodbImage: "mongodb-community-server", + mongodbImageType: "ubuntu2204", + version: "6.0.5", + expectedImage: "docker.io/mongodb/mongodb-community-server:6.0.5-ubuntu2204", + }, + "Overridden UBI Community image": { + mongodbRepoUrl: "docker.io/mongodb", + mongodbImage: "mongodb-community-server", + mongodbImageType: "ubi8", + version: "6.0.5", + expectedImage: "docker.io/mongodb/mongodb-community-server:6.0.5-ubi8", + }, + "Docker Inc images": { + mongodbRepoUrl: "docker.io", + mongodbImage: "mongo", + mongodbImageType: "ubi8", + version: "6.0.5", + expectedImage: "docker.io/mongo:6.0.5", + }, + "Deprecated AppDB images defined the old way": { + mongodbRepoUrl: "quay.io", + mongodbImage: "mongodb/mongodb-enterprise-appdb-database-ubi", + // In this example, we intentionally don't use the suffix from the env. variable and let users + // define it in the version instead. There are some known customers who do this. + // This is a backwards compatibility case. + mongodbImageType: "will-be-ignored", + version: "5.0.14-ent", + expectedImage: "quay.io/mongodb/mongodb-enterprise-appdb-database-ubi:5.0.14-ent", + }, + } + for testName := range tests { + t.Run(testName, func(t *testing.T) { + testConfig := tests[testName] + image := getMongoDBImage(testConfig.mongodbRepoUrl, testConfig.mongodbImage, testConfig.mongodbImageType, testConfig.version) + assert.Equal(t, testConfig.expectedImage, image) + }) + } +} diff --git a/test/e2e/replica_set_enterprise_upgrade/replica_set_enterprise_upgrade.go b/test/e2e/replica_set_enterprise_upgrade/replica_set_enterprise_upgrade.go index d8c0bfb23..ff6930252 100644 --- a/test/e2e/replica_set_enterprise_upgrade/replica_set_enterprise_upgrade.go +++ b/test/e2e/replica_set_enterprise_upgrade/replica_set_enterprise_upgrade.go @@ -7,14 +7,14 @@ import ( "time" "github.com/mongodb/mongodb-kubernetes-operator/controllers/construct" - "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" + e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e" "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/mongodbtests" "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup" "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/util/mongotester" ) func DeployEnterpriseAndUpgradeTest(ctx context.Context, t *testing.T, versionsToBeTested []string) { - t.Setenv(construct.MongodbRepoUrl, "docker.io/mongodb") + t.Setenv(construct.MongodbRepoUrlEnv, "docker.io/mongodb") t.Setenv(construct.MongodbImageEnv, "mongodb-enterprise-server") testCtx := setup.Setup(ctx, t) defer testCtx.Teardown() diff --git a/test/e2e/setup/test_config.go b/test/e2e/setup/test_config.go index 06f2018fe..d4b2ab8c1 100644 --- a/test/e2e/setup/test_config.go +++ b/test/e2e/setup/test_config.go @@ -39,7 +39,7 @@ func LoadTestConfigFromEnv() TestConfig { CertManagerVersion: envvar.GetEnvOrDefault(testCertManagerVersionEnvName, "v1.5.3"), OperatorImage: envvar.GetEnvOrDefault(operatorImageEnvName, "quay.io/mongodb/community-operator-dev:latest"), MongoDBImage: envvar.GetEnvOrDefault(construct.MongodbImageEnv, "mongodb-community-server"), - MongoDBRepoUrl: envvar.GetEnvOrDefault(construct.MongodbRepoUrl, "quay.io/mongodb"), + MongoDBRepoUrl: envvar.GetEnvOrDefault(construct.MongodbRepoUrlEnv, "quay.io/mongodb"), VersionUpgradeHookImage: envvar.GetEnvOrDefault(construct.VersionUpgradeHookImageEnv, "quay.io/mongodb/mongodb-kubernetes-operator-version-upgrade-post-start-hook:1.0.2"), AgentImage: envvar.GetEnvOrDefault(construct.AgentImageEnv, "quay.io/mongodb/mongodb-agent-ubi:10.29.0.6830-1"), // TODO: better way to decide default agent image. ClusterWide: envvar.ReadBool(clusterWideEnvName), From 0e7208998804803f5e953557785fb114b2c3bab3 Mon Sep 17 00:00:00 2001 From: Mikalai Radchuk <509198+m1kola@users.noreply.github.com> Date: Tue, 11 Mar 2025 09:49:16 +0100 Subject: [PATCH 778/790] Remove annotations arg from GetMongoDBVersion (#1677) --- api/v1/mongodbcommunity_types.go | 4 ++-- controllers/construct/mongodbstatefulset.go | 2 +- controllers/replica_set_controller.go | 4 ++-- test/e2e/mongodbtests/mongodbtests.go | 2 +- test/e2e/replica_set_recovery/replica_set_recovery_test.go | 2 +- test/e2e/replica_set_scale/replica_set_scaling_test.go | 2 +- .../e2e/replica_set_scale_down/replica_set_scale_down_test.go | 2 +- test/e2e/statefulset_delete/statefulset_delete_test.go | 2 +- 8 files changed, 10 insertions(+), 10 deletions(-) diff --git a/api/v1/mongodbcommunity_types.go b/api/v1/mongodbcommunity_types.go index f1e7f8710..e3455ea21 100644 --- a/api/v1/mongodbcommunity_types.go +++ b/api/v1/mongodbcommunity_types.go @@ -1114,7 +1114,7 @@ func (m *MongoDBCommunity) CurrentArbiters() int { return m.Status.CurrentStatefulSetArbitersReplicas } -func (m *MongoDBCommunity) GetMongoDBVersion(map[string]string) string { +func (m *MongoDBCommunity) GetMongoDBVersion() string { return m.Spec.Version } @@ -1122,7 +1122,7 @@ func (m *MongoDBCommunity) GetMongoDBVersion(map[string]string) string { // Here it's the same as GetMongoDBVersion, but a different name is used in order to make // the usage clearer in enterprise (where it's a method of OpsManager but is used for the AppDB) func (m *MongoDBCommunity) GetMongoDBVersionForAnnotation() string { - return m.GetMongoDBVersion(nil) + return m.GetMongoDBVersion() } func (m *MongoDBCommunity) StatefulSetReplicasThisReconciliation() int { diff --git a/controllers/construct/mongodbstatefulset.go b/controllers/construct/mongodbstatefulset.go index cec2e5be7..08a33939a 100644 --- a/controllers/construct/mongodbstatefulset.go +++ b/controllers/construct/mongodbstatefulset.go @@ -95,7 +95,7 @@ type MongoDBStatefulSetOwner interface { // GetNamespace returns the namespace the resource is defined in. GetNamespace() string // GetMongoDBVersion returns the version of MongoDB to be used for this resource. - GetMongoDBVersion(annotations map[string]string) string + GetMongoDBVersion() string // AutomationConfigSecretName returns the name of the secret which will contain the automation config. AutomationConfigSecretName() string // GetUpdateStrategyType returns the UpdateStrategyType of the statefulset. diff --git a/controllers/replica_set_controller.go b/controllers/replica_set_controller.go index d7f2ee65d..5b3594812 100644 --- a/controllers/replica_set_controller.go +++ b/controllers/replica_set_controller.go @@ -226,7 +226,7 @@ func (r ReplicaSetReconciler) Reconcile(ctx context.Context, request reconcile.R withMongoDBArbiters(mdb.AutomationConfigArbitersThisReconciliation()). withMessage(None, ""). withRunningPhase(). - withVersion(mdb.GetMongoDBVersion(nil))) + withVersion(mdb.GetMongoDBVersion())) if err != nil { r.log.Errorf("Error updating the status of the MongoDB resource: %s", err) return res, err @@ -491,7 +491,7 @@ func (r *ReplicaSetReconciler) createOrUpdateStatefulSet(ctx context.Context, md return fmt.Errorf("error getting StatefulSet: %s", err) } - mongodbImage := getMongoDBImage(r.mongodbRepoUrl, r.mongodbImage, r.mongodbImageType, mdb.GetMongoDBVersion(nil)) + mongodbImage := getMongoDBImage(r.mongodbRepoUrl, r.mongodbImage, r.mongodbImageType, mdb.GetMongoDBVersion()) buildStatefulSetModificationFunction(mdb, mongodbImage, r.agentImage, r.versionUpgradeHookImage, r.readinessProbeImage)(&set) if isArbiter { buildArbitersModificationFunction(mdb)(&set) diff --git a/test/e2e/mongodbtests/mongodbtests.go b/test/e2e/mongodbtests/mongodbtests.go index aad73a450..a7bbf30df 100644 --- a/test/e2e/mongodbtests/mongodbtests.go +++ b/test/e2e/mongodbtests/mongodbtests.go @@ -503,7 +503,7 @@ func BasicFunctionality(ctx context.Context, mdb *mdbv1.MongoDBCommunity, skipSt t.Run("Test Status Was Updated", Status(ctx, mdb, mdbv1.MongoDBCommunityStatus{ MongoURI: mdb.MongoURI(""), Phase: mdbv1.Running, - Version: mdb.GetMongoDBVersion(nil), + Version: mdb.GetMongoDBVersion(), CurrentMongoDBMembers: mdb.Spec.Members, CurrentStatefulSetReplicas: mdb.Spec.Members, })) diff --git a/test/e2e/replica_set_recovery/replica_set_recovery_test.go b/test/e2e/replica_set_recovery/replica_set_recovery_test.go index a6ab6f7a0..91c9426b7 100644 --- a/test/e2e/replica_set_recovery/replica_set_recovery_test.go +++ b/test/e2e/replica_set_recovery/replica_set_recovery_test.go @@ -59,7 +59,7 @@ func TestReplicaSetRecovery(t *testing.T) { t.Run("Test Status Was Updated", mongodbtests.Status(ctx, &mdb, mdbv1.MongoDBCommunityStatus{ MongoURI: mdb.MongoURI(""), Phase: mdbv1.Running, - Version: mdb.GetMongoDBVersion(nil), + Version: mdb.GetMongoDBVersion(), CurrentMongoDBMembers: 3, CurrentStatefulSetReplicas: 3, })) diff --git a/test/e2e/replica_set_scale/replica_set_scaling_test.go b/test/e2e/replica_set_scale/replica_set_scaling_test.go index ee0fe2f19..0361ba9f0 100644 --- a/test/e2e/replica_set_scale/replica_set_scaling_test.go +++ b/test/e2e/replica_set_scale/replica_set_scaling_test.go @@ -54,7 +54,7 @@ func TestReplicaSetScaleUp(t *testing.T) { t.Run("Test Status Was Updated", mongodbtests.Status(ctx, &mdb, mdbv1.MongoDBCommunityStatus{ MongoURI: mdb.MongoURI(""), Phase: mdbv1.Running, - Version: mdb.GetMongoDBVersion(nil), + Version: mdb.GetMongoDBVersion(), CurrentMongoDBMembers: 5, CurrentStatefulSetReplicas: 5, })) diff --git a/test/e2e/replica_set_scale_down/replica_set_scale_down_test.go b/test/e2e/replica_set_scale_down/replica_set_scale_down_test.go index 529738712..fd03fdafc 100644 --- a/test/e2e/replica_set_scale_down/replica_set_scale_down_test.go +++ b/test/e2e/replica_set_scale_down/replica_set_scale_down_test.go @@ -57,7 +57,7 @@ func TestReplicaSetScaleDown(t *testing.T) { t.Run("Test Status Was Updated", mongodbtests.Status(ctx, &mdb, mdbv1.MongoDBCommunityStatus{ MongoURI: mdb.MongoURI(""), Phase: mdbv1.Running, - Version: mdb.GetMongoDBVersion(nil), + Version: mdb.GetMongoDBVersion(), CurrentMongoDBMembers: 1, CurrentStatefulSetReplicas: 1, })) diff --git a/test/e2e/statefulset_delete/statefulset_delete_test.go b/test/e2e/statefulset_delete/statefulset_delete_test.go index 686d50071..3117109e6 100644 --- a/test/e2e/statefulset_delete/statefulset_delete_test.go +++ b/test/e2e/statefulset_delete/statefulset_delete_test.go @@ -42,7 +42,7 @@ func TestStatefulSetDelete(t *testing.T) { t.Run("Test Status Was Updated", mongodbtests.Status(ctx, &mdb, mdbv1.MongoDBCommunityStatus{ MongoURI: mdb.MongoURI(""), Phase: mdbv1.Running, - Version: mdb.GetMongoDBVersion(nil), + Version: mdb.GetMongoDBVersion(), CurrentMongoDBMembers: mdb.DesiredReplicas(), })) }) From bbc718507c7233fb8a04a568f70665e8c2116087 Mon Sep 17 00:00:00 2001 From: Mikalai Radchuk <509198+m1kola@users.noreply.github.com> Date: Wed, 12 Mar 2025 19:33:09 +0100 Subject: [PATCH 779/790] CLOUDP-302068: Add a linter to forbid env var usage (#1690) We want to avoid spreading the practice of accessing environment variables deep in the call hierarchy. We only want to access env vars in the `main` package or very close to it. This commit adds a linter to help us with that. It also adds `nolint:forbidigo` exceptions into places: * Where we allow access (`main` package) * Where we still unfortunately have access. These places ideally need to change with time. To find remaining undesired places, from repo root you can `grep` all files for `nolint:forbidigo` excluding files with `main` package. For example: `grep "nolint:forbidigo" --exclude main.go -r .` --- .golangci.yml | 21 +++++++++++++++ .../construct/build_statefulset_test.go | 2 +- controllers/construct/mongodbstatefulset.go | 2 +- controllers/replica_set_controller.go | 10 +++---- pkg/kube/container/container_test.go | 2 +- pkg/kube/container/containers.go | 2 +- pkg/kube/podtemplatespec/podspec_template.go | 2 +- pkg/readiness/config/config.go | 8 +++--- test/e2e/e2eutil.go | 2 +- test/e2e/setup/setup.go | 6 ++--- test/e2e/setup/test_config.go | 27 ++++++++++--------- 11 files changed, 53 insertions(+), 31 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 0fae128cd..795e08728 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -16,6 +16,12 @@ issues: - goconst - golint text: "underscore" + - path: ^pkg\/util\/envvar + linters: + - forbidigo + - path: ^cmd\/(readiness|versionhook|manager)\/main\.go$ + linters: + - forbidigo linters: enable: - govet @@ -28,10 +34,25 @@ linters: - rowserrcheck - gosec - unconvert + - forbidigo linters-settings: gosec: excludes: - G115 + forbidigo: + forbid: + - p: os\.(Getenv|LookupEnv|Environ|ExpandEnv) + pkg: os + msg: "Reading environemnt variables here is prohibited. Please read environment variables in the main package." + - p: os\.(Clearenv|Unsetenv|Setenv) + msg: "Modifying environemnt variables is prohibited." + pkg: os + - p: envvar\.(Read.*?|MergeWithOverride|GetEnvOrDefault) + pkg: github.com/mongodb/mongodb-kubernetes-operator/pkg/util/envvar + msg: "Using this envvar package here is prohibited. Please work with environment variables in the main package." + # Rules with the `pkg` depend on it + analyze-types: true + run: modules-download-mode: mod # timeout for analysis, e.g. 30s, 5m, default is 1m diff --git a/controllers/construct/build_statefulset_test.go b/controllers/construct/build_statefulset_test.go index e02095c63..791fa5a8b 100644 --- a/controllers/construct/build_statefulset_test.go +++ b/controllers/construct/build_statefulset_test.go @@ -109,7 +109,7 @@ func assertStatefulSetIsBuiltCorrectly(t *testing.T, mdb mdbv1.MongoDBCommunity, assert.Len(t, sts.Spec.Template.Spec.Containers[0].Env, 4) assert.Len(t, sts.Spec.Template.Spec.Containers[1].Env, 1) - managedSecurityContext := envvar.ReadBool(podtemplatespec.ManagedSecurityContextEnv) + managedSecurityContext := envvar.ReadBool(podtemplatespec.ManagedSecurityContextEnv) // nolint:forbidigo if !managedSecurityContext { assert.NotNil(t, sts.Spec.Template.Spec.SecurityContext) assert.Equal(t, podtemplatespec.DefaultPodSecurityContext(), *sts.Spec.Template.Spec.SecurityContext) diff --git a/controllers/construct/mongodbstatefulset.go b/controllers/construct/mongodbstatefulset.go index 08a33939a..ec94a6eac 100644 --- a/controllers/construct/mongodbstatefulset.go +++ b/controllers/construct/mongodbstatefulset.go @@ -417,7 +417,7 @@ func collectEnvVars() []corev1.EnvVar { }) addEnvVarIfSet := func(name string) { - value := os.Getenv(name) + value := os.Getenv(name) // nolint:forbidigo if value != "" { envVars = append(envVars, corev1.EnvVar{ Name: name, diff --git a/controllers/replica_set_controller.go b/controllers/replica_set_controller.go index 5b3594812..890c88216 100644 --- a/controllers/replica_set_controller.go +++ b/controllers/replica_set_controller.go @@ -219,7 +219,7 @@ func (r ReplicaSetReconciler) Reconcile(ctx context.Context, request reconcile.R } res, err := status.Update(ctx, r.client.Status(), &mdb, statusOptions(). - withMongoURI(mdb.MongoURI(os.Getenv(clusterDomain))). + withMongoURI(mdb.MongoURI(os.Getenv(clusterDomain))). // nolint:forbidigo withMongoDBMembers(mdb.AutomationConfigMembersThisReconciliation()). withStatefulSetReplicas(mdb.StatefulSetReplicasThisReconciliation()). withStatefulSetArbiters(mdb.StatefulSetArbitersThisReconciliation()). @@ -232,7 +232,7 @@ func (r ReplicaSetReconciler) Reconcile(ctx context.Context, request reconcile.R return res, err } - if err := r.updateConnectionStringSecrets(ctx, mdb, os.Getenv(clusterDomain)); err != nil { + if err := r.updateConnectionStringSecrets(ctx, mdb, os.Getenv(clusterDomain)); err != nil { // nolint:forbidigo r.log.Errorf("Could not update connection string secrets: %s", err) } @@ -515,8 +515,8 @@ func (r ReplicaSetReconciler) ensureAutomationConfig(mdb mdbv1.MongoDBCommunity, } func buildAutomationConfig(mdb mdbv1.MongoDBCommunity, isEnterprise bool, auth automationconfig.Auth, currentAc automationconfig.AutomationConfig, modifications ...automationconfig.Modification) (automationconfig.AutomationConfig, error) { - domain := getDomain(mdb.ServiceName(), mdb.Namespace, os.Getenv(clusterDomain)) - arbiterDomain := getDomain(mdb.ServiceName(), mdb.Namespace, os.Getenv(clusterDomain)) + domain := getDomain(mdb.ServiceName(), mdb.Namespace, os.Getenv(clusterDomain)) // nolint:forbidigo + arbiterDomain := getDomain(mdb.ServiceName(), mdb.Namespace, os.Getenv(clusterDomain)) // nolint:forbidigo zap.S().Debugw("AutomationConfigMembersThisReconciliation", "mdb.AutomationConfigMembersThisReconciliation()", mdb.AutomationConfigMembersThisReconciliation()) @@ -557,7 +557,7 @@ func buildAutomationConfig(mdb mdbv1.MongoDBCommunity, isEnterprise bool, auth a } func guessEnterprise(mdb mdbv1.MongoDBCommunity, mongodbImage string) bool { - overrideAssumption, err := strconv.ParseBool(os.Getenv(construct.MongoDBAssumeEnterpriseEnv)) + overrideAssumption, err := strconv.ParseBool(os.Getenv(construct.MongoDBAssumeEnterpriseEnv)) // nolint:forbidigo if err == nil { return overrideAssumption } diff --git a/pkg/kube/container/container_test.go b/pkg/kube/container/container_test.go index 5c08b14b9..a61a0be15 100644 --- a/pkg/kube/container/container_test.go +++ b/pkg/kube/container/container_test.go @@ -146,7 +146,7 @@ func TestMergeEnvs(t *testing.T) { }, } - merged := envvar.MergeWithOverride(existing, desired) + merged := envvar.MergeWithOverride(existing, desired) // nolint:forbidigo t.Run("EnvVars should be sorted", func(t *testing.T) { assert.Equal(t, "A_env", merged[0].Name) diff --git a/pkg/kube/container/containers.go b/pkg/kube/container/containers.go index debd4b115..687befc5b 100644 --- a/pkg/kube/container/containers.go +++ b/pkg/kube/container/containers.go @@ -129,7 +129,7 @@ func WithLifecycle(lifeCycleMod lifecycle.Modification) Modification { // WithEnvs ensures all of the provided envs exist in the container func WithEnvs(envs ...corev1.EnvVar) Modification { return func(container *corev1.Container) { - container.Env = envvar.MergeWithOverride(container.Env, envs) + container.Env = envvar.MergeWithOverride(container.Env, envs) // nolint:forbidigo } } diff --git a/pkg/kube/podtemplatespec/podspec_template.go b/pkg/kube/podtemplatespec/podspec_template.go index e79b4cf05..f908a214a 100644 --- a/pkg/kube/podtemplatespec/podspec_template.go +++ b/pkg/kube/podtemplatespec/podspec_template.go @@ -297,7 +297,7 @@ func FindContainerByName(name string, podTemplateSpec *corev1.PodTemplateSpec) * } func WithDefaultSecurityContextsModifications() (Modification, container.Modification) { - managedSecurityContext := envvar.ReadBool(ManagedSecurityContextEnv) + managedSecurityContext := envvar.ReadBool(ManagedSecurityContextEnv) // nolint:forbidigo configureContainerSecurityContext := container.NOOP() configurePodSpecSecurityContext := NOOP() if !managedSecurityContext { diff --git a/pkg/readiness/config/config.go b/pkg/readiness/config/config.go index 09b8ca50e..7f3e64714 100644 --- a/pkg/readiness/config/config.go +++ b/pkg/readiness/config/config.go @@ -43,15 +43,15 @@ func BuildFromEnvVariables(clientSet kubernetes.Interface, isHeadless bool, file var namespace, automationConfigName, hostname string if isHeadless { var ok bool - namespace, ok = os.LookupEnv(podNamespaceEnv) + namespace, ok = os.LookupEnv(podNamespaceEnv) // nolint:forbidigo if !ok { return Config{}, fmt.Errorf("the '%s' environment variable must be set", podNamespaceEnv) } - automationConfigName, ok = os.LookupEnv(automationConfigSecretEnv) + automationConfigName, ok = os.LookupEnv(automationConfigSecretEnv) // nolint:forbidigo if !ok { return Config{}, fmt.Errorf("the '%s' environment variable must be set", automationConfigSecretEnv) } - hostname, ok = os.LookupEnv(hostNameEnv) + hostname, ok = os.LookupEnv(hostNameEnv) // nolint:forbidigo if !ok { return Config{}, fmt.Errorf("the '%s' environment variable must be set", hostNameEnv) } @@ -85,7 +85,7 @@ func readinessProbeLogFilePath() string { } func GetEnvOrDefault(envVar, defaultValue string) string { - value := strings.TrimSpace(os.Getenv(envVar)) + value := strings.TrimSpace(os.Getenv(envVar)) // nolint:forbidigo if value == "" { return defaultValue } diff --git a/test/e2e/e2eutil.go b/test/e2e/e2eutil.go index 41f64c287..d29fd9abb 100644 --- a/test/e2e/e2eutil.go +++ b/test/e2e/e2eutil.go @@ -36,7 +36,7 @@ func TestAnnotations() map[string]string { } func TestDataDir() string { - return envvar.GetEnvOrDefault(testDataDirEnv, "/workspace/testdata") + return envvar.GetEnvOrDefault(testDataDirEnv, "/workspace/testdata") // nolint:forbidigo } func TlsTestDataDir() string { diff --git a/test/e2e/setup/setup.go b/test/e2e/setup/setup.go index 88769bf78..8bf9595bd 100644 --- a/test/e2e/setup/setup.go +++ b/test/e2e/setup/setup.go @@ -42,7 +42,7 @@ const ( ) func Setup(ctx context.Context, t *testing.T) *e2eutil.TestContext { - testCtx, err := e2eutil.NewContext(ctx, t, envvar.ReadBool(performCleanupEnv)) + testCtx, err := e2eutil.NewContext(ctx, t, envvar.ReadBool(performCleanupEnv)) // nolint:forbidigo if err != nil { t.Fatal(err) @@ -57,7 +57,7 @@ func Setup(ctx context.Context, t *testing.T) *e2eutil.TestContext { } func SetupWithTLS(ctx context.Context, t *testing.T, resourceName string, additionalHelmArgs ...HelmArg) (*e2eutil.TestContext, TestConfig) { - textCtx, err := e2eutil.NewContext(ctx, t, envvar.ReadBool(performCleanupEnv)) + textCtx, err := e2eutil.NewContext(ctx, t, envvar.ReadBool(performCleanupEnv)) // nolint:forbidigo if err != nil { t.Fatal(err) @@ -76,7 +76,7 @@ func SetupWithTLS(ctx context.Context, t *testing.T, resourceName string, additi } func SetupWithTestConfig(ctx context.Context, t *testing.T, testConfig TestConfig, withTLS, defaultOperator bool, resourceName string) *e2eutil.TestContext { - testCtx, err := e2eutil.NewContext(ctx, t, envvar.ReadBool(performCleanupEnv)) + testCtx, err := e2eutil.NewContext(ctx, t, envvar.ReadBool(performCleanupEnv)) // nolint:forbidigo if err != nil { t.Fatal(err) diff --git a/test/e2e/setup/test_config.go b/test/e2e/setup/test_config.go index d4b2ab8c1..1fc247021 100644 --- a/test/e2e/setup/test_config.go +++ b/test/e2e/setup/test_config.go @@ -34,18 +34,19 @@ type TestConfig struct { func LoadTestConfigFromEnv() TestConfig { return TestConfig{ - Namespace: envvar.GetEnvOrDefault(testNamespaceEnvName, "mongodb"), - CertManagerNamespace: envvar.GetEnvOrDefault(testCertManagerNamespaceEnvName, "cert-manager"), - CertManagerVersion: envvar.GetEnvOrDefault(testCertManagerVersionEnvName, "v1.5.3"), - OperatorImage: envvar.GetEnvOrDefault(operatorImageEnvName, "quay.io/mongodb/community-operator-dev:latest"), - MongoDBImage: envvar.GetEnvOrDefault(construct.MongodbImageEnv, "mongodb-community-server"), - MongoDBRepoUrl: envvar.GetEnvOrDefault(construct.MongodbRepoUrlEnv, "quay.io/mongodb"), - VersionUpgradeHookImage: envvar.GetEnvOrDefault(construct.VersionUpgradeHookImageEnv, "quay.io/mongodb/mongodb-kubernetes-operator-version-upgrade-post-start-hook:1.0.2"), - AgentImage: envvar.GetEnvOrDefault(construct.AgentImageEnv, "quay.io/mongodb/mongodb-agent-ubi:10.29.0.6830-1"), // TODO: better way to decide default agent image. - ClusterWide: envvar.ReadBool(clusterWideEnvName), - PerformCleanup: envvar.ReadBool(performCleanupEnvName), - ReadinessProbeImage: envvar.GetEnvOrDefault(construct.ReadinessProbeImageEnv, "quay.io/mongodb/mongodb-kubernetes-readinessprobe:1.0.3"), - HelmChartPath: envvar.GetEnvOrDefault(helmChartPathEnvName, "/workspace/helm-charts/charts/community-operator"), - LocalOperator: envvar.ReadBool(LocalOperatorEnvName), + Namespace: envvar.GetEnvOrDefault(testNamespaceEnvName, "mongodb"), // nolint:forbidigo + CertManagerNamespace: envvar.GetEnvOrDefault(testCertManagerNamespaceEnvName, "cert-manager"), // nolint:forbidigo + CertManagerVersion: envvar.GetEnvOrDefault(testCertManagerVersionEnvName, "v1.5.3"), // nolint:forbidigo + OperatorImage: envvar.GetEnvOrDefault(operatorImageEnvName, "quay.io/mongodb/community-operator-dev:latest"), // nolint:forbidigo + MongoDBImage: envvar.GetEnvOrDefault(construct.MongodbImageEnv, "mongodb-community-server"), // nolint:forbidigo + MongoDBRepoUrl: envvar.GetEnvOrDefault(construct.MongodbRepoUrlEnv, "quay.io/mongodb"), // nolint:forbidigo + VersionUpgradeHookImage: envvar.GetEnvOrDefault(construct.VersionUpgradeHookImageEnv, "quay.io/mongodb/mongodb-kubernetes-operator-version-upgrade-post-start-hook:1.0.2"), // nolint:forbidigo + // TODO: better way to decide default agent image. + AgentImage: envvar.GetEnvOrDefault(construct.AgentImageEnv, "quay.io/mongodb/mongodb-agent-ubi:10.29.0.6830-1"), // nolint:forbidigo + ClusterWide: envvar.ReadBool(clusterWideEnvName), // nolint:forbidigo + PerformCleanup: envvar.ReadBool(performCleanupEnvName), // nolint:forbidigo + ReadinessProbeImage: envvar.GetEnvOrDefault(construct.ReadinessProbeImageEnv, "quay.io/mongodb/mongodb-kubernetes-readinessprobe:1.0.3"), // nolint:forbidigo + HelmChartPath: envvar.GetEnvOrDefault(helmChartPathEnvName, "/workspace/helm-charts/charts/community-operator"), // nolint:forbidigo + LocalOperator: envvar.ReadBool(LocalOperatorEnvName), // nolint:forbidigo } } From 61d715fb405de3e88e2017e7cc41f67e915dd1ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Kara=C5=9B?= <6159874+MaciejKaras@users.noreply.github.com> Date: Tue, 18 Mar 2025 10:47:42 +0100 Subject: [PATCH 780/790] [dependabot] Ignore k8s api dependencies bumps (#1696) We compile and release with the lowest API we support currently, so there is no reason to create bumps for controller-runtime. --- .github/dependabot.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index d19a62dbe..eb3084c66 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,6 +5,12 @@ updates: schedule: interval: weekly day: monday + ignore: + - dependency-name: k8s.io/api + - dependency-name: k8s.io/apimachinery + - dependency-name: k8s.io/client-go + - dependency-name: k8s.io/code-generator + - dependency-name: sigs.k8s.io/controller-runtime - package-ecosystem: pip directory: "/" schedule: From 96624d06f31a77080d2904f0184ab8847f32b9e3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 18 Mar 2025 12:25:10 +0100 Subject: [PATCH 781/790] Bump github.com/stretchr/testify from 1.9.0 to 1.10.0 (#1697) Bumps [github.com/stretchr/testify](https://github.com/stretchr/testify) from 1.9.0 to 1.10.0. - [Release notes](https://github.com/stretchr/testify/releases) - [Commits](https://github.com/stretchr/testify/compare/v1.9.0...v1.10.0) --- updated-dependencies: - dependency-name: github.com/stretchr/testify dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 77ad4173c..ce3e344be 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/imdario/mergo v0.3.15 github.com/spf13/cast v1.6.0 github.com/stretchr/objx v0.5.2 - github.com/stretchr/testify v1.9.0 + github.com/stretchr/testify v1.10.0 github.com/xdg/stringprep v1.0.3 go.mongodb.org/mongo-driver v1.16.0 go.uber.org/zap v1.27.0 diff --git a/go.sum b/go.sum index c85107ab4..aaa674bfd 100644 --- a/go.sum +++ b/go.sum @@ -125,8 +125,8 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= From 255a21c80165c63cd22f5c635a6bc0c7e89f6bf5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 18 Mar 2025 12:25:21 +0100 Subject: [PATCH 782/790] Bump github.com/spf13/cast from 1.6.0 to 1.7.1 (#1698) Bumps [github.com/spf13/cast](https://github.com/spf13/cast) from 1.6.0 to 1.7.1. - [Release notes](https://github.com/spf13/cast/releases) - [Commits](https://github.com/spf13/cast/compare/v1.6.0...v1.7.1) --- updated-dependencies: - dependency-name: github.com/spf13/cast dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index ce3e344be..49eeb06d3 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/go-logr/logr v1.4.2 github.com/hashicorp/go-multierror v1.1.1 github.com/imdario/mergo v0.3.15 - github.com/spf13/cast v1.6.0 + github.com/spf13/cast v1.7.1 github.com/stretchr/objx v0.5.2 github.com/stretchr/testify v1.10.0 github.com/xdg/stringprep v1.0.3 diff --git a/go.sum b/go.sum index aaa674bfd..0d8f4ac2d 100644 --- a/go.sum +++ b/go.sum @@ -112,8 +112,8 @@ github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= -github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= -github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= +github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= From 7637d160086f9fc5d795e7eb0a6f147a4e20dd06 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 18 Mar 2025 12:25:33 +0100 Subject: [PATCH 783/790] Bump golang.org/x/net from 0.34.0 to 0.36.0 (#1691) Bumps [golang.org/x/net](https://github.com/golang/net) from 0.34.0 to 0.36.0. - [Commits](https://github.com/golang/net/compare/v0.34.0...v0.36.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 12 ++++++------ go.sum | 24 ++++++++++++------------ 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index 49eeb06d3..5894aee21 100644 --- a/go.mod +++ b/go.mod @@ -68,14 +68,14 @@ require ( github.com/xdg-go/stringprep v1.0.4 // indirect github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.32.0 // indirect + golang.org/x/crypto v0.35.0 // indirect golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect - golang.org/x/net v0.34.0 // indirect + golang.org/x/net v0.36.0 // indirect golang.org/x/oauth2 v0.12.0 // indirect - golang.org/x/sync v0.10.0 // indirect - golang.org/x/sys v0.29.0 // indirect - golang.org/x/term v0.28.0 // indirect - golang.org/x/text v0.21.0 // indirect + golang.org/x/sync v0.11.0 // indirect + golang.org/x/sys v0.30.0 // indirect + golang.org/x/term v0.29.0 // indirect + golang.org/x/text v0.22.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.23.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index 0d8f4ac2d..4b8969f32 100644 --- a/go.sum +++ b/go.sum @@ -152,8 +152,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= -golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= +golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs= +golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ= golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA= golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -166,16 +166,16 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= -golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= +golang.org/x/net v0.36.0 h1:vWF2fRbw4qslQsQzgFqZff+BItCvGFQqKzKIzx1rmoA= +golang.org/x/net v0.36.0/go.mod h1:bFmbeoIPfrw4sMHNhb4J9f6+tPziuGjq7Jk/38fxi1I= golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4= golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= -golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= +golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -183,19 +183,19 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= -golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= +golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg= -golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek= +golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU= +golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= -golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= +golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From 51761b993543fb4865741bf4ef15f7b1c99a2e72 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 25 Mar 2025 09:47:22 +0100 Subject: [PATCH 784/790] Bump setuptools from 71.0.3 to 78.0.1 (#1702) Bumps [setuptools](https://github.com/pypa/setuptools) from 71.0.3 to 78.0.1. - [Release notes](https://github.com/pypa/setuptools/releases) - [Changelog](https://github.com/pypa/setuptools/blob/main/NEWS.rst) - [Commits](https://github.com/pypa/setuptools/compare/v71.0.3...v78.0.1) --- updated-dependencies: - dependency-name: setuptools dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e86c8aa2f..e264e2e09 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,6 +15,6 @@ requests==2.32.3 ruamel.yaml==0.17.9 semver==2.13.0 rsa>=4.7 # not directly required, pinned by Snyk to avoid a vulnerability -setuptools==71.0.3 # not directly required, pinned by Snyk to avoid a vulnerability +setuptools==78.0.1 # not directly required, pinned by Snyk to avoid a vulnerability certifi>=2022.12.7 # not directly required, pinned by Snyk to avoid a vulnerability urllib3<2 # not directly required, pinned by Snyk to avoid a vulnerability From 6c683e70dedb78ca158012fd0cd7e929e8c17a8a Mon Sep 17 00:00:00 2001 From: unknowvwake Date: Tue, 1 Apr 2025 15:03:34 +0530 Subject: [PATCH 785/790] Changed "repo" to "repository" for clarity and formality. (#1643) Changed "repo" to "repository" for clarity and formality. --- LICENSE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE.md b/LICENSE.md index 5463e3426..9c600b1bc 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,5 +1,5 @@ The MongoDB Agent binary in the agent/ directory may be used under the "Free for Commercial Use - Oct 2020" license found in [agent/LICENSE](scripts/dev/templates/agent/LICENSE). -The source code of this Operator, and all other content in this repo are available under the Apache v2 license. The text of this license is available in [APACHE2](APACHE2) +The source code of this Operator, and all other content in this repository are available under the Apache v2 license. The text of this license is available in [APACHE2](APACHE2) To use this Operator, you must agree to both licenses. From 0c46290355085d85c649e629b462aff597c0ee12 Mon Sep 17 00:00:00 2001 From: Christoph Geschwind Date: Tue, 1 Apr 2025 11:35:07 +0200 Subject: [PATCH 786/790] Implement OverrideReplicaSet Id for https://github.com/mongodb/mongodb-kubernetes-operator/issues/1650 (#1656) --- api/v1/mongodbcommunity_types.go | 3 +++ api/v1/zz_generated.deepcopy.go | 5 +++++ ...bcommunity.mongodb.com_mongodbcommunity.yaml | 5 +++++ controllers/replica_set_controller.go | 3 +++ .../automation_config_builder.go | 13 ++++++++++++- pkg/automationconfig/automation_config_test.go | 17 +++++++++++++++++ 6 files changed, 45 insertions(+), 1 deletion(-) diff --git a/api/v1/mongodbcommunity_types.go b/api/v1/mongodbcommunity_types.go index e3455ea21..6a5e4bf0c 100644 --- a/api/v1/mongodbcommunity_types.go +++ b/api/v1/mongodbcommunity_types.go @@ -328,6 +328,9 @@ type AutomationConfigOverride struct { } type OverrideReplicaSet struct { + // Id can be used together with additionalMongodConfig.replication.replSetName + // to manage clusters where replSetName differs from the MongoDBCommunity resource name + Id *string `json:"id,omitempty"` // +kubebuilder:validation:Type=object // +kubebuilder:pruning:PreserveUnknownFields Settings MapWrapper `json:"settings,omitempty"` diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index bf58f2b77..df22b4876 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -366,6 +366,11 @@ func (in *OverrideProcess) DeepCopy() *OverrideProcess { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OverrideReplicaSet) DeepCopyInto(out *OverrideReplicaSet) { *out = *in + if in.Id != nil { + in, out := &in.Id, &out.Id + *out = new(string) + **out = **in + } in.Settings.DeepCopyInto(&out.Settings) } diff --git a/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml b/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml index 42c2258d0..12207a6bd 100644 --- a/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml +++ b/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml @@ -225,6 +225,11 @@ spec: type: array replicaSet: properties: + id: + description: |- + Id can be used together with additionalMongodConfig.replication.replSetName + to manage clusters where replSetName differs from the MongoDBCommunity resource name + type: string settings: description: |- MapWrapper is a wrapper for a map to be used by other structs. diff --git a/controllers/replica_set_controller.go b/controllers/replica_set_controller.go index 890c88216..cf3e9d526 100644 --- a/controllers/replica_set_controller.go +++ b/controllers/replica_set_controller.go @@ -527,8 +527,10 @@ func buildAutomationConfig(mdb mdbv1.MongoDBCommunity, isEnterprise bool, auth a } var acOverrideSettings map[string]interface{} + var acReplicaSetId *string if mdb.Spec.AutomationConfigOverride != nil { acOverrideSettings = mdb.Spec.AutomationConfigOverride.ReplicaSet.Settings.Object + acReplicaSetId = mdb.Spec.AutomationConfigOverride.ReplicaSet.Id } return automationconfig.NewBuilder(). @@ -545,6 +547,7 @@ func buildAutomationConfig(mdb mdbv1.MongoDBCommunity, isEnterprise bool, auth a SetFCV(mdb.Spec.FeatureCompatibilityVersion). SetOptions(automationconfig.Options{DownloadBase: "/var/lib/mongodb-mms-automation"}). SetAuth(auth). + SetReplicaSetId(acReplicaSetId). SetSettings(acOverrideSettings). SetMemberOptions(mdb.Spec.MemberConfig). SetDataDir(mdb.GetMongodConfiguration().GetDBDataDir()). diff --git a/pkg/automationconfig/automation_config_builder.go b/pkg/automationconfig/automation_config_builder.go index be69e3f29..3091734dc 100644 --- a/pkg/automationconfig/automation_config_builder.go +++ b/pkg/automationconfig/automation_config_builder.go @@ -54,6 +54,7 @@ type Builder struct { port int memberOptions []MemberOptions forceReconfigureToVersion *int64 + replicaSetId *string settings map[string]interface{} } @@ -194,6 +195,11 @@ func (b *Builder) SetAuth(auth Auth) *Builder { return b } +func (b *Builder) SetReplicaSetId(id *string) *Builder { + b.replicaSetId = id + return b +} + func (b *Builder) SetSettings(settings map[string]interface{}) *Builder { b.settings = settings return b @@ -372,12 +378,17 @@ func (b *Builder) Build() (AutomationConfig, error) { replSetForceConfig = &ReplSetForceConfig{CurrentVersion: *b.forceReconfigureToVersion} } + replicaSetId := b.name + if b.replicaSetId != nil { + replicaSetId = *b.replicaSetId + } + currentAc := AutomationConfig{ Version: b.previousAC.Version, Processes: processes, ReplicaSets: []ReplicaSet{ { - Id: b.name, + Id: replicaSetId, Members: members, ProtocolVersion: "1", NumberArbiters: b.arbiters, diff --git a/pkg/automationconfig/automation_config_test.go b/pkg/automationconfig/automation_config_test.go index 2950eb4d7..19b3bcfe8 100644 --- a/pkg/automationconfig/automation_config_test.go +++ b/pkg/automationconfig/automation_config_test.go @@ -530,3 +530,20 @@ func createAutomationConfig(name, mongodbVersion, domain string, opts Options, a ac.Version = acVersion return ac } + +func TestReplicaSetId(t *testing.T) { + id := "rs0" + ac, err := NewBuilder(). + SetName("my-rs"). + SetDomain("my-ns.svc.cluster.local"). + SetMongoDBVersion("4.2.0"). + SetMembers(3). + AddVersion(defaultMongoDbVersion("4.3.2")). + SetReplicaSetId(&id). + Build() + + assert.NoError(t, err) + assert.Len(t, ac.ReplicaSets, 1) + rs := ac.ReplicaSets[0] + assert.Equal(t, rs.Id, id, "The provided id should be used") +} From e2467068a94f77757714eff9677be3b79bc4e114 Mon Sep 17 00:00:00 2001 From: mircea-cosbuc Date: Fri, 11 Apr 2025 20:14:10 +0200 Subject: [PATCH 787/790] Release MongoDB Kubernetes Operator v0.13.0 (#1705) * Release MongoDB Kubernetes Operator v0.13.0 * Remove agent images from release matrix --- .github/workflows/release-images.yml | 2 -- config/manager/manager.yaml | 8 +++--- deploy/openshift/operator_openshift.yaml | 8 +++--- docs/RELEASE_NOTES.md | 17 ++++++++++-- go.mod | 15 +++++------ go.sum | 33 ++++++++++-------------- release.json | 10 +++---- 7 files changed, 49 insertions(+), 44 deletions(-) diff --git a/.github/workflows/release-images.yml b/.github/workflows/release-images.yml index b0fb77dbc..5ced57eae 100644 --- a/.github/workflows/release-images.yml +++ b/.github/workflows/release-images.yml @@ -18,8 +18,6 @@ jobs: release-key: version-upgrade-hook - pipeline-argument: readiness-probe release-key: readiness-probe - - pipeline-argument: agent - release-key: agent steps: - name: Checkout Code diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 9013a8451..0705e7eae 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -45,16 +45,16 @@ spec: - name: OPERATOR_NAME value: mongodb-kubernetes-operator - name: AGENT_IMAGE - value: quay.io/mongodb/mongodb-agent-ubi:108.0.2.8729-1 + value: quay.io/mongodb/mongodb-agent-ubi:108.0.6.8796-1 - name: VERSION_UPGRADE_HOOK_IMAGE - value: quay.io/mongodb/mongodb-kubernetes-operator-version-upgrade-post-start-hook:1.0.9 + value: quay.io/mongodb/mongodb-kubernetes-operator-version-upgrade-post-start-hook:1.0.10 - name: READINESS_PROBE_IMAGE - value: quay.io/mongodb/mongodb-kubernetes-readinessprobe:1.0.22 + value: quay.io/mongodb/mongodb-kubernetes-readinessprobe:1.0.23 - name: MONGODB_IMAGE value: mongodb-community-server - name: MONGODB_REPO_URL value: quay.io/mongodb - image: quay.io/mongodb/mongodb-kubernetes-operator:0.12.0 + image: quay.io/mongodb/mongodb-kubernetes-operator:0.13.0 imagePullPolicy: Always name: mongodb-kubernetes-operator resources: diff --git a/deploy/openshift/operator_openshift.yaml b/deploy/openshift/operator_openshift.yaml index c0f3cf1ce..b7011a1cc 100644 --- a/deploy/openshift/operator_openshift.yaml +++ b/deploy/openshift/operator_openshift.yaml @@ -47,16 +47,16 @@ spec: - name: OPERATOR_NAME value: mongodb-kubernetes-operator - name: AGENT_IMAGE - value: quay.io/mongodb/mongodb-agent-ubi:108.0.2.8729-1 + value: quay.io/mongodb/mongodb-agent-ubi:108.0.6.8796-1 - name: READINESS_PROBE_IMAGE - value: quay.io/mongodb/mongodb-kubernetes-readinessprobe:1.0.22 + value: quay.io/mongodb/mongodb-kubernetes-readinessprobe:1.0.23 - name: VERSION_UPGRADE_HOOK_IMAGE - value: quay.io/mongodb/mongodb-kubernetes-operator-version-upgrade-post-start-hook:1.0.9 + value: quay.io/mongodb/mongodb-kubernetes-operator-version-upgrade-post-start-hook:1.0.10 - name: MONGODB_IMAGE value: mongo - name: MONGODB_REPO_URL value: quay.io - image: quay.io/mongodb/mongodb-kubernetes-operator:0.12.0 + image: quay.io/mongodb/mongodb-kubernetes-operator:0.13.0 imagePullPolicy: Always name: mongodb-kubernetes-operator resources: diff --git a/docs/RELEASE_NOTES.md b/docs/RELEASE_NOTES.md index 13beb2133..6109fac02 100644 --- a/docs/RELEASE_NOTES.md +++ b/docs/RELEASE_NOTES.md @@ -1,5 +1,18 @@ # MongoDB Kubernetes Operator 0.13.0 -## Minor fixes and improvements +## Dependency updates + - Updated kubernetes dependencies to 1.30 + - Bumped Go dependency to 1.24 + - Updated packages `crypto`, `net` and `oauth2` to remediate multiple CVEs + +## MongoDBCommunity Resource + - Added support for overriding the ReplicaSet ID ([#1656](https://github.com/mongodb/mongodb-kubernetes-operator/pull/1656)). + +## Improvements + - Refactored environment variable propagation ([#1676](https://github.com/mongodb/mongodb-kubernetes-operator/pull/1676)). + - Introduced a linter to limit inappropriate usage of environment variables within the codebase ([#1690](https://github.com/mongodb/mongodb-kubernetes-operator/pull/1690)). + +## Security & Dependency Updates + - **CVE Updates**: Updated packages `crypto`, `net` and `oauth2` to remediate multiple CVEs + - Upgraded to Go 1.24 and Kubernetes dependencies to 1.30.x . - - Bumped kube APIs to 1.30 diff --git a/go.mod b/go.mod index 5894aee21..35b8ccebc 100644 --- a/go.mod +++ b/go.mod @@ -68,18 +68,17 @@ require ( github.com/xdg-go/stringprep v1.0.4 // indirect github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.35.0 // indirect + golang.org/x/crypto v0.37.0 // indirect golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect - golang.org/x/net v0.36.0 // indirect - golang.org/x/oauth2 v0.12.0 // indirect - golang.org/x/sync v0.11.0 // indirect - golang.org/x/sys v0.30.0 // indirect - golang.org/x/term v0.29.0 // indirect - golang.org/x/text v0.22.0 // indirect + golang.org/x/net v0.39.0 // indirect + golang.org/x/oauth2 v0.29.0 // indirect + golang.org/x/sync v0.13.0 // indirect + golang.org/x/sys v0.32.0 // indirect + golang.org/x/term v0.31.0 // indirect + golang.org/x/text v0.24.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.23.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect - google.golang.org/appengine v1.6.7 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 4b8969f32..4e2b54c85 100644 --- a/go.sum +++ b/go.sum @@ -36,7 +36,6 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= @@ -152,30 +151,29 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs= -golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ= +golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= +golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA= golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.36.0 h1:vWF2fRbw4qslQsQzgFqZff+BItCvGFQqKzKIzx1rmoA= -golang.org/x/net v0.36.0/go.mod h1:bFmbeoIPfrw4sMHNhb4J9f6+tPziuGjq7Jk/38fxi1I= -golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4= -golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4= +golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY= +golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E= +golang.org/x/oauth2 v0.29.0 h1:WdYw2tdTK1S8olAzWHdgeqfy+Mtm9XNhv/xJsY65d98= +golang.org/x/oauth2 v0.29.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= -golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= +golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -183,19 +181,18 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= -golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= +golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU= -golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s= +golang.org/x/term v0.31.0 h1:erwDkOK1Msy6offm1mOgvspSkslFnIGsFnxOKoufg3o= +golang.org/x/term v0.31.0/go.mod h1:R4BeIy7D95HzImkxGkTW1UQTtP54tio2RyHz7PwK0aw= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= -golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= +golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= +golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -211,8 +208,6 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/release.json b/release.json index 88983b5b3..078b90861 100644 --- a/release.json +++ b/release.json @@ -1,8 +1,8 @@ { "golang-builder-image": "golang:1.24", - "operator": "0.12.0", - "version-upgrade-hook": "1.0.9", - "readiness-probe": "1.0.22", - "agent": "108.0.2.8729-1", - "agent-tools-version": "100.10.0" + "operator": "0.13.0", + "version-upgrade-hook": "1.0.10", + "readiness-probe": "1.0.23", + "agent": "108.0.6.8796-1", + "agent-tools-version": "100.11.0" } From 39025950721c5e2a2b7e69665729018adceb7ce7 Mon Sep 17 00:00:00 2001 From: mircea-cosbuc Date: Mon, 14 Apr 2025 09:31:24 +0200 Subject: [PATCH 788/790] Pin helm-chart to 0.13.0 release (#1706) --- helm-charts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm-charts b/helm-charts index ad091e266..c6b6488a2 160000 --- a/helm-charts +++ b/helm-charts @@ -1 +1 @@ -Subproject commit ad091e2667b42cdbe240806322659525cb45663e +Subproject commit c6b6488a2a84cb806eadac0e286b6060914082d5 From 5cf108f929c440839c31ada415ab0f970331b6a2 Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Mon, 12 May 2025 10:07:51 +0200 Subject: [PATCH 789/790] CLOUDP-315307 - mck - MCO deprecation notice (#1712) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * mck - add deprecation notice and add linkages Co-authored-by: Simon Bäumer --------- Co-authored-by: Simon Bäumer --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 8af30d276..5476e4383 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,10 @@ +> **DEPRECATED:** This repository is deprecated but we will continue a best-effort support until November 2025. Please use the new repository at [mongodb/mongodb-kubernetes](https://github.com/mongodb/mongodb-kubernetes) instead. +> +> For more information on this decision - what it means and entails - see the [announcement](https://github.com/mongodb/mongodb-kubernetes/releases/tag/v1.0.0) and our [public documentation](https://www.mongodb.com/docs/kubernetes/current/). +> +> A detailed migration guide is available to help you transition smoothly - see [guide](https://github.com/mongodb/mongodb-kubernetes/blob/master/docs/migration/community-operator-migration.md). There will be no functional changes in the new repository - only a better and unified experience as well as improved visibility into the development process. + + # MongoDB Community Kubernetes Operator # From f6a53c96128e3e998f4c274964876206ce409305 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 23 May 2025 10:46:01 +0200 Subject: [PATCH 790/790] Bump setuptools from 78.0.1 to 78.1.1 (#1717) Bumps [setuptools](https://github.com/pypa/setuptools) from 78.0.1 to 78.1.1. - [Release notes](https://github.com/pypa/setuptools/releases) - [Changelog](https://github.com/pypa/setuptools/blob/main/NEWS.rst) - [Commits](https://github.com/pypa/setuptools/compare/v78.0.1...v78.1.1) --- updated-dependencies: - dependency-name: setuptools dependency-version: 78.1.1 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e264e2e09..3247df769 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,6 +15,6 @@ requests==2.32.3 ruamel.yaml==0.17.9 semver==2.13.0 rsa>=4.7 # not directly required, pinned by Snyk to avoid a vulnerability -setuptools==78.0.1 # not directly required, pinned by Snyk to avoid a vulnerability +setuptools==78.1.1 # not directly required, pinned by Snyk to avoid a vulnerability certifi>=2022.12.7 # not directly required, pinned by Snyk to avoid a vulnerability urllib3<2 # not directly required, pinned by Snyk to avoid a vulnerability