From fc987b77d42ce779e319a0597b56783c5445a1ae Mon Sep 17 00:00:00 2001 From: Fabian Lindfors Date: Tue, 14 Jul 2020 15:22:10 +0200 Subject: [PATCH 01/11] Rename pre-stop hook to version upgrade post-start hook --- .evergreen.yml | 2 +- architecture.md | 2 +- cmd/testrunner/main.go | 48 +++++++++---------- cmd/{prestop => versionhook}/main.go | 2 +- contributing.md | 2 +- .../mongodb/build_statefulset_test.go | 7 ++- pkg/controller/mongodb/mongodb_controller.go | 26 +++++----- scripts/ci/config.json | 2 +- scripts/dev/dev_config.py | 4 +- scripts/dev/e2e.py | 16 +++---- scripts/dev/templates/Dockerfile.prehook | 18 ------- scripts/dev/templates/Dockerfile.versionhook | 18 +++++++ 12 files changed, 73 insertions(+), 74 deletions(-) rename cmd/{prestop => versionhook}/main.go (99%) delete mode 100644 scripts/dev/templates/Dockerfile.prehook create mode 100644 scripts/dev/templates/Dockerfile.versionhook diff --git a/.evergreen.yml b/.evergreen.yml index b3f12758c..9c2a416aa 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -184,7 +184,7 @@ tasks: - func: build_and_push_image vars: image: quay.io/mongodb/community-operator-pre-stop-hook:${version_id} - image_type: prehook + image_type: versionhook - name: build_testrunner_image priority: 60 diff --git a/architecture.md b/architecture.md index f4e6e9b05..63d656be2 100644 --- a/architecture.md +++ b/architecture.md @@ -16,7 +16,7 @@ You create and update MongoDB resources by defining a MongoDB resource definitio 1. Writes the Automation configuration as a [ConfigMap](https://kubernetes.io/docs/concepts/configuration/configmap/) and mounts it to each pod. 1. Creates one [init container](https://kubernetes.io/docs/concepts/workloads/pods/init-containers/) and two [containers](https://kubernetes.io/docs/concepts/containers/overview/) in each pod: - - An init container which copies the `cmd/prestop` binary to the main `mongod` container. [This pre-stop hook](https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/) is used during [version upgrades](#example-mongodb-version-upgrade). + - An init container which copies the `cmd/versionhook` binary to the main `mongod` container. This is used as a [pre-stop hook](https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/) for [version upgrades](#example-mongodb-version-upgrade). - A container for the [`mongod`](https://docs.mongodb.com/manual/reference/program/mongod/index.html) process binary. `mongod` is the primary daemon process for the MongoDB system. It handles data requests, manages data access, and performs background management operations. diff --git a/cmd/testrunner/main.go b/cmd/testrunner/main.go index 81614085b..b0b1f81d1 100644 --- a/cmd/testrunner/main.go +++ b/cmd/testrunner/main.go @@ -30,34 +30,34 @@ import ( ) type flags struct { - deployDir string - namespace string - operatorImage string - preHookImage string - testImage string - test string - performCleanup string + deployDir string + namespace string + operatorImage string + versionUpgradeHookImage string + testImage string + test string + performCleanup string } func parseFlags() flags { - var namespace, deployDir, operatorImage, preHookImage, testImage, test, performCleanup *string + var namespace, deployDir, operatorImage, versionUpgradeHookImage, testImage, test, performCleanup *string namespace = flag.String("namespace", "default", "the namespace the operator and tests should be deployed in") deployDir = flag.String("deployDir", "deploy/", "the path to the directory which contains the yaml deployment files") operatorImage = flag.String("operatorImage", "quay.io/mongodb/community-operator-dev:latest", "the image which should be used for the operator deployment") - preHookImage = flag.String("preHookImage", "quay.io/mongodb/community-operator-prehook:latest", "the prestophook image") + versionUpgradeHookImage = flag.String("versionUpgradeHookImage", "quay.io/mongodb/community-operator-pre-stop-hook:latest", "the version upgrade post-start hook image") testImage = flag.String("testImage", "quay.io/mongodb/community-operator-e2e:latest", "the image which should be used for the operator e2e tests") test = flag.String("test", "", "test e2e test that should be run. (name of folder containing the test)") performCleanup = flag.String("performCleanup", "1", "specifies whether to performing a cleanup the context or not") flag.Parse() return flags{ - deployDir: *deployDir, - namespace: *namespace, - operatorImage: *operatorImage, - preHookImage: *preHookImage, - testImage: *testImage, - test: *test, - performCleanup: *performCleanup, + deployDir: *deployDir, + namespace: *namespace, + operatorImage: *operatorImage, + versionUpgradeHookImage: *versionUpgradeHookImage, + testImage: *testImage, + test: *test, + performCleanup: *performCleanup, } } @@ -174,7 +174,7 @@ func deployOperator(f flags, c client.Client) error { &appsv1.Deployment{}, withNamespace(f.namespace), withOperatorImage(f.operatorImage), - withPreHookImage(f.preHookImage)); err != nil { + withVersionUpgradeHookImage(f.versionUpgradeHookImage)); err != nil { return fmt.Errorf("error building operator deployment: %v", err) } fmt.Println("Successfully created the operator Deployment") @@ -219,25 +219,25 @@ func withEnvVar(key, val string) func(obj runtime.Object) { } } -// withPreHookImage sets the value of the PRE_STOP_HOOK_IMAGE +// withVersionUpgradeHookImage sets the value of the POD_DELETER_IMAGE // EnvVar from first container to `image`. The EnvVar is updated // if it exists. Or appended if there is no EnvVar with this `Name`. -func withPreHookImage(image string) func(runtime.Object) { +func withVersionUpgradeHookImage(image string) func(runtime.Object) { return func(obj runtime.Object) { if dep, ok := obj.(*appsv1.Deployment); ok { - preHookEnv := corev1.EnvVar{ - Name: "PRE_STOP_HOOK_IMAGE", + versionUpgradeHookEnv := corev1.EnvVar{ + Name: "VERSION_UPGRADE_HOOK_IMAGE", Value: image, } found := false for idx := range dep.Spec.Template.Spec.Containers[0].Env { - if dep.Spec.Template.Spec.Containers[0].Env[idx].Name == preHookEnv.Name { - dep.Spec.Template.Spec.Containers[0].Env[idx].Value = preHookEnv.Value + if dep.Spec.Template.Spec.Containers[0].Env[idx].Name == versionUpgradeHookEnv.Name { + dep.Spec.Template.Spec.Containers[0].Env[idx].Value = versionUpgradeHookEnv.Value found = true } } if !found { - dep.Spec.Template.Spec.Containers[0].Env = append(dep.Spec.Template.Spec.Containers[0].Env, preHookEnv) + dep.Spec.Template.Spec.Containers[0].Env = append(dep.Spec.Template.Spec.Containers[0].Env, versionUpgradeHookEnv) } } } diff --git a/cmd/prestop/main.go b/cmd/versionhook/main.go similarity index 99% rename from cmd/prestop/main.go rename to cmd/versionhook/main.go index feff0735b..72fe1104e 100644 --- a/cmd/prestop/main.go +++ b/cmd/versionhook/main.go @@ -20,7 +20,7 @@ import ( const ( agentStatusFilePathEnv = "AGENT_STATUS_FILEPATH" - logFilePathEnv = "PRE_STOP_HOOK_LOG_PATH" + logFilePathEnv = "POD_DELETER_LOG_PATH" defaultNamespace = "default" diff --git a/contributing.md b/contributing.md index 30f57e6d3..0d8498970 100644 --- a/contributing.md +++ b/contributing.md @@ -31,7 +31,7 @@ by the operator and mounted in the Agent's Pod. * MongoDB image: Docker image that includes the MongoDB server. -* Pre-stop Hook: This image includes a binary that helps orchestrate the +* Version upgrade post-start hook image: This image includes a binary that helps orchestrate the restarts of the MongoDB Replica Set members, in particular, when dealing with version upgrades, which requires a very precise set of operations to allow for seamless upgrades and downgrades, with no downtime. diff --git a/pkg/controller/mongodb/build_statefulset_test.go b/pkg/controller/mongodb/build_statefulset_test.go index 5bd15c380..f79c326f5 100644 --- a/pkg/controller/mongodb/build_statefulset_test.go +++ b/pkg/controller/mongodb/build_statefulset_test.go @@ -14,8 +14,7 @@ import ( ) func init() { - - os.Setenv(preStopHookImageEnv, "pre-stop-hook-image") + os.Setenv(versionUpgradeHookImageEnv, "version-upgrade-hook-image") } func TestMultipleCalls_DoNotCauseSideEffects(t *testing.T) { @@ -62,7 +61,7 @@ func assertStatefulSetIsBuiltCorrectly(t *testing.T, mdb mdbv1.MongoDB, sts *app assert.Len(t, mongodContainer.VolumeMounts, 3) initContainer := sts.Spec.Template.Spec.InitContainers[0] - assert.Equal(t, preStopHookName, initContainer.Name) - assert.Equal(t, "pre-stop-hook-image", initContainer.Image) + assert.Equal(t, versionUpgradeHookName, initContainer.Name) + assert.Equal(t, "version-upgrade-hook-image", initContainer.Image) assert.Len(t, initContainer.VolumeMounts, 1) } diff --git a/pkg/controller/mongodb/mongodb_controller.go b/pkg/controller/mongodb/mongodb_controller.go index b1dd26366..53a987baf 100644 --- a/pkg/controller/mongodb/mongodb_controller.go +++ b/pkg/controller/mongodb/mongodb_controller.go @@ -42,15 +42,15 @@ import ( ) const ( - agentImageEnv = "AGENT_IMAGE" - preStopHookImageEnv = "PRE_STOP_HOOK_IMAGE" - agentHealthStatusFilePathEnv = "AGENT_STATUS_FILEPATH" - preStopHookLogFilePathEnv = "PRE_STOP_HOOK_LOG_PATH" + agentImageEnv = "AGENT_IMAGE" + versionUpgradeHookImageEnv = "VERSION_UPGRADE_HOOK_IMAGE" + agentHealthStatusFilePathEnv = "AGENT_STATUS_FILEPATH" + versionUpgradeHookLogFilePathEnv = "VERSION_UPGRADE_HOOK_LOG_PATH" AutomationConfigKey = "automation-config" agentName = "mongodb-agent" mongodbName = "mongod" - preStopHookName = "mongod-prehook" + versionUpgradeHookName = "mongod-posthook" dataVolumeName = "data-volume" versionManifestFilePath = "/usr/local/version_manifest.json" readinessProbePath = "/var/lib/mongodb-mms-automation/probes/readinessprobe" @@ -526,11 +526,11 @@ func mongodbAgentContainer(volumeMounts []corev1.VolumeMount) container.Modifica ) } -func preStopHookInit(volumeMount []corev1.VolumeMount) container.Modification { +func versionUpgradeHookInit(volumeMount []corev1.VolumeMount) container.Modification { return container.Apply( - container.WithName(preStopHookName), - container.WithCommand([]string{"cp", "pre-stop-hook", "/hooks/pre-stop-hook"}), - container.WithImage(os.Getenv(preStopHookImageEnv)), + container.WithName(versionUpgradeHookName), + container.WithCommand([]string{"cp", "version-upgrade-hook", "/hooks/version-upgrade-hook"}), + container.WithImage(os.Getenv(versionUpgradeHookImageEnv)), container.WithImagePullPolicy(corev1.PullAlways), container.WithVolumeMounts(volumeMount), ) @@ -548,7 +548,7 @@ mongod -f /data/automation-mongod.conf ; # start the pre-stop-hook to restart the Pod when needed # If the Pod does not require to be restarted, the pre-stop-hook will # exit(0) for Kubernetes to restart the container. -/hooks/pre-stop-hook ; +/hooks/version-upgrade-hook; `, } @@ -563,8 +563,8 @@ mongod -f /data/automation-mongod.conf ; Value: "/healthstatus/agent-health-status.json", }, corev1.EnvVar{ - Name: preStopHookLogFilePathEnv, - Value: "/hooks/pre-stop-hook.log", + Name: versionUpgradeHookLogFilePathEnv, + Value: "/hooks/pod-deleter.log", }, ), container.WithVolumeMounts(volumeMounts), @@ -619,7 +619,7 @@ func buildStatefulSetModificationFunction(mdb mdbv1.MongoDB) statefulset.Modific podtemplatespec.WithServiceAccount(operatorServiceAccountName), podtemplatespec.WithContainer(agentName, mongodbAgentContainer([]corev1.VolumeMount{agentHealthStatusVolumeMount, automationConfigVolumeMount, dataVolume})), podtemplatespec.WithContainer(mongodbName, mongodbContainer(mdb.Spec.Version, []corev1.VolumeMount{mongodHealthStatusVolumeMount, dataVolume, hooksVolumeMount})), - podtemplatespec.WithInitContainer(preStopHookName, preStopHookInit([]corev1.VolumeMount{hooksVolumeMount})), + podtemplatespec.WithInitContainer(versionUpgradeHookName, versionUpgradeHookInit([]corev1.VolumeMount{hooksVolumeMount})), buildTLSPodSpecModification(mdb), buildScramPodSpecModification(mdb), ), diff --git a/scripts/ci/config.json b/scripts/ci/config.json index bfa0385c3..405ef778f 100644 --- a/scripts/ci/config.json +++ b/scripts/ci/config.json @@ -3,6 +3,6 @@ "repo_url": "quay.io/mongodb", "operator_image": "community-operator-dev", "e2e_image": "community-operator-e2e", - "prestop_hook_image": "community-operator-pre-stop-hook", + "version_upgrade_hook_image": "community-operator-version-upgrade-post-start-hook", "testrunner_image": "community-operator-testrunner" } diff --git a/scripts/dev/dev_config.py b/scripts/dev/dev_config.py index dfadb3181..8529ec85a 100644 --- a/scripts/dev/dev_config.py +++ b/scripts/dev/dev_config.py @@ -35,8 +35,8 @@ def e2e_image(self) -> str: return self._config["e2e_image"] @property - def prestop_hook_image(self) -> str: - return self._config["prestop_hook_image"] + def version_upgrade_hook_image(self) -> str: + return self._config["version_upgrade_hook_image"] @property def testrunner_image(self) -> str: diff --git a/scripts/dev/e2e.py b/scripts/dev/e2e.py index 9318ed61f..06284c620 100644 --- a/scripts/dev/e2e.py +++ b/scripts/dev/e2e.py @@ -123,11 +123,11 @@ def build_and_push_e2e(repo_url: str, tag: str, path: str) -> None: build_and_push_image(repo_url, tag, path, "e2e") -def build_and_push_prehook(repo_url: str, tag: str, path: str) -> None: +def build_and_push_version_upgrade_hook(repo_url: str, tag: str, path: str) -> None: """ - build_and_push_prehook builds and pushes the pre-stop-hook image. + build_and_push_version_upgrade_hook builds and pushes the pod-deleter image. """ - build_and_push_image(repo_url, tag, path, "prehook") + build_and_push_image(repo_url, tag, path, "versionhook") def _delete_testrunner_pod(config_file: str) -> None: @@ -216,8 +216,8 @@ def _get_testrunner_pod_body( "./runner", "--operatorImage", f"{dev_config.repo_url}/{dev_config.operator_image}:{tag}", - "--preHookImage", - f"{dev_config.repo_url}/{dev_config.prestop_hook_image}:{tag}", + "--versionUpgradeHookImage", + f"{dev_config.repo_url}/{dev_config.version_upgrade_hook_image}:{tag}", "--testImage", f"{dev_config.repo_url}/{dev_config.e2e_image}:{tag}", f"--test={test}", @@ -240,7 +240,7 @@ def parse_args() -> argparse.Namespace: ) parser.add_argument( "--build-images", - help="Build testrunner, e2e and prestop-hook images", + help="Build testrunner, e2e and version upgrade hook images", action="store_true", ) parser.add_argument( @@ -284,10 +284,10 @@ def build_and_push_images(args: argparse.Namespace, dev_config: DevConfig) -> No "{}/{}:{}".format(dev_config.repo_url, dev_config.e2e_image, args.tag), ".", ) - build_and_push_prehook( + build_and_push_version_upgrade_hook( dev_config.repo_url, "{}/{}:{}".format( - dev_config.repo_url, dev_config.prestop_hook_image, args.tag + dev_config.repo_url, dev_config.version_upgrade_hook_image, args.tag ), ".", ) diff --git a/scripts/dev/templates/Dockerfile.prehook b/scripts/dev/templates/Dockerfile.prehook deleted file mode 100644 index 14bd00544..000000000 --- a/scripts/dev/templates/Dockerfile.prehook +++ /dev/null @@ -1,18 +0,0 @@ -# TODO: template this -FROM golang AS builder - -ENV GO111MODULE=on -ENV GOFLAGS="-mod=vendor" -ENV GOPATH "" - -COPY go.mod go.sum ./ -RUN go mod download - -ADD . . - -RUN go mod vendor && \ - go build -o build/_output/pre-stop-hook -mod=vendor github.com/mongodb/mongodb-kubernetes-operator/cmd/prestop - -FROM busybox - -COPY --from=builder /go/build/_output/pre-stop-hook /pre-stop-hook diff --git a/scripts/dev/templates/Dockerfile.versionhook b/scripts/dev/templates/Dockerfile.versionhook new file mode 100644 index 000000000..a3df35627 --- /dev/null +++ b/scripts/dev/templates/Dockerfile.versionhook @@ -0,0 +1,18 @@ +# TODO: template this +FROM golang AS builder + +ENV GO111MODULE=on +ENV GOFLAGS="-mod=vendor" +ENV GOPATH "" + +COPY go.mod go.sum ./ +RUN go mod download + +ADD . . + +RUN go mod vendor && \ + go build -o build/_output/version-upgrade-hook -mod=vendor github.com/mongodb/mongodb-kubernetes-operator/cmd/versionhook + +FROM busybox + +COPY --from=builder /go/build/_output/version-upgrade-hook /version-upgrade-hook From a21e62672431721fdf50a2d829cb0d008a2cd9f2 Mon Sep 17 00:00:00 2001 From: Fabian Lindfors Date: Tue, 14 Jul 2020 23:38:21 +0200 Subject: [PATCH 02/11] Update Evergreen --- .evergreen.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen.yml b/.evergreen.yml index 9c2a416aa..b501e2964 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -183,7 +183,7 @@ tasks: - func: setup_virtualenv - func: build_and_push_image vars: - image: quay.io/mongodb/community-operator-pre-stop-hook:${version_id} + image: quay.io/mongodb/community-operator-version-upgrade-post-start-hook:${version_id} image_type: versionhook - name: build_testrunner_image From cee9022f8e9c97f8596e831ee3e6a91d890b19a8 Mon Sep 17 00:00:00 2001 From: Fabian Lindfors Date: Wed, 15 Jul 2020 09:46:55 +0200 Subject: [PATCH 03/11] Update hook log path --- cmd/versionhook/main.go | 2 +- pkg/controller/mongodb/mongodb_controller.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/versionhook/main.go b/cmd/versionhook/main.go index 72fe1104e..407cb5f04 100644 --- a/cmd/versionhook/main.go +++ b/cmd/versionhook/main.go @@ -20,7 +20,7 @@ import ( const ( agentStatusFilePathEnv = "AGENT_STATUS_FILEPATH" - logFilePathEnv = "POD_DELETER_LOG_PATH" + logFilePathEnv = "VERSION_UPGRADE_HOOK_LOG_PATH" defaultNamespace = "default" diff --git a/pkg/controller/mongodb/mongodb_controller.go b/pkg/controller/mongodb/mongodb_controller.go index 53a987baf..89ca6e436 100644 --- a/pkg/controller/mongodb/mongodb_controller.go +++ b/pkg/controller/mongodb/mongodb_controller.go @@ -564,7 +564,7 @@ mongod -f /data/automation-mongod.conf ; }, corev1.EnvVar{ Name: versionUpgradeHookLogFilePathEnv, - Value: "/hooks/pod-deleter.log", + Value: "/hooks/version-upgrade-hook.log", }, ), container.WithVolumeMounts(volumeMounts), From 8d57c67976808c5e15ce3438df1c654f8cd121be Mon Sep 17 00:00:00 2001 From: Fabian Lindfors Date: Wed, 15 Jul 2020 09:56:56 +0200 Subject: [PATCH 04/11] Fix outdated comment --- cmd/testrunner/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/testrunner/main.go b/cmd/testrunner/main.go index b0b1f81d1..f9da43a33 100644 --- a/cmd/testrunner/main.go +++ b/cmd/testrunner/main.go @@ -219,7 +219,7 @@ func withEnvVar(key, val string) func(obj runtime.Object) { } } -// withVersionUpgradeHookImage sets the value of the POD_DELETER_IMAGE +// withVersionUpgradeHookImage sets the value of the VERSION_UPGRADE_HOOK_IMAGE // EnvVar from first container to `image`. The EnvVar is updated // if it exists. Or appended if there is no EnvVar with this `Name`. func withVersionUpgradeHookImage(image string) func(runtime.Object) { From 38e6f75d0e3f1d6688d7b384715319d15c365410 Mon Sep 17 00:00:00 2001 From: Fabian Lindfors Date: Wed, 15 Jul 2020 11:06:51 +0200 Subject: [PATCH 05/11] Improve version hook log messages --- cmd/versionhook/main.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/cmd/versionhook/main.go b/cmd/versionhook/main.go index 407cb5f04..cecc346db 100644 --- a/cmd/versionhook/main.go +++ b/cmd/versionhook/main.go @@ -29,7 +29,7 @@ const ( ) func main() { - fmt.Println("Calling pre-stop hook!") + fmt.Println("Calling version change post-start hook!") if err := ensureEnvironmentVariables(logFilePathEnv, agentStatusFilePathEnv); err != nil { zap.S().Fatal("Not all required environment variables are present: %s", err) @@ -41,7 +41,15 @@ func main() { logger.Info("Waiting for agent health status...") health, err := waitForAgentHealthStatus() if err != nil { - logger.Errorf("Error getting the agent health file: %s", err) + // If the pod has just restarted then the status file will not exist. + // In that case we return and let mongod start again. + if os.IsNotExist(err) { + logger.Info("Agent health status file not found, mongod will start") + } else { + logger.Errorf("Error getting the agent health file: %s", err) + } + + return } shouldDelete, err := shouldDeletePod(health) @@ -63,10 +71,10 @@ func main() { // is killed by Kubernetes, bringing the new container image // into play. var quit = make(chan struct{}) - logger.Info("A Pod killed itself, waiting...") + logger.Info("Pod killed itself, waiting...") <-quit } else { - logger.Info("Pod should not be deleted, container will restart...") + logger.Info("Pod should not be deleted, mongod started") } } From c2920c02979c959c99382992dcd7bfc3645bb486 Mon Sep 17 00:00:00 2001 From: Fabian Lindfors Date: Wed, 15 Jul 2020 11:07:08 +0200 Subject: [PATCH 06/11] Switch from pre-stop hook to post-start hook with mongod as PID 1 --- pkg/controller/mongodb/mongodb_controller.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/pkg/controller/mongodb/mongodb_controller.go b/pkg/controller/mongodb/mongodb_controller.go index 89ca6e436..04d300192 100644 --- a/pkg/controller/mongodb/mongodb_controller.go +++ b/pkg/controller/mongodb/mongodb_controller.go @@ -540,15 +540,17 @@ func mongodbContainer(version string, volumeMounts []corev1.VolumeMount) contain mongoDbCommand := []string{ "/bin/sh", "-c", - // we execute the pre-stop hook once the mongod has been gracefully shut down by the agent. - `while [ ! -f /data/automation-mongod.conf ]; do sleep 3 ; done ; sleep 2 ; -# start mongod with this configuration -mongod -f /data/automation-mongod.conf ; + ` +# run post-start hook to handle version changes. +# during a version change, mongod will be stopped and the +# container will be restarted, at which points this hook runs. +/hooks/version-upgrade-hook + +# wait for config to be created by the agent +while [ ! -f /data/automation-mongod.conf ]; do sleep 3 ; done ; sleep 2 ; -# start the pre-stop-hook to restart the Pod when needed -# If the Pod does not require to be restarted, the pre-stop-hook will -# exit(0) for Kubernetes to restart the container. -/hooks/version-upgrade-hook; +# start mongod with this configuration +exec mongod -f /data/automation-mongod.conf ; `, } From b107ac2ef6652acb1fa88d432928abdd07264f1d Mon Sep 17 00:00:00 2001 From: Fabian Lindfors Date: Wed, 15 Jul 2020 13:44:09 +0200 Subject: [PATCH 07/11] Switch to logging to stdout --- cmd/versionhook/main.go | 32 +++++--------------- pkg/controller/mongodb/mongodb_controller.go | 11 ++----- 2 files changed, 10 insertions(+), 33 deletions(-) diff --git a/cmd/versionhook/main.go b/cmd/versionhook/main.go index cecc346db..97b1df691 100644 --- a/cmd/versionhook/main.go +++ b/cmd/versionhook/main.go @@ -20,7 +20,6 @@ import ( const ( agentStatusFilePathEnv = "AGENT_STATUS_FILEPATH" - logFilePathEnv = "VERSION_UPGRADE_HOOK_LOG_PATH" defaultNamespace = "default" @@ -29,14 +28,14 @@ const ( ) func main() { - fmt.Println("Calling version change post-start hook!") + logger := setupLogger() - if err := ensureEnvironmentVariables(logFilePathEnv, agentStatusFilePathEnv); err != nil { - zap.S().Fatal("Not all required environment variables are present: %s", err) - os.Exit(1) - } + logger.Info("Running version change post-start hook") - logger := setupLogger() + if statusPath := os.Getenv(agentStatusFilePathEnv); statusPath == "" { + logger.Fatalf(`Required environment variable "%s" not set`, agentStatusFilePathEnv) + return + } logger.Info("Waiting for agent health status...") health, err := waitForAgentHealthStatus() @@ -78,25 +77,8 @@ func main() { } } -func ensureEnvironmentVariables(requiredEnvVars ...string) error { - var missingEnvVars []string - for _, envVar := range requiredEnvVars { - if val := os.Getenv(envVar); val == "" { - missingEnvVars = append(missingEnvVars, envVar) - } - } - if len(missingEnvVars) > 0 { - return fmt.Errorf("missing envars: %s", strings.Join(missingEnvVars, ",")) - } - return nil -} - func setupLogger() *zap.SugaredLogger { - cfg := zap.NewDevelopmentConfig() - cfg.OutputPaths = []string{ - os.Getenv(logFilePathEnv), - } - log, err := cfg.Build() + log, err := zap.NewDevelopment() if err != nil { zap.S().Errorf("Error building logger config: %s", err) os.Exit(1) diff --git a/pkg/controller/mongodb/mongodb_controller.go b/pkg/controller/mongodb/mongodb_controller.go index 04d300192..8e3feb236 100644 --- a/pkg/controller/mongodb/mongodb_controller.go +++ b/pkg/controller/mongodb/mongodb_controller.go @@ -42,10 +42,9 @@ import ( ) const ( - agentImageEnv = "AGENT_IMAGE" - versionUpgradeHookImageEnv = "VERSION_UPGRADE_HOOK_IMAGE" - agentHealthStatusFilePathEnv = "AGENT_STATUS_FILEPATH" - versionUpgradeHookLogFilePathEnv = "VERSION_UPGRADE_HOOK_LOG_PATH" + agentImageEnv = "AGENT_IMAGE" + versionUpgradeHookImageEnv = "VERSION_UPGRADE_HOOK_IMAGE" + agentHealthStatusFilePathEnv = "AGENT_STATUS_FILEPATH" AutomationConfigKey = "automation-config" agentName = "mongodb-agent" @@ -564,10 +563,6 @@ exec mongod -f /data/automation-mongod.conf ; Name: agentHealthStatusFilePathEnv, Value: "/healthstatus/agent-health-status.json", }, - corev1.EnvVar{ - Name: versionUpgradeHookLogFilePathEnv, - Value: "/hooks/version-upgrade-hook.log", - }, ), container.WithVolumeMounts(volumeMounts), ) From 160ab4a64bcfd218c8eb73de6b2d7f104737af4b Mon Sep 17 00:00:00 2001 From: Fabian Lindfors Date: Wed, 15 Jul 2020 13:48:17 +0200 Subject: [PATCH 08/11] Move version change hook from comment to architecture.md --- architecture.md | 4 ++-- pkg/controller/mongodb/mongodb_controller.go | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/architecture.md b/architecture.md index 63d656be2..a72483ecb 100644 --- a/architecture.md +++ b/architecture.md @@ -16,7 +16,7 @@ You create and update MongoDB resources by defining a MongoDB resource definitio 1. Writes the Automation configuration as a [ConfigMap](https://kubernetes.io/docs/concepts/configuration/configmap/) and mounts it to each pod. 1. Creates one [init container](https://kubernetes.io/docs/concepts/workloads/pods/init-containers/) and two [containers](https://kubernetes.io/docs/concepts/containers/overview/) in each pod: - - An init container which copies the `cmd/versionhook` binary to the main `mongod` container. This is used as a [pre-stop hook](https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/) for [version upgrades](#example-mongodb-version-upgrade). + - An init container which copies the `cmd/versionhook` binary to the main `mongod` container. This is run before `mongod` starts to handle [version upgrades](#example-mongodb-version-upgrade). - A container for the [`mongod`](https://docs.mongodb.com/manual/reference/program/mongod/index.html) process binary. `mongod` is the primary daemon process for the MongoDB system. It handles data requests, manages data access, and performs background management operations. @@ -61,7 +61,7 @@ When you update the MongoDB version in your resource definition and reapply it t 1. The MongoDB Agent chooses the first pod to upgrade and stops the `mongod` process using a local connection and [`db.shutdownServer`](https://docs.mongodb.com/manual/reference/method/db.shutdownServer/#db.shutdownServer). -1. A pre-stop hook on the database container checks the state of the MongoDB Agent. If the MongoDB Agent expects the `mongod` process to start with a new version, the hook uses a Kubernetes API call to delete the pod. +1. Kubernetes will restart the `mongod` container causing the version change hook to run and check the state of the MongoDB Agent. If the MongoDB Agent expects the `mongod` process to start with a new version, the hook uses a Kubernetes API call to delete the pod. 1. The Kubernetes Controller downloads the target version of MongoDB from its default docker registry and restarts the pod with the target version of `mongod` in the database container. diff --git a/pkg/controller/mongodb/mongodb_controller.go b/pkg/controller/mongodb/mongodb_controller.go index 8e3feb236..d6c5ad051 100644 --- a/pkg/controller/mongodb/mongodb_controller.go +++ b/pkg/controller/mongodb/mongodb_controller.go @@ -540,9 +540,7 @@ func mongodbContainer(version string, volumeMounts []corev1.VolumeMount) contain "/bin/sh", "-c", ` -# run post-start hook to handle version changes. -# during a version change, mongod will be stopped and the -# container will be restarted, at which points this hook runs. +# run post-start hook to handle version changes /hooks/version-upgrade-hook # wait for config to be created by the agent From 37e808589b7a95d6c415cd5bd75cff17b01034ef Mon Sep 17 00:00:00 2001 From: Fabian Lindfors Date: Wed, 15 Jul 2020 13:52:19 +0200 Subject: [PATCH 09/11] Remove redundant "hook" from name --- pkg/controller/mongodb/mongodb_controller.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/controller/mongodb/mongodb_controller.go b/pkg/controller/mongodb/mongodb_controller.go index d6c5ad051..b83d4368c 100644 --- a/pkg/controller/mongodb/mongodb_controller.go +++ b/pkg/controller/mongodb/mongodb_controller.go @@ -528,7 +528,7 @@ func mongodbAgentContainer(volumeMounts []corev1.VolumeMount) container.Modifica func versionUpgradeHookInit(volumeMount []corev1.VolumeMount) container.Modification { return container.Apply( container.WithName(versionUpgradeHookName), - container.WithCommand([]string{"cp", "version-upgrade-hook", "/hooks/version-upgrade-hook"}), + container.WithCommand([]string{"cp", "version-upgrade-hook", "/hooks/version-upgrade"}), container.WithImage(os.Getenv(versionUpgradeHookImageEnv)), container.WithImagePullPolicy(corev1.PullAlways), container.WithVolumeMounts(volumeMount), @@ -541,7 +541,7 @@ func mongodbContainer(version string, volumeMounts []corev1.VolumeMount) contain "-c", ` # run post-start hook to handle version changes -/hooks/version-upgrade-hook +/hooks/version-upgrade # wait for config to be created by the agent while [ ! -f /data/automation-mongod.conf ]; do sleep 3 ; done ; sleep 2 ; From a947848517beaca7791664ba8d43514381bfb437 Mon Sep 17 00:00:00 2001 From: Fabian Lindfors Date: Wed, 15 Jul 2020 14:35:24 +0200 Subject: [PATCH 10/11] Fix failing unit test --- pkg/controller/mongodb/build_statefulset_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/controller/mongodb/build_statefulset_test.go b/pkg/controller/mongodb/build_statefulset_test.go index f79c326f5..fdf65c46b 100644 --- a/pkg/controller/mongodb/build_statefulset_test.go +++ b/pkg/controller/mongodb/build_statefulset_test.go @@ -45,7 +45,7 @@ func assertStatefulSetIsBuiltCorrectly(t *testing.T, mdb mdbv1.MongoDB, sts *app assert.Equal(t, appsv1.RollingUpdateStatefulSetStrategyType, sts.Spec.UpdateStrategy.Type) assert.Equal(t, operatorServiceAccountName, sts.Spec.Template.Spec.ServiceAccountName) assert.Len(t, sts.Spec.Template.Spec.Containers[0].Env, 1) - assert.Len(t, sts.Spec.Template.Spec.Containers[1].Env, 2) + assert.Len(t, sts.Spec.Template.Spec.Containers[1].Env, 1) agentContainer := sts.Spec.Template.Spec.Containers[0] assert.Equal(t, "agent-image", agentContainer.Image) From bb10a2d65492644f12872f92504a08cfdd508f61 Mon Sep 17 00:00:00 2001 From: Fabian Lindfors Date: Thu, 16 Jul 2020 15:52:52 +0200 Subject: [PATCH 11/11] Update outdated comment --- scripts/dev/e2e.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/dev/e2e.py b/scripts/dev/e2e.py index 06284c620..7b5d9cf7d 100644 --- a/scripts/dev/e2e.py +++ b/scripts/dev/e2e.py @@ -125,7 +125,7 @@ def build_and_push_e2e(repo_url: str, tag: str, path: str) -> None: def build_and_push_version_upgrade_hook(repo_url: str, tag: str, path: str) -> None: """ - build_and_push_version_upgrade_hook builds and pushes the pod-deleter image. + build_and_push_version_upgrade_hook builds and pushes the version upgrade hook image. """ build_and_push_image(repo_url, tag, path, "versionhook")