New release pipeline in operator (ydb-platform#260)

Author: Jorres

.changes/header.tpl.md

@@ -0,0 +1 @@
+# Changelog

.changes/unreleased/.gitkeep

@@ -0,0 +1,3 @@
+## v0.5.31 - 2024-11-04
+### Added
+* Initialized a changelog

.changes/v0.5.31.md

@@ -0,0 +1,26 @@
+changesDir: .changes
+unreleasedDir: unreleased
+headerPath: header.tpl.md
+changelogPath: CHANGELOG.md
+versionExt: md
+versionFormat: '## {{.Version}} - {{.Time.Format "2006-01-02"}}'
+kindFormat: '### {{.Kind}}'
+changeFormat: '* {{.Body}}'
+kinds:
+    - label: Added
+      auto: minor
+    - label: Changed
+      auto: major
+    - label: Deprecated
+      auto: minor
+    - label: Removed
+      auto: major
+    - label: Fixed
+      auto: patch
+    - label: Security
+      auto: patch
+newlines:
+    afterChangelogHeader: 1
+    beforeChangelogVersion: 1
+    endOfVersion: 1
+envPrefix: CHANGIE_

.changie.yaml

@@ -0,0 +1,58 @@
+name: create-release-pr
+
+on:
+  workflow_dispatch:
+
+jobs:
+  create-release-pr:
+    runs-on: ubuntu-latest
+    steps:
+    - name: checkout
+      uses: actions/checkout@v4
+
+    - name: batch-changes
+      uses: miniscruff/changie-action@v2
+      with:
+        version: latest
+        args: batch patch
+
+    - name: merge-changes
+      uses: miniscruff/changie-action@v2
+      with:
+        version: latest
+        args: merge
+
+    - name: print the latest version
+      id: latest
+      uses: miniscruff/changie-action@v2
+      with:
+        version: latest
+        args: latest
+
+    - name: print the latest version without "v"
+      id: latest-no-v
+      uses: miniscruff/changie-action@v2
+      with:
+        version: latest
+        args: latest --remove-prefix
+
+    - name: check-chart-version
+      run: |
+        VERSION=${{ steps.latest-no-v.outputs.output }}
+        APP_VERSION=$(grep '^appVersion:' ./deploy/ydb-operator/Chart.yaml | awk '{print $2}' | tr -d '"')
+        CHART_VERSION=$(grep '^version:' ./deploy/ydb-operator/Chart.yaml | awk '{print $2}' | tr -d '"')
+
+        if [ "$APP_VERSION" != "$VERSION" ] || [ "$CHART_VERSION" != "$VERSION" ]; then
+          echo "Version mismatch: appVersion ($APP_VERSION) or version ($CHART_VERSION) does not match expected version ($VERSION). You most likely forgot to bump the version in Chart.yaml, please do so."
+          exit 1
+        else
+          echo "Version matches: appVersion ($APP_VERSION) and version ($CHART_VERSION) are both $VERSION."
+        fi
+
+    - name: Create Pull Request
+      uses: peter-evans/create-pull-request@v7
+      with:
+        title: Release ${{ steps.latest.outputs.output }}
+        branch: release/${{ steps.latest.outputs.output }}
+        commit-message: Release ${{ steps.latest.outputs.output }}
+        token: ${{ github.token }}

.github/scripts/check-work-copy-equals-to-committed.sh

@@ -12,35 +12,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: checkout
-        uses: actions/checkout@v3
-      - name: setup-go
-        uses: actions/setup-go@v3
-        with:
-          go-version: '1.20'
+        uses: actions/checkout@v4
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v2
-        with:
-          version: v1.58.1
-  code-format-check:
-    concurrency:
-      group: lint-autoformat-${{ github.head_ref || github.ref_name }}
-      cancel-in-progress: true
-    runs-on: ubuntu-latest
-    steps:
-      - name: checkout
-        uses: actions/checkout@v3
-      - name: setup-go
-        uses: actions/setup-go@v3
+        uses: golangci/golangci-lint-action@v6
         with:
-          go-version: '1.20'
-      - name: install-utilities
-        run: |
-          go install mvdan.cc/gofumpt@v0.5.0
-          go install github.com/rinchsan/gosimports/cmd/gosimports@v0.3.8
-      - name: format all files with auto-formatter
-        run: bash ./.github/scripts/format-all-go-code.sh "$PWD"
-      - name: check-repository-diff
-        run: bash ./.github/scripts/check-work-copy-equals-to-committed.sh "auto-format broken"
+          version: v1.61.0
   run-unit-tests:
     concurrency:
       group: run-unit-tests-${{ github.head_ref || github.ref_name }}

.github/scripts/format-all-go-code.sh

@@ -1,29 +1,45 @@
 name: upload-artifacts
+
 on:
   push:
     branches:
       - master
+    paths:
+      - 'CHANGELOG.md'
+  workflow_dispatch:
+
 jobs:
-  tag-job:
+  tag-and-release:
     runs-on: ubuntu-latest
-    outputs:
-      tagcreated: ${{steps.tag-step.outputs.tagcreated}}
     steps:
-    - uses: actions/checkout@v3
-    - id: tag-step
-      uses: butlerlogic/action-autotag@1.1.2
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    - uses: actions/checkout@v4
+
+    - name: get-latest-version
+      uses: miniscruff/changie-action@v2
       with:
-        strategy: regex
-        root: "./deploy/ydb-operator/Chart.yaml"
-        regex_pattern: 'version:[\s]*(["]?[0-9\.]{3,}["]?)'
-  upload-artifacts:
-    runs-on: ubuntu-latest
-    needs: tag-job
-    if: ${{ needs.tag-job.outputs.tagcreated == 'yes' }}
-    steps:
-    - uses: actions/checkout@v3
+        version: latest
+        args: latest
+    - name: get-latest-no-v-version
+      uses: miniscruff/changie-action@v2
+      with:
+        version: latest
+        # Echoes the same version as previous step, but without "v" prefix. 
+        # Is used as a docker image tag in the release step.
+        # E.g. "v0.5.31" -> "0.5.31"
+        args: latest --remove-prefix
+
+    - name: update-chart-version-with-release-version
+      run: |
+        RELEASE_VERSION=${{ steps.get-latest-no-v-version.outputs.output }}
+        sed -i 's/<IS_SUBSTITUTED_IN_CI_RELEASE_JOB>/'"$RELEASE_VERSION"'/g' ./deploy/ydb-operator/Chart.yaml
+
+    - name: create-tag
+      uses: mathieudutour/github-tag-action@v6.2
+      with:
+        tag_prefix: ""
+        custom_tag: ${{ steps.get-latest-version.outputs.output }}
+        github_token: ${{ github.token }}
+
     - name: install-dependencies
       run: |
         HELM_PKG="helm-v3.10.3-linux-amd64.tar.gz"

.github/scripts/format-go-code.sh

@@ -56,7 +56,7 @@ linters-settings:
     check-blank: false
   govet:
     # report about shadowed variables
-    check-shadowing: true
+    shadow: true
     fieldalignment: true
   golint:
     # minimal confidence for issues, default is 0.8
@@ -76,6 +76,7 @@ linters-settings:
   gosec:
     excludes:
      - G101
+     - G115
   fieldalignment:
     # print struct with more effective memory layout or not, false by default
     suggest-new: true
@@ -129,6 +130,8 @@ linters-settings:
 linters:
   disable-all: true
   enable:
+#    - cyclop
+    # - depguardgolang
     - dogsled
     - errcheck
     - errorlint

.github/scripts/is_autogenerated_file.sh

@@ -0,0 +1,6 @@
+# Changelog
+
+
+## v0.5.31 - 2024-11-04
+### Added
+* Initialized a changelog

.github/workflows/create-release-pr.yml

@@ -1,4 +1,4 @@
-[![check-pr](https://github.com/ydb-platform/ydb-kubernetes-operator/actions/workflows/check-pr.yml/badge.svg)](https://github.com/ydb-platform/ydb-kubernetes-operator/actions/workflows/check-pr.yml)
+[![run-tests](https://github.com/ydb-platform/ydb-kubernetes-operator/actions/workflows/run-tests.yml/badge.svg)](https://github.com/ydb-platform/ydb-kubernetes-operator/actions/workflows/run-tests.yml)
 [![upload-artifacts](https://github.com/ydb-platform/ydb-kubernetes-operator/actions/workflows/upload-artifacts.yml/badge.svg)](https://github.com/ydb-platform/ydb-kubernetes-operator/actions/workflows/upload-artifacts.yml)
 
 # YDB Kubernetes Operator

.github/workflows/run-tests.yml

@@ -15,10 +15,10 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.5.30
+version: "0.5.31"
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "0.5.30"
+appVersion: "0.5.31"

.github/workflows/upload-artifacts.yml

@@ -1,16 +1,8 @@
 This document attempts to be some sort of a table of contents:
 
-## Operator release flow
 
-See [here](./release-flow.md).
+## Working on a feature
 
-## Writing and running tests
-
-See [here](./tests.md).
-
-## Storage
-
-#### [Storage state machine schema](./storage-state-machine-schema.md)
-
-This describes which states the Storage object has and in what order they are
-traversed.
+- don't forget to use `changie` to generate a changelog entry, whenever you complete a feature
+- see [here](./tests.md) to learn how to run tests
+- see [here](./tests.md) to learn how to release a new version

.golangci.yml

@@ -2,34 +2,15 @@
 
 #### How and when the operator version is changed
 
-The single source of truth is the version number in
-[Chart.yaml](https://github.com/ydb-platform/ydb-kubernetes-operator/blob/master/deploy/ydb-operator/Chart.yaml#L18)
-file.
-
-It is incremented according to semver practices. Essentially, it is incremented every
-time any change is made into either chart or the operator code.
-
-For the contrast, changing some details of Github workflows or rewriting the docs
-does not initiate a new release.
-
-#### What the CI does
-
-When you increment the version in `Chart.yaml` and your PR is merged into master, tag
-is automatically extracted from the Chart like this:
-
-```
-"version: 0.4.22"  ->  "0.4.22"
-```
-
-If the version is new (no previous commit was tagged with this version), then:
-
-- this merge commit gets tagged with version extracted from `Chart.yaml`;
-- new docker image is built and uploaded to `cr.yandex/yc/ydb-kubernetes-operator`;
-- new chart version is uploaded to https://charts.ydb.tech.
-
-#### What if I forget to bump up the chart version?
-
-Then these changes won't get automatically tagged, new version won't be built and
-uploaded and you most likely will notice it immediately when you'll want to use an
-updated version. Then you make another PR with a chart version bump and get your
-changes rebuilt.
+The single source of truth is the changelog. 
+
+Currently, version is updated when a developer decides to release a new version by manually invoking
+`create-release-pr` workflow in Github Actions:
+
+- invoke `create-release-pr` workflow
+- `changie` tool automatically bumps the version and generates an updated CHANGELOG.md
+- if a generated `CHANGELOG.md` looks okay, just merge it
+- the `upload-artifacts` workflow will:
+    - substitute the latest version in `Chart.yaml`
+    - build artifacts (docker image and helm chart) and upload them to all configured registries
+    - create a new Github release 

CHANGELOG.md

@@ -58,6 +58,7 @@ func InPod(
 	if err != nil {
 		return stdout.String(), stderr.String(), errors.Wrapf(
 			err,
+			//nolint:govet  // TODO @jorres figure out why non-const error messages are not recommended
 			fmt.Sprintf("failed to stream execution results back, stdout:\n\"%s\"stderr:\n\"%s\"", stdout.String(), stderr.String()),
 		)
 	}