ci: prevent CI cache from growing indefinitely (#41814) (#41855)

Previously, the fallback key used for the CircleCI cache could match a
cache indefinitely (as long as `.bazelversion` didn't change). This
would allow the cache to grow quite large, which in turn would lead to
slow-down in CI jobs. See, also, angular/angular-cli#17533 for more
details of the impact of a growing CircleCI cache.

Unfortunately, using something like the lockfile checksum in the
fallback cache key would cause too many cache misses (esp. with
automatic updates via Renovate), again slowing CI down.

(The problem was originally discussed [here][2].)

This commit uses the technique described in [this blogpost][1] to
invalidate the cache monthly. This keeps the extra cache misses low
(essentially once per month per fork), while also preventing the cache
from growing indefinitely.

[1]: https://support.circleci.com/hc/en-us/articles/360012618473-Creating-a-daily-cache
[2]: angular/angular#41467 (comment)

PR Close #41814

PR Close #41855

Author: gkalpak

.circleci/config.yml

@@ -19,23 +19,25 @@ version: 2.1
 # 1) yarn lock file changes --> cached "node_modules" are different.
 # 2) bazel repository definitions change --> cached bazel repositories are different.
 # Windows needs its own cache key because binaries in node_modules are different.
-# **NOTE 1 **: If you change the cache key prefix, also sync the cache_key_fallback to match.
-# **NOTE 2 **: Keep the static part of the cache key as prefix to enable correct fallbacks.
+# **NOTE 1 **: In order to avoid the cache from growing indefinitely and causing slow-downs, we invalidate the cache monthly.
+#              (See https://support.circleci.com/hc/en-us/articles/360012618473-Creating-a-daily-cache.)
+# **NOTE 2 **: If you change the cache key prefix, also sync the cache_key_fallback to match.
+# **NOTE 3 **: Keep the static part of the cache key as prefix to enable correct fallbacks.
 # See https://circleci.com/docs/2.0/caching/#restoring-cache for how prefixes work in CircleCI.
-var_3: &cache_key v4-angular-node-12-{{ checksum ".bazelversion" }}-{{ checksum "yarn.lock" }}-{{ checksum "WORKSPACE" }}-{{ checksum "packages/bazel/package.bzl" }}-{{ checksum "aio/yarn.lock" }}
+var_3: &cache_key v4-angular-node-12-{{ checksum "month.txt" }}-{{ checksum ".bazelversion" }}-{{ checksum "yarn.lock" }}-{{ checksum "WORKSPACE" }}-{{ checksum "packages/bazel/package.bzl" }}-{{ checksum "aio/yarn.lock" }}
 # We invalidate the cache if the Bazel version changes because otherwise the `bazelisk` cache
 # folder will contain all previously used versions and ultimately cause the cache restoring to
 # be slower due to its growing size.
-var_4: &cache_key_fallback v4-angular-node-12-{{ checksum ".bazelversion" }}
+var_4: &cache_key_fallback v4-angular-node-12-{{ checksum "month.txt" }}-{{ checksum ".bazelversion" }}
 
 # Windows needs its own cache key because binaries in node_modules are different.
-var_3_win: &cache_key_win v4-angular-win-node-12-{{ checksum ".bazelversion" }}-{{ checksum "yarn.lock" }}-{{ checksum "WORKSPACE" }}-{{ checksum "packages/bazel/package.bzl" }}-{{ checksum "aio/yarn.lock" }}
-var_4_win: &cache_key_win_fallback v4-angular-win-node-12-{{ checksum ".bazelversion" }}
+var_3_win: &cache_key_win v4-angular-win-node-12-{{ checksum "month.txt" }}-{{ checksum ".bazelversion" }}-{{ checksum "yarn.lock" }}-{{ checksum "WORKSPACE" }}-{{ checksum "packages/bazel/package.bzl" }}-{{ checksum "aio/yarn.lock" }}
+var_4_win: &cache_key_win_fallback v4-angular-win-node-12-{{ checksum "month.txt" }}-{{ checksum ".bazelversion" }}
 
 # Cache key for the `components-repo-unit-tests` job. **Note** when updating the SHA in the
 # cache keys also update the SHA for the "COMPONENTS_REPO_COMMIT" environment variable.
-var_5: &components_repo_unit_tests_cache_key v1-angular-components-09e68db8ed5b1253f2fe38ff954ef0df019fc25a
-var_6: &components_repo_unit_tests_cache_key_fallback v1-angular-components-
+var_5: &components_repo_unit_tests_cache_key v1-angular-components-{{ checksum "month.txt" }}-09e68db8ed5b1253f2fe38ff954ef0df019fc25a
+var_6: &components_repo_unit_tests_cache_key_fallback v1-angular-components-{{ checksum "month.txt" }}
 
 # Workspace initially persisted by the `setup` job, and then enhanced by `build-npm-packages` and
 # `build-ivy-npm-packages`.
@@ -180,13 +182,21 @@ commands:
           name: Setting up alias domain for local host.
           command: echo "127.0.0.1 $SAUCE_LOCALHOST_ALIAS_DOMAIN" | sudo tee -a /etc/hosts
 
+  save_month_to_file:
+    description: Store the current year and month in a file, so that it can be used for computing the cache key.
+    steps:
+      - run:
+          name: Save month to file
+          command: date +%Y-%m > month.txt
+
   # Normally this would be an individual job instead of a command.
   # But startup and setup time for each individual windows job are high enough to discourage
   # many small jobs, so instead we use a command for setup unless the gain becomes significant.
   setup_win:
     description: Setup windows node environment
     steps:
       - checkout
+      - save_month_to_file
       # Install Bazel pre-requisites that aren't in the preconfigured CircleCI Windows VM.
       - run: ./.circleci/windows-env.ps1
       - run: node --version
@@ -220,6 +230,7 @@ jobs:
     executor: default-executor
     steps:
       - checkout
+      - save_month_to_file
       - init_environment
       - run:
           name: Rebase PR on target branch