MCLOUD-6023: Investigate Redis scan keys eviction

Console/Command/CacheEvict.php

@@ -0,0 +1,57 @@
+<?php
+/**
+ * Copyright © Magento, Inc. All rights reserved.
+ * See COPYING.txt for license details.
+ */
+declare(strict_types=1);
+
+namespace Magento\CloudComponents\Console\Command;
+
+use Magento\CloudComponents\Model\Cache\Evictor;
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Output\OutputInterface;
+
+/**
+ * Performs force key eviction with a "scan" command.
+ */
+class CacheEvict extends Command
+{
+    /**
+     * @var Evictor
+     */
+    private $evictor;
+
+    /**
+     * @param Evictor $evictor
+     */
+    public function __construct(Evictor $evictor)
+    {
+        $this->evictor = $evictor;
+
+        parent::__construct('cache:evict');
+    }
+
+    /**
+     * @inheritDoc
+     */
+    protected function configure()
+    {
+        $this->setDescription('Evicts unused keys by performing scan command');
+    }
+
+    /**
+     * @inheritDoc
+     */
+    public function execute(InputInterface $input, OutputInterface $output)
+    {
+        $output->writeln('Begin scanning of cache keys');
+
+        $count = $this->evictor->evict();
+
+        $output->writeln(sprintf(
+            'Total scanned keys: %s',
+            $count
+        ));
+    }
+}

Cron/Evict.php

@@ -0,0 +1,59 @@
+<?php
+/**
+ * Copyright © Magento, Inc. All rights reserved.
+ * See COPYING.txt for license details.
+ */
+declare(strict_types=1);
+
+namespace Magento\CloudComponents\Cron;
+
+use Magento\CloudComponents\Model\Cache\Evictor;
+use Magento\Framework\App\DeploymentConfig;
+use Psr\Log\LoggerInterface;
+
+/**
+ * The cron cprocess to evict keys.
+ */
+class Evict
+{
+    /**
+     * @var Evictor
+     */
+    private $evictor;
+
+    /**
+     * @var DeploymentConfig
+     */
+    private $deploymentConfig;
+
+    /**
+     * @var LoggerInterface
+     */
+    private $logger;
+
+    /**
+     * @param Evictor $evictor
+     * @param DeploymentConfig $deploymentConfig
+     * @param LoggerInterface $logger
+     */
+    public function __construct(Evictor $evictor, DeploymentConfig $deploymentConfig, LoggerInterface $logger)
+    {
+        $this->evictor = $evictor;
+        $this->deploymentConfig = $deploymentConfig;
+        $this->logger = $logger;
+    }
+
+    /**
+     * Perform keys eviction.
+     */
+    public function execute()
+    {
+        if (!$this->deploymentConfig->get(Evictor::CONFIG_PATH_ENABLED)) {
+            $this->logger->info('Keys eviction is disabled');
+
+            return;
+        }
+
+        $this->evictor->evict();
+    }
+}

Model/Cache/Evictor.php

@@ -0,0 +1,120 @@
+<?php
+/**
+ * Copyright © Magento, Inc. All rights reserved.
+ * See COPYING.txt for license details.
+ */
+declare(strict_types=1);
+
+namespace Magento\CloudComponents\Model\Cache;
+
+use Magento\Framework\App\Cache\Type\FrontendPool;
+use Magento\Framework\App\DeploymentConfig;
+use Psr\Log\LoggerInterface;
+use Credis_Client as Client;
+use Cm_Cache_Backend_Redis as Backend;
+
+/**
+ * Performs force key eviction with a "scan" command.
+ */
+class Evictor
+{
+    const DEFAULT_EVICTION_LIMIT = 10000;
+    const DEFAULT_SLEEP_TIMEOUT = 20000;
+    const CONFIG_PATH_ENABLED = 'cache_evict/enabled';
+    const CONFIG_PATH_LIMIT = 'cache_evict/limit';
+
+    /**
+     * @var DeploymentConfig
+     */
+    private $deploymentConfig;
+
+    /**
+     * @var LoggerInterface
+     */
+    private $logger;
+
+    /**
+     * @param DeploymentConfig $deploymentConfig
+     * @param LoggerInterface $logger
+     */
+    public function __construct(DeploymentConfig $deploymentConfig, LoggerInterface $logger)
+    {
+        $this->deploymentConfig = $deploymentConfig;
+        $this->logger = $logger;
+    }
+
+    /**
+     * Evicts all keys using iterator.
+     *
+     * @return int
+     */
+    public function evict(): int
+    {
+        $options = $this->deploymentConfig->getConfigData(FrontendPool::KEY_CACHE)[FrontendPool::KEY_FRONTEND_CACHE]
+            ?? [];
+        $evictedKeys = 0;
+
+        foreach ($options as $name => $cacheConfig) {
+            $this->logger->info(sprintf(
+                'Scanning keys for "%s" database',
+                $name
+            ));
+
+            if (!isset(
+                $cacheConfig['backend_options']['server'],
+                $cacheConfig['backend_options']['port'],
+                $cacheConfig['backend_options']['database']
+            )) {
+                $this->logger->debug(sprintf(
+                    'Cache config for database "%s" config is not valid',
+                    $name
+                ));
+
+                continue;
+            }
+
+            $dbKeys = $this->run(
+                $cacheConfig['backend_options']['server'],
+                $cacheConfig['backend_options']['port'],
+                $cacheConfig['backend_options']['database']
+            );
+            $evictedKeys += $dbKeys;
+
+            $this->logger->info(sprintf('Keys scanned: %s', $dbKeys));
+        }
+
+        return $evictedKeys;
+    }
+
+    /**
+     * @param string $host
+     * @param int $port
+     * @param int $db
+     * @return int
+     */
+    private function run(string $host, int $port, int $db): int
+    {
+        $client = new Client($host, $port, null, '', $db);
+        $evictedKeys = 0;
+
+        do {
+            $keys = $client->scan(
+                $iterator,
+                Backend::PREFIX_KEY . '*',
+                (int)$this->deploymentConfig->get(self::CONFIG_PATH_LIMIT, self::DEFAULT_EVICTION_LIMIT)
+            );
+
+            if ($keys === false) {
+                $this->logger->debug('Reached end');
+            } else {
+                $keysCount = count($keys);
+                $evictedKeys += $keysCount;
+            }
+
+            /* Give Redis some time to handle other requests */
+            usleep(self::DEFAULT_SLEEP_TIMEOUT);
+        } while ($iterator > 0);
+
+        return $evictedKeys;
+    }
+}

composer.json

@@ -6,6 +6,8 @@
   "require": {
     "php": "^7.0",
     "ext-json": "*",
+    "colinmollenhour/cache-backend-redis": "^1.9",
+    "colinmollenhour/credis": "^1.6",
     "newrelic/monolog-enricher": "^1.0"
   },
   "suggest": {
@@ -21,6 +23,9 @@
     "phpunit/phpunit": "^6.2",
     "squizlabs/php_codesniffer": "^3.0"
   },
+  "config": {
+    "sort-packages": true
+  },
   "scripts": {
     "test": [
       "@phpstan",

etc/crontab.xml

@@ -0,0 +1,14 @@
+<?xml version="1.0"?>
+<!--
+/**
+ * Copyright © Magento, Inc. All rights reserved.
+ * See COPYING.txt for license details.
+ */
+-->
+<config xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="urn:magento:module:Magento_Cron:etc/crontab.xsd">
+    <group id="cache_evict">
+        <job name="cache_evict_keys" instance="Magento\CloudComponents\Cron\Evict" method="execute">
+            <schedule>0 */12 * * *</schedule>
+        </job>
+    </group>
+</config>

etc/di.xml

@@ -12,6 +12,7 @@
                 <item name="configShowStoreUrlCommand" xsi:type="object">Magento\CloudComponents\Console\Command\ConfigShowStoreUrlCommand</item>
                 <item name="configShowEntityUrlsCommand" xsi:type="object">Magento\CloudComponents\Console\Command\ConfigShowEntityUrlsCommand</item>
                 <item name="ConfigShowDefaultUrlCommand" xsi:type="object">Magento\CloudComponents\Console\Command\ConfigShowDefaultUrlCommand</item>
+                <item name="CacheEvict" xsi:type="object">Magento\CloudComponents\Console\Command\CacheEvict</item>
             </argument>
         </arguments>
     </type>