From ffec377dd2d8b3d091dcc20c33c1354eed89836d Mon Sep 17 00:00:00 2001 From: Ji Lu Date: Fri, 10 Jan 2020 14:44:00 -0600 Subject: [PATCH 1/7] MQE-1918: MFTF AWS Secrets Manager - Local Use --- composer.json | 3 + composer.lock | 148 ++++++++++++++++- .../AwsSecretManagerStorageTest.php | 65 ++++++++ docs/configuration.md | 22 +++ docs/credentials.md | 67 +++++++- etc/config/.env.example | 6 +- .../Handlers/CredentialStore.php | 27 ++- .../SecretStorage/AwsSecretManagerStorage.php | 155 ++++++++++++++++++ .../Handlers/SecretStorage/VaultStorage.php | 2 +- 9 files changed, 483 insertions(+), 12 deletions(-) create mode 100644 dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/SecretStorage/AwsSecretManagerStorageTest.php create mode 100644 src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/AwsSecretManagerStorage.php diff --git a/composer.json b/composer.json index 158287fbf..ca1c02572 100755 --- a/composer.json +++ b/composer.json @@ -11,7 +11,10 @@ "require": { "php": "7.0.2||7.0.4||~7.0.6||~7.1.0||~7.2.0||~7.3.0", "ext-curl": "*", + "ext-json": "*", + "ext-openssl": "*", "allure-framework/allure-codeception": "~1.3.0", + "aws/aws-sdk-php": "^3.132", "codeception/codeception": "~2.4.5", "composer/composer": "^1.4", "consolidation/robo": "^1.0.0", diff --git a/composer.lock b/composer.lock index 8f2fcb8e9..6bfc30ab9 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "59e95cc1ae6311e93111bd7ced180d29", + "content-hash": "2325a3a38edb33b24f6e33bd3009fd8a", "packages": [ { "name": "allure-framework/allure-codeception", @@ -109,6 +109,90 @@ ], "time": "2016-12-07T12:15:46+00:00" }, + { + "name": "aws/aws-sdk-php", + "version": "3.132.2", + "source": { + "type": "git", + "url": "https://github.com/aws/aws-sdk-php.git", + "reference": "f890689c6db27625522ea2e7e9b8420b6fccb063" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/aws/aws-sdk-php/zipball/f890689c6db27625522ea2e7e9b8420b6fccb063", + "reference": "f890689c6db27625522ea2e7e9b8420b6fccb063", + "shasum": "" + }, + "require": { + "ext-json": "*", + "ext-pcre": "*", + "ext-simplexml": "*", + "guzzlehttp/guzzle": "^5.3.3|^6.2.1|^7.0", + "guzzlehttp/promises": "^1.0", + "guzzlehttp/psr7": "^1.4.1", + "mtdowling/jmespath.php": "^2.5", + "php": ">=5.5" + }, + "require-dev": { + "andrewsville/php-token-reflection": "^1.4", + "aws/aws-php-sns-message-validator": "~1.0", + "behat/behat": "~3.0", + "doctrine/cache": "~1.4", + "ext-dom": "*", + "ext-openssl": "*", + "ext-pcntl": "*", + "ext-sockets": "*", + "nette/neon": "^2.3", + "phpunit/phpunit": "^4.8.35|^5.4.3", + "psr/cache": "^1.0", + "psr/simple-cache": "^1.0", + "sebastian/comparator": "^1.2.3" + }, + "suggest": { + "aws/aws-php-sns-message-validator": "To validate incoming SNS notifications", + "doctrine/cache": "To use the DoctrineCacheAdapter", + "ext-curl": "To send requests using cURL", + "ext-openssl": "Allows working with CloudFront private distributions and verifying received SNS messages", + "ext-sockets": "To use client-side monitoring" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "3.0-dev" + } + }, + "autoload": { + "psr-4": { + "Aws\\": "src/" + }, + "files": [ + "src/functions.php" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "Apache-2.0" + ], + "authors": [ + { + "name": "Amazon Web Services", + "homepage": "http://aws.amazon.com" + } + ], + "description": "AWS SDK for PHP - Use Amazon Web Services in your PHP project", + "homepage": "http://aws.amazon.com/sdkforphp", + "keywords": [ + "amazon", + "aws", + "cloud", + "dynamodb", + "ec2", + "glacier", + "s3", + "sdk" + ], + "time": "2020-01-09T19:09:31+00:00" + }, { "name": "behat/gherkin", "version": "v4.4.5", @@ -2542,8 +2626,66 @@ "bcmath", "math" ], + "abandoned": "brick/math", "time": "2017-02-16T16:54:46+00:00" }, + { + "name": "mtdowling/jmespath.php", + "version": "2.5.0", + "source": { + "type": "git", + "url": "https://github.com/jmespath/jmespath.php.git", + "reference": "52168cb9472de06979613d365c7f1ab8798be895" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/jmespath/jmespath.php/zipball/52168cb9472de06979613d365c7f1ab8798be895", + "reference": "52168cb9472de06979613d365c7f1ab8798be895", + "shasum": "" + }, + "require": { + "php": ">=5.4.0", + "symfony/polyfill-mbstring": "^1.4" + }, + "require-dev": { + "composer/xdebug-handler": "^1.2", + "phpunit/phpunit": "^4.8.36|^7.5.15" + }, + "bin": [ + "bin/jp.php" + ], + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "2.5-dev" + } + }, + "autoload": { + "psr-4": { + "JmesPath\\": "src/" + }, + "files": [ + "src/JmesPath.php" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Michael Dowling", + "email": "mtdowling@gmail.com", + "homepage": "https://github.com/mtdowling" + } + ], + "description": "Declaratively specify how to extract elements from a JSON document", + "keywords": [ + "json", + "jsonpath" + ], + "time": "2019-12-30T18:03:34+00:00" + }, { "name": "mustache/mustache", "version": "v2.12.0", @@ -6542,7 +6684,9 @@ "prefer-lowest": false, "platform": { "php": "7.0.2||7.0.4||~7.0.6||~7.1.0||~7.2.0||~7.3.0", - "ext-curl": "*" + "ext-curl": "*", + "ext-json": "*", + "ext-openssl": "*" }, "platform-dev": [] } diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/SecretStorage/AwsSecretManagerStorageTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/SecretStorage/AwsSecretManagerStorageTest.php new file mode 100644 index 000000000..015c7a009 --- /dev/null +++ b/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/SecretStorage/AwsSecretManagerStorageTest.php @@ -0,0 +1,65 @@ + 'mftf/magento/' . $testShortKey, + 'SecretString' => json_encode([$testShortKey => $testValue]) + ]; + /** @var Result */ + $result = new Result($data); + + $mockClient = $this->getMockBuilder(SecretsManagerClient::class) + ->disableOriginalConstructor() + ->setMethods(['__call']) + ->getMock(); + + $mockClient->expects($this->once()) + ->method('__call') + ->willReturnCallback(function($name, $args) use ($result) { + return $result; + }); + + /** @var SecretsManagerClient */ + $credentialStorage = new AwsSecretManagerStorage($testRegion, $testProfile); + $reflection = new ReflectionClass($credentialStorage); + $reflection_property = $reflection->getProperty('client'); + $reflection_property->setAccessible(true); + $reflection_property->setValue($credentialStorage, $mockClient); + + // Test getEncryptedValue() + $encryptedCred = $credentialStorage->getEncryptedValue($testLongKey); + + // Assert the value we've gotten is in fact not identical to our test value + $this->assertNotEquals($testValue, $encryptedCred); + + // Test getDecryptedValue() + $actualValue = $credentialStorage->getDecryptedValue($encryptedCred); + + // Assert that we are able to successfully decrypt our secret value + $this->assertEquals($testValue, $actualValue); + } +} diff --git a/docs/configuration.md b/docs/configuration.md index 5140c01e7..f89090c06 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -277,6 +277,28 @@ Example: CREDENTIAL_VAULT_SECRET_BASE_PATH=secret ``` +### CREDENTIAL_AWS_SECRET_MANAGER_REGION + +The region that Aws Secret Manager is located. + +Example: + +```conf +# Region of Aws Secret Manager +CREDENTIAL_AWS_SECRET_MANAGER_REGION=us-east-1 +``` + +### CREDENTIAL_AWS_SECRET_MANAGER_PROFILE + +The profile used to connect to Aws Secret Manager. + +Example: + +```conf +# Profile used to connect to Aws Secret Manager. +CREDENTIAL_AWS_SECRET_MANAGER_PROFILE=default +``` + ### ENABLE_BROWSER_LOG Enables addition of browser logs to Allure steps diff --git a/docs/credentials.md b/docs/credentials.md index a2850cfe8..6ca9900cb 100644 --- a/docs/credentials.md +++ b/docs/credentials.md @@ -3,10 +3,11 @@ When you test functionality that involves external services such as UPS, FedEx, PayPal, or SignifyD, use the MFTF credentials feature to hide sensitive [data][] like integration tokens and API keys. -Currently the MFTF supports two types of credential storage: +Currently the MFTF supports three types of credential storage: - **.credentials file** -- **HashiCorp vault** +- **HashiCorp Vault** +- **Aws Secret Manager** ## Configure File Storage @@ -135,11 +136,64 @@ CREDENTIAL_VAULT_ADDRESS=http://127.0.0.1:8200 CREDENTIAL_VAULT_SECRET_BASE_PATH=secret ``` -## Configure both File Storage and Vault Storage +## Configure Aws Secret Manager -It is possible and sometimes useful to setup and use both `.credentials` file and vault for secret storage at the same time. -In this case, the MFTF tests are able to read secret data at runtime from both storage options, but the local `.credentials` file will take precedence. +Aws Secrets Manager offers secret management that supports: +- Secret rotation with built-in integration for Amazon RDS, Amazon Redshift, and Amazon DocumentDB +- Fine-grained policies and permissions +- Audit secret rotation centrally for resources in the AWS Cloud, third-party services, and on-premises +### Prerequisites +- AWS account +- AWS Secret Manger is created and configured +- IAM User or Role is created + +### Store secrets in Aws Secret Manager + +#### Secrets format +`Secret Name`, `Secret Key`, `Secret Value` are three key pieces of information to construct an Aws Secret. +`Secret Key` and `Secret Value` can be any content you want to secure, `Secret Name` must follow the format: + +```conf +mftf// +``` + +```conf +# Secret name for carriers_usps_userid +mftf/magento/carriers_usps_userid + +# Secret key for carriers_usps_userid +carriers_usps_userid + +# Secret name for carriers_usps_password +mftf/magento/carriers_usps_password + +# Secret key for carriers_usps_password +carriers_usps_password +``` + +### Setup MFTF to use Aws Secret Manager + +To use Aws Secret Manager, the Aws region to connect to is required. You can set it through environment variable [`CREDENTIAL_AWS_SECRET_MANAGER_REGION`][] in `.env`. + +MFTF uses the recommended [Default Credential Provider Chain][credential chain] to establish connection to Aws Secret Manager service. +You can setup credentials according to [Default Credential Provider Chain][credential chain] and there is no MFTF specific setup required. +Optionally, however, you can explicitly set Aws profile through environment variable [`CREDENTIAL_AWS_SECRET_MANAGER_PROFILE`][] in `.env`. + +```conf +# Sample Aws Secret Manager configuration +CREDENTIAL_AWS_SECRET_MANAGER_REGION=us-east-1 +CREDENTIAL_AWS_SECRET_MANAGER_PROFILE=default +``` + +## Configure multiple credential storage + +It is possible and sometimes useful to setup and use multiple credential storage at the same time. +In this case, the MFTF tests are able to read secret data at runtime from all storage options, in this case MFTF use the following precedence: + +``` +.credentials File > HashiCorp Vault > Aws Secret Manager +``` ## Use credentials in a test @@ -183,3 +237,6 @@ The MFTF tests delivered with Magento application do not use credentials and do [Vault KV2]: https://www.vaultproject.io/docs/secrets/kv/kv-v2.html [`CREDENTIAL_VAULT_ADDRESS`]: configuration.md#credential_vault_address [`CREDENTIAL_VAULT_SECRET_BASE_PATH`]: configuration.md#credential_vault_secret_base_path +[credential chain]: https://docs.aws.amazon.com/sdk-for-php/v3/developer-guide/guide_credentials.html +[`CREDENTIAL_AWS_SECRET_MANAGER_PROFILE`]: configuration.md#credential_aws_secret_manager_profile +[`CREDENTIAL_AWS_SECRET_MANAGER_REGION`]: configuration.md#credential_aws_secret_manager_region \ No newline at end of file diff --git a/etc/config/.env.example b/etc/config/.env.example index 7320d8b8b..a772b1e9e 100644 --- a/etc/config/.env.example +++ b/etc/config/.env.example @@ -30,10 +30,14 @@ BROWSER=chrome #MAGENTO_RESTAPI_SERVER_PORT=8080 #MAGENTO_RESTAPI_SERVER_PROTOCOL=https -#*** Uncomment and set vault address and secret base path if you want to use vault to manage _CREDS secrets ***# +#*** To use HashiCorp Vault to manage _CREDS secrets, uncomment and set vault address and secret base path ***# #CREDENTIAL_VAULT_ADDRESS=http://127.0.0.1:8200 #CREDENTIAL_VAULT_SECRET_BASE_PATH=secret +#*** To use AWS Secret Manager to manage _CREDS secrets, uncomment and set region, profile is optional, when omitted, AWS default credential provider chain will be used ***# +#CREDENTIAL_AWS_SECRET_MANAGER_PROFILE=default +#CREDENTIAL_AWS_SECRET_MANAGER_REGION=us-east-1 + #*** Uncomment these properties to set up a dev environment with symlinked projects ***# #TESTS_BP= #FW_BP= diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php index 94ff40069..bff3e9b00 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php @@ -9,12 +9,18 @@ use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; use Magento\FunctionalTestingFramework\DataGenerator\Handlers\SecretStorage\FileStorage; use Magento\FunctionalTestingFramework\DataGenerator\Handlers\SecretStorage\VaultStorage; +use Magento\FunctionalTestingFramework\DataGenerator\Handlers\SecretStorage\AwsSecretManagerStorage; use Magento\FunctionalTestingFramework\Util\Path\UrlFormatter; class CredentialStore { const ARRAY_KEY_FOR_VAULT = 'vault'; const ARRAY_KEY_FOR_FILE = 'file'; + const ARRAY_KEY_FOR_AWS_SECRET_MANAGER = 'aws'; + + const CREDENTIAL_STORAGE_INFO = 'MFTF uses Credential Storage in the following precedence: ' + . '.credentials file, HashiCorp Vault and AWS Secret Manager. ' + . 'You need to configure at least one to use _CREDS in tests.'; /** * Credential storage array @@ -71,9 +77,25 @@ private function __construct() } } + // Initialize AWS secret manager storage + $awsRegion = getenv('CREDENTIAL_AWS_SECRET_MANAGER_REGION'); + $awsProfile = getenv('CREDENTIAL_AWS_SECRET_MANAGER_PROFILE'); + if ($awsRegion !== false) { + if ($awsProfile === false) { + $awsProfile = null; + } + try { + $this->credStorage[self::ARRAY_KEY_FOR_AWS_SECRET_MANAGER] = new AwsSecretManagerStorage( + $awsRegion, + $awsProfile + ); + } catch (TestFrameworkException $e) { + } + } + if (empty($this->credStorage)) { throw new TestFrameworkException( - "No credential storage is properly configured. Please configure vault or .credentials file." + 'Invalid Credential Storage. ' . self::CREDENTIAL_STORAGE_INFO ); } } @@ -97,8 +119,7 @@ public function getSecret($key) } throw new TestFrameworkException( - "\"{$key}\" not defined in vault or .credentials file, " - . "please provide a value in order to use this secret in a test." + "{$key} not found. " . self::CREDENTIAL_STORAGE_INFO . ' And make sure key/value exists.' ); } diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/AwsSecretManagerStorage.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/AwsSecretManagerStorage.php new file mode 100644 index 000000000..c617f1cac --- /dev/null +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/AwsSecretManagerStorage.php @@ -0,0 +1,155 @@ +createAwsSecretManagerClient($region, $profile); + } + + /** + * Returns the value of a secret based on corresponding key + * + * @param string $key + * @return string|null + * @throws Exception + */ + public function getEncryptedValue($key) + { + // Check if secret is in cached array + if (null !== ($value = parent::getEncryptedValue($key))) { + return $value; + } + + if (MftfApplicationConfig::getConfig()->verboseEnabled()) { + LoggingUtil::getInstance()->getLogger(VaultStorage::class)->debug( + "Retrieving secret for key name {$key} from AWS Secret Manager" + ); + } + + $reValue = null; + try { + // Split vendor/key to construct secret id + list($vendor, $key) = explode('/', trim($key, '/'), 2); + $secretId = self::MFTF_PATH + . '/' + . $vendor + . '/' + . $key; + // Read value by id from AWS Secret Manager, and parse the result + $value = $this->parseAwsSecretResult( + $this->client->getSecretValue(['SecretId' => $secretId]), + $key + ); + // Encrypt value for return + $reValue = openssl_encrypt($value, parent::ENCRYPTION_ALGO, parent::$encodedKey, 0, parent::$iv); + parent::$cachedSecretData[$key] = $reValue; + } catch (AwsException $e) { + $error = $e->getAwsErrorCode(); + if (MftfApplicationConfig::getConfig()->verboseEnabled()) { + LoggingUtil::getInstance()->getLogger(VaultStorage::class)->debug( + "AWS error code: {$error}. Unable to read secret for key {$key} from AWS Secret Manager" + ); + } + } catch (\Exception $e) { + if (MftfApplicationConfig::getConfig()->verboseEnabled()) { + LoggingUtil::getInstance()->getLogger(VaultStorage::class)->debug( + "Unable to read secret for key {$key} from AWS Secret Manager" + ); + } + } + return $reValue; + } + + /** + * Parse AWS result object and return secret for key + * + * @param Result $awsResult + * @param string $key + * @return string + * @throws TestFrameworkException + */ + private function parseAwsSecretResult($awsResult, $key) + { + // Return secret from the associated KMS CMK + if (isset($awsResult['SecretString'])) { + $rawSecret = $awsResult['SecretString']; + } else { + throw new TestFrameworkException("Error parsing AWS secret result"); + } + $secret = json_decode($rawSecret, true); + if (isset($secret[$key])) { + return $secret[$key]; + } + throw new TestFrameworkException("Error parsing AWS secret result"); + } + + /** + * Create Aws Secret Manager client + * + * @param string $region + * @param string $profile + * @throws TestFrameworkException + * @throws InvalidArgumentException + */ + private function createAwsSecretManagerClient($region, $profile) + { + if (null !== $this->client) { + return; + } + + // Create AWS Secret Manager client + $this->client = new SecretsManagerClient([ + 'profile' => $profile, + 'region' => $region, + 'version' => self::LATEST_VERSION + ]); + + if ($this->client === null) { + throw new TestFrameworkException("Unable to create AWS Secret Manager client"); + } + } +} diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php index 6a9e9f0cf..798ac660c 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php @@ -67,7 +67,7 @@ class VaultStorage extends BaseStorage private $secretBasePath; /** - * CredentialVault constructor + * VaultStorage constructor * * @param string $baseUrl * @param string $secretBasePath From 579c96d82c9786392cf85b78162947d42f686802 Mon Sep 17 00:00:00 2001 From: Ji Lu Date: Fri, 10 Jan 2020 14:49:15 -0600 Subject: [PATCH 2/7] MQE-1918: MFTF AWS Secrets Manager - Local Use --- .../Handlers/SecretStorage/AwsSecretManagerStorageTest.php | 2 +- .../Handlers/SecretStorage/AwsSecretManagerStorage.php | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/SecretStorage/AwsSecretManagerStorageTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/SecretStorage/AwsSecretManagerStorageTest.php index 015c7a009..f11d14920 100644 --- a/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/SecretStorage/AwsSecretManagerStorageTest.php +++ b/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/SecretStorage/AwsSecretManagerStorageTest.php @@ -39,7 +39,7 @@ public function testEncryptAndDecrypt() $mockClient->expects($this->once()) ->method('__call') - ->willReturnCallback(function($name, $args) use ($result) { + ->willReturnCallback(function ($name, $args) use ($result) { return $result; }); diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/AwsSecretManagerStorage.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/AwsSecretManagerStorage.php index c617f1cac..0c10c53c4 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/AwsSecretManagerStorage.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/AwsSecretManagerStorage.php @@ -132,6 +132,7 @@ private function parseAwsSecretResult($awsResult, $key) * * @param string $region * @param string $profile + * @return void * @throws TestFrameworkException * @throws InvalidArgumentException */ From d0d98051488658d8325b7e2e0fe3a969c9cd45ee Mon Sep 17 00:00:00 2001 From: Ji Lu Date: Tue, 14 Jan 2020 15:36:30 -0600 Subject: [PATCH 3/7] MQE-1918: MFTF AWS Secrets Manager - Local Use --- ...t.php => AwsSecretsManagerStorageTest.php} | 8 ++--- docs/configuration.md | 16 ++++----- docs/credentials.md | 34 +++++++++---------- etc/config/.env.example | 6 ++-- .../Handlers/CredentialStore.php | 18 +++++----- ...orage.php => AwsSecretsManagerStorage.php} | 34 +++++++++---------- 6 files changed, 58 insertions(+), 58 deletions(-) rename dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/SecretStorage/{AwsSecretManagerStorageTest.php => AwsSecretsManagerStorageTest.php} (88%) rename src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/{AwsSecretManagerStorage.php => AwsSecretsManagerStorage.php} (75%) diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/SecretStorage/AwsSecretManagerStorageTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/SecretStorage/AwsSecretsManagerStorageTest.php similarity index 88% rename from dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/SecretStorage/AwsSecretManagerStorageTest.php rename to dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/SecretStorage/AwsSecretsManagerStorageTest.php index f11d14920..e1f4e4879 100644 --- a/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/SecretStorage/AwsSecretManagerStorageTest.php +++ b/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/SecretStorage/AwsSecretsManagerStorageTest.php @@ -7,15 +7,15 @@ namespace tests\unit\Magento\FunctionalTestFramework\DataGenerator\Handlers\SecretStorage; use Aws\SecretsManager\SecretsManagerClient; -use Magento\FunctionalTestingFramework\DataGenerator\Handlers\SecretStorage\AwsSecretManagerStorage; +use Magento\FunctionalTestingFramework\DataGenerator\Handlers\SecretStorage\AwsSecretsManagerStorage; use Aws\Result; use Magento\FunctionalTestingFramework\Util\MagentoTestCase; use ReflectionClass; -class AwsSecretManagerStorageTest extends MagentoTestCase +class AwsSecretsManagerStorageTest extends MagentoTestCase { /** - * Test encryption/decryption functionality in AwsSecretManagerStorage class. + * Test encryption/decryption functionality in AwsSecretsManagerStorage class. */ public function testEncryptAndDecrypt() { @@ -44,7 +44,7 @@ public function testEncryptAndDecrypt() }); /** @var SecretsManagerClient */ - $credentialStorage = new AwsSecretManagerStorage($testRegion, $testProfile); + $credentialStorage = new AwsSecretsManagerStorage($testRegion, $testProfile); $reflection = new ReflectionClass($credentialStorage); $reflection_property = $reflection->getProperty('client'); $reflection_property->setAccessible(true); diff --git a/docs/configuration.md b/docs/configuration.md index f89090c06..9466f2bcc 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -277,26 +277,26 @@ Example: CREDENTIAL_VAULT_SECRET_BASE_PATH=secret ``` -### CREDENTIAL_AWS_SECRET_MANAGER_REGION +### CREDENTIAL_AWS_SECRETS_MANAGER_REGION -The region that Aws Secret Manager is located. +The region that AWS Secrets Manager is located. Example: ```conf -# Region of Aws Secret Manager -CREDENTIAL_AWS_SECRET_MANAGER_REGION=us-east-1 +# Region of AWS Secrets Manager +CREDENTIAL_AWS_SECRETS_MANAGER_REGION=us-east-1 ``` -### CREDENTIAL_AWS_SECRET_MANAGER_PROFILE +### CREDENTIAL_AWS_SECRETS_MANAGER_PROFILE -The profile used to connect to Aws Secret Manager. +The profile used to connect to AWS Secrets Manager. Example: ```conf -# Profile used to connect to Aws Secret Manager. -CREDENTIAL_AWS_SECRET_MANAGER_PROFILE=default +# Profile used to connect to AWS Secrets Manager. +CREDENTIAL_AWS_SECRETS_MANAGER_PROFILE=default ``` ### ENABLE_BROWSER_LOG diff --git a/docs/credentials.md b/docs/credentials.md index 6ca9900cb..4b20e68dc 100644 --- a/docs/credentials.md +++ b/docs/credentials.md @@ -7,7 +7,7 @@ Currently the MFTF supports three types of credential storage: - **.credentials file** - **HashiCorp Vault** -- **Aws Secret Manager** +- **AWS Secrets Manager** ## Configure File Storage @@ -136,22 +136,22 @@ CREDENTIAL_VAULT_ADDRESS=http://127.0.0.1:8200 CREDENTIAL_VAULT_SECRET_BASE_PATH=secret ``` -## Configure Aws Secret Manager +## Configure AWS Secrets Manager -Aws Secrets Manager offers secret management that supports: +AWS Secrets Manager offers secret management that supports: - Secret rotation with built-in integration for Amazon RDS, Amazon Redshift, and Amazon DocumentDB - Fine-grained policies and permissions - Audit secret rotation centrally for resources in the AWS Cloud, third-party services, and on-premises ### Prerequisites - AWS account -- AWS Secret Manger is created and configured -- IAM User or Role is created +- AWS Secrets Manger is created and configured +- IAM User or Role is created with appropriate AWS Secrets Manger access permission -### Store secrets in Aws Secret Manager +### Store secrets in AWS Secrets Manager #### Secrets format -`Secret Name`, `Secret Key`, `Secret Value` are three key pieces of information to construct an Aws Secret. +`Secret Name`, `Secret Key`, `Secret Value` are three key pieces of information to construct an AWS Secret. `Secret Key` and `Secret Value` can be any content you want to secure, `Secret Name` must follow the format: ```conf @@ -172,18 +172,18 @@ mftf/magento/carriers_usps_password carriers_usps_password ``` -### Setup MFTF to use Aws Secret Manager +### Setup MFTF to use AWS Secrets Manager -To use Aws Secret Manager, the Aws region to connect to is required. You can set it through environment variable [`CREDENTIAL_AWS_SECRET_MANAGER_REGION`][] in `.env`. +To use AWS Secrets Manager, the AWS region to connect to is required. You can set it through environment variable [`CREDENTIAL_AWS_SECRETS_MANAGER_REGION`][] in `.env`. -MFTF uses the recommended [Default Credential Provider Chain][credential chain] to establish connection to Aws Secret Manager service. +MFTF uses the recommended [Default Credential Provider Chain][credential chain] to establish connection to AWS Secrets Manager service. You can setup credentials according to [Default Credential Provider Chain][credential chain] and there is no MFTF specific setup required. -Optionally, however, you can explicitly set Aws profile through environment variable [`CREDENTIAL_AWS_SECRET_MANAGER_PROFILE`][] in `.env`. +Optionally, however, you can explicitly set AWS profile through environment variable [`CREDENTIAL_AWS_SECRETS_MANAGER_PROFILE`][] in `.env`. ```conf -# Sample Aws Secret Manager configuration -CREDENTIAL_AWS_SECRET_MANAGER_REGION=us-east-1 -CREDENTIAL_AWS_SECRET_MANAGER_PROFILE=default +# Sample AWS Secrets Manager configuration +CREDENTIAL_AWS_SECRETS_MANAGER_REGION=us-east-1 +CREDENTIAL_AWS_SECRETS_MANAGER_PROFILE=default ``` ## Configure multiple credential storage @@ -192,7 +192,7 @@ It is possible and sometimes useful to setup and use multiple credential storage In this case, the MFTF tests are able to read secret data at runtime from all storage options, in this case MFTF use the following precedence: ``` -.credentials File > HashiCorp Vault > Aws Secret Manager +.credentials File > HashiCorp Vault > AWS Secrets Manager ``` @@ -238,5 +238,5 @@ The MFTF tests delivered with Magento application do not use credentials and do [`CREDENTIAL_VAULT_ADDRESS`]: configuration.md#credential_vault_address [`CREDENTIAL_VAULT_SECRET_BASE_PATH`]: configuration.md#credential_vault_secret_base_path [credential chain]: https://docs.aws.amazon.com/sdk-for-php/v3/developer-guide/guide_credentials.html -[`CREDENTIAL_AWS_SECRET_MANAGER_PROFILE`]: configuration.md#credential_aws_secret_manager_profile -[`CREDENTIAL_AWS_SECRET_MANAGER_REGION`]: configuration.md#credential_aws_secret_manager_region \ No newline at end of file +[`CREDENTIAL_AWS_SECRETS_MANAGER_PROFILE`]: configuration.md#credential_aws_secrets_manager_profile +[`CREDENTIAL_AWS_SECRETS_MANAGER_REGION`]: configuration.md#credential_aws_secrets_manager_region \ No newline at end of file diff --git a/etc/config/.env.example b/etc/config/.env.example index a772b1e9e..f5b6ef40e 100644 --- a/etc/config/.env.example +++ b/etc/config/.env.example @@ -34,9 +34,9 @@ BROWSER=chrome #CREDENTIAL_VAULT_ADDRESS=http://127.0.0.1:8200 #CREDENTIAL_VAULT_SECRET_BASE_PATH=secret -#*** To use AWS Secret Manager to manage _CREDS secrets, uncomment and set region, profile is optional, when omitted, AWS default credential provider chain will be used ***# -#CREDENTIAL_AWS_SECRET_MANAGER_PROFILE=default -#CREDENTIAL_AWS_SECRET_MANAGER_REGION=us-east-1 +#*** To use AWS Secrets Manager to manage _CREDS secrets, uncomment and set region, profile is optional, when omitted, AWS default credential provider chain will be used ***# +#CREDENTIAL_AWS_SECRETS_MANAGER_PROFILE=default +#CREDENTIAL_AWS_SECRETS_MANAGER_REGION=us-east-1 #*** Uncomment these properties to set up a dev environment with symlinked projects ***# #TESTS_BP= diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php index bff3e9b00..76560bcf1 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php @@ -9,17 +9,17 @@ use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; use Magento\FunctionalTestingFramework\DataGenerator\Handlers\SecretStorage\FileStorage; use Magento\FunctionalTestingFramework\DataGenerator\Handlers\SecretStorage\VaultStorage; -use Magento\FunctionalTestingFramework\DataGenerator\Handlers\SecretStorage\AwsSecretManagerStorage; +use Magento\FunctionalTestingFramework\DataGenerator\Handlers\SecretStorage\AwsSecretsManagerStorage; use Magento\FunctionalTestingFramework\Util\Path\UrlFormatter; class CredentialStore { const ARRAY_KEY_FOR_VAULT = 'vault'; const ARRAY_KEY_FOR_FILE = 'file'; - const ARRAY_KEY_FOR_AWS_SECRET_MANAGER = 'aws'; + const ARRAY_KEY_FOR_AWS_SECRETS_MANAGER = 'aws'; const CREDENTIAL_STORAGE_INFO = 'MFTF uses Credential Storage in the following precedence: ' - . '.credentials file, HashiCorp Vault and AWS Secret Manager. ' + . '.credentials file, HashiCorp Vault and AWS Secrets Manager. ' . 'You need to configure at least one to use _CREDS in tests.'; /** @@ -77,15 +77,15 @@ private function __construct() } } - // Initialize AWS secret manager storage - $awsRegion = getenv('CREDENTIAL_AWS_SECRET_MANAGER_REGION'); - $awsProfile = getenv('CREDENTIAL_AWS_SECRET_MANAGER_PROFILE'); + // Initialize AWS Secrets Manager storage + $awsRegion = getenv('CREDENTIAL_AWS_SECRETS_MANAGER_REGION'); + $awsProfile = getenv('CREDENTIAL_AWS_SECRETS_MANAGER_PROFILE'); if ($awsRegion !== false) { if ($awsProfile === false) { $awsProfile = null; } try { - $this->credStorage[self::ARRAY_KEY_FOR_AWS_SECRET_MANAGER] = new AwsSecretManagerStorage( + $this->credStorage[self::ARRAY_KEY_FOR_AWS_SECRETS_MANAGER] = new AwsSecretsManagerStorage( $awsRegion, $awsProfile ); @@ -109,8 +109,8 @@ private function __construct() */ public function getSecret($key) { - // Get secret data from storage according to the order they are stored - // File storage is preferred over vault storage to allow local secret value overriding remote secret value + // Get secret data from storage according to the order they are stored which follows this precedence: + // FileStorage > VaultStorage > AwsSecretsManagerStorage foreach ($this->credStorage as $storage) { $value = $storage->getEncryptedValue($key); if (null !== $value) { diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/AwsSecretManagerStorage.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/AwsSecretsManagerStorage.php similarity index 75% rename from src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/AwsSecretManagerStorage.php rename to src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/AwsSecretsManagerStorage.php index 0c10c53c4..e7a858254 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/AwsSecretManagerStorage.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/AwsSecretsManagerStorage.php @@ -15,7 +15,7 @@ use InvalidArgumentException; use Exception; -class AwsSecretManagerStorage extends BaseStorage +class AwsSecretsManagerStorage extends BaseStorage { /** * Mftf project path @@ -23,7 +23,7 @@ class AwsSecretManagerStorage extends BaseStorage const MFTF_PATH = 'mftf'; /** - * AWS Secret Manager version + * AWS Secrets Manager version * * Last tested version '2017-10-17' */ @@ -37,7 +37,7 @@ class AwsSecretManagerStorage extends BaseStorage private $client = null; /** - * AwsSecretManagerStorage constructor + * AwsSecretsManagerStorage constructor * * @param string $region * @param string $profile @@ -47,7 +47,7 @@ class AwsSecretManagerStorage extends BaseStorage public function __construct($region, $profile = null) { parent::__construct(); - $this->createAwsSecretManagerClient($region, $profile); + $this->createAwsSecretsManagerClient($region, $profile); } /** @@ -65,8 +65,8 @@ public function getEncryptedValue($key) } if (MftfApplicationConfig::getConfig()->verboseEnabled()) { - LoggingUtil::getInstance()->getLogger(VaultStorage::class)->debug( - "Retrieving secret for key name {$key} from AWS Secret Manager" + LoggingUtil::getInstance()->getLogger(AwsSecretsManagerStorage::class)->debug( + "Retrieving value for key name {$key} from AWS Secrets Manager" ); } @@ -79,7 +79,7 @@ public function getEncryptedValue($key) . $vendor . '/' . $key; - // Read value by id from AWS Secret Manager, and parse the result + // Read value by id from AWS Secrets Manager, and parse the result $value = $this->parseAwsSecretResult( $this->client->getSecretValue(['SecretId' => $secretId]), $key @@ -90,14 +90,14 @@ public function getEncryptedValue($key) } catch (AwsException $e) { $error = $e->getAwsErrorCode(); if (MftfApplicationConfig::getConfig()->verboseEnabled()) { - LoggingUtil::getInstance()->getLogger(VaultStorage::class)->debug( - "AWS error code: {$error}. Unable to read secret for key {$key} from AWS Secret Manager" + LoggingUtil::getInstance()->getLogger(AwsSecretsManagerStorage::class)->debug( + "AWS error code: {$error}. Unable to read value for key {$key} from AWS Secrets Manager" ); } } catch (\Exception $e) { if (MftfApplicationConfig::getConfig()->verboseEnabled()) { - LoggingUtil::getInstance()->getLogger(VaultStorage::class)->debug( - "Unable to read secret for key {$key} from AWS Secret Manager" + LoggingUtil::getInstance()->getLogger(AwsSecretsManagerStorage::class)->debug( + "Unable to read value for key {$key} from AWS Secrets Manager" ); } } @@ -118,17 +118,17 @@ private function parseAwsSecretResult($awsResult, $key) if (isset($awsResult['SecretString'])) { $rawSecret = $awsResult['SecretString']; } else { - throw new TestFrameworkException("Error parsing AWS secret result"); + throw new TestFrameworkException("Error parsing result from AWS Secrets Manager"); } $secret = json_decode($rawSecret, true); if (isset($secret[$key])) { return $secret[$key]; } - throw new TestFrameworkException("Error parsing AWS secret result"); + throw new TestFrameworkException("Error parsing result from AWS Secrets Manager"); } /** - * Create Aws Secret Manager client + * Create Aws Secrets Manager client * * @param string $region * @param string $profile @@ -136,13 +136,13 @@ private function parseAwsSecretResult($awsResult, $key) * @throws TestFrameworkException * @throws InvalidArgumentException */ - private function createAwsSecretManagerClient($region, $profile) + private function createAwsSecretsManagerClient($region, $profile) { if (null !== $this->client) { return; } - // Create AWS Secret Manager client + // Create AWS Secrets Manager client $this->client = new SecretsManagerClient([ 'profile' => $profile, 'region' => $region, @@ -150,7 +150,7 @@ private function createAwsSecretManagerClient($region, $profile) ]); if ($this->client === null) { - throw new TestFrameworkException("Unable to create AWS Secret Manager client"); + throw new TestFrameworkException("Unable to create AWS Secrets Manager client"); } } } From bb62cfe317f88c7e18df7039d72c2bbbb78b1bf9 Mon Sep 17 00:00:00 2001 From: Ji Lu Date: Wed, 15 Jan 2020 10:07:44 -0600 Subject: [PATCH 4/7] MQE-1918: MFTF AWS Secrets Manager - Local Use --- .../DataGenerator/Handlers/CredentialStore.php | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php index 76560bcf1..84d58ade8 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php @@ -18,9 +18,8 @@ class CredentialStore const ARRAY_KEY_FOR_FILE = 'file'; const ARRAY_KEY_FOR_AWS_SECRETS_MANAGER = 'aws'; - const CREDENTIAL_STORAGE_INFO = 'MFTF uses Credential Storage in the following precedence: ' - . '.credentials file, HashiCorp Vault and AWS Secrets Manager. ' - . 'You need to configure at least one to use _CREDS in tests.'; + const CREDENTIAL_STORAGE_INFO = 'You need to configure at least one of these options: ' + . '.credentials file, HashiCorp Vault or AWS Secrets Manager correctly'; /** * Credential storage array @@ -95,7 +94,7 @@ private function __construct() if (empty($this->credStorage)) { throw new TestFrameworkException( - 'Invalid Credential Storage. ' . self::CREDENTIAL_STORAGE_INFO + 'Invalid Credential Storage. ' . self::CREDENTIAL_STORAGE_INFO . '.' ); } } @@ -119,7 +118,8 @@ public function getSecret($key) } throw new TestFrameworkException( - "{$key} not found. " . self::CREDENTIAL_STORAGE_INFO . ' And make sure key/value exists.' + "{$key} not found. " . self::CREDENTIAL_STORAGE_INFO + . ' and ensure key, value exists to use _CREDS in tests.' ); } From 9a1fdd1530b8005dd2cf8f3d62edab612330adb9 Mon Sep 17 00:00:00 2001 From: Ji Lu Date: Thu, 16 Jan 2020 16:02:52 -0600 Subject: [PATCH 5/7] MQE-1919: MFTF AWS Secrets Manager - CI Use --- etc/config/.env.example | 2 ++ .../Handlers/CredentialStore.php | 7 ++++- .../AwsSecretsManagerStorage.php | 31 +++++++++++++++++-- 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/etc/config/.env.example b/etc/config/.env.example index f5b6ef40e..7f988c9f0 100644 --- a/etc/config/.env.example +++ b/etc/config/.env.example @@ -37,6 +37,8 @@ BROWSER=chrome #*** To use AWS Secrets Manager to manage _CREDS secrets, uncomment and set region, profile is optional, when omitted, AWS default credential provider chain will be used ***# #CREDENTIAL_AWS_SECRETS_MANAGER_PROFILE=default #CREDENTIAL_AWS_SECRETS_MANAGER_REGION=us-east-1 +#*** If using non-default AWS account ***# +#CREDENTIAL_AWS_ACCOUNT_ID= #*** Uncomment these properties to set up a dev environment with symlinked projects ***# #TESTS_BP= diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php index 84d58ade8..7769decc6 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php @@ -79,14 +79,19 @@ private function __construct() // Initialize AWS Secrets Manager storage $awsRegion = getenv('CREDENTIAL_AWS_SECRETS_MANAGER_REGION'); $awsProfile = getenv('CREDENTIAL_AWS_SECRETS_MANAGER_PROFILE'); + $awsId = getenv('CREDENTIAL_AWS_ACCOUNT_ID'); if ($awsRegion !== false) { if ($awsProfile === false) { $awsProfile = null; } + if ($awsId === false) { + $awsId = null; + } try { $this->credStorage[self::ARRAY_KEY_FOR_AWS_SECRETS_MANAGER] = new AwsSecretsManagerStorage( $awsRegion, - $awsProfile + $awsProfile, + $awsId ); } catch (TestFrameworkException $e) { } diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/AwsSecretsManagerStorage.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/AwsSecretsManagerStorage.php index e7a858254..1e58e3bee 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/AwsSecretsManagerStorage.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/AwsSecretsManagerStorage.php @@ -22,6 +22,11 @@ class AwsSecretsManagerStorage extends BaseStorage */ const MFTF_PATH = 'mftf'; + /** + * AWS Secrets Manager partial ARN + */ + const AWS_SM_PARTIAL_ARN = 'arn:aws:secretsmanager:'; + /** * AWS Secrets Manager version * @@ -36,18 +41,35 @@ class AwsSecretsManagerStorage extends BaseStorage */ private $client = null; + /** + * AWS account id + * + * @var string + */ + private $awsAccountId; + + /** + * AWS account region + * + * @var string + */ + private $region; + /** * AwsSecretsManagerStorage constructor * * @param string $region * @param string $profile + * @param string $accountId * @throws TestFrameworkException * @throws InvalidArgumentException */ - public function __construct($region, $profile = null) + public function __construct($region, $profile = null, $accountId = null) { parent::__construct(); $this->createAwsSecretsManagerClient($region, $profile); + $this->region = $region; + $this->awsAccountId = $accountId; } /** @@ -74,7 +96,12 @@ public function getEncryptedValue($key) try { // Split vendor/key to construct secret id list($vendor, $key) = explode('/', trim($key, '/'), 2); - $secretId = self::MFTF_PATH + // If AWS account id is specified, create and use full ARN, otherwise use partial ARN as secret id + $secretId = ''; + if (!empty($this->awsAccountId)) { + $secretId = self::AWS_SM_PARTIAL_ARN . $this->region . ':' . $this->awsAccountId . ':secret:'; + } + $secretId .= self::MFTF_PATH . '/' . $vendor . '/' From 93e221c4a99db73a129d05b3fd203942022617dc Mon Sep 17 00:00:00 2001 From: Ji Lu Date: Tue, 21 Jan 2020 11:14:56 -0600 Subject: [PATCH 6/7] MQE-1919: MFTF AWS Secrets Manager - CI Use --- docs/credentials.md | 64 ++++++++--- etc/config/.env.example | 2 - .../Handlers/CredentialStore.php | 107 +++++++++++------- .../AwsSecretsManagerStorage.php | 42 ++++--- 4 files changed, 145 insertions(+), 70 deletions(-) diff --git a/docs/credentials.md b/docs/credentials.md index 4b20e68dc..1ea91a5c3 100644 --- a/docs/credentials.md +++ b/docs/credentials.md @@ -144,32 +144,54 @@ AWS Secrets Manager offers secret management that supports: - Audit secret rotation centrally for resources in the AWS Cloud, third-party services, and on-premises ### Prerequisites -- AWS account -- AWS Secrets Manger is created and configured + +#### Use AWS Secrets Manager from your own AWS account + +- AWS account with Secrets Manager service available - IAM User or Role is created with appropriate AWS Secrets Manger access permission +#### Use AWS Secrets Manager from other AWS account + +- AWS account ID where the AWS Secrets Manager service is hosted +- IAM User or Role with appropriate access permission + ### Store secrets in AWS Secrets Manager + #### Secrets format -`Secret Name`, `Secret Key`, `Secret Value` are three key pieces of information to construct an AWS Secret. -`Secret Key` and `Secret Value` can be any content you want to secure, `Secret Name` must follow the format: + +`Secret Name` and `Secret Value` are two key pieces of information for creating a secret. + +`Secret Value` can be either plaintext or key/value pairs in JSON format. + +`Secrets Name` must use the following format: ```conf -mftf// +mftf// ``` -```conf -# Secret name for carriers_usps_userid -mftf/magento/carriers_usps_userid +`Secrets Value` in plaintext format can be any content you want to secure. `Secrets Value` in key/value pairs format, however, the `key` must be same as the `Secret Name` with `mftf//` part removed. +e.g. in above example, `key` should be `` + +##### Create Secrets using AWS CLI -# Secret key for carriers_usps_userid -carriers_usps_userid +```bash +aws secretsmanager create-secret --name "mftf/magento/shipping/carriers_usps_userid" --description "Carriers USPS user id" --secret-string "1234567" +``` + +##### Create Secrets using AWS Console + +To save the same secret in key/value JSON format, you should use + +```conf +# Secret Name +mftf/magento/shipping/carriers_usps_userid -# Secret name for carriers_usps_password -mftf/magento/carriers_usps_password +# Secret Key +shipping/carriers_usps_userid -# Secret key for carriers_usps_password -carriers_usps_password +# Secret Value +1234567 ``` ### Setup MFTF to use AWS Secrets Manager @@ -186,6 +208,16 @@ CREDENTIAL_AWS_SECRETS_MANAGER_REGION=us-east-1 CREDENTIAL_AWS_SECRETS_MANAGER_PROFILE=default ``` +### Optionally set CREDENTIAL_AWS_ACCOUNT_ID environment variable + +Full AWS KMS ([Key Management Service][]) key ARN ([Amazon Resource Name][]) is required when accessing secrets stored in other AWS account. +If this is the case, you will also need to set `CREDENTIAL_AWS_ACCOUNT_ID` environment variable so that MFTF can construct the full ARN. +This is also commonly used in CI system. + +```bash +export CREDENTIAL_AWS_ACCOUNT_ID= +``` + ## Configure multiple credential storage It is possible and sometimes useful to setup and use multiple credential storage at the same time. @@ -239,4 +271,6 @@ The MFTF tests delivered with Magento application do not use credentials and do [`CREDENTIAL_VAULT_SECRET_BASE_PATH`]: configuration.md#credential_vault_secret_base_path [credential chain]: https://docs.aws.amazon.com/sdk-for-php/v3/developer-guide/guide_credentials.html [`CREDENTIAL_AWS_SECRETS_MANAGER_PROFILE`]: configuration.md#credential_aws_secrets_manager_profile -[`CREDENTIAL_AWS_SECRETS_MANAGER_REGION`]: configuration.md#credential_aws_secrets_manager_region \ No newline at end of file +[`CREDENTIAL_AWS_SECRETS_MANAGER_REGION`]: configuration.md#credential_aws_secrets_manager_region +[Key Management Service]: https://aws.amazon.com/kms/ +[Amazon Resource Name]: https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html \ No newline at end of file diff --git a/etc/config/.env.example b/etc/config/.env.example index 7f988c9f0..f5b6ef40e 100644 --- a/etc/config/.env.example +++ b/etc/config/.env.example @@ -37,8 +37,6 @@ BROWSER=chrome #*** To use AWS Secrets Manager to manage _CREDS secrets, uncomment and set region, profile is optional, when omitted, AWS default credential provider chain will be used ***# #CREDENTIAL_AWS_SECRETS_MANAGER_PROFILE=default #CREDENTIAL_AWS_SECRETS_MANAGER_REGION=us-east-1 -#*** If using non-default AWS account ***# -#CREDENTIAL_AWS_ACCOUNT_ID= #*** Uncomment these properties to set up a dev environment with symlinked projects ***# #TESTS_BP= diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php index 7769decc6..b66d19799 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php @@ -57,45 +57,10 @@ public static function getInstance() */ private function __construct() { - // Initialize file storage - try { - $this->credStorage[self::ARRAY_KEY_FOR_FILE] = new FileStorage(); - } catch (TestFrameworkException $e) { - } - - // Initialize vault storage - $cvAddress = getenv('CREDENTIAL_VAULT_ADDRESS'); - $cvSecretPath = getenv('CREDENTIAL_VAULT_SECRET_BASE_PATH'); - if ($cvAddress !== false && $cvSecretPath !== false) { - try { - $this->credStorage[self::ARRAY_KEY_FOR_VAULT] = new VaultStorage( - UrlFormatter::format($cvAddress, false), - '/' . trim($cvSecretPath, '/') - ); - } catch (TestFrameworkException $e) { - } - } - - // Initialize AWS Secrets Manager storage - $awsRegion = getenv('CREDENTIAL_AWS_SECRETS_MANAGER_REGION'); - $awsProfile = getenv('CREDENTIAL_AWS_SECRETS_MANAGER_PROFILE'); - $awsId = getenv('CREDENTIAL_AWS_ACCOUNT_ID'); - if ($awsRegion !== false) { - if ($awsProfile === false) { - $awsProfile = null; - } - if ($awsId === false) { - $awsId = null; - } - try { - $this->credStorage[self::ARRAY_KEY_FOR_AWS_SECRETS_MANAGER] = new AwsSecretsManagerStorage( - $awsRegion, - $awsProfile, - $awsId - ); - } catch (TestFrameworkException $e) { - } - } + // Initialize credential storage by defined order of precedence as the following + $this->initializeFileStorage(); + $this->initializeVaultStorage(); + $this->initializeAwsSecretsManagerStorage(); if (empty($this->credStorage)) { throw new TestFrameworkException( @@ -155,4 +120,68 @@ public function decryptAllSecretsInString($string) return $storage->getAllDecryptedValuesInString($string); } } + + /** + * Initialize file storage + * + * @return void + */ + private function initializeFileStorage() + { + // Initialize file storage + try { + $this->credStorage[self::ARRAY_KEY_FOR_FILE] = new FileStorage(); + } catch (TestFrameworkException $e) { + } + } + + /** + * Initialize Vault storage + * + * @return void + */ + private function initializeVaultStorage() + { + // Initialize vault storage + $cvAddress = getenv('CREDENTIAL_VAULT_ADDRESS'); + $cvSecretPath = getenv('CREDENTIAL_VAULT_SECRET_BASE_PATH'); + if ($cvAddress !== false && $cvSecretPath !== false) { + try { + $this->credStorage[self::ARRAY_KEY_FOR_VAULT] = new VaultStorage( + UrlFormatter::format($cvAddress, false), + '/' . trim($cvSecretPath, '/') + ); + } catch (TestFrameworkException $e) { + } + } + } + + /** + * Initialize AWS Secrets Manager storage + * + * @return void + */ + private function initializeAwsSecretsManagerStorage() + { + // Initialize AWS Secrets Manager storage + $awsRegion = getenv('CREDENTIAL_AWS_SECRETS_MANAGER_REGION'); + $awsProfile = getenv('CREDENTIAL_AWS_SECRETS_MANAGER_PROFILE'); + $awsId = getenv('CREDENTIAL_AWS_ACCOUNT_ID'); + if (!empty($awsRegion)) { + if (empty($awsProfile)) { + $awsProfile = null; + } + if (empty($awsId)) { + $awsId = null; + } + try { + $this->credStorage[self::ARRAY_KEY_FOR_AWS_SECRETS_MANAGER] = new AwsSecretsManagerStorage( + $awsRegion, + $awsProfile, + $awsId + ); + } catch (TestFrameworkException $e) { + } + } + } } diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/AwsSecretsManagerStorage.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/AwsSecretsManagerStorage.php index 1e58e3bee..bb6044c7f 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/AwsSecretsManagerStorage.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/AwsSecretsManagerStorage.php @@ -115,17 +115,18 @@ public function getEncryptedValue($key) $reValue = openssl_encrypt($value, parent::ENCRYPTION_ALGO, parent::$encodedKey, 0, parent::$iv); parent::$cachedSecretData[$key] = $reValue; } catch (AwsException $e) { - $error = $e->getAwsErrorCode(); + $errMessage = "\nAWS Exception:\n" . $e->getAwsErrorMessage() + . "\nUnable to read value for key {$key} from AWS Secrets Manager\n"; + print_r($errMessage); if (MftfApplicationConfig::getConfig()->verboseEnabled()) { - LoggingUtil::getInstance()->getLogger(AwsSecretsManagerStorage::class)->debug( - "AWS error code: {$error}. Unable to read value for key {$key} from AWS Secrets Manager" - ); + LoggingUtil::getInstance()->getLogger(AwsSecretsManagerStorage::class)->debug($errMessage); } } catch (\Exception $e) { + $errMessage = "\nException:\n" . $e->getMessage() + . "\nUnable to read value for key {$key} from AWS Secrets Manager\n"; + print_r($errMessage); if (MftfApplicationConfig::getConfig()->verboseEnabled()) { - LoggingUtil::getInstance()->getLogger(AwsSecretsManagerStorage::class)->debug( - "Unable to read value for key {$key} from AWS Secrets Manager" - ); + LoggingUtil::getInstance()->getLogger(AwsSecretsManagerStorage::class)->debug($errMessage); } } return $reValue; @@ -145,13 +146,22 @@ private function parseAwsSecretResult($awsResult, $key) if (isset($awsResult['SecretString'])) { $rawSecret = $awsResult['SecretString']; } else { - throw new TestFrameworkException("Error parsing result from AWS Secrets Manager"); + throw new TestFrameworkException( + "'SecretString' field is not set in AWS Result. Error parsing result from AWS Secrets Manager" + ); } + + // Secrets are saved as JSON structures of key/value pairs if using AWS Secrets Manager console, and + // Secrets are saved as plain text if using AWS CLI. We need to handle both cases. $secret = json_decode($rawSecret, true); if (isset($secret[$key])) { return $secret[$key]; + } elseif (is_string($rawSecret)) { + return $rawSecret; } - throw new TestFrameworkException("Error parsing result from AWS Secrets Manager"); + throw new TestFrameworkException( + "$key not found or value is not string . Error parsing result from AWS Secrets Manager" + ); } /** @@ -169,13 +179,17 @@ private function createAwsSecretsManagerClient($region, $profile) return; } - // Create AWS Secrets Manager client - $this->client = new SecretsManagerClient([ - 'profile' => $profile, + $options = [ 'region' => $region, - 'version' => self::LATEST_VERSION - ]); + 'version' => self::LATEST_VERSION, + ]; + if (!empty($profile)) { + $options['profile'] = $profile; + } + + // Create AWS Secrets Manager client + $this->client = new SecretsManagerClient($options); if ($this->client === null) { throw new TestFrameworkException("Unable to create AWS Secrets Manager client"); } From c0a885cbc5e75e67925736152dc592105021563d Mon Sep 17 00:00:00 2001 From: Ji Lu Date: Fri, 24 Jan 2020 13:21:26 -0600 Subject: [PATCH 7/7] MQE-1919: MFTF AWS Secrets Manager - CI Use --- docs/credentials.md | 32 ++-- .../Handlers/CredentialStore.php | 155 +++++++++++++++--- .../Handlers/PersistedObjectHandler.php | 8 +- .../AwsSecretsManagerStorage.php | 15 ++ .../Handlers/SecretStorage/BaseStorage.php | 24 ++- .../Handlers/SecretStorage/VaultStorage.php | 18 +- .../Util/RuntimeDataReferenceResolver.php | 6 +- .../Collector/ExceptionCollector.php | 20 +++ .../Module/MagentoWebDriver.php | 6 + 9 files changed, 231 insertions(+), 53 deletions(-) diff --git a/docs/credentials.md b/docs/credentials.md index 1ea91a5c3..402030985 100644 --- a/docs/credentials.md +++ b/docs/credentials.md @@ -147,31 +147,33 @@ AWS Secrets Manager offers secret management that supports: #### Use AWS Secrets Manager from your own AWS account -- AWS account with Secrets Manager service available -- IAM User or Role is created with appropriate AWS Secrets Manger access permission +- An AWS account with Secrets Manager service +- An IAM user with AWS Secrets Manager access permission -#### Use AWS Secrets Manager from other AWS account +#### Use AWS Secrets Manager in CI/CD - AWS account ID where the AWS Secrets Manager service is hosted -- IAM User or Role with appropriate access permission +- Authorized CI/CD EC2 instances with AWS Secrets Manager service access IAM role attached ### Store secrets in AWS Secrets Manager - #### Secrets format `Secret Name` and `Secret Value` are two key pieces of information for creating a secret. `Secret Value` can be either plaintext or key/value pairs in JSON format. -`Secrets Name` must use the following format: +`Secret Name` must use the following format: ```conf mftf// ``` -`Secrets Value` in plaintext format can be any content you want to secure. `Secrets Value` in key/value pairs format, however, the `key` must be same as the `Secret Name` with `mftf//` part removed. -e.g. in above example, `key` should be `` +`Secret Value` can be stored in two different formats: plaintext or key/value pairs. + +For plaintext format, `Secret Value` can be any string you want to secure. + +For key/value pairs format, `Secret Value` is a key/value pair with `key` the same as `Secret Name` without `mftf//` prefix, which is ``, and value can be any string you want to secure. ##### Create Secrets using AWS CLI @@ -181,8 +183,11 @@ aws secretsmanager create-secret --name "mftf/magento/shipping/carriers_usps_use ##### Create Secrets using AWS Console -To save the same secret in key/value JSON format, you should use - +- Sign in to the AWS Secrets Manager console +- Choose Store a new secret +- In the Select secret type section, specify "Other type of secret" +- For `Secret Name`, `Secret Key` and `Secret Value` field, for example, to save the same secret in key/value JSON format, you should use + ```conf # Secret Name mftf/magento/shipping/carriers_usps_userid @@ -210,9 +215,8 @@ CREDENTIAL_AWS_SECRETS_MANAGER_PROFILE=default ### Optionally set CREDENTIAL_AWS_ACCOUNT_ID environment variable -Full AWS KMS ([Key Management Service][]) key ARN ([Amazon Resource Name][]) is required when accessing secrets stored in other AWS account. -If this is the case, you will also need to set `CREDENTIAL_AWS_ACCOUNT_ID` environment variable so that MFTF can construct the full ARN. -This is also commonly used in CI system. +In case AWS credentials cannot resolve to a valid AWS account, full AWS KMS ([Key Management Service][]) key ARN ([Amazon Resource Name][]) is required. +You will also need to set `CREDENTIAL_AWS_ACCOUNT_ID` environment variable so that MFTF can construct the full ARN. This is mostly used for CI/CD. ```bash export CREDENTIAL_AWS_ACCOUNT_ID= @@ -236,7 +240,7 @@ Define the value as a reference to the corresponding key in the credentials file - `_CREDS` is an environment constant pointing to the `.credentials` file - `my_data_key` is a key in the the `.credentials` file or vault that contains the value to be used in a test step - - for File Storage, ensure your key contains the vendor prefix, i.e. `vendor/my_data_key` + - for File Storage, ensure your key contains the vendor prefix, which is `vendor/my_data_key` For example, to reference secret data in the [`fillField`][] action, use the `userInput` attribute using a typical File Storage: diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php index b66d19799..d6e6b9a69 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php @@ -6,7 +6,9 @@ namespace Magento\FunctionalTestingFramework\DataGenerator\Handlers; +use Magento\FunctionalTestingFramework\Exceptions\Collector\ExceptionCollector; use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; +use Magento\FunctionalTestingFramework\DataGenerator\Handlers\SecretStorage\BaseStorage; use Magento\FunctionalTestingFramework\DataGenerator\Handlers\SecretStorage\FileStorage; use Magento\FunctionalTestingFramework\DataGenerator\Handlers\SecretStorage\VaultStorage; use Magento\FunctionalTestingFramework\DataGenerator\Handlers\SecretStorage\AwsSecretsManagerStorage; @@ -24,10 +26,17 @@ class CredentialStore /** * Credential storage array * - * @var array + * @var BaseStorage[] */ private $credStorage = []; + /** + * Boolean to indicate if credential storage have been initialized + * + * @var boolean + */ + private $initialized; + /** * Singleton instance * @@ -35,11 +44,17 @@ class CredentialStore */ private static $INSTANCE = null; + /** + * Exception contexts + * + * @var ExceptionCollector + */ + private $exceptionContexts; + /** * Static singleton getter for CredentialStore Instance * * @return CredentialStore - * @throws TestFrameworkException */ public static function getInstance() { @@ -52,21 +67,11 @@ public static function getInstance() /** * CredentialStore constructor - * - * @throws TestFrameworkException */ private function __construct() { - // Initialize credential storage by defined order of precedence as the following - $this->initializeFileStorage(); - $this->initializeVaultStorage(); - $this->initializeAwsSecretsManagerStorage(); - - if (empty($this->credStorage)) { - throw new TestFrameworkException( - 'Invalid Credential Storage. ' . self::CREDENTIAL_STORAGE_INFO . '.' - ); - } + $this->initialized = false; + $this->exceptionContexts = new ExceptionCollector(); } /** @@ -78,6 +83,9 @@ private function __construct() */ public function getSecret($key) { + // Initialize credential storage if it's not been done + $this->initializeCredentialStorage(); + // Get secret data from storage according to the order they are stored which follows this precedence: // FileStorage > VaultStorage > AwsSecretsManagerStorage foreach ($this->credStorage as $storage) { @@ -87,9 +95,12 @@ public function getSecret($key) } } + $exceptionContexts = $this->getExceptionContexts(); + $this->resetExceptionContext(); throw new TestFrameworkException( "{$key} not found. " . self::CREDENTIAL_STORAGE_INFO - . ' and ensure key, value exists to use _CREDS in tests.' + . " and ensure key, value exists to use _CREDS in tests." + . $exceptionContexts ); } @@ -97,28 +108,112 @@ public function getSecret($key) * Return decrypted input value * * @param string $value - * @return string + * @return string|false The decrypted string on success or false on failure + * @throws TestFrameworkException */ public function decryptSecretValue($value) { - // Loop through storage to decrypt value - foreach ($this->credStorage as $storage) { - return $storage->getDecryptedValue($value); - } + // Initialize credential storage if it's not been done + $this->initializeCredentialStorage(); + + // Decrypt secret value + return BaseStorage::getDecryptedValue($value); } /** * Return decrypted values for all occurrences from input string * * @param string $string - * @return mixed + * @return string|false The decrypted string on success or false on failure + * @throws TestFrameworkException */ public function decryptAllSecretsInString($string) { - // Loop through storage to decrypt all occurrences from input string - foreach ($this->credStorage as $storage) { - return $storage->getAllDecryptedValuesInString($string); + // Initialize credential storage if it's not been done + $this->initializeCredentialStorage(); + + // Decrypt all secret values in string + return BaseStorage::getAllDecryptedValuesInString($string); + } + + /** + * Setter for exception contexts + * + * @param string $type + * @param string $context + * @return void + */ + public function setExceptionContexts($type, $context) + { + $typeArray = [self::ARRAY_KEY_FOR_FILE, self::ARRAY_KEY_FOR_VAULT, self::ARRAY_KEY_FOR_AWS_SECRETS_MANAGER]; + if (in_array($type, $typeArray) && !empty($context)) { + $this->exceptionContexts->addError($type, $context); + } + } + + /** + * Return collected exception contexts + * + * @return string + */ + private function getExceptionContexts() + { + // Gather all exceptions collected + $exceptionMessage = "\n"; + foreach ($this->exceptionContexts->getErrors() as $type => $exceptions) { + $exceptionMessage .= "\nException from "; + if ($type == self::ARRAY_KEY_FOR_FILE) { + $exceptionMessage .= "File Storage: \n"; + } + if ($type == self::ARRAY_KEY_FOR_VAULT) { + $exceptionMessage .= "Vault Storage: \n"; + } + if ($type == self::ARRAY_KEY_FOR_AWS_SECRETS_MANAGER) { + $exceptionMessage .= "AWS Secrets Manager Storage: \n"; + } + + if (is_array($exceptions)) { + $exceptionMessage .= implode("\n", $exceptions) . "\n"; + } else { + $exceptionMessage .= $exceptions . "\n"; + } + } + return $exceptionMessage; + } + + /** + * Reset exception contexts to empty array + * + * @return void + */ + private function resetExceptionContext() + { + $this->exceptionContexts->reset(); + } + + /** + * Initialize all available credential storage + * + * @return void + * @throws TestFrameworkException + */ + private function initializeCredentialStorage() + { + if (!$this->initialized) { + // Initialize credential storage by defined order of precedence as the following + $this->initializeFileStorage(); + $this->initializeVaultStorage(); + $this->initializeAwsSecretsManagerStorage(); + $this->initialized = true; + } + + if (empty($this->credStorage)) { + throw new TestFrameworkException( + 'Invalid Credential Storage. ' . self::CREDENTIAL_STORAGE_INFO + . '.' . $this->getExceptionContexts() + ); } + $this->resetExceptionContext(); } /** @@ -132,6 +227,10 @@ private function initializeFileStorage() try { $this->credStorage[self::ARRAY_KEY_FOR_FILE] = new FileStorage(); } catch (TestFrameworkException $e) { + // Print error message in console + print_r($e->getMessage()); + // Save to exception context for Allure report + $this->setExceptionContexts(self::ARRAY_KEY_FOR_FILE, $e->getMessage()); } } @@ -152,6 +251,10 @@ private function initializeVaultStorage() '/' . trim($cvSecretPath, '/') ); } catch (TestFrameworkException $e) { + // Print error message in console + print_r($e->getMessage()); + // Save to exception context for Allure report + $this->setExceptionContexts(self::ARRAY_KEY_FOR_VAULT, $e->getMessage()); } } } @@ -181,6 +284,10 @@ private function initializeAwsSecretsManagerStorage() $awsId ); } catch (TestFrameworkException $e) { + // Print error message in console + print_r($e->getMessage()); + // Save to exception context for Allure report + $this->setExceptionContexts(self::ARRAY_KEY_FOR_AWS_SECRETS_MANAGER, $e->getMessage()); } } } diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/PersistedObjectHandler.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/PersistedObjectHandler.php index 00353856b..927c0ab4e 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/PersistedObjectHandler.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/PersistedObjectHandler.php @@ -89,10 +89,12 @@ public function createEntity( foreach ($overrideFields as $index => $field) { try { - $overrideFields[$index] = CredentialStore::getInstance()->decryptAllSecretsInString($field); + $decrptedField = CredentialStore::getInstance()->decryptAllSecretsInString($field); + if ($decrptedField !== false) { + $overrideFields[$index] = $decrptedField; + } } catch (TestFrameworkException $e) { - //do not rethrow if Credentials are not defined - $overrideFields[$index] = $field; + //catch exception if Credentials are not defined } } diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/AwsSecretsManagerStorage.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/AwsSecretsManagerStorage.php index bb6044c7f..6bd1ff144 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/AwsSecretsManagerStorage.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/AwsSecretsManagerStorage.php @@ -7,6 +7,7 @@ namespace Magento\FunctionalTestingFramework\DataGenerator\Handlers\SecretStorage; use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig; +use Magento\FunctionalTestingFramework\DataGenerator\Handlers\CredentialStore; use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; use Magento\FunctionalTestingFramework\Util\Logger\LoggingUtil; use Aws\SecretsManager\SecretsManagerClient; @@ -117,17 +118,31 @@ public function getEncryptedValue($key) } catch (AwsException $e) { $errMessage = "\nAWS Exception:\n" . $e->getAwsErrorMessage() . "\nUnable to read value for key {$key} from AWS Secrets Manager\n"; + // Print error message in console print_r($errMessage); + // Add error message in mftf log if verbose is enable if (MftfApplicationConfig::getConfig()->verboseEnabled()) { LoggingUtil::getInstance()->getLogger(AwsSecretsManagerStorage::class)->debug($errMessage); } + // Save to exception context for Allure report + CredentialStore::getInstance()->setExceptionContexts( + CredentialStore::ARRAY_KEY_FOR_AWS_SECRETS_MANAGER, + $errMessage + ); } catch (\Exception $e) { $errMessage = "\nException:\n" . $e->getMessage() . "\nUnable to read value for key {$key} from AWS Secrets Manager\n"; + // Print error message in console print_r($errMessage); + // Add error message in mftf log if verbose is enable if (MftfApplicationConfig::getConfig()->verboseEnabled()) { LoggingUtil::getInstance()->getLogger(AwsSecretsManagerStorage::class)->debug($errMessage); } + // Save to exception context for Allure report + CredentialStore::getInstance()->setExceptionContexts( + CredentialStore::ARRAY_KEY_FOR_AWS_SECRETS_MANAGER, + $errMessage + ); } return $reValue; } diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/BaseStorage.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/BaseStorage.php index cb892a545..6eb7fa7e8 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/BaseStorage.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/BaseStorage.php @@ -58,30 +58,38 @@ public function getEncryptedValue($key) /** * Takes a value encrypted at runtime and decrypts it using the object's initial vector + * return the decrypted string on success or false on failure * * @param string $value - * @return string + * @return string|false The decrypted string on success or false on failure */ - public function getDecryptedValue($value) + public static function getDecryptedValue($value) { return openssl_decrypt($value, self::ENCRYPTION_ALGO, self::$encodedKey, 0, self::$iv); } /** * Takes a string that contains encrypted data at runtime and decrypts each value + * return false if no decryption happens or a failure occurs * * @param string $string - * @return mixed + * @return string|false The decrypted string on success or false on failure */ - public function getAllDecryptedValuesInString($string) + public static function getAllDecryptedValuesInString($string) { - $newString = $string; + $decrypted = false; foreach (self::$cachedSecretData as $key => $secretValue) { - if (strpos($newString, $secretValue) !== false) { + if (strpos($string, $secretValue) !== false) { $decryptedValue = self::getDecryptedValue($secretValue); - $newString = str_replace($secretValue, $decryptedValue, $newString); + if ($decryptedValue === false) { + return false; + } + if (!$decrypted) { + $decrypted = true; + } + $string = str_replace($secretValue, $decryptedValue, $string); } } - return $newString; + return $decrypted ? $string : false; } } diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php index 798ac660c..39839e8f9 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php @@ -7,6 +7,7 @@ namespace Magento\FunctionalTestingFramework\DataGenerator\Handlers\SecretStorage; use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig; +use Magento\FunctionalTestingFramework\DataGenerator\Handlers\CredentialStore; use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; use Magento\FunctionalTestingFramework\Util\Logger\LoggingUtil; use Vault\Client; @@ -123,10 +124,14 @@ public function getEncryptedValue($key) $reValue = openssl_encrypt($value, parent::ENCRYPTION_ALGO, parent::$encodedKey, 0, parent::$iv); parent::$cachedSecretData[$key] = $reValue; } catch (\Exception $e) { + $errMessage = "\nUnable to read secret for key name {$key} from vault." . $e->getMessage(); + // Print error message in console + print_r($errMessage); + // Save to exception context for Allure report + CredentialStore::getInstance()->setExceptionContexts('vault', $errMessage); + // Add error message in mftf log if verbose is enable if (MftfApplicationConfig::getConfig()->verboseEnabled()) { - LoggingUtil::getInstance()->getLogger(VaultStorage::class)->debug( - "Unable to read secret for key name {$key} from vault" - ); + LoggingUtil::getInstance()->getLogger(VaultStorage::class)->debug($errMessage); } } return $reValue; @@ -149,6 +154,13 @@ private function authenticated() return true; } } catch (\Exception $e) { + // Print error message in console + print_r($e->getMessage()); + // Save to exception context for Allure report + CredentialStore::getInstance()->setExceptionContexts( + CredentialStore::ARRAY_KEY_FOR_VAULT, + $e->getMessage() + ); } return false; } diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Util/RuntimeDataReferenceResolver.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Util/RuntimeDataReferenceResolver.php index a09afd01a..220d4ab7f 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Util/RuntimeDataReferenceResolver.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Util/RuntimeDataReferenceResolver.php @@ -9,6 +9,7 @@ use Magento\FunctionalTestingFramework\DataGenerator\Handlers\CredentialStore; use Magento\FunctionalTestingFramework\DataGenerator\Handlers\DataObjectHandler; use Magento\FunctionalTestingFramework\Exceptions\TestReferenceException; +use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; use Magento\FunctionalTestingFramework\Test\Objects\ActionObject; /** @@ -23,7 +24,7 @@ class RuntimeDataReferenceResolver implements DataReferenceResolverInterface * @param string $originalDataEntity * @return array|false|string|null * @throws TestReferenceException - * @throws \Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException + * @throws TestFrameworkException */ public function getDataReference(string $data, string $originalDataEntity) { @@ -43,6 +44,9 @@ public function getDataReference(string $data, string $originalDataEntity) case ActionObject::__CREDS: $value = CredentialStore::getInstance()->getSecret($var); $result = CredentialStore::getInstance()->decryptSecretValue($value); + if ($result === false) { + throw new TestFrameworkException("\nFailed to decrypt value {$value}\n"); + } $result = str_replace($matches['reference'], $result, $data); break; default: diff --git a/src/Magento/FunctionalTestingFramework/Exceptions/Collector/ExceptionCollector.php b/src/Magento/FunctionalTestingFramework/Exceptions/Collector/ExceptionCollector.php index c1e023ef4..f64b35446 100644 --- a/src/Magento/FunctionalTestingFramework/Exceptions/Collector/ExceptionCollector.php +++ b/src/Magento/FunctionalTestingFramework/Exceptions/Collector/ExceptionCollector.php @@ -43,6 +43,26 @@ public function throwException() throw new \Exception("\n" . $errorMsg); } + /** + * Return all errors + * + * @return array + */ + public function getErrors() + { + return $this->errors ?? []; + } + + /** + * Reset error to empty array + * + * @return void + */ + public function reset() + { + $this->errors = []; + } + /** * If there are multiple exceptions for a single file, the function flattens the array so they can be printed * as separate messages. diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php index f44d7e1a0..cd2768a06 100644 --- a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php +++ b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php @@ -769,6 +769,9 @@ public function fillSecretField($field, $value) // decrypted value $decryptedValue = CredentialStore::getInstance()->decryptSecretValue($value); + if ($decryptedValue === false) { + throw new TestFrameworkException("\nFailed to decrypt value {$value} for field {$field}\n"); + } $this->fillField($field, $decryptedValue); } @@ -788,6 +791,9 @@ public function magentoCLISecret($command, $timeout = null, $arguments = null) // decrypted value $decryptedCommand = CredentialStore::getInstance()->decryptAllSecretsInString($command); + if ($decryptedCommand === false) { + throw new TestFrameworkException("\nFailed to decrypt magentoCLI command {$command}\n"); + } return $this->magentoCLI($decryptedCommand, $timeout, $arguments); }