diff --git a/composer.json b/composer.json index 158287fbf..ca1c02572 100755 --- a/composer.json +++ b/composer.json @@ -11,7 +11,10 @@ "require": { "php": "7.0.2||7.0.4||~7.0.6||~7.1.0||~7.2.0||~7.3.0", "ext-curl": "*", + "ext-json": "*", + "ext-openssl": "*", "allure-framework/allure-codeception": "~1.3.0", + "aws/aws-sdk-php": "^3.132", "codeception/codeception": "~2.4.5", "composer/composer": "^1.4", "consolidation/robo": "^1.0.0", diff --git a/composer.lock b/composer.lock index 8f2fcb8e9..6bfc30ab9 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "59e95cc1ae6311e93111bd7ced180d29", + "content-hash": "2325a3a38edb33b24f6e33bd3009fd8a", "packages": [ { "name": "allure-framework/allure-codeception", @@ -109,6 +109,90 @@ ], "time": "2016-12-07T12:15:46+00:00" }, + { + "name": "aws/aws-sdk-php", + "version": "3.132.2", + "source": { + "type": "git", + "url": "https://github.com/aws/aws-sdk-php.git", + "reference": "f890689c6db27625522ea2e7e9b8420b6fccb063" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/aws/aws-sdk-php/zipball/f890689c6db27625522ea2e7e9b8420b6fccb063", + "reference": "f890689c6db27625522ea2e7e9b8420b6fccb063", + "shasum": "" + }, + "require": { + "ext-json": "*", + "ext-pcre": "*", + "ext-simplexml": "*", + "guzzlehttp/guzzle": "^5.3.3|^6.2.1|^7.0", + "guzzlehttp/promises": "^1.0", + "guzzlehttp/psr7": "^1.4.1", + "mtdowling/jmespath.php": "^2.5", + "php": ">=5.5" + }, + "require-dev": { + "andrewsville/php-token-reflection": "^1.4", + "aws/aws-php-sns-message-validator": "~1.0", + "behat/behat": "~3.0", + "doctrine/cache": "~1.4", + "ext-dom": "*", + "ext-openssl": "*", + "ext-pcntl": "*", + "ext-sockets": "*", + "nette/neon": "^2.3", + "phpunit/phpunit": "^4.8.35|^5.4.3", + "psr/cache": "^1.0", + "psr/simple-cache": "^1.0", + "sebastian/comparator": "^1.2.3" + }, + "suggest": { + "aws/aws-php-sns-message-validator": "To validate incoming SNS notifications", + "doctrine/cache": "To use the DoctrineCacheAdapter", + "ext-curl": "To send requests using cURL", + "ext-openssl": "Allows working with CloudFront private distributions and verifying received SNS messages", + "ext-sockets": "To use client-side monitoring" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "3.0-dev" + } + }, + "autoload": { + "psr-4": { + "Aws\\": "src/" + }, + "files": [ + "src/functions.php" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "Apache-2.0" + ], + "authors": [ + { + "name": "Amazon Web Services", + "homepage": "http://aws.amazon.com" + } + ], + "description": "AWS SDK for PHP - Use Amazon Web Services in your PHP project", + "homepage": "http://aws.amazon.com/sdkforphp", + "keywords": [ + "amazon", + "aws", + "cloud", + "dynamodb", + "ec2", + "glacier", + "s3", + "sdk" + ], + "time": "2020-01-09T19:09:31+00:00" + }, { "name": "behat/gherkin", "version": "v4.4.5", @@ -2542,8 +2626,66 @@ "bcmath", "math" ], + "abandoned": "brick/math", "time": "2017-02-16T16:54:46+00:00" }, + { + "name": "mtdowling/jmespath.php", + "version": "2.5.0", + "source": { + "type": "git", + "url": "https://github.com/jmespath/jmespath.php.git", + "reference": "52168cb9472de06979613d365c7f1ab8798be895" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/jmespath/jmespath.php/zipball/52168cb9472de06979613d365c7f1ab8798be895", + "reference": "52168cb9472de06979613d365c7f1ab8798be895", + "shasum": "" + }, + "require": { + "php": ">=5.4.0", + "symfony/polyfill-mbstring": "^1.4" + }, + "require-dev": { + "composer/xdebug-handler": "^1.2", + "phpunit/phpunit": "^4.8.36|^7.5.15" + }, + "bin": [ + "bin/jp.php" + ], + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "2.5-dev" + } + }, + "autoload": { + "psr-4": { + "JmesPath\\": "src/" + }, + "files": [ + "src/JmesPath.php" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Michael Dowling", + "email": "mtdowling@gmail.com", + "homepage": "https://github.com/mtdowling" + } + ], + "description": "Declaratively specify how to extract elements from a JSON document", + "keywords": [ + "json", + "jsonpath" + ], + "time": "2019-12-30T18:03:34+00:00" + }, { "name": "mustache/mustache", "version": "v2.12.0", @@ -6542,7 +6684,9 @@ "prefer-lowest": false, "platform": { "php": "7.0.2||7.0.4||~7.0.6||~7.1.0||~7.2.0||~7.3.0", - "ext-curl": "*" + "ext-curl": "*", + "ext-json": "*", + "ext-openssl": "*" }, "platform-dev": [] } diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/SecretStorage/AwsSecretsManagerStorageTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/SecretStorage/AwsSecretsManagerStorageTest.php new file mode 100644 index 000000000..e1f4e4879 --- /dev/null +++ b/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/SecretStorage/AwsSecretsManagerStorageTest.php @@ -0,0 +1,65 @@ + 'mftf/magento/' . $testShortKey, + 'SecretString' => json_encode([$testShortKey => $testValue]) + ]; + /** @var Result */ + $result = new Result($data); + + $mockClient = $this->getMockBuilder(SecretsManagerClient::class) + ->disableOriginalConstructor() + ->setMethods(['__call']) + ->getMock(); + + $mockClient->expects($this->once()) + ->method('__call') + ->willReturnCallback(function ($name, $args) use ($result) { + return $result; + }); + + /** @var SecretsManagerClient */ + $credentialStorage = new AwsSecretsManagerStorage($testRegion, $testProfile); + $reflection = new ReflectionClass($credentialStorage); + $reflection_property = $reflection->getProperty('client'); + $reflection_property->setAccessible(true); + $reflection_property->setValue($credentialStorage, $mockClient); + + // Test getEncryptedValue() + $encryptedCred = $credentialStorage->getEncryptedValue($testLongKey); + + // Assert the value we've gotten is in fact not identical to our test value + $this->assertNotEquals($testValue, $encryptedCred); + + // Test getDecryptedValue() + $actualValue = $credentialStorage->getDecryptedValue($encryptedCred); + + // Assert that we are able to successfully decrypt our secret value + $this->assertEquals($testValue, $actualValue); + } +} diff --git a/docs/configuration.md b/docs/configuration.md index 5140c01e7..9466f2bcc 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -277,6 +277,28 @@ Example: CREDENTIAL_VAULT_SECRET_BASE_PATH=secret ``` +### CREDENTIAL_AWS_SECRETS_MANAGER_REGION + +The region that AWS Secrets Manager is located. + +Example: + +```conf +# Region of AWS Secrets Manager +CREDENTIAL_AWS_SECRETS_MANAGER_REGION=us-east-1 +``` + +### CREDENTIAL_AWS_SECRETS_MANAGER_PROFILE + +The profile used to connect to AWS Secrets Manager. + +Example: + +```conf +# Profile used to connect to AWS Secrets Manager. +CREDENTIAL_AWS_SECRETS_MANAGER_PROFILE=default +``` + ### ENABLE_BROWSER_LOG Enables addition of browser logs to Allure steps diff --git a/docs/credentials.md b/docs/credentials.md index a2850cfe8..402030985 100644 --- a/docs/credentials.md +++ b/docs/credentials.md @@ -3,10 +3,11 @@ When you test functionality that involves external services such as UPS, FedEx, PayPal, or SignifyD, use the MFTF credentials feature to hide sensitive [data][] like integration tokens and API keys. -Currently the MFTF supports two types of credential storage: +Currently the MFTF supports three types of credential storage: - **.credentials file** -- **HashiCorp vault** +- **HashiCorp Vault** +- **AWS Secrets Manager** ## Configure File Storage @@ -135,11 +136,100 @@ CREDENTIAL_VAULT_ADDRESS=http://127.0.0.1:8200 CREDENTIAL_VAULT_SECRET_BASE_PATH=secret ``` -## Configure both File Storage and Vault Storage +## Configure AWS Secrets Manager -It is possible and sometimes useful to setup and use both `.credentials` file and vault for secret storage at the same time. -In this case, the MFTF tests are able to read secret data at runtime from both storage options, but the local `.credentials` file will take precedence. +AWS Secrets Manager offers secret management that supports: +- Secret rotation with built-in integration for Amazon RDS, Amazon Redshift, and Amazon DocumentDB +- Fine-grained policies and permissions +- Audit secret rotation centrally for resources in the AWS Cloud, third-party services, and on-premises +### Prerequisites + +#### Use AWS Secrets Manager from your own AWS account + +- An AWS account with Secrets Manager service +- An IAM user with AWS Secrets Manager access permission + +#### Use AWS Secrets Manager in CI/CD + +- AWS account ID where the AWS Secrets Manager service is hosted +- Authorized CI/CD EC2 instances with AWS Secrets Manager service access IAM role attached + +### Store secrets in AWS Secrets Manager + +#### Secrets format + +`Secret Name` and `Secret Value` are two key pieces of information for creating a secret. + +`Secret Value` can be either plaintext or key/value pairs in JSON format. + +`Secret Name` must use the following format: + +```conf +mftf// +``` + +`Secret Value` can be stored in two different formats: plaintext or key/value pairs. + +For plaintext format, `Secret Value` can be any string you want to secure. + +For key/value pairs format, `Secret Value` is a key/value pair with `key` the same as `Secret Name` without `mftf//` prefix, which is ``, and value can be any string you want to secure. + +##### Create Secrets using AWS CLI + +```bash +aws secretsmanager create-secret --name "mftf/magento/shipping/carriers_usps_userid" --description "Carriers USPS user id" --secret-string "1234567" +``` + +##### Create Secrets using AWS Console + +- Sign in to the AWS Secrets Manager console +- Choose Store a new secret +- In the Select secret type section, specify "Other type of secret" +- For `Secret Name`, `Secret Key` and `Secret Value` field, for example, to save the same secret in key/value JSON format, you should use + +```conf +# Secret Name +mftf/magento/shipping/carriers_usps_userid + +# Secret Key +shipping/carriers_usps_userid + +# Secret Value +1234567 +``` + +### Setup MFTF to use AWS Secrets Manager + +To use AWS Secrets Manager, the AWS region to connect to is required. You can set it through environment variable [`CREDENTIAL_AWS_SECRETS_MANAGER_REGION`][] in `.env`. + +MFTF uses the recommended [Default Credential Provider Chain][credential chain] to establish connection to AWS Secrets Manager service. +You can setup credentials according to [Default Credential Provider Chain][credential chain] and there is no MFTF specific setup required. +Optionally, however, you can explicitly set AWS profile through environment variable [`CREDENTIAL_AWS_SECRETS_MANAGER_PROFILE`][] in `.env`. + +```conf +# Sample AWS Secrets Manager configuration +CREDENTIAL_AWS_SECRETS_MANAGER_REGION=us-east-1 +CREDENTIAL_AWS_SECRETS_MANAGER_PROFILE=default +``` + +### Optionally set CREDENTIAL_AWS_ACCOUNT_ID environment variable + +In case AWS credentials cannot resolve to a valid AWS account, full AWS KMS ([Key Management Service][]) key ARN ([Amazon Resource Name][]) is required. +You will also need to set `CREDENTIAL_AWS_ACCOUNT_ID` environment variable so that MFTF can construct the full ARN. This is mostly used for CI/CD. + +```bash +export CREDENTIAL_AWS_ACCOUNT_ID= +``` + +## Configure multiple credential storage + +It is possible and sometimes useful to setup and use multiple credential storage at the same time. +In this case, the MFTF tests are able to read secret data at runtime from all storage options, in this case MFTF use the following precedence: + +``` +.credentials File > HashiCorp Vault > AWS Secrets Manager +``` ## Use credentials in a test @@ -150,7 +240,7 @@ Define the value as a reference to the corresponding key in the credentials file - `_CREDS` is an environment constant pointing to the `.credentials` file - `my_data_key` is a key in the the `.credentials` file or vault that contains the value to be used in a test step - - for File Storage, ensure your key contains the vendor prefix, i.e. `vendor/my_data_key` + - for File Storage, ensure your key contains the vendor prefix, which is `vendor/my_data_key` For example, to reference secret data in the [`fillField`][] action, use the `userInput` attribute using a typical File Storage: @@ -183,3 +273,8 @@ The MFTF tests delivered with Magento application do not use credentials and do [Vault KV2]: https://www.vaultproject.io/docs/secrets/kv/kv-v2.html [`CREDENTIAL_VAULT_ADDRESS`]: configuration.md#credential_vault_address [`CREDENTIAL_VAULT_SECRET_BASE_PATH`]: configuration.md#credential_vault_secret_base_path +[credential chain]: https://docs.aws.amazon.com/sdk-for-php/v3/developer-guide/guide_credentials.html +[`CREDENTIAL_AWS_SECRETS_MANAGER_PROFILE`]: configuration.md#credential_aws_secrets_manager_profile +[`CREDENTIAL_AWS_SECRETS_MANAGER_REGION`]: configuration.md#credential_aws_secrets_manager_region +[Key Management Service]: https://aws.amazon.com/kms/ +[Amazon Resource Name]: https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html \ No newline at end of file diff --git a/etc/config/.env.example b/etc/config/.env.example index 7320d8b8b..f5b6ef40e 100644 --- a/etc/config/.env.example +++ b/etc/config/.env.example @@ -30,10 +30,14 @@ BROWSER=chrome #MAGENTO_RESTAPI_SERVER_PORT=8080 #MAGENTO_RESTAPI_SERVER_PROTOCOL=https -#*** Uncomment and set vault address and secret base path if you want to use vault to manage _CREDS secrets ***# +#*** To use HashiCorp Vault to manage _CREDS secrets, uncomment and set vault address and secret base path ***# #CREDENTIAL_VAULT_ADDRESS=http://127.0.0.1:8200 #CREDENTIAL_VAULT_SECRET_BASE_PATH=secret +#*** To use AWS Secrets Manager to manage _CREDS secrets, uncomment and set region, profile is optional, when omitted, AWS default credential provider chain will be used ***# +#CREDENTIAL_AWS_SECRETS_MANAGER_PROFILE=default +#CREDENTIAL_AWS_SECRETS_MANAGER_REGION=us-east-1 + #*** Uncomment these properties to set up a dev environment with symlinked projects ***# #TESTS_BP= #FW_BP= diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php index 94ff40069..d6e6b9a69 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php @@ -6,23 +6,37 @@ namespace Magento\FunctionalTestingFramework\DataGenerator\Handlers; +use Magento\FunctionalTestingFramework\Exceptions\Collector\ExceptionCollector; use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; +use Magento\FunctionalTestingFramework\DataGenerator\Handlers\SecretStorage\BaseStorage; use Magento\FunctionalTestingFramework\DataGenerator\Handlers\SecretStorage\FileStorage; use Magento\FunctionalTestingFramework\DataGenerator\Handlers\SecretStorage\VaultStorage; +use Magento\FunctionalTestingFramework\DataGenerator\Handlers\SecretStorage\AwsSecretsManagerStorage; use Magento\FunctionalTestingFramework\Util\Path\UrlFormatter; class CredentialStore { const ARRAY_KEY_FOR_VAULT = 'vault'; const ARRAY_KEY_FOR_FILE = 'file'; + const ARRAY_KEY_FOR_AWS_SECRETS_MANAGER = 'aws'; + + const CREDENTIAL_STORAGE_INFO = 'You need to configure at least one of these options: ' + . '.credentials file, HashiCorp Vault or AWS Secrets Manager correctly'; /** * Credential storage array * - * @var array + * @var BaseStorage[] */ private $credStorage = []; + /** + * Boolean to indicate if credential storage have been initialized + * + * @var boolean + */ + private $initialized; + /** * Singleton instance * @@ -30,11 +44,17 @@ class CredentialStore */ private static $INSTANCE = null; + /** + * Exception contexts + * + * @var ExceptionCollector + */ + private $exceptionContexts; + /** * Static singleton getter for CredentialStore Instance * * @return CredentialStore - * @throws TestFrameworkException */ public static function getInstance() { @@ -47,35 +67,11 @@ public static function getInstance() /** * CredentialStore constructor - * - * @throws TestFrameworkException */ private function __construct() { - // Initialize file storage - try { - $this->credStorage[self::ARRAY_KEY_FOR_FILE] = new FileStorage(); - } catch (TestFrameworkException $e) { - } - - // Initialize vault storage - $cvAddress = getenv('CREDENTIAL_VAULT_ADDRESS'); - $cvSecretPath = getenv('CREDENTIAL_VAULT_SECRET_BASE_PATH'); - if ($cvAddress !== false && $cvSecretPath !== false) { - try { - $this->credStorage[self::ARRAY_KEY_FOR_VAULT] = new VaultStorage( - UrlFormatter::format($cvAddress, false), - '/' . trim($cvSecretPath, '/') - ); - } catch (TestFrameworkException $e) { - } - } - - if (empty($this->credStorage)) { - throw new TestFrameworkException( - "No credential storage is properly configured. Please configure vault or .credentials file." - ); - } + $this->initialized = false; + $this->exceptionContexts = new ExceptionCollector(); } /** @@ -87,8 +83,11 @@ private function __construct() */ public function getSecret($key) { - // Get secret data from storage according to the order they are stored - // File storage is preferred over vault storage to allow local secret value overriding remote secret value + // Initialize credential storage if it's not been done + $this->initializeCredentialStorage(); + + // Get secret data from storage according to the order they are stored which follows this precedence: + // FileStorage > VaultStorage > AwsSecretsManagerStorage foreach ($this->credStorage as $storage) { $value = $storage->getEncryptedValue($key); if (null !== $value) { @@ -96,9 +95,12 @@ public function getSecret($key) } } + $exceptionContexts = $this->getExceptionContexts(); + $this->resetExceptionContext(); throw new TestFrameworkException( - "\"{$key}\" not defined in vault or .credentials file, " - . "please provide a value in order to use this secret in a test." + "{$key} not found. " . self::CREDENTIAL_STORAGE_INFO + . " and ensure key, value exists to use _CREDS in tests." + . $exceptionContexts ); } @@ -106,27 +108,187 @@ public function getSecret($key) * Return decrypted input value * * @param string $value - * @return string + * @return string|false The decrypted string on success or false on failure + * @throws TestFrameworkException */ public function decryptSecretValue($value) { - // Loop through storage to decrypt value - foreach ($this->credStorage as $storage) { - return $storage->getDecryptedValue($value); - } + // Initialize credential storage if it's not been done + $this->initializeCredentialStorage(); + + // Decrypt secret value + return BaseStorage::getDecryptedValue($value); } /** * Return decrypted values for all occurrences from input string * * @param string $string - * @return mixed + * @return string|false The decrypted string on success or false on failure + * @throws TestFrameworkException */ public function decryptAllSecretsInString($string) { - // Loop through storage to decrypt all occurrences from input string - foreach ($this->credStorage as $storage) { - return $storage->getAllDecryptedValuesInString($string); + // Initialize credential storage if it's not been done + $this->initializeCredentialStorage(); + + // Decrypt all secret values in string + return BaseStorage::getAllDecryptedValuesInString($string); + } + + /** + * Setter for exception contexts + * + * @param string $type + * @param string $context + * @return void + */ + public function setExceptionContexts($type, $context) + { + $typeArray = [self::ARRAY_KEY_FOR_FILE, self::ARRAY_KEY_FOR_VAULT, self::ARRAY_KEY_FOR_AWS_SECRETS_MANAGER]; + if (in_array($type, $typeArray) && !empty($context)) { + $this->exceptionContexts->addError($type, $context); + } + } + + /** + * Return collected exception contexts + * + * @return string + */ + private function getExceptionContexts() + { + // Gather all exceptions collected + $exceptionMessage = "\n"; + foreach ($this->exceptionContexts->getErrors() as $type => $exceptions) { + $exceptionMessage .= "\nException from "; + if ($type == self::ARRAY_KEY_FOR_FILE) { + $exceptionMessage .= "File Storage: \n"; + } + if ($type == self::ARRAY_KEY_FOR_VAULT) { + $exceptionMessage .= "Vault Storage: \n"; + } + if ($type == self::ARRAY_KEY_FOR_AWS_SECRETS_MANAGER) { + $exceptionMessage .= "AWS Secrets Manager Storage: \n"; + } + + if (is_array($exceptions)) { + $exceptionMessage .= implode("\n", $exceptions) . "\n"; + } else { + $exceptionMessage .= $exceptions . "\n"; + } + } + return $exceptionMessage; + } + + /** + * Reset exception contexts to empty array + * + * @return void + */ + private function resetExceptionContext() + { + $this->exceptionContexts->reset(); + } + + /** + * Initialize all available credential storage + * + * @return void + * @throws TestFrameworkException + */ + private function initializeCredentialStorage() + { + if (!$this->initialized) { + // Initialize credential storage by defined order of precedence as the following + $this->initializeFileStorage(); + $this->initializeVaultStorage(); + $this->initializeAwsSecretsManagerStorage(); + $this->initialized = true; + } + + if (empty($this->credStorage)) { + throw new TestFrameworkException( + 'Invalid Credential Storage. ' . self::CREDENTIAL_STORAGE_INFO + . '.' . $this->getExceptionContexts() + ); + } + $this->resetExceptionContext(); + } + + /** + * Initialize file storage + * + * @return void + */ + private function initializeFileStorage() + { + // Initialize file storage + try { + $this->credStorage[self::ARRAY_KEY_FOR_FILE] = new FileStorage(); + } catch (TestFrameworkException $e) { + // Print error message in console + print_r($e->getMessage()); + // Save to exception context for Allure report + $this->setExceptionContexts(self::ARRAY_KEY_FOR_FILE, $e->getMessage()); + } + } + + /** + * Initialize Vault storage + * + * @return void + */ + private function initializeVaultStorage() + { + // Initialize vault storage + $cvAddress = getenv('CREDENTIAL_VAULT_ADDRESS'); + $cvSecretPath = getenv('CREDENTIAL_VAULT_SECRET_BASE_PATH'); + if ($cvAddress !== false && $cvSecretPath !== false) { + try { + $this->credStorage[self::ARRAY_KEY_FOR_VAULT] = new VaultStorage( + UrlFormatter::format($cvAddress, false), + '/' . trim($cvSecretPath, '/') + ); + } catch (TestFrameworkException $e) { + // Print error message in console + print_r($e->getMessage()); + // Save to exception context for Allure report + $this->setExceptionContexts(self::ARRAY_KEY_FOR_VAULT, $e->getMessage()); + } + } + } + + /** + * Initialize AWS Secrets Manager storage + * + * @return void + */ + private function initializeAwsSecretsManagerStorage() + { + // Initialize AWS Secrets Manager storage + $awsRegion = getenv('CREDENTIAL_AWS_SECRETS_MANAGER_REGION'); + $awsProfile = getenv('CREDENTIAL_AWS_SECRETS_MANAGER_PROFILE'); + $awsId = getenv('CREDENTIAL_AWS_ACCOUNT_ID'); + if (!empty($awsRegion)) { + if (empty($awsProfile)) { + $awsProfile = null; + } + if (empty($awsId)) { + $awsId = null; + } + try { + $this->credStorage[self::ARRAY_KEY_FOR_AWS_SECRETS_MANAGER] = new AwsSecretsManagerStorage( + $awsRegion, + $awsProfile, + $awsId + ); + } catch (TestFrameworkException $e) { + // Print error message in console + print_r($e->getMessage()); + // Save to exception context for Allure report + $this->setExceptionContexts(self::ARRAY_KEY_FOR_AWS_SECRETS_MANAGER, $e->getMessage()); + } } } } diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/PersistedObjectHandler.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/PersistedObjectHandler.php index 00353856b..927c0ab4e 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/PersistedObjectHandler.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/PersistedObjectHandler.php @@ -89,10 +89,12 @@ public function createEntity( foreach ($overrideFields as $index => $field) { try { - $overrideFields[$index] = CredentialStore::getInstance()->decryptAllSecretsInString($field); + $decrptedField = CredentialStore::getInstance()->decryptAllSecretsInString($field); + if ($decrptedField !== false) { + $overrideFields[$index] = $decrptedField; + } } catch (TestFrameworkException $e) { - //do not rethrow if Credentials are not defined - $overrideFields[$index] = $field; + //catch exception if Credentials are not defined } } diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/AwsSecretsManagerStorage.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/AwsSecretsManagerStorage.php new file mode 100644 index 000000000..6bd1ff144 --- /dev/null +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/AwsSecretsManagerStorage.php @@ -0,0 +1,212 @@ +createAwsSecretsManagerClient($region, $profile); + $this->region = $region; + $this->awsAccountId = $accountId; + } + + /** + * Returns the value of a secret based on corresponding key + * + * @param string $key + * @return string|null + * @throws Exception + */ + public function getEncryptedValue($key) + { + // Check if secret is in cached array + if (null !== ($value = parent::getEncryptedValue($key))) { + return $value; + } + + if (MftfApplicationConfig::getConfig()->verboseEnabled()) { + LoggingUtil::getInstance()->getLogger(AwsSecretsManagerStorage::class)->debug( + "Retrieving value for key name {$key} from AWS Secrets Manager" + ); + } + + $reValue = null; + try { + // Split vendor/key to construct secret id + list($vendor, $key) = explode('/', trim($key, '/'), 2); + // If AWS account id is specified, create and use full ARN, otherwise use partial ARN as secret id + $secretId = ''; + if (!empty($this->awsAccountId)) { + $secretId = self::AWS_SM_PARTIAL_ARN . $this->region . ':' . $this->awsAccountId . ':secret:'; + } + $secretId .= self::MFTF_PATH + . '/' + . $vendor + . '/' + . $key; + // Read value by id from AWS Secrets Manager, and parse the result + $value = $this->parseAwsSecretResult( + $this->client->getSecretValue(['SecretId' => $secretId]), + $key + ); + // Encrypt value for return + $reValue = openssl_encrypt($value, parent::ENCRYPTION_ALGO, parent::$encodedKey, 0, parent::$iv); + parent::$cachedSecretData[$key] = $reValue; + } catch (AwsException $e) { + $errMessage = "\nAWS Exception:\n" . $e->getAwsErrorMessage() + . "\nUnable to read value for key {$key} from AWS Secrets Manager\n"; + // Print error message in console + print_r($errMessage); + // Add error message in mftf log if verbose is enable + if (MftfApplicationConfig::getConfig()->verboseEnabled()) { + LoggingUtil::getInstance()->getLogger(AwsSecretsManagerStorage::class)->debug($errMessage); + } + // Save to exception context for Allure report + CredentialStore::getInstance()->setExceptionContexts( + CredentialStore::ARRAY_KEY_FOR_AWS_SECRETS_MANAGER, + $errMessage + ); + } catch (\Exception $e) { + $errMessage = "\nException:\n" . $e->getMessage() + . "\nUnable to read value for key {$key} from AWS Secrets Manager\n"; + // Print error message in console + print_r($errMessage); + // Add error message in mftf log if verbose is enable + if (MftfApplicationConfig::getConfig()->verboseEnabled()) { + LoggingUtil::getInstance()->getLogger(AwsSecretsManagerStorage::class)->debug($errMessage); + } + // Save to exception context for Allure report + CredentialStore::getInstance()->setExceptionContexts( + CredentialStore::ARRAY_KEY_FOR_AWS_SECRETS_MANAGER, + $errMessage + ); + } + return $reValue; + } + + /** + * Parse AWS result object and return secret for key + * + * @param Result $awsResult + * @param string $key + * @return string + * @throws TestFrameworkException + */ + private function parseAwsSecretResult($awsResult, $key) + { + // Return secret from the associated KMS CMK + if (isset($awsResult['SecretString'])) { + $rawSecret = $awsResult['SecretString']; + } else { + throw new TestFrameworkException( + "'SecretString' field is not set in AWS Result. Error parsing result from AWS Secrets Manager" + ); + } + + // Secrets are saved as JSON structures of key/value pairs if using AWS Secrets Manager console, and + // Secrets are saved as plain text if using AWS CLI. We need to handle both cases. + $secret = json_decode($rawSecret, true); + if (isset($secret[$key])) { + return $secret[$key]; + } elseif (is_string($rawSecret)) { + return $rawSecret; + } + throw new TestFrameworkException( + "$key not found or value is not string . Error parsing result from AWS Secrets Manager" + ); + } + + /** + * Create Aws Secrets Manager client + * + * @param string $region + * @param string $profile + * @return void + * @throws TestFrameworkException + * @throws InvalidArgumentException + */ + private function createAwsSecretsManagerClient($region, $profile) + { + if (null !== $this->client) { + return; + } + + $options = [ + 'region' => $region, + 'version' => self::LATEST_VERSION, + ]; + + if (!empty($profile)) { + $options['profile'] = $profile; + } + + // Create AWS Secrets Manager client + $this->client = new SecretsManagerClient($options); + if ($this->client === null) { + throw new TestFrameworkException("Unable to create AWS Secrets Manager client"); + } + } +} diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/BaseStorage.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/BaseStorage.php index cb892a545..6eb7fa7e8 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/BaseStorage.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/BaseStorage.php @@ -58,30 +58,38 @@ public function getEncryptedValue($key) /** * Takes a value encrypted at runtime and decrypts it using the object's initial vector + * return the decrypted string on success or false on failure * * @param string $value - * @return string + * @return string|false The decrypted string on success or false on failure */ - public function getDecryptedValue($value) + public static function getDecryptedValue($value) { return openssl_decrypt($value, self::ENCRYPTION_ALGO, self::$encodedKey, 0, self::$iv); } /** * Takes a string that contains encrypted data at runtime and decrypts each value + * return false if no decryption happens or a failure occurs * * @param string $string - * @return mixed + * @return string|false The decrypted string on success or false on failure */ - public function getAllDecryptedValuesInString($string) + public static function getAllDecryptedValuesInString($string) { - $newString = $string; + $decrypted = false; foreach (self::$cachedSecretData as $key => $secretValue) { - if (strpos($newString, $secretValue) !== false) { + if (strpos($string, $secretValue) !== false) { $decryptedValue = self::getDecryptedValue($secretValue); - $newString = str_replace($secretValue, $decryptedValue, $newString); + if ($decryptedValue === false) { + return false; + } + if (!$decrypted) { + $decrypted = true; + } + $string = str_replace($secretValue, $decryptedValue, $string); } } - return $newString; + return $decrypted ? $string : false; } } diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php index 6a9e9f0cf..39839e8f9 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php @@ -7,6 +7,7 @@ namespace Magento\FunctionalTestingFramework\DataGenerator\Handlers\SecretStorage; use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig; +use Magento\FunctionalTestingFramework\DataGenerator\Handlers\CredentialStore; use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; use Magento\FunctionalTestingFramework\Util\Logger\LoggingUtil; use Vault\Client; @@ -67,7 +68,7 @@ class VaultStorage extends BaseStorage private $secretBasePath; /** - * CredentialVault constructor + * VaultStorage constructor * * @param string $baseUrl * @param string $secretBasePath @@ -123,10 +124,14 @@ public function getEncryptedValue($key) $reValue = openssl_encrypt($value, parent::ENCRYPTION_ALGO, parent::$encodedKey, 0, parent::$iv); parent::$cachedSecretData[$key] = $reValue; } catch (\Exception $e) { + $errMessage = "\nUnable to read secret for key name {$key} from vault." . $e->getMessage(); + // Print error message in console + print_r($errMessage); + // Save to exception context for Allure report + CredentialStore::getInstance()->setExceptionContexts('vault', $errMessage); + // Add error message in mftf log if verbose is enable if (MftfApplicationConfig::getConfig()->verboseEnabled()) { - LoggingUtil::getInstance()->getLogger(VaultStorage::class)->debug( - "Unable to read secret for key name {$key} from vault" - ); + LoggingUtil::getInstance()->getLogger(VaultStorage::class)->debug($errMessage); } } return $reValue; @@ -149,6 +154,13 @@ private function authenticated() return true; } } catch (\Exception $e) { + // Print error message in console + print_r($e->getMessage()); + // Save to exception context for Allure report + CredentialStore::getInstance()->setExceptionContexts( + CredentialStore::ARRAY_KEY_FOR_VAULT, + $e->getMessage() + ); } return false; } diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Util/RuntimeDataReferenceResolver.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Util/RuntimeDataReferenceResolver.php index a09afd01a..220d4ab7f 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Util/RuntimeDataReferenceResolver.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Util/RuntimeDataReferenceResolver.php @@ -9,6 +9,7 @@ use Magento\FunctionalTestingFramework\DataGenerator\Handlers\CredentialStore; use Magento\FunctionalTestingFramework\DataGenerator\Handlers\DataObjectHandler; use Magento\FunctionalTestingFramework\Exceptions\TestReferenceException; +use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; use Magento\FunctionalTestingFramework\Test\Objects\ActionObject; /** @@ -23,7 +24,7 @@ class RuntimeDataReferenceResolver implements DataReferenceResolverInterface * @param string $originalDataEntity * @return array|false|string|null * @throws TestReferenceException - * @throws \Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException + * @throws TestFrameworkException */ public function getDataReference(string $data, string $originalDataEntity) { @@ -43,6 +44,9 @@ public function getDataReference(string $data, string $originalDataEntity) case ActionObject::__CREDS: $value = CredentialStore::getInstance()->getSecret($var); $result = CredentialStore::getInstance()->decryptSecretValue($value); + if ($result === false) { + throw new TestFrameworkException("\nFailed to decrypt value {$value}\n"); + } $result = str_replace($matches['reference'], $result, $data); break; default: diff --git a/src/Magento/FunctionalTestingFramework/Exceptions/Collector/ExceptionCollector.php b/src/Magento/FunctionalTestingFramework/Exceptions/Collector/ExceptionCollector.php index c1e023ef4..f64b35446 100644 --- a/src/Magento/FunctionalTestingFramework/Exceptions/Collector/ExceptionCollector.php +++ b/src/Magento/FunctionalTestingFramework/Exceptions/Collector/ExceptionCollector.php @@ -43,6 +43,26 @@ public function throwException() throw new \Exception("\n" . $errorMsg); } + /** + * Return all errors + * + * @return array + */ + public function getErrors() + { + return $this->errors ?? []; + } + + /** + * Reset error to empty array + * + * @return void + */ + public function reset() + { + $this->errors = []; + } + /** * If there are multiple exceptions for a single file, the function flattens the array so they can be printed * as separate messages. diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php index f44d7e1a0..cd2768a06 100644 --- a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php +++ b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php @@ -769,6 +769,9 @@ public function fillSecretField($field, $value) // decrypted value $decryptedValue = CredentialStore::getInstance()->decryptSecretValue($value); + if ($decryptedValue === false) { + throw new TestFrameworkException("\nFailed to decrypt value {$value} for field {$field}\n"); + } $this->fillField($field, $decryptedValue); } @@ -788,6 +791,9 @@ public function magentoCLISecret($command, $timeout = null, $arguments = null) // decrypted value $decryptedCommand = CredentialStore::getInstance()->decryptAllSecretsInString($command); + if ($decryptedCommand === false) { + throw new TestFrameworkException("\nFailed to decrypt magentoCLI command {$command}\n"); + } return $this->magentoCLI($decryptedCommand, $timeout, $arguments); }