From 272fb1e24f8f9dc6b0c360fe60352cab0e9ff900 Mon Sep 17 00:00:00 2001 From: Andrii Meysar Date: Wed, 3 Jul 2019 20:43:17 +0300 Subject: [PATCH 01/82] - Fix assignment of correct value for object return property - Fix operation method receive --- .../DataGenerator/Persist/Curl/AdminExecutor.php | 5 ++++- .../DataGenerator/Persist/Curl/FrontendExecutor.php | 5 ++++- .../DataGenerator/Persist/CurlHandler.php | 3 ++- .../DataGenerator/etc/dataOperation.xsd | 10 +++++++++- 4 files changed, 19 insertions(+), 4 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/AdminExecutor.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/AdminExecutor.php index 1f7ae70d7..70dafa75a 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/AdminExecutor.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/AdminExecutor.php @@ -158,7 +158,10 @@ public function read($successRegex = null, $returnRegex = null) if (!empty($returnRegex)) { preg_match($returnRegex, $this->response, $returnMatches); if (!empty($returnMatches)) { - return $returnMatches; + if (count($returnMatches) > 1) { + unset($returnMatches); + } + return reset($returnMatches); } } return $this->response; diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/FrontendExecutor.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/FrontendExecutor.php index 7e7485a82..20f8c1af8 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/FrontendExecutor.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/FrontendExecutor.php @@ -179,7 +179,10 @@ public function read($successRegex = null, $returnRegex = null) if (!empty($returnRegex)) { preg_match($returnRegex, $this->response, $returnMatches); if (!empty($returnMatches)) { - return $returnMatches; + if (count($returnMatches) > 1) { + unset($returnMatches); + } + return reset($returnMatches); } } return $this->response; diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/CurlHandler.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/CurlHandler.php index 9c8a9e175..ad81b9e9b 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/CurlHandler.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/CurlHandler.php @@ -119,6 +119,7 @@ public function executeRequest($dependentEntities) $contentType = $this->operationDefinition->getContentType(); $successRegex = $this->operationDefinition->getSuccessRegex(); $returnRegex = $this->operationDefinition->getReturnRegex(); + $method = $this->operationDefinition->getApiMethod(); $operationDataResolver = new OperationDataArrayResolver($dependentEntities); $this->requestData = $operationDataResolver->resolveOperationDataArray( @@ -156,7 +157,7 @@ public function executeRequest($dependentEntities) $executor->write( $apiUrl, $this->requestData, - self::$curlMethodMapping[$this->operation], + $method ?? self::$curlMethodMapping[$this->operation], $headers ); diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/etc/dataOperation.xsd b/src/Magento/FunctionalTestingFramework/DataGenerator/etc/dataOperation.xsd index 7a455dc2b..ccc2b8436 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/etc/dataOperation.xsd +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/etc/dataOperation.xsd @@ -26,7 +26,7 @@ - + @@ -93,4 +93,12 @@ + + + + + + + + \ No newline at end of file From b1adef436b3efdb72efe309849e8b51741317ad6 Mon Sep 17 00:00:00 2001 From: Tom Reece Date: Wed, 31 Jul 2019 15:12:25 -0500 Subject: [PATCH 02/82] MQE-1306: Can not get actual entity data --- .../Objects/EntityDataObjectTest.php | 17 +++++++++++++++++ .../DataGenerator/Objects/EntityDataObject.php | 13 ++++++++++--- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Objects/EntityDataObjectTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Objects/EntityDataObjectTest.php index 5b63f0c94..802a6c108 100644 --- a/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Objects/EntityDataObjectTest.php +++ b/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Objects/EntityDataObjectTest.php @@ -120,6 +120,23 @@ public function testGetLinkedEntities() $this->assertEquals("linkedEntity2", $dataObject->getLinkedEntitiesOfType("otherEntityType")[0]); } + public function testGetCamelCaseKeys() + { + $data = [ + "lowercasekey1" => "value1", + "camelCaseKey2" => "value2", + "lowercasekey3" => "value3", + "camelCaseKey4" => "value4" + ]; + + $dataObject = new EntityDataObject("name", "type", $data, null, null, null); + + $this->assertEquals("value1", $dataObject->getDataByName("lowercasekey1", 0)); + $this->assertEquals("value2", $dataObject->getDataByName("camelCaseKey2", 0)); + $this->assertEquals("value3", $dataObject->getDataByName("lowercasekey3", 0)); + $this->assertEquals("value4", $dataObject->getDataByName("camelCaseKey4", 0)); + } + /** * After class functionality * @return void diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Objects/EntityDataObject.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Objects/EntityDataObject.php index 070a08bf1..09c95df9e 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Objects/EntityDataObject.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Objects/EntityDataObject.php @@ -177,15 +177,22 @@ public function getDataByName($name, $uniquenessFormat) $name_lower = strtolower($name); - if ($this->data !== null && array_key_exists($name_lower, $this->data)) { + if ($this->data === null) { + return null; + } + + if (array_key_exists($name_lower, $this->data)) { $uniquenessData = $this->getUniquenessDataByName($name_lower); if (null === $uniquenessData || $uniquenessFormat == self::NO_UNIQUE_PROCESS) { return $this->data[$name_lower]; } return $this->formatUniqueData($name_lower, $uniquenessData, $uniquenessFormat); + } elseif (array_key_exists($name, $this->data)) { + // Data returned by the API may be camelCase so we need to check for the original $name also. + return $this->data[$name]; + } else { + return null; } - - return null; } /** From 9a09a90bf7ddc4a84c61bc0f8d4c8380b8fd0dfc Mon Sep 17 00:00:00 2001 From: Ji Lu Date: Mon, 5 Aug 2019 12:01:09 -0500 Subject: [PATCH 03/82] Revert "Revert "Merge pull request #382 from magento/MQE-1600"" This reverts commit 1359e26 --- composer.json | 4 +- .../FileStorageTest.php} | 17 +- etc/config/.credentials.example | 150 ++++++++--------- etc/config/.env.example | 4 + .../Handlers/CredentialStore.php | 154 ++++++------------ .../Handlers/SecretStorage/BaseStorage.php | 87 ++++++++++ .../Handlers/SecretStorage/FileStorage.php | 113 +++++++++++++ .../Handlers/SecretStorage/VaultStorage.php | 123 ++++++++++++++ .../SecretStorage/VaultTokenAuthStrategy.php | 47 ++++++ .../Test/Util/ActionMergeUtil.php | 3 +- .../Util/TestGenerator.php | 3 +- 11 files changed, 519 insertions(+), 186 deletions(-) rename dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/{CredentialStoreTest.php => SecretStorage/FileStorageTest.php} (59%) create mode 100644 src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/BaseStorage.php create mode 100644 src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/FileStorage.php create mode 100644 src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php create mode 100644 src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultTokenAuthStrategy.php diff --git a/composer.json b/composer.json index 39d206ee8..a37dd06c7 100755 --- a/composer.json +++ b/composer.json @@ -10,10 +10,12 @@ }, "require": { "php": "7.0.2||7.0.4||~7.0.6||~7.1.0||~7.2.0||~7.3.0", - "allure-framework/allure-codeception": "~1.3.0", "ext-curl": "*", + "allure-framework/allure-codeception": "~1.3.0", "codeception/codeception": "~2.3.4 || ~2.4.0 ", "consolidation/robo": "^1.0.0", + "csharpru/vault-php": "~3.5.3", + "csharpru/vault-php-guzzle6-transport": "^2.0", "flow/jsonpath": ">0.2", "fzaninotto/faker": "^1.6", "monolog/monolog": "^1.0", diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/CredentialStoreTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/SecretStorage/FileStorageTest.php similarity index 59% rename from dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/CredentialStoreTest.php rename to dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/SecretStorage/FileStorageTest.php index a451f8dc9..7e5824c8e 100644 --- a/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/CredentialStoreTest.php +++ b/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/SecretStorage/FileStorageTest.php @@ -4,33 +4,34 @@ * See COPYING.txt for license details. */ -namespace tests\unit\Magento\FunctionalTestFramework\DataGenerator\Handlers; +namespace tests\unit\Magento\FunctionalTestFramework\DataGenerator\Handlers\SecretStorage; -use Magento\FunctionalTestingFramework\DataGenerator\Handlers\CredentialStore; +use Magento\FunctionalTestingFramework\DataGenerator\Handlers\SecretStorage\FileStorage; use Magento\FunctionalTestingFramework\Util\MagentoTestCase; use AspectMock\Test as AspectMock; -class CredentialStoreTest extends MagentoTestCase +class FileStorageTest extends MagentoTestCase { /** - * Test basic encryption/decryption functionality in CredentialStore class. + * Test basic encryption/decryption functionality in FileStorage class. */ public function testBasicEncryptDecrypt() { - $testKey = 'myKey'; + $testKey = 'magento/myKey'; $testValue = 'myValue'; - AspectMock::double(CredentialStore::class, [ + AspectMock::double(FileStorage::class, [ 'readInCredentialsFile' => ["$testKey=$testValue"] ]); - $encryptedCred = CredentialStore::getInstance()->getSecret($testKey); + $fileStorage = new FileStorage(); + $encryptedCred = $fileStorage->getEncryptedValue($testKey); // assert the value we've gotten is in fact not identical to our test value $this->assertNotEquals($testValue, $encryptedCred); - $actualValue = CredentialStore::getInstance()->decryptSecretValue($encryptedCred); + $actualValue = $fileStorage->getDecryptedValue($encryptedCred); // assert that we are able to successfully decrypt our secret value $this->assertEquals($testValue, $actualValue); diff --git a/etc/config/.credentials.example b/etc/config/.credentials.example index ea8b03480..d9c73ac66 100644 --- a/etc/config/.credentials.example +++ b/etc/config/.credentials.example @@ -1,75 +1,75 @@ -#carriers/fedex/account= -#carriers/fedex/meter_number= -#carriers/fedex/key= -#carriers/fedex/password= - -#carriers/ups/password= -#carriers/ups/username= -#carriers/ups/access_license_number= -#carriers/ups/shipper_number= - -#carriers/usps/userid= -#carriers/usps/password= - -#carriers_dhl_id_us= -#carriers_dhl_password_us= -#carriers_dhl_account_us= - -#carriers_dhl_id_eu= -#carriers_dhl_password_eu= -#carriers_dhl_account_eu= - - -#payment_authorizenet_login= -#payment_authorizenet_trans_key= -#payment_authorizenet_trans_md5= - -#authorizenet_fraud_review_login= -#authorizenet_fraud_review_trans_key= -#authorizenet_fraud_review_md5= - -#braintree_enabled_fraud_merchant_account_id= -#braintree_enabled_fraud_merchant_id= -#braintree_enabled_fraud_public_key= -#braintree_enabled_fraud_private_key= - -#braintree_disabled_fraud_merchant_account_id= -#braintree_disabled_fraud_merchant_id= -#braintree_disabled_fraud_public_key= -#braintree_disabled_fraud_private_key= - -#payment/paypal_group_all_in_one/wpp_usuk/wpp_required_settings/wpp_and_express_checkout/business_account= -#payment/paypal_group_all_in_one/wpp_usuk/wpp_required_settings/wpp_and_express_checkout/api_username= -#payment/paypal_group_all_in_one/wpp_usuk/wpp_required_settings/wpp_and_express_checkout/api_password= -#payment/paypal_group_all_in_one/wpp_usuk/wpp_required_settings/wpp_and_express_checkout/api_signature= -#payment/paypal_express/merchant_id= - -#payflow_pro_fraud_protection_enabled_business_account= -#payflow_pro_fraud_protection_enabled_partner= -#payflow_pro_fraud_protection_enabled_user= -#payflow_pro_fraud_protection_enabled_pwd= -#payflow_pro_fraud_protection_enabled_vendor= - -#payflow_pro_business_account= -#payflow_pro_partner= -#payflow_pro_user= -#payflow_pro_pwd= -#payflow_pro_vendor= - -#payflow_link_business_account_email= -#payflow_link_partner= -#payflow_link_user= -#payflow_link_password= -#payflow_link_vendor= - -#payment/paypal_group_all_in_one/payments_pro_hosted_solution_with_express_checkout/pphs_required_settings/pphs_required_settings_pphs/business_account= -#payment/paypal_group_all_in_one/payments_pro_hosted_solution_with_express_checkout/pphs_required_settings/pphs_required_settings_pphs/api_username= -#payment/paypal_group_all_in_one/payments_pro_hosted_solution_with_express_checkout/pphs_required_settings/pphs_required_settings_pphs/api_password= -#payment/paypal_group_all_in_one/payments_pro_hosted_solution_with_express_checkout/pphs_required_settings/pphs_required_settings_pphs/api_signature= - -#payment/paypal_alternative_payment_methods/express_checkout_us/express_checkout_required/express_checkout_required_express_checkout/business_account= -#payment/paypal_alternative_payment_methods/express_checkout_us/express_checkout_required/express_checkout_required_express_checkout/api_username= -#payment/paypal_alternative_payment_methods/express_checkout_us/express_checkout_required/express_checkout_required_express_checkout/api_password= -#payment/paypal_alternative_payment_methods/express_checkout_us/express_checkout_required/express_checkout_required_express_checkout/api_signature= - -#fraud_protection/signifyd/api_key= \ No newline at end of file +#magento/magento/carriers_fedex_account= +#magento/carriers_fedex_meter_number= +#magento/carriers_fedex_key= +#magento/carriers_fedex_password= + +#magento/carriers_ups_password= +#magento/carriers_ups_username= +#magento/carriers_ups_access_license_number= +#magento/carriers_ups_shipper_number= + +#magento/carriers_usps_userid= +#magento/carriers_usps_password= + +#magento/carriers_dhl_id_us= +#magento/carriers_dhl_password_us= +#magento/carriers_dhl_account_us= + +#magento/carriers_dhl_id_eu= +#magento/carriers_dhl_password_eu= +#magento/carriers_dhl_account_eu= + + +#magento/payment_authorizenet_login= +#magento/payment_authorizenet_trans_key= +#magento/payment_authorizenet_trans_md5= + +#magento/authorizenet_fraud_review_login= +#magento/authorizenet_fraud_review_trans_key= +#magento/authorizenet_fraud_review_md5= + +#magento/braintree_enabled_fraud_merchant_account_id= +#magento/braintree_enabled_fraud_merchant_id= +#magento/braintree_enabled_fraud_public_key= +#magento/braintree_enabled_fraud_private_key= + +#magento/braintree_disabled_fraud_merchant_account_id= +#magento/braintree_disabled_fraud_merchant_id= +#magento/braintree_disabled_fraud_public_key= +#magento/braintree_disabled_fraud_private_key= + +#magento/payment_paypal_group_all_in_one_wpp_usuk_wpp_required_settings_wpp_and_express_checkout_business_account= +#magento/payment_paypal_group_all_in_one_wpp_usuk_wpp_required_settings_wpp_and_express_checkout_api_username= +#magento/payment_paypal_group_all_in_one_wpp_usuk_wpp_required_settings_wpp_and_express_checkout_api_password= +#magento/payment_paypal_group_all_in_one_wpp_usuk_wpp_required_settings_wpp_and_express_checkout_api_signature= +#magento/payment_paypal_express_merchant_id= + +#magento/payflow_pro_fraud_protection_enabled_business_account= +#magento/payflow_pro_fraud_protection_enabled_partner= +#magento/payflow_pro_fraud_protection_enabled_user= +#magento/payflow_pro_fraud_protection_enabled_pwd= +#magento/payflow_pro_fraud_protection_enabled_vendor= + +#magento/payflow_pro_business_account= +#magento/payflow_pro_partner= +#magento/payflow_pro_user= +#magento/payflow_pro_pwd= +#magento/payflow_pro_vendor= + +#magento/payflow_link_business_account_email= +#magento/payflow_link_partner= +#magento/payflow_link_user= +#magento/payflow_link_password= +#magento/payflow_link_vendor= + +#magento/payment_paypal_group_all_in_one_payments_pro_hosted_solution_with_express_checkout_pphs_required_settings_pphs_required_settings_pphs_business_account= +#magento/payment_paypal_group_all_in_one_payments_pro_hosted_solution_with_express_checkout_pphs_required_settings_pphs_required_settings_pphs_api_username= +#magento/payment_paypal_group_all_in_one_payments_pro_hosted_solution_with_express_checkout_pphs_required_settings_pphs_required_settings_pphs_api_password= +#magento/payment_paypal_group_all_in_one_payments_pro_hosted_solution_with_express_checkout_pphs_required_settings_pphs_required_settings_pphs_api_signature= + +#magento/payment_paypal_alternative_payment_methods_express_checkout_us_express_checkout_required_express_checkout_required_express_checkout_business_account= +#magento/payment_paypal_alternative_payment_methods_express_checkout_us_express_checkout_required_express_checkout_required_express_checkout_api_username= +#magento/payment_paypal_alternative_payment_methods_express_checkout_us_express_checkout_required_express_checkout_required_express_checkout_api_password= +#magento/payment_paypal_alternative_payment_methods_express_checkout_us_express_checkout_required_express_checkout_required_express_checkout_api_signature= + +#magento/fraud_protection_signifyd_api_key= \ No newline at end of file diff --git a/etc/config/.env.example b/etc/config/.env.example index 5dc7168be..cc82ae447 100644 --- a/etc/config/.env.example +++ b/etc/config/.env.example @@ -30,6 +30,10 @@ BROWSER=chrome #MAGENTO_RESTAPI_SERVER_PORT=8080 #MAGENTO_RESTAPI_SERVER_PROTOCOL=https +#*** Uncomment and set vault base url and access token if you want to use vault to manage _CREDS secrets ***# +#CREDENTIAL_VAULT_BASE_URL= +#CREDENTIAL_VAULT_TOKEN= + #*** Uncomment these properties to set up a dev environment with symlinked projects ***# #TESTS_BP= #FW_BP= diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php index a83540683..016360752 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php @@ -6,47 +6,34 @@ namespace Magento\FunctionalTestingFramework\DataGenerator\Handlers; -use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig; -use Magento\FunctionalTestingFramework\Console\BuildProjectCommand; use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; -use Magento\FunctionalTestingFramework\Util\Logger\LoggingUtil; +use Magento\FunctionalTestingFramework\DataGenerator\Handlers\SecretStorage\FileStorage; +use Magento\FunctionalTestingFramework\DataGenerator\Handlers\SecretStorage\VaultStorage; class CredentialStore { - const ENCRYPTION_ALGO = "AES-256-CBC"; + const ARRAY_KEY_FOR_VAULT = 'vault'; + const ARRAY_KEY_FOR_FILE = 'file'; /** - * Singleton instance - * - * @var CredentialStore - */ - private static $INSTANCE = null; - - /** - * Initial vector for open_ssl encryption. + * Credential storage array * - * @var string - */ - private $iv = null; - - /** - * Key for open_ssl encryption/decryption - * - * @var string + * @var array */ - private $encodedKey = null; + private $credStorage = []; /** - * Key/Value paris of credential names and their corresponding values + * Singleton instance * - * @var array + * @var CredentialStore */ - private $credentials = []; + private static $INSTANCE = null; /** * Static singleton getter for CredentialStore Instance * * @return CredentialStore + * @throws TestFrameworkException */ public static function getInstance() { @@ -58,120 +45,87 @@ public static function getInstance() } /** - * CredentialStore constructor. - */ - private function __construct() - { - $this->encodedKey = base64_encode(openssl_random_pseudo_bytes(16)); - $this->iv = substr(hash('sha256', $this->encodedKey), 0, 16); - $creds = $this->readInCredentialsFile(); - $this->credentials = $this->encryptCredFileContents($creds); - } - - /** - * Returns the value of a secret based on corresponding key + * CredentialStore constructor * - * @param string $key - * @return string|null * @throws TestFrameworkException */ - public function getSecret($key) + private function __construct() { - if (!array_key_exists($key, $this->credentials)) { - throw new TestFrameworkException( - "{$key} not defined in .credentials, please provide a value in order to use this secret in a test." - ); + // Initialize file storage + try { + $this->credStorage[self::ARRAY_KEY_FOR_FILE] = new FileStorage(); + } catch (TestFrameworkException $e) { } - // log here for verbose config - if (MftfApplicationConfig::getConfig()->verboseEnabled()) { - LoggingUtil::getInstance()->getLogger(CredentialStore::class)->debug( - "retrieving secret for key name {$key}" - ); + // Initialize vault storage + $csBaseUrl = getenv('CREDENTIAL_VAULT_BASE_URL'); + $csToken = getenv('CREDENTIAL_VAULT_TOKEN'); + if ($csBaseUrl !== false && $csToken !== false) { + try { + $this->credStorage[self::ARRAY_KEY_FOR_VAULT] = new VaultStorage( + rtrim($csBaseUrl, '/'), + $csToken + ); + } catch (TestFrameworkException $e) { + } } - return $this->credentials[$key] ?? null; - } - - /** - * Private function which reads in secret key/values from .credentials file and stores in memory as key/value pair. - * - * @return array - * @throws TestFrameworkException - */ - private function readInCredentialsFile() - { - $credsFilePath = str_replace( - '.credentials.example', - '.credentials', - BuildProjectCommand::CREDENTIALS_FILE_PATH - ); - - if (!file_exists($credsFilePath)) { + if (empty($this->credStorage)) { throw new TestFrameworkException( - "Cannot find .credentials file, please create in " - . TESTS_BP . " in order to reference sensitive information" + "No credential storage is properly configured. Please configure vault or .credentials file." ); } - - return file($credsFilePath, FILE_IGNORE_NEW_LINES); } /** - * Function which takes the contents of the credentials file and encrypts the entries. + * Get encrypted value by key * - * @param array $credContents - * @return array + * @param string $key + * @return string|null + * @throws TestFrameworkException */ - private function encryptCredFileContents($credContents) + public function getSecret($key) { - $encryptedCreds = []; - foreach ($credContents as $credValue) { - if (substr($credValue, 0, 1) === '#' || empty($credValue)) { - continue; - } - - list($key, $value) = explode("=", $credValue, 2); - if (!empty($value)) { - $encryptedCreds[$key] = openssl_encrypt( - $value, - self::ENCRYPTION_ALGO, - $this->encodedKey, - 0, - $this->iv - ); + // Get secret data from storage according to the order they are stored + // File storage is preferred over vault storage to allow local secret value overriding remote secret value + foreach ($this->credStorage as $storage) { + $value = $storage->getEncryptedValue($key); + if (null !== $value) { + return $value; } } - return $encryptedCreds; + throw new TestFrameworkException( + "\"{$key}\" not defined in vault or .credentials file, " + . "please provide a value in order to use this secret in a test." + ); } /** - * Takes a value encrypted at runtime and descrypts using the object's initial vector. + * Return decrypted input value * * @param string $value * @return string */ public function decryptSecretValue($value) { - return openssl_decrypt($value, self::ENCRYPTION_ALGO, $this->encodedKey, 0, $this->iv); + // Loop through storage to decrypt value + foreach ($this->credStorage as $storage) { + return $storage->getDecryptedValue($value); + } } /** - * Takes a string that contains encrypted data at runtime and decrypts each value. + * Return decrypted values for all occurrences from input string * * @param string $string * @return mixed */ public function decryptAllSecretsInString($string) { - $newString = $string; - foreach ($this->credentials as $name => $secretValue) { - if (strpos($newString, $secretValue) !== false) { - $decryptedValue = $this->decryptSecretValue($secretValue); - $newString = str_replace($secretValue, $decryptedValue, $newString); - } + // Loop through storage to decrypt all occurrences from input string + foreach ($this->credStorage as $storage) { + return $storage->getAllDecryptedValuesInString($string); } - return $newString; } } diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/BaseStorage.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/BaseStorage.php new file mode 100644 index 000000000..cb892a545 --- /dev/null +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/BaseStorage.php @@ -0,0 +1,87 @@ + $secretValue) { + if (strpos($newString, $secretValue) !== false) { + $decryptedValue = self::getDecryptedValue($secretValue); + $newString = str_replace($secretValue, $decryptedValue, $newString); + } + } + return $newString; + } +} diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/FileStorage.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/FileStorage.php new file mode 100644 index 000000000..064610c79 --- /dev/null +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/FileStorage.php @@ -0,0 +1,113 @@ +readInCredentialsFile(); + $this->secretData = $this->encryptCredFileContents($creds); + } + + /** + * Returns the value of a secret based on corresponding key + * + * @param string $key + * @return string|null + */ + public function getEncryptedValue($key) + { + $value = null; + // Check if secret is in cached array + if (null !== ($value = parent::getEncryptedValue($key))) { + return $value; + } + + // log here for verbose config + if (MftfApplicationConfig::getConfig()->verboseEnabled()) { + LoggingUtil::getInstance()->getLogger(FileStorage::class)->debug( + "retrieving secret for key name {$key} from file" + ); + } + + // Retrieve from file storage + if (array_key_exists($key, $this->secretData) && (null !== ($value = $this->secretData[$key]))) { + parent::$cachedSecretData[$key] = $value; + } + + return $value; + } + + /** + * Private function which reads in secret key/values from .credentials file and stores in memory as key/value pair + * + * @return array + * @throws TestFrameworkException + */ + private function readInCredentialsFile() + { + $credsFilePath = str_replace( + '.credentials.example', + '.credentials', + BuildProjectCommand::CREDENTIALS_FILE_PATH + ); + + if (!file_exists($credsFilePath)) { + throw new TestFrameworkException( + "Credential file is not used: .credentials file not found in " . TESTS_BP + ); + } + + return file($credsFilePath, FILE_IGNORE_NEW_LINES); + } + + /** + * Function which takes the contents of the credentials file and encrypts the entries + * + * @param array $credContents + * @return array + */ + private function encryptCredFileContents($credContents) + { + $encryptedCreds = []; + foreach ($credContents as $credValue) { + if (substr($credValue, 0, 1) === '#' || empty($credValue)) { + continue; + } + + list($key, $value) = explode("=", $credValue, 2); + if (!empty($value)) { + $encryptedCreds[$key] = openssl_encrypt( + $value, + parent::ENCRYPTION_ALGO, + parent::$encodedKey, + 0, + parent::$iv + ); + } + } + return $encryptedCreds; + } +} diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php new file mode 100644 index 000000000..de80b83e3 --- /dev/null +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php @@ -0,0 +1,123 @@ +client) { + // Creating the client using Guzzle6 Transport and passing a custom url + $this->client = new Client(new Guzzle6Transport(['base_uri' => $baseUrl])); + } + $this->token = $token; + if (!$this->authenticated()) { + throw new TestFrameworkException("Credential vault is not used: cannot authenticate"); + } + } + + /** + * Returns the value of a secret based on corresponding key + * + * @param string $key + * @return string|null + */ + public function getEncryptedValue($key) + { + // Check if secret is in cached array + if (null !== ($value = parent::getEncryptedValue($key))) { + return $value; + } + + if (MftfApplicationConfig::getConfig()->verboseEnabled()) { + LoggingUtil::getInstance()->getLogger(VaultStorage::class)->debug( + "Retrieving secret for key name {$key} from vault" + ); + } + + $reValue = null; + try { + // Split vendor/key to construct secret path + list($vendor, $key) = explode('/', trim($key, '/'), 2); + $url = self::BASE_PATH + . (empty(self::KV_DATA) ? '' : '/' . self::KV_DATA) + . self::MFTF_PATH + . '/' + . $vendor + . '/' + . $key; + // Read value by key from vault + $value = $this->client->read($url)->getData()[self::KV_DATA][$key]; + // Encrypt value for return + $reValue = openssl_encrypt($value, parent::ENCRYPTION_ALGO, parent::$encodedKey, 0, parent::$iv); + parent::$cachedSecretData[$key] = $reValue; + } catch (\Exception $e) { + if (MftfApplicationConfig::getConfig()->verboseEnabled()) { + LoggingUtil::getInstance()->getLogger(VaultStorage::class)->debug( + "Unable to read secret for key name {$key} from vault" + ); + } + } + return $reValue; + } + + /** + * Check if vault token is valid + * + * @return boolean + */ + private function authenticated() + { + try { + // Authenticating using token auth backend + $authenticated = $this->client + ->setAuthenticationStrategy(new VaultTokenAuthStrategy($this->token)) + ->authenticate(); + + if ($authenticated) { + return true; + } + } catch (\Exception $e) { + } + return false; + } +} diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultTokenAuthStrategy.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultTokenAuthStrategy.php new file mode 100644 index 000000000..716344ca1 --- /dev/null +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultTokenAuthStrategy.php @@ -0,0 +1,47 @@ +token = $token; + } + + /** + * Returns auth for further interactions with Vault + * + * @return Auth + * @throws TestFrameworkException + */ + public function authenticate() + { + try { + return new Auth(['clientToken' => $this->token]); + } catch (\Exception $e) { + throw new TestFrameworkException("Cannot authenticate Vault token."); + } + } +} diff --git a/src/Magento/FunctionalTestingFramework/Test/Util/ActionMergeUtil.php b/src/Magento/FunctionalTestingFramework/Test/Util/ActionMergeUtil.php index 2a939a8c7..cd81ade24 100644 --- a/src/Magento/FunctionalTestingFramework/Test/Util/ActionMergeUtil.php +++ b/src/Magento/FunctionalTestingFramework/Test/Util/ActionMergeUtil.php @@ -31,6 +31,7 @@ class ActionMergeUtil const DEFAULT_WAIT_ORDER = 'after'; const APPROVED_ACTIONS = ['fillField', 'magentoCLI', 'field']; const SECRET_MAPPING = ['fillField' => 'fillSecretField', 'magentoCLI' => 'magentoCLISecret']; + const CREDS_REGEX = "/{{_CREDS\.([\w|\/]+)}}/"; /** * Array holding final resulting steps @@ -149,7 +150,7 @@ private function actionAttributeContainsSecretRef($actionAttributes) return $this->actionAttributeContainsSecretRef($actionAttribute); } - preg_match_all("/{{_CREDS\.([\w]+)}}/", $actionAttribute, $matches); + preg_match_all(self::CREDS_REGEX, $actionAttribute, $matches); if (!empty($matches[0])) { return true; diff --git a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php index 415e22956..3f1417fd1 100644 --- a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php +++ b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php @@ -24,6 +24,7 @@ use Magento\FunctionalTestingFramework\Test\Util\ActionObjectExtractor; use Magento\FunctionalTestingFramework\Test\Util\TestObjectExtractor; use Magento\FunctionalTestingFramework\Util\Filesystem\DirSetupUtil; +use Magento\FunctionalTestingFramework\Test\Util\ActionMergeUtil; /** * Class TestGenerator @@ -1904,7 +1905,7 @@ private function resolveAllRuntimeReferences($args) { $runtimeReferenceRegex = [ "/{{_ENV\.([\w]+)}}/" => 'getenv', - "/{{_CREDS\.([\w]+)}}/" => 'CredentialStore::getInstance()->getSecret' + ActionMergeUtil::CREDS_REGEX => 'CredentialStore::getInstance()->getSecret' ]; $argResult = $args; From 82371b1549756ae9450bcea29bb1d5b741b7e939 Mon Sep 17 00:00:00 2001 From: Ji Lu Date: Mon, 5 Aug 2019 13:06:49 -0500 Subject: [PATCH 04/82] MQE-1600: MFTF Vault integration - added composer.lock change --- composer.lock | 603 ++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 532 insertions(+), 71 deletions(-) diff --git a/composer.lock b/composer.lock index 7875ec7e1..5ce61bae1 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "8c865770654a7053c62a505c2ca31942", + "content-hash": "39e380b2acfd8adcd2c3642655f5a2a1", "packages": [ { "name": "allure-framework/allure-codeception", @@ -168,6 +168,99 @@ ], "time": "2016-10-30T11:50:56+00:00" }, + { + "name": "cache/cache", + "version": "0.4.0", + "source": { + "type": "git", + "url": "https://github.com/php-cache/cache.git", + "reference": "902b2e5b54ea57e3a801437748652228c4c58604" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-cache/cache/zipball/902b2e5b54ea57e3a801437748652228c4c58604", + "reference": "902b2e5b54ea57e3a801437748652228c4c58604", + "shasum": "" + }, + "require": { + "doctrine/cache": "^1.3", + "league/flysystem": "^1.0", + "php": "^5.6 || ^7.0", + "psr/cache": "^1.0", + "psr/log": "^1.0", + "psr/simple-cache": "^1.0" + }, + "conflict": { + "cache/adapter-common": "*", + "cache/apc-adapter": "*", + "cache/apcu-adapter": "*", + "cache/array-adapter": "*", + "cache/chain-adapter": "*", + "cache/doctrine-adapter": "*", + "cache/filesystem-adapter": "*", + "cache/hierarchical-cache": "*", + "cache/illuminate-adapter": "*", + "cache/memcache-adapter": "*", + "cache/memcached-adapter": "*", + "cache/mongodb-adapter": "*", + "cache/predis-adapter": "*", + "cache/psr-6-doctrine-bridge": "*", + "cache/redis-adapter": "*", + "cache/session-handler": "*", + "cache/taggable-cache": "*", + "cache/void-adapter": "*" + }, + "require-dev": { + "cache/integration-tests": "^0.16", + "defuse/php-encryption": "^2.0", + "illuminate/cache": "^5.4", + "mockery/mockery": "^0.9", + "phpunit/phpunit": "^4.0 || ^5.1", + "predis/predis": "^1.0", + "symfony/cache": "dev-master" + }, + "suggest": { + "ext-apc": "APC extension is required to use the APC Adapter", + "ext-apcu": "APCu extension is required to use the APCu Adapter", + "ext-memcache": "Memcache extension is required to use the Memcache Adapter", + "ext-memcached": "Memcached extension is required to use the Memcached Adapter", + "ext-mongodb": "Mongodb extension required to use the Mongodb adapter", + "ext-redis": "Redis extension is required to use the Redis adapter", + "mongodb/mongodb": "Mongodb lib required to use the Mongodb adapter" + }, + "type": "library", + "autoload": { + "psr-4": { + "Cache\\": "src/" + }, + "exclude-from-classmap": [ + "**/Tests/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Aaron Scherer", + "email": "aequasi@gmail.com", + "homepage": "https://github.com/aequasi" + }, + { + "name": "Tobias Nyholm", + "email": "tobias.nyholm@gmail.com", + "homepage": "https://github.com/nyholm" + } + ], + "description": "Library of all the php-cache adapters", + "homepage": "http://www.php-cache.com/en/latest/", + "keywords": [ + "cache", + "psr6" + ], + "time": "2017-03-28T16:08:48+00:00" + }, { "name": "codeception/codeception", "version": "2.3.9", @@ -668,6 +761,92 @@ "homepage": "https://github.com/container-interop/container-interop", "time": "2017-02-14T19:40:03+00:00" }, + { + "name": "csharpru/vault-php", + "version": "3.5.3", + "source": { + "type": "git", + "url": "https://github.com/CSharpRU/vault-php.git", + "reference": "04be9776310fe7d1afb97795645f95c21e6b4fcf" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/CSharpRU/vault-php/zipball/04be9776310fe7d1afb97795645f95c21e6b4fcf", + "reference": "04be9776310fe7d1afb97795645f95c21e6b4fcf", + "shasum": "" + }, + "require": { + "cache/cache": "^0.4.0", + "doctrine/inflector": "~1.1.0", + "guzzlehttp/promises": "^1.3", + "guzzlehttp/psr7": "^1.4", + "psr/cache": "^1.0", + "psr/log": "^1.0", + "weew/helpers-array": "^1.3" + }, + "require-dev": { + "codacy/coverage": "^1.1", + "codeception/codeception": "^2.2", + "csharpru/vault-php-guzzle6-transport": "~2.0", + "php-vcr/php-vcr": "^1.3" + }, + "type": "library", + "autoload": { + "psr-4": { + "Vault\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Yaroslav Lukyanov", + "email": "c_sharp@mail.ru" + } + ], + "description": "Best Vault client for PHP that you can find", + "time": "2018-04-28T04:52:17+00:00" + }, + { + "name": "csharpru/vault-php-guzzle6-transport", + "version": "2.0.4", + "source": { + "type": "git", + "url": "https://github.com/CSharpRU/vault-php-guzzle6-transport.git", + "reference": "33c392120ac9f253b62b034e0e8ffbbdb3513bd8" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/CSharpRU/vault-php-guzzle6-transport/zipball/33c392120ac9f253b62b034e0e8ffbbdb3513bd8", + "reference": "33c392120ac9f253b62b034e0e8ffbbdb3513bd8", + "shasum": "" + }, + "require": { + "guzzlehttp/guzzle": "~6.2", + "guzzlehttp/promises": "^1.3", + "guzzlehttp/psr7": "^1.4" + }, + "type": "library", + "autoload": { + "psr-4": { + "VaultTransports\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Yaroslav Lukyanov", + "email": "c_sharp@mail.ru" + } + ], + "description": "Guzzle6 transport for Vault PHP client", + "time": "2019-03-10T06:17:37+00:00" + }, { "name": "dflydev/dot-access-data", "version": "v1.1.0", @@ -795,6 +974,143 @@ ], "time": "2017-02-24T16:22:25+00:00" }, + { + "name": "doctrine/cache", + "version": "v1.6.2", + "source": { + "type": "git", + "url": "https://github.com/doctrine/cache.git", + "reference": "eb152c5100571c7a45470ff2a35095ab3f3b900b" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/doctrine/cache/zipball/eb152c5100571c7a45470ff2a35095ab3f3b900b", + "reference": "eb152c5100571c7a45470ff2a35095ab3f3b900b", + "shasum": "" + }, + "require": { + "php": "~5.5|~7.0" + }, + "conflict": { + "doctrine/common": ">2.2,<2.4" + }, + "require-dev": { + "phpunit/phpunit": "~4.8|~5.0", + "predis/predis": "~1.0", + "satooshi/php-coveralls": "~0.6" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.6.x-dev" + } + }, + "autoload": { + "psr-4": { + "Doctrine\\Common\\Cache\\": "lib/Doctrine/Common/Cache" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Roman Borschel", + "email": "roman@code-factory.org" + }, + { + "name": "Benjamin Eberlei", + "email": "kontakt@beberlei.de" + }, + { + "name": "Guilherme Blanco", + "email": "guilhermeblanco@gmail.com" + }, + { + "name": "Jonathan Wage", + "email": "jonwage@gmail.com" + }, + { + "name": "Johannes Schmitt", + "email": "schmittjoh@gmail.com" + } + ], + "description": "Caching library offering an object-oriented API for many cache backends", + "homepage": "http://www.doctrine-project.org", + "keywords": [ + "cache", + "caching" + ], + "time": "2017-07-22T12:49:21+00:00" + }, + { + "name": "doctrine/inflector", + "version": "v1.1.0", + "source": { + "type": "git", + "url": "https://github.com/doctrine/inflector.git", + "reference": "90b2128806bfde671b6952ab8bea493942c1fdae" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/doctrine/inflector/zipball/90b2128806bfde671b6952ab8bea493942c1fdae", + "reference": "90b2128806bfde671b6952ab8bea493942c1fdae", + "shasum": "" + }, + "require": { + "php": ">=5.3.2" + }, + "require-dev": { + "phpunit/phpunit": "4.*" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.1.x-dev" + } + }, + "autoload": { + "psr-0": { + "Doctrine\\Common\\Inflector\\": "lib/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Roman Borschel", + "email": "roman@code-factory.org" + }, + { + "name": "Benjamin Eberlei", + "email": "kontakt@beberlei.de" + }, + { + "name": "Guilherme Blanco", + "email": "guilhermeblanco@gmail.com" + }, + { + "name": "Jonathan Wage", + "email": "jonwage@gmail.com" + }, + { + "name": "Johannes Schmitt", + "email": "schmittjoh@gmail.com" + } + ], + "description": "Common String Manipulations with regard to casing and singular/plural rules.", + "homepage": "http://www.doctrine-project.org", + "keywords": [ + "inflection", + "pluralize", + "singularize", + "string" + ], + "time": "2015-11-06T14:35:42+00:00" + }, { "name": "doctrine/instantiator", "version": "1.0.5", @@ -1608,6 +1924,90 @@ ], "time": "2017-05-10T09:20:27+00:00" }, + { + "name": "league/flysystem", + "version": "1.0.53", + "source": { + "type": "git", + "url": "https://github.com/thephpleague/flysystem.git", + "reference": "08e12b7628f035600634a5e76d95b5eb66cea674" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/thephpleague/flysystem/zipball/08e12b7628f035600634a5e76d95b5eb66cea674", + "reference": "08e12b7628f035600634a5e76d95b5eb66cea674", + "shasum": "" + }, + "require": { + "ext-fileinfo": "*", + "php": ">=5.5.9" + }, + "conflict": { + "league/flysystem-sftp": "<1.0.6" + }, + "require-dev": { + "phpspec/phpspec": "^3.4", + "phpunit/phpunit": "^5.7.10" + }, + "suggest": { + "ext-fileinfo": "Required for MimeType", + "ext-ftp": "Allows you to use FTP server storage", + "ext-openssl": "Allows you to use FTPS server storage", + "league/flysystem-aws-s3-v2": "Allows you to use S3 storage with AWS SDK v2", + "league/flysystem-aws-s3-v3": "Allows you to use S3 storage with AWS SDK v3", + "league/flysystem-azure": "Allows you to use Windows Azure Blob storage", + "league/flysystem-cached-adapter": "Flysystem adapter decorator for metadata caching", + "league/flysystem-eventable-filesystem": "Allows you to use EventableFilesystem", + "league/flysystem-rackspace": "Allows you to use Rackspace Cloud Files", + "league/flysystem-sftp": "Allows you to use SFTP server storage via phpseclib", + "league/flysystem-webdav": "Allows you to use WebDAV storage", + "league/flysystem-ziparchive": "Allows you to use ZipArchive adapter", + "spatie/flysystem-dropbox": "Allows you to use Dropbox storage", + "srmklive/flysystem-dropbox-v2": "Allows you to use Dropbox storage for PHP 5 applications" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.1-dev" + } + }, + "autoload": { + "psr-4": { + "League\\Flysystem\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Frank de Jonge", + "email": "info@frenky.net" + } + ], + "description": "Filesystem abstraction: Many filesystems, one API.", + "keywords": [ + "Cloud Files", + "WebDAV", + "abstraction", + "aws", + "cloud", + "copy.com", + "dropbox", + "file systems", + "files", + "filesystem", + "filesystems", + "ftp", + "rackspace", + "remote", + "s3", + "sftp", + "storage" + ], + "time": "2019-06-18T20:09:29+00:00" + }, { "name": "monolog/monolog", "version": "1.24.0", @@ -2679,6 +3079,52 @@ "abandoned": true, "time": "2018-08-09T05:50:03+00:00" }, + { + "name": "psr/cache", + "version": "1.0.1", + "source": { + "type": "git", + "url": "https://github.com/php-fig/cache.git", + "reference": "d11b50ad223250cf17b86e38383413f5a6764bf8" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-fig/cache/zipball/d11b50ad223250cf17b86e38383413f5a6764bf8", + "reference": "d11b50ad223250cf17b86e38383413f5a6764bf8", + "shasum": "" + }, + "require": { + "php": ">=5.3.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.0.x-dev" + } + }, + "autoload": { + "psr-4": { + "Psr\\Cache\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "PHP-FIG", + "homepage": "http://www.php-fig.org/" + } + ], + "description": "Common interface for caching libraries", + "keywords": [ + "cache", + "psr", + "psr-6" + ], + "time": "2016-08-06T20:24:11+00:00" + }, { "name": "psr/container", "version": "1.0.0", @@ -2825,6 +3271,54 @@ ], "time": "2016-10-10T12:19:37+00:00" }, + { + "name": "psr/simple-cache", + "version": "1.0.1", + "source": { + "type": "git", + "url": "https://github.com/php-fig/simple-cache.git", + "reference": "408d5eafb83c57f6365a3ca330ff23aa4a5fa39b" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-fig/simple-cache/zipball/408d5eafb83c57f6365a3ca330ff23aa4a5fa39b", + "reference": "408d5eafb83c57f6365a3ca330ff23aa4a5fa39b", + "shasum": "" + }, + "require": { + "php": ">=5.3.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.0.x-dev" + } + }, + "autoload": { + "psr-4": { + "Psr\\SimpleCache\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "PHP-FIG", + "homepage": "http://www.php-fig.org/" + } + ], + "description": "Common interfaces for simple caching", + "keywords": [ + "cache", + "caching", + "psr", + "psr-16", + "simple-cache" + ], + "time": "2017-10-23T01:57:42+00:00" + }, { "name": "ramsey/uuid", "version": "3.8.0", @@ -4397,6 +4891,43 @@ "validate" ], "time": "2018-01-29T19:49:41+00:00" + }, + { + "name": "weew/helpers-array", + "version": "v1.3.1", + "source": { + "type": "git", + "url": "https://github.com/weew/helpers-array.git", + "reference": "9bff63111f9765b4277750db8d276d92b3e16ed0" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/weew/helpers-array/zipball/9bff63111f9765b4277750db8d276d92b3e16ed0", + "reference": "9bff63111f9765b4277750db8d276d92b3e16ed0", + "shasum": "" + }, + "require-dev": { + "phpunit/phpunit": "^4.7", + "satooshi/php-coveralls": "^0.6.1" + }, + "type": "library", + "autoload": { + "files": [ + "src/array.php" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Maxim Kott", + "email": "maximkott@gmail.com" + } + ], + "description": "Useful collection of php array helpers.", + "time": "2016-07-21T11:18:01+00:00" } ], "packages-dev": [ @@ -4556,76 +5087,6 @@ "description": "Experimental Mocking Framework powered by Aspects", "time": "2018-10-07T16:21:11+00:00" }, - { - "name": "doctrine/cache", - "version": "v1.6.2", - "source": { - "type": "git", - "url": "https://github.com/doctrine/cache.git", - "reference": "eb152c5100571c7a45470ff2a35095ab3f3b900b" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/doctrine/cache/zipball/eb152c5100571c7a45470ff2a35095ab3f3b900b", - "reference": "eb152c5100571c7a45470ff2a35095ab3f3b900b", - "shasum": "" - }, - "require": { - "php": "~5.5|~7.0" - }, - "conflict": { - "doctrine/common": ">2.2,<2.4" - }, - "require-dev": { - "phpunit/phpunit": "~4.8|~5.0", - "predis/predis": "~1.0", - "satooshi/php-coveralls": "~0.6" - }, - "type": "library", - "extra": { - "branch-alias": { - "dev-master": "1.6.x-dev" - } - }, - "autoload": { - "psr-4": { - "Doctrine\\Common\\Cache\\": "lib/Doctrine/Common/Cache" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Roman Borschel", - "email": "roman@code-factory.org" - }, - { - "name": "Benjamin Eberlei", - "email": "kontakt@beberlei.de" - }, - { - "name": "Guilherme Blanco", - "email": "guilhermeblanco@gmail.com" - }, - { - "name": "Jonathan Wage", - "email": "jonwage@gmail.com" - }, - { - "name": "Johannes Schmitt", - "email": "schmittjoh@gmail.com" - } - ], - "description": "Caching library offering an object-oriented API for many cache backends", - "homepage": "http://www.doctrine-project.org", - "keywords": [ - "cache", - "caching" - ], - "time": "2017-07-22T12:49:21+00:00" - }, { "name": "gitonomy/gitlib", "version": "v1.0.4", From 56f8ac0d278ce2851fb63eb1f351c6525a624098 Mon Sep 17 00:00:00 2001 From: Ji Lu Date: Wed, 24 Jul 2019 11:41:30 -0500 Subject: [PATCH 05/82] MQE-1647: read vault token from local file system (cherry picked from commit 9f63033) --- .../Handlers/CredentialStore.php | 8 +- .../Handlers/SecretStorage/VaultStorage.php | 98 ++++++++++++++++++- 2 files changed, 96 insertions(+), 10 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php index 016360752..75707b941 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php @@ -59,13 +59,9 @@ private function __construct() // Initialize vault storage $csBaseUrl = getenv('CREDENTIAL_VAULT_BASE_URL'); - $csToken = getenv('CREDENTIAL_VAULT_TOKEN'); - if ($csBaseUrl !== false && $csToken !== false) { + if ($csBaseUrl !== false) { try { - $this->credStorage[self::ARRAY_KEY_FOR_VAULT] = new VaultStorage( - rtrim($csBaseUrl, '/'), - $csToken - ); + $this->credStorage[self::ARRAY_KEY_FOR_VAULT] = new VaultStorage(rtrim($csBaseUrl, '/')); } catch (TestFrameworkException $e) { } } diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php index de80b83e3..415627cda 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php @@ -21,6 +21,21 @@ class VaultStorage extends BaseStorage const BASE_PATH = '/dx_magento_qe'; const KV_DATA = 'data'; + /** + * Default vault token file + */ + const TOKEN_FILE = '.vault-token'; + /** + * Default vault config file + */ + const CONFIG_FILE = '.vault'; + /** + * Environment variable name for vault config path + */ + const CONFIG_PATH_ENV_VAR = 'VAULT_CONFIG_PATH'; + + const TOKEN_HELPER_REGEX = "~\s*token_helper\s*=(.+)$~"; + /** * Vault client * @@ -33,23 +48,22 @@ class VaultStorage extends BaseStorage * * @var string */ - private $token; + private $token = null; /** * CredentialVault constructor * * @param string $baseUrl - * @param string $token * @throws TestFrameworkException */ - public function __construct($baseUrl, $token) + public function __construct($baseUrl) { parent::__construct(); if (null === $this->client) { // Creating the client using Guzzle6 Transport and passing a custom url $this->client = new Client(new Guzzle6Transport(['base_uri' => $baseUrl])); } - $this->token = $token; + $this->readVaultTokenFromFileSystem(); if (!$this->authenticated()) { throw new TestFrameworkException("Credential vault is not used: cannot authenticate"); } @@ -120,4 +134,80 @@ private function authenticated() } return false; } + + /** + * Read vault token from file system + * + * @return void + * @throws TestFrameworkException + */ + private function readVaultTokenFromFileSystem() + { + // Find user home directory + $homeDir = getenv('HOME'); + if ($homeDir === false) { + // If HOME is not set, don't fail right away + $homeDir = '~/'; + } else { + $homeDir = rtrim($homeDir, '/') . '/'; + } + + $vaultTokenFile = $homeDir . self::TOKEN_FILE; + if (file_exists($vaultTokenFile)) { + // Found .vault-token file in default location, construct command + $cmd = 'cat ' . $vaultTokenFile; + } else { + // Otherwise search vault config file for custom token helper script + $vaultConfigPath = getenv(self::CONFIG_PATH_ENV_VAR); + if ($vaultConfigPath === false) { + $vaultConfigFile = $homeDir . self::CONFIG_FILE; + } else { + $vaultConfigFile = rtrim($vaultConfigPath, '/') . '/' . self::CONFIG_FILE; + } + // Found .vault config file, read custom token helper script and construct command + if (file_exists($vaultConfigFile) + && !empty($cmd = $this->getTokenHelperScript(file($vaultConfigFile, FILE_IGNORE_NEW_LINES)))) { + $cmd = $cmd . ' get'; + } else { + throw new TestFrameworkException( + 'Unable to read .vault-token file. Please authenticate to vault through vault CLI first.' + ); + } + } + $this->token = $this->execVaultTokenHelper($cmd); + } + + /** + * Get vault token helper script by parsing lines in vault config file + * + * @param array $lines + * @return array + */ + private function getTokenHelperScript($lines) + { + $tokenHelper = ''; + foreach ($lines as $line) { + preg_match(self::TOKEN_HELPER_REGEX, $line, $matches); + if (isset($matches[1])) { + $tokenHelper = trim(trim(trim($matches[1]), '"')); + } + } + return $tokenHelper; + } + + /** + * Execute vault token helper script and return the token it contains + * + * @param string $cmd + * @return string + */ + private function execVaultTokenHelper($cmd) + { + $output = ''; + exec($cmd, $out, $status); + if ($status === 0 && isset($out[0])) { + $output = $out[0]; + } + return $output; + } } From 242045c8a02cddc533ba0f262448d2c531dc4a5d Mon Sep 17 00:00:00 2001 From: Ji Lu Date: Wed, 24 Jul 2019 12:01:00 -0500 Subject: [PATCH 06/82] MQE-1647: read vault token from local file system (cherry picked from commit 2ddaa61) --- .../Handlers/SecretStorage/VaultStorage.php | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php index 415627cda..bb88b18c5 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php @@ -181,7 +181,7 @@ private function readVaultTokenFromFileSystem() * Get vault token helper script by parsing lines in vault config file * * @param array $lines - * @return array + * @return string */ private function getTokenHelperScript($lines) { @@ -200,14 +200,16 @@ private function getTokenHelperScript($lines) * * @param string $cmd * @return string + * @throws TestFrameworkException */ private function execVaultTokenHelper($cmd) { - $output = ''; exec($cmd, $out, $status); - if ($status === 0 && isset($out[0])) { - $output = $out[0]; + if ($status === 0 && isset($out[0]) && !empty($out[0])) { + return $out[0]; } - return $output; + throw new TestFrameworkException( + 'Error running custom vault token helper script. Please make sure vault CLI works in your environment.' + ); } } From 7d1104202bd3e709f51ac6d598d74f0076dfbf2c Mon Sep 17 00:00:00 2001 From: Ji Lu Date: Thu, 25 Jul 2019 09:04:51 -0500 Subject: [PATCH 07/82] MQE-1647: read vault token from local file system (cherry picked from commit 599d8fd) --- etc/config/.env.example | 1 - 1 file changed, 1 deletion(-) diff --git a/etc/config/.env.example b/etc/config/.env.example index cc82ae447..f091688f5 100644 --- a/etc/config/.env.example +++ b/etc/config/.env.example @@ -32,7 +32,6 @@ BROWSER=chrome #*** Uncomment and set vault base url and access token if you want to use vault to manage _CREDS secrets ***# #CREDENTIAL_VAULT_BASE_URL= -#CREDENTIAL_VAULT_TOKEN= #*** Uncomment these properties to set up a dev environment with symlinked projects ***# #TESTS_BP= From 3304dca851aa2c1bb13abf4caade99449460c254 Mon Sep 17 00:00:00 2001 From: Ji Lu Date: Tue, 30 Jul 2019 16:51:32 -0500 Subject: [PATCH 08/82] =?UTF-8?q?MQE-1647:=C2=A0=20read=20vault=20token=20?= =?UTF-8?q?from=20local=20file=20system?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (cherry picked from commit 84d7a44) --- .../Handlers/SecretStorage/VaultStorage.php | 59 +++++++++++-------- 1 file changed, 34 insertions(+), 25 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php index bb88b18c5..785db036b 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php @@ -34,7 +34,11 @@ class VaultStorage extends BaseStorage */ const CONFIG_PATH_ENV_VAR = 'VAULT_CONFIG_PATH'; - const TOKEN_HELPER_REGEX = "~\s*token_helper\s*=(.+)$~"; + /** + * Regex to grab token helper script + */ + const TOKEN_HELPER_REGEX_GROUP_NAME = 'GROUP_NAME'; + const TOKEN_HELPER_REGEX = "~\s*token_helper\s*=(?<" . self::TOKEN_HELPER_REGEX_GROUP_NAME . ">.+)$~"; /** * Vault client @@ -146,35 +150,40 @@ private function readVaultTokenFromFileSystem() // Find user home directory $homeDir = getenv('HOME'); if ($homeDir === false) { - // If HOME is not set, don't fail right away - $homeDir = '~/'; - } else { - $homeDir = rtrim($homeDir, '/') . '/'; + throw new TestFrameworkException( + "HOME environment variable is not set. It's required when using vault." + ); } + $homeDir = realpath($homeDir) . DIRECTORY_SEPARATOR; + // Read .vault-token file if it is found in default location $vaultTokenFile = $homeDir . self::TOKEN_FILE; if (file_exists($vaultTokenFile)) { - // Found .vault-token file in default location, construct command - $cmd = 'cat ' . $vaultTokenFile; - } else { - // Otherwise search vault config file for custom token helper script - $vaultConfigPath = getenv(self::CONFIG_PATH_ENV_VAR); - if ($vaultConfigPath === false) { - $vaultConfigFile = $homeDir . self::CONFIG_FILE; - } else { - $vaultConfigFile = rtrim($vaultConfigPath, '/') . '/' . self::CONFIG_FILE; + $token = file_get_contents($vaultTokenFile); + if ($token !== false) { + $this->token = $token; + return; } - // Found .vault config file, read custom token helper script and construct command - if (file_exists($vaultConfigFile) - && !empty($cmd = $this->getTokenHelperScript(file($vaultConfigFile, FILE_IGNORE_NEW_LINES)))) { - $cmd = $cmd . ' get'; - } else { - throw new TestFrameworkException( - 'Unable to read .vault-token file. Please authenticate to vault through vault CLI first.' - ); + } + + // Otherwise search vault config file for custom token helper script + $vaultConfigPath = getenv(self::CONFIG_PATH_ENV_VAR); + if ($vaultConfigPath === false) { + $vaultConfigFile = $homeDir . self::CONFIG_FILE; + } else { + $vaultConfigFile = realpath($vaultConfigPath) . DIRECTORY_SEPARATOR . self::CONFIG_FILE; + } + // Get custom token helper script file from .vault config file + if (file_exists($vaultConfigFile)) { + $cmd = $this->getTokenHelperScript(file($vaultConfigFile, FILE_IGNORE_NEW_LINES)); + if (!empty($cmd)) { + $this->token = $this->execVaultTokenHelper($cmd . ' get'); + return; } } - $this->token = $this->execVaultTokenHelper($cmd); + throw new TestFrameworkException( + 'Unable to read .vault-token file. Please authenticate to vault through vault CLI first.' + ); } /** @@ -188,8 +197,8 @@ private function getTokenHelperScript($lines) $tokenHelper = ''; foreach ($lines as $line) { preg_match(self::TOKEN_HELPER_REGEX, $line, $matches); - if (isset($matches[1])) { - $tokenHelper = trim(trim(trim($matches[1]), '"')); + if (isset($matches[self::TOKEN_HELPER_REGEX_GROUP_NAME])) { + $tokenHelper = trim(trim(trim($matches[self::TOKEN_HELPER_REGEX_GROUP_NAME]), '"')); } } return $tokenHelper; From f9baa07162a9b26fadbab194eb539bc373190bf7 Mon Sep 17 00:00:00 2001 From: Ji Lu Date: Wed, 7 Aug 2019 17:51:01 -0500 Subject: [PATCH 09/82] =?UTF-8?q?MQE-1647:=C2=A0=20read=20vault=20token=20?= =?UTF-8?q?from=20local=20file=20system?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - allow secret base path configurable --- etc/config/.env.example | 5 ++-- .../Handlers/CredentialStore.php | 10 +++++--- .../Handlers/SecretStorage/VaultStorage.php | 25 +++++++++++++------ 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/etc/config/.env.example b/etc/config/.env.example index f091688f5..369fd946b 100644 --- a/etc/config/.env.example +++ b/etc/config/.env.example @@ -30,8 +30,9 @@ BROWSER=chrome #MAGENTO_RESTAPI_SERVER_PORT=8080 #MAGENTO_RESTAPI_SERVER_PROTOCOL=https -#*** Uncomment and set vault base url and access token if you want to use vault to manage _CREDS secrets ***# -#CREDENTIAL_VAULT_BASE_URL= +#*** Uncomment and set vault address and secret base path if you want to use vault to manage _CREDS secrets ***# +#CREDENTIAL_VAULT_ADDRESS=http://127.0.0.1:8200 +#CREDENTIAL_VAULT_SECRET_BASE_PATH=secret #*** Uncomment these properties to set up a dev environment with symlinked projects ***# #TESTS_BP= diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php index 75707b941..0514cfe57 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php @@ -58,10 +58,14 @@ private function __construct() } // Initialize vault storage - $csBaseUrl = getenv('CREDENTIAL_VAULT_BASE_URL'); - if ($csBaseUrl !== false) { + $cvAddress = getenv('CREDENTIAL_VAULT_ADDRESS'); + $cvSecretPath = getenv('CREDENTIAL_VAULT_SECRET_BASE_PATH'); + if ($cvAddress !== false && $cvSecretPath !== false) { try { - $this->credStorage[self::ARRAY_KEY_FOR_VAULT] = new VaultStorage(rtrim($csBaseUrl, '/')); + $this->credStorage[self::ARRAY_KEY_FOR_VAULT] = new VaultStorage( + rtrim($cvAddress, '/'), + '/' . trim($cvSecretPath, '/') + ); } catch (TestFrameworkException $e) { } } diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php index 785db036b..86e06b89a 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php @@ -14,12 +14,14 @@ class VaultStorage extends BaseStorage { + /** + * Mftf project path + */ const MFTF_PATH = '/mftf'; /** - * Adobe Vault + * Vault kv version 2 data */ - const BASE_PATH = '/dx_magento_qe'; - const KV_DATA = 'data'; + const KV2_DATA = 'data'; /** * Default vault token file @@ -54,18 +56,27 @@ class VaultStorage extends BaseStorage */ private $token = null; + /** + * Vault secret base path + * + * @var string + */ + private $secretBasePath; + /** * CredentialVault constructor * * @param string $baseUrl + * @param string $secretBasePath * @throws TestFrameworkException */ - public function __construct($baseUrl) + public function __construct($baseUrl, $secretBasePath) { parent::__construct(); if (null === $this->client) { // Creating the client using Guzzle6 Transport and passing a custom url $this->client = new Client(new Guzzle6Transport(['base_uri' => $baseUrl])); + $this->secretBasePath = $secretBasePath; } $this->readVaultTokenFromFileSystem(); if (!$this->authenticated()) { @@ -96,15 +107,15 @@ public function getEncryptedValue($key) try { // Split vendor/key to construct secret path list($vendor, $key) = explode('/', trim($key, '/'), 2); - $url = self::BASE_PATH - . (empty(self::KV_DATA) ? '' : '/' . self::KV_DATA) + $url = $this->secretBasePath + . (empty(self::KV2_DATA) ? '' : '/' . self::KV2_DATA) . self::MFTF_PATH . '/' . $vendor . '/' . $key; // Read value by key from vault - $value = $this->client->read($url)->getData()[self::KV_DATA][$key]; + $value = $this->client->read($url)->getData()[self::KV2_DATA][$key]; // Encrypt value for return $reValue = openssl_encrypt($value, parent::ENCRYPTION_ALGO, parent::$encodedKey, 0, parent::$iv); parent::$cachedSecretData[$key] = $reValue; From 81dc8eccbc6e5bede51da82671ec4a9288e7a689 Mon Sep 17 00:00:00 2001 From: Ji Lu Date: Thu, 8 Aug 2019 09:48:41 -0500 Subject: [PATCH 10/82] =?UTF-8?q?MQE-1647:=C2=A0=20read=20vault=20token=20?= =?UTF-8?q?from=20local=20file=20system?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - allow secret base path configurable --- .../DataGenerator/Handlers/SecretStorage/VaultStorage.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php index 86e06b89a..6a9e9f0cf 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php @@ -18,6 +18,7 @@ class VaultStorage extends BaseStorage * Mftf project path */ const MFTF_PATH = '/mftf'; + /** * Vault kv version 2 data */ @@ -27,10 +28,12 @@ class VaultStorage extends BaseStorage * Default vault token file */ const TOKEN_FILE = '.vault-token'; + /** * Default vault config file */ const CONFIG_FILE = '.vault'; + /** * Environment variable name for vault config path */ From 3a9b868966430f4a5401d5d881aeb121b96ad948 Mon Sep 17 00:00:00 2001 From: Alex Kolesnyk Date: Mon, 12 Aug 2019 13:42:46 -0500 Subject: [PATCH 11/82] MQE-1561: CreateData action must be able to parse Data entity which references another data --- .../Objects/EntityDataObject.php | 28 ++++-- .../Persist/OperationDataArrayResolver.php | 12 +++ .../Util/DataReferenceResolverInterface.php | 26 ++++++ .../Util/GenerationDataReferenceResolver.php | 82 +++++++++++++++++ .../Util/RuntimeDataReferenceResolver.php | 88 +++++++++++++++++++ 5 files changed, 231 insertions(+), 5 deletions(-) create mode 100644 src/Magento/FunctionalTestingFramework/DataGenerator/Util/DataReferenceResolverInterface.php create mode 100644 src/Magento/FunctionalTestingFramework/DataGenerator/Util/GenerationDataReferenceResolver.php create mode 100644 src/Magento/FunctionalTestingFramework/DataGenerator/Util/RuntimeDataReferenceResolver.php diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Objects/EntityDataObject.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Objects/EntityDataObject.php index 09c95df9e..3fb511d3b 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Objects/EntityDataObject.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Objects/EntityDataObject.php @@ -7,6 +7,7 @@ namespace Magento\FunctionalTestingFramework\DataGenerator\Objects; use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig; +use Magento\FunctionalTestingFramework\DataGenerator\Util\GenerationDataReferenceResolver; use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; use Magento\FunctionalTestingFramework\Util\Logger\LoggingUtil; @@ -81,14 +82,14 @@ class EntityDataObject /** * Constructor * - * @param string $name - * @param string $type + * @param string $name + * @param string $type * @param string[] $data * @param string[] $linkedEntities * @param string[] $uniquenessData * @param string[] $vars - * @param string $parentEntity - * @param string $filename + * @param string $parentEntity + * @param string $filename */ public function __construct( $name, @@ -181,13 +182,30 @@ public function getDataByName($name, $uniquenessFormat) return null; } + $dataReferenceResolver = new GenerationDataReferenceResolver(); if (array_key_exists($name_lower, $this->data)) { - $uniquenessData = $this->getUniquenessDataByName($name_lower); + $uniquenessData = $this->getUniquenessDataByName($name_lower) === null + ? $dataReferenceResolver->getDataUniqueness( + $this->data[$name_lower], + $this->name . '.' . $name + ) + : $this->getUniquenessDataByName($name_lower); + if ($uniquenessData !== null) { + $this->uniquenessData[$name] = $uniquenessData; + } + $this->data[$name_lower] = $dataReferenceResolver->getDataReference( + $this->data[$name_lower], + $this->name . '.' . $name + ); if (null === $uniquenessData || $uniquenessFormat == self::NO_UNIQUE_PROCESS) { return $this->data[$name_lower]; } return $this->formatUniqueData($name_lower, $uniquenessData, $uniquenessFormat); } elseif (array_key_exists($name, $this->data)) { + $this->data[$name] = $dataReferenceResolver->getDataReference( + $this->data[$name], + $this->name . '.' . $name + ); // Data returned by the API may be camelCase so we need to check for the original $name also. return $this->data[$name]; } else { diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/OperationDataArrayResolver.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/OperationDataArrayResolver.php index 4e7915b87..7a76c54d1 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/OperationDataArrayResolver.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/OperationDataArrayResolver.php @@ -11,6 +11,7 @@ use Magento\FunctionalTestingFramework\DataGenerator\Objects\EntityDataObject; use Magento\FunctionalTestingFramework\DataGenerator\Objects\OperationElement; use Magento\FunctionalTestingFramework\DataGenerator\Util\OperationElementExtractor; +use Magento\FunctionalTestingFramework\DataGenerator\Util\RuntimeDataReferenceResolver; use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; class OperationDataArrayResolver @@ -121,6 +122,17 @@ public function resolveOperationDataArray($entityObject, $operationMetadata, $op } } + $dataReferenceResolver = new RuntimeDataReferenceResolver(); + foreach ($operationDataArray as $key => $operationDataValue) { + if (is_array($operationDataValue)){ + continue; + } + $operationDataArray[$key] = $dataReferenceResolver->getDataReference( + $operationDataValue, + $entityObject->getName() + ); + } + return $operationDataArray; } diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Util/DataReferenceResolverInterface.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Util/DataReferenceResolverInterface.php new file mode 100644 index 000000000..488a87bfe --- /dev/null +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Util/DataReferenceResolverInterface.php @@ -0,0 +1,26 @@ +{{[\w]+\.[\w\[\]]+}})/"; + + /** + * @param string $data + * @param string $originalDataEntity + * @return mixed + */ + public function getDataReference(string $data, string $originalDataEntity); + + /** + * @param string $data + * @param string $originalDataEntity + * @return mixed + */ + public function getDataUniqueness(string $data, string $originalDataEntity); +} diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Util/GenerationDataReferenceResolver.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Util/GenerationDataReferenceResolver.php new file mode 100644 index 000000000..bea84b7a9 --- /dev/null +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Util/GenerationDataReferenceResolver.php @@ -0,0 +1,82 @@ +getObject($entity); + if ($entityObject === null) { + throw new TestReferenceException( + "Could not resolve entity reference \"{$matches['reference']}\" " + . "in Data entity \"{$originalDataEntity}\"" + ); + } + $entityData = $entityObject->getAllData(); + $result = $entityData[$var]; + } + + return $result; + } + + /** + * @param string $data + * @param string $originalDataEntity + * @return string|null + * @throws TestReferenceException + */ + public function getDataUniqueness(string $data, string $originalDataEntity) + { + preg_match(ActionObject::ACTION_ATTRIBUTE_VARIABLE_REGEX_PATTERN, + $data, + $matches + ); + + if (empty($matches['reference'])) { + return null; + } + + $strippedReference = str_replace(['{{', '}}'], '', $matches['reference']); + list($entity, $var) = explode('.', $strippedReference); + $entityObject = DataObjectHandler::getInstance()->getObject($entity); + if ($entityObject === null) { + throw new TestReferenceException( + "Could not resolve entity reference \"{$matches['reference']}\" " + . "in Data entity \"{$originalDataEntity}\"" + ); + + } + + return $entityObject->getUniquenessDataByName($var); + } +} diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Util/RuntimeDataReferenceResolver.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Util/RuntimeDataReferenceResolver.php new file mode 100644 index 000000000..68f083cad --- /dev/null +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Util/RuntimeDataReferenceResolver.php @@ -0,0 +1,88 @@ +getSecret($var); + $result = CredentialStore::getInstance()->decryptSecretValue($value); + $result = str_replace($matches['reference'], $result, $data); + break; + default: + $entityObject = DataObjectHandler::getInstance()->getObject($entity); + if ($entityObject === null) { + throw new TestReferenceException( + "Could not resolve entity reference \"{$matches['reference']}\" " + . "in Data entity \"{$originalDataEntity}\"" + ); + } + $entityData = $entityObject->getAllData(); + $result = $entityData[$var]; + } + + return $result; + } + + /** + * @param string $data + * @param string $originalDataEntity + * @return string|null + * @throws TestReferenceException + */ + public function getDataUniqueness(string $data, string $originalDataEntity) + { + preg_match(ActionObject::ACTION_ATTRIBUTE_VARIABLE_REGEX_PATTERN, + $data, + $matches + ); + + if (empty($matches['reference'])) { + return null; + } + + $strippedReference = str_replace(['{{', '}}'], '', $matches['reference']); + list($entity, $var) = explode('.', $strippedReference); + $entityObject = DataObjectHandler::getInstance()->getObject($entity); + if ($entityObject === null) { + throw new TestReferenceException( + "Could not resolve entity reference \"{$matches['reference']}\" " + . "in Data entity \"{$originalDataEntity}\"" + ); + + } + + return $entityObject->getUniquenessDataByName($var); + } +} From 124e1069f285287348d5fc473343a9ff503b6e1e Mon Sep 17 00:00:00 2001 From: Alex Kolesnyk Date: Mon, 12 Aug 2019 15:00:45 -0500 Subject: [PATCH 12/82] MQE-1561: CreateData action must be able to parse Data entity which references another data - fix an issue when data is an array --- .../DataGenerator/Objects/EntityDataObject.php | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Objects/EntityDataObject.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Objects/EntityDataObject.php index 3fb511d3b..59e02d09d 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Objects/EntityDataObject.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Objects/EntityDataObject.php @@ -184,6 +184,9 @@ public function getDataByName($name, $uniquenessFormat) $dataReferenceResolver = new GenerationDataReferenceResolver(); if (array_key_exists($name_lower, $this->data)) { + if (is_array($this->data[$name_lower])) { + return $this->data[$name_lower]; + } $uniquenessData = $this->getUniquenessDataByName($name_lower) === null ? $dataReferenceResolver->getDataUniqueness( $this->data[$name_lower], @@ -202,6 +205,9 @@ public function getDataByName($name, $uniquenessFormat) } return $this->formatUniqueData($name_lower, $uniquenessData, $uniquenessFormat); } elseif (array_key_exists($name, $this->data)) { + if (is_array($this->data[$name_lower])) { + return $this->data[$name]; + } $this->data[$name] = $dataReferenceResolver->getDataReference( $this->data[$name], $this->name . '.' . $name From 37ea58d47aeb9168965837db5f57c3078ebf83aa Mon Sep 17 00:00:00 2001 From: Alex Kolesnyk Date: Tue, 13 Aug 2019 12:01:16 -0500 Subject: [PATCH 13/82] MQE-1561: CreateData action must be able to parse Data entity which references another data - fix static check failures --- .../Objects/EntityDataObject.php | 9 +-- .../Persist/OperationDataArrayResolver.php | 5 +- .../Util/GenerationDataReferenceResolver.php | 62 +++++++++---------- .../Util/RuntimeDataReferenceResolver.php | 4 +- 4 files changed, 40 insertions(+), 40 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Objects/EntityDataObject.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Objects/EntityDataObject.php index 59e02d09d..110869187 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Objects/EntityDataObject.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Objects/EntityDataObject.php @@ -82,14 +82,14 @@ class EntityDataObject /** * Constructor * - * @param string $name - * @param string $type + * @param string $name + * @param string $type * @param string[] $data * @param string[] $linkedEntities * @param string[] $uniquenessData * @param string[] $vars - * @param string $parentEntity - * @param string $filename + * @param string $parentEntity + * @param string $filename */ public function __construct( $name, @@ -161,6 +161,7 @@ public function getAllData() * @param integer $uniquenessFormat * @return string|null * @throws TestFrameworkException + * @SuppressWarnings(PHPMD) */ public function getDataByName($name, $uniquenessFormat) { diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/OperationDataArrayResolver.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/OperationDataArrayResolver.php index 7a76c54d1..6d1ebc3fc 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/OperationDataArrayResolver.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/OperationDataArrayResolver.php @@ -66,8 +66,7 @@ public function __construct($dependentEntities = null) * @param boolean $fromArray * @return array * @throws \Exception - * @SuppressWarnings(PHPMD.NPathComplexity) - * @SuppressWarnings(PHPMD.ExcessiveMethodLength) + * @SuppressWarnings(PHPMD) */ public function resolveOperationDataArray($entityObject, $operationMetadata, $operation, $fromArray = false) { @@ -124,7 +123,7 @@ public function resolveOperationDataArray($entityObject, $operationMetadata, $op $dataReferenceResolver = new RuntimeDataReferenceResolver(); foreach ($operationDataArray as $key => $operationDataValue) { - if (is_array($operationDataValue)){ + if (is_array($operationDataValue)) { continue; } $operationDataArray[$key] = $dataReferenceResolver->getDataReference( diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Util/GenerationDataReferenceResolver.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Util/GenerationDataReferenceResolver.php index bea84b7a9..903dcdf29 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Util/GenerationDataReferenceResolver.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Util/GenerationDataReferenceResolver.php @@ -12,6 +12,37 @@ class GenerationDataReferenceResolver implements DataReferenceResolverInterface { + /** + * @param string $data + * @param string $originalDataEntity + * @return string|null + * @throws TestReferenceException + */ + public function getDataUniqueness(string $data, string $originalDataEntity) + { + preg_match( + ActionObject::ACTION_ATTRIBUTE_VARIABLE_REGEX_PATTERN, + $data, + $matches + ); + + if (empty($matches['reference'])) { + return null; + } + + $strippedReference = str_replace(['{{', '}}'], '', $matches['reference']); + list($entity, $var) = explode('.', $strippedReference); + $entityObject = DataObjectHandler::getInstance()->getObject($entity); + if ($entityObject === null) { + throw new TestReferenceException( + "Could not resolve entity reference \"{$matches['reference']}\" " + . "in Data entity \"{$originalDataEntity}\"" + ); + } + + return $entityObject->getUniquenessDataByName($var); + } + /** * @param string $data * @param string $originalDataEntity @@ -48,35 +79,4 @@ public function getDataReference(string $data, string $originalDataEntity) return $result; } - - /** - * @param string $data - * @param string $originalDataEntity - * @return string|null - * @throws TestReferenceException - */ - public function getDataUniqueness(string $data, string $originalDataEntity) - { - preg_match(ActionObject::ACTION_ATTRIBUTE_VARIABLE_REGEX_PATTERN, - $data, - $matches - ); - - if (empty($matches['reference'])) { - return null; - } - - $strippedReference = str_replace(['{{', '}}'], '', $matches['reference']); - list($entity, $var) = explode('.', $strippedReference); - $entityObject = DataObjectHandler::getInstance()->getObject($entity); - if ($entityObject === null) { - throw new TestReferenceException( - "Could not resolve entity reference \"{$matches['reference']}\" " - . "in Data entity \"{$originalDataEntity}\"" - ); - - } - - return $entityObject->getUniquenessDataByName($var); - } } diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Util/RuntimeDataReferenceResolver.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Util/RuntimeDataReferenceResolver.php index 68f083cad..a8bb4154e 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Util/RuntimeDataReferenceResolver.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Util/RuntimeDataReferenceResolver.php @@ -63,7 +63,8 @@ public function getDataReference(string $data, string $originalDataEntity) */ public function getDataUniqueness(string $data, string $originalDataEntity) { - preg_match(ActionObject::ACTION_ATTRIBUTE_VARIABLE_REGEX_PATTERN, + preg_match( + ActionObject::ACTION_ATTRIBUTE_VARIABLE_REGEX_PATTERN, $data, $matches ); @@ -80,7 +81,6 @@ public function getDataUniqueness(string $data, string $originalDataEntity) "Could not resolve entity reference \"{$matches['reference']}\" " . "in Data entity \"{$originalDataEntity}\"" ); - } return $entityObject->getUniquenessDataByName($var); From 32967a2cfb07724bdc98c269cca2a5e1598d2c62 Mon Sep 17 00:00:00 2001 From: Alex Kolesnyk Date: Tue, 13 Aug 2019 15:09:24 -0500 Subject: [PATCH 14/82] MQE-1561: CreateData action must be able to parse Data entity which references another data - fix regex to match all paths --- .../DataGenerator/Util/DataReferenceResolverInterface.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Util/DataReferenceResolverInterface.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Util/DataReferenceResolverInterface.php index 488a87bfe..4c7070646 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Util/DataReferenceResolverInterface.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Util/DataReferenceResolverInterface.php @@ -8,7 +8,7 @@ interface DataReferenceResolverInterface { - const REFERENCE_REGEX_PATTERN = "/(?{{[\w]+\.[\w\[\]]+}})/"; + const REFERENCE_REGEX_PATTERN = "/(?{{[\w]+\..+}})/"; /** * @param string $data From dd2da2e13efa2afb8f7744c3bda0e5a2f0cf204f Mon Sep 17 00:00:00 2001 From: Alex Kolesnyk Date: Thu, 15 Aug 2019 14:37:28 -0500 Subject: [PATCH 15/82] MQE-1561: CreateData action must be able to parse Data entity which references another data - Added phpdoc blocks --- .../Util/GenerationDataReferenceResolver.php | 18 ++++++++++++++++-- .../Util/RuntimeDataReferenceResolver.php | 17 +++++++++++++++-- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Util/GenerationDataReferenceResolver.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Util/GenerationDataReferenceResolver.php index 903dcdf29..c9e0bd9a9 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Util/GenerationDataReferenceResolver.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Util/GenerationDataReferenceResolver.php @@ -10,9 +10,15 @@ use Magento\FunctionalTestingFramework\Exceptions\TestReferenceException; use Magento\FunctionalTestingFramework\Test\Objects\ActionObject; +/** + * Class resolves data references in data entities while generating static test files. + */ class GenerationDataReferenceResolver implements DataReferenceResolverInterface { /** + * Returns data uniqueness for data entity field. + * + * * @param string $data * @param string $originalDataEntity * @return string|null @@ -44,6 +50,8 @@ public function getDataUniqueness(string $data, string $originalDataEntity) } /** + * Returns data by reference if reference exist. + * * @param string $data * @param string $originalDataEntity * @return string|null @@ -69,11 +77,17 @@ public function getDataReference(string $data, string $originalDataEntity) $entityObject = DataObjectHandler::getInstance()->getObject($entity); if ($entityObject === null) { throw new TestReferenceException( - "Could not resolve entity reference \"{$matches['reference']}\" " - . "in Data entity \"{$originalDataEntity}\"" + "Could not find data entity by name \"{$entityObject}\" " + . "referenced in Data entity \"{$originalDataEntity}\"" . PHP_EOL ); } $entityData = $entityObject->getAllData(); + if (!isset($entityData[$var])) { + throw new TestReferenceException( + "Could not resolve entity reference \"{$matches['reference']}\" " + . "in Data entity \"{$originalDataEntity}\"" . PHP_EOL + ); + } $result = $entityData[$var]; } diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Util/RuntimeDataReferenceResolver.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Util/RuntimeDataReferenceResolver.php index a8bb4154e..a09afd01a 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Util/RuntimeDataReferenceResolver.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Util/RuntimeDataReferenceResolver.php @@ -11,9 +11,14 @@ use Magento\FunctionalTestingFramework\Exceptions\TestReferenceException; use Magento\FunctionalTestingFramework\Test\Objects\ActionObject; +/** + * Class resolves data references in data entities at the runtime of tests. + */ class RuntimeDataReferenceResolver implements DataReferenceResolverInterface { /** + * Returns data by reference if reference exist. + * * @param string $data * @param string $originalDataEntity * @return array|false|string|null @@ -44,11 +49,17 @@ public function getDataReference(string $data, string $originalDataEntity) $entityObject = DataObjectHandler::getInstance()->getObject($entity); if ($entityObject === null) { throw new TestReferenceException( - "Could not resolve entity reference \"{$matches['reference']}\" " - . "in Data entity \"{$originalDataEntity}\"" + "Could not find data entity by name \"{$entityObject}\" " + . "referenced in Data entity \"{$originalDataEntity}\"" . PHP_EOL ); } $entityData = $entityObject->getAllData(); + if (!isset($entityData[$var])) { + throw new TestReferenceException( + "Could not resolve entity reference \"{$matches['reference']}\" " + . "in Data entity \"{$originalDataEntity}\"" . PHP_EOL + ); + } $result = $entityData[$var]; } @@ -56,6 +67,8 @@ public function getDataReference(string $data, string $originalDataEntity) } /** + * Returns data uniqueness for data entity field. + * * @param string $data * @param string $originalDataEntity * @return string|null From 932c4355d4a1ed1ff47c58852f5e10538766e6ee Mon Sep 17 00:00:00 2001 From: Soumya Unnikrishnan Date: Mon, 19 Aug 2019 09:06:31 -0500 Subject: [PATCH 16/82] MQE-1373: Cannot Use Data Returned By Test Actions As Substitutable Value Inside Parameterized Selector From Within Action Group to replace custom field key in create data fixed regex to include action group parameter references fixed test generator to make full word replacement of stepKey references --- .../Test/Objects/ActionGroupObject.php | 2 +- src/Magento/FunctionalTestingFramework/Util/TestGenerator.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/Test/Objects/ActionGroupObject.php b/src/Magento/FunctionalTestingFramework/Test/Objects/ActionGroupObject.php index 1e86cff77..15a5f7ba7 100644 --- a/src/Magento/FunctionalTestingFramework/Test/Objects/ActionGroupObject.php +++ b/src/Magento/FunctionalTestingFramework/Test/Objects/ActionGroupObject.php @@ -226,7 +226,7 @@ private function resolveAttributesWithArguments($arguments, $attributes) // $regexPattern match on: $matches[0] {{section.element(arg.field)}} // $matches[1] = section.element // $matches[2] = arg.field - $regexPattern = '/{{([^(}]+)\(*([^)}]+)*\)*}}/'; + $regexPattern = '/{{([^(}]+)\(*([^)]+)*?\)*}}/'; $newActionAttributes = []; foreach ($attributes as $attributeKey => $attributeValue) { diff --git a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php index 3f1417fd1..10fece7b0 100644 --- a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php +++ b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php @@ -1483,7 +1483,7 @@ private function resolveStepKeyReferences($input, $actionGroupOrigin, $matchAll . $stepKey . $testInvocationKey . "', 'field', 'test')"; if (strpos($output, $stepKeyVarRef) !== false) { - $output = str_replace($stepKeyVarRef, $stepKeyVarRef . $testInvocationKey, $output); + $output = preg_replace('/\B\\' .$stepKeyVarRef. '\b/', $stepKeyVarRef . $testInvocationKey, $output); } if (strpos($output, $persistedVarRef) !== false) { From 51e24100128a683931dddbac1adb26793c00c5bf Mon Sep 17 00:00:00 2001 From: Kevin Kozan Date: Tue, 20 Aug 2019 13:32:55 -0500 Subject: [PATCH 17/82] MQE-1628: Error loading Yaml config from - Added quotes to default config --- etc/config/functional.suite.dist.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/etc/config/functional.suite.dist.yml b/etc/config/functional.suite.dist.yml index 12efe5b50..12658515b 100644 --- a/etc/config/functional.suite.dist.yml +++ b/etc/config/functional.suite.dist.yml @@ -28,10 +28,10 @@ modules: username: "%MAGENTO_ADMIN_USERNAME%" password: "%MAGENTO_ADMIN_PASSWORD%" pageload_timeout: 30 - host: %SELENIUM_HOST% - port: %SELENIUM_PORT% - protocol: %SELENIUM_PROTOCOL% - path: %SELENIUM_PATH% + host: "%SELENIUM_HOST%" + port: "%SELENIUM_PORT%" + protocol: "%SELENIUM_PROTOCOL%" + path: "%SELENIUM_PATH%" capabilities: chromeOptions: args: ["--window-size=1280,1024", "--disable-extensions", "--enable-automation", "--disable-gpu", "--enable-Passthrough"] From 36c21d3f429c13aa0f489f68fb76ed5d22bce672 Mon Sep 17 00:00:00 2001 From: Soumya Unnikrishnan Date: Tue, 20 Aug 2019 14:27:56 -0500 Subject: [PATCH 18/82] MQE-1373: Cannot Use Data Returned By Test Actions As Substitutable Value Inside Parameterized Selector From Within Action Group to replace custom field key in create data added verification test + comments --- ...meterizedElementsWithStepKeyReferences.txt | 47 +++++++++++++++++++ .../ActionGroup/FunctionalActionGroup.xml | 12 +++++ .../TestModule/Section/SampleSection.xml | 1 + .../TestModule/Test/ActionGroupTest.xml | 7 +++ .../Tests/ActionGroupGenerationTest.php | 11 +++++ .../Util/TestGenerator.php | 2 + 6 files changed, 80 insertions(+) create mode 100644 dev/tests/verification/Resources/ActionGroupWithParameterizedElementsWithStepKeyReferences.txt diff --git a/dev/tests/verification/Resources/ActionGroupWithParameterizedElementsWithStepKeyReferences.txt b/dev/tests/verification/Resources/ActionGroupWithParameterizedElementsWithStepKeyReferences.txt new file mode 100644 index 000000000..ba62ba479 --- /dev/null +++ b/dev/tests/verification/Resources/ActionGroupWithParameterizedElementsWithStepKeyReferences.txt @@ -0,0 +1,47 @@ +comment("Entering Action Group [actionGroup] actionGroupWithParametrizedSelectors"); + $testVariableActionGroup = $I->executeJS("return 1"); // stepKey: testVariableActionGroup + $testVariable2ActionGroup = $I->executeJS("return 'test'"); // stepKey: testVariable2ActionGroup + $I->comment("[createSimpleDataActionGroup] create 'simpleData' entity"); + PersistedObjectHandler::getInstance()->createEntity( + "createSimpleDataActionGroup", + "test", + "simpleData", + [], + [] + ); + + $I->click("#{$testVariable2ActionGroup} .John"); // stepKey: click1ActionGroup + $I->click("#Doe-" . msq("simpleParamData") . "prename .{$testVariableActionGroup}"); // stepKey: click2ActionGroup + $I->seeElement("//div[@name='Tiberius'][@class={$testVariableActionGroup}][@data-element='{$testVariable2ActionGroup}'][" . PersistedObjectHandler::getInstance()->retrieveEntityField('createSimpleData', 'name', 'test') . "]"); // stepKey: see1ActionGroup + $I->comment("Exiting Action Group [actionGroup] actionGroupWithParametrizedSelectors"); + } +} diff --git a/dev/tests/verification/TestModule/ActionGroup/FunctionalActionGroup.xml b/dev/tests/verification/TestModule/ActionGroup/FunctionalActionGroup.xml index e6a7bad00..30cba3848 100644 --- a/dev/tests/verification/TestModule/ActionGroup/FunctionalActionGroup.xml +++ b/dev/tests/verification/TestModule/ActionGroup/FunctionalActionGroup.xml @@ -91,4 +91,16 @@ + + + + + + + + + + + + diff --git a/dev/tests/verification/TestModule/Section/SampleSection.xml b/dev/tests/verification/TestModule/Section/SampleSection.xml index a560e2f1e..920ee6e1b 100644 --- a/dev/tests/verification/TestModule/Section/SampleSection.xml +++ b/dev/tests/verification/TestModule/Section/SampleSection.xml @@ -15,6 +15,7 @@ + diff --git a/dev/tests/verification/TestModule/Test/ActionGroupTest.xml b/dev/tests/verification/TestModule/Test/ActionGroupTest.xml index 56be24441..5dce96934 100644 --- a/dev/tests/verification/TestModule/Test/ActionGroupTest.xml +++ b/dev/tests/verification/TestModule/Test/ActionGroupTest.xml @@ -171,4 +171,11 @@ + + + + + + + diff --git a/dev/tests/verification/Tests/ActionGroupGenerationTest.php b/dev/tests/verification/Tests/ActionGroupGenerationTest.php index 604f546b2..3f9412aec 100644 --- a/dev/tests/verification/Tests/ActionGroupGenerationTest.php +++ b/dev/tests/verification/Tests/ActionGroupGenerationTest.php @@ -217,4 +217,15 @@ public function testActionGroupWithXmlComments() { $this->generateAndCompareTest('XmlCommentedActionGroupTest'); } + + /** + * Test generation of a test referencing an action group with selectors referencing stepKeys. + * + * @throws \Exception + * @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException + */ + public function testActionGroupWithActionStepKeyReferencesInSelectors() + { + $this->generateAndCompareTest('ActionGroupWithParameterizedElementsWithStepKeyReferences'); + } } diff --git a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php index 10fece7b0..a9eb15152 100644 --- a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php +++ b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php @@ -1482,6 +1482,8 @@ private function resolveStepKeyReferences($input, $actionGroupOrigin, $matchAll $persistedVarRefInvoked = "PersistedObjectHandler::getInstance()->retrieveEntityField('" . $stepKey . $testInvocationKey . "', 'field', 'test')"; + // only replace when whole word matches exactly + // e.g. testVar => $testVar but not $testVar2 if (strpos($output, $stepKeyVarRef) !== false) { $output = preg_replace('/\B\\' .$stepKeyVarRef. '\b/', $stepKeyVarRef . $testInvocationKey, $output); } From b41049927e4988211197d07dc8f50f78c62bc21c Mon Sep 17 00:00:00 2001 From: Kevin Kozan Date: Tue, 20 Aug 2019 14:53:45 -0500 Subject: [PATCH 19/82] MQE-1705: Fix PHP Notice in Unit Test - Fixed potential bug --- .../DataGenerator/Objects/EntityDataObject.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Objects/EntityDataObject.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Objects/EntityDataObject.php index 110869187..34edeee1d 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Objects/EntityDataObject.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Objects/EntityDataObject.php @@ -206,7 +206,7 @@ public function getDataByName($name, $uniquenessFormat) } return $this->formatUniqueData($name_lower, $uniquenessData, $uniquenessFormat); } elseif (array_key_exists($name, $this->data)) { - if (is_array($this->data[$name_lower])) { + if (is_array($this->data[$name])) { return $this->data[$name]; } $this->data[$name] = $dataReferenceResolver->getDataReference( From dd35d9db2d2750a4aaab3b38791812aed8a7e7b2 Mon Sep 17 00:00:00 2001 From: Soumya Unnikrishnan Date: Tue, 20 Aug 2019 15:39:28 -0500 Subject: [PATCH 20/82] MQE-1331: MFTF Run:Test Will Run All Tests That Are Already Generated updated code to run only tests specified in run command --- .../Console/RunTestCommand.php | 37 +++++++++++++------ 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php index 76a8bc8f4..1dae2a7d4 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php @@ -8,6 +8,7 @@ namespace Magento\FunctionalTestingFramework\Console; use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig; +use Magento\FunctionalTestingFramework\Util\TestGenerator; use Symfony\Component\Console\Input\ArrayInput; use Symfony\Component\Console\Input\InputArgument; use Symfony\Component\Console\Input\InputInterface; @@ -88,18 +89,32 @@ protected function execute(InputInterface $input, OutputInterface $output): int $command->run(new ArrayInput($args), $output); } - // we only generate relevant tests here so we can execute "all tests" - $codeceptionCommand = realpath(PROJECT_ROOT . '/vendor/bin/codecept') . " run functional --verbose --steps"; + $returnCode = 0; + //execute only tests specified as arguments in run command + foreach ($tests as $test) { + $codeceptionCommand = realpath(PROJECT_ROOT . '/vendor/bin/codecept') . ' run functional '; + $test = TESTS_MODULE_PATH . + DIRECTORY_SEPARATOR . + TestGenerator::GENERATED_DIR . + DIRECTORY_SEPARATOR . + TestGenerator::DEFAULT_DIR . + DIRECTORY_SEPARATOR . + $test . + 'Cest.php'; + $codeceptionCommand .= $test; + $codeceptionCommand .= ' --verbose --steps'; - $process = new Process($codeceptionCommand); - $process->setWorkingDirectory(TESTS_BP); - $process->setIdleTimeout(600); - $process->setTimeout(0); + $process = new Process($codeceptionCommand); + $process->setWorkingDirectory(TESTS_BP); + $process->setIdleTimeout(600); + $process->setTimeout(0); - return $process->run( - function ($type, $buffer) use ($output) { - $output->write($buffer); - } - ); + $returnCode = max($returnCode, $process->run( + function ($type, $buffer) use ($output) { + $output->write($buffer); + } + )); + } + return $returnCode; } } From b27e2a9b6ff8602551ca8535912a942022d1ebf7 Mon Sep 17 00:00:00 2001 From: Kevin Kozan Date: Wed, 21 Aug 2019 08:47:56 -0500 Subject: [PATCH 21/82] MQE-1582: Enable Testers To Run Skipped Tests - added --allowSkipped flag, added to base generate command - moved --force flag to base generate command - fixed some existing issues around not all BaseGenerate inheriting commands not making proper use of --force flag --- .../Config/MftfApplicationConfig.php | 35 ++++++++++++++++--- .../Console/BaseGenerateCommand.php | 10 ++++++ .../Console/GenerateTestsCommand.php | 14 ++++---- .../Console/RunTestCommand.php | 10 +++--- .../Console/RunTestFailedCommand.php | 12 ++++--- .../Console/RunTestGroupCommand.php | 12 +++---- .../Util/TestGenerator.php | 3 +- 7 files changed, 65 insertions(+), 31 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php b/src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php index 29278f761..799798640 100644 --- a/src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php +++ b/src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php @@ -9,6 +9,9 @@ class MftfApplicationConfig { + /** + * MFTF Execution Phases + */ const GENERATION_PHASE = "generation"; const EXECUTION_PHASE = "execution"; const UNIT_TEST_PHASE = "testing"; @@ -50,6 +53,12 @@ class MftfApplicationConfig */ private $debugLevel; + /** + * Boolean which allows MFTF to fully generate skipped tests + * @var bool + */ + private $allowSkipped; + /** * MftfApplicationConfig Singelton Instance * @@ -64,13 +73,15 @@ class MftfApplicationConfig * @param string $phase * @param boolean $verboseEnabled * @param string $debugLevel + * @param boolean $allowSkipped * @throws TestFrameworkException */ private function __construct( $forceGenerate = false, $phase = self::EXECUTION_PHASE, $verboseEnabled = null, - $debugLevel = self::LEVEL_NONE + $debugLevel = self::LEVEL_NONE, + $allowSkipped = false ) { $this->forceGenerate = $forceGenerate; @@ -89,6 +100,7 @@ private function __construct( default: $this->debugLevel = self::LEVEL_DEVELOPER; } + $this->allowSkipped = $allowSkipped; } /** @@ -96,16 +108,18 @@ private function __construct( * returns void and is only run once during the lifetime of the application. * * @param boolean $forceGenerate - * @param string $phase + * @param string $phase * @param boolean $verboseEnabled - * @param string $debugLevel + * @param string $debugLevel + * @param boolean $allowSkipped * @return void + * @throws TestFrameworkException */ - public static function create($forceGenerate, $phase, $verboseEnabled, $debugLevel) + public static function create($forceGenerate, $phase, $verboseEnabled, $debugLevel, $allowSkipped) { if (self::$MFTF_APPLICATION_CONTEXT == null) { self::$MFTF_APPLICATION_CONTEXT = - new MftfApplicationConfig($forceGenerate, $phase, $verboseEnabled, $debugLevel); + new MftfApplicationConfig($forceGenerate, $phase, $verboseEnabled, $debugLevel, $allowSkipped); } } @@ -113,6 +127,7 @@ public static function create($forceGenerate, $phase, $verboseEnabled, $debugLev * This function returns an instance of the MftfApplicationConfig which is created once the application starts. * * @return MftfApplicationConfig + * @throws TestFrameworkException */ public static function getConfig() { @@ -157,6 +172,16 @@ public function getDebugLevel() return $this->debugLevel ?? getenv('MFTF_DEBUG'); } + /** + * Returns a boolean indicating whether mftf is generating skipped tests. + * + * @return boolean + */ + public function allowSkipped() + { + return $this->allowSkipped ?? getenv('ALLOW_SKIPPED'); + } + /** * Returns a string which indicates the phase of mftf execution. * diff --git a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php index d4044c74b..db175ff3f 100644 --- a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php @@ -28,6 +28,16 @@ protected function configure() 'r', InputOption::VALUE_NONE, 'remove previous generated suites and tests' + )->addOption( + "force", + 'f', + InputOption::VALUE_NONE, + 'force generation of tests regardless of Magento Instance Configuration' + )->addOption( + "allowSkipped", + 'a', + InputOption::VALUE_NONE, + 'Allows MFTF to generate and run skipped tests.' ); } diff --git a/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php b/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php index 23651b492..4b4af5855 100644 --- a/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php @@ -36,11 +36,6 @@ protected function configure() 'name(s) of specific tests to generate' )->addOption("config", 'c', InputOption::VALUE_REQUIRED, 'default, singleRun, or parallel', 'default') ->addOption( - "force", - 'f', - InputOption::VALUE_NONE, - 'Force generation of tests regardless of Magento Instance Configuration' - )->addOption( 'time', 'i', InputOption::VALUE_REQUIRED, @@ -83,6 +78,7 @@ protected function execute(InputInterface $input, OutputInterface $output) $debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility $remove = $input->getOption('remove'); $verbose = $output->isVerbose(); + $allowSkipped = $input->getOption('allowSkipped'); if ($json !== null && !json_decode($json)) { // stop execution if we have failed to properly parse any json passed in by the user @@ -100,7 +96,7 @@ protected function execute(InputInterface $input, OutputInterface $output) ($debug !== MftfApplicationConfig::LEVEL_NONE)); } - $testConfiguration = $this->createTestConfiguration($json, $tests, $force, $debug, $verbose); + $testConfiguration = $this->createTestConfiguration($json, $tests, $force, $debug, $verbose, $allowSkipped); // create our manifest file here $testManifest = TestManifestFactory::makeManifest($config, $testConfiguration['suites']); @@ -128,18 +124,20 @@ protected function execute(InputInterface $input, OutputInterface $output) * @param boolean $force * @param string $debug * @param boolean $verbose + * @param boolean $allowSkipped * @return array * @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException * @throws \Magento\FunctionalTestingFramework\Exceptions\XmlException */ - private function createTestConfiguration($json, array $tests, bool $force, $debug, bool $verbose) + private function createTestConfiguration($json, array $tests, bool $force, $debug, bool $verbose, bool $allowSkipped) { // set our application configuration so we can references the user options in our framework MftfApplicationConfig::create( $force, MftfApplicationConfig::GENERATION_PHASE, $verbose, - $debug + $debug, + $allowSkipped ); $testConfiguration = []; diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php index 76a8bc8f4..4bd64dec3 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php @@ -33,11 +33,6 @@ protected function configure() "name of tests to generate and execute" )->addOption('skip-generate', 'k', InputOption::VALUE_NONE, "skip generation and execute existing test") ->addOption( - "force", - 'f', - InputOption::VALUE_NONE, - 'force generation of tests regardless of Magento Instance Configuration' - )->addOption( 'debug', 'd', InputOption::VALUE_OPTIONAL, @@ -66,6 +61,8 @@ protected function execute(InputInterface $input, OutputInterface $output): int $force = $input->getOption('force'); $remove = $input->getOption('remove'); $debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility + $allowSkipped = $input->getOption('allowSkipped'); + if ($skipGeneration and $remove) { // "skip-generate" and "remove" options cannot be used at the same time @@ -83,7 +80,8 @@ protected function execute(InputInterface $input, OutputInterface $output): int ]), '--force' => $force, '--remove' => $remove, - '--debug' => $debug + '--debug' => $debug, + '--allowSkipped' => $allowSkipped ]; $command->run(new ArrayInput($args), $output); } diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php index d6a580d52..bff232c0c 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php @@ -76,13 +76,17 @@ protected function configure() */ protected function execute(InputInterface $input, OutputInterface $output): int { + $force = $input->getOption('force'); $debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility + $allowSkipped = $input->getOption('allowSkipped'); + // Create Mftf Configuration MftfApplicationConfig::create( - false, + $force, MftfApplicationConfig::GENERATION_PHASE, false, - $debug + $debug, + $allowSkipped ); $testConfiguration = $this->getFailedTestList(); @@ -96,9 +100,9 @@ protected function execute(InputInterface $input, OutputInterface $output): int $args = [ '--tests' => $testConfiguration, '--remove' => true, - '--debug' => $debug + '--debug' => $debug, + '--allowSkipped' => $allowSkipped ]; - $command->run(new ArrayInput($args), $output); $testManifestList = $this->readTestManifestFile(); diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php index d714a1dd1..dfbeda211 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php @@ -34,11 +34,6 @@ protected function configure() 'k', InputOption::VALUE_NONE, "only execute a group of tests without generating from source xml" - )->addOption( - "force", - 'f', - InputOption::VALUE_NONE, - 'force generation of tests regardless of Magento Instance Configuration' )->addOption( 'debug', 'd', @@ -72,6 +67,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int $groups = $input->getArgument('groups'); $remove = $input->getOption('remove'); $debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility + $allowSkipped = $input->getOption('allowSkipped'); if ($skipGeneration and $remove) { // "skip-generate" and "remove" options cannot be used at the same time @@ -85,7 +81,8 @@ protected function execute(InputInterface $input, OutputInterface $output): int $force, MftfApplicationConfig::GENERATION_PHASE, false, - $debug + $debug, + $allowSkipped ); if (!$skipGeneration) { @@ -95,7 +92,8 @@ protected function execute(InputInterface $input, OutputInterface $output): int '--tests' => $testConfiguration, '--force' => $force, '--remove' => $remove, - '--debug' => $debug + '--debug' => $debug, + '--allowSkipped' => $allowSkipped ]; $command->run(new ArrayInput($args), $output); diff --git a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php index 3f1417fd1..d541a5eaf 100644 --- a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php +++ b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php @@ -6,6 +6,7 @@ namespace Magento\FunctionalTestingFramework\Util; +use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig; use Magento\FunctionalTestingFramework\DataGenerator\Handlers\CredentialStore; use Magento\FunctionalTestingFramework\DataGenerator\Handlers\PersistedObjectHandler; use Magento\FunctionalTestingFramework\DataGenerator\Objects\EntityDataObject; @@ -1604,7 +1605,7 @@ private function generateTestPhp($test) $testName = str_replace(' ', '', $testName); $testAnnotations = $this->generateAnnotationsPhp($test->getAnnotations(), true); $dependencies = 'AcceptanceTester $I'; - if ($test->isSkipped()) { + if ($test->isSkipped() && !MftfApplicationConfig::getConfig()->allowSkipped()) { $skipString = "This test is skipped due to the following issues:\\n"; $issues = $test->getAnnotations()['skip'] ?? null; if (isset($issues)) { From 9dfcefb884278d07c9e08a94059b3658883a52d4 Mon Sep 17 00:00:00 2001 From: Kevin Kozan Date: Wed, 21 Aug 2019 08:59:08 -0500 Subject: [PATCH 22/82] MQE-1582: Enable Testers To Run Skipped Tests - Unit/Static check fixes --- dev/tests/_bootstrap.php | 3 ++- .../Config/MftfApplicationConfig.php | 6 +++--- .../Console/BaseGenerateCommand.php | 7 +++++++ .../Console/GenerateTestsCommand.php | 17 ++++++++--------- .../Console/RunTestCommand.php | 11 +---------- .../Console/RunTestGroupCommand.php | 7 ------- .../DataGenerator/Objects/EntityDataObject.php | 2 +- 7 files changed, 22 insertions(+), 31 deletions(-) diff --git a/dev/tests/_bootstrap.php b/dev/tests/_bootstrap.php index d9a4208b2..e06e373d4 100644 --- a/dev/tests/_bootstrap.php +++ b/dev/tests/_bootstrap.php @@ -32,7 +32,8 @@ true, \Magento\FunctionalTestingFramework\Config\MftfApplicationConfig::UNIT_TEST_PHASE, true, - \Magento\FunctionalTestingFramework\Config\MftfApplicationConfig::LEVEL_NONE + \Magento\FunctionalTestingFramework\Config\MftfApplicationConfig::LEVEL_NONE, + false ); // Load needed framework env params diff --git a/src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php b/src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php index 799798640..6faf322bc 100644 --- a/src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php +++ b/src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php @@ -55,7 +55,7 @@ class MftfApplicationConfig /** * Boolean which allows MFTF to fully generate skipped tests - * @var bool + * @var boolean */ private $allowSkipped; @@ -108,9 +108,9 @@ private function __construct( * returns void and is only run once during the lifetime of the application. * * @param boolean $forceGenerate - * @param string $phase + * @param string $phase * @param boolean $verboseEnabled - * @param string $debugLevel + * @param string $debugLevel * @param boolean $allowSkipped * @return void * @throws TestFrameworkException diff --git a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php index db175ff3f..9f989f956 100644 --- a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php @@ -38,6 +38,13 @@ protected function configure() 'a', InputOption::VALUE_NONE, 'Allows MFTF to generate and run skipped tests.' + )->addOption( + 'debug', + 'd', + InputOption::VALUE_OPTIONAL, + 'Run extra validation when running tests. Use option \'none\' to turn off debugging -- + added for backward compatibility, will be removed in the next MAJOR release', + MftfApplicationConfig::LEVEL_DEFAULT ); } diff --git a/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php b/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php index 4b4af5855..98ec87e11 100644 --- a/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php @@ -46,13 +46,6 @@ protected function configure() 't', InputOption::VALUE_REQUIRED, 'A parameter accepting a JSON string used to determine the test configuration' - )->addOption( - 'debug', - 'd', - InputOption::VALUE_OPTIONAL, - 'Run extra validation when generating tests. Use option \'none\' to turn off debugging -- - added for backward compatibility, will be removed in the next MAJOR release', - MftfApplicationConfig::LEVEL_DEFAULT ); parent::configure(); @@ -129,8 +122,14 @@ protected function execute(InputInterface $input, OutputInterface $output) * @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException * @throws \Magento\FunctionalTestingFramework\Exceptions\XmlException */ - private function createTestConfiguration($json, array $tests, bool $force, $debug, bool $verbose, bool $allowSkipped) - { + private function createTestConfiguration( + $json, + array $tests, + bool $force, + $debug, + bool $verbose, + bool $allowSkipped + ) { // set our application configuration so we can references the user options in our framework MftfApplicationConfig::create( $force, diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php index 4bd64dec3..fab65a8e6 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php @@ -31,15 +31,7 @@ protected function configure() 'name', InputArgument::REQUIRED | InputArgument::IS_ARRAY, "name of tests to generate and execute" - )->addOption('skip-generate', 'k', InputOption::VALUE_NONE, "skip generation and execute existing test") - ->addOption( - 'debug', - 'd', - InputOption::VALUE_OPTIONAL, - 'Run extra validation when running tests. Use option \'none\' to turn off debugging -- - added for backward compatibility, will be removed in the next MAJOR release', - MftfApplicationConfig::LEVEL_DEFAULT - ); + )->addOption('skip-generate', 'k', InputOption::VALUE_NONE, "skip generation and execute existing test"); parent::configure(); } @@ -63,7 +55,6 @@ protected function execute(InputInterface $input, OutputInterface $output): int $debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility $allowSkipped = $input->getOption('allowSkipped'); - if ($skipGeneration and $remove) { // "skip-generate" and "remove" options cannot be used at the same time throw new TestFrameworkException( diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php index dfbeda211..81a6b864f 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php @@ -34,13 +34,6 @@ protected function configure() 'k', InputOption::VALUE_NONE, "only execute a group of tests without generating from source xml" - )->addOption( - 'debug', - 'd', - InputOption::VALUE_OPTIONAL, - 'Run extra validation when running tests. Use option \'none\' to turn off debugging -- - added for backward compatibility, will be removed in the next MAJOR release', - MftfApplicationConfig::LEVEL_DEFAULT )->addArgument( 'groups', InputArgument::IS_ARRAY | InputArgument::REQUIRED, diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Objects/EntityDataObject.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Objects/EntityDataObject.php index 110869187..34edeee1d 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Objects/EntityDataObject.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Objects/EntityDataObject.php @@ -206,7 +206,7 @@ public function getDataByName($name, $uniquenessFormat) } return $this->formatUniqueData($name_lower, $uniquenessData, $uniquenessFormat); } elseif (array_key_exists($name, $this->data)) { - if (is_array($this->data[$name_lower])) { + if (is_array($this->data[$name])) { return $this->data[$name]; } $this->data[$name] = $dataReferenceResolver->getDataReference( From a9d19170fccaa0ea75e72c44b5de8be5e9a0750e Mon Sep 17 00:00:00 2001 From: Kevin Kozan Date: Wed, 21 Aug 2019 09:42:34 -0500 Subject: [PATCH 23/82] MQE-1582: Enable Testers To Run Skipped Tests - Added new unit test for TestGenerator, also changed visibility of function for testing --- .../Util/TestGeneratorTest.php | 50 +++++++++++++++++++ .../Console/BaseGenerateCommand.php | 1 + .../Util/TestGenerator.php | 4 +- 3 files changed, 53 insertions(+), 2 deletions(-) diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Util/TestGeneratorTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Util/TestGeneratorTest.php index 097150888..c11a40d2c 100644 --- a/dev/tests/unit/Magento/FunctionalTestFramework/Util/TestGeneratorTest.php +++ b/dev/tests/unit/Magento/FunctionalTestFramework/Util/TestGeneratorTest.php @@ -12,6 +12,9 @@ use Magento\FunctionalTestingFramework\Test\Objects\TestObject; use Magento\FunctionalTestingFramework\Util\MagentoTestCase; use Magento\FunctionalTestingFramework\Util\TestGenerator; +use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig; +use Magento\FunctionalTestingFramework\ObjectManager; +use Magento\FunctionalTestingFramework\ObjectManagerFactory; class TestGeneratorTest extends MagentoTestCase { @@ -38,4 +41,51 @@ public function testEntityException() $testGeneratorObject->createAllTestFiles(null, []); } + + /** + * Tests that skipped tests do not have a fully generated body + * + * @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException + */ + public function testSkippedNoGeneration() + { + $actionInput = 'fakeInput'; + $actionObject = new ActionObject('fakeAction', 'comment', [ + 'userInput' => $actionInput + ]); + + $annotations = ['skip' => ['issue']]; + $testObject = new TestObject("sampleTest", ["merge123" => $actionObject], $annotations, [], "filename"); + + $testGeneratorObject = TestGenerator::getInstance("", ["sampleTest" => $testObject]); + $output = $testGeneratorObject->assembleTestPhp($testObject); + + $this->assertContains('This test is skipped', $output); + $this->assertNotContains( $actionInput, $output); + } + + /** + * Tests that skipped tests have a fully generated body when --allowSkipped is passed in + * + * @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException + */ + public function testAllowSkipped() + { + // Mock allowSkipped for TestGenerator + AspectMock::double(MftfApplicationConfig::class, ['allowSkipped' => true]); + + $actionInput = 'fakeInput'; + $actionObject = new ActionObject('fakeAction', 'comment', [ + 'userInput' => $actionInput + ]); + + $annotations = ['skip' => ['issue']]; + $testObject = new TestObject("sampleTest", ["merge123" => $actionObject], $annotations, [], "filename"); + + $testGeneratorObject = TestGenerator::getInstance("", ["sampleTest" => $testObject]); + $output = $testGeneratorObject->assembleTestPhp($testObject); + + $this->assertNotContains('This test is skipped', $output); + $this->assertContains($actionInput, $output); + } } diff --git a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php index 9f989f956..6acbf8b4d 100644 --- a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php @@ -13,6 +13,7 @@ use Symfony\Component\Console\Output\OutputInterface; use Magento\FunctionalTestingFramework\Util\Filesystem\DirSetupUtil; use Magento\FunctionalTestingFramework\Util\TestGenerator; +use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig; class BaseGenerateCommand extends Command { diff --git a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php index d541a5eaf..94868031f 100644 --- a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php +++ b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php @@ -228,14 +228,14 @@ public function createAllTestFiles($testManifest = null, $testsToIgnore = null) * @throws TestReferenceException * @throws \Exception */ - private function assembleTestPhp($testObject) + public function assembleTestPhp($testObject) { $usePhp = $this->generateUseStatementsPhp(); $classAnnotationsPhp = $this->generateAnnotationsPhp($testObject->getAnnotations()); $className = $testObject->getCodeceptionName(); try { - if (!$testObject->isSkipped()) { + if (!$testObject->isSkipped() && !MftfApplicationConfig::getConfig()->allowSkipped()) { $hookPhp = $this->generateHooksPhp($testObject->getHooks()); } else { $hookPhp = null; From 3409ac8fbb690fd00c6333ee978988cf3ade1c21 Mon Sep 17 00:00:00 2001 From: Kevin Kozan Date: Wed, 21 Aug 2019 09:43:16 -0500 Subject: [PATCH 24/82] MQE-1582: Enable Testers To Run Skipped Tests - Unit Test Cleanup --- .../Magento/FunctionalTestFramework/Util/TestGeneratorTest.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Util/TestGeneratorTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Util/TestGeneratorTest.php index c11a40d2c..b5c8537aa 100644 --- a/dev/tests/unit/Magento/FunctionalTestFramework/Util/TestGeneratorTest.php +++ b/dev/tests/unit/Magento/FunctionalTestFramework/Util/TestGeneratorTest.php @@ -13,8 +13,6 @@ use Magento\FunctionalTestingFramework\Util\MagentoTestCase; use Magento\FunctionalTestingFramework\Util\TestGenerator; use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig; -use Magento\FunctionalTestingFramework\ObjectManager; -use Magento\FunctionalTestingFramework\ObjectManagerFactory; class TestGeneratorTest extends MagentoTestCase { From 75cc720af7171c9afe245efd86f8622daad0af58 Mon Sep 17 00:00:00 2001 From: Kevin Kozan Date: Thu, 22 Aug 2019 11:43:56 -0500 Subject: [PATCH 25/82] MQE-1582: Enable Testers To Run Skipped Tests - static check fix --- .../Magento/FunctionalTestFramework/Util/TestGeneratorTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Util/TestGeneratorTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Util/TestGeneratorTest.php index b5c8537aa..cadc1c6a8 100644 --- a/dev/tests/unit/Magento/FunctionalTestFramework/Util/TestGeneratorTest.php +++ b/dev/tests/unit/Magento/FunctionalTestFramework/Util/TestGeneratorTest.php @@ -59,7 +59,7 @@ public function testSkippedNoGeneration() $output = $testGeneratorObject->assembleTestPhp($testObject); $this->assertContains('This test is skipped', $output); - $this->assertNotContains( $actionInput, $output); + $this->assertNotContains($actionInput, $output); } /** From 60fc5715ab9d8042de57283c75fb08fa0acf1760 Mon Sep 17 00:00:00 2001 From: Kevin Kozan Date: Fri, 23 Aug 2019 11:30:19 -0500 Subject: [PATCH 26/82] MQE-1582: Enable Testers To Run Skipped Tests - CR Fixes --- .../Console/BaseGenerateCommand.php | 4 ++-- .../Console/GenerateTestsCommand.php | 4 ++-- .../Console/RunTestFailedCommand.php | 10 +--------- 3 files changed, 5 insertions(+), 13 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php index 6acbf8b4d..75c521379 100644 --- a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php @@ -33,7 +33,7 @@ protected function configure() "force", 'f', InputOption::VALUE_NONE, - 'force generation of tests regardless of Magento Instance Configuration' + 'force generation and running of tests regardless of Magento Instance Configuration' )->addOption( "allowSkipped", 'a', @@ -43,7 +43,7 @@ protected function configure() 'debug', 'd', InputOption::VALUE_OPTIONAL, - 'Run extra validation when running tests. Use option \'none\' to turn off debugging -- + 'Run extra validation when generating and running tests. Use option \'none\' to turn off debugging -- added for backward compatibility, will be removed in the next MAJOR release', MftfApplicationConfig::LEVEL_DEFAULT ); diff --git a/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php b/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php index 98ec87e11..581688615 100644 --- a/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php @@ -123,10 +123,10 @@ protected function execute(InputInterface $input, OutputInterface $output) * @throws \Magento\FunctionalTestingFramework\Exceptions\XmlException */ private function createTestConfiguration( - $json, + string $json, array $tests, bool $force, - $debug, + string $debug, bool $verbose, bool $allowSkipped ) { diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php index bff232c0c..ed00322ff 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php @@ -50,15 +50,7 @@ class RunTestFailedCommand extends BaseGenerateCommand protected function configure() { $this->setName('run:failed') - ->setDescription('Execute a set of tests referenced via failed file') - ->addOption( - 'debug', - 'd', - InputOption::VALUE_OPTIONAL, - 'Run extra validation when running failed tests. Use option \'none\' to turn off debugging -- - added for backward compatibility, will be removed in the next MAJOR release', - MftfApplicationConfig::LEVEL_DEFAULT - ); + ->setDescription('Execute a set of tests referenced via failed file'); parent::configure(); } From 51fc64f40f825e485410bd2cf7ff7d5d17349975 Mon Sep 17 00:00:00 2001 From: Kevin Kozan Date: Fri, 23 Aug 2019 14:10:16 -0500 Subject: [PATCH 27/82] MQE-1582: Enable Testers To Run Skipped Tests - Added documentation for new argument --- docs/commands/mftf.md | 1 + etc/config/.env.example | 3 +++ 2 files changed, 4 insertions(+) diff --git a/docs/commands/mftf.md b/docs/commands/mftf.md index 2301d6f44..aea53c96f 100644 --- a/docs/commands/mftf.md +++ b/docs/commands/mftf.md @@ -122,6 +122,7 @@ vendor/bin/mftf generate:tests [option] [] [] [--remove] | `--force` | Forces test generation, regardless of the module merge order defined in the Magento instance. Example: `generate:tests --force`. | | `-i,--time` | Set time in minutes to determine the group size when `--config=parallel` is used. The __default value__ is `10`. Example: `generate:tests --config=parallel --time=15`| | `--tests` | Defines the test configuration as a JSON string.| +| `--allow-skipped` | Allows MFTF to generate and run tests marked with `.`| | `--debug or --debug=[]`| Performs schema validations on XML files.
DEFAULT: `generate:tests` implicitly performs schema validation on merged files. It does not indicate the file name where the error is encountered.
DEVELOPER: `--debug` performs per-file validation and returns additional debug information (such as the filename where an error occurred) when test generation fails because of an invalid XML schema. This option takes extra processing time. Use it after test generation has failed once.

NONE: `--debug=none` skips debugging during test generation. Added for backward compatibility, it will be removed in the next MAJOR release.
| | `-r,--remove`| Removes the existing generated suites and tests cleaning up the `_generated` directory before the actual run. For example, `generate:tests SampleTest --remove` cleans up the entire `_generated` directory and generates `SampleTest` only.| diff --git a/etc/config/.env.example b/etc/config/.env.example index 369fd946b..f4aa90e51 100644 --- a/etc/config/.env.example +++ b/etc/config/.env.example @@ -49,6 +49,9 @@ MODULE_WHITELIST=Magento_Framework,Magento_ConfigurableProductWishlist,Magento_C #*** Bool property which allows the user to toggle debug output during test execution #MFTF_DEBUG= +#*** Bool property which allows the user to generate and run tests marked as skipped +#ALLOW_SKIPPED=true + #*** Default timeout for wait actions #WAIT_TIMEOUT=10 #*** End of .env ***# From d2010f00ac650475d6a3f8f99e308f5752ad794b Mon Sep 17 00:00:00 2001 From: Soumya Unnikrishnan Date: Fri, 23 Aug 2019 16:04:01 -0500 Subject: [PATCH 28/82] MQE-1331: MFTF Run:Test Will Run All Tests That Are Already Generated restructured command formation --- .../Console/RunTestCommand.php | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php index 1dae2a7d4..bf4aa711b 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php @@ -90,21 +90,14 @@ protected function execute(InputInterface $input, OutputInterface $output): int } $returnCode = 0; + $codeceptionCommand = realpath(PROJECT_ROOT . '/vendor/bin/codecept') . ' run functional '; + $testsDirectory = TESTS_MODULE_PATH . DIRECTORY_SEPARATOR . TestGenerator::GENERATED_DIR . DIRECTORY_SEPARATOR; //execute only tests specified as arguments in run command foreach ($tests as $test) { - $codeceptionCommand = realpath(PROJECT_ROOT . '/vendor/bin/codecept') . ' run functional '; - $test = TESTS_MODULE_PATH . - DIRECTORY_SEPARATOR . - TestGenerator::GENERATED_DIR . - DIRECTORY_SEPARATOR . - TestGenerator::DEFAULT_DIR . - DIRECTORY_SEPARATOR . - $test . - 'Cest.php'; - $codeceptionCommand .= $test; - $codeceptionCommand .= ' --verbose --steps'; - - $process = new Process($codeceptionCommand); + $testGroup = TestGenerator::DEFAULT_DIR . DIRECTORY_SEPARATOR; + $testName = $test . 'Cest.php'; + $fullCommand = $codeceptionCommand . $testsDirectory . $testGroup . $testName . ' --verbose --steps'; + $process = new Process($fullCommand); $process->setWorkingDirectory(TESTS_BP); $process->setIdleTimeout(600); $process->setTimeout(0); From 4f25d64886fa1b851e7a67333f4ae30b8a7bdd94 Mon Sep 17 00:00:00 2001 From: Soumya Unnikrishnan Date: Mon, 26 Aug 2019 09:27:22 -0500 Subject: [PATCH 29/82] MQE-1331: MFTF Run:Test Will Run All Tests That Are Already Generated added error handling --- .../FunctionalTestingFramework/Console/RunTestCommand.php | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php index bf4aa711b..6d261b6d7 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php @@ -96,6 +96,11 @@ protected function execute(InputInterface $input, OutputInterface $output): int foreach ($tests as $test) { $testGroup = TestGenerator::DEFAULT_DIR . DIRECTORY_SEPARATOR; $testName = $test . 'Cest.php'; + if (!realpath($testsDirectory . $testGroup . $testName)) { + throw new TestFrameworkException( + $testName . " is not available under " . $testsDirectory . $testGroup + ); + } $fullCommand = $codeceptionCommand . $testsDirectory . $testGroup . $testName . ' --verbose --steps'; $process = new Process($fullCommand); $process->setWorkingDirectory(TESTS_BP); From b328d66ee12778c63b0a7085b8299c0b9b4560de Mon Sep 17 00:00:00 2001 From: Kevin Kozan Date: Mon, 26 Aug 2019 09:44:14 -0500 Subject: [PATCH 30/82] MQE-1582: Enable Testers To Run Skipped Tests - Fix nullable type hinting --- .../FunctionalTestingFramework/Console/GenerateTestsCommand.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php b/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php index 581688615..ca9d6e3cc 100644 --- a/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php @@ -123,7 +123,7 @@ protected function execute(InputInterface $input, OutputInterface $output) * @throws \Magento\FunctionalTestingFramework\Exceptions\XmlException */ private function createTestConfiguration( - string $json, + $json, array $tests, bool $force, string $debug, From 49c96499a74bd5089b70f444a1c5179a5af56eb1 Mon Sep 17 00:00:00 2001 From: Tom Reece Date: Mon, 26 Aug 2019 10:32:44 -0500 Subject: [PATCH 31/82] MQE-1055: [SPIKE] Audit performance issues with test generation in framework --- .../Test/Objects/ActionGroupObject.php | 21 +++++++++++++------ .../Test/Objects/TestObject.php | 15 +++++++++++-- 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/Test/Objects/ActionGroupObject.php b/src/Magento/FunctionalTestingFramework/Test/Objects/ActionGroupObject.php index 15a5f7ba7..fc38a7fce 100644 --- a/src/Magento/FunctionalTestingFramework/Test/Objects/ActionGroupObject.php +++ b/src/Magento/FunctionalTestingFramework/Test/Objects/ActionGroupObject.php @@ -87,6 +87,13 @@ class ActionGroupObject */ private $filename; + /** + * Holds on to the result of extractStepKeys() to increase test generation performance. + * + * @var string[] + */ + private $cachedStepKeys = null; + /** * ActionGroupObject constructor. * @@ -409,16 +416,18 @@ private function replacePersistedArgument($replacement, $attributeValue, $fullVa */ public function extractStepKeys() { - $originalKeys = []; - foreach ($this->parsedActions as $action) { - //limit actions returned to list that are relevant - foreach (self::STEPKEY_REPLACEMENT_ENABLED_TYPES as $actionValue) { - if ($actionValue === $action->getType()) { + if ($this->cachedStepKeys === null) { + $originalKeys = []; + foreach ($this->parsedActions as $action) { + //limit actions returned to list that are relevant + if (in_array($action->getType(), self::STEPKEY_REPLACEMENT_ENABLED_TYPES)) { $originalKeys[] = $action->getStepKey(); } } + $this->cachedStepKeys = $originalKeys; } - return $originalKeys; + + return $this->cachedStepKeys; } /** diff --git a/src/Magento/FunctionalTestingFramework/Test/Objects/TestObject.php b/src/Magento/FunctionalTestingFramework/Test/Objects/TestObject.php index 30c3a9004..504d207b4 100644 --- a/src/Magento/FunctionalTestingFramework/Test/Objects/TestObject.php +++ b/src/Magento/FunctionalTestingFramework/Test/Objects/TestObject.php @@ -71,6 +71,13 @@ class TestObject */ private $parentTest; + /** + * Holds on to the result of getOrderedActions() to increase test generation performance. + * + * @var ActionObject[] + */ + private $cachedOrderedActions = null; + /** * TestObject constructor. * @@ -255,8 +262,12 @@ public function getCustomData() */ public function getOrderedActions() { - $mergeUtil = new ActionMergeUtil($this->getName(), "Test"); - return $mergeUtil->resolveActionSteps($this->parsedSteps); + if ($this->cachedOrderedActions === null) { + $mergeUtil = new ActionMergeUtil($this->getName(), "Test"); + $this->cachedOrderedActions = $mergeUtil->resolveActionSteps($this->parsedSteps); + } + + return $this->cachedOrderedActions; } /** From 322bd6cfd65a9bd81e00fbaa6751893b3a16f696 Mon Sep 17 00:00:00 2001 From: Andrii Meysar Date: Tue, 27 Aug 2019 07:39:31 +0300 Subject: [PATCH 32/82] - Fix CR notes --- .../OperationDefinitionObjectHandler.php | 5 ++++- .../Objects/OperationDefinitionObject.php | 22 ++++++++++++++++++- .../Persist/Curl/AdminExecutor.php | 15 +++++-------- .../Persist/Curl/FrontendExecutor.php | 15 +++++-------- .../Persist/Curl/WebapiExecutor.php | 8 ++----- .../DataGenerator/Persist/CurlHandler.php | 4 +++- .../DataGenerator/etc/dataOperation.xsd | 3 ++- .../Util/Protocol/CurlInterface.php | 3 ++- .../Util/Protocol/CurlTransport.php | 8 ++----- 9 files changed, 46 insertions(+), 37 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/OperationDefinitionObjectHandler.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/OperationDefinitionObjectHandler.php index 78deef0ef..cb53b2d68 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/OperationDefinitionObjectHandler.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/OperationDefinitionObjectHandler.php @@ -24,6 +24,7 @@ class OperationDefinitionObjectHandler implements ObjectHandlerInterface const ENTITY_OPERATION_STORE_CODE = 'storeCode'; const ENTITY_OPERATION_SUCCESS_REGEX = 'successRegex'; const ENTITY_OPERATION_RETURN_REGEX = 'returnRegex'; + const ENTITY_OPERATION_RETURN_INDEX = 'returnIndex'; const ENTITY_OPERATION_HEADER = 'header'; const ENTITY_OPERATION_CONTENT_TYPE = 'contentType'; const ENTITY_OPERATION_HEADER_PARAM = 'param'; @@ -144,6 +145,7 @@ private function initialize() $auth = $opDefArray[OperationDefinitionObjectHandler::ENTITY_OPERATION_AUTH] ?? null; $successRegex = $opDefArray[OperationDefinitionObjectHandler::ENTITY_OPERATION_SUCCESS_REGEX] ?? null; $returnRegex = $opDefArray[OperationDefinitionObjectHandler::ENTITY_OPERATION_RETURN_REGEX] ?? null; + $returnIndex = $opDefArray[OperationDefinitionObjectHandler::ENTITY_OPERATION_RETURN_INDEX] ?? null; $contentType = $opDefArray[OperationDefinitionObjectHandler::ENTITY_OPERATION_CONTENT_TYPE][0]['value'] ?? null; $headers = $this->initializeHeaders($opDefArray); @@ -216,7 +218,8 @@ private function initialize() $contentType, $removeBackend, $successRegex, - $returnRegex + $returnRegex, + $returnIndex ); } } diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Objects/OperationDefinitionObject.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Objects/OperationDefinitionObject.php index d6237a437..721a9dcc9 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Objects/OperationDefinitionObject.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Objects/OperationDefinitionObject.php @@ -104,6 +104,13 @@ class OperationDefinitionObject */ private $returnRegex; + /** + * Index of element to be returned from "returnRegex" matches. + * + * @var string + */ + private $returnIndex; + /** * Determines if operation should remove backend_name from URL. * @var boolean @@ -125,6 +132,7 @@ class OperationDefinitionObject * @param boolean $removeBackend * @param string $successRegex * @param string $returnRegex + * @param string $returnIndex * @SuppressWarnings(PHPMD.ExcessiveParameterList) */ public function __construct( @@ -140,7 +148,8 @@ public function __construct( $contentType, $removeBackend, $successRegex = null, - $returnRegex = null + $returnRegex = null, + $returnIndex = null ) { $this->name = $name; $this->operation = $operation; @@ -153,6 +162,7 @@ public function __construct( $this->operationMetadata = $metaData; $this->successRegex = $successRegex; $this->returnRegex = $returnRegex; + $this->returnIndex = $returnIndex; $this->removeBackend = $removeBackend; $this->apiUrl = null; @@ -284,6 +294,16 @@ public function getReturnRegex() return $this->returnRegex; } + /** + * Getter for return regex matches index. + * + * @return string|null + */ + public function getReturnIndex() + { + return $this->returnIndex; + } + /** * Function to append or add query parameters * diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/AdminExecutor.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/AdminExecutor.php index 70dafa75a..3da851299 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/AdminExecutor.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/AdminExecutor.php @@ -136,14 +136,10 @@ public function write($url, $data = [], $method = CurlInterface::POST, $headers } /** - * Read response from server. - * - * @param string $successRegex - * @param string $returnRegex - * @return string|array + * @inheritdoc * @throws TestFrameworkException */ - public function read($successRegex = null, $returnRegex = null) + public function read($successRegex = null, $returnRegex = null, $returnIndex = null) { $this->response = $this->transport->read(); $this->setFormKey(); @@ -158,10 +154,9 @@ public function read($successRegex = null, $returnRegex = null) if (!empty($returnRegex)) { preg_match($returnRegex, $this->response, $returnMatches); if (!empty($returnMatches)) { - if (count($returnMatches) > 1) { - unset($returnMatches); - } - return reset($returnMatches); + return isset($returnIndex, $returnMatches[$returnIndex]) + ? $returnMatches[$returnIndex] + : $returnMatches; } } return $this->response; diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/FrontendExecutor.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/FrontendExecutor.php index 20f8c1af8..06ea78d8c 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/FrontendExecutor.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/FrontendExecutor.php @@ -156,14 +156,10 @@ public function write($url, $data = [], $method = CurlInterface::POST, $headers } /** - * Read response from server. - * - * @param string $successRegex - * @param string $returnRegex - * @return string|array + * @inheritdoc * @throws TestFrameworkException */ - public function read($successRegex = null, $returnRegex = null) + public function read($successRegex = null, $returnRegex = null, $returnIndex = null) { $this->response = $this->transport->read(); $this->setCookies(); @@ -179,10 +175,9 @@ public function read($successRegex = null, $returnRegex = null) if (!empty($returnRegex)) { preg_match($returnRegex, $this->response, $returnMatches); if (!empty($returnMatches)) { - if (count($returnMatches) > 1) { - unset($returnMatches); - } - return reset($returnMatches); + return isset($returnIndex, $returnMatches[$returnIndex]) + ? $returnMatches[$returnIndex] + : $returnMatches; } } return $this->response; diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/WebapiExecutor.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/WebapiExecutor.php index 6aec3e58a..ce867c9fb 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/WebapiExecutor.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/WebapiExecutor.php @@ -135,14 +135,10 @@ public function write($url, $data = [], $method = CurlInterface::POST, $headers } /** - * Read response from server. - * - * @param string $successRegex - * @param string $returnRegex - * @return string + * @inheritdoc * @throws TestFrameworkException */ - public function read($successRegex = null, $returnRegex = null) + public function read($successRegex = null, $returnRegex = null, $returnIndex = null) { $this->response = $this->transport->read(); return $this->response; diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/CurlHandler.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/CurlHandler.php index ad81b9e9b..f6d8773bc 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/CurlHandler.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/CurlHandler.php @@ -107,6 +107,7 @@ public function executeRequest($dependentEntities) $executor = null; $successRegex = null; $returnRegex = null; + $returnIndex = null; if ((null !== $dependentEntities) && is_array($dependentEntities)) { $entities = array_merge([$this->entityObject], $dependentEntities); @@ -119,6 +120,7 @@ public function executeRequest($dependentEntities) $contentType = $this->operationDefinition->getContentType(); $successRegex = $this->operationDefinition->getSuccessRegex(); $returnRegex = $this->operationDefinition->getReturnRegex(); + $returnIndex = $this->operationDefinition->getReturnIndex(); $method = $this->operationDefinition->getApiMethod(); $operationDataResolver = new OperationDataArrayResolver($dependentEntities); @@ -161,7 +163,7 @@ public function executeRequest($dependentEntities) $headers ); - $response = $executor->read($successRegex, $returnRegex); + $response = $executor->read($successRegex, $returnRegex, $returnIndex); $executor->close(); return $response; diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/etc/dataOperation.xsd b/src/Magento/FunctionalTestingFramework/DataGenerator/etc/dataOperation.xsd index ccc2b8436..8b59ccae0 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/etc/dataOperation.xsd +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/etc/dataOperation.xsd @@ -29,6 +29,7 @@ + @@ -93,7 +94,7 @@ - + diff --git a/src/Magento/FunctionalTestingFramework/Util/Protocol/CurlInterface.php b/src/Magento/FunctionalTestingFramework/Util/Protocol/CurlInterface.php index 171fd01e6..5f48a4140 100644 --- a/src/Magento/FunctionalTestingFramework/Util/Protocol/CurlInterface.php +++ b/src/Magento/FunctionalTestingFramework/Util/Protocol/CurlInterface.php @@ -44,9 +44,10 @@ public function write($url, $body = [], $method = CurlInterface::POST, $headers * * @param string $successRegex * @param string $returnRegex + * @param string|null $returnIndex * @return string|array */ - public function read($successRegex = null, $returnRegex = null); + public function read($successRegex = null, $returnRegex = null, $returnIndex = null); /** * Close the connection to the server. diff --git a/src/Magento/FunctionalTestingFramework/Util/Protocol/CurlTransport.php b/src/Magento/FunctionalTestingFramework/Util/Protocol/CurlTransport.php index 5a9133a05..fd0932a82 100644 --- a/src/Magento/FunctionalTestingFramework/Util/Protocol/CurlTransport.php +++ b/src/Magento/FunctionalTestingFramework/Util/Protocol/CurlTransport.php @@ -159,14 +159,10 @@ public function write($url, $body = [], $method = CurlInterface::POST, $headers } /** - * Read response from server. - * - * @param string $successRegex - * @param string $returnRegex - * @return string + * @inheritdoc * @throws TestFrameworkException */ - public function read($successRegex = null, $returnRegex = null) + public function read($successRegex = null, $returnRegex = null, $returnIndex = null) { $response = curl_exec($this->getResource()); From 23edff1a9f73446eee3dfc8fe0bde9be36dd0e01 Mon Sep 17 00:00:00 2001 From: Andrii Meysar Date: Tue, 27 Aug 2019 08:32:37 +0300 Subject: [PATCH 33/82] - Fix static tests --- .../DataGenerator/Persist/Curl/AdminExecutor.php | 7 ++++++- .../DataGenerator/Persist/Curl/FrontendExecutor.php | 7 ++++++- .../DataGenerator/Persist/Curl/WebapiExecutor.php | 7 ++++++- .../Util/Protocol/CurlInterface.php | 4 ++-- .../Util/Protocol/CurlTransport.php | 7 ++++++- 5 files changed, 26 insertions(+), 6 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/AdminExecutor.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/AdminExecutor.php index 3da851299..b23f3be65 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/AdminExecutor.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/AdminExecutor.php @@ -136,7 +136,12 @@ public function write($url, $data = [], $method = CurlInterface::POST, $headers } /** - * @inheritdoc + * Read response from server. + * + * @param string $successRegex + * @param string $returnRegex + * @param string|null $returnIndex + * @return string|array * @throws TestFrameworkException */ public function read($successRegex = null, $returnRegex = null, $returnIndex = null) diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/FrontendExecutor.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/FrontendExecutor.php index 06ea78d8c..08d6508a8 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/FrontendExecutor.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/FrontendExecutor.php @@ -156,7 +156,12 @@ public function write($url, $data = [], $method = CurlInterface::POST, $headers } /** - * @inheritdoc + * Read response from server. + * + * @param string $successRegex + * @param string $returnRegex + * @param string|null $returnIndex + * @return string|array * @throws TestFrameworkException */ public function read($successRegex = null, $returnRegex = null, $returnIndex = null) diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/WebapiExecutor.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/WebapiExecutor.php index ce867c9fb..281eaa24d 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/WebapiExecutor.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/WebapiExecutor.php @@ -135,7 +135,12 @@ public function write($url, $data = [], $method = CurlInterface::POST, $headers } /** - * @inheritdoc + * Read response from server. + * + * @param string $successRegex + * @param string $returnRegex + * @param string|null $returnIndex + * @return string * @throws TestFrameworkException */ public function read($successRegex = null, $returnRegex = null, $returnIndex = null) diff --git a/src/Magento/FunctionalTestingFramework/Util/Protocol/CurlInterface.php b/src/Magento/FunctionalTestingFramework/Util/Protocol/CurlInterface.php index 5f48a4140..a2d6bc344 100644 --- a/src/Magento/FunctionalTestingFramework/Util/Protocol/CurlInterface.php +++ b/src/Magento/FunctionalTestingFramework/Util/Protocol/CurlInterface.php @@ -42,8 +42,8 @@ public function write($url, $body = [], $method = CurlInterface::POST, $headers /** * Read response from server. * - * @param string $successRegex - * @param string $returnRegex + * @param string $successRegex + * @param string $returnRegex * @param string|null $returnIndex * @return string|array */ diff --git a/src/Magento/FunctionalTestingFramework/Util/Protocol/CurlTransport.php b/src/Magento/FunctionalTestingFramework/Util/Protocol/CurlTransport.php index fd0932a82..16c716e22 100644 --- a/src/Magento/FunctionalTestingFramework/Util/Protocol/CurlTransport.php +++ b/src/Magento/FunctionalTestingFramework/Util/Protocol/CurlTransport.php @@ -159,7 +159,12 @@ public function write($url, $body = [], $method = CurlInterface::POST, $headers } /** - * @inheritdoc + * Read response from server. + * + * @param string $successRegex + * @param string $returnRegex + * @param string|null $returnIndex + * @return string * @throws TestFrameworkException */ public function read($successRegex = null, $returnRegex = null, $returnIndex = null) From 75b6b88527869d8b3034d6ea3f26bf811da8d983 Mon Sep 17 00:00:00 2001 From: Kevin Kozan Date: Tue, 27 Aug 2019 09:48:35 -0500 Subject: [PATCH 34/82] MQE-1514: Failure For seeCurrentUrlEquals Action Does Not Show Actual & Expected Results in Allure - Added Allure firing of attachment to steps --- .../Module/MagentoWebDriver.php | 33 +++++++++++++++---- 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php index d268e23b4..0c8f632b3 100644 --- a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php +++ b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php @@ -9,6 +9,7 @@ use Codeception\Module\WebDriver; use Codeception\Test\Descriptor; use Codeception\TestInterface; +use Yandex\Allure\Adapter\Allure; use Facebook\WebDriver\Interactions\WebDriverActions; use Codeception\Exception\ModuleConfigException; use Codeception\Exception\ModuleException; @@ -18,6 +19,8 @@ use Magento\FunctionalTestingFramework\Util\Protocol\CurlTransport; use Magento\FunctionalTestingFramework\Util\Protocol\CurlInterface; use Magento\FunctionalTestingFramework\Util\ConfigSanitizerUtil; +use Yandex\Allure\Adapter\Event\AddAttachmentEvent; +use Yandex\Allure\Adapter\Event\AddParameterEvent; use Yandex\Allure\Adapter\Support\AttachmentSupport; use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; @@ -198,7 +201,10 @@ public function _getCurrentUri() */ public function dontSeeCurrentUrlEquals($url) { - $this->assertNotEquals($url, $this->webDriver->getCurrentURL()); + $actualUrl = $this->webDriver->getCurrentURL(); + $comparison = "Expected: $url\nActual: $actualUrl"; + Allure::lifecycle()->fire(new AddAttachmentEvent($comparison, 'Comparison')); + $this->assertNotEquals($url, $actualUrl); } /** @@ -209,7 +215,10 @@ public function dontSeeCurrentUrlEquals($url) */ public function dontSeeCurrentUrlMatches($regex) { - $this->assertNotRegExp($regex, $this->webDriver->getCurrentURL()); + $actualUrl = $this->webDriver->getCurrentURL(); + $comparison = "Expected: $regex\nActual: $actualUrl"; + Allure::lifecycle()->fire(new AddAttachmentEvent($comparison, 'Comparison')); + $this->assertNotRegExp($regex, $actualUrl); } /** @@ -220,7 +229,10 @@ public function dontSeeCurrentUrlMatches($regex) */ public function dontSeeInCurrentUrl($needle) { - $this->assertNotContains($needle, $this->webDriver->getCurrentURL()); + $actualUrl = $this->webDriver->getCurrentURL(); + $comparison = "Expected: $needle\nActual: $actualUrl"; + Allure::lifecycle()->fire(new AddAttachmentEvent($comparison, 'Comparison')); + $this->assertNotContains($needle, $actualUrl); } /** @@ -254,7 +266,10 @@ public function grabFromCurrentUrl($regex = null) */ public function seeCurrentUrlEquals($url) { - $this->assertEquals($url, $this->webDriver->getCurrentURL()); + $actualUrl = $this->webDriver->getCurrentURL(); + $comparison = "Expected: $url\nActual: $actualUrl"; + Allure::lifecycle()->fire(new AddAttachmentEvent($comparison, 'Comparison')); + $this->assertEquals($url, $actualUrl); } /** @@ -265,7 +280,10 @@ public function seeCurrentUrlEquals($url) */ public function seeCurrentUrlMatches($regex) { - $this->assertRegExp($regex, $this->webDriver->getCurrentURL()); + $actualUrl = $this->webDriver->getCurrentURL(); + $comparison = "Expected: $regex\nActual: $actualUrl"; + Allure::lifecycle()->fire(new AddAttachmentEvent($comparison, 'Comparison')); + $this->assertRegExp($regex, $url); } /** @@ -276,7 +294,10 @@ public function seeCurrentUrlMatches($regex) */ public function seeInCurrentUrl($needle) { - $this->assertContains($needle, $this->webDriver->getCurrentURL()); + $actualUrl = $this->webDriver->getCurrentURL(); + $comparison = "Expected: $needle\nActual: $actualUrl"; + Allure::lifecycle()->fire(new AddAttachmentEvent($comparison, 'Comparison')); + $this->assertContains($needle, $actualUrl); } /** From 4d46f9b14642962ac29b8f818ebd7ab72f8360f1 Mon Sep 17 00:00:00 2001 From: Ji Lu Date: Tue, 27 Aug 2019 15:34:49 -0500 Subject: [PATCH 35/82] MQE-588: CreatedData should throw Error/Warning when undefined data is returned --- .../Handlers/PersistedObjectHandlerTest.php | 104 ++++++++++++++++++ .../Handlers/PersistedObjectHandler.php | 9 +- 2 files changed, 111 insertions(+), 2 deletions(-) diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/PersistedObjectHandlerTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/PersistedObjectHandlerTest.php index 30dc3ec40..fd04ee5fe 100644 --- a/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/PersistedObjectHandlerTest.php +++ b/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/PersistedObjectHandlerTest.php @@ -11,6 +11,8 @@ use Magento\FunctionalTestingFramework\DataGenerator\Handlers\PersistedObjectHandler; use Magento\FunctionalTestingFramework\DataGenerator\Parsers\DataProfileSchemaParser; use Magento\FunctionalTestingFramework\DataGenerator\Persist\CurlHandler; +use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; +use Magento\FunctionalTestingFramework\Exceptions\TestReferenceException; use Magento\FunctionalTestingFramework\ObjectManager; use Magento\FunctionalTestingFramework\ObjectManagerFactory; use Magento\FunctionalTestingFramework\Util\MagentoTestCase; @@ -330,6 +332,108 @@ public function testRetrieveEntityAcrossScopes() $this->assertEquals($dataValueThree, $retrievedFromSuite); } + /** + * @param string $name + * @param string $key + * @param string $value + * @param string $type + * @param string $scope + * @param string $stepKey + * @dataProvider entityDataProvider + */ + public function testRetrieveEntityValidField($name, $key, $value, $type, $scope, $stepKey) + { + $parserOutputOne = [ + 'entity' => [ + $name => [ + 'type' => $type, + 'data' => [ + 0 => [ + 'key' => $key, + 'value' => $value + ] + ] + ] + ] + ]; + $jsonReponseOne = " + { + \"" . strtolower($key) . "\" : \"{$value}\" + } + "; + + // Mock Classes and Create Entities + $handler = PersistedObjectHandler::getInstance(); + + $this->mockDataHandlerWithOutput($parserOutputOne); + $this->mockCurlHandler($jsonReponseOne); + $handler->createEntity($stepKey, $scope, $name); + + // Call method + $retrieved = $handler->retrieveEntityField($stepKey, $key, $scope); + + $this->assertEquals($value, $retrieved); + } + + /** + * @param string $name + * @param string $key + * @param string $value + * @param string $type + * @param string $scope + * @param string $stepKey + * @dataProvider entityDataProvider + * @throws TestReferenceException + * @throws TestFrameworkException + */ + public function testRetrieveEntityInValidField($name, $key, $value, $type, $scope, $stepKey) + { + $invalidDataKey = "invalidDataKey"; + + $parserOutputOne = [ + 'entity' => [ + $name => [ + 'type' => $type, + 'data' => [ + 0 => [ + 'key' => $key, + 'value' => $value + ] + ] + ] + ] + ]; + $jsonReponseOne = " + { + \"" . strtolower($key) . "\" : \"{$value}\" + } + "; + + // Mock Classes and Create Entities + $handler = PersistedObjectHandler::getInstance(); + + $this->mockDataHandlerWithOutput($parserOutputOne); + $this->mockCurlHandler($jsonReponseOne); + $handler->createEntity($stepKey, $scope, $name); + + $this->expectException(\Magento\FunctionalTestingFramework\Exceptions\TestReferenceException::class); + + // Call method + $handler->retrieveEntityField($stepKey, $invalidDataKey, $scope); + } + + /** + * Data provider for testRetrieveEntityField + */ + public static function entityDataProvider() + { + return [ + ['Entity1', 'testKey1', 'testValue1', 'testType', PersistedObjectHandler::HOOK_SCOPE, 'StepKey1'], + ['Entity2', 'testKey2', 'testValue2', 'testType', PersistedObjectHandler::SUITE_SCOPE, 'StepKey2'], + ['Entity3', 'testKey3', 'testValue3', 'testType', PersistedObjectHandler::TEST_SCOPE, 'StepKey3'] + ]; + } + /** * Mocks DataObjectHandler to use given output to create * @param $parserOutput diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/PersistedObjectHandler.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/PersistedObjectHandler.php index cc35bbc52..55dd599f1 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/PersistedObjectHandler.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/PersistedObjectHandler.php @@ -7,6 +7,7 @@ namespace Magento\FunctionalTestingFramework\DataGenerator\Handlers; use Magento\FunctionalTestingFramework\DataGenerator\Persist\DataPersistenceHandler; +use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; use Magento\FunctionalTestingFramework\Exceptions\TestReferenceException; class PersistedObjectHandler @@ -178,11 +179,15 @@ public function getEntity($key, $scope, $entity, $dependentObjectKeys = [], $sto * @param string $scope * @return string * @throws TestReferenceException - * @throws \Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException + * @throws TestFrameworkException */ public function retrieveEntityField($stepKey, $field, $scope) { - return $this->retrieveEntity($stepKey, $scope)->getCreatedDataByName($field); + $fieldValue = $this->retrieveEntity($stepKey, $scope)->getCreatedDataByName($field); + if ($fieldValue === null) { + throw new TestReferenceException("Undefined field {$field} in entity object with a stepKey of {$stepKey}"); + } + return $fieldValue; } /** From 0c8560b5d2cba5feb3fe9ee53a39acb66bd8bce0 Mon Sep 17 00:00:00 2001 From: Andrii Meysar Date: Wed, 28 Aug 2019 06:50:22 +0300 Subject: [PATCH 36/82] - Fix CR notes --- .../Handlers/OperationDefinitionObjectHandler.php | 2 +- .../DataGenerator/Persist/Curl/AdminExecutor.php | 4 +--- .../DataGenerator/Persist/Curl/FrontendExecutor.php | 4 +--- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/OperationDefinitionObjectHandler.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/OperationDefinitionObjectHandler.php index cb53b2d68..b768d9bd8 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/OperationDefinitionObjectHandler.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/OperationDefinitionObjectHandler.php @@ -145,7 +145,7 @@ private function initialize() $auth = $opDefArray[OperationDefinitionObjectHandler::ENTITY_OPERATION_AUTH] ?? null; $successRegex = $opDefArray[OperationDefinitionObjectHandler::ENTITY_OPERATION_SUCCESS_REGEX] ?? null; $returnRegex = $opDefArray[OperationDefinitionObjectHandler::ENTITY_OPERATION_RETURN_REGEX] ?? null; - $returnIndex = $opDefArray[OperationDefinitionObjectHandler::ENTITY_OPERATION_RETURN_INDEX] ?? null; + $returnIndex = $opDefArray[OperationDefinitionObjectHandler::ENTITY_OPERATION_RETURN_INDEX] ?? 0; $contentType = $opDefArray[OperationDefinitionObjectHandler::ENTITY_OPERATION_CONTENT_TYPE][0]['value'] ?? null; $headers = $this->initializeHeaders($opDefArray); diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/AdminExecutor.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/AdminExecutor.php index b23f3be65..4e11f5cec 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/AdminExecutor.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/AdminExecutor.php @@ -159,9 +159,7 @@ public function read($successRegex = null, $returnRegex = null, $returnIndex = n if (!empty($returnRegex)) { preg_match($returnRegex, $this->response, $returnMatches); if (!empty($returnMatches)) { - return isset($returnIndex, $returnMatches[$returnIndex]) - ? $returnMatches[$returnIndex] - : $returnMatches; + return $returnMatches[$returnIndex] ?? $returnMatches[0]; } } return $this->response; diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/FrontendExecutor.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/FrontendExecutor.php index 08d6508a8..799cd6d5c 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/FrontendExecutor.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/FrontendExecutor.php @@ -180,9 +180,7 @@ public function read($successRegex = null, $returnRegex = null, $returnIndex = n if (!empty($returnRegex)) { preg_match($returnRegex, $this->response, $returnMatches); if (!empty($returnMatches)) { - return isset($returnIndex, $returnMatches[$returnIndex]) - ? $returnMatches[$returnIndex] - : $returnMatches; + return $returnMatches[$returnIndex] ?? $returnMatches[0]; } } return $this->response; From 315616f97db14186e8f4f805e615cd5ac6342347 Mon Sep 17 00:00:00 2001 From: Soumya Unnikrishnan Date: Wed, 28 Aug 2019 08:55:12 -0500 Subject: [PATCH 37/82] MQE-1703: Implicit Suite Generation for Tests --- .../Console/BaseGenerateCommand.php | 32 ++++++++++++++ .../Console/GenerateTestsCommand.php | 11 +++-- .../Console/RunTestCommand.php | 44 +++++++++++++++---- 3 files changed, 75 insertions(+), 12 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php index 75c521379..afe2f6916 100644 --- a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php @@ -14,6 +14,7 @@ use Magento\FunctionalTestingFramework\Util\Filesystem\DirSetupUtil; use Magento\FunctionalTestingFramework\Util\TestGenerator; use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig; +use Magento\FunctionalTestingFramework\Suite\Handlers\SuiteObjectHandler; class BaseGenerateCommand extends Command { @@ -67,4 +68,35 @@ protected function removeGeneratedDirectory(OutputInterface $output, bool $verbo } } } + + /** + * Returns a 2D array of tests with their suites references that can be encoded into a json test configuration + * @param array $tests + * @return false|string + * @throws \Magento\FunctionalTestingFramework\Exceptions\XmlException + */ + + protected function getTestAndSuiteConfiguration(array $tests) + { + $testConfiguration['tests'] = []; + $testConfiguration['suites'] = []; + $allSuiteTests = SuiteObjectHandler::getInstance()->getAllTestReferences(); + $suiteGroup = []; + + foreach($tests as $test) { + if (array_key_exists($test, $allSuiteTests)) { + $suiteGroup[$test] = $allSuiteTests[$test]; + } + else $testConfiguration['tests'][] = $test; + } + + foreach ($suiteGroup as $test => $suites) { + + foreach ($suites as $suite) { + $testConfiguration['suites'][$suite][] = $test; + } + + } + return $testConfiguration; + } } diff --git a/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php b/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php index ca9d6e3cc..0a92e5dab 100644 --- a/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php @@ -65,7 +65,7 @@ protected function execute(InputInterface $input, OutputInterface $output) { $tests = $input->getArgument('name'); $config = $input->getOption('config'); - $json = $input->getOption('tests'); + $json = $input->getOption('tests'); // for backward compatibility $force = $input->getOption('force'); $time = $input->getOption('time') * 60 * 1000; // convert from minutes to milliseconds $debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility @@ -73,6 +73,11 @@ protected function execute(InputInterface $input, OutputInterface $output) $verbose = $output->isVerbose(); $allowSkipped = $input->getOption('allowSkipped'); + // if test configuration is not specified, set implicitly + if ($json === null && !empty($tests)) { + $json = json_encode($this->getTestAndSuiteConfiguration($tests)); + } + if ($json !== null && !json_decode($json)) { // stop execution if we have failed to properly parse any json passed in by the user throw new TestFrameworkException("JSON could not be parsed: " . json_last_error_msg()); @@ -100,9 +105,7 @@ protected function execute(InputInterface $input, OutputInterface $output) $testManifest->createTestGroups($time); } - if (empty($tests)) { - SuiteGenerator::getInstance()->generateAllSuites($testManifest); - } + SuiteGenerator::getInstance()->generateAllSuites($testManifest); $testManifest->generate(); diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php index b17131459..b0c47c7eb 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php @@ -64,12 +64,10 @@ protected function execute(InputInterface $input, OutputInterface $output): int } if (!$skipGeneration) { + $testConfiguration = $this->getTestAndSuiteConfiguration($tests); $command = $this->getApplication()->find('generate:tests'); $args = [ - '--tests' => json_encode([ - 'tests' => $tests, - 'suites' => null - ]), + '--tests' => json_encode($testConfiguration), '--force' => $force, '--remove' => $remove, '--debug' => $debug, @@ -77,14 +75,23 @@ protected function execute(InputInterface $input, OutputInterface $output): int ]; $command->run(new ArrayInput($args), $output); } - + // tests with resolved suite references + $resolvedTests = $this->getResolvedTests($testConfiguration); $returnCode = 0; $codeceptionCommand = realpath(PROJECT_ROOT . '/vendor/bin/codecept') . ' run functional '; $testsDirectory = TESTS_MODULE_PATH . DIRECTORY_SEPARATOR . TestGenerator::GENERATED_DIR . DIRECTORY_SEPARATOR; //execute only tests specified as arguments in run command - foreach ($tests as $test) { - $testGroup = TestGenerator::DEFAULT_DIR . DIRECTORY_SEPARATOR; - $testName = $test . 'Cest.php'; + foreach ($resolvedTests as $test){ + // for tests in suite, set directory as suite name + if (strpos($test, ':' )) { + list($suite, $testName) = explode(":", $test); + } + // for standalone tests set directory as "default" + else { + list($suite, $testName) = [TestGenerator::DEFAULT_DIR, $test]; + } + $testGroup = $suite . DIRECTORY_SEPARATOR; + $testName .= 'Cest.php'; if (!realpath($testsDirectory . $testGroup . $testName)) { throw new TestFrameworkException( $testName . " is not available under " . $testsDirectory . $testGroup @@ -104,4 +111,25 @@ function ($type, $buffer) use ($output) { } return $returnCode; } + + /** Get an array of tests with resolved suite references from $testConfiguration + * eg: if test is referenced in a suite, it'll be stored in format "SuiteName:Testname"; + * if not it'll be stored as is. + * @param $testConfiguration + * @return array + */ + private function getResolvedTests($testConfiguration) + { + $testsArray = $testConfiguration['tests'] ?? []; + $suitesArray = $testConfiguration['suites'] ?? []; + $testArrayBuilder = []; + foreach ($suitesArray as $suite => $tests) { + $testArrayBuilder = array_merge($testArrayBuilder, + array_map(function($test) use ($suite) + { return $suite . ':' . $test ; }, $tests)); + } + return array_merge($testArrayBuilder, $testsArray); + + + } } From 5ba993ad1e81c1af743e346d5bc5f2833b8bc26f Mon Sep 17 00:00:00 2001 From: Soumya Unnikrishnan Date: Wed, 28 Aug 2019 10:10:42 -0500 Subject: [PATCH 38/82] MQE-1703: Implicit Suite Generation for Tests fixing unit tests --- .../Console/BaseGenerateCommand.php | 2 -- .../Console/RunTestCommand.php | 21 ++++++++++--------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php index afe2f6916..a14b83c0f 100644 --- a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php @@ -91,11 +91,9 @@ protected function getTestAndSuiteConfiguration(array $tests) } foreach ($suiteGroup as $test => $suites) { - foreach ($suites as $suite) { $testConfiguration['suites'][$suite][] = $test; } - } return $testConfiguration; } diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php index b0c47c7eb..ecbf9ec7b 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php @@ -81,9 +81,9 @@ protected function execute(InputInterface $input, OutputInterface $output): int $codeceptionCommand = realpath(PROJECT_ROOT . '/vendor/bin/codecept') . ' run functional '; $testsDirectory = TESTS_MODULE_PATH . DIRECTORY_SEPARATOR . TestGenerator::GENERATED_DIR . DIRECTORY_SEPARATOR; //execute only tests specified as arguments in run command - foreach ($resolvedTests as $test){ + foreach ($resolvedTests as $test) { // for tests in suite, set directory as suite name - if (strpos($test, ':' )) { + if (strpos($test, ':')) { list($suite, $testName) = explode(":", $test); } // for standalone tests set directory as "default" @@ -114,22 +114,23 @@ function ($type, $buffer) use ($output) { /** Get an array of tests with resolved suite references from $testConfiguration * eg: if test is referenced in a suite, it'll be stored in format "SuiteName:Testname"; - * if not it'll be stored as is. - * @param $testConfiguration + * @param array $testConfiguration * @return array */ - private function getResolvedTests($testConfiguration) + private function getResolvedTests(array $testConfiguration) { $testsArray = $testConfiguration['tests'] ?? []; $suitesArray = $testConfiguration['suites'] ?? []; $testArrayBuilder = []; + foreach ($suitesArray as $suite => $tests) { - $testArrayBuilder = array_merge($testArrayBuilder, - array_map(function($test) use ($suite) - { return $suite . ':' . $test ; }, $tests)); + $testArrayBuilder = array_merge( + $testArrayBuilder, + array_map(function ($test) use ($suite) { + return $suite . ':' . $test; + }, $tests) + ); } return array_merge($testArrayBuilder, $testsArray); - - } } From 32827704c8f1e93710232c6a7a588d0dcb3afbf4 Mon Sep 17 00:00:00 2001 From: Soumya Unnikrishnan Date: Wed, 28 Aug 2019 10:25:49 -0500 Subject: [PATCH 39/82] MQE-1703: Implicit Suite Generation for Tests fixing unit tests --- .../FunctionalTestingFramework/Console/RunTestCommand.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php index ecbf9ec7b..d58d95fdc 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php @@ -82,11 +82,10 @@ protected function execute(InputInterface $input, OutputInterface $output): int $testsDirectory = TESTS_MODULE_PATH . DIRECTORY_SEPARATOR . TestGenerator::GENERATED_DIR . DIRECTORY_SEPARATOR; //execute only tests specified as arguments in run command foreach ($resolvedTests as $test) { - // for tests in suite, set directory as suite name + //set directory as suite name for tests in suite, if not set to "default" if (strpos($test, ':')) { list($suite, $testName) = explode(":", $test); } - // for standalone tests set directory as "default" else { list($suite, $testName) = [TestGenerator::DEFAULT_DIR, $test]; } From 79ff34748397aeb46377f0a0ae75c11c307f241a Mon Sep 17 00:00:00 2001 From: Soumya Unnikrishnan Date: Wed, 28 Aug 2019 10:59:33 -0500 Subject: [PATCH 40/82] MQE-1703: Implicit Suite Generation for Tests fixing unit tests --- .../FunctionalTestingFramework/Console/BaseGenerateCommand.php | 2 +- .../FunctionalTestingFramework/Console/RunTestCommand.php | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php index a14b83c0f..46f3db95e 100644 --- a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php @@ -70,7 +70,7 @@ protected function removeGeneratedDirectory(OutputInterface $output, bool $verbo } /** - * Returns a 2D array of tests with their suites references that can be encoded into a json test configuration + * Returns an array of test configuration to be used as an argument for generation of tests * @param array $tests * @return false|string * @throws \Magento\FunctionalTestingFramework\Exceptions\XmlException diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php index d58d95fdc..09f36ae41 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php @@ -85,8 +85,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int //set directory as suite name for tests in suite, if not set to "default" if (strpos($test, ':')) { list($suite, $testName) = explode(":", $test); - } - else { + } else { list($suite, $testName) = [TestGenerator::DEFAULT_DIR, $test]; } $testGroup = $suite . DIRECTORY_SEPARATOR; From beec38835e3e38b60e881bc7a504b980984b2d5d Mon Sep 17 00:00:00 2001 From: Kevin Kozan Date: Wed, 28 Aug 2019 13:12:09 -0500 Subject: [PATCH 41/82] MQE-1514: Failure For seeCurrentUrlEquals Action Does Not Show Actual & Expected Results in Allure - bugfix for variable name --- .../FunctionalTestingFramework/Module/MagentoWebDriver.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php index 0c8f632b3..830fd40c4 100644 --- a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php +++ b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php @@ -283,7 +283,7 @@ public function seeCurrentUrlMatches($regex) $actualUrl = $this->webDriver->getCurrentURL(); $comparison = "Expected: $regex\nActual: $actualUrl"; Allure::lifecycle()->fire(new AddAttachmentEvent($comparison, 'Comparison')); - $this->assertRegExp($regex, $url); + $this->assertRegExp($regex, $actualUrl); } /** From 1a2dc67dacb23590f7debe3218b78b48b5f52236 Mon Sep 17 00:00:00 2001 From: Soumya Unnikrishnan Date: Wed, 28 Aug 2019 16:39:23 -0500 Subject: [PATCH 42/82] MQE-1703: Implicit Suite Generation for Tests fixing unit tests --- .../Console/BaseGenerateCommand.php | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php index 46f3db95e..d8c7a53ce 100644 --- a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php @@ -80,20 +80,26 @@ protected function getTestAndSuiteConfiguration(array $tests) { $testConfiguration['tests'] = []; $testConfiguration['suites'] = []; - $allSuiteTests = SuiteObjectHandler::getInstance()->getAllTestReferences(); - $suiteGroup = []; + $testsReferencedInSuites = SuiteObjectHandler::getInstance()->getAllTestReferences(); + $resolvedTests = []; foreach($tests as $test) { - if (array_key_exists($test, $allSuiteTests)) { - $suiteGroup[$test] = $allSuiteTests[$test]; + if (array_key_exists($test, $testsReferencedInSuites)) { + $suites = $testsReferencedInSuites[$test]; + $resolvedTests = array_merge( + $resolvedTests, + array_map(function ($value) use ($test) { + return $value . ':' . $test; + }, $suites) + ); } + // configuration for tests else $testConfiguration['tests'][] = $test; } - - foreach ($suiteGroup as $test => $suites) { - foreach ($suites as $suite) { - $testConfiguration['suites'][$suite][] = $test; - } + // configuration for suites + foreach ($resolvedTests as $test) { + list($suite, $test) = explode(":", $test); + $testConfiguration['suites'][$suite][] = $test; } return $testConfiguration; } From 4bbfb25f1b47c6f755edd11e23f607cc8e111130 Mon Sep 17 00:00:00 2001 From: Soumya Unnikrishnan Date: Wed, 28 Aug 2019 20:35:32 -0500 Subject: [PATCH 43/82] MQE-1703: Implicit Suite Generation for Tests fixing unit tests --- .../Console/BaseGenerateCommand.php | 3 ++- .../Console/GenerateTestsCommand.php | 5 ++-- .../Console/RunTestCommand.php | 25 +++++++++++-------- 3 files changed, 18 insertions(+), 15 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php index d8c7a53ce..1cfa34e5a 100644 --- a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php @@ -101,6 +101,7 @@ protected function getTestAndSuiteConfiguration(array $tests) list($suite, $test) = explode(":", $test); $testConfiguration['suites'][$suite][] = $test; } - return $testConfiguration; + $testConfigurationJson = json_encode($testConfiguration); + return $testConfigurationJson; } } diff --git a/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php b/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php index 0a92e5dab..c410973c7 100644 --- a/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php @@ -73,9 +73,8 @@ protected function execute(InputInterface $input, OutputInterface $output) $verbose = $output->isVerbose(); $allowSkipped = $input->getOption('allowSkipped'); - // if test configuration is not specified, set implicitly - if ($json === null && !empty($tests)) { - $json = json_encode($this->getTestAndSuiteConfiguration($tests)); + if (!empty($tests)) { + $json = $this->getTestAndSuiteConfiguration($tests); } if ($json !== null && !json_decode($json)) { diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php index 09f36ae41..6dbd8fc5b 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php @@ -63,11 +63,12 @@ protected function execute(InputInterface $input, OutputInterface $output): int ); } + $testConfiguration = $this->getTestAndSuiteConfiguration($tests); + if (!$skipGeneration) { - $testConfiguration = $this->getTestAndSuiteConfiguration($tests); $command = $this->getApplication()->find('generate:tests'); $args = [ - '--tests' => json_encode($testConfiguration), + '--tests' => $testConfiguration, '--force' => $force, '--remove' => $remove, '--debug' => $debug, @@ -76,20 +77,21 @@ protected function execute(InputInterface $input, OutputInterface $output): int $command->run(new ArrayInput($args), $output); } // tests with resolved suite references - $resolvedTests = $this->getResolvedTests($testConfiguration); - $returnCode = 0; + $resolvedTests = $this->resolveSuiteReferences($testConfiguration); + $codeceptionCommand = realpath(PROJECT_ROOT . '/vendor/bin/codecept') . ' run functional '; $testsDirectory = TESTS_MODULE_PATH . DIRECTORY_SEPARATOR . TestGenerator::GENERATED_DIR . DIRECTORY_SEPARATOR; + $returnCode = 0; //execute only tests specified as arguments in run command foreach ($resolvedTests as $test) { //set directory as suite name for tests in suite, if not set to "default" if (strpos($test, ':')) { - list($suite, $testName) = explode(":", $test); + list($testGroup, $testName) = explode(":", $test); } else { - list($suite, $testName) = [TestGenerator::DEFAULT_DIR, $test]; + list($testGroup, $testName) = [TestGenerator::DEFAULT_DIR, $test]; } - $testGroup = $suite . DIRECTORY_SEPARATOR; - $testName .= 'Cest.php'; + $testGroup = $testGroup . DIRECTORY_SEPARATOR; + $testName = $testName . 'Cest.php'; if (!realpath($testsDirectory . $testGroup . $testName)) { throw new TestFrameworkException( $testName . " is not available under " . $testsDirectory . $testGroup @@ -111,12 +113,13 @@ function ($type, $buffer) use ($output) { } /** Get an array of tests with resolved suite references from $testConfiguration - * eg: if test is referenced in a suite, it'll be stored in format "SuiteName:Testname"; - * @param array $testConfiguration + * eg: if test is referenced in a suite, it'll be stored in format suite:test + * @param $testConfigurationJson * @return array */ - private function getResolvedTests(array $testConfiguration) + private function resolveSuiteReferences($testConfigurationJson) { + $testConfiguration = json_decode($testConfigurationJson, true); $testsArray = $testConfiguration['tests'] ?? []; $suitesArray = $testConfiguration['suites'] ?? []; $testArrayBuilder = []; From da509f2bf4e295d400a9ec8ea1db6d6bfad5215e Mon Sep 17 00:00:00 2001 From: Soumya Unnikrishnan Date: Wed, 28 Aug 2019 20:56:12 -0500 Subject: [PATCH 44/82] MQE-1703: Implicit Suite Generation for Tests fixing unit tests --- .../FunctionalTestingFramework/Console/RunTestCommand.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php index 6dbd8fc5b..0a355e798 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php @@ -114,7 +114,7 @@ function ($type, $buffer) use ($output) { /** Get an array of tests with resolved suite references from $testConfiguration * eg: if test is referenced in a suite, it'll be stored in format suite:test - * @param $testConfigurationJson + * @param string $testConfigurationJson * @return array */ private function resolveSuiteReferences($testConfigurationJson) From 037d8aa359c67a32c15a707b42f9c62c1dffbd8a Mon Sep 17 00:00:00 2001 From: Tom Reece Date: Thu, 29 Aug 2019 10:17:32 -0500 Subject: [PATCH 45/82] MQE-1626: bin/mftf run:manifest - Initial implementation --- .../Console/CommandList.php | 1 + .../Console/RunManifestCommand.php | 144 ++++++++++++++++++ 2 files changed, 145 insertions(+) create mode 100644 src/Magento/FunctionalTestingFramework/Console/RunManifestCommand.php diff --git a/src/Magento/FunctionalTestingFramework/Console/CommandList.php b/src/Magento/FunctionalTestingFramework/Console/CommandList.php index 5bcf1ed31..34d221840 100644 --- a/src/Magento/FunctionalTestingFramework/Console/CommandList.php +++ b/src/Magento/FunctionalTestingFramework/Console/CommandList.php @@ -37,6 +37,7 @@ public function __construct(array $commands = []) 'run:test' => new RunTestCommand(), 'run:group' => new RunTestGroupCommand(), 'run:failed' => new RunTestFailedCommand(), + 'run:manifest' => new RunManifestCommand(), 'setup:env' => new SetupEnvCommand(), 'upgrade:tests' => new UpgradeTestsCommand(), 'generate:docs' => new GenerateDocsCommand(), diff --git a/src/Magento/FunctionalTestingFramework/Console/RunManifestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunManifestCommand.php new file mode 100644 index 000000000..9ef59e296 --- /dev/null +++ b/src/Magento/FunctionalTestingFramework/Console/RunManifestCommand.php @@ -0,0 +1,144 @@ +setName("run:manifest") + ->setDescription("runs a manifest file") + ->addArgument("path", InputArgument::REQUIRED, "path to a manifest file"); + } + + /** + * Executes the run:manifest command. + * + * @param InputInterface $input + * @param OutputInterface $output + * @return integer + */ + protected function execute(InputInterface $input, OutputInterface $output): int + { + $manifestFile = file($input->getArgument("path"), FILE_IGNORE_NEW_LINES); + + // Delete the Codeception failed file just in case it exists from any previous test runs + $this->deleteFailedFile(); + + foreach ($manifestFile as $manifestLine) { + if (empty($manifestLine)) { + continue; + } + + $this->runManifestLine($manifestLine, $output); + $this->aggregateFailed(); + } + + if (!empty($this->failedTests)) { + $this->deleteFailedFile(); + $this->writeFailedFile(); + } + + return $this->returnCode; + } + + /** + * Runs a test (or group) line from the manifest file + * + * @param string $manifestLine + * @param OutputInterface $output + * @return void + * + * @SuppressWarnings(PHPMD.UnusedLocalVariable) Need this because of the unused $type variable in the closure + */ + private function runManifestLine(string $manifestLine, OutputInterface $output) + { + $codeceptionCommand = realpath(PROJECT_ROOT . "/vendor/bin/codecept") + . " run functional --verbose --steps " + . $manifestLine; + + // run the codecept command in a sub process + $process = new Process($codeceptionCommand); + $process->setWorkingDirectory(TESTS_BP); + $process->setIdleTimeout(600); + $process->setTimeout(0); + $subReturnCode = $process->run(function ($type, $buffer) use ($output) { + $output->write($buffer); + }); + $this->returnCode = max($this->returnCode, $subReturnCode); + } + + /** + * Keeps track of any tests that failed while running the manifest file. + * + * Each codecept command executions overwrites the failed file. Since we are running multiple codecept commands, + * we need to hold on to any failures in order to write a final failed file containing all tests. + * + * @return void + */ + private function aggregateFailed() + { + if (file_exists(RunTestFailedCommand::TESTS_FAILED_FILE)) { + $currentFile = file(RunTestFailedCommand::TESTS_FAILED_FILE, FILE_IGNORE_NEW_LINES); + $this->failedTests = array_merge( + $this->failedTests, + $currentFile + ); + } + } + + /** + * Delete the Codeception failed file. + * + * @return void + */ + private function deleteFailedFile() + { + if (file_exists(RunTestFailedCommand::TESTS_FAILED_FILE)) { + unlink(RunTestFailedCommand::TESTS_FAILED_FILE); + } + } + + /** + * Writes any tests that failed to the Codeception failed file. + * + * @return void + */ + private function writeFailedFile() + { + foreach ($this->failedTests as $test) { + file_put_contents(RunTestFailedCommand::TESTS_FAILED_FILE, $test . "\n", FILE_APPEND); + } + } +} From 300619dcccefaaf83ccb1f3f5efce8273c43a5aa Mon Sep 17 00:00:00 2001 From: Kevin Kozan Date: Thu, 29 Aug 2019 14:22:28 -0500 Subject: [PATCH 46/82] MQE-1633: Add API Request To Output for MFTF API Actions - Added AllureHelper + Unit tests - Added attachment event to curlHandler classes --- .../Allure/AllureHelperTest.php | 97 +++++++++++++++++++ .../Allure/AllureHelper.php | 39 ++++++++ .../DataGenerator/Persist/CurlHandler.php | 9 ++ 3 files changed, 145 insertions(+) create mode 100644 dev/tests/unit/Magento/FunctionalTestFramework/Allure/AllureHelperTest.php create mode 100644 src/Magento/FunctionalTestingFramework/Allure/AllureHelper.php diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Allure/AllureHelperTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Allure/AllureHelperTest.php new file mode 100644 index 000000000..048c6f7de --- /dev/null +++ b/dev/tests/unit/Magento/FunctionalTestFramework/Allure/AllureHelperTest.php @@ -0,0 +1,97 @@ +mockAttachmentWriteEvent(); + $expectedData = "string"; + $expectedCaption = "caption"; + + //Prepare Allure lifecycle + Allure::lifecycle()->fire(new StepStartedEvent('firstStep')); + + //Call function + AllureHelper::addAttachmentToCurrentStep($expectedData, $expectedCaption); + + // Assert Attachment is created as expected + $step = Allure::lifecycle()->getStepStorage()->pollLast(); + $expectedAttachment = new Attachment($expectedCaption, self::MOCK_FILENAME, null); + $this->assertEquals($step->getAttachments()[0], $expectedAttachment); + } + + /** + * AddAttachmentToLastStep should add an attachment only to the last step + * @throws \Yandex\Allure\Adapter\AllureException + */ + public function testAddAttachmentToLastStep() + { + $this->mockAttachmentWriteEvent(); + $expectedData = "string"; + $expectedCaption = "caption"; + + //Prepare Allure lifecycle + Allure::lifecycle()->fire(new StepStartedEvent('firstStep')); + Allure::lifecycle()->fire(new StepFinishedEvent('firstStep')); + Allure::lifecycle()->fire(new StepStartedEvent('secondStep')); + Allure::lifecycle()->fire(new StepFinishedEvent('secondStep')); + + //Call function + AllureHelper::addAttachmentToLastStep($expectedData, $expectedCaption); + + //Continue Allure lifecycle + Allure::lifecycle()->fire(new StepStartedEvent('thirdStep')); + Allure::lifecycle()->fire(new StepFinishedEvent('thirdStep')); + + // Assert Attachment is created as expected on the right step + $rootStep = Allure::lifecycle()->getStepStorage()->pollLast(); + + $firstStep = $rootStep->getSteps()[0]; + $secondStep = $rootStep->getSteps()[1]; + $thirdStep = $rootStep->getSteps()[2]; + + $expectedAttachment = new Attachment($expectedCaption, self::MOCK_FILENAME, null); + $this->assertEmpty($firstStep->getAttachments()); + $this->assertEquals($secondStep->getAttachments()[0], $expectedAttachment); + $this->assertEmpty($thirdStep->getAttachments()); + } + + /** + * Mock file system manipulation function + * @throws \Exception + */ + public function mockAttachmentWriteEvent() + { + AspectMock::double(AddAttachmentEvent::class, [ + "getAttachmentFileName" => self::MOCK_FILENAME + ]); + } +} diff --git a/src/Magento/FunctionalTestingFramework/Allure/AllureHelper.php b/src/Magento/FunctionalTestingFramework/Allure/AllureHelper.php new file mode 100644 index 000000000..232c3387a --- /dev/null +++ b/src/Magento/FunctionalTestingFramework/Allure/AllureHelper.php @@ -0,0 +1,39 @@ +fire(new AddAttachmentEvent($data, $caption)); + } + + /** + * Adds Attachment to the last executed step. Required due to Allure root-step behavior + * @param mixed $data + * @param string $caption + * @return void + */ + public static function addAttachmentToLastStep($data, $caption) + { + $rootStep = Allure::lifecycle()->getStepStorage()->getLast(); + $trueLastStep = array_last($rootStep->getSteps()); + + $attachmentEvent = new AddAttachmentEvent($data, $caption); + $attachmentEvent->process($trueLastStep); + } +} diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/CurlHandler.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/CurlHandler.php index f6d8773bc..4b184c206 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/CurlHandler.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/CurlHandler.php @@ -5,6 +5,7 @@ */ namespace Magento\FunctionalTestingFramework\DataGenerator\Persist; +use Magento\FunctionalTestingFramework\Allure\AllureHelper; use Magento\FunctionalTestingFramework\DataGenerator\Persist\Curl\AdminExecutor; use Magento\FunctionalTestingFramework\DataGenerator\Persist\Curl\FrontendExecutor; use Magento\FunctionalTestingFramework\DataGenerator\Persist\Curl\WebapiExecutor; @@ -166,6 +167,14 @@ public function executeRequest($dependentEntities) $response = $executor->read($successRegex, $returnRegex, $returnIndex); $executor->close(); + AllureHelper::addAttachmentToLastStep($apiUrl, 'API Endpoint'); + AllureHelper::addAttachmentToLastStep(json_encode($headers, JSON_PRETTY_PRINT), 'Request Headers'); + AllureHelper::addAttachmentToLastStep(json_encode($this->requestData, JSON_PRETTY_PRINT), 'Request Body'); + AllureHelper::addAttachmentToLastStep( + json_encode(json_decode($response, true), JSON_PRETTY_PRINT+JSON_UNESCAPED_UNICODE+JSON_UNESCAPED_SLASHES), + 'Response Data' + ); + return $response; } From fdcda9a93cbeb243d90f2384a9bbe02959140ef9 Mon Sep 17 00:00:00 2001 From: Kevin Kozan Date: Thu, 29 Aug 2019 14:26:36 -0500 Subject: [PATCH 47/82] MQE-1514: Failure For seeCurrentUrlEquals Action Does Not Show Actual - Added contents of 1633, changed the calls to use Helper class instead --- .../Module/MagentoWebDriver.php | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php index 830fd40c4..731643eaf 100644 --- a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php +++ b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php @@ -9,7 +9,7 @@ use Codeception\Module\WebDriver; use Codeception\Test\Descriptor; use Codeception\TestInterface; -use Yandex\Allure\Adapter\Allure; +use Magento\FunctionalTestingFramework\Allure\AllureHelper; use Facebook\WebDriver\Interactions\WebDriverActions; use Codeception\Exception\ModuleConfigException; use Codeception\Exception\ModuleException; @@ -19,8 +19,6 @@ use Magento\FunctionalTestingFramework\Util\Protocol\CurlTransport; use Magento\FunctionalTestingFramework\Util\Protocol\CurlInterface; use Magento\FunctionalTestingFramework\Util\ConfigSanitizerUtil; -use Yandex\Allure\Adapter\Event\AddAttachmentEvent; -use Yandex\Allure\Adapter\Event\AddParameterEvent; use Yandex\Allure\Adapter\Support\AttachmentSupport; use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; @@ -203,7 +201,7 @@ public function dontSeeCurrentUrlEquals($url) { $actualUrl = $this->webDriver->getCurrentURL(); $comparison = "Expected: $url\nActual: $actualUrl"; - Allure::lifecycle()->fire(new AddAttachmentEvent($comparison, 'Comparison')); + AllureHelper::addAttachmentToCurrentStep($comparison, 'Comparison'); $this->assertNotEquals($url, $actualUrl); } @@ -217,7 +215,7 @@ public function dontSeeCurrentUrlMatches($regex) { $actualUrl = $this->webDriver->getCurrentURL(); $comparison = "Expected: $regex\nActual: $actualUrl"; - Allure::lifecycle()->fire(new AddAttachmentEvent($comparison, 'Comparison')); + AllureHelper::addAttachmentToCurrentStep($comparison, 'Comparison'); $this->assertNotRegExp($regex, $actualUrl); } @@ -231,7 +229,7 @@ public function dontSeeInCurrentUrl($needle) { $actualUrl = $this->webDriver->getCurrentURL(); $comparison = "Expected: $needle\nActual: $actualUrl"; - Allure::lifecycle()->fire(new AddAttachmentEvent($comparison, 'Comparison')); + AllureHelper::addAttachmentToCurrentStep($comparison, 'Comparison'); $this->assertNotContains($needle, $actualUrl); } @@ -268,7 +266,7 @@ public function seeCurrentUrlEquals($url) { $actualUrl = $this->webDriver->getCurrentURL(); $comparison = "Expected: $url\nActual: $actualUrl"; - Allure::lifecycle()->fire(new AddAttachmentEvent($comparison, 'Comparison')); + AllureHelper::addAttachmentToCurrentStep($comparison, 'Comparison'); $this->assertEquals($url, $actualUrl); } @@ -282,7 +280,7 @@ public function seeCurrentUrlMatches($regex) { $actualUrl = $this->webDriver->getCurrentURL(); $comparison = "Expected: $regex\nActual: $actualUrl"; - Allure::lifecycle()->fire(new AddAttachmentEvent($comparison, 'Comparison')); + AllureHelper::addAttachmentToCurrentStep($comparison, 'Comparison'); $this->assertRegExp($regex, $actualUrl); } @@ -296,7 +294,7 @@ public function seeInCurrentUrl($needle) { $actualUrl = $this->webDriver->getCurrentURL(); $comparison = "Expected: $needle\nActual: $actualUrl"; - Allure::lifecycle()->fire(new AddAttachmentEvent($comparison, 'Comparison')); + AllureHelper::addAttachmentToCurrentStep($comparison, 'Comparison'); $this->assertContains($needle, $actualUrl); } From f38196387a5de58fff1a451b4a8b8bf59ed4bfe9 Mon Sep 17 00:00:00 2001 From: Tom Reece Date: Thu, 29 Aug 2019 14:43:03 -0500 Subject: [PATCH 48/82] MQE-1626: bin/mftf run:manifest - Add documentation --- docs/commands/mftf.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/docs/commands/mftf.md b/docs/commands/mftf.md index aea53c96f..1347ac13e 100644 --- a/docs/commands/mftf.md +++ b/docs/commands/mftf.md @@ -58,6 +58,14 @@ vendor/bin/mftf run:test LoginAsAdminTest LoginAsCustomerTest -r This command cleans up the previously generated tests; generates and runs the `LoginAsAdminTest` and `LoginAsCustomerTest` tests. +### Generate and run a testManifest.txt file + +```bash +vendor/bin/mftf run:manifest path/to/your/testManifest.txt +``` + +This command runs all tests specified in a testManifest.txt file. When you generate tests, a testManifest.txt file is also generated for you. You can pass this file directly to the run:manifest command and it will execute all tests. Or you can create your own file of the same format to execute a subset of tests. Note: This command does not generate tests. + ### Generate and run previously failed tests ```bash @@ -338,6 +346,30 @@ Generate the `LoginCustomerTest` and `StorefrontCreateCustomerTest` tests from X vendor/bin/mftf run:test LoginCustomerTest StorefrontCreateCustomerTest ``` +### `run:manifest` + +Runs a testManifest.txt file. + +This command runs all tests specified in a testManifest.xml file. It does not generate tests for you. You must do that as first. + +#### Usage + +```bash +vendor/bin/mftf run:manifest path/to/your/testManifest.txt +``` + +#### Example testManifest.xml file + +Each line should contain either: one test path or one group (-g) reference. + +``` +tests/functional/tests/MFTF/_generated/default/AdminLoginTestCest.php +-g PaypalTestSuite +tests/functional/tests/MFTF/_generated/default/SomeOtherTestCest.php +tests/functional/tests/MFTF/_generated/default/ThirdTestCest.php +-g SomeOtherSuite +``` + ### `run:failed` Regenerates and reruns tests that previously failed. From 18030d3b1f6943c5dccb030dc6da9d6cec129fc5 Mon Sep 17 00:00:00 2001 From: Ji Lu Date: Fri, 30 Aug 2019 09:14:00 -0500 Subject: [PATCH 49/82] MQE-1685: Test/Data self extension memory limit error --- .../Handlers/DataObjectHandlerTest.php | 224 ++++++++++++++++-- .../Handlers/ActionGroupObjectHandlerTest.php | 98 ++++++++ .../Test/Handlers/TestObjectHandlerTest.php | 62 +++++ .../Test/Util/ObjectExtensionUtilTest.php | 8 +- .../unit/Util/ActionGroupArrayBuilder.php | 170 +++++++++++++ .../Handlers/DataObjectHandler.php | 5 + .../Handlers/ActionGroupObjectHandler.php | 5 + .../Test/Handlers/TestObjectHandler.php | 5 + 8 files changed, 549 insertions(+), 28 deletions(-) create mode 100644 dev/tests/unit/Magento/FunctionalTestFramework/Test/Handlers/ActionGroupObjectHandlerTest.php create mode 100644 dev/tests/unit/Util/ActionGroupArrayBuilder.php diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/DataObjectHandlerTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/DataObjectHandlerTest.php index b77659aa2..d55f35247 100644 --- a/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/DataObjectHandlerTest.php +++ b/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/DataObjectHandlerTest.php @@ -30,41 +30,105 @@ class DataObjectHandlerTest extends MagentoTestCase 'value' => 'testValue' ] ] - ] + ], + 'EntityTwo' => [ + 'type' => 'testType', + 'extends' => 'EntityOne', + 'data' => [ + 0 => [ + 'key' => 'testKeyTwo', + 'value' => 'testValueTwo' + ] + ] + ], ] ]; - /** - * Set up everything required to mock DataObjectHander::getInstance() - * The first call to getInstance() uses these mocks to emulate the parser, initializing internal state - * according to the PARSER_OUTPUT value - */ - public static function setUpBeforeClass() - { - $mockDataProfileSchemaParser = AspectMock::double(DataProfileSchemaParser::class, [ - 'readDataProfiles' => self::PARSER_OUTPUT - ])->make(); - - $mockObjectManager = AspectMock::double(ObjectManager::class, [ - 'create' => $mockDataProfileSchemaParser - ])->make(); + const PARSER_OUTPUT_WITH_EXTEND = [ + 'entity' => [ + 'EntityOne' => [ + 'name' => 'EntityOne', + 'type' => 'testType', + 'data' => [ + 0 => [ + 'key' => 'testKey', + 'value' => 'testValue' + ] + ] + ], + 'EntityTwo' => [ + 'name' => 'EntityTwo', + 'type' => 'testType', + 'extends' => 'EntityOne', + 'data' => [ + 0 => [ + 'key' => 'testKeyTwo', + 'value' => 'testValueTwo' + ] + ], + ], + 'EntityThree' => [ + 'name' => 'EntityThree', + 'type' => 'testType', + 'extends' => 'EntityOne', + 'data' => [ + 0 => [ + 'key' => 'testKeyThree', + 'value' => 'testValueThree' + ] + ], + ] + ] + ]; - AspectMock::double(ObjectManagerFactory::class, [ - 'getObjectManager' => $mockObjectManager - ]); - } + const PARSER_OUTPUT_WITH_EXTEND_INVALID = [ + 'entity' => [ + 'EntityOne' => [ + 'name' => 'EntityOne', + 'type' => 'testType', + 'extends' => 'EntityOne', + 'data' => [ + 0 => [ + 'key' => 'testKey', + 'value' => 'testValue' + ] + ] + ], + 'EntityTwo' => [ + 'name' => 'EntityTwo', + 'type' => 'testType', + 'data' => [ + 0 => [ + 'key' => 'testKeyTwo', + 'value' => 'testValueTwo' + ] + ], + ], + 'EntityThree' => [ + 'name' => 'EntityThree', + 'type' => 'testType', + 'extends' => 'EntityThree', + 'data' => [ + 0 => [ + 'key' => 'testKeyThree', + 'value' => 'testValueThree' + ] + ], + ] + ] + ]; /** * getAllObjects should contain the expected data object */ public function testGetAllObjects() { - // Call the method under test + $this->setUpMockDataObjectHander(self::PARSER_OUTPUT); + // Call the method under test $actual = DataObjectHandler::getInstance()->getAllObjects(); // Assert - $expected = new EntityDataObject('EntityOne', 'testType', ['testkey' => 'testValue'], [], null, []); $this->assertArrayHasKey('EntityOne', $actual); $this->assertEquals($expected, $actual['EntityOne']); @@ -75,22 +139,134 @@ public function testGetAllObjects() */ public function testGetObject() { - // Call the method under test + $this->setUpMockDataObjectHander(self::PARSER_OUTPUT); + // Call the method under test $actual = DataObjectHandler::getInstance()->getObject('EntityOne'); // Assert - $expected = new EntityDataObject('EntityOne', 'testType', ['testkey' => 'testValue'], [], null, []); $this->assertEquals($expected, $actual); } /** - * getObject should return null if the data object does not exist + * getAllObjects should return the expected data object if it exists */ public function testGetObjectNull() { + $this->setUpMockDataObjectHander(self::PARSER_OUTPUT); + $actual = DataObjectHandler::getInstance()->getObject('h953u789h0g73t521'); // doesnt exist $this->assertNull($actual); } + + /** + * getAllObjects should contain the expected data object with extends + */ + public function testGetAllObjectsWithDataExtends() + { + $this->setUpMockDataObjectHander(self::PARSER_OUTPUT_WITH_EXTEND); + + // Call the method under test + $actual = DataObjectHandler::getInstance()->getAllObjects(); + + // Assert + $expected = new EntityDataObject( + 'EntityTwo', + 'testType', + ['testkey' => 'testValue', 'testkeytwo' => 'testValueTwo'], + [], + null, + [], + 'EntityOne' + ); + $this->assertArrayHasKey('EntityTwo', $actual); + $this->assertEquals($expected, $actual['EntityTwo']); + } + + /** + * getObject should return the expected data object with extended data if it exists + */ + public function testGetObjectWithDataExtends() + { + $this->setUpMockDataObjectHander(self::PARSER_OUTPUT_WITH_EXTEND); + + // Call the method under test + $actual = DataObjectHandler::getInstance()->getObject('EntityTwo'); + + // Assert + $expected = new EntityDataObject( + 'EntityTwo', + 'testType', + ['testkey' => 'testValue', 'testkeytwo' => 'testValueTwo'], + [], + null, + [], + 'EntityOne' + ); + $this->assertEquals($expected, $actual); + } + + /** + * getAllObjects should throw TestFrameworkException exception if some data extends itself + */ + public function testGetAllObjectsWithDataExtendsItself() + { + $this->setUpMockDataObjectHander(self::PARSER_OUTPUT_WITH_EXTEND_INVALID); + + $this->expectException(\Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException::class); + $this->expectExceptionMessage( + "Mftf Data can not extend from itself: " + . self::PARSER_OUTPUT_WITH_EXTEND_INVALID['entity']['EntityOne']['name'] + ); + + // Call the method under test + DataObjectHandler::getInstance()->getAllObjects(); + } + + /** + * getObject should throw TestFrameworkException exception if requested data extends itself + */ + public function testGetObjectWithDataExtendsItself() + { + $this->setUpMockDataObjectHander(self::PARSER_OUTPUT_WITH_EXTEND_INVALID); + + $this->expectException(\Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException::class); + $this->expectExceptionMessage( + "Mftf Data can not extend from itself: " + . self::PARSER_OUTPUT_WITH_EXTEND_INVALID['entity']['EntityOne']['name'] + ); + + // Call the method under test + DataObjectHandler::getInstance()->getObject( + self::PARSER_OUTPUT_WITH_EXTEND_INVALID['entity']['EntityOne']['name'] + ); + } + + /** + * Set up everything required to mock DataObjectHander::getInstance() + * The first call to getInstance() uses these mocks to emulate the parser, initializing internal state + * according to the PARSER_OUTPUT value + * + * @param array $entityDataArray + */ + private function setUpMockDataObjectHander($entityDataArray) + { + // Clear DataObjectHandler singleton if already set + $property = new \ReflectionProperty(DataObjectHandler::class, "INSTANCE"); + $property->setAccessible(true); + $property->setValue(null); + + $mockDataProfileSchemaParser = AspectMock::double(DataProfileSchemaParser::class, [ + 'readDataProfiles' => $entityDataArray + ])->make(); + + $mockObjectManager = AspectMock::double(ObjectManager::class, [ + 'create' => $mockDataProfileSchemaParser + ])->make(); + + AspectMock::double(ObjectManagerFactory::class, [ + 'getObjectManager' => $mockObjectManager + ]); + } } diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Test/Handlers/ActionGroupObjectHandlerTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Test/Handlers/ActionGroupObjectHandlerTest.php new file mode 100644 index 000000000..4a7c6101e --- /dev/null +++ b/dev/tests/unit/Magento/FunctionalTestFramework/Test/Handlers/ActionGroupObjectHandlerTest.php @@ -0,0 +1,98 @@ +withName($nameOne) + ->withExtendedAction($nameOne) + ->withAnnotations() + ->withFilename() + ->withActionObjects() + ->build(); + $this->setMockParserOutput(['actionGroups' => $actionGroupOne]); + + $handler = ActionGroupObjectHandler::getInstance(); + + $this->expectException(\Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException::class); + $this->expectExceptionMessage("Mftf Action Group can not extend from itself: " . $nameOne); + $handler->getObject('actionGroupOne'); + } + + /** + * getAllObjects should throw exception if test extends from itself + * + * @throws \Exception + */ + public function testGetAllTestObjectsWithInvalidExtends() + { + // Set up action group data + $nameOne = 'actionGroupOne'; + $nameTwo = 'actionGroupTwo'; + $actionGroupOne = (new ActionGroupArrayBuilder()) + ->withName($nameOne) + ->withExtendedAction($nameOne) + ->withAnnotations() + ->withFilename() + ->withActionObjects() + ->build(); + $actionGroupTwo = (new ActionGroupArrayBuilder()) + ->withName($nameTwo) + ->withExtendedAction() + ->withAnnotations() + ->withFilename() + ->withActionObjects() + ->build(); + + $this->setMockParserOutput(['actionGroups' => array_merge($actionGroupOne, $actionGroupTwo)]); + + $handler = ActionGroupObjectHandler::getInstance(); + + $this->expectException(\Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException::class); + $this->expectExceptionMessage("Mftf Action Group can not extend from itself: " . $nameOne); + $handler->getAllObjects(); + } + + /** + * Function used to set mock for parser return and force init method to run between tests. + * + * @param array $data + * @throws \Exception + */ + private function setMockParserOutput($data) + { + // Clear action group object handler value to inject parsed content + $property = new \ReflectionProperty(ActionGroupObjectHandler::class, 'instance'); + $property->setAccessible(true); + $property->setValue(null); + + $mockDataParser = AspectMock::double(ActionGroupDataParser::class, ['readActionGroupData' => $data])->make(); + $instance = AspectMock::double(ObjectManager::class, ['create' => $mockDataParser]) + ->make(); // bypass the private constructor + AspectMock::double(ObjectManagerFactory::class, ['getObjectManager' => $instance]); + } +} diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Test/Handlers/TestObjectHandlerTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Test/Handlers/TestObjectHandlerTest.php index 1dbeb50db..a94c41ae4 100644 --- a/dev/tests/unit/Magento/FunctionalTestFramework/Test/Handlers/TestObjectHandlerTest.php +++ b/dev/tests/unit/Magento/FunctionalTestFramework/Test/Handlers/TestObjectHandlerTest.php @@ -180,6 +180,68 @@ public function testGetTestWithModuleName() $this->assertEquals($moduleExpected, $moduleName); } + /** + * getObject should throw exception if test extends from itself + * + * @throws \Exception + */ + public function testGetTestObjectWithInvalidExtends() + { + // set up Test Data + $testOne = (new TestDataArrayBuilder()) + ->withName('testOne') + ->withTestReference('testOne') + ->withAnnotations() + ->withFailedHook() + ->withAfterHook() + ->withBeforeHook() + ->withTestActions() + ->build(); + $this->setMockParserOutput(['tests' => $testOne]); + + $toh = TestObjectHandler::getInstance(); + + $this->expectException(\Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException::class); + $this->expectExceptionMessage("Mftf Test can not extend from itself: " . "testOne"); + + $toh->getObject('testOne'); + } + + /** + * getAllObjects should throw exception if test extends from itself + * + * @throws \Exception + */ + public function testGetAllTestObjectsWithInvalidExtends() + { + // set up Test Data + $testOne = (new TestDataArrayBuilder()) + ->withName('testOne') + ->withTestReference('testOne') + ->withAnnotations() + ->withFailedHook() + ->withAfterHook() + ->withBeforeHook() + ->withTestActions() + ->build(); + $testTwo = (new TestDataArrayBuilder()) + ->withName('testTwo') + ->withAnnotations() + ->withFailedHook() + ->withAfterHook() + ->withBeforeHook() + ->withTestActions() + ->build(); + + $this->setMockParserOutput(['tests' => array_merge($testOne, $testTwo)]); + + $toh = TestObjectHandler::getInstance(); + + $this->expectException(\Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException::class); + $this->expectExceptionMessage("Mftf Test can not extend from itself: " . "testOne"); + $toh->getAllObjects(); + } + /** * Function used to set mock for parser return and force init method to run between tests. * diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Test/Util/ObjectExtensionUtilTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Test/Util/ObjectExtensionUtilTest.php index b023b16c7..ce184e564 100644 --- a/dev/tests/unit/Magento/FunctionalTestFramework/Test/Util/ObjectExtensionUtilTest.php +++ b/dev/tests/unit/Magento/FunctionalTestFramework/Test/Util/ObjectExtensionUtilTest.php @@ -263,7 +263,7 @@ public function testGenerateExtendedActionGroupNoParent() { $mockExtendedActionGroup = [ "nodeName" => "actionGroup", - "name" => "mockSimpleActionGroup", + "name" => "mockExtendedActionGroup", "filename" => "someFile", "extends" => "mockSimpleActionGroup", "commentHere" => [ @@ -281,7 +281,7 @@ public function testGenerateExtendedActionGroupNoParent() $this->setMockTestOutput(null, $mockActionGroupData); $this->expectExceptionMessage( - "Parent Action Group mockSimpleActionGroup not defined for Test " . $mockExtendedActionGroup['extends'] + "Parent Action Group mockSimpleActionGroup not defined for Test " . $mockExtendedActionGroup['name'] ); // parse and generate test object with mocked data @@ -309,7 +309,7 @@ public function testExtendingExtendedActionGroup() $mockExtendedActionGroup = [ "nodeName" => "actionGroup", - "name" => "mockSimpleActionGroup", + "name" => "mockExtendedActionGroup", "filename" => "someFile", "extends" => "mockSimpleActionGroup", ]; @@ -336,7 +336,7 @@ public function testExtendingExtendedActionGroup() 'error', "Cannot extend an action group that already extends another action group. " . $mockSimpleActionGroup['name'], - ['parent' => $mockSimpleActionGroup['name'], 'actionGroup' => $mockSimpleActionGroup['name']] + ['parent' => $mockSimpleActionGroup['name'], 'actionGroup' => $mockExtendedActionGroup['name']] ); throw $e; diff --git a/dev/tests/unit/Util/ActionGroupArrayBuilder.php b/dev/tests/unit/Util/ActionGroupArrayBuilder.php new file mode 100644 index 000000000..14877f4dd --- /dev/null +++ b/dev/tests/unit/Util/ActionGroupArrayBuilder.php @@ -0,0 +1,170 @@ +name = $name; + return $this; + } + + /** + * Setter for action group annotations + * + * @param array $annotations + * @return $this + */ + public function withAnnotations($annotations = []) + { + $this->annotations = $annotations; + return $this; + } + + /** + * Setter for action group arguments + * + * @param array $args + * @return $this + */ + public function withArguments($args = []) + { + $this->arguments = $args; + return $this; + } + + /** + * Setter for action group actions + * + * @param array $actionObjs + * @return $this + */ + public function withActionObjects($actionObjs = []) + { + if (!empty($actionObjs)) { + $this->actionObjects = $actionObjs; + } + return $this; + } + + /** + * Setter for action group extended action group name + * + * @param string $extendedActionGroup + * @return $this + */ + public function withExtendedAction($extendedActionGroup = null) + { + $this->extends = $extendedActionGroup; + return $this; + } + + /** + * Setter for action group filename + * + * @param string $filename + * @return $this + */ + public function withFilename($filename = '') + { + if (empty($filename)) { + $this->filename = "/magento2-functional-testing-framework/dev/tests/verification/" + . "TestModule/ActionGroup/BasicActionGroup.xml"; + } else { + $this->filename = $filename; + } + + return $this; + } + + /** + * ActionGroupArrayBuilder constructor + */ + public function __construct() + { + $this->actionObjects = [ + self::DEFAULT_ACTION_GROUP_KEY => [ + ActionObjectExtractor::NODE_NAME => 'testActionType', + ActionObjectExtractor::TEST_STEP_MERGE_KEY => self::DEFAULT_ACTION_GROUP_KEY, + ] + ]; + } + + /** + * Function which takes builder parameters and returns an action group array + * + * @return array + */ + public function build() + { + // Build and return action group array + return [$this->name => array_merge( + [ + ActionGroupObjectExtractor::NAME => $this->name, + ActionGroupObjectExtractor::ACTION_GROUP_ANNOTATIONS => $this->annotations, + ActionGroupObjectExtractor::ACTION_GROUP_ARGUMENTS => $this->arguments, + ActionGroupObjectExtractor::FILENAME => $this->filename, + ActionGroupObjectExtractor::EXTENDS_ACTION_GROUP => $this->extends + ], + $this->actionObjects + )]; + } +} diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/DataObjectHandler.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/DataObjectHandler.php index 7cc9e8a0e..95a2b8b93 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/DataObjectHandler.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/DataObjectHandler.php @@ -8,6 +8,7 @@ use Magento\FunctionalTestingFramework\DataGenerator\Objects\EntityDataObject; use Magento\FunctionalTestingFramework\DataGenerator\Parsers\DataProfileSchemaParser; +use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; use Magento\FunctionalTestingFramework\Exceptions\XmlException; use Magento\FunctionalTestingFramework\ObjectManager\ObjectHandlerInterface; use Magento\FunctionalTestingFramework\ObjectManagerFactory; @@ -272,10 +273,14 @@ private function processVarElements($entityData) * * @param EntityDataObject $dataObject * @return EntityDataObject + * @throws TestFrameworkException */ private function extendDataObject($dataObject) { if ($dataObject->getParentName() != null) { + if ($dataObject->getParentName() == $dataObject->getName()) { + throw new TestFrameworkException("Mftf Data can not extend from itself: " . $dataObject->getName()); + } return $this->extendUtil->extendEntity($dataObject); } return $dataObject; diff --git a/src/Magento/FunctionalTestingFramework/Test/Handlers/ActionGroupObjectHandler.php b/src/Magento/FunctionalTestingFramework/Test/Handlers/ActionGroupObjectHandler.php index 18a6402ea..4a98eaa68 100644 --- a/src/Magento/FunctionalTestingFramework/Test/Handlers/ActionGroupObjectHandler.php +++ b/src/Magento/FunctionalTestingFramework/Test/Handlers/ActionGroupObjectHandler.php @@ -5,6 +5,7 @@ */ namespace Magento\FunctionalTestingFramework\Test\Handlers; +use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; use Magento\FunctionalTestingFramework\ObjectManager\ObjectHandlerInterface; use Magento\FunctionalTestingFramework\ObjectManagerFactory; use Magento\FunctionalTestingFramework\Test\Objects\ActionGroupObject; @@ -124,10 +125,14 @@ private function initActionGroups() * * @param ActionGroupObject $actionGroupObject * @return ActionGroupObject + * @throws TestFrameworkException */ private function extendActionGroup($actionGroupObject): ActionGroupObject { if ($actionGroupObject->getParentName() !== null) { + if ($actionGroupObject->getParentName() == $actionGroupObject->getName()) { + throw new TestFrameworkException("Mftf Action Group can not extend from itself: " . $actionGroupObject->getName()); + } return $this->extendUtil->extendActionGroup($actionGroupObject); } return $actionGroupObject; diff --git a/src/Magento/FunctionalTestingFramework/Test/Handlers/TestObjectHandler.php b/src/Magento/FunctionalTestingFramework/Test/Handlers/TestObjectHandler.php index 1c0f7e2fc..a0d490df5 100644 --- a/src/Magento/FunctionalTestingFramework/Test/Handlers/TestObjectHandler.php +++ b/src/Magento/FunctionalTestingFramework/Test/Handlers/TestObjectHandler.php @@ -6,6 +6,7 @@ namespace Magento\FunctionalTestingFramework\Test\Handlers; use Magento\FunctionalTestingFramework\Exceptions\Collector\ExceptionCollector; +use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; use Magento\FunctionalTestingFramework\Exceptions\TestReferenceException; use Magento\FunctionalTestingFramework\Exceptions\XmlException; use Magento\FunctionalTestingFramework\ObjectManager\ObjectHandlerInterface; @@ -160,10 +161,14 @@ private function initTestData() * * @param TestObject $testObject * @return TestObject + * @throws TestFrameworkException */ private function extendTest($testObject) { if ($testObject->getParentName() !== null) { + if ($testObject->getParentName() == $testObject->getName()) { + throw new TestFrameworkException("Mftf Test can not extend from itself: " . $testObject->getName()); + } return $this->extendUtil->extendTest($testObject); } return $testObject; From 0dd8840fd7f2029098b9caef6e33dd098093ab7e Mon Sep 17 00:00:00 2001 From: Kevin Kozan Date: Fri, 30 Aug 2019 09:54:00 -0500 Subject: [PATCH 50/82] MQE-1633: Add API Request To Output for MFTF API Actions - Unit test fix --- dev/tests/_bootstrap.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dev/tests/_bootstrap.php b/dev/tests/_bootstrap.php index e06e373d4..b41f80394 100644 --- a/dev/tests/_bootstrap.php +++ b/dev/tests/_bootstrap.php @@ -17,7 +17,10 @@ $kernel = \AspectMock\Kernel::getInstance(); $kernel->init([ 'debug' => true, - 'includePaths' => [PROJECT_ROOT . DIRECTORY_SEPARATOR . 'src'], + 'includePaths' => [ + PROJECT_ROOT . DIRECTORY_SEPARATOR . 'src', + PROJECT_ROOT . DIRECTORY_SEPARATOR . 'vendor' . DIRECTORY_SEPARATOR . 'allure-framework' + ], 'cacheDir' => PROJECT_ROOT . DIRECTORY_SEPARATOR . 'dev' . From 4a055bf7e59ce59a2d2c68a5e8748be6dbe839d1 Mon Sep 17 00:00:00 2001 From: Ji Lu Date: Fri, 30 Aug 2019 10:05:11 -0500 Subject: [PATCH 51/82] MQE-1685: Test/Data self extension memory limit error --- .../Test/Handlers/ActionGroupObjectHandler.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Magento/FunctionalTestingFramework/Test/Handlers/ActionGroupObjectHandler.php b/src/Magento/FunctionalTestingFramework/Test/Handlers/ActionGroupObjectHandler.php index 4a98eaa68..131ae0a26 100644 --- a/src/Magento/FunctionalTestingFramework/Test/Handlers/ActionGroupObjectHandler.php +++ b/src/Magento/FunctionalTestingFramework/Test/Handlers/ActionGroupObjectHandler.php @@ -131,7 +131,9 @@ private function extendActionGroup($actionGroupObject): ActionGroupObject { if ($actionGroupObject->getParentName() !== null) { if ($actionGroupObject->getParentName() == $actionGroupObject->getName()) { - throw new TestFrameworkException("Mftf Action Group can not extend from itself: " . $actionGroupObject->getName()); + throw new TestFrameworkException( + "Mftf Action Group can not extend from itself: " . $actionGroupObject->getName() + ); } return $this->extendUtil->extendActionGroup($actionGroupObject); } From fdd5ec09e5c94f0cb874b6cb207ab6da6cceb0fa Mon Sep 17 00:00:00 2001 From: Kevin Kozan Date: Fri, 30 Aug 2019 10:17:41 -0500 Subject: [PATCH 52/82] MQE-1633: Add API Request To Output for MFTF API Actions - Updated docblock --- src/Magento/FunctionalTestingFramework/Allure/AllureHelper.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Magento/FunctionalTestingFramework/Allure/AllureHelper.php b/src/Magento/FunctionalTestingFramework/Allure/AllureHelper.php index 232c3387a..d8f1c63d8 100644 --- a/src/Magento/FunctionalTestingFramework/Allure/AllureHelper.php +++ b/src/Magento/FunctionalTestingFramework/Allure/AllureHelper.php @@ -23,7 +23,8 @@ public static function addAttachmentToCurrentStep($data, $caption) } /** - * Adds Attachment to the last executed step. Required due to Allure root-step behavior + * Adds Attachment to the last executed step. + * Use this when adding attachments outside of an $I->doSomething() step/context. * @param mixed $data * @param string $caption * @return void From d4a1237b032e3669ee8c4e7e1e1f3744a1978acd Mon Sep 17 00:00:00 2001 From: Ji Lu Date: Fri, 30 Aug 2019 11:05:36 -0500 Subject: [PATCH 53/82] MQE-588: CreatedData should throw Error/Warning when undefined data is returned - changed to print warning message for backward compatibility --- .../Handlers/PersistedObjectHandlerTest.php | 31 +++++++++++++++++-- .../Handlers/PersistedObjectHandler.php | 9 ++++-- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/PersistedObjectHandlerTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/PersistedObjectHandlerTest.php index fd04ee5fe..c79ad322a 100644 --- a/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/PersistedObjectHandlerTest.php +++ b/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/PersistedObjectHandlerTest.php @@ -16,12 +16,22 @@ use Magento\FunctionalTestingFramework\ObjectManager; use Magento\FunctionalTestingFramework\ObjectManagerFactory; use Magento\FunctionalTestingFramework\Util\MagentoTestCase; +use tests\unit\Util\TestLoggingUtil; /** * Class PersistedObjectHandlerTest */ class PersistedObjectHandlerTest extends MagentoTestCase { + /** + * Before test functionality + * @return void + */ + public function setUp() + { + TestLoggingUtil::getInstance()->setMockLoggingUtil(); + } + public function testCreateSimpleEntity() { // Test Data and Variables @@ -389,6 +399,8 @@ public function testRetrieveEntityValidField($name, $key, $value, $type, $scope, public function testRetrieveEntityInValidField($name, $key, $value, $type, $scope, $stepKey) { $invalidDataKey = "invalidDataKey"; + $warnMsg = "Undefined field {$invalidDataKey} in entity object with a stepKey of {$stepKey}\n"; + $warnMsg .= "Please fix the invalid reference. This will result in fatal error in next major release."; $parserOutputOne = [ 'entity' => [ @@ -416,10 +428,15 @@ public function testRetrieveEntityInValidField($name, $key, $value, $type, $scop $this->mockCurlHandler($jsonReponseOne); $handler->createEntity($stepKey, $scope, $name); - $this->expectException(\Magento\FunctionalTestingFramework\Exceptions\TestReferenceException::class); - // Call method $handler->retrieveEntityField($stepKey, $invalidDataKey, $scope); + + // validate log statement + TestLoggingUtil::getInstance()->validateMockLogStatement( + "warning", + $warnMsg, + [] + ); } /** @@ -478,4 +495,14 @@ public function tearDown() parent::tearDown(); // TODO: Change the autogenerated stub } + + /** + * After class functionality + * @return void + */ + public static function tearDownAfterClass() + { + TestLoggingUtil::getInstance()->clearMockLoggingUtil(); + parent::tearDownAfterClass(); + } } diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/PersistedObjectHandler.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/PersistedObjectHandler.php index 55dd599f1..3845e52e9 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/PersistedObjectHandler.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/PersistedObjectHandler.php @@ -9,6 +9,7 @@ use Magento\FunctionalTestingFramework\DataGenerator\Persist\DataPersistenceHandler; use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; use Magento\FunctionalTestingFramework\Exceptions\TestReferenceException; +use Magento\FunctionalTestingFramework\Util\Logger\LoggingUtil; class PersistedObjectHandler { @@ -178,14 +179,16 @@ public function getEntity($key, $scope, $entity, $dependentObjectKeys = [], $sto * @param string $field * @param string $scope * @return string - * @throws TestReferenceException - * @throws TestFrameworkException */ public function retrieveEntityField($stepKey, $field, $scope) { $fieldValue = $this->retrieveEntity($stepKey, $scope)->getCreatedDataByName($field); if ($fieldValue === null) { - throw new TestReferenceException("Undefined field {$field} in entity object with a stepKey of {$stepKey}"); + $warnMsg = "Undefined field {$field} in entity object with a stepKey of {$stepKey}\n"; + $warnMsg .= "Please fix the invalid reference. This will result in fatal error in next major release."; + //TODO: change this to throw an exception in next major release + LoggingUtil::getInstance()->getLogger(PersistedObjectHandler::class)->warn($warnMsg); + print($warnMsg . PHP_EOL); } return $fieldValue; } From 11e69d69862d3dd051925b98c8811b62efa0f2d6 Mon Sep 17 00:00:00 2001 From: Donald Booth Date: Tue, 3 Sep 2019 10:50:13 -0500 Subject: [PATCH 54/82] Small formatting fixes. --- docs/commands/mftf.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/commands/mftf.md b/docs/commands/mftf.md index 1347ac13e..ac029779e 100644 --- a/docs/commands/mftf.md +++ b/docs/commands/mftf.md @@ -64,7 +64,7 @@ This command cleans up the previously generated tests; generates and runs the `L vendor/bin/mftf run:manifest path/to/your/testManifest.txt ``` -This command runs all tests specified in a testManifest.txt file. When you generate tests, a testManifest.txt file is also generated for you. You can pass this file directly to the run:manifest command and it will execute all tests. Or you can create your own file of the same format to execute a subset of tests. Note: This command does not generate tests. +This command runs all tests specified in a `testManifest.txt` file. When you generate tests, a `testManifest.txt` file is also generated for you. You can pass this file directly to the `run:manifest` command and it will execute all listed tests. You can also create your own file in the same format to execute a subset of tests. Note: This command does not generate tests. ### Generate and run previously failed tests From 08354caabfdda0c5d25ea0d87c49fbf59a8a0510 Mon Sep 17 00:00:00 2001 From: Tom Reece Date: Tue, 3 Sep 2019 11:15:59 -0500 Subject: [PATCH 55/82] Update reporting.md --- docs/reporting.md | 37 +++++++++++++++---------------------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/docs/reporting.md b/docs/reporting.md index f303945af..26c7f8fc9 100644 --- a/docs/reporting.md +++ b/docs/reporting.md @@ -18,7 +18,7 @@ The `mftf` tool logs output continuously to the `dev/tests/acceptance/mftf.log` ## Command line -The MFTF reports about its progress during test run when you run the `mftf` CLI tool with [`run:test`][] or [`run:group`][] commands. +MFTF reports about its progress during test run when you run the `mftf` CLI tool with [`run:test`][] or [`run:group`][] commands. The report can contain three main parts: @@ -35,14 +35,7 @@ The following sections demonstrate an example interpretation of a complete log s ### Pre-run check report -First, the MFTF reports about issues with environment. -In our case, there is an issue with PHP library loading. - -```terminal -PHP Warning: PHP Startup: Unable to load dynamic library '/usr/local/lib/php/pecl/20160303/php_intl.dll' - dlopen(/usr/local/lib/php/pecl/20160303/php_intl.dll, 9): image not found in Unknown on line 0 -``` - -Next, the MFTF returns `DEPRECATION` reports alerting you that required test components are missing in XML test definitions. +First, MFTF returns `DEPRECATION` warnings alerting you that required test components are missing in XML test definitions. ```terminal DEPRECATION: Test AdminFilteringCategoryProductsUsingScopeSelectorTest is missing required annotations.{"testName":"AdminFilteringCategoryProductsUsingScopeSelectorTest","missingAnnotations":"stories"} @@ -51,7 +44,7 @@ DEPRECATION: Test AdminRemoveProductWeeeAttributeOptionTest is missing required Generate Tests Command Run ``` -`Generate Tests Command Run` indicates the moment when the MFTF has run the tests generation command actually. +`Generate Tests Command Run` indicates that test generation is finished and tests are able to be executed. ### Test execution report @@ -72,7 +65,7 @@ Magento\FunctionalTestingFramework.functional Tests (2) ------------------------ Modules: \Magento\FunctionalTestingFramework\Module\MagentoWebDriver, \Magento\FunctionalTestingFramework\Helper\Acceptance, \Magento\FunctionalTestingFramework\Helper\MagentoFakerData, \Magento\FunctionalTestingFramework\Module\MagentoRestDriver, PhpBrowser, \Magento\FunctionalTestingFramework\Module\MagentoSequence, \Magento\FunctionalTes ``` -After the test generation command (mentioned in the previous section), the MFTF delegates control to the `vendor/codeception` tool, which is the `Codeception PHP Testing Framework` of version `2.3.9` that uses `PHPUnit` of version `6.5.13`. +After the test generation command (mentioned in the previous section), MFTF delegates control to the `vendor/codeception` tool, which is the `Codeception PHP Testing Framework` of version `2.3.9` that uses `PHPUnit` of version `6.5.13`. The tool runs `2 Tests` using the configuration defined in the `functional` suite under the `Magento\FunctionalTestingFramework` namespace. The corresponding configuration file is `acceptance/tests/functional.suite.yml`. @@ -154,7 +147,7 @@ I save screenshot FAIL ``` -When a test step fails, the MFTF always saves a screenshot of the web page with the failing state immediately after the failure occurs. +When a test step fails, MFTF always saves a screenshot of the web page with the failing state immediately after the failure occurs. `I save screenshot` follows the failing test step `I see "#something"` in our case. A screenshot of the fail goes at the `acceptance/tests/_output` directory in both PNG and HTML formats: @@ -176,7 +169,7 @@ Actions after `FAIL` are run as a part of the [`after`][] hook of the test. ### Test result report -After the MFTF completed test execution, it generates a general report about test results along with detailed information about each fail. +After MFTF completed test execution, it generates a general report about test results along with detailed information about each fail. ```terminal -------------------------------------------------------------------------------- @@ -192,7 +185,7 @@ First you see warnings and deprecations. The `DEPRECATION` here is thrown by an MFTF dependency (Symfony) that is out of the scope for test writers and should be considered by MFTF contributors. If you encounter this type of reporting, [report an issue][]. -Then, the MFTF reports that the test run took 52.43 seconds using 16 MB of system RAM. +Then, MFTF reports that the test run took 52.43 seconds using 16 MB of system RAM. And, finally, that there was `1 failure`. Next, the report provides details about the test failure. @@ -268,12 +261,12 @@ FAILURES! Tests: 2, Assertions: 3, Failures: 1. ``` -The MFTF encountered failures due to the last test run, that included *2* tests with *3* assertions. +MFTF encountered failures due to the last test run, that included *2* tests with *3* assertions. *1* assertion fails. ## Allure -Each time you run tests, the MFTF appends an XML file with results at the `tests/_output/allure-results/` directory. +Each time you run tests, MFTF appends an XML file with results at the `tests/_output/allure-results/` directory. The official [Allure Test Report][] documentation is well-covered, so we'll list only the CLI commands that you would need for your day-to-day work. @@ -281,6 +274,12 @@ The official [Allure Test Report][] documentation is well-covered, so we'll list The following commands are relative to the Magento installation directory. +To generate the HTML Allure report in a temporary folder and open the report in your default web browser: + +```bash +allure serve dev/tests/acceptance/tests/_output/allure-results/ +``` + To generate a report to the `allure-report/` at the current directory: ```bash @@ -334,12 +333,6 @@ To clean up existing reports before generation (for example after getting new re allure generate dev/tests/acceptance/tests/_output/allure-result --clean ``` -To generate the HTML Allure report in a temporary folder and open the report in your default web browser: - -```bash -allure serve dev/tests/acceptance/tests/_output/allure-results/ -``` - Refer to the [Reporting section][] for more Allure CLI details. From 69101eff1723ab2e5360bddb557212ce7fa8b48b Mon Sep 17 00:00:00 2001 From: Tom Reece Date: Tue, 3 Sep 2019 14:00:13 -0500 Subject: [PATCH 56/82] MQE-1626: bin/mftf run:manifest - Add check for input path exists --- .../Console/RunManifestCommand.php | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/Magento/FunctionalTestingFramework/Console/RunManifestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunManifestCommand.php index 9ef59e296..e487d16cf 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunManifestCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunManifestCommand.php @@ -7,6 +7,7 @@ namespace Magento\FunctionalTestingFramework\Console; +use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputArgument; use Symfony\Component\Console\Input\InputInterface; @@ -47,11 +48,18 @@ protected function configure() * * @param InputInterface $input * @param OutputInterface $output + * @throws TestFrameworkException * @return integer */ protected function execute(InputInterface $input, OutputInterface $output): int { - $manifestFile = file($input->getArgument("path"), FILE_IGNORE_NEW_LINES); + $path = $input->getArgument("path"); + + if (!file_exists($path)) { + throw new TestFrameworkException("Could not find file $path. Check the path and try again."); + } + + $manifestFile = file($path, FILE_IGNORE_NEW_LINES); // Delete the Codeception failed file just in case it exists from any previous test runs $this->deleteFailedFile(); From a344d34128305238b5ad2ac51842728e0aac60b8 Mon Sep 17 00:00:00 2001 From: Ji Lu Date: Wed, 4 Sep 2019 10:30:05 -0500 Subject: [PATCH 57/82] MQE-588: CreatedData should throw Error/Warning when undefined data is returned - prevent printing in unit phase --- .../DataGenerator/Handlers/PersistedObjectHandler.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/PersistedObjectHandler.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/PersistedObjectHandler.php index 3845e52e9..ea3aa919a 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/PersistedObjectHandler.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/PersistedObjectHandler.php @@ -10,6 +10,7 @@ use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; use Magento\FunctionalTestingFramework\Exceptions\TestReferenceException; use Magento\FunctionalTestingFramework\Util\Logger\LoggingUtil; +use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig; class PersistedObjectHandler { @@ -188,7 +189,9 @@ public function retrieveEntityField($stepKey, $field, $scope) $warnMsg .= "Please fix the invalid reference. This will result in fatal error in next major release."; //TODO: change this to throw an exception in next major release LoggingUtil::getInstance()->getLogger(PersistedObjectHandler::class)->warn($warnMsg); - print($warnMsg . PHP_EOL); + if (MftfApplicationConfig::getConfig()->getPhase() !== MftfApplicationConfig::UNIT_TEST_PHASE) { + print("\n$warnMsg\n"); + } } return $fieldValue; } From d8a8afedee26a163f69d21f6ab9572d5212bfc3a Mon Sep 17 00:00:00 2001 From: Tom Reece Date: Wed, 4 Sep 2019 11:28:07 -0500 Subject: [PATCH 58/82] MQE-1692: Improve makeScreenshot Action to Add Screenshot to Allure Report - Initial implementation --- .../Module/MagentoWebDriver.php | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php index 731643eaf..5ab5cc1ca 100644 --- a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php +++ b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php @@ -19,6 +19,7 @@ use Magento\FunctionalTestingFramework\Util\Protocol\CurlTransport; use Magento\FunctionalTestingFramework\Util\Protocol\CurlInterface; use Magento\FunctionalTestingFramework\Util\ConfigSanitizerUtil; +use Yandex\Allure\Adapter\AllureException; use Yandex\Allure\Adapter\Support\AttachmentSupport; use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; @@ -196,6 +197,7 @@ public function _getCurrentUri() * * @param string $url * @return void + * @throws AllureException */ public function dontSeeCurrentUrlEquals($url) { @@ -210,6 +212,7 @@ public function dontSeeCurrentUrlEquals($url) * * @param string $regex * @return void + * @throws AllureException */ public function dontSeeCurrentUrlMatches($regex) { @@ -224,6 +227,7 @@ public function dontSeeCurrentUrlMatches($regex) * * @param string $needle * @return void + * @throws AllureException */ public function dontSeeInCurrentUrl($needle) { @@ -261,6 +265,7 @@ public function grabFromCurrentUrl($regex = null) * * @param string $url * @return void + * @throws AllureException */ public function seeCurrentUrlEquals($url) { @@ -275,6 +280,7 @@ public function seeCurrentUrlEquals($url) * * @param string $regex * @return void + * @throws AllureException */ public function seeCurrentUrlMatches($regex) { @@ -289,6 +295,7 @@ public function seeCurrentUrlMatches($regex) * * @param string $needle * @return void + * @throws AllureException */ public function seeInCurrentUrl($needle) { @@ -645,6 +652,7 @@ public function dragAndDrop($source, $target, $xOffset = null, $yOffset = null) * @param string $field * @param string $value * @return void + * @throws TestFrameworkException */ public function fillSecretField($field, $value) { @@ -791,4 +799,28 @@ public function dontSeeJsError() { $this->assertEmpty($this->jsErrors, $this->getJsErrors()); } + + /** + * Takes a screenshot of the current window and saves it to `tests/_output/debug`. + * + * This function is copied over from the original Codeception WebDriver so that we still have visibility of + * the screenshot filename to be passed to the AllureHelper. + * + * @param string $name + * @throws AllureException + */ + public function makeScreenshot($name = null) + { + if (empty($name)) { + $name = uniqid(date("Y-m-d_H-i-s_")); + } + $debugDir = codecept_log_dir() . 'debug'; + if (!is_dir($debugDir)) { + mkdir($debugDir, 0777); + } + $screenName = $debugDir . DIRECTORY_SEPARATOR . $name . '.png'; + $this->_saveScreenshot($screenName); + $this->debug("Screenshot saved to $screenName"); + AllureHelper::addAttachmentToCurrentStep($screenName, 'Screenshot'); + } } From 7cce52054c07df8aee8cb8969f424811bb7244b1 Mon Sep 17 00:00:00 2001 From: Tom Reece Date: Wed, 4 Sep 2019 12:57:33 -0500 Subject: [PATCH 59/82] MQE-1692: Improve makeScreenshot Action to Add Screenshot to Allure Report - Add missing phpdoc @return --- .../FunctionalTestingFramework/Module/MagentoWebDriver.php | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php index 5ab5cc1ca..ac6e86c0a 100644 --- a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php +++ b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php @@ -807,6 +807,7 @@ public function dontSeeJsError() * the screenshot filename to be passed to the AllureHelper. * * @param string $name + * @return void * @throws AllureException */ public function makeScreenshot($name = null) From f353afe16cf002135973208ed52f682c5dfd59f2 Mon Sep 17 00:00:00 2001 From: Soumya Unnikrishnan Date: Wed, 4 Sep 2019 13:53:57 -0500 Subject: [PATCH 60/82] MQE-1703: Implicit Suite Generation for Tests fixing bug with test generation --- .../Console/BaseGenerateCommand.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php index 1cfa34e5a..61bd4cb66 100644 --- a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php @@ -78,8 +78,8 @@ protected function removeGeneratedDirectory(OutputInterface $output, bool $verbo protected function getTestAndSuiteConfiguration(array $tests) { - $testConfiguration['tests'] = []; - $testConfiguration['suites'] = []; + $testConfiguration['tests'] = null; + $testConfiguration['suites'] = null; $testsReferencedInSuites = SuiteObjectHandler::getInstance()->getAllTestReferences(); $resolvedTests = []; From 25c57dcbfbce1a9e90cd2521d5be45f7f3240ff3 Mon Sep 17 00:00:00 2001 From: Ji Lu Date: Wed, 4 Sep 2019 16:27:15 -0500 Subject: [PATCH 61/82] MQE-588: CreatedData should throw Error/Warning when undefined data is returned - prevent printing in unit phase --- .../DataGenerator/Handlers/PersistedObjectHandlerTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/PersistedObjectHandlerTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/PersistedObjectHandlerTest.php index c79ad322a..6c3823466 100644 --- a/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/PersistedObjectHandlerTest.php +++ b/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/PersistedObjectHandlerTest.php @@ -433,7 +433,7 @@ public function testRetrieveEntityInValidField($name, $key, $value, $type, $scop // validate log statement TestLoggingUtil::getInstance()->validateMockLogStatement( - "warning", + 'warning', $warnMsg, [] ); From 9a91579deceeec43a33503d5f7750f3d92701f2b Mon Sep 17 00:00:00 2001 From: Ji Lu Date: Thu, 5 Sep 2019 10:21:57 -0500 Subject: [PATCH 62/82] MQE-1729: Added document for PR #377 --- docs/metadata.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/metadata.md b/docs/metadata.md index 87f2da7e0..f1e4e1fe2 100644 --- a/docs/metadata.md +++ b/docs/metadata.md @@ -25,7 +25,7 @@ This directory contains XML files with metadata required to create a valid reque A metadata file contains a list of operations with different types (defined in `type`). Each [operation] includes: -- The set of adjustments for processing a request in [attributes][operation], and in some cases, a response (see `successRegex` and `returnRegex` in [reference details][operation]). +- The set of adjustments for processing a request in [attributes][operation], and in some cases, a response (see `successRegex`, `returnRegex` and `returnIndex` in [reference details][operation]). - The type of body content encoding in [contentType]. - The body of the request represented as a tree of objects, arrays, and fields. @@ -336,7 +336,7 @@ There are two different attributes to split access to different areas: - `auth="adminFormKey"` is used for objects in an Admin area. - `auth="customerFormKey"` is used for objects in a storefront. -You are able to create assurances with `successRegex`, and even return values with `returnRegex`. +You are able to create assurances with `successRegex`, and even optionally return values with `returnRegex`. You can also use `returnIndex` when `returnRegex` matches multiple values. ### Create an object in Admin {#create-object-as-adminFormKey} @@ -418,6 +418,7 @@ Root element that points to the corresponding XML Schema. | `method` | string | optional | HTTP method of the operation. Possible values: `POST`, `DELETE`, `PUT`, `GET`. | | `successRegex` | string | optional | Determines if the operation was successful. Parses the HTML body in response and asserts if the value assigned to the `successRegex` exists. | | `returnRegex` | string | optional | Determines if the response contains the matching value to return. | +| `returnIndex` | string | optional | Specifies index at which the value will be returned when `returnRegex` matches multiple values | | `removeBackend` | boolean | optional | Removes backend name from requested URL. Applicable when `auth="adminFormKey"`. | | `filename` | string | optional | | From 4a519fe0bbedfadf4bddfc3434ce371e2da04744 Mon Sep 17 00:00:00 2001 From: Soumya Unnikrishnan Date: Thu, 5 Sep 2019 10:21:59 -0500 Subject: [PATCH 63/82] MQE-1703: Implicit Suite Generation for Tests Style fixes --- .../Console/BaseGenerateCommand.php | 19 +++++++++---------- .../Console/RunTestCommand.php | 12 +++++------- 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php index 61bd4cb66..40582db86 100644 --- a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php @@ -81,24 +81,23 @@ protected function getTestAndSuiteConfiguration(array $tests) $testConfiguration['tests'] = null; $testConfiguration['suites'] = null; $testsReferencedInSuites = SuiteObjectHandler::getInstance()->getAllTestReferences(); - $resolvedTests = []; + $suiteToTestPair = []; foreach($tests as $test) { if (array_key_exists($test, $testsReferencedInSuites)) { $suites = $testsReferencedInSuites[$test]; - $resolvedTests = array_merge( - $resolvedTests, - array_map(function ($value) use ($test) { - return $value . ':' . $test; - }, $suites) - ); + foreach ($suites as $suite) { + $suiteToTestPair[] = "$suite:$test"; + } } // configuration for tests - else $testConfiguration['tests'][] = $test; + else { + $testConfiguration['tests'][] = $test; + } } // configuration for suites - foreach ($resolvedTests as $test) { - list($suite, $test) = explode(":", $test); + foreach ($suiteToTestPair as $pair) { + list($suite, $test) = explode(":", $pair); $testConfiguration['suites'][$suite][] = $test; } $testConfigurationJson = json_encode($testConfiguration); diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php index 0a355e798..cc23300c4 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php @@ -112,7 +112,8 @@ function ($type, $buffer) use ($output) { return $returnCode; } - /** Get an array of tests with resolved suite references from $testConfiguration + /** + * Get an array of tests with resolved suite references from $testConfiguration * eg: if test is referenced in a suite, it'll be stored in format suite:test * @param string $testConfigurationJson * @return array @@ -125,12 +126,9 @@ private function resolveSuiteReferences($testConfigurationJson) $testArrayBuilder = []; foreach ($suitesArray as $suite => $tests) { - $testArrayBuilder = array_merge( - $testArrayBuilder, - array_map(function ($test) use ($suite) { - return $suite . ':' . $test; - }, $tests) - ); + foreach($tests as $test){ + $testArrayBuilder[] = "$suite:$test"; + } } return array_merge($testArrayBuilder, $testsArray); } From 5f81deb45497c4c9f2bb3f7f93856e47bf078f42 Mon Sep 17 00:00:00 2001 From: Soumya Unnikrishnan Date: Thu, 5 Sep 2019 10:32:24 -0500 Subject: [PATCH 64/82] MQE-1703: Implicit Suite Generation for Tests style fixes --- .../FunctionalTestingFramework/Console/RunTestCommand.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php index cc23300c4..e1e30589d 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php @@ -126,7 +126,7 @@ private function resolveSuiteReferences($testConfigurationJson) $testArrayBuilder = []; foreach ($suitesArray as $suite => $tests) { - foreach($tests as $test){ + foreach ($tests as $test) { $testArrayBuilder[] = "$suite:$test"; } } From 55c30aca68af3ea73e72054b6bf81669c2155133 Mon Sep 17 00:00:00 2001 From: Donald Booth Date: Fri, 6 Sep 2019 10:17:12 -0500 Subject: [PATCH 65/82] Small grammar change. --- docs/metadata.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/metadata.md b/docs/metadata.md index f1e4e1fe2..91ecbe2e9 100644 --- a/docs/metadata.md +++ b/docs/metadata.md @@ -336,7 +336,7 @@ There are two different attributes to split access to different areas: - `auth="adminFormKey"` is used for objects in an Admin area. - `auth="customerFormKey"` is used for objects in a storefront. -You are able to create assurances with `successRegex`, and even optionally return values with `returnRegex`. You can also use `returnIndex` when `returnRegex` matches multiple values. +You are able to create assurances with `successRegex`, and, optionally, return values with `returnRegex`. You can also use `returnIndex` when `returnRegex` matches multiple values. ### Create an object in Admin {#create-object-as-adminFormKey} From d1c1914abe63630751752316de5910ac64fdd582 Mon Sep 17 00:00:00 2001 From: Ji Lu Date: Tue, 10 Sep 2019 15:03:54 -0500 Subject: [PATCH 66/82] MQE-1753: Uncaught ArgumentCountError when running bin/mftf static-checks - bug fix --- .../StaticCheck/TestDependencyCheck.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Magento/FunctionalTestingFramework/StaticCheck/TestDependencyCheck.php b/src/Magento/FunctionalTestingFramework/StaticCheck/TestDependencyCheck.php index a891d42e1..3abbd40c2 100644 --- a/src/Magento/FunctionalTestingFramework/StaticCheck/TestDependencyCheck.php +++ b/src/Magento/FunctionalTestingFramework/StaticCheck/TestDependencyCheck.php @@ -88,7 +88,8 @@ public function execute(InputInterface $input) true, MftfApplicationConfig::UNIT_TEST_PHASE, false, - MftfApplicationConfig::LEVEL_NONE + MftfApplicationConfig::LEVEL_NONE, + true ); ModuleResolver::getInstance()->getModulesPath(); From 72222595abbecb06b6aa0c0e2cabe22b19f7493f Mon Sep 17 00:00:00 2001 From: Kevin Kozan Date: Tue, 10 Sep 2019 15:11:42 -0500 Subject: [PATCH 67/82] MQE-1510 (#446) --- etc/config/command.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/config/command.php b/etc/config/command.php index f018e3014..047af324a 100644 --- a/etc/config/command.php +++ b/etc/config/command.php @@ -12,8 +12,8 @@ $tokenModel = $magentoObjectManager->get(\Magento\Integration\Model\Oauth\Token::class); $tokenPassedIn = urldecode($_POST['token']); - $command = str_replace([';', '&', '|'], '', urldecode($_POST['command'])); - $arguments = str_replace([';', '&', '|'], '', urldecode($_POST['arguments'])); + $command = urldecode($_POST['command']); + $arguments = urldecode($_POST['arguments']); // Token returned will be null if the token we passed in is invalid $tokenFromMagento = $tokenModel->loadByToken($tokenPassedIn)->getToken(); From fa579d1fa07806259d995310439e0e5b4a8215d5 Mon Sep 17 00:00:00 2001 From: Soumya Unnikrishnan Date: Tue, 10 Sep 2019 16:32:33 -0500 Subject: [PATCH 68/82] MQE-1755: mftf run:test Test1 Test2 does not run before/after hooks correctly --- .../Console/RunTestCommand.php | 104 +++++++++++------- 1 file changed, 64 insertions(+), 40 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php index e1e30589d..10f3defc6 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php @@ -19,6 +19,13 @@ class RunTestCommand extends BaseGenerateCommand { + /** + * The return code. Determined by all tests that run. + * + * @var integer + */ + private $returnCode = 0; + /** * Configures the current command. * @@ -76,60 +83,77 @@ protected function execute(InputInterface $input, OutputInterface $output): int ]; $command->run(new ArrayInput($args), $output); } - // tests with resolved suite references - $resolvedTests = $this->resolveSuiteReferences($testConfiguration); + $testConfigArray = json_decode($testConfiguration, true); + + // run tests not referenced in suites + $this->runTests($testConfigArray['tests'], $output); + + // run tests in suites + $this->runTestsInSuite($testConfigArray['suites'], $output); + + return $this->returnCode; + + } + + /** + * Run tests not referenced in suites + * @param array $testsConfig + * @param OutputInterface $output + * @throws TestFrameworkException + */ + private function runTests($testsConfig, OutputInterface $output) { + + + $tests = $testsConfig ?? []; $codeceptionCommand = realpath(PROJECT_ROOT . '/vendor/bin/codecept') . ' run functional '; - $testsDirectory = TESTS_MODULE_PATH . DIRECTORY_SEPARATOR . TestGenerator::GENERATED_DIR . DIRECTORY_SEPARATOR; - $returnCode = 0; - //execute only tests specified as arguments in run command - foreach ($resolvedTests as $test) { - //set directory as suite name for tests in suite, if not set to "default" - if (strpos($test, ':')) { - list($testGroup, $testName) = explode(":", $test); - } else { - list($testGroup, $testName) = [TestGenerator::DEFAULT_DIR, $test]; - } - $testGroup = $testGroup . DIRECTORY_SEPARATOR; - $testName = $testName . 'Cest.php'; - if (!realpath($testsDirectory . $testGroup . $testName)) { + $testsDirectory = TESTS_MODULE_PATH . + DIRECTORY_SEPARATOR . + TestGenerator::GENERATED_DIR . + DIRECTORY_SEPARATOR . + TestGenerator::DEFAULT_DIR . + DIRECTORY_SEPARATOR ; + + foreach ($tests as $test) { + $testName = $test . 'Cest.php'; + if (!realpath($testsDirectory . $testName)) { throw new TestFrameworkException( - $testName . " is not available under " . $testsDirectory . $testGroup + $testName . " is not available under " . $testsDirectory ); } - $fullCommand = $codeceptionCommand . $testsDirectory . $testGroup . $testName . ' --verbose --steps'; + $fullCommand = $codeceptionCommand . $testsDirectory . $testName . ' --verbose --steps'; $process = new Process($fullCommand); $process->setWorkingDirectory(TESTS_BP); $process->setIdleTimeout(600); $process->setTimeout(0); - - $returnCode = max($returnCode, $process->run( - function ($type, $buffer) use ($output) { - $output->write($buffer); - } - )); + $subReturnCode = $process->run(function ($type, $buffer) use ($output) { + $output->write($buffer); + }); + $this->returnCode = max($this->returnCode, $subReturnCode); } - return $returnCode; } /** - * Get an array of tests with resolved suite references from $testConfiguration - * eg: if test is referenced in a suite, it'll be stored in format suite:test - * @param string $testConfigurationJson - * @return array + * Run tests referenced in suites within suites' context. + * @param array $suitesConfig + * @param OutputInterface $output */ - private function resolveSuiteReferences($testConfigurationJson) - { - $testConfiguration = json_decode($testConfigurationJson, true); - $testsArray = $testConfiguration['tests'] ?? []; - $suitesArray = $testConfiguration['suites'] ?? []; - $testArrayBuilder = []; - - foreach ($suitesArray as $suite => $tests) { - foreach ($tests as $test) { - $testArrayBuilder[] = "$suite:$test"; - } + private function runTestsInSuite($suitesConfig, OutputInterface $output) { + + $suites = $suitesConfig ?? []; + $codeceptionCommand = realpath(PROJECT_ROOT . '/vendor/bin/codecept') . ' run functional --verbose --steps '; + $testGroups = array_keys($suites); + //for tests in suites, run them as a group to run before and after block + foreach ($testGroups as $testGroup) { + $fullCommand = $codeceptionCommand . " -g {$testGroup}"; + $process = new Process($fullCommand); + $process->setWorkingDirectory(TESTS_BP); + $process->setIdleTimeout(600); + $process->setTimeout(0); + $subReturnCode = $process->run(function ($type, $buffer) use ($output) { + $output->write($buffer); + }); + $this->returnCode = max($this->returnCode, $subReturnCode); } - return array_merge($testArrayBuilder, $testsArray); } } From 3b84f12a963a789bded006226d0dfd46463de279 Mon Sep 17 00:00:00 2001 From: Soumya Unnikrishnan Date: Tue, 10 Sep 2019 19:08:02 -0500 Subject: [PATCH 69/82] MQE-1755: mftf run:test Test1 Test2 does not run before/after hooks correctly fixed unit tests --- .../Console/RunTestCommand.php | 39 ++++++++++--------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php index 10f3defc6..668d5152c 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php @@ -51,8 +51,6 @@ protected function configure() * @param OutputInterface $output * @return integer * @throws \Exception - * - * @SuppressWarnings(PHPMD.UnusedLocalVariable) */ protected function execute(InputInterface $input, OutputInterface $output): int { @@ -86,26 +84,29 @@ protected function execute(InputInterface $input, OutputInterface $output): int $testConfigArray = json_decode($testConfiguration, true); - // run tests not referenced in suites - $this->runTests($testConfigArray['tests'], $output); + if (isset($testConfigArray['tests'])) { + $this->runTests($testConfigArray['tests'], $output); + } - // run tests in suites - $this->runTestsInSuite($testConfigArray['suites'], $output); + if (isset($testConfigArray['suites'])) { + $this->runTestsInSuite($testConfigArray['suites'], $output); + } return $this->returnCode; - } /** * Run tests not referenced in suites - * @param array $testsConfig + * + * @param array $tests * @param OutputInterface $output + * @return void * @throws TestFrameworkException + * + * @SuppressWarnings(PHPMD.UnusedLocalVariable) */ - private function runTests($testsConfig, OutputInterface $output) { - - - $tests = $testsConfig ?? []; + private function runTests(array $tests, OutputInterface $output) + { $codeceptionCommand = realpath(PROJECT_ROOT . '/vendor/bin/codecept') . ' run functional '; $testsDirectory = TESTS_MODULE_PATH . DIRECTORY_SEPARATOR . @@ -135,17 +136,19 @@ private function runTests($testsConfig, OutputInterface $output) { /** * Run tests referenced in suites within suites' context. + * * @param array $suitesConfig * @param OutputInterface $output + * @return void + * + * @SuppressWarnings(PHPMD.UnusedLocalVariable) */ - private function runTestsInSuite($suitesConfig, OutputInterface $output) { - - $suites = $suitesConfig ?? []; + private function runTestsInSuite(array $suitesConfig, OutputInterface $output) + { $codeceptionCommand = realpath(PROJECT_ROOT . '/vendor/bin/codecept') . ' run functional --verbose --steps '; - $testGroups = array_keys($suites); //for tests in suites, run them as a group to run before and after block - foreach ($testGroups as $testGroup) { - $fullCommand = $codeceptionCommand . " -g {$testGroup}"; + foreach (array_keys($suitesConfig) as $suite) { + $fullCommand = $codeceptionCommand . " -g {$suite}"; $process = new Process($fullCommand); $process->setWorkingDirectory(TESTS_BP); $process->setIdleTimeout(600); From 8597fcb8313a022f83c9ff47a20dd9c421fbdb4c Mon Sep 17 00:00:00 2001 From: Soumya Unnikrishnan Date: Tue, 10 Sep 2019 19:33:09 -0500 Subject: [PATCH 70/82] MQE-1755: mftf run:test Test1 Test2 does not run before/after hooks correctly fix unit tests --- .../FunctionalTestingFramework/Console/RunTestCommand.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php index 668d5152c..dc763b9b0 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php @@ -98,7 +98,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int /** * Run tests not referenced in suites * - * @param array $tests + * @param array $tests * @param OutputInterface $output * @return void * @throws TestFrameworkException @@ -137,7 +137,7 @@ private function runTests(array $tests, OutputInterface $output) /** * Run tests referenced in suites within suites' context. * - * @param array $suitesConfig + * @param array $suitesConfig * @param OutputInterface $output * @return void * From f572a7a0f91d403ff40c1e71ac42e0421dff5975 Mon Sep 17 00:00:00 2001 From: Ji Lu Date: Wed, 11 Sep 2019 11:47:03 -0500 Subject: [PATCH 71/82] MQE-1754: mftf generate:tests AdminLoginTest -f does not obey the -f flag - set MftfApplicationConfig for generate and run console commands --- .../Config/MftfApplicationConfig.php | 9 +++++-- .../Console/GenerateDocsCommand.php | 3 ++- .../Console/GenerateSuiteCommand.php | 13 ++++++++++ .../Console/GenerateTestsCommand.php | 26 ++++++++----------- .../Console/RunTestCommand.php | 20 ++++++++++++-- .../Console/RunTestFailedCommand.php | 9 ++++--- .../Console/RunTestGroupCommand.php | 8 +++--- 7 files changed, 62 insertions(+), 26 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php b/src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php index 6faf322bc..80db27de0 100644 --- a/src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php +++ b/src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php @@ -115,8 +115,13 @@ private function __construct( * @return void * @throws TestFrameworkException */ - public static function create($forceGenerate, $phase, $verboseEnabled, $debugLevel, $allowSkipped) - { + public static function create( + $forceGenerate = false, + $phase = self::EXECUTION_PHASE, + $verboseEnabled = null, + $debugLevel = self::LEVEL_NONE, + $allowSkipped = false + ) { if (self::$MFTF_APPLICATION_CONTEXT == null) { self::$MFTF_APPLICATION_CONTEXT = new MftfApplicationConfig($forceGenerate, $phase, $verboseEnabled, $debugLevel, $allowSkipped); diff --git a/src/Magento/FunctionalTestingFramework/Console/GenerateDocsCommand.php b/src/Magento/FunctionalTestingFramework/Console/GenerateDocsCommand.php index 4ed2b6b29..7def3894f 100644 --- a/src/Magento/FunctionalTestingFramework/Console/GenerateDocsCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/GenerateDocsCommand.php @@ -67,7 +67,8 @@ protected function execute(InputInterface $input, OutputInterface $output) $force, MftfApplicationConfig::GENERATION_PHASE, false, - MftfApplicationConfig::LEVEL_NONE + MftfApplicationConfig::LEVEL_NONE, + true ); $allActionGroups = ActionGroupObjectHandler::getInstance()->getAllObjects(); diff --git a/src/Magento/FunctionalTestingFramework/Console/GenerateSuiteCommand.php b/src/Magento/FunctionalTestingFramework/Console/GenerateSuiteCommand.php index 815133e4d..7dd3b8cb4 100644 --- a/src/Magento/FunctionalTestingFramework/Console/GenerateSuiteCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/GenerateSuiteCommand.php @@ -42,7 +42,20 @@ protected function configure() */ protected function execute(InputInterface $input, OutputInterface $output) { + $force = $input->getOption('force'); + $debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility $remove = $input->getOption('remove'); + $verbose = $output->isVerbose(); + $allowSkipped = $input->getOption('allowSkipped'); + + // Set application configuration so we can references the user options in our framework + MftfApplicationConfig::create( + $force, + MftfApplicationConfig::GENERATION_PHASE, + $verbose, + $debug, + $allowSkipped + ); // Remove previous GENERATED_DIR if --remove option is used if ($remove) { diff --git a/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php b/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php index c410973c7..6f73e38a1 100644 --- a/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php @@ -73,6 +73,15 @@ protected function execute(InputInterface $input, OutputInterface $output) $verbose = $output->isVerbose(); $allowSkipped = $input->getOption('allowSkipped'); + // Set application configuration so we can references the user options in our framework + MftfApplicationConfig::create( + $force, + MftfApplicationConfig::GENERATION_PHASE, + $verbose, + $debug, + $allowSkipped + ); + if (!empty($tests)) { $json = $this->getTestAndSuiteConfiguration($tests); } @@ -93,7 +102,7 @@ protected function execute(InputInterface $input, OutputInterface $output) ($debug !== MftfApplicationConfig::LEVEL_NONE)); } - $testConfiguration = $this->createTestConfiguration($json, $tests, $force, $debug, $verbose, $allowSkipped); + $testConfiguration = $this->createTestConfiguration($json, $tests); // create our manifest file here $testManifest = TestManifestFactory::makeManifest($config, $testConfiguration['suites']); @@ -126,21 +135,8 @@ protected function execute(InputInterface $input, OutputInterface $output) */ private function createTestConfiguration( $json, - array $tests, - bool $force, - string $debug, - bool $verbose, - bool $allowSkipped + array $tests ) { - // set our application configuration so we can references the user options in our framework - MftfApplicationConfig::create( - $force, - MftfApplicationConfig::GENERATION_PHASE, - $verbose, - $debug, - $allowSkipped - ); - $testConfiguration = []; $testConfiguration['tests'] = $tests; $testConfiguration['suites'] = []; diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php index e1e30589d..b83953dce 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php @@ -32,7 +32,12 @@ protected function configure() 'name', InputArgument::REQUIRED | InputArgument::IS_ARRAY, "name of tests to generate and execute" - )->addOption('skip-generate', 'k', InputOption::VALUE_NONE, "skip generation and execute existing test"); + )->addOption( + 'skip-generate', + 'k', + InputOption::VALUE_NONE, + "skip generation and execute existing test" + ); parent::configure(); } @@ -55,6 +60,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int $remove = $input->getOption('remove'); $debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility $allowSkipped = $input->getOption('allowSkipped'); + $verbose = $output->isVerbose(); if ($skipGeneration and $remove) { // "skip-generate" and "remove" options cannot be used at the same time @@ -63,6 +69,15 @@ protected function execute(InputInterface $input, OutputInterface $output): int ); } + // Set application configuration so we can references the user options in our framework + MftfApplicationConfig::create( + $force, + MftfApplicationConfig::EXECUTION_PHASE, + $verbose, + $debug, + $allowSkipped + ); + $testConfiguration = $this->getTestAndSuiteConfiguration($tests); if (!$skipGeneration) { @@ -72,7 +87,8 @@ protected function execute(InputInterface $input, OutputInterface $output): int '--force' => $force, '--remove' => $remove, '--debug' => $debug, - '--allowSkipped' => $allowSkipped + '--allowSkipped' => $allowSkipped, + '-v' => $verbose ]; $command->run(new ArrayInput($args), $output); } diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php index ed00322ff..15ba723e6 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php @@ -71,12 +71,13 @@ protected function execute(InputInterface $input, OutputInterface $output): int $force = $input->getOption('force'); $debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility $allowSkipped = $input->getOption('allowSkipped'); + $verbose = $output->isVerbose(); // Create Mftf Configuration MftfApplicationConfig::create( $force, - MftfApplicationConfig::GENERATION_PHASE, - false, + MftfApplicationConfig::EXECUTION_PHASE, + $verbose, $debug, $allowSkipped ); @@ -91,9 +92,11 @@ protected function execute(InputInterface $input, OutputInterface $output): int $command = $this->getApplication()->find('generate:tests'); $args = [ '--tests' => $testConfiguration, + '--force' => $force, '--remove' => true, '--debug' => $debug, - '--allowSkipped' => $allowSkipped + '--allowSkipped' => $allowSkipped, + '-v' => $verbose ]; $command->run(new ArrayInput($args), $output); diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php index 81a6b864f..20d290af9 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php @@ -61,6 +61,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int $remove = $input->getOption('remove'); $debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility $allowSkipped = $input->getOption('allowSkipped'); + $verbose = $output->isVerbose(); if ($skipGeneration and $remove) { // "skip-generate" and "remove" options cannot be used at the same time @@ -72,8 +73,8 @@ protected function execute(InputInterface $input, OutputInterface $output): int // Create Mftf Configuration MftfApplicationConfig::create( $force, - MftfApplicationConfig::GENERATION_PHASE, - false, + MftfApplicationConfig::EXECUTION_PHASE, + $verbose, $debug, $allowSkipped ); @@ -86,7 +87,8 @@ protected function execute(InputInterface $input, OutputInterface $output): int '--force' => $force, '--remove' => $remove, '--debug' => $debug, - '--allowSkipped' => $allowSkipped + '--allowSkipped' => $allowSkipped, + '-v' => $verbose ]; $command->run(new ArrayInput($args), $output); From 319001b2e341380dd20df12d503cb09ac6a7aead Mon Sep 17 00:00:00 2001 From: Ji Lu Date: Wed, 11 Sep 2019 13:37:44 -0500 Subject: [PATCH 72/82] MQE-1754: mftf generate:tests AdminLoginTest -f does not obey the -f flag - set MftfApplicationConfig for generate and run console commands --- .../Console/GenerateTestsCommand.php | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php b/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php index 6f73e38a1..d70aaaf3a 100644 --- a/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php @@ -125,10 +125,6 @@ protected function execute(InputInterface $input, OutputInterface $output) * * @param string $json * @param array $tests - * @param boolean $force - * @param string $debug - * @param boolean $verbose - * @param boolean $allowSkipped * @return array * @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException * @throws \Magento\FunctionalTestingFramework\Exceptions\XmlException From 9cfeb9c71b573482f18ce5f51ebe342bdd861e90 Mon Sep 17 00:00:00 2001 From: Soumya Unnikrishnan Date: Wed, 11 Sep 2019 13:42:23 -0500 Subject: [PATCH 73/82] MQE-1755: mftf run:test Test1 Test2 does not run before/after hooks correctly --- .../Console/RunTestCommand.php | 42 ++++++++++--------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php index dc763b9b0..b65409b70 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php @@ -102,8 +102,6 @@ protected function execute(InputInterface $input, OutputInterface $output): int * @param OutputInterface $output * @return void * @throws TestFrameworkException - * - * @SuppressWarnings(PHPMD.UnusedLocalVariable) */ private function runTests(array $tests, OutputInterface $output) { @@ -123,14 +121,7 @@ private function runTests(array $tests, OutputInterface $output) ); } $fullCommand = $codeceptionCommand . $testsDirectory . $testName . ' --verbose --steps'; - $process = new Process($fullCommand); - $process->setWorkingDirectory(TESTS_BP); - $process->setIdleTimeout(600); - $process->setTimeout(0); - $subReturnCode = $process->run(function ($type, $buffer) use ($output) { - $output->write($buffer); - }); - $this->returnCode = max($this->returnCode, $subReturnCode); + $this->returnCode = max($this->returnCode, $this->executeTestCommand($fullCommand, $output)); } } @@ -140,8 +131,6 @@ private function runTests(array $tests, OutputInterface $output) * @param array $suitesConfig * @param OutputInterface $output * @return void - * - * @SuppressWarnings(PHPMD.UnusedLocalVariable) */ private function runTestsInSuite(array $suitesConfig, OutputInterface $output) { @@ -149,14 +138,27 @@ private function runTestsInSuite(array $suitesConfig, OutputInterface $output) //for tests in suites, run them as a group to run before and after block foreach (array_keys($suitesConfig) as $suite) { $fullCommand = $codeceptionCommand . " -g {$suite}"; - $process = new Process($fullCommand); - $process->setWorkingDirectory(TESTS_BP); - $process->setIdleTimeout(600); - $process->setTimeout(0); - $subReturnCode = $process->run(function ($type, $buffer) use ($output) { - $output->write($buffer); - }); - $this->returnCode = max($this->returnCode, $subReturnCode); + $this->returnCode = max($this->returnCode, $this->executeTestCommand($fullCommand, $output)); } } + + /** + * Runs the codeception test command and returns exit code + * + * @param String $command + * @param OutputInterface $output + * @return int + * + * @SuppressWarnings(PHPMD.UnusedLocalVariable) + */ + private function executeTestCommand(String $command, OutputInterface $output) + { + $process = new Process($command); + $process->setWorkingDirectory(TESTS_BP); + $process->setIdleTimeout(600); + $process->setTimeout(0); + return $process->run(function ($type, $buffer) use ($output) { + $output->write($buffer); + }); + } } From 6a5798ed082a73372cf06b37f3d2168eb4835333 Mon Sep 17 00:00:00 2001 From: Soumya Unnikrishnan Date: Wed, 11 Sep 2019 13:57:27 -0500 Subject: [PATCH 74/82] MQE-1755: mftf run:test Test1 Test2 does not run before/after hooks correctly fixed unit test failures --- .../FunctionalTestingFramework/Console/RunTestCommand.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php index b65409b70..c848a3010 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php @@ -145,13 +145,13 @@ private function runTestsInSuite(array $suitesConfig, OutputInterface $output) /** * Runs the codeception test command and returns exit code * - * @param String $command + * @param string $command * @param OutputInterface $output - * @return int + * @return integer * * @SuppressWarnings(PHPMD.UnusedLocalVariable) */ - private function executeTestCommand(String $command, OutputInterface $output) + private function executeTestCommand(string $command, OutputInterface $output) { $process = new Process($command); $process->setWorkingDirectory(TESTS_BP); From 03c1f19c76745d8779d423ef80a3f6a87750c5fa Mon Sep 17 00:00:00 2001 From: Donald Booth Date: Fri, 13 Sep 2019 09:00:11 -0500 Subject: [PATCH 75/82] Removed breaking "\r\n" instances from command. (#441) * Removed breaking "\r\n" instances from command. * Put back required backslashes --- docs/commands/mftf.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/commands/mftf.md b/docs/commands/mftf.md index ac029779e..58be6501e 100644 --- a/docs/commands/mftf.md +++ b/docs/commands/mftf.md @@ -181,7 +181,7 @@ Complex configuration to generate a few non-suite tests, a single test in a suit The command that encodes this complex configuration: ```bash -vendor/bin/mftf generate:tests --tests "{\r\n\"tests\":[\r\n\"general_test1\",\r\n\"general_test2\",\r\n\"general_test3\"\r\n],\r\n\"suites\":{\r\n\"sample\":[\r\n\"suite_test1\"\r\n],\r\n\"sample2\":null\r\n}\r\n}" +vendor/bin/mftf generate:tests --tests '{"tests":["general_test1","general_test2","general_test3"],"suites":{"sample":["suite_test1"],"sample2":null}}' ``` Note that the strings must be escaped and surrounded in quotes. From 04ff5b55d3f861883fea73842ac3ae869ef8597a Mon Sep 17 00:00:00 2001 From: Ji Lu Date: Fri, 13 Sep 2019 09:41:07 -0500 Subject: [PATCH 76/82] MQE-1754: mftf generate:tests AdminLoginTest -f does not obey the -f flag - set MftfApplicationConfig for generate and run console commands --- .../Console/GenerateTestsCommand.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php b/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php index d70aaaf3a..64c7c01d5 100644 --- a/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php @@ -123,8 +123,8 @@ protected function execute(InputInterface $input, OutputInterface $output) /** * Function which builds up a configuration including test and suites for consumption of Magento generation methods. * - * @param string $json - * @param array $tests + * @param string $json + * @param array $tests * @return array * @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException * @throws \Magento\FunctionalTestingFramework\Exceptions\XmlException From c113b9f70ba78756b84df6b787b9ba3d67b886f3 Mon Sep 17 00:00:00 2001 From: Kevin Kozan Date: Fri, 13 Sep 2019 11:21:49 -0500 Subject: [PATCH 77/82] MQE-1743: Changelog and Composer Bump (#449) - CHANGELOG update - Version bump --- CHANGELOG.md | 34 ++++++++++++++++++++++++++++++++++ bin/mftf | 2 +- composer.json | 2 +- composer.lock | 4 ++-- docs/data.md | 10 ++++++++++ 5 files changed, 48 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d3f82a2f9..5750f0ab2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,40 @@ Magento Functional Testing Framework Changelog ================================================ +2.5.0 +----- +* Traceability + * Allure output has been enhanced to contain new test artifacts created and used per MFTF step: + * `makeScreenshot` will contain an attachment under its Allure step. + * `seeInCurrentUrl` and all other `Url` asserts now contain an attachment with the expects vs actual comparison. + * `createData` and all other `Data` actions now contain attachments with `ApiUrl`, `Headers`, `Request Body`, and `Response Body`. +* Modularity + * Added a new `mftf run:manifest` command to run testManifest files generated by `generate:tests`. + * See DevDocs for details + * `mftf generate/run:test` commands now implicitly generates the `suite` the test exists in. + * If a test exists in multiple suites, it will generate it in all suite contexts. + * `mftf run:test ` will now only run the exact test provided, regardless of what is generated. +* Maintainability + * Added an `--allow-skipped` flag that allows MFTF to ignore the `` annotation. This was added to the following commands: + * `generate:test` + * `run:test` + * `run:group` + * `run:failed` +* Customizability + * `` defined in data.xml can now reference other `` directly + * See DevDocs for details + +### Fixes +* Fixed an issue where `grab` action variables were not substituting correctly when used as an element parameter. +* Framework will not throw a descriptive error when referencing a `$persisted.field$` that does not exist. +* MFTF test materials that `extends=""` itself will no longer cause infinite recursion. +* Fixed an issue where a test could not reference a `$data.field$` whose casing was modified by the API that it used. +* Fixed an issue with the default `functional.suite.yml` where it was incompatible with `symfony/yaml 4.0.0`. +* Improved test generation performance via class refactors (`~10%` faster). + +### GitHub Issues/Pull requests: +* [#377](https://github.com/magento/magento2-functional-testing-framework/pull/377) -- Non-API operations fixes + 2.4.4 ----- ### Fixes diff --git a/bin/mftf b/bin/mftf index 0f2bf274d..7f9db3524 100755 --- a/bin/mftf +++ b/bin/mftf @@ -29,7 +29,7 @@ try { try { $application = new Symfony\Component\Console\Application(); $application->setName('Magento Functional Testing Framework CLI'); - $application->setVersion('2.4.4'); + $application->setVersion('2.5.0'); /** @var \Magento\FunctionalTestingFramework\Console\CommandListInterface $commandList */ $commandList = new \Magento\FunctionalTestingFramework\Console\CommandList; foreach ($commandList->getCommands() as $command) { diff --git a/composer.json b/composer.json index 66936938c..b33567303 100755 --- a/composer.json +++ b/composer.json @@ -2,7 +2,7 @@ "name": "magento/magento2-functional-testing-framework", "description": "Magento2 Functional Testing Framework", "type": "library", - "version": "2.4.4", + "version": "2.5.0", "license": "AGPL-3.0", "keywords": ["magento", "automation", "functional", "testing"], "config": { diff --git a/composer.lock b/composer.lock index 10169fb08..dd7d6ad3f 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "1b7f01a11ff57f3b64f08352c33a506f", + "content-hash": "01cbd9e237e76de7070a3c0de4ee8b9f", "packages": [ { "name": "allure-framework/allure-codeception", @@ -250,7 +250,7 @@ { "name": "Tobias Nyholm", "email": "tobias.nyholm@gmail.com", - "homepage": "https://github.com/nyholm" + "homepage": "https://github.com/Nyholm" } ], "description": "Library of all the php-cache adapters", diff --git a/docs/data.md b/docs/data.md index 875df0198..4c618ae45 100644 --- a/docs/data.md +++ b/docs/data.md @@ -173,6 +173,16 @@ The following is an example of a call in test: This action inputs data from the `name` of the `_defaultCategory` entity (for example, `simpleCategory598742365`) into the field with the locator defined in the selector of the `categoryNameInput` element of the `AdminCategoryBasicFieldSection`. +You can also call data from the xml definition of a `data` tag directly: + +```xml + + admin + {{AnotherUser.current_password}} + {{_ENV.MAGENTO_ADMIN_PASSWORD}} + +``` + ## Reference ### entities {#entities-tag} From d97b01352a1fda16f59f1a676fa815217ae3f1b3 Mon Sep 17 00:00:00 2001 From: Kevin Kozan Date: Fri, 13 Sep 2019 14:39:46 -0500 Subject: [PATCH 78/82] MQE-1582: Enable Testers To Run Skipped Tests - Fix tag name to allow-skipped as per AC --- .../Console/BaseGenerateCommand.php | 2 +- .../Console/GenerateSuiteCommand.php | 2 +- .../Console/GenerateTestsCommand.php | 2 +- .../FunctionalTestingFramework/Console/RunTestCommand.php | 4 ++-- .../Console/RunTestFailedCommand.php | 4 ++-- .../Console/RunTestGroupCommand.php | 4 ++-- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php index 40582db86..87b203d66 100644 --- a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php @@ -36,7 +36,7 @@ protected function configure() InputOption::VALUE_NONE, 'force generation and running of tests regardless of Magento Instance Configuration' )->addOption( - "allowSkipped", + "allow-skipped", 'a', InputOption::VALUE_NONE, 'Allows MFTF to generate and run skipped tests.' diff --git a/src/Magento/FunctionalTestingFramework/Console/GenerateSuiteCommand.php b/src/Magento/FunctionalTestingFramework/Console/GenerateSuiteCommand.php index 7dd3b8cb4..cd798f420 100644 --- a/src/Magento/FunctionalTestingFramework/Console/GenerateSuiteCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/GenerateSuiteCommand.php @@ -46,7 +46,7 @@ protected function execute(InputInterface $input, OutputInterface $output) $debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility $remove = $input->getOption('remove'); $verbose = $output->isVerbose(); - $allowSkipped = $input->getOption('allowSkipped'); + $allowSkipped = $input->getOption('allow-skipped'); // Set application configuration so we can references the user options in our framework MftfApplicationConfig::create( diff --git a/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php b/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php index 64c7c01d5..621f29d03 100644 --- a/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php @@ -71,7 +71,7 @@ protected function execute(InputInterface $input, OutputInterface $output) $debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility $remove = $input->getOption('remove'); $verbose = $output->isVerbose(); - $allowSkipped = $input->getOption('allowSkipped'); + $allowSkipped = $input->getOption('allow-skipped'); // Set application configuration so we can references the user options in our framework MftfApplicationConfig::create( diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php index ad9f5795b..77b8be513 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php @@ -64,7 +64,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int $force = $input->getOption('force'); $remove = $input->getOption('remove'); $debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility - $allowSkipped = $input->getOption('allowSkipped'); + $allowSkipped = $input->getOption('allow-skipped'); $verbose = $output->isVerbose(); if ($skipGeneration and $remove) { @@ -92,7 +92,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int '--force' => $force, '--remove' => $remove, '--debug' => $debug, - '--allowSkipped' => $allowSkipped, + '--allow-skipped' => $allowSkipped, '-v' => $verbose ]; $command->run(new ArrayInput($args), $output); diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php index 15ba723e6..336fd5d87 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php @@ -70,7 +70,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int { $force = $input->getOption('force'); $debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility - $allowSkipped = $input->getOption('allowSkipped'); + $allowSkipped = $input->getOption('allow-skipped'); $verbose = $output->isVerbose(); // Create Mftf Configuration @@ -95,7 +95,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int '--force' => $force, '--remove' => true, '--debug' => $debug, - '--allowSkipped' => $allowSkipped, + '--allow-skipped' => $allowSkipped, '-v' => $verbose ]; $command->run(new ArrayInput($args), $output); diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php index 20d290af9..7f954c8fe 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php @@ -60,7 +60,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int $groups = $input->getArgument('groups'); $remove = $input->getOption('remove'); $debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility - $allowSkipped = $input->getOption('allowSkipped'); + $allowSkipped = $input->getOption('allow-skipped'); $verbose = $output->isVerbose(); if ($skipGeneration and $remove) { @@ -87,7 +87,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int '--force' => $force, '--remove' => $remove, '--debug' => $debug, - '--allowSkipped' => $allowSkipped, + '--allow-skipped' => $allowSkipped, '-v' => $verbose ]; From f63fac83e89bec461ef85a43cb120e48e3cad5fd Mon Sep 17 00:00:00 2001 From: Kevin Kozan Date: Mon, 16 Sep 2019 15:39:30 -0500 Subject: [PATCH 79/82] MQE-1765: Introduce API Endpoint and Request Headers to Allure artifacts - Removed both artifacts to be reintroduced later. --- .../DataGenerator/Persist/CurlHandler.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/CurlHandler.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/CurlHandler.php index 4b184c206..691ce3606 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/CurlHandler.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/CurlHandler.php @@ -167,8 +167,6 @@ public function executeRequest($dependentEntities) $response = $executor->read($successRegex, $returnRegex, $returnIndex); $executor->close(); - AllureHelper::addAttachmentToLastStep($apiUrl, 'API Endpoint'); - AllureHelper::addAttachmentToLastStep(json_encode($headers, JSON_PRETTY_PRINT), 'Request Headers'); AllureHelper::addAttachmentToLastStep(json_encode($this->requestData, JSON_PRETTY_PRINT), 'Request Body'); AllureHelper::addAttachmentToLastStep( json_encode(json_decode($response, true), JSON_PRETTY_PRINT+JSON_UNESCAPED_UNICODE+JSON_UNESCAPED_SLASHES), From 93fd35d2f5f8be9595d2b0f0fe9ea90f14b1ecb0 Mon Sep 17 00:00:00 2001 From: Kevin Kozan Date: Mon, 16 Sep 2019 15:49:11 -0500 Subject: [PATCH 80/82] MQE-1765: Introduce API Endpoint and Request Headers to Allure artifacts - Changelog redaction --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5750f0ab2..6e297f189 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ Magento Functional Testing Framework Changelog * Allure output has been enhanced to contain new test artifacts created and used per MFTF step: * `makeScreenshot` will contain an attachment under its Allure step. * `seeInCurrentUrl` and all other `Url` asserts now contain an attachment with the expects vs actual comparison. - * `createData` and all other `Data` actions now contain attachments with `ApiUrl`, `Headers`, `Request Body`, and `Response Body`. + * `createData` and all other `Data` actions now contain attachments with `Request Body` and `Response Body`. * Modularity * Added a new `mftf run:manifest` command to run testManifest files generated by `generate:tests`. * See DevDocs for details From 7accc4cce53ef0cf30a844b49c7d55387643bb3a Mon Sep 17 00:00:00 2001 From: Ji Lu Date: Tue, 17 Sep 2019 10:15:43 -0500 Subject: [PATCH 81/82] MQE-1671: updated CHANGELOG.md to include vault integration --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e297f189..6471920c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,7 +21,9 @@ Magento Functional Testing Framework Changelog * `run:group` * `run:failed` * Customizability - * `` defined in data.xml can now reference other `` directly + * `` defined in data.xml can now reference other `` directly. + * See DevDocs for details + * Added vault as an alternative credential storage. * See DevDocs for details ### Fixes From d58eb6bc9b5a6f52b55d58cdafbc5b07aa451b26 Mon Sep 17 00:00:00 2001 From: Ji Lu Date: Tue, 17 Sep 2019 12:12:30 -0500 Subject: [PATCH 82/82] MQE-1768: incorrect credential key for carriers_fedex_account in .credentials.example --- etc/config/.credentials.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/config/.credentials.example b/etc/config/.credentials.example index d9c73ac66..429e9d19f 100644 --- a/etc/config/.credentials.example +++ b/etc/config/.credentials.example @@ -1,4 +1,4 @@ -#magento/magento/carriers_fedex_account= +#magento/carriers_fedex_account= #magento/carriers_fedex_meter_number= #magento/carriers_fedex_key= #magento/carriers_fedex_password=