From a07bf6b64a1514ef72bb4091224bd8df621d75a9 Mon Sep 17 00:00:00 2001 From: Ajith Date: Sun, 19 Apr 2020 14:57:43 +0530 Subject: [PATCH 01/52] command added to modifiy the web server rewrites config --- docs/getting-started.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/getting-started.md b/docs/getting-started.md index 89d228df9..903a77372 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -117,6 +117,18 @@ The MFTF does not support executing CLI commands if your web server points to `< If the Nginx Web server is used on your development environment, then **Use Web Server Rewrites** setting in **Stores** > Settings > **Configuration** > **General** > **Web** > **Search Engine Optimization** must be set to **Yes**. +or via command line: + +```bash +bin/magento config:set web/seo/use_rewrites 1 +``` + +Clean the cache after changing the configuration values: + +```bash +bin/magento cache:clean config full_page +``` + To be able to run Magento command line commands in tests, add the following location block to the Nginx configuration file in the Magento root directory: ```conf From 54004ba8506aa93d5bd6f7047a9a0f90f7ad4568 Mon Sep 17 00:00:00 2001 From: Ajith Date: Thu, 23 Apr 2020 00:47:13 +0530 Subject: [PATCH 02/52] Renamed sample test name with the correct one --- docs/getting-started.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index 89d228df9..8f30601af 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -241,10 +241,10 @@ See more commands in [`codecept`][]. #### Run a simple test {#run-test} -To clean up the previously generated tests, and then generate and run a single test `AdminLoginTest`, run: +To clean up the previously generated tests, and then generate and run a single test `AdminLoginSuccessfulTest`, run: ```bash -vendor/bin/mftf run:test AdminLoginTest --remove +vendor/bin/mftf run:test AdminLoginSuccessfulTest --remove ``` See more commands in [`mftf`][]. @@ -319,7 +319,7 @@ composer remove magento/magento2-functional-testing-framework --dev -d Date: Fri, 24 Apr 2020 09:30:48 -0500 Subject: [PATCH 03/52] Grammar --- docs/getting-started.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index 903a77372..51dabe4b5 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -117,13 +117,13 @@ The MFTF does not support executing CLI commands if your web server points to `< If the Nginx Web server is used on your development environment, then **Use Web Server Rewrites** setting in **Stores** > Settings > **Configuration** > **General** > **Web** > **Search Engine Optimization** must be set to **Yes**. -or via command line: +Or via command line: ```bash bin/magento config:set web/seo/use_rewrites 1 ``` -Clean the cache after changing the configuration values: +You must clean the cache after changing the configuration values: ```bash bin/magento cache:clean config full_page From d35455e9b8de4ef2a3d7b5dddd4702065b003462 Mon Sep 17 00:00:00 2001 From: Ajith Date: Wed, 29 Apr 2020 23:09:28 +0530 Subject: [PATCH 04/52] Branch name changed --- docs/getting-started.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index 89d228df9..0c543042f 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -44,7 +44,7 @@ cd magento2/ ``` ```bash -git checkout 2.3-develop +git checkout 2.4-develop ``` Install the Magento application. From d3d5b2864bfed03e2b344a4a17690d2dade7a9d0 Mon Sep 17 00:00:00 2001 From: David Haecker Date: Tue, 16 Jun 2020 11:27:27 -0500 Subject: [PATCH 05/52] PWA-691: Fix Waits In PWA MFTF MagentoPwaWebDriver - Adding locators to wait for in PWA webdriver - Updating Magento webdriver to correctly use loading icon locators - Updating Magento webdriver to use pageload_timeout --- .../Module/MagentoPwaWebDriver.php | 24 ++++++++++++++++++- .../Module/MagentoWebDriver.php | 6 +++-- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoPwaWebDriver.php b/src/Magento/FunctionalTestingFramework/Module/MagentoPwaWebDriver.php index 9b75cb10d..abfbaf751 100644 --- a/src/Magento/FunctionalTestingFramework/Module/MagentoPwaWebDriver.php +++ b/src/Magento/FunctionalTestingFramework/Module/MagentoPwaWebDriver.php @@ -17,6 +17,18 @@ */ class MagentoPwaWebDriver extends MagentoWebDriver { + /** + * List of known PWA loading masks by selector + * + * Overriding the MagentoWebDriver array to contain applicable PWA locators. + * + * @var array + */ + protected $loadingMasksLocators = [ + '//div[contains(@class, "indicator-global-3ae")]', + '//div[contains(@class, "indicator-message-2he")]' + ]; + /** * Go to the page. * @@ -24,12 +36,14 @@ class MagentoPwaWebDriver extends MagentoWebDriver * The AJAX check in 'waitForPageLoad' does NOT work with a PWA. * * @param string $page + * @param null $timeout * @throws \Exception * @return void */ - public function amOnPage($page) + public function amOnPage($page, $timeout = null) { WebDriver::amOnPage($page); + $this->waitForLoadingMaskToDisappear($timeout); } /** @@ -43,11 +57,15 @@ public function amOnPage($page) */ public function waitForPwaElementNotVisible($selector, $timeout = null) { + $timeout = $timeout ?? $this->_getConfig()['pageload_timeout']; + // Determine what type of Selector is used. // Then use the correct JavaScript to locate the Element. if (\Codeception\Util\Locator::isXPath($selector)) { + $this->waitForLoadingMaskToDisappear($timeout); $this->waitForJS("return !document.evaluate(`$selector`, document);", $timeout); } else { + $this->waitForLoadingMaskToDisappear($timeout); $this->waitForJS("return !document.querySelector(`$selector`);", $timeout); } } @@ -63,11 +81,15 @@ public function waitForPwaElementNotVisible($selector, $timeout = null) */ public function waitForPwaElementVisible($selector, $timeout = null) { + $timeout = $timeout ?? $this->_getConfig()['pageload_timeout']; + // Determine what type of Selector is used. // Then use the correct JavaScript to locate the Element. if (\Codeception\Util\Locator::isXPath($selector)) { + $this->waitForLoadingMaskToDisappear($timeout); $this->waitForJS("return !!document && !!document.evaluate(`$selector`, document);", $timeout); } else { + $this->waitForLoadingMaskToDisappear($timeout); $this->waitForJS("return !!document && !!document.querySelector(`$selector`);", $timeout); } } diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php index 984881e4e..2f4745df1 100644 --- a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php +++ b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php @@ -63,7 +63,7 @@ class MagentoWebDriver extends WebDriver * * @var array */ - public static $loadingMasksLocators = [ + protected $loadingMasksLocators = [ '//div[contains(@class, "loading-mask")]', '//div[contains(@class, "admin_data-grid-loading-mask")]', '//div[contains(@class, "admin__data-grid-loading-mask")]', @@ -439,7 +439,9 @@ public function waitForPageLoad($timeout = null) */ public function waitForLoadingMaskToDisappear($timeout = null) { - foreach (self::$loadingMasksLocators as $maskLocator) { + $timeout = $timeout ?? $this->_getConfig()['pageload_timeout']; + + foreach ($this->loadingMasksLocators as $maskLocator) { // Get count of elements found for looping. // Elements are NOT useful for interaction, as they cannot be fed to codeception actions. $loadingMaskElements = $this->_findElements($maskLocator); From 15b9b2955f046e5e7d5e4066523825fff66c2f77 Mon Sep 17 00:00:00 2001 From: David Haecker Date: Tue, 16 Jun 2020 12:32:16 -0500 Subject: [PATCH 06/52] PWA-691: Fix Waits In PWA MFTF MagentoPwaWebDriver - Adding locators to wait for in PWA webdriver --- .../FunctionalTestingFramework/Module/MagentoPwaWebDriver.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoPwaWebDriver.php b/src/Magento/FunctionalTestingFramework/Module/MagentoPwaWebDriver.php index abfbaf751..9d6031e18 100644 --- a/src/Magento/FunctionalTestingFramework/Module/MagentoPwaWebDriver.php +++ b/src/Magento/FunctionalTestingFramework/Module/MagentoPwaWebDriver.php @@ -26,6 +26,8 @@ class MagentoPwaWebDriver extends MagentoWebDriver */ protected $loadingMasksLocators = [ '//div[contains(@class, "indicator-global-3ae")]', + '//div[contains(@class, "indicator-root-3J-")]', + '//div[contains(@class, "indicator-indicator-JHR")]', '//div[contains(@class, "indicator-message-2he")]' ]; From c502e467db838d4d13eadcda887ae8e63b672cc2 Mon Sep 17 00:00:00 2001 From: David Haecker Date: Tue, 16 Jun 2020 12:59:30 -0500 Subject: [PATCH 07/52] PWA-691: Fix Waits In PWA MFTF MagentoPwaWebDriver - Fixing locators to wait for in PWA webdriver --- .../Module/MagentoPwaWebDriver.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoPwaWebDriver.php b/src/Magento/FunctionalTestingFramework/Module/MagentoPwaWebDriver.php index 9d6031e18..d16249d98 100644 --- a/src/Magento/FunctionalTestingFramework/Module/MagentoPwaWebDriver.php +++ b/src/Magento/FunctionalTestingFramework/Module/MagentoPwaWebDriver.php @@ -25,10 +25,10 @@ class MagentoPwaWebDriver extends MagentoWebDriver * @var array */ protected $loadingMasksLocators = [ - '//div[contains(@class, "indicator-global-3ae")]', - '//div[contains(@class, "indicator-root-3J-")]', - '//div[contains(@class, "indicator-indicator-JHR")]', - '//div[contains(@class, "indicator-message-2he")]' + '//div[contains(@class, "indicator-global-")]', + '//div[contains(@class, "indicator-root-")]', + '//img[contains(@class, "indicator-indicator-")]', + '//span[contains(@class, "indicator-message-")]' ]; /** From fb3da40f264318fd54ee9fb48153c8b46ae5814a Mon Sep 17 00:00:00 2001 From: David Haecker Date: Tue, 16 Jun 2020 15:35:57 -0500 Subject: [PATCH 08/52] PWA-691: Fix Waits In PWA MFTF MagentoPwaWebDriver - Fixing parameter types --- .../Module/MagentoPwaWebDriver.php | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoPwaWebDriver.php b/src/Magento/FunctionalTestingFramework/Module/MagentoPwaWebDriver.php index d16249d98..46860b59d 100644 --- a/src/Magento/FunctionalTestingFramework/Module/MagentoPwaWebDriver.php +++ b/src/Magento/FunctionalTestingFramework/Module/MagentoPwaWebDriver.php @@ -38,7 +38,7 @@ class MagentoPwaWebDriver extends MagentoWebDriver * The AJAX check in 'waitForPageLoad' does NOT work with a PWA. * * @param string $page - * @param null $timeout + * @param integer $timeout * @throws \Exception * @return void */ @@ -52,8 +52,8 @@ public function amOnPage($page, $timeout = null) * Wait for a PWA Element to NOT be visible using JavaScript. * Add the WAIT_TIMEOUT variable to your .env file for this action. * - * @param null $selector - * @param null $timeout + * @param string $selector + * @param integer $timeout * @throws \Exception * @return void */ @@ -76,8 +76,8 @@ public function waitForPwaElementNotVisible($selector, $timeout = null) * Wait for a PWA Element to be visible using JavaScript. * Add the WAIT_TIMEOUT variable to your .env file for this action. * - * @param null $selector - * @param null $timeout + * @param string $selector + * @param integer $timeout * @throws \Exception * @return void */ From 72ffec286ad431c54601ff96cdd7d95e1484ff13 Mon Sep 17 00:00:00 2001 From: David Haecker Date: Tue, 16 Jun 2020 18:37:40 -0500 Subject: [PATCH 09/52] PWA-691: Fix Waits In PWA MFTF MagentoPwaWebDriver - Fixing static test failures --- .../Module/MagentoPwaWebDriver.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoPwaWebDriver.php b/src/Magento/FunctionalTestingFramework/Module/MagentoPwaWebDriver.php index 46860b59d..272f4206d 100644 --- a/src/Magento/FunctionalTestingFramework/Module/MagentoPwaWebDriver.php +++ b/src/Magento/FunctionalTestingFramework/Module/MagentoPwaWebDriver.php @@ -37,7 +37,7 @@ class MagentoPwaWebDriver extends MagentoWebDriver * Overriding the MagentoWebDriver version because it contains 'waitForPageLoad'. * The AJAX check in 'waitForPageLoad' does NOT work with a PWA. * - * @param string $page + * @param string $page * @param integer $timeout * @throws \Exception * @return void @@ -52,7 +52,7 @@ public function amOnPage($page, $timeout = null) * Wait for a PWA Element to NOT be visible using JavaScript. * Add the WAIT_TIMEOUT variable to your .env file for this action. * - * @param string $selector + * @param string $selector * @param integer $timeout * @throws \Exception * @return void @@ -76,7 +76,7 @@ public function waitForPwaElementNotVisible($selector, $timeout = null) * Wait for a PWA Element to be visible using JavaScript. * Add the WAIT_TIMEOUT variable to your .env file for this action. * - * @param string $selector + * @param string $selector * @param integer $timeout * @throws \Exception * @return void From 3cc7ae5b749e0f620ea1db8ad471ba1896e11bdc Mon Sep 17 00:00:00 2001 From: Mohamed Azarudeen Date: Wed, 24 Jun 2020 20:13:08 +0530 Subject: [PATCH 10/52] Removed invalid sample test name --- docs/commands/mftf.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/commands/mftf.md b/docs/commands/mftf.md index f1ef3955d..427cdbbf7 100644 --- a/docs/commands/mftf.md +++ b/docs/commands/mftf.md @@ -39,13 +39,13 @@ vendor/bin/mftf generate:tests ### Generate tests by test name ```bash -vendor/bin/mftf generate:tests AdminLoginTest StorefrontPersistedCustomerLoginTest +vendor/bin/mftf generate:tests AdminLoginSuccessfulTest StorefrontPersistedCustomerLoginTest ``` ### Generate test by test and suite name ```bash -vendor/bin/mftf generate:tests LoginSuite:AdminLoginTest +vendor/bin/mftf generate:tests LoginSuite:AdminLoginSuccessfulTest ``` ### Generate and run the tests for a specified group @@ -59,7 +59,7 @@ This command cleans up the previously generated tests; generates and runs tests ### Generate and run particular tests ```bash -vendor/bin/mftf run:test AdminLoginTest StorefrontPersistedCustomerLoginTest -r +vendor/bin/mftf run:test AdminLoginSuccessfulTest StorefrontPersistedCustomerLoginTest -r ``` This command cleans up the previously generated tests; generates and runs the `LoginAsAdminTest` and `LoginAsCustomerTest` tests. @@ -67,10 +67,10 @@ This command cleans up the previously generated tests; generates and runs the `L ### Generate and run particular test in a specific suite's context ```bash -vendor/bin/mftf run:test LoginSuite:AdminLoginTest -r +vendor/bin/mftf run:test LoginSuite:AdminLoginSuccessfulTest -r ``` -This command cleans up previously generated tests; generates and run `AdminLoginTest` within the context of the `LoginSuite`. +This command cleans up previously generated tests; generates and run `AdminLoginSuccessfulTest` within the context of the `LoginSuite`. ### Generate and run a testManifest.txt file @@ -362,7 +362,7 @@ vendor/bin/mftf run:manifest path/to/your/testManifest.txt Each line should contain either: one test path or one group (-g) reference. ``` -tests/functional/tests/MFTF/_generated/default/AdminLoginTestCest.php +tests/functional/tests/MFTF/_generated/default/AdminLoginSuccessfulTestCest.php -g PaypalTestSuite tests/functional/tests/MFTF/_generated/default/SomeOtherTestCest.php tests/functional/tests/MFTF/_generated/default/ThirdTestCest.php From c0255eece466c1871e679b5f9077927508c4b9f8 Mon Sep 17 00:00:00 2001 From: soumyau Date: Wed, 24 Jun 2020 18:25:15 -0500 Subject: [PATCH 11/52] =?UTF-8?q?MQE-1861:=20Suite=20precondition=20failur?= =?UTF-8?q?e=20when=20using=20=20with=20 with * MQE-1861: Suite precondition failure when using with Added verification test --- .../Resources/functionalSuiteHooks.txt | 29 +++++++++++++++---- .../Resources/functionalSuiteWithComments.txt | 3 +- .../TestModule/Suite/functionalSuiteHooks.xml | 12 ++++++-- .../Suite/Generators/GroupClassGenerator.php | 8 ++--- .../Suite/views/partials/testActions.mustache | 6 ++-- 5 files changed, 43 insertions(+), 15 deletions(-) diff --git a/dev/tests/verification/Resources/functionalSuiteHooks.txt b/dev/tests/verification/Resources/functionalSuiteHooks.txt index 3b5f6bae0..041b088b1 100644 --- a/dev/tests/verification/Resources/functionalSuiteHooks.txt +++ b/dev/tests/verification/Resources/functionalSuiteHooks.txt @@ -53,14 +53,33 @@ class functionalSuiteHooks extends \Codeception\GroupObject $webDriver->_initializeSession(); } $webDriver->amOnPage("some.url"); // stepKey: before - $createFields['someKey'] = "dataHere"; + $createOneFields['someKey'] = "dataHere"; PersistedObjectHandler::getInstance()->createEntity( - "create", + "createOne", "suite", - "createThis", - $createFields + "createEntityOne", + [], + $createOneFields ); - $webDriver->click(PersistedObjectHandler::getInstance()->retrieveEntityField('create', 'data', 'suite')); // stepKey: clickWithData + PersistedObjectHandler::getInstance()->createEntity( + "createTwo", + "suite", + "createEntityTwo", + ["createEntityOne"] + ); + PersistedObjectHandler::getInstance()->createEntity( + "createThree", + "suite", + "createEntityThree", + [] + ); + PersistedObjectHandler::getInstance()->createEntity( + "createFour", + "suite", + "createEntityFour", + ["createEntityTwo", "createEntityThree"] + ); + $webDriver->click(PersistedObjectHandler::getInstance()->retrieveEntityField('createTwo', 'data', 'suite')); // stepKey: clickWithData print("Entering Action Group [AC] actionGroupWithTwoArguments"); $webDriver->see("John", msq("uniqueData") . "John"); // stepKey: seeFirstNameAC print("Exiting Action Group [AC] actionGroupWithTwoArguments"); diff --git a/dev/tests/verification/Resources/functionalSuiteWithComments.txt b/dev/tests/verification/Resources/functionalSuiteWithComments.txt index c8523d0c1..6c646dd18 100644 --- a/dev/tests/verification/Resources/functionalSuiteWithComments.txt +++ b/dev/tests/verification/Resources/functionalSuiteWithComments.txt @@ -59,7 +59,8 @@ class functionalSuiteWithComments extends \Codeception\GroupObject "create", "suite", "createThis", - $createFields + [], + $createFields ); print(""); $webDriver->click(PersistedObjectHandler::getInstance()->retrieveEntityField('create', 'data', 'suite')); // stepKey: clickWithData diff --git a/dev/tests/verification/TestModule/Suite/functionalSuiteHooks.xml b/dev/tests/verification/TestModule/Suite/functionalSuiteHooks.xml index 4fc459e7d..b4dca7f8d 100644 --- a/dev/tests/verification/TestModule/Suite/functionalSuiteHooks.xml +++ b/dev/tests/verification/TestModule/Suite/functionalSuiteHooks.xml @@ -13,10 +13,18 @@ - + dataHere - + + + + + + + + + diff --git a/src/Magento/FunctionalTestingFramework/Suite/Generators/GroupClassGenerator.php b/src/Magento/FunctionalTestingFramework/Suite/Generators/GroupClassGenerator.php index 67930f09a..bb0157152 100644 --- a/src/Magento/FunctionalTestingFramework/Suite/Generators/GroupClassGenerator.php +++ b/src/Magento/FunctionalTestingFramework/Suite/Generators/GroupClassGenerator.php @@ -225,16 +225,16 @@ private function buildPersistenceMustacheArray($action, $entityArray) $action->getCustomActionAttributes()[TestGenerator::REQUIRED_ENTITY_REFERENCE]; // append entries for any required entities to this entry - if (array_key_exists('requiredEntities', $action->getCustomActionAttributes())) { - $entityArray[self::REQUIRED_ENTITY_KEY] = - $this->buildReqEntitiesMustacheArray($action->getCustomActionAttributes()); + $requiredEntities = $this->buildReqEntitiesMustacheArray($action->getCustomActionAttributes()); + if (!array_key_exists(-1, $requiredEntities)) { + $entityArray[self::REQUIRED_ENTITY_KEY] = $requiredEntities; } // append entries for customFields if specified by the user. if (array_key_exists('customFields', $action->getCustomActionAttributes())) { $entityArray['customFields'] = $action->getStepKey() . 'Fields'; } - + return $entityArray; } diff --git a/src/Magento/FunctionalTestingFramework/Suite/views/partials/testActions.mustache b/src/Magento/FunctionalTestingFramework/Suite/views/partials/testActions.mustache index 73079f169..eaa0959cb 100644 --- a/src/Magento/FunctionalTestingFramework/Suite/views/partials/testActions.mustache +++ b/src/Magento/FunctionalTestingFramework/Suite/views/partials/testActions.mustache @@ -13,9 +13,9 @@ if ($webDriver->webDriver != null) { PersistedObjectHandler::getInstance()->createEntity( "{{stepKey}}", "suite", - "{{entityName}}"{{#requiredEntities}}, - [$this->{{entityName}}{{^last}}, {{/last}}]{{/requiredEntities}}{{#customFields}}, - ${{customFields}}{{/customFields}} + "{{entityName}}", + [{{#requiredEntities}}"{{entityName}}"{{^last}}, {{/last}}{{/requiredEntities}}]{{#customFields}}, + ${{customFields}}{{/customFields}} ); {{/createData}} {{#deleteData}} From f630b5dc5f0be24f7fc5b5c87179b403535eb7f0 Mon Sep 17 00:00:00 2001 From: soumyau Date: Fri, 26 Jun 2020 15:04:58 -0500 Subject: [PATCH 12/52] MQE-2203: Create CHANGELOG.MD entry for 3.0.0 (#748) consolidated change log for 3.0.0 --- CHANGELOG.md | 158 +++++++++++++++++++-------------------------------- 1 file changed, 60 insertions(+), 98 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2401d413f..f1c5fad6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,118 +1,53 @@ Magento Functional Testing Framework Changelog ================================================ -3.0.0 RC5 ---------- - -### Enhancements - -* Readability - * Removed blacklist/whitelist terminology in MFTF. - -### Fixes - -* Fixed javascript error seen on chrome 81 for dragAndDrop action. -* Fixed allure issue when `WebDriverCurlException` is encountered in `afterStep`. - -3.0.0 RC4 +3.0.0 --------- ### Enhancements * Customizability + * Introduced MFTF helpers `` to create custom actions outside of MFTF.[See custom-helpers page for details](./docs/custom-helpers.md) + * Removed deprecated actions `` and ``. * `` no longer skips a test. Instead, the test is added to the `skip` group. -* Maintainability - * `mftf.log` no longer includes notices and warnings at test execution time. - * Added the Chrome option `--ignore-certificate-errors` to `functional.suite.dist.yml`. -* Traceability - * Changed the `bin/mftf static-checks` error file directory from the current working directory to `TESTS_BP/tests/_output/static-results/`. -* Readability - * Documented [3.0.0 Backward Incompatible Changes.](./docs/backward-incompatible-changes.md) - -### Fixes - -* Fixed issue where an extended data entity would not merge array items. Array items should merge properly now. -* Fixed issue where Chrome remains running after MFTF suite finishes. - -3.0.0 RC3 ---------- - -### Enhancements - -* Maintainability - * Added support for Two-Factor Authentication (2FA). [See configure-2fa page for details](./docs/configure-2fa.md) - * Added new static check `annotationsCheck` that checks and reports missing annotations in tests. - * Updated `bin/mftf static-checks` command to allow executing static-checks defined in `staticRuleSet.json` by default. [See command page for details](./docs/commands/mftf.md#static-checks) - * Added new upgrade script to remove unused arguments from action groups. - * Added unhandledPromptBehavior driver capability for Chrome 75+ support. - * Removed redundant and unused classes. - -### Fixes - -* Fixed issue with custom helper usage in suites. -* Fixed issue with decryption of secrets during data entity creation. -* Fixed issue with merging of `array` items in data entity. - -3.0.0 RC2 ---------- - -### Enhancements - -* Maintainability - * Added support for PHP 7.4. - * Removed support for PHP 7.2. - * Added support for PHPUnit 9. - * Improved assertion actions to support PHPUnit 9 changes. [See assertions page for details](./docs/test/assertions.md) - * Added new actions: `assertEqualsWithDelta`, `assertNotEqualsWithDelta`, `assertEqualsCanonicalizing`, `assertNotEqualsCanonicalizing`, `assertEqualsIgnoringCase`, `assertNotEqualsIgnoringCase`. - * Added new actions: `assertStringContainsString`, `assertStringNotContainsString`, `assertStringContainsStringIgnoringCase`, `assertStringNotContainsStringIgnoringCase` for string haystacks. - * Removed actions: `assertInternalType`, `assertNotInternalType`, `assertArraySubset`. - * Removed delta option from `assertEquals` and `assertNotEquals`. - * Removed action `pauseExecution` and added `pause`. [See actions page for details](./docs/test/actions.md#pause) - * Removed action `formatMoney` and added `formatCurrency`. [See actions page for details](./docs/test/actions.md#formatcurrency) - * Added new static check that checks and reports references to deprecated test entities. -* Bumped dependencies to support PHP/PHPUnit upgrade. - -* Traceability - * Introduced new `.env` configuration `VERBOSE_ARTIFACTS` to toggle saving attachments in Allure. [See configuration page for details](./docs/configuration.md) -### Fixes - -* Fixed issue of resolving arguments of type `entity` in action groups within a custom helper. -* Fixed reporting issue in output file for `testDependencies` static check. -* Fixed a bug in `actionGroupArguments` static check when action group filename is missing `ActionGroup`. -* Fixed issue of running suites under root `_suite` directory in Standalone MFTF. - -### GitHub Issues/Pull Requests - -* [#567](https://github.com/magento/magento2-functional-testing-framework/pull/567) -- log attachments for failed requests. - -3.0.0 RC1 ---------- - -### Enhancements - -* Customizability - * Introduced MFTF helpers `` to create custom actions outside of MFTF. - * Removed deprecated actions `` and ``. * Maintainability + * Added support for PHP 7.4. + * Added support for PHPUnit 9. + * Dropped support for PHP 7.0, 7.1, 7.2. * Schema updates for test entities to only allow single entity per file except Data and Metadata. * Support for sub-folders in test modules. * Removed support to read test entities from `dev/tests/acceptance/tests/functional/Magento/FunctionalTest`. - * Removed support for PHP 7.0 and 7.1. * Removed file attribute for `` in suiteSchema. + * Removed action `pauseExecution` and added `pause`. [See actions page for details](./docs/test/actions.md#pause) + * Removed action `formatMoney` and added `formatCurrency`. [See actions page for details](./docs/test/actions.md#formatcurrency) + * Improved assertion actions to support PHPUnit 9 changes. [See assertions page for details](./docs/test/assertions.md) + * Added new actions: `assertEqualsWithDelta`, `assertNotEqualsWithDelta`, `assertEqualsCanonicalizing`, `assertNotEqualsCanonicalizing`, `assertEqualsIgnoringCase`, `assertNotEqualsIgnoringCase`. + * Added new actions: `assertStringContainsString`, `assertStringNotContainsString`, `assertStringContainsStringIgnoringCase`, `assertStringNotContainsStringIgnoringCase` for string haystacks. + * Removed actions: `assertInternalType`, `assertNotInternalType`, `assertArraySubset`. + * Removed delta option from `assertEquals` and `assertNotEquals`. + * Added static check `deprecatedEntityUsage` that checks and reports references to deprecated test entities. + * Added static check `annotations` that checks and reports missing annotations in tests. + * Updated `bin/mftf static-checks` command to allow executing static-checks defined in `staticRuleSet.json` by default. [See command page for details](./docs/commands/mftf.md#static-checks) + * Added support for Two-Factor Authentication (2FA). [See configure-2fa page for details](./docs/configure-2fa.md) + * Added new upgrade script to remove unused arguments from action groups. + * `mftf.log` no longer includes notices and warnings at test execution time. + * Added unhandledPromptBehavior driver capability for Chrome 75+ support. + * Added the Chrome option `--ignore-certificate-errors` to `functional.suite.dist.yml`. + * Traceability - * Removed `--debug` option NONE to disallow ability to turn off schema validation. + * Removed `--debug` option `NONE` to disallow ability to turn off schema validation. * Notices added for test entity naming convention violations. * Metadata file names changed to `*Meta.xml`. + * Introduced new `.env` configuration `VERBOSE_ARTIFACTS` to toggle saving attachments in Allure. [See configuration page for details](./docs/configuration.md) + * Changed the `bin/mftf static-checks` error file directory from the current working directory to `TESTS_BP/tests/_output/static-results/`. + * Readability - * Support only nested assertion syntax [See assertions page for details](./docs/test/assertions.md) + * Support only nested assertion syntax [See assertions page for details](./docs/test/assertions.md). + * Documented [3.0.0 Backward Incompatible Changes](./docs/backward-incompatible-changes.md). + * Removed blacklist/whitelist terminology in MFTF. + * Upgrade scripts added to upgrade tests to MFTF major version requirements. See upgrade instructions below. -* Bumped dependencies to latest possible versions. - -### Fixes - -* Throw exception during generation when leaving out .url for `amOnPage`. -* `request_timeout` and `connection_timeout` added to functional.suite.yml.dist. -* Fixed `ModuleResolver` to resolve test modules moved out of deprecated path. +* Bumped dependencies to support PHP/PHPUnit upgrade. ### Upgrade Instructions @@ -120,12 +55,39 @@ Magento Functional Testing Framework Changelog * Run `bin/mftf build:project` to generate new configurations. * Run `bin/mftf upgrade:tests`. [See command page for details](./docs/commands/mftf.md#upgradetests). * After running the above command, some tests may need manually updates: - * Remove all occurrences of `` and `` - * Remove all occurrences of `` from any ``s + * Remove all occurrences of `` and ``. + * Remove all occurrences of `` from any ``s. * Ensure all `` actions in your tests have a valid schema. * Lastly, try to generate all tests. Tests should all be generated as a result of the upgrades. * If not, the most likely issue will be a changed XML schema. Check error messaging and search your codebase for the attributes listed. +### Fixes + +* Throw exception during generation when leaving out .url for `amOnPage`. +* `request_timeout` and `connection_timeout` added to functional.suite.yml.dist. +* Fixed `ModuleResolver` to resolve test modules moved out of deprecated path. +* Fixed issue of resolving arguments of type `entity` in action groups within a custom helper. +* Fixed reporting issue in output file for `testDependencies` static check. +* Fixed a bug in `actionGroupArguments` static check when action group filename is missing `ActionGroup`. +* Fixed issue of running suites under root `_suite` directory in Standalone MFTF. +* Fixed issue with custom helper usage in suites. +* Fixed issue with decryption of secrets during data entity creation. +* Fixed issue with merging of `array` items in data entity. +* Fixed issue where an extended data entity would not merge array items. Array items should merge properly now. +* Fixed issue where Chrome remains running after MFTF suite finishes. +* Fixed javascript error seen on Chrome 83 for dragAndDrop action. +* Fixed allure issue when `WebDriverCurlException` is encountered in `afterStep`. + +### GitHub Issues/Pull Requests + +* [#567](https://github.com/magento/magento2-functional-testing-framework/pull/567) -- log attachments for failed requests. + +### Demo Video links + +* [MFTF 3.0.0 RC1](https://www.youtube.com/watch?v=z0ZaZCmnw-A&t=2s) +* [MFTF 3.0.0 RC2](https://www.youtube.com/watch?v=BJOQAw6dX5o) +* [MFTF 3.0.0 RC3](https://www.youtube.com/watch?v=scLb7pi8pR0) + 2.6.3 ----- From 185852385432c7260087e717fc9093fb2aad09d8 Mon Sep 17 00:00:00 2001 From: Alastair Mucklow Date: Wed, 8 Jul 2020 11:19:44 +0100 Subject: [PATCH 13/52] Amend element name --- docs/test/annotations.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/test/annotations.md b/docs/test/annotations.md index 4e9194bb0..8fdcd2240 100644 --- a/docs/test/annotations.md +++ b/docs/test/annotations.md @@ -112,7 +112,7 @@ Attribute|Type|Use ### severity -The `` element is an implementation of a [`@Severity`] Allure tag; Metadata for report. +The `` element is an implementation of a [`@Severity`] Allure tag; Metadata for report. Attribute|Type|Use|Acceptable values ---|---|---|--- From 1941831d7d9e44b91843b5260ad8f1d434e910de Mon Sep 17 00:00:00 2001 From: Alastair Mucklow Date: Wed, 8 Jul 2020 11:45:40 +0100 Subject: [PATCH 14/52] Replace broken devhub.io/zh paths with github.com paths --- docs/test/annotations.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/test/annotations.md b/docs/test/annotations.md index 8fdcd2240..055ca9450 100644 --- a/docs/test/annotations.md +++ b/docs/test/annotations.md @@ -213,14 +213,14 @@ Attribute|Type|Use -[`@Description`]: https://devhub.io/zh/repos/allure-framework-allure-phpunit#extended-test-class-or-test-method-description -[`@Features`]: https://devhub.io/zh/repos/allure-framework-allure-phpunit#map-test-classes-and-test-methods-to-features-and-stories +[`@Description`]: https://github.com/allure-framework/allure-phpunit#extended-test-class-or-test-method-description +[`@Features`]: https://github.com/allure-framework/allure-phpunit#map-test-classes-and-test-methods-to-features-and-stories [`@group`]: http://codeception.com/docs/07-AdvancedUsage#Groups [`@return`]: http://codeception.com/docs/07-AdvancedUsage#Examples -[`@Severity`]: https://devhub.io/zh/repos/allure-framework-allure-phpunit#set-test-severity -[`@Stories`]: https://devhub.io/zh/repos/allure-framework-allure-phpunit#map-test-classes-and-test-methods-to-features-and-stories +[`@Severity`]: https://github.com/allure-framework/allure-phpunit#set-test-severity +[`@Stories`]: https://github.com/allure-framework/allure-phpunit#map-test-classes-and-test-methods-to-features-and-stories [`@TestCaseId`]: https://github.com/allure-framework/allure1/wiki/Test-Case-ID -[`@Title`]: https://devhub.io/zh/repos/allure-framework-allure-phpunit#human-readable-test-class-or-test-method-title +[`@Title`]: https://github.com/allure-framework/allure-phpunit#human-readable-test-class-or-test-method-title [description]: #description [features]: #features [group]: #group From 30f2c653c669c98e116e6bde77b7692d5a97b4e0 Mon Sep 17 00:00:00 2001 From: Alastair Mucklow Date: Wed, 8 Jul 2020 12:22:27 +0100 Subject: [PATCH 15/52] Make intro text clearer and more accurate --- docs/test/annotations.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/test/annotations.md b/docs/test/annotations.md index 055ca9450..d61565aec 100644 --- a/docs/test/annotations.md +++ b/docs/test/annotations.md @@ -112,7 +112,7 @@ Attribute|Type|Use ### severity -The `` element is an implementation of a [`@Severity`] Allure tag; Metadata for report. +The `` element is an implementation of the [`@Severity`] Allure annotation, which is used to prioritise test methods by severity. Attribute|Type|Use|Acceptable values ---|---|---|--- From ffbcac774edec1d9f718ef4c0de8b80d861fbb3a Mon Sep 17 00:00:00 2001 From: Alastair Mucklow Date: Wed, 8 Jul 2020 12:22:42 +0100 Subject: [PATCH 16/52] Fix order (Blocker is the most severe) --- docs/test/annotations.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/test/annotations.md b/docs/test/annotations.md index d61565aec..69c75a2d0 100644 --- a/docs/test/annotations.md +++ b/docs/test/annotations.md @@ -116,7 +116,7 @@ The `` element is an implementation of the [`@Severity`] Allure annota Attribute|Type|Use|Acceptable values ---|---|---|--- -`value`|string|required|`MINOR`, `AVERAGE`, `MAJOR`, `BLOCKER`, `CRITICAL` +`value`|string|required|`MINOR`, `AVERAGE`, `MAJOR`, `CRITICAL`, `BLOCKER` #### Example From a0a966fdcc4b17b8245a4c0e119c67cfa2d40409 Mon Sep 17 00:00:00 2001 From: Alastair Mucklow Date: Wed, 8 Jul 2020 12:23:14 +0100 Subject: [PATCH 17/52] Add usage guidelines table outlining the difference between levels --- docs/test/annotations.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/test/annotations.md b/docs/test/annotations.md index 69c75a2d0..1e2e50f58 100644 --- a/docs/test/annotations.md +++ b/docs/test/annotations.md @@ -124,6 +124,16 @@ Attribute|Type|Use|Acceptable values ``` +#### Usage guidelines + +Severity Level|Usage +---|--- +`BLOCKER`|If this test fails, the customer is completely blocked from purchasing a product. +`CRITICAL`|This is a serious problem impacting conversion, or affecting the operation of the store. +`MAJOR`|Store conversion rate is reduced owing to this issue. For example, something is broken or missing that impacts checkout frequency or cart volume. +`AVERAGE`|A fault on the storefront that can negatively impact conversion rate (like UI errors or omissions), or problems with Magento admin functionality. +`MINOR`|An application or configuration fault that has no impact on conversion rate. + ### skip Use the `` element to skip a test. From f958045af0dcef47ca75fdf2adc81bf4bb418829 Mon Sep 17 00:00:00 2001 From: Alastair Mucklow Date: Wed, 8 Jul 2020 12:34:24 +0100 Subject: [PATCH 18/52] Trim text --- docs/test/annotations.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/test/annotations.md b/docs/test/annotations.md index 1e2e50f58..79cc00e80 100644 --- a/docs/test/annotations.md +++ b/docs/test/annotations.md @@ -112,7 +112,7 @@ Attribute|Type|Use ### severity -The `` element is an implementation of the [`@Severity`] Allure annotation, which is used to prioritise test methods by severity. +The `` element is an implementation of the [`@Severity`] Allure annotation, which is used to prioritise tests by severity. Attribute|Type|Use|Acceptable values ---|---|---|--- From ef5be1dcece0cc9b451a255d076e1e64c29444c1 Mon Sep 17 00:00:00 2001 From: soumyau Date: Wed, 22 Jul 2020 11:03:45 -0500 Subject: [PATCH 19/52] MQE-1886: Ability to use grab data between ActionGroups (#755) * MQE-1886: Ability to use grab data between ActionGroups * MQE-1886: Ability to use grab data between ActionGroups verification tests + other fixes * MQE-1886: Ability to use grab data between ActionGroups added functional test. * MQE-1886: Ability to use grab data between ActionGroups added functional test. Removed functional test for return. --- .../ActionGroupReturningValueTest.txt | 75 ++++++++++++++++++ .../ExtendedActionGroupReturningValueTest.txt | 40 ++++++++++ ...ndedChildActionGroupReturningValueTest.txt | 43 +++++++++++ .../MergedActionGroupReturningValueTest.txt | 77 +++++++++++++++++++ .../ActionGroupReturningValueActionGroup.xml | 21 +++++ ...edActionGroupReturningValueActionGroup.xml | 21 +++++ ...lActionGroupWithReturnValueActionGroup.xml | 14 ++++ ...geActionGroupReturningValueActionGroup.xml | 19 +++++ ...geActionGroupReturningValueActionGroup.xml | 14 ++++ .../ActionGroupReturningValueTest.xml | 30 ++++++++ .../ExtendedActionGroupReturningValueTest.xml | 22 ++++++ ...ndedChildActionGroupReturningValueTest.xml | 23 ++++++ .../MergedActionGroupReturningValueTest.xml | 32 ++++++++ .../ActionGroupWithReturnGenerationTest.php | 52 +++++++++++++ docs/data.md | 2 + docs/test/action-groups.md | 28 +++++++ docs/test/actions.md | 16 ++++ etc/di.xml | 2 +- .../Module/MagentoWebDriver.php | 12 +++ .../Test/etc/Actions/customActions.xsd | 26 +++++++ .../Test/etc/actionTypeTags.xsd | 6 ++ .../Test/etc/mergedActionGroupSchema.xsd | 62 +++++++++------ .../Util/TestGenerator.php | 10 +++ 23 files changed, 623 insertions(+), 24 deletions(-) create mode 100644 dev/tests/verification/Resources/ActionGroupReturningValueTest.txt create mode 100644 dev/tests/verification/Resources/ExtendedActionGroupReturningValueTest.txt create mode 100644 dev/tests/verification/Resources/ExtendedChildActionGroupReturningValueTest.txt create mode 100644 dev/tests/verification/Resources/MergedActionGroupReturningValueTest.txt create mode 100644 dev/tests/verification/TestModule/ActionGroup/BasicActionGroup/ActionGroupReturningValueActionGroup.xml create mode 100644 dev/tests/verification/TestModule/ActionGroup/BasicActionGroup/ExtendedActionGroupReturningValueActionGroup.xml create mode 100644 dev/tests/verification/TestModule/ActionGroup/FunctionalActionGroup/FunctionalActionGroupWithReturnValueActionGroup.xml create mode 100644 dev/tests/verification/TestModule/ActionGroup/FunctionalActionGroup/MergeActionGroupReturningValueActionGroup.xml create mode 100644 dev/tests/verification/TestModule/ActionGroup/MergeFunctionalActionGroup/MergeActionGroupReturningValueActionGroup.xml create mode 100644 dev/tests/verification/TestModule/Test/ActionGroupFunctionalTest/ActionGroupReturningValueTest.xml create mode 100644 dev/tests/verification/TestModule/Test/ActionGroupTest/ExtendedActionGroupReturningValueTest.xml create mode 100644 dev/tests/verification/TestModule/Test/ActionGroupTest/ExtendedChildActionGroupReturningValueTest.xml create mode 100644 dev/tests/verification/TestModule/Test/MergeFunctionalTest/MergedActionGroupReturningValueTest.xml create mode 100644 dev/tests/verification/Tests/ActionGroupWithReturnGenerationTest.php diff --git a/dev/tests/verification/Resources/ActionGroupReturningValueTest.txt b/dev/tests/verification/Resources/ActionGroupReturningValueTest.txt new file mode 100644 index 000000000..08e0b44f8 --- /dev/null +++ b/dev/tests/verification/Resources/ActionGroupReturningValueTest.txt @@ -0,0 +1,75 @@ +Test filesverification/TestModule/Test/ActionGroupFunctionalTest/ActionGroupReturningValueTest.xml
") + */ +class ActionGroupReturningValueTestCest +{ + /** + * @param AcceptanceTester $I + * @throws \Exception + */ + public function _before(AcceptanceTester $I) + { + $I->createEntity("createPersonParam", "hook", "ReplacementPerson", [], []); // stepKey: createPersonParam + $I->comment("Entering Action Group [beforeGroup] FunctionalActionGroup"); + $I->fillField("#foo", "myData1"); // stepKey: fillField1BeforeGroup + $I->fillField("#bar", "myData2"); // stepKey: fillField2BeforeGroup + $I->comment("Exiting Action Group [beforeGroup] FunctionalActionGroup"); + } + + /** + * @param AcceptanceTester $I + * @throws \Exception + */ + public function _after(AcceptanceTester $I) + { + $I->comment("Entering Action Group [afterGroup] FunctionalActionGroup"); + $I->fillField("#foo", "myData1"); // stepKey: fillField1AfterGroup + $I->fillField("#bar", "myData2"); // stepKey: fillField2AfterGroup + $I->comment("Exiting Action Group [afterGroup] FunctionalActionGroup"); + } + + /** + * @param AcceptanceTester $I + * @throws \Exception + */ + public function _failed(AcceptanceTester $I) + { + $I->saveScreenshot(); // stepKey: saveScreenshot + } + + /** + * @Severity(level = SeverityLevel::CRITICAL) + * @Features({"TestModule"}) + * @Stories({"MQE-433"}) + * @Parameter(name = "AcceptanceTester", value="$I") + * @param AcceptanceTester $I + * @return void + * @throws \Exception + */ + public function ActionGroupReturningValueTest(AcceptanceTester $I) + { + $I->amOnPage("/someUrl"); // stepKey: step1 + $I->comment("Entering Action Group [actionGroupWithReturnValue1] FunctionalActionGroupWithReturnValueActionGroup"); + $grabTextFrom1ActionGroupWithReturnValue1 = $I->grabTextFrom("#foo"); // stepKey: grabTextFrom1ActionGroupWithReturnValue1 + $actionGroupWithReturnValue1 = $I->return($grabTextFrom1ActionGroupWithReturnValue1); // stepKey: returnActionGroupWithReturnValue1 + $I->comment("Exiting Action Group [actionGroupWithReturnValue1] FunctionalActionGroupWithReturnValueActionGroup"); + $I->comment("Entering Action Group [actionGroupWithStringUsage1] actionGroupWithStringUsage"); + $I->see($actionGroupWithReturnValue1, "#element .{$actionGroupWithReturnValue1}"); // stepKey: see1ActionGroupWithStringUsage1 + $I->comment("Exiting Action Group [actionGroupWithStringUsage1] actionGroupWithStringUsage"); + } +} diff --git a/dev/tests/verification/Resources/ExtendedActionGroupReturningValueTest.txt b/dev/tests/verification/Resources/ExtendedActionGroupReturningValueTest.txt new file mode 100644 index 000000000..2f26a6463 --- /dev/null +++ b/dev/tests/verification/Resources/ExtendedActionGroupReturningValueTest.txt @@ -0,0 +1,40 @@ +Test filesverification/TestModule/Test/ActionGroupTest/ExtendedActionGroupReturningValueTest.xml
") + */ +class ExtendedActionGroupReturningValueTestCest +{ + /** + * @Severity(level = SeverityLevel::CRITICAL) + * @Features({"TestModule"}) + * @Parameter(name = "AcceptanceTester", value="$I") + * @param AcceptanceTester $I + * @return void + * @throws \Exception + */ + public function ExtendedActionGroupReturningValueTest(AcceptanceTester $I) + { + $I->comment("Entering Action Group [actionGroupReturningValue] ActionGroupReturningValueActionGroup"); + $grabProducts1ActionGroupReturningValue = $I->grabMultiple("selector"); // stepKey: grabProducts1ActionGroupReturningValue + $I->assertCount(99, $grabProducts1ActionGroupReturningValue); // stepKey: assertCountActionGroupReturningValue + $actionGroupReturningValue = $I->return($grabProducts1ActionGroupReturningValue); // stepKey: returnProducts1ActionGroupReturningValue + $I->comment("Exiting Action Group [actionGroupReturningValue] ActionGroupReturningValueActionGroup"); + $I->comment("Entering Action Group [actionGroupWithStringUsage1] actionGroupWithStringUsage"); + $I->see($actionGroupReturningValue, "#element .{$actionGroupReturningValue}"); // stepKey: see1ActionGroupWithStringUsage1 + $I->comment("Exiting Action Group [actionGroupWithStringUsage1] actionGroupWithStringUsage"); + } +} diff --git a/dev/tests/verification/Resources/ExtendedChildActionGroupReturningValueTest.txt b/dev/tests/verification/Resources/ExtendedChildActionGroupReturningValueTest.txt new file mode 100644 index 000000000..d043a5c04 --- /dev/null +++ b/dev/tests/verification/Resources/ExtendedChildActionGroupReturningValueTest.txt @@ -0,0 +1,43 @@ +Test filesverification/TestModule/Test/ActionGroupTest/ExtendedChildActionGroupReturningValueTest.xml
") + */ +class ExtendedChildActionGroupReturningValueTestCest +{ + /** + * @Severity(level = SeverityLevel::CRITICAL) + * @Features({"TestModule"}) + * @Parameter(name = "AcceptanceTester", value="$I") + * @param AcceptanceTester $I + * @return void + * @throws \Exception + */ + public function ExtendedChildActionGroupReturningValueTest(AcceptanceTester $I) + { + $I->comment("Entering Action Group [extendedActionGroupReturningValue] ExtendedActionGroupReturningValueActionGroup"); + $grabProducts1ExtendedActionGroupReturningValue = $I->grabMultiple("selector"); // stepKey: grabProducts1ExtendedActionGroupReturningValue + $I->assertCount(99, $grabProducts1ExtendedActionGroupReturningValue); // stepKey: assertCountExtendedActionGroupReturningValue + $extendedActionGroupReturningValue = $I->return($grabProducts1ExtendedActionGroupReturningValue); // stepKey: returnProducts1ExtendedActionGroupReturningValue + $grabProducts2ExtendedActionGroupReturningValue = $I->grabMultiple("otherSelector"); // stepKey: grabProducts2ExtendedActionGroupReturningValue + $I->assertCount(8000, $grabProducts2ExtendedActionGroupReturningValue); // stepKey: assertSecondCountExtendedActionGroupReturningValue + $extendedActionGroupReturningValue = $I->return($grabProducts2ExtendedActionGroupReturningValue); // stepKey: returnProducts2ExtendedActionGroupReturningValue + $I->comment("Exiting Action Group [extendedActionGroupReturningValue] ExtendedActionGroupReturningValueActionGroup"); + $I->comment("Entering Action Group [actionGroupWithStringUsage1] actionGroupWithStringUsage"); + $I->see($extendedActionGroupReturningValue, "#element .{$extendedActionGroupReturningValue}"); // stepKey: see1ActionGroupWithStringUsage1 + $I->comment("Exiting Action Group [actionGroupWithStringUsage1] actionGroupWithStringUsage"); + } +} diff --git a/dev/tests/verification/Resources/MergedActionGroupReturningValueTest.txt b/dev/tests/verification/Resources/MergedActionGroupReturningValueTest.txt new file mode 100644 index 000000000..a4689779b --- /dev/null +++ b/dev/tests/verification/Resources/MergedActionGroupReturningValueTest.txt @@ -0,0 +1,77 @@ +Test filesverification/TestModule/Test/MergeFunctionalTest/MergedActionGroupReturningValueTest.xml
") + */ +class MergedActionGroupReturningValueTestCest +{ + /** + * @param AcceptanceTester $I + * @throws \Exception + */ + public function _before(AcceptanceTester $I) + { + $I->createEntity("createPersonParam", "hook", "ReplacementPerson", [], []); // stepKey: createPersonParam + $I->comment("Entering Action Group [beforeGroup] FunctionalActionGroup"); + $I->fillField("#foo", "myData1"); // stepKey: fillField1BeforeGroup + $I->fillField("#bar", "myData2"); // stepKey: fillField2BeforeGroup + $I->comment("Exiting Action Group [beforeGroup] FunctionalActionGroup"); + } + + /** + * @param AcceptanceTester $I + * @throws \Exception + */ + public function _after(AcceptanceTester $I) + { + $I->comment("Entering Action Group [afterGroup] FunctionalActionGroup"); + $I->fillField("#foo", "myData1"); // stepKey: fillField1AfterGroup + $I->fillField("#bar", "myData2"); // stepKey: fillField2AfterGroup + $I->comment("Exiting Action Group [afterGroup] FunctionalActionGroup"); + } + + /** + * @param AcceptanceTester $I + * @throws \Exception + */ + public function _failed(AcceptanceTester $I) + { + $I->saveScreenshot(); // stepKey: saveScreenshot + } + + /** + * @Severity(level = SeverityLevel::CRITICAL) + * @Features({"TestModule"}) + * @Stories({"MQE-433"}) + * @Parameter(name = "AcceptanceTester", value="$I") + * @param AcceptanceTester $I + * @return void + * @throws \Exception + */ + public function MergedActionGroupReturningValueTest(AcceptanceTester $I) + { + $I->amOnPage("/someUrl"); // stepKey: step1 + $I->comment("Entering Action Group [actionGroupWithReturnValue1] MergeActionGroupReturningValueActionGroup"); + $I->amOnPage("/Jane/Dane.html"); // stepKey: amOnPage1ActionGroupWithReturnValue1 + $I->click(".merge .Jane"); // stepKey: myMergedClickActionGroupWithReturnValue1 + $grabMultiple1ActionGroupWithReturnValue1 = $I->grabMultiple("#foo"); // stepKey: grabMultiple1ActionGroupWithReturnValue1 + $actionGroupWithReturnValue1 = $I->return($grabMultiple1ActionGroupWithReturnValue1); // stepKey: returnValueActionGroupWithReturnValue1 + $I->comment("Exiting Action Group [actionGroupWithReturnValue1] MergeActionGroupReturningValueActionGroup"); + $I->comment("Entering Action Group [actionGroupWithStringUsage1] actionGroupWithStringUsage"); + $I->see($actionGroupWithReturnValue1, "#element .{$actionGroupWithReturnValue1}"); // stepKey: see1ActionGroupWithStringUsage1 + $I->comment("Exiting Action Group [actionGroupWithStringUsage1] actionGroupWithStringUsage"); + } +} diff --git a/dev/tests/verification/TestModule/ActionGroup/BasicActionGroup/ActionGroupReturningValueActionGroup.xml b/dev/tests/verification/TestModule/ActionGroup/BasicActionGroup/ActionGroupReturningValueActionGroup.xml new file mode 100644 index 000000000..ddb93c0c1 --- /dev/null +++ b/dev/tests/verification/TestModule/ActionGroup/BasicActionGroup/ActionGroupReturningValueActionGroup.xml @@ -0,0 +1,21 @@ + + + + + + + + + + {{count}} + grabProducts1 + + + + diff --git a/dev/tests/verification/TestModule/ActionGroup/BasicActionGroup/ExtendedActionGroupReturningValueActionGroup.xml b/dev/tests/verification/TestModule/ActionGroup/BasicActionGroup/ExtendedActionGroupReturningValueActionGroup.xml new file mode 100644 index 000000000..18933a2f2 --- /dev/null +++ b/dev/tests/verification/TestModule/ActionGroup/BasicActionGroup/ExtendedActionGroupReturningValueActionGroup.xml @@ -0,0 +1,21 @@ + + + + + + + + + + {{otherCount}} + grabProducts2 + + + + diff --git a/dev/tests/verification/TestModule/ActionGroup/FunctionalActionGroup/FunctionalActionGroupWithReturnValueActionGroup.xml b/dev/tests/verification/TestModule/ActionGroup/FunctionalActionGroup/FunctionalActionGroupWithReturnValueActionGroup.xml new file mode 100644 index 000000000..77900efd3 --- /dev/null +++ b/dev/tests/verification/TestModule/ActionGroup/FunctionalActionGroup/FunctionalActionGroupWithReturnValueActionGroup.xml @@ -0,0 +1,14 @@ + + + + + + + + diff --git a/dev/tests/verification/TestModule/ActionGroup/FunctionalActionGroup/MergeActionGroupReturningValueActionGroup.xml b/dev/tests/verification/TestModule/ActionGroup/FunctionalActionGroup/MergeActionGroupReturningValueActionGroup.xml new file mode 100644 index 000000000..83afbce0c --- /dev/null +++ b/dev/tests/verification/TestModule/ActionGroup/FunctionalActionGroup/MergeActionGroupReturningValueActionGroup.xml @@ -0,0 +1,19 @@ + + + + + + + + + + + + + diff --git a/dev/tests/verification/TestModule/ActionGroup/MergeFunctionalActionGroup/MergeActionGroupReturningValueActionGroup.xml b/dev/tests/verification/TestModule/ActionGroup/MergeFunctionalActionGroup/MergeActionGroupReturningValueActionGroup.xml new file mode 100644 index 000000000..d952fdddf --- /dev/null +++ b/dev/tests/verification/TestModule/ActionGroup/MergeFunctionalActionGroup/MergeActionGroupReturningValueActionGroup.xml @@ -0,0 +1,14 @@ + + + + + + + + diff --git a/dev/tests/verification/TestModule/Test/ActionGroupFunctionalTest/ActionGroupReturningValueTest.xml b/dev/tests/verification/TestModule/Test/ActionGroupFunctionalTest/ActionGroupReturningValueTest.xml new file mode 100644 index 000000000..ade1f51db --- /dev/null +++ b/dev/tests/verification/TestModule/Test/ActionGroupFunctionalTest/ActionGroupReturningValueTest.xml @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/dev/tests/verification/TestModule/Test/ActionGroupTest/ExtendedActionGroupReturningValueTest.xml b/dev/tests/verification/TestModule/Test/ActionGroupTest/ExtendedActionGroupReturningValueTest.xml new file mode 100644 index 000000000..68042344c --- /dev/null +++ b/dev/tests/verification/TestModule/Test/ActionGroupTest/ExtendedActionGroupReturningValueTest.xml @@ -0,0 +1,22 @@ + + + + + + + + </annotations> + <actionGroup ref="ActionGroupReturningValueActionGroup" stepKey="actionGroupReturningValue"> + <argument name="count" value="99"/> + </actionGroup> + <actionGroup ref="actionGroupWithStringUsage" stepKey="actionGroupWithStringUsage1"> + <argument name="someArgument" value="{$actionGroupReturningValue}"/> + </actionGroup> + </test> +</tests> diff --git a/dev/tests/verification/TestModule/Test/ActionGroupTest/ExtendedChildActionGroupReturningValueTest.xml b/dev/tests/verification/TestModule/Test/ActionGroupTest/ExtendedChildActionGroupReturningValueTest.xml new file mode 100644 index 000000000..ecd46c3c4 --- /dev/null +++ b/dev/tests/verification/TestModule/Test/ActionGroupTest/ExtendedChildActionGroupReturningValueTest.xml @@ -0,0 +1,23 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + /** + * Copyright © Magento, Inc. All rights reserved. + * See COPYING.txt for license details. + */ +--> +<tests xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:noNamespaceSchemaLocation="urn:magento:mftf:Test/etc/testSchema.xsd"> + <test name="ExtendedChildActionGroupReturningValueTest"> + <annotations> + <severity value="CRITICAL"/> + <title value="Extended Child ActionGroup Returning Value Test"/> + </annotations> + <actionGroup ref="ExtendedActionGroupReturningValueActionGroup" stepKey="extendedActionGroupReturningValue"> + <argument name="count" value="99"/> + <argument name="otherCount" value="8000"/> + </actionGroup> + <actionGroup ref="actionGroupWithStringUsage" stepKey="actionGroupWithStringUsage1"> + <argument name="someArgument" value="{$extendedActionGroupReturningValue}"/> + </actionGroup> + </test> +</tests> diff --git a/dev/tests/verification/TestModule/Test/MergeFunctionalTest/MergedActionGroupReturningValueTest.xml b/dev/tests/verification/TestModule/Test/MergeFunctionalTest/MergedActionGroupReturningValueTest.xml new file mode 100644 index 000000000..baf7c0ded --- /dev/null +++ b/dev/tests/verification/TestModule/Test/MergeFunctionalTest/MergedActionGroupReturningValueTest.xml @@ -0,0 +1,32 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + /** + * Copyright © Magento, Inc. All rights reserved. + * See COPYING.txt for license details. + */ +--> +<tests xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:noNamespaceSchemaLocation="urn:magento:mftf:Test/etc/testSchema.xsd"> + <test name="MergedActionGroupReturningValueTest"> + <annotations> + <severity value="CRITICAL"/> + <group value="functional"/> + <features value="Action Group Functional Cest"/> + <stories value="MQE-433"/> + </annotations> + <before> + <createData entity="ReplacementPerson" stepKey="createPersonParam"/> + <actionGroup ref="FunctionalActionGroup" stepKey="beforeGroup"/> + </before> + <amOnPage url="/someUrl" stepKey="step1"/> + <actionGroup ref="MergeActionGroupReturningValueActionGroup" stepKey="actionGroupWithReturnValue1"> + <argument name="myArg" value="DefaultPerson"/> + </actionGroup> + <actionGroup ref="actionGroupWithStringUsage" stepKey="actionGroupWithStringUsage1"> + <argument name="someArgument" value="{$actionGroupWithReturnValue1}"/> + </actionGroup> + <after> + <actionGroup ref="FunctionalActionGroup" stepKey="afterGroup"/> + </after> + </test> +</tests> diff --git a/dev/tests/verification/Tests/ActionGroupWithReturnGenerationTest.php b/dev/tests/verification/Tests/ActionGroupWithReturnGenerationTest.php new file mode 100644 index 000000000..e369a797d --- /dev/null +++ b/dev/tests/verification/Tests/ActionGroupWithReturnGenerationTest.php @@ -0,0 +1,52 @@ +<?php +/** + * Copyright © Magento, Inc. All rights reserved. + * See COPYING.txt for license details. + */ +namespace tests\verification\Tests; + +use tests\util\MftfTestCase; + +class ActionGroupWithReturnGenerationTest extends MftfTestCase +{ + /** + * Test generation of a test referencing an action group that returns a value. + * + * @throws \Exception + * @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException + */ + public function testActionGroupReturningValue() + { + $this->generateAndCompareTest('ActionGroupReturningValueTest'); + } + /** + * Test generation of a test referencing a merged action group that returns a value. + * + * @throws \Exception + * @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException + */ + public function testMergedActionGroupReturningValue() + { + $this->generateAndCompareTest('MergedActionGroupReturningValueTest'); + } + /** + * Test generation of a test referencing an extended action group that returns a value. + * + * @throws \Exception + * @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException + */ + public function testExtendedActionGroupReturningValue() + { + $this->generateAndCompareTest('ExtendedActionGroupReturningValueTest'); + } + /** + * Test generation of a test referencing an extending child action group that returns a value. + * + * @throws \Exception + * @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException + */ + public function testExtendedChildActionGroupReturningValue() + { + $this->generateAndCompareTest('ExtendedChildActionGroupReturningValueTest'); + } +} diff --git a/docs/data.md b/docs/data.md index 8b0a86e3d..d642763d0 100644 --- a/docs/data.md +++ b/docs/data.md @@ -78,6 +78,8 @@ A test can also reference data that was returned as a result of [test actions][] Further in the test, the data grabbed by the `someSelector` selector can be referenced using the `stepKey` value. In this case, it is `grabStepKey`. +The `stepKey` value can only be referenced within the test scope that it is defined in (`test`, `before/after`). + The following example shows the usage of `grabValueFrom` in testing, where the returned value is used by action's `stepKey`: ```xml diff --git a/docs/test/action-groups.md b/docs/test/action-groups.md index 70af0621a..05adf795e 100644 --- a/docs/test/action-groups.md +++ b/docs/test/action-groups.md @@ -180,6 +180,34 @@ MFTF resolves `{{myCustomEntity.field1}}` the same as it would in a `selector` o </actionGroup> ``` +## Return a value + +Action groups can return a value using a `return` tag. + +```xml +<actionGroup name="GetOrderIdActionGroup"> + <seeElement selector="{{CheckoutSuccessMainSection.orderLink}}" stepKey="assertOrderLink"/> + <grabTextFrom selector="{{CheckoutSuccessMainSection.orderLink}}" stepKey="orderId"/> + <return value="{$orderId}" stepKey="returnOrderId"/> +</actionGroup> +``` + +The value returned can be accessed in later steps using action group step key `{$getOrderId}`. +```xml +<actionGroup ref="GetOrderIdActionGroup" stepKey="getOrderId"/> +<!--Filter the Order using Order ID --> +<actionGroup ref="FilterOrderGridByIdActionGroup" stepKey="filterOrderGridById"> + <argument name="orderId" value="{$getOrderId}"/> +</actionGroup> +``` +### Convention to return a value + +The following conventions apply to action groups returning a value: +- Only action groups can return value. Use of `return` tag is dis-allowed in tests and suites. +- An action group does not support multiple `return` tags. +- For [merging action groups](../merging.md#merge-action-groups), `return` is allowed only in one of the merging action groups. +- Value returned by an action group can only be referenced within the scope that the action group is defined in (`test`, `before/after`). + ## Optimizing action group structures Structuring properly an action group increases code reusability and readability. diff --git a/docs/test/actions.md b/docs/test/actions.md index 1d2f83802..562f6e8ff 100644 --- a/docs/test/actions.md +++ b/docs/test/actions.md @@ -151,6 +151,7 @@ The following test actions return a variable: * [grabValueFrom](#grabvaluefrom) * [executeJS](#executejs) * [getOTP](#getotp) +* [return](#return) Learn more in [Using data returned by test actions](../data.md#use-data-returned-by-test-actions). @@ -1240,6 +1241,21 @@ To access this value, use `{$grabInputName}` in later actions. --> <grabValueFrom selector="input#name" stepKey="grabInputName"/> ``` +### return + +Specifies what value is returned by an action group. The value can be then accessed in later steps using the action group stepKey. See [Action groups returning a value](./action-groups.md#return-a-value) for usage information. + +Attribute|Type|Use|Description +---|---|---|--- +`value`|string|required| value returned by action group. +`stepKey`|string|required| A unique identifier of the action. + +#### Example +```xml +<!-- Returns value of $grabInputName to the calling +<return value="{$grabInputName}" stepKey="returnInputName"/> +``` + ### loadSessionSnapshot See [loadSessionSnapshot docs on codeception.com](http://codeception.com/docs/modules/WebDriver#loadSessionSnapshot). diff --git a/etc/di.xml b/etc/di.xml index e561911da..f5184808c 100644 --- a/etc/di.xml +++ b/etc/di.xml @@ -8,7 +8,7 @@ <!-- Entity value gets replaced in Dom.php before reading $xml --> <!DOCTYPE config [ - <!ENTITY commonTestActions "acceptPopup|actionGroup|amOnPage|amOnUrl|amOnSubdomain|appendField|assertArrayIsSortasserted|assertElementContainsAttribute|attachFile|cancelPopup|checkOption|clearField|click|clickWithLeftButton|clickWithRightButton|closeAdminNotification|closeTab|comment|conditionalClick|createData|deleteData|updateData|getData|dontSee|dontSeeJsError|dontSeeCheckboxIsChecked|dontSeeCookie|dontSeeCurrentUrlEquals|dontSeeCurrentUrlMatches|dontSeeElement|dontSeeElementInDOM|dontSeeInCurrentUrl|dontSeeInField|dontSeeInFormFields|dontSeeInPageSource|dontSeeInSource|dontSeeInTitle|dontSeeLink|dontSeeOptionIsSelected|doubleClick|dragAndDrop|entity|executeJS|fillField|formatCurrency|generateDate|getOTP|grabAttributeFrom|grabCookie|grabFromCurrentUrl|grabMultiple|grabPageSource|grabTextFrom|grabValueFrom|loadSessionSnapshot|loginAsAdmin|magentoCLI|magentoCron|makeScreenshot|maximizeWindow|moveBack|moveForward|moveMouseOver|mSetLocale|mResetLocale|openNewTab|pause|parseFloat|pressKey|reloadPage|resetCookie|submitForm|resizeWindow|saveSessionSnapshot|scrollTo|scrollToTopOfPage|searchAndMultiSelectOption|see|seeCheckboxIsChecked|seeCookie|seeCurrentUrlEquals|seeCurrentUrlMatches|seeElement|seeElementInDOM|seeInCurrentUrl|seeInField|seeInFormFields|seeInPageSource|seeInPopup|seeInSource|seeInTitle|seeLink|seeNumberOfElements|seeOptionIsSelected|selectOption|setCookie|submitForm|switchToIFrame|switchToNextTab|switchToPreviousTab|switchToWindow|typeInPopup|uncheckOption|unselectOption|wait|waitForAjaxLoad|waitForElement|waitForElementChange|waitForElementNotVisible|waitForElementVisible|waitForPwaElementNotVisible|waitForPwaElementVisible|waitForJS|waitForLoadingMaskToDisappear|waitForPageLoad|waitForText|assertArrayHasKey|assertArrayNotHasKey|assertContains|assertStringContainsString|assertStringContainsStringIgnoringCase|assertCount|assertEmpty|assertEquals|assertFalse|assertFileExists|assertFileNotExists|assertGreaterOrEquals|assertGreaterThan|assertGreaterThanOrEqual|assertInstanceOf|assertIsEmpty|assertLessOrEquals|assertLessThan|assertLessThanOrEqual|assertNotContains|assertStringNotContainsString|assertStringNotContainsStringIgnoringCase|assertNotEmpty|assertNotEquals|assertNotInstanceOf|assertNotNull|assertNotRegExp|assertNotSame|assertNull|assertRegExp|assertSame|assertStringStartsNotWith|assertStringStartsWith|assertTrue|expectException|fail|dontSeeFullUrlEquals|dontSee|dontSeeFullUrlMatches|dontSeeInFullUrl|seeFullUrlEquals|seeFullUrlMatches|seeInFullUrl|grabFromFullUrl|helper|assertEqualsWithDelta|assertEqualsCanonicalizing|assertEqualsIgnoringCase|assertNotEqualsWithDelta|assertNotEqualsCanonicalizing|assertNotEqualsIgnoringCase"> + <!ENTITY commonTestActions "acceptPopup|actionGroup|amOnPage|amOnUrl|amOnSubdomain|appendField|assertArrayIsSortasserted|assertElementContainsAttribute|attachFile|cancelPopup|checkOption|clearField|click|clickWithLeftButton|clickWithRightButton|closeAdminNotification|closeTab|comment|conditionalClick|createData|deleteData|updateData|getData|dontSee|dontSeeJsError|dontSeeCheckboxIsChecked|dontSeeCookie|dontSeeCurrentUrlEquals|dontSeeCurrentUrlMatches|dontSeeElement|dontSeeElementInDOM|dontSeeInCurrentUrl|dontSeeInField|dontSeeInFormFields|dontSeeInPageSource|dontSeeInSource|dontSeeInTitle|dontSeeLink|dontSeeOptionIsSelected|doubleClick|dragAndDrop|entity|executeJS|fillField|formatCurrency|generateDate|getOTP|grabAttributeFrom|grabCookie|grabFromCurrentUrl|grabMultiple|grabPageSource|grabTextFrom|grabValueFrom|return|loadSessionSnapshot|loginAsAdmin|magentoCLI|magentoCron|makeScreenshot|maximizeWindow|moveBack|moveForward|moveMouseOver|mSetLocale|mResetLocale|openNewTab|pause|parseFloat|pressKey|reloadPage|resetCookie|submitForm|resizeWindow|saveSessionSnapshot|scrollTo|scrollToTopOfPage|searchAndMultiSelectOption|see|seeCheckboxIsChecked|seeCookie|seeCurrentUrlEquals|seeCurrentUrlMatches|seeElement|seeElementInDOM|seeInCurrentUrl|seeInField|seeInFormFields|seeInPageSource|seeInPopup|seeInSource|seeInTitle|seeLink|seeNumberOfElements|seeOptionIsSelected|selectOption|setCookie|submitForm|switchToIFrame|switchToNextTab|switchToPreviousTab|switchToWindow|typeInPopup|uncheckOption|unselectOption|wait|waitForAjaxLoad|waitForElement|waitForElementChange|waitForElementNotVisible|waitForElementVisible|waitForPwaElementNotVisible|waitForPwaElementVisible|waitForJS|waitForLoadingMaskToDisappear|waitForPageLoad|waitForText|assertArrayHasKey|assertArrayNotHasKey|assertContains|assertStringContainsString|assertStringContainsStringIgnoringCase|assertCount|assertEmpty|assertEquals|assertFalse|assertFileExists|assertFileNotExists|assertGreaterOrEquals|assertGreaterThan|assertGreaterThanOrEqual|assertInstanceOf|assertIsEmpty|assertLessOrEquals|assertLessThan|assertLessThanOrEqual|assertNotContains|assertStringNotContainsString|assertStringNotContainsStringIgnoringCase|assertNotEmpty|assertNotEquals|assertNotInstanceOf|assertNotNull|assertNotRegExp|assertNotSame|assertNull|assertRegExp|assertSame|assertStringStartsNotWith|assertStringStartsWith|assertTrue|expectException|fail|dontSeeFullUrlEquals|dontSee|dontSeeFullUrlMatches|dontSeeInFullUrl|seeFullUrlEquals|seeFullUrlMatches|seeInFullUrl|grabFromFullUrl|helper|assertEqualsWithDelta|assertEqualsCanonicalizing|assertEqualsIgnoringCase|assertNotEqualsWithDelta|assertNotEqualsCanonicalizing|assertNotEqualsIgnoringCase"> ]> <config xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="../src/Magento/FunctionalTestingFramework/ObjectManager/etc/config.xsd"> diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php index 2f4745df1..944735677 100644 --- a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php +++ b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php @@ -1122,4 +1122,16 @@ public function switchToIFrame($locator = null) $this->webDriver->switchTo()->frame($els[0]); } } + + /** + * Returns a value to origin of the action. + * TODO: move this function to MagentoActionProxies after MQE-1904 + * + * @param mixed $value + * @return mixed + */ + public function return($value) + { + return $value; + } } diff --git a/src/Magento/FunctionalTestingFramework/Test/etc/Actions/customActions.xsd b/src/Magento/FunctionalTestingFramework/Test/etc/Actions/customActions.xsd index 2aad30c75..b1d6e1b02 100644 --- a/src/Magento/FunctionalTestingFramework/Test/etc/Actions/customActions.xsd +++ b/src/Magento/FunctionalTestingFramework/Test/etc/Actions/customActions.xsd @@ -29,6 +29,12 @@ </xs:choice> </xs:group> + <xs:group name="returnTags"> + <xs:choice> + <xs:element type="returnType" name="return" minOccurs="0" maxOccurs="1"/> + </xs:choice> + </xs:group> + <!-- Complex Types --> <xs:complexType name="helperType"> @@ -346,6 +352,26 @@ </xs:simpleContent> </xs:complexType> + <xs:complexType name="returnType"> + <xs:annotation> + <xs:documentation> + Used in an action group to return a value. Must be used only once in action group. Do not use in tests or suites. + </xs:documentation> + </xs:annotation> + <xs:simpleContent> + <xs:extension base="xs:string"> + <xs:attribute name="value" use="required" type="xs:string"> + <xs:annotation> + <xs:documentation> + Value or variable to be returned. + </xs:documentation> + </xs:annotation> + </xs:attribute> + <xs:attributeGroup ref="commonActionAttributes"/> + </xs:extension> + </xs:simpleContent> + </xs:complexType> + <xs:simpleType name="sortEnum" final="restriction"> <xs:annotation> <xs:documentation> diff --git a/src/Magento/FunctionalTestingFramework/Test/etc/actionTypeTags.xsd b/src/Magento/FunctionalTestingFramework/Test/etc/actionTypeTags.xsd index def2964af..d68cf43db 100644 --- a/src/Magento/FunctionalTestingFramework/Test/etc/actionTypeTags.xsd +++ b/src/Magento/FunctionalTestingFramework/Test/etc/actionTypeTags.xsd @@ -68,6 +68,12 @@ </xs:choice> </xs:group> + <xs:group name="returnTypeTags"> + <xs:choice> + <xs:group ref="returnTags"/> + </xs:choice> + </xs:group> + <!-- Complex Types --> <xs:complexType name="failType"> diff --git a/src/Magento/FunctionalTestingFramework/Test/etc/mergedActionGroupSchema.xsd b/src/Magento/FunctionalTestingFramework/Test/etc/mergedActionGroupSchema.xsd index bd32ba879..f45c33acd 100644 --- a/src/Magento/FunctionalTestingFramework/Test/etc/mergedActionGroupSchema.xsd +++ b/src/Magento/FunctionalTestingFramework/Test/etc/mergedActionGroupSchema.xsd @@ -15,29 +15,21 @@ </xs:choice> </xs:complexType> <xs:complexType name="actionsRefType"> - <xs:choice minOccurs="0" maxOccurs="unbounded"> - <xs:group ref="actionTypeTags"/> - <xs:element name="arguments"> - <xs:complexType> - <xs:sequence> - <xs:element name="argument" maxOccurs="unbounded" minOccurs="0"> - <xs:complexType> - <xs:attribute type="xs:string" name="name" use="required"/> - <xs:attribute type="xs:string" name="defaultValue"/> - <xs:attribute type="dataTypeEnum" name="type" default="entity"/> - </xs:complexType> - </xs:element> - </xs:sequence> - </xs:complexType> - </xs:element> - <xs:element name="annotations"> - <xs:complexType> - <xs:sequence> - <xs:element name="description"/> - </xs:sequence> - </xs:complexType> - </xs:element> - </xs:choice> + <xs:sequence> + <xs:choice minOccurs="0" maxOccurs="unbounded"> + <xs:group ref="actionTypeTags"/> + <xs:element ref="arguments"/> + <xs:element ref="annotations"/> + </xs:choice> + <xs:sequence minOccurs="0"> + <xs:group ref="returnTags"/> + <xs:choice minOccurs="0" maxOccurs="unbounded"> + <xs:group ref="actionTypeTags"/> + <xs:element ref="arguments"/> + <xs:element ref="annotations"/> + </xs:choice> + </xs:sequence> + </xs:sequence> <xs:attribute type="xs:string" name="name" use="required"/> <xs:attribute type="xs:string" name="filename"/> <xs:attribute type="xs:string" name="insertBefore"/> @@ -57,4 +49,28 @@ <xs:enumeration value="entity"/> </xs:restriction> </xs:simpleType> + + <!-- elements --> + + <xs:element name="arguments"> + <xs:complexType> + <xs:sequence> + <xs:element name="argument" maxOccurs="unbounded" minOccurs="0"> + <xs:complexType> + <xs:attribute type="xs:string" name="name" use="required"/> + <xs:attribute type="xs:string" name="defaultValue"/> + <xs:attribute type="dataTypeEnum" name="type" default="entity"/> + </xs:complexType> + </xs:element> + </xs:sequence> + </xs:complexType> + </xs:element> + + <xs:element name="annotations"> + <xs:complexType> + <xs:sequence> + <xs:element name="description"/> + </xs:sequence> + </xs:complexType> + </xs:element> </xs:schema> diff --git a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php index 8253a75ba..80056b3b9 100644 --- a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php +++ b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php @@ -1172,6 +1172,16 @@ public function generateStepsPhp($actionObjects, $generationScope = TestGenerato $selector ); break; + case "return": + $actionOrigin = $actionObject->getActionOrigin(); + $actionOriginStepKey = $actionOrigin[ActionGroupObject::ACTION_GROUP_ORIGIN_TEST_REF]; + $testSteps .= $this->wrapFunctionCallWithReturnValue( + $actionOriginStepKey, + $actor, + $actionObject, + $value + ); + break; case "formatCurrency": $testSteps .= $this->wrapFunctionCallWithReturnValue( $stepKey, From 8c73f2cf767e2f1326d199927e5bd2afe4ff0de3 Mon Sep 17 00:00:00 2001 From: Ji Lu <jilu1@adobe.com> Date: Tue, 28 Jul 2020 13:24:57 -0500 Subject: [PATCH 20/52] MQE-2231: Adobe open source repository standards compliance --- .github/CONTRIBUTING.md | 13 ++++++-- CODE_OF_CONDUCT.md | 74 +++++++++++++++++++++++++++++++++++++++++ README.md | 10 +++--- 3 files changed, 90 insertions(+), 7 deletions(-) create mode 100644 CODE_OF_CONDUCT.md diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 271b66244..11334ae73 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -24,8 +24,16 @@ If there is no response from the contributor for two weeks, the issue is closed. Often when the MFTF team works on reviewing the suggested changes, we will add a label to the issue to indicate to our internal team certain information, like status or who is working the issue. If you’re ever curious what the different labels mean, see the [table][labels] below for an explanation of each one. -Refer to [Magento Contributor Agreement] for detailed information about the License Agreement. -All contributors are required to submit a click-through form to agree to the terms. +## Code Of Conduct + +This project adheres to the Adobe [Code Of Conduct](../CODE_OF_CONDUCT.md). By participating, you are expected to uphold this code. +Please report unacceptable behavior to [Grp-opensourceoffice@adobe.com](mailto:Grp-opensourceoffice@adobe.com). + +## Contributor License Agreement + +All third-party contributions to this project must be accompanied by a signed Contributor License Agreement (CLA). +This gives Adobe permission to redistribute your contributions as part of the project. +[Sign our CLA](https://opensource.adobe.com/cla.html). You only need to sign it once. ## Contribution requirements @@ -152,6 +160,7 @@ Label| Description **bugfix**| The issue or pull request is about fixing a bug. **enhancement**| The issue or pull request that makes the MFTF even more awesome (for example new features, optimization, refactoring, etc). + [fork]: #fork-a-repository [issue]: #report-an-issue [labels]: #read-labels diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 000000000..549b492a0 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,74 @@ +# Adobe Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, gender identity and expression, level of experience, +nationality, personal appearance, race, religion, or sexual identity and +orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language. +* Being respectful of differing viewpoints and experiences. +* Gracefully accepting constructive criticism. +* Focusing on what is best for the community. +* Showing empathy towards other community members. + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or +advances. +* Trolling, insulting/derogatory comments, and personal or political attacks. +* Public or private harassment. +* Publishing others' private information, such as a physical or electronic + address, without explicit permission. +* Other conduct which could reasonably be considered inappropriate in a + professional setting. + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at Grp-opensourceoffice@adobe.com. All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at [https://contributor-covenant.org/version/1/4][version]. + +[homepage]: https://contributor-covenant.org +[version]: https://contributor-covenant.org/version/1/4/ \ No newline at end of file diff --git a/README.md b/README.md index fc42f0603..76bd66cc7 100755 --- a/README.md +++ b/README.md @@ -58,7 +58,7 @@ These labels are applied by the MFTF development team to community contributed i To report security vulnerabilities and other security issues in the Magento software or web sites, send an email with the report at [security@magento.com][]. Do not report security issues using GitHub. Be sure to encrypt your e-mail with our [encryption key][] if it includes sensitive information. -Learn more about reporting security issues [here][]. +Learn more about reporting security issues [security][]. Stay up-to-date on the latest security news and patches for Magento by signing up for [Security Alert Notifications][]. @@ -66,15 +66,15 @@ Stay up-to-date on the latest security news and patches for Magento by signing u Each Magento source file included in this distribution is licensed under AGPL 3.0. -See the license [here][] or contact [license@magentocommerce.com][] for a copy. +See the license [license][] or contact [license@magentocommerce.com][] for a copy. <!-- Link Definitions --> [Getting Started]: docs/getting-started.md -[Contribution Guidelines]: https://github.com/magento/magento2-functional-testing-framework/blob/develop/.github/CONTRIBUTING.md +[Contribution Guidelines]: .github/CONTRIBUTING.md [DevDocs Contributing]: https://github.com/magento/devdocs/blob/master/.github/CONTRIBUTING.md [security@magento.com]: mailto:security@magento.com [encryption key]: https://info2.magento.com/rs/magentoenterprise/images/security_at_magento.asc -[here]: https://magento.com/security/reporting-magento-security-issue +[security]: https://magento.com/security/reporting-magento-security-issue [Security Alert Notifications]: https://magento.com/security/sign-up -[here]: LICENSE_AGPL3.txt +[license]: LICENSE_AGPL3.txt [license@magentocommerce.com]: mailto:license@magentocommerce.com From 8218bc238e1f39bd70ae7f1efa89edd13026b272 Mon Sep 17 00:00:00 2001 From: Ji Lu <jilu1@adobe.com> Date: Tue, 28 Jul 2020 14:00:30 -0500 Subject: [PATCH 21/52] MQE-2231: Adobe open source repository standards compliance --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 76bd66cc7..ddeacd6de 100755 --- a/README.md +++ b/README.md @@ -58,7 +58,7 @@ These labels are applied by the MFTF development team to community contributed i To report security vulnerabilities and other security issues in the Magento software or web sites, send an email with the report at [security@magento.com][]. Do not report security issues using GitHub. Be sure to encrypt your e-mail with our [encryption key][] if it includes sensitive information. -Learn more about reporting security issues [security][]. +Learn more about reporting [security issues][]. Stay up-to-date on the latest security news and patches for Magento by signing up for [Security Alert Notifications][]. @@ -66,7 +66,7 @@ Stay up-to-date on the latest security news and patches for Magento by signing u Each Magento source file included in this distribution is licensed under AGPL 3.0. -See the license [license][] or contact [license@magentocommerce.com][] for a copy. +See the [license][] or contact [license@magentocommerce.com][] for a copy. <!-- Link Definitions --> [Getting Started]: docs/getting-started.md @@ -74,7 +74,7 @@ See the license [license][] or contact [license@magentocommerce.com][] for a cop [DevDocs Contributing]: https://github.com/magento/devdocs/blob/master/.github/CONTRIBUTING.md [security@magento.com]: mailto:security@magento.com [encryption key]: https://info2.magento.com/rs/magentoenterprise/images/security_at_magento.asc -[security]: https://magento.com/security/reporting-magento-security-issue +[security issues]: https://magento.com/security/reporting-magento-security-issue [Security Alert Notifications]: https://magento.com/security/sign-up [license]: LICENSE_AGPL3.txt [license@magentocommerce.com]: mailto:license@magentocommerce.com From b2bd6a873290ce80e96123b8531607e03b7c17ee Mon Sep 17 00:00:00 2001 From: Ji Lu <jilu1@adobe.com> Date: Tue, 28 Jul 2020 14:02:03 -0500 Subject: [PATCH 22/52] MQE-2231: Adobe open source repository standards compliance --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ddeacd6de..07c43fff1 100755 --- a/README.md +++ b/README.md @@ -58,7 +58,7 @@ These labels are applied by the MFTF development team to community contributed i To report security vulnerabilities and other security issues in the Magento software or web sites, send an email with the report at [security@magento.com][]. Do not report security issues using GitHub. Be sure to encrypt your e-mail with our [encryption key][] if it includes sensitive information. -Learn more about reporting [security issues][]. +Learn more about reporting [security][] issues. Stay up-to-date on the latest security news and patches for Magento by signing up for [Security Alert Notifications][]. @@ -74,7 +74,7 @@ See the [license][] or contact [license@magentocommerce.com][] for a copy. [DevDocs Contributing]: https://github.com/magento/devdocs/blob/master/.github/CONTRIBUTING.md [security@magento.com]: mailto:security@magento.com [encryption key]: https://info2.magento.com/rs/magentoenterprise/images/security_at_magento.asc -[security issues]: https://magento.com/security/reporting-magento-security-issue +[security]: https://magento.com/security/reporting-magento-security-issue [Security Alert Notifications]: https://magento.com/security/sign-up [license]: LICENSE_AGPL3.txt [license@magentocommerce.com]: mailto:license@magentocommerce.com From 91b1d1ac210a0ac7737972838022b9bcabf18ce9 Mon Sep 17 00:00:00 2001 From: Ji Lu <jilu1@adobe.com> Date: Wed, 29 Jul 2020 10:17:13 -0500 Subject: [PATCH 23/52] MQE-2231: Adobe open source repository standards compliance --- .github/CONTRIBUTING.md | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 11334ae73..9a38dfc65 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -160,7 +160,6 @@ Label| Description **bugfix**| The issue or pull request is about fixing a bug. **enhancement**| The issue or pull request that makes the MFTF even more awesome (for example new features, optimization, refactoring, etc). - [fork]: #fork-a-repository [issue]: #report-an-issue [labels]: #read-labels From 21c625c776705a6fd4e627142b18913ce204fd04 Mon Sep 17 00:00:00 2001 From: Ji Lu <jilu1@adobe.com> Date: Wed, 29 Jul 2020 11:59:52 -0500 Subject: [PATCH 24/52] MQE-2231: Adobe open source repository standards compliance --- README.md | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 07c43fff1..5a4bbab11 100755 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ Learn more about contributing in our [Contribution Guidelines][]. If you want to participate in the documentation work, see [DevDocs Contributing][]. -### Labels applied by the MFTF team +### Labels Applied by the MFTF Team Refer to the tables with descriptions of each label below. These labels are applied by the MFTF development team to community contributed issues and pull requests, to communicate status, impact, or which team is working on it. @@ -53,12 +53,10 @@ These labels are applied by the MFTF development team to community contributed i | **bugfix** | The issue or pull request relates to bug fixing. | | **enhancement** | The issue or pull request that raises the MFTF to a higher degree (for example new features, optimization, refactoring, etc). | -## Reporting security issues +## Reporting Security Issues -To report security vulnerabilities and other security issues in the Magento software or web sites, send an email with the report at [security@magento.com][]. -Do not report security issues using GitHub. -Be sure to encrypt your e-mail with our [encryption key][] if it includes sensitive information. -Learn more about reporting [security][] issues. +To report security vulnerabilities or learn more about reporting security issues in Magento software or web sites visit the [Magento Bug Bounty Program](https://hackerone.com/magento) on hackerone. +Please create a hackerone account [there](https://hackerone.com/magento) to submit and follow-up on your issue. Stay up-to-date on the latest security news and patches for Magento by signing up for [Security Alert Notifications][]. @@ -72,9 +70,6 @@ See the [license][] or contact [license@magentocommerce.com][] for a copy. [Getting Started]: docs/getting-started.md [Contribution Guidelines]: .github/CONTRIBUTING.md [DevDocs Contributing]: https://github.com/magento/devdocs/blob/master/.github/CONTRIBUTING.md -[security@magento.com]: mailto:security@magento.com -[encryption key]: https://info2.magento.com/rs/magentoenterprise/images/security_at_magento.asc -[security]: https://magento.com/security/reporting-magento-security-issue [Security Alert Notifications]: https://magento.com/security/sign-up [license]: LICENSE_AGPL3.txt [license@magentocommerce.com]: mailto:license@magentocommerce.com From 967043ee5ede64793155df1ab1936b5df1ab4d6a Mon Sep 17 00:00:00 2001 From: Ji Lu <jilu1@adobe.com> Date: Thu, 30 Jul 2020 17:15:47 -0500 Subject: [PATCH 25/52] MQE-2158: support using actions from multiple modules in Suites --- .../Module/MagentoActionProxies.php | 119 ++++++++++++++- .../Module/MagentoWebDriver.php | 137 ++---------------- .../Suite/Generators/GroupClassGenerator.php | 34 ++++- .../Suite/views/SuiteClass.mustache | 50 +++++-- .../Suite/views/partials/testActions.mustache | 6 +- 5 files changed, 194 insertions(+), 152 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoActionProxies.php b/src/Magento/FunctionalTestingFramework/Module/MagentoActionProxies.php index 5a3b2360b..55b19a7c1 100644 --- a/src/Magento/FunctionalTestingFramework/Module/MagentoActionProxies.php +++ b/src/Magento/FunctionalTestingFramework/Module/MagentoActionProxies.php @@ -20,5 +20,122 @@ */ class MagentoActionProxies extends CodeceptionModule { - // TODO: placeholder for proxy functions currently in MagentoWebDriver (MQE-1904) + /** + * Create an entity + * + * @param string $key StepKey of the createData action. + * @param string $scope + * @param string $entity Name of xml entity to create. + * @param array $dependentObjectKeys StepKeys of other createData actions that are required. + * @param array $overrideFields Array of FieldName => Value of override fields. + * @param string $storeCode + * @return void + */ + public function createEntity( + $key, + $scope, + $entity, + $dependentObjectKeys = [], + $overrideFields = [], + $storeCode = '' + ) { + PersistedObjectHandler::getInstance()->createEntity( + $key, + $scope, + $entity, + $dependentObjectKeys, + $overrideFields, + $storeCode + ); + } + + /** + * Retrieves and updates a previously created entity + * + * @param string $key StepKey of the createData action. + * @param string $scope + * @param string $updateEntity Name of the static XML data to update the entity with. + * @param array $dependentObjectKeys StepKeys of other createData actions that are required. + * @return void + */ + public function updateEntity($key, $scope, $updateEntity, $dependentObjectKeys = []) + { + PersistedObjectHandler::getInstance()->updateEntity( + $key, + $scope, + $updateEntity, + $dependentObjectKeys + ); + } + + /** + * Performs GET on given entity and stores entity for use + * + * @param string $key StepKey of getData action. + * @param string $scope + * @param string $entity Name of XML static data to use. + * @param array $dependentObjectKeys StepKeys of other createData actions that are required. + * @param string $storeCode + * @param integer $index + * @return void + */ + public function getEntity($key, $scope, $entity, $dependentObjectKeys = [], $storeCode = '', $index = null) + { + PersistedObjectHandler::getInstance()->getEntity( + $key, + $scope, + $entity, + $dependentObjectKeys, + $storeCode, + $index + ); + } + + /** + * Retrieves and deletes a previously created entity + * + * @param string $key StepKey of the createData action. + * @param string $scope + * @return void + */ + public function deleteEntity($key, $scope) + { + PersistedObjectHandler::getInstance()->deleteEntity($key, $scope); + } + + /** + * Retrieves a field from an entity, according to key and scope given + * + * @param string $stepKey + * @param string $field + * @param string $scope + * @return string + */ + public function retrieveEntityField($stepKey, $field, $scope) + { + return PersistedObjectHandler::getInstance()->retrieveEntityField($stepKey, $field, $scope); + } + + /** + * Get encrypted value by key + * + * @param string $key + * @return string|null + * @throws TestFrameworkException + */ + public function getSecret($key) + { + return CredentialStore::getInstance()->getSecret($key); + } + + /** + * Returns a value to origin of the action + * + * @param mixed $value + * @return mixed + */ + public function return($value) + { + return $value; + } } diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php index 944735677..ef57aa3f1 100644 --- a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php +++ b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php @@ -15,6 +15,7 @@ use Codeception\Exception\ModuleConfigException; use Codeception\Exception\ModuleException; use Codeception\Util\Uri; +use Codeception\Lib\ModuleContainer; use Magento\FunctionalTestingFramework\DataTransport\WebApiExecutor; use Magento\FunctionalTestingFramework\DataTransport\Auth\WebApiAuth; use Magento\FunctionalTestingFramework\DataTransport\Auth\Tfa\OTP; @@ -180,6 +181,16 @@ public function _after(TestInterface $test) // DO NOT RESET SESSIONS } + /** + * Return ModuleContainer + * + * @return ModuleContainer + */ + public function getModuleContainer() + { + return $this->moduleContainer; + } + /** * Returns URL of a host. * @@ -956,120 +967,6 @@ public function getOTP() return OTP::getOTP(); } - /** - * Create an entity - * TODO: move this function to MagentoActionProxies after MQE-1904 - * - * @param string $key StepKey of the createData action. - * @param string $scope - * @param string $entity Name of xml entity to create. - * @param array $dependentObjectKeys StepKeys of other createData actions that are required. - * @param array $overrideFields Array of FieldName => Value of override fields. - * @param string $storeCode - * @return void - */ - public function createEntity( - $key, - $scope, - $entity, - $dependentObjectKeys = [], - $overrideFields = [], - $storeCode = '' - ) { - PersistedObjectHandler::getInstance()->createEntity( - $key, - $scope, - $entity, - $dependentObjectKeys, - $overrideFields, - $storeCode - ); - } - - /** - * Retrieves and updates a previously created entity - * TODO: move this function to MagentoActionProxies after MQE-1904 - * - * @param string $key StepKey of the createData action. - * @param string $scope - * @param string $updateEntity Name of the static XML data to update the entity with. - * @param array $dependentObjectKeys StepKeys of other createData actions that are required. - * @return void - */ - public function updateEntity($key, $scope, $updateEntity, $dependentObjectKeys = []) - { - PersistedObjectHandler::getInstance()->updateEntity( - $key, - $scope, - $updateEntity, - $dependentObjectKeys - ); - } - - /** - * Performs GET on given entity and stores entity for use - * TODO: move this function to MagentoActionProxies after MQE-1904 - * - * @param string $key StepKey of getData action. - * @param string $scope - * @param string $entity Name of XML static data to use. - * @param array $dependentObjectKeys StepKeys of other createData actions that are required. - * @param string $storeCode - * @param integer $index - * @return void - */ - public function getEntity($key, $scope, $entity, $dependentObjectKeys = [], $storeCode = '', $index = null) - { - PersistedObjectHandler::getInstance()->getEntity( - $key, - $scope, - $entity, - $dependentObjectKeys, - $storeCode, - $index - ); - } - - /** - * Retrieves and deletes a previously created entity - * TODO: move this function to MagentoActionProxies after MQE-1904 - * - * @param string $key StepKey of the createData action. - * @param string $scope - * @return void - */ - public function deleteEntity($key, $scope) - { - PersistedObjectHandler::getInstance()->deleteEntity($key, $scope); - } - - /** - * Retrieves a field from an entity, according to key and scope given - * TODO: move this function to MagentoActionProxies after MQE-1904 - * - * @param string $stepKey - * @param string $field - * @param string $scope - * @return string - */ - public function retrieveEntityField($stepKey, $field, $scope) - { - return PersistedObjectHandler::getInstance()->retrieveEntityField($stepKey, $field, $scope); - } - - /** - * Get encrypted value by key - * TODO: move this function to MagentoActionProxies after MQE-1904 - * - * @param string $key - * @return string|null - * @throws TestFrameworkException - */ - public function getSecret($key) - { - return CredentialStore::getInstance()->getSecret($key); - } - /** * Waits proper amount of time to perform Cron execution * @@ -1122,16 +1019,4 @@ public function switchToIFrame($locator = null) $this->webDriver->switchTo()->frame($els[0]); } } - - /** - * Returns a value to origin of the action. - * TODO: move this function to MagentoActionProxies after MQE-1904 - * - * @param mixed $value - * @return mixed - */ - public function return($value) - { - return $value; - } } diff --git a/src/Magento/FunctionalTestingFramework/Suite/Generators/GroupClassGenerator.php b/src/Magento/FunctionalTestingFramework/Suite/Generators/GroupClassGenerator.php index bb0157152..ccac30529 100644 --- a/src/Magento/FunctionalTestingFramework/Suite/Generators/GroupClassGenerator.php +++ b/src/Magento/FunctionalTestingFramework/Suite/Generators/GroupClassGenerator.php @@ -6,6 +6,7 @@ namespace Magento\FunctionalTestingFramework\Suite\Generators; +use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; use Magento\FunctionalTestingFramework\Exceptions\TestReferenceException; use Magento\FunctionalTestingFramework\Suite\Objects\SuiteObject; use Magento\FunctionalTestingFramework\Test\Objects\ActionObject; @@ -32,6 +33,10 @@ class GroupClassGenerator 'comment' => 'print' ]; const GROUP_DIR_NAME = 'Group'; + const FUNCTION_PLACEHOLDER = 'PLACEHOLDER'; + const FUNCTION_START = 'this->getModuleForAction("'; + const FUNCTION_END = '")'; + const FUNCTION_REPLACE_REGEX = '/(PLACEHOLDER->([^\(]+))\(/'; /** * Mustache_Engine instance for template loading @@ -142,7 +147,7 @@ private function buildHookMustacheArray($hookObj) //deleteData contains either url or createDataKey, if it contains the former it needs special formatting if ($action->getType() !== "createData" && !array_key_exists(TestGenerator::REQUIRED_ENTITY_REFERENCE, $action->getCustomActionAttributes())) { - $actions = $this->buildWebDriverActionsMustacheArray($action, $actions); + $actions = $this->buildModuleActionsMustacheArray($action, $actions); continue; } @@ -165,7 +170,7 @@ private function buildHookMustacheArray($hookObj) } /** - * Takes an action object and array of generated action steps. Converst the action object into generated php and + * Takes an action object and array of generated action steps. Convert the action object into generated php and * appends the entry to the given array. The result is returned by the function. * * @param ActionObject $action @@ -173,14 +178,21 @@ private function buildHookMustacheArray($hookObj) * @return array * @throws TestReferenceException */ - private function buildWebDriverActionsMustacheArray($action, $actionEntries) + private function buildModuleActionsMustacheArray($action, $actionEntries) { - $step = TestGenerator::getInstance()->generateStepsPhp([$action], TestGenerator::SUITE_SCOPE, 'webDriver'); + $step = TestGenerator::getInstance()->generateStepsPhp( + [$action], + TestGenerator::SUITE_SCOPE, + self::FUNCTION_PLACEHOLDER + ); $rawPhp = str_replace(["\t"], "", $step); $multipleCommands = explode(PHP_EOL, $rawPhp, -1); $multipleCommands = array_filter($multipleCommands); foreach ($multipleCommands as $command) { - $actionEntries = $this->replaceReservedTesterFunctions($command . PHP_EOL, $actionEntries, 'webDriver'); + $actionEntries = $this->replaceReservedTesterFunctions( + $command . PHP_EOL, $actionEntries, + self::FUNCTION_PLACEHOLDER + ); } return $actionEntries; @@ -204,7 +216,17 @@ private function replaceReservedTesterFunctions($formattedStep, $actionEntries, $resultingStep = str_replace($testActionCall, $replacement, $formattedStep); $actionEntries[] = ['action' => $resultingStep]; } else { - $actionEntries[] = ['action' => $formattedStep]; + $placeholder = self::FUNCTION_PLACEHOLDER; + $begin = self::FUNCTION_START; + $end = self::FUNCTION_END; + $resultingStep = preg_replace_callback( + self::FUNCTION_REPLACE_REGEX, + function($matches) use ($placeholder, $begin, $end) { + return str_replace($placeholder, $begin . $matches[2] . $end, $matches[1]) . '('; + }, + $formattedStep + ); + $actionEntries[] = ['action' => $resultingStep]; } } diff --git a/src/Magento/FunctionalTestingFramework/Suite/views/SuiteClass.mustache b/src/Magento/FunctionalTestingFramework/Suite/views/SuiteClass.mustache index a9fdcd6df..dc2fde915 100644 --- a/src/Magento/FunctionalTestingFramework/Suite/views/SuiteClass.mustache +++ b/src/Magento/FunctionalTestingFramework/Suite/views/SuiteClass.mustache @@ -6,6 +6,11 @@ use Facebook\WebDriver\Remote\RemoteWebDriver; use Magento\FunctionalTestingFramework\DataGenerator\Handlers\PersistedObjectHandler; use Magento\FunctionalTestingFramework\DataGenerator\Handlers\CredentialStore; use Magento\FunctionalTestingFramework\Module\MagentoWebDriver; +use Magento\FunctionalTestingFramework\Module\MagentoAssert; +use Magento\FunctionalTestingFramework\Module\MagentoActionProxies; +use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; +use Codeception\Lib\ModuleContainer; +use Codeception\Module; /** * Group class is Codeception Extension which is allowed to handle to all internal events. @@ -30,10 +35,16 @@ class {{suiteName}} extends \Codeception\GroupObject {{/helpers}} private static $HOOK_EXECUTION_INIT = "\n/******** Beginning execution of {{suiteName}} suite %s block ********/\n"; private static $HOOK_EXECUTION_END = "\n/******** Execution of {{suiteName}} suite %s block complete ********/\n"; + /** @var MagentoWebDriver */ + private $webDriver; + /** @var ModuleContainer */ + private $moduleContainer; {{#before}} public function _before(\Codeception\Event\TestEvent $e) { + $this->webDriver = $this->getModule('\Magento\FunctionalTestingFramework\Module\MagentoWebDriver'); + $this->moduleContainer = $this->webDriver->getModuleContainer(); {{#helpers}} /** @var \Magento\FunctionalTestingFramework\Helper\HelperContainer $helperContainer */ $this->helperContainer = $this->getModule('\Magento\FunctionalTestingFramework\Helper\HelperContainer'); @@ -51,15 +62,11 @@ class {{suiteName}} extends \Codeception\GroupObject } } - private function executePreConditions() { if ($this->currentTestRun == 1) { print sprintf(self::$HOOK_EXECUTION_INIT, "before"); - /** @var MagentoWebDriver $webDriver */ - $webDriver = $this->getModule('\Magento\FunctionalTestingFramework\Module\MagentoWebDriver'); - try { {{> testActions}} } catch (\Exception $exception) { @@ -67,9 +74,9 @@ class {{suiteName}} extends \Codeception\GroupObject } // reset configuration and close session - $webDriver->_resetConfig(); - $webDriver->webDriver->close(); - $webDriver->webDriver = null; + $this->webDriver->_resetConfig(); + $this->webDriver->webDriver->close(); + $this->webDriver->webDriver = null; print sprintf(self::$HOOK_EXECUTION_END, "before"); } @@ -82,15 +89,11 @@ class {{suiteName}} extends \Codeception\GroupObject $this->executePostConditions($e); } - private function executePostConditions(\Codeception\Event\TestEvent $e) { if ($this->currentTestRun == $this->testCount) { print sprintf(self::$HOOK_EXECUTION_INIT, "after"); - /** @var MagentoWebDriver $webDriver */ - $webDriver = $this->getModule('\Magento\FunctionalTestingFramework\Module\MagentoWebDriver'); - try { // Find out if Test in Suite failed, will cause potential failures in suite after $cest = $e->getTest(); @@ -120,7 +123,7 @@ class {{suiteName}} extends \Codeception\GroupObject PersistedObjectHandler::getInstance()->clearSuiteObjects(); - $this->closeSession($webDriver); + $this->closeSession($this->webDriver); print sprintf(self::$HOOK_EXECUTION_END, "after"); } @@ -131,13 +134,12 @@ class {{suiteName}} extends \Codeception\GroupObject * Close session method closes current session. * If config 'close_all_sessions' is set to 'true' all sessions will be closed. * - * @param MagentoWebDriver $webDriver * return void */ - private function closeSession(MagentoWebDriver $webDriver): void + private function closeSession(): void { - $webDriverConfig = $webDriver->_getConfig(); - $webDriver->_closeSession(); + $webDriverConfig = $this->webDriver->_getConfig(); + $this->webDriver->_closeSession(); if (isset($webDriverConfig['close_all_sessions']) && $webDriverConfig['close_all_sessions'] === "true") { $wdHost = sprintf( '%s://%s:%s%s', @@ -153,4 +155,20 @@ class {{suiteName}} extends \Codeception\GroupObject } } } + + /** + * Return the module for an action. + * + * @param string $action + * @return Module + * @throws \Exception + */ + private function getModuleForAction($action) + { + $module = $this->moduleContainer->moduleForAction($action); + if ($module === null) { + throw new TestFrameworkException('Invalid action "' . $action . '"' . PHP_EOL); + } + return $module; + } } diff --git a/src/Magento/FunctionalTestingFramework/Suite/views/partials/testActions.mustache b/src/Magento/FunctionalTestingFramework/Suite/views/partials/testActions.mustache index eaa0959cb..c07958778 100644 --- a/src/Magento/FunctionalTestingFramework/Suite/views/partials/testActions.mustache +++ b/src/Magento/FunctionalTestingFramework/Suite/views/partials/testActions.mustache @@ -1,9 +1,9 @@ {{#actions}} {{#webDriverInit}} -if ($webDriver->webDriver != null) { - $webDriver->_restart(); +if ($this->webDriver->webDriver != null) { + $this->webDriver->_restart(); } else { - $webDriver->_initializeSession(); + $this->webDriver->_initializeSession(); } {{/webDriverInit}} {{#action}} From fc05c77aa68f72eae7bdc294933935c4148127f5 Mon Sep 17 00:00:00 2001 From: Ji Lu <jilu1@adobe.com> Date: Thu, 30 Jul 2020 17:33:17 -0500 Subject: [PATCH 26/52] MQE-2158: support using actions from multiple modules in Suites --- .../Suite/Generators/GroupClassGenerator.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/Suite/Generators/GroupClassGenerator.php b/src/Magento/FunctionalTestingFramework/Suite/Generators/GroupClassGenerator.php index ccac30529..f90a25322 100644 --- a/src/Magento/FunctionalTestingFramework/Suite/Generators/GroupClassGenerator.php +++ b/src/Magento/FunctionalTestingFramework/Suite/Generators/GroupClassGenerator.php @@ -190,7 +190,8 @@ private function buildModuleActionsMustacheArray($action, $actionEntries) $multipleCommands = array_filter($multipleCommands); foreach ($multipleCommands as $command) { $actionEntries = $this->replaceReservedTesterFunctions( - $command . PHP_EOL, $actionEntries, + $command . PHP_EOL, + $actionEntries, self::FUNCTION_PLACEHOLDER ); } @@ -221,7 +222,7 @@ private function replaceReservedTesterFunctions($formattedStep, $actionEntries, $end = self::FUNCTION_END; $resultingStep = preg_replace_callback( self::FUNCTION_REPLACE_REGEX, - function($matches) use ($placeholder, $begin, $end) { + function ($matches) use ($placeholder, $begin, $end) { return str_replace($placeholder, $begin . $matches[2] . $end, $matches[1]) . '('; }, $formattedStep From 7b53949b3b6bc9a0d0129e01f14155d436e1a215 Mon Sep 17 00:00:00 2001 From: Ji Lu <jilu1@adobe.com> Date: Fri, 31 Jul 2020 08:59:26 -0500 Subject: [PATCH 27/52] MQE-2158: support using actions from multiple modules in Suites --- .../Resources/functionalSuiteHooks.txt | 74 ++++--- .../functionalSuiteUsingSecretData.txt | 205 ++++++++++++++++++ .../Resources/functionalSuiteWithComments.txt | 68 +++--- .../Data/PersistedReplacementData.xml | 4 + .../Suite/functionalSuiteUsingSecretData.xml | 36 +++ .../IncludeUsingSecretDataTest.xml | 17 ++ .../Tests/SuiteGenerationTest.php | 59 +++++ 7 files changed, 410 insertions(+), 53 deletions(-) create mode 100644 dev/tests/verification/Resources/functionalSuiteUsingSecretData.txt create mode 100644 dev/tests/verification/TestModule/Suite/functionalSuiteUsingSecretData.xml create mode 100644 dev/tests/verification/TestModule/Test/SampleSuiteTest/IncludeUsingSecretDataTest.xml diff --git a/dev/tests/verification/Resources/functionalSuiteHooks.txt b/dev/tests/verification/Resources/functionalSuiteHooks.txt index 041b088b1..3156a59dd 100644 --- a/dev/tests/verification/Resources/functionalSuiteHooks.txt +++ b/dev/tests/verification/Resources/functionalSuiteHooks.txt @@ -6,6 +6,11 @@ use Facebook\WebDriver\Remote\RemoteWebDriver; use Magento\FunctionalTestingFramework\DataGenerator\Handlers\PersistedObjectHandler; use Magento\FunctionalTestingFramework\DataGenerator\Handlers\CredentialStore; use Magento\FunctionalTestingFramework\Module\MagentoWebDriver; +use Magento\FunctionalTestingFramework\Module\MagentoAssert; +use Magento\FunctionalTestingFramework\Module\MagentoActionProxies; +use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; +use Codeception\Lib\ModuleContainer; +use Codeception\Module; /** * Group class is Codeception Extension which is allowed to handle to all internal events. @@ -24,9 +29,15 @@ class functionalSuiteHooks extends \Codeception\GroupObject private $currentTestRun = 0; private static $HOOK_EXECUTION_INIT = "\n/******** Beginning execution of functionalSuiteHooks suite %s block ********/\n"; private static $HOOK_EXECUTION_END = "\n/******** Execution of functionalSuiteHooks suite %s block complete ********/\n"; + /** @var MagentoWebDriver */ + private $webDriver; + /** @var ModuleContainer */ + private $moduleContainer; public function _before(\Codeception\Event\TestEvent $e) { + $this->webDriver = $this->getModule('\Magento\FunctionalTestingFramework\Module\MagentoWebDriver'); + $this->moduleContainer = $this->webDriver->getModuleContainer(); // increment test count per execution $this->currentTestRun++; $this->executePreConditions(); @@ -37,22 +48,18 @@ class functionalSuiteHooks extends \Codeception\GroupObject } } - private function executePreConditions() { if ($this->currentTestRun == 1) { print sprintf(self::$HOOK_EXECUTION_INIT, "before"); - /** @var MagentoWebDriver $webDriver */ - $webDriver = $this->getModule('\Magento\FunctionalTestingFramework\Module\MagentoWebDriver'); - try { - if ($webDriver->webDriver != null) { - $webDriver->_restart(); + if ($this->webDriver->webDriver != null) { + $this->webDriver->_restart(); } else { - $webDriver->_initializeSession(); + $this->webDriver->_initializeSession(); } - $webDriver->amOnPage("some.url"); // stepKey: before + $this->getModuleForAction("amOnPage")->amOnPage("some.url"); // stepKey: before $createOneFields['someKey'] = "dataHere"; PersistedObjectHandler::getInstance()->createEntity( "createOne", @@ -79,18 +86,18 @@ class functionalSuiteHooks extends \Codeception\GroupObject "createEntityFour", ["createEntityTwo", "createEntityThree"] ); - $webDriver->click(PersistedObjectHandler::getInstance()->retrieveEntityField('createTwo', 'data', 'suite')); // stepKey: clickWithData + $this->getModuleForAction("click")->click(PersistedObjectHandler::getInstance()->retrieveEntityField('createTwo', 'data', 'suite')); // stepKey: clickWithData print("Entering Action Group [AC] actionGroupWithTwoArguments"); - $webDriver->see("John", msq("uniqueData") . "John"); // stepKey: seeFirstNameAC + $this->getModuleForAction("see")->see("John", msq("uniqueData") . "John"); // stepKey: seeFirstNameAC print("Exiting Action Group [AC] actionGroupWithTwoArguments"); } catch (\Exception $exception) { $this->preconditionFailure = $exception->getMessage(); } // reset configuration and close session - $webDriver->_resetConfig(); - $webDriver->webDriver->close(); - $webDriver->webDriver = null; + $this->webDriver->_resetConfig(); + $this->webDriver->webDriver->close(); + $this->webDriver->webDriver = null; print sprintf(self::$HOOK_EXECUTION_END, "before"); } @@ -101,15 +108,11 @@ class functionalSuiteHooks extends \Codeception\GroupObject $this->executePostConditions($e); } - private function executePostConditions(\Codeception\Event\TestEvent $e) { if ($this->currentTestRun == $this->testCount) { print sprintf(self::$HOOK_EXECUTION_INIT, "after"); - /** @var MagentoWebDriver $webDriver */ - $webDriver = $this->getModule('\Magento\FunctionalTestingFramework\Module\MagentoWebDriver'); - try { // Find out if Test in Suite failed, will cause potential failures in suite after $cest = $e->getTest(); @@ -132,15 +135,15 @@ class functionalSuiteHooks extends \Codeception\GroupObject } } } - if ($webDriver->webDriver != null) { - $webDriver->_restart(); + if ($this->webDriver->webDriver != null) { + $this->webDriver->_restart(); } else { - $webDriver->_initializeSession(); + $this->webDriver->_initializeSession(); } - $webDriver->amOnPage("some.url"); // stepKey: after - $webDriver->deleteEntityByUrl("deleteThis"); // stepKey: delete + $this->getModuleForAction("amOnPage")->amOnPage("some.url"); // stepKey: after + $this->getModuleForAction("deleteEntityByUrl")->deleteEntityByUrl("deleteThis"); // stepKey: delete print("Entering Action Group [AC] actionGroupWithTwoArguments"); - $webDriver->see("John", msq("uniqueData") . "John"); // stepKey: seeFirstNameAC + $this->getModuleForAction("see")->see("John", msq("uniqueData") . "John"); // stepKey: seeFirstNameAC print("Exiting Action Group [AC] actionGroupWithTwoArguments"); } catch (\Exception $exception) { print $exception->getMessage(); @@ -148,7 +151,7 @@ class functionalSuiteHooks extends \Codeception\GroupObject PersistedObjectHandler::getInstance()->clearSuiteObjects(); - $this->closeSession($webDriver); + $this->closeSession($this->webDriver); print sprintf(self::$HOOK_EXECUTION_END, "after"); } @@ -158,13 +161,12 @@ class functionalSuiteHooks extends \Codeception\GroupObject * Close session method closes current session. * If config 'close_all_sessions' is set to 'true' all sessions will be closed. * - * @param MagentoWebDriver $webDriver * return void */ - private function closeSession(MagentoWebDriver $webDriver): void + private function closeSession(): void { - $webDriverConfig = $webDriver->_getConfig(); - $webDriver->_closeSession(); + $webDriverConfig = $this->webDriver->_getConfig(); + $this->webDriver->_closeSession(); if (isset($webDriverConfig['close_all_sessions']) && $webDriverConfig['close_all_sessions'] === "true") { $wdHost = sprintf( '%s://%s:%s%s', @@ -180,4 +182,20 @@ class functionalSuiteHooks extends \Codeception\GroupObject } } } + + /** + * Return the module for an action. + * + * @param string $action + * @return Module + * @throws \Exception + */ + private function getModuleForAction($action) + { + $module = $this->moduleContainer->moduleForAction($action); + if ($module === null) { + throw new TestFrameworkException('Invalid action "' . $action . '"' . PHP_EOL); + } + return $module; + } } diff --git a/dev/tests/verification/Resources/functionalSuiteUsingSecretData.txt b/dev/tests/verification/Resources/functionalSuiteUsingSecretData.txt new file mode 100644 index 000000000..1be43a544 --- /dev/null +++ b/dev/tests/verification/Resources/functionalSuiteUsingSecretData.txt @@ -0,0 +1,205 @@ +<?php + +namespace Group; + +use Facebook\WebDriver\Remote\RemoteWebDriver; +use Magento\FunctionalTestingFramework\DataGenerator\Handlers\PersistedObjectHandler; +use Magento\FunctionalTestingFramework\DataGenerator\Handlers\CredentialStore; +use Magento\FunctionalTestingFramework\Module\MagentoWebDriver; +use Magento\FunctionalTestingFramework\Module\MagentoAssert; +use Magento\FunctionalTestingFramework\Module\MagentoActionProxies; +use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; +use Codeception\Lib\ModuleContainer; +use Codeception\Module; + +/** + * Group class is Codeception Extension which is allowed to handle to all internal events. + * This class itself can be used to listen events for test execution of one particular group. + * It may be especially useful to create fixtures data, prepare server, etc. + * + * INSTALLATION: + * + * To use this group extension, include it to "extensions" option of global Codeception config. + */ +class functionalSuiteUsingSecretData extends \Codeception\GroupObject +{ + public static $group = 'functionalSuiteUsingSecretData'; + private $testCount = 1; + private $preconditionFailure = null; + private $currentTestRun = 0; + private static $HOOK_EXECUTION_INIT = "\n/******** Beginning execution of functionalSuiteUsingSecretData suite %s block ********/\n"; + private static $HOOK_EXECUTION_END = "\n/******** Execution of functionalSuiteUsingSecretData suite %s block complete ********/\n"; + /** @var MagentoWebDriver */ + private $webDriver; + /** @var ModuleContainer */ + private $moduleContainer; + + public function _before(\Codeception\Event\TestEvent $e) + { + $this->webDriver = $this->getModule('\Magento\FunctionalTestingFramework\Module\MagentoWebDriver'); + $this->moduleContainer = $this->webDriver->getModuleContainer(); + // increment test count per execution + $this->currentTestRun++; + $this->executePreConditions(); + + if ($this->preconditionFailure != null) { + //if our preconditions fail, we need to mark all the tests as incomplete. + $e->getTest()->getMetadata()->setIncomplete("SUITE PRECONDITION FAILED:" . PHP_EOL . $this->preconditionFailure); + } + } + + private function executePreConditions() + { + if ($this->currentTestRun == 1) { + print sprintf(self::$HOOK_EXECUTION_INIT, "before"); + + try { + if ($this->webDriver->webDriver != null) { + $this->webDriver->_restart(); + } else { + $this->webDriver->_initializeSession(); + } + $cli = $this->getModuleForAction("magentoCLISecret")->magentoCLISecret($this->getModuleForAction("getSecret")->getSecret("magento/some/secret"), 60); // stepKey: cli + print($cli); // stepKey: cli + $create1Fields['someKey'] = "dataHere"; + PersistedObjectHandler::getInstance()->createEntity( + "create1", + "suite", + "SecretData", + [], + $create1Fields + ); + PersistedObjectHandler::getInstance()->createEntity( + "create2", + "suite", + "SecretData", + [] + ); + PersistedObjectHandler::getInstance()->createEntity( + "create3", + "suite", + "SecretData", + ["create1", "create2"] + ); + $this->getModuleForAction("fillSecretField")->fillSecretField("#fill", $this->getModuleForAction("getSecret")->getSecret("magento/some/secret") . "+" . $this->getModuleForAction("getSecret")->getSecret("magento/some/secret")); // stepKey: fillBefore + $this->getModuleForAction("click")->click(PersistedObjectHandler::getInstance()->retrieveEntityField('create2', 'key2', 'suite')); // stepKey: click + } catch (\Exception $exception) { + $this->preconditionFailure = $exception->getMessage(); + } + + // reset configuration and close session + $this->webDriver->_resetConfig(); + $this->webDriver->webDriver->close(); + $this->webDriver->webDriver = null; + + print sprintf(self::$HOOK_EXECUTION_END, "before"); + } + } + + public function _after(\Codeception\Event\TestEvent $e) + { + $this->executePostConditions($e); + } + + private function executePostConditions(\Codeception\Event\TestEvent $e) + { + if ($this->currentTestRun == $this->testCount) { + print sprintf(self::$HOOK_EXECUTION_INIT, "after"); + + try { + // Find out if Test in Suite failed, will cause potential failures in suite after + $cest = $e->getTest(); + + //Access private TestResultObject to find stack and if there are any errors (as opposed to failures) + $testResultObject = call_user_func(\Closure::bind( + function () use ($cest) { + return $cest->getTestResultObject(); + }, + $cest + )); + $errors = $testResultObject->errors(); + + if (!empty($errors)) { + foreach ($errors as $error) { + if ($error->failedTest()->getTestMethod() == $cest->getName()) { + // Do not attempt to run _after if failure was in the _after block + // Try to run _after but catch exceptions to prevent them from overwriting original failure. + print("LAST TEST IN SUITE FAILED, TEST AFTER MAY NOT BE SUCCESSFUL\n"); + } + } + } + if ($this->webDriver->webDriver != null) { + $this->webDriver->_restart(); + } else { + $this->webDriver->_initializeSession(); + } + PersistedObjectHandler::getInstance()->deleteEntity( + "create1", + "suite" + ); + PersistedObjectHandler::getInstance()->deleteEntity( + "create2", + "suite" + ); + PersistedObjectHandler::getInstance()->deleteEntity( + "create3", + "suite" + ); + $this->getModuleForAction("deleteEntityByUrl")->deleteEntityByUrl("deleteThis"); // stepKey: deleteThis + $this->getModuleForAction("fillSecretField")->fillSecretField("#fill", $this->getModuleForAction("getSecret")->getSecret("magento/some/secret")); // stepKey: fillAfter + $cli2 = $this->getModuleForAction("magentoCLISecret")->magentoCLISecret($this->getModuleForAction("getSecret")->getSecret("magento/some/secret") . "-some/data-" . $this->getModuleForAction("getSecret")->getSecret("magento/some/secret"), 60); // stepKey: cli2 + print($cli2); // stepKey: cli2 + } catch (\Exception $exception) { + print $exception->getMessage(); + } + + PersistedObjectHandler::getInstance()->clearSuiteObjects(); + + $this->closeSession($this->webDriver); + + print sprintf(self::$HOOK_EXECUTION_END, "after"); + } + } + + /** + * Close session method closes current session. + * If config 'close_all_sessions' is set to 'true' all sessions will be closed. + * + * return void + */ + private function closeSession(): void + { + $webDriverConfig = $this->webDriver->_getConfig(); + $this->webDriver->_closeSession(); + if (isset($webDriverConfig['close_all_sessions']) && $webDriverConfig['close_all_sessions'] === "true") { + $wdHost = sprintf( + '%s://%s:%s%s', + $webDriverConfig['protocol'], + $webDriverConfig['host'], + $webDriverConfig['port'], + $webDriverConfig['path'] + ); + $availableSessions = RemoteWebDriver::getAllSessions($wdHost); + foreach ($availableSessions as $session) { + $remoteWebDriver = RemoteWebDriver::createBySessionID($session['id'], $wdHost); + $remoteWebDriver->quit(); + } + } + } + + /** + * Return the module for an action. + * + * @param string $action + * @return Module + * @throws \Exception + */ + private function getModuleForAction($action) + { + $module = $this->moduleContainer->moduleForAction($action); + if ($module === null) { + throw new TestFrameworkException('Invalid action "' . $action . '"' . PHP_EOL); + } + return $module; + } +} diff --git a/dev/tests/verification/Resources/functionalSuiteWithComments.txt b/dev/tests/verification/Resources/functionalSuiteWithComments.txt index 6c646dd18..02b50f092 100644 --- a/dev/tests/verification/Resources/functionalSuiteWithComments.txt +++ b/dev/tests/verification/Resources/functionalSuiteWithComments.txt @@ -6,6 +6,11 @@ use Facebook\WebDriver\Remote\RemoteWebDriver; use Magento\FunctionalTestingFramework\DataGenerator\Handlers\PersistedObjectHandler; use Magento\FunctionalTestingFramework\DataGenerator\Handlers\CredentialStore; use Magento\FunctionalTestingFramework\Module\MagentoWebDriver; +use Magento\FunctionalTestingFramework\Module\MagentoAssert; +use Magento\FunctionalTestingFramework\Module\MagentoActionProxies; +use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; +use Codeception\Lib\ModuleContainer; +use Codeception\Module; /** * Group class is Codeception Extension which is allowed to handle to all internal events. @@ -24,9 +29,15 @@ class functionalSuiteWithComments extends \Codeception\GroupObject private $currentTestRun = 0; private static $HOOK_EXECUTION_INIT = "\n/******** Beginning execution of functionalSuiteWithComments suite %s block ********/\n"; private static $HOOK_EXECUTION_END = "\n/******** Execution of functionalSuiteWithComments suite %s block complete ********/\n"; + /** @var MagentoWebDriver */ + private $webDriver; + /** @var ModuleContainer */ + private $moduleContainer; public function _before(\Codeception\Event\TestEvent $e) { + $this->webDriver = $this->getModule('\Magento\FunctionalTestingFramework\Module\MagentoWebDriver'); + $this->moduleContainer = $this->webDriver->getModuleContainer(); // increment test count per execution $this->currentTestRun++; $this->executePreConditions(); @@ -37,23 +48,19 @@ class functionalSuiteWithComments extends \Codeception\GroupObject } } - private function executePreConditions() { if ($this->currentTestRun == 1) { print sprintf(self::$HOOK_EXECUTION_INIT, "before"); - /** @var MagentoWebDriver $webDriver */ - $webDriver = $this->getModule('\Magento\FunctionalTestingFramework\Module\MagentoWebDriver'); - try { - if ($webDriver->webDriver != null) { - $webDriver->_restart(); + if ($this->webDriver->webDriver != null) { + $this->webDriver->_restart(); } else { - $webDriver->_initializeSession(); + $this->webDriver->_initializeSession(); } print("Comment in Before"); - $webDriver->amOnPage("some.url"); // stepKey: before + $this->getModuleForAction("amOnPage")->amOnPage("some.url"); // stepKey: before $createFields['someKey'] = "dataHere"; PersistedObjectHandler::getInstance()->createEntity( "create", @@ -63,18 +70,18 @@ class functionalSuiteWithComments extends \Codeception\GroupObject $createFields ); print("<click stepKey=\"comment with element\" userInput=\"helloworld\"/>"); - $webDriver->click(PersistedObjectHandler::getInstance()->retrieveEntityField('create', 'data', 'suite')); // stepKey: clickWithData + $this->getModuleForAction("click")->click(PersistedObjectHandler::getInstance()->retrieveEntityField('create', 'data', 'suite')); // stepKey: clickWithData print("Entering Action Group [AC] actionGroupWithTwoArguments"); - $webDriver->see("John", msq("uniqueData") . "John"); // stepKey: seeFirstNameAC + $this->getModuleForAction("see")->see("John", msq("uniqueData") . "John"); // stepKey: seeFirstNameAC print("Exiting Action Group [AC] actionGroupWithTwoArguments"); } catch (\Exception $exception) { $this->preconditionFailure = $exception->getMessage(); } // reset configuration and close session - $webDriver->_resetConfig(); - $webDriver->webDriver->close(); - $webDriver->webDriver = null; + $this->webDriver->_resetConfig(); + $this->webDriver->webDriver->close(); + $this->webDriver->webDriver = null; print sprintf(self::$HOOK_EXECUTION_END, "before"); } @@ -85,15 +92,11 @@ class functionalSuiteWithComments extends \Codeception\GroupObject $this->executePostConditions($e); } - private function executePostConditions(\Codeception\Event\TestEvent $e) { if ($this->currentTestRun == $this->testCount) { print sprintf(self::$HOOK_EXECUTION_INIT, "after"); - /** @var MagentoWebDriver $webDriver */ - $webDriver = $this->getModule('\Magento\FunctionalTestingFramework\Module\MagentoWebDriver'); - try { // Find out if Test in Suite failed, will cause potential failures in suite after $cest = $e->getTest(); @@ -116,10 +119,10 @@ class functionalSuiteWithComments extends \Codeception\GroupObject } } } - if ($webDriver->webDriver != null) { - $webDriver->_restart(); + if ($this->webDriver->webDriver != null) { + $this->webDriver->_restart(); } else { - $webDriver->_initializeSession(); + $this->webDriver->_initializeSession(); } print("afterBlock"); } catch (\Exception $exception) { @@ -128,7 +131,7 @@ class functionalSuiteWithComments extends \Codeception\GroupObject PersistedObjectHandler::getInstance()->clearSuiteObjects(); - $this->closeSession($webDriver); + $this->closeSession($this->webDriver); print sprintf(self::$HOOK_EXECUTION_END, "after"); } @@ -138,13 +141,12 @@ class functionalSuiteWithComments extends \Codeception\GroupObject * Close session method closes current session. * If config 'close_all_sessions' is set to 'true' all sessions will be closed. * - * @param MagentoWebDriver $webDriver * return void */ - private function closeSession(MagentoWebDriver $webDriver): void + private function closeSession(): void { - $webDriverConfig = $webDriver->_getConfig(); - $webDriver->_closeSession(); + $webDriverConfig = $this->webDriver->_getConfig(); + $this->webDriver->_closeSession(); if (isset($webDriverConfig['close_all_sessions']) && $webDriverConfig['close_all_sessions'] === "true") { $wdHost = sprintf( '%s://%s:%s%s', @@ -160,4 +162,20 @@ class functionalSuiteWithComments extends \Codeception\GroupObject } } } + + /** + * Return the module for an action. + * + * @param string $action + * @return Module + * @throws \Exception + */ + private function getModuleForAction($action) + { + $module = $this->moduleContainer->moduleForAction($action); + if ($module === null) { + throw new TestFrameworkException('Invalid action "' . $action . '"' . PHP_EOL); + } + return $module; + } } diff --git a/dev/tests/verification/TestModule/Data/PersistedReplacementData.xml b/dev/tests/verification/TestModule/Data/PersistedReplacementData.xml index 514b5eec6..7213fa6b9 100644 --- a/dev/tests/verification/TestModule/Data/PersistedReplacementData.xml +++ b/dev/tests/verification/TestModule/Data/PersistedReplacementData.xml @@ -22,4 +22,8 @@ <data key="lastName">Dane</data> <data key="mergedField">unmerged</data> </entity> + <entity name="SecretData" type="SecretData"> + <data key="key1">some/data</data> + <data key="key2">{{_CREDS.magento/some/secret}}</data> + </entity> </entities> diff --git a/dev/tests/verification/TestModule/Suite/functionalSuiteUsingSecretData.xml b/dev/tests/verification/TestModule/Suite/functionalSuiteUsingSecretData.xml new file mode 100644 index 000000000..fdb9164cb --- /dev/null +++ b/dev/tests/verification/TestModule/Suite/functionalSuiteUsingSecretData.xml @@ -0,0 +1,36 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + /** + * Copyright © Magento, Inc. All rights reserved. + * See COPYING.txt for license details. + */ +--> + +<suites xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="urn:magento:mftf:Suite/etc/suiteSchema.xsd"> + <suite name="functionalSuiteUsingSecretData"> + <include> + <test name="IncludeUsingSecretDataTest"/> + </include> + <before> + <magentoCLI command="{{SecretData.key2}}" stepKey="cli"/> + <createData entity="SecretData" stepKey="create1"> + <field key="someKey">dataHere</field> + </createData> + <createData entity="SecretData" stepKey="create2"/> + <createData entity="SecretData" stepKey="create3"> + <requiredEntity createDataKey="create1"/> + <requiredEntity createDataKey="create2"/> + </createData> + <fillField selector="#fill" userInput="{{SecretData.key2}}+{{SecretData.key2}}" stepKey="fillBefore"/> + <click userInput="$create2.key2$" stepKey="click"/> + </before> + <after> + <deleteData createDataKey="create1" stepKey="delete1"/> + <deleteData createDataKey="create2" stepKey="delete2"/> + <deleteData createDataKey="create3" stepKey="delete3"/> + <deleteData url="deleteThis" stepKey="deleteThis"/> + <fillField selector="#fill" userInput="{{SecretData.key2}}" stepKey="fillAfter"/> + <magentoCLI command="{{SecretData.key2}}-{{SecretData.key1}}-{{SecretData.key2}}" stepKey="cli2"/> + </after> + </suite> +</suites> diff --git a/dev/tests/verification/TestModule/Test/SampleSuiteTest/IncludeUsingSecretDataTest.xml b/dev/tests/verification/TestModule/Test/SampleSuiteTest/IncludeUsingSecretDataTest.xml new file mode 100644 index 000000000..14360e2f3 --- /dev/null +++ b/dev/tests/verification/TestModule/Test/SampleSuiteTest/IncludeUsingSecretDataTest.xml @@ -0,0 +1,17 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + /** + * Copyright © Magento, Inc. All rights reserved. + * See COPYING.txt for license details. + */ +--> +<tests xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:noNamespaceSchemaLocation="urn:magento:mftf:Test/etc/testSchema.xsd"> + <test name="IncludeUsingSecretDataTest"> + <createData entity="SecretData" stepKey="create"> + <field key="someKey">dataHere</field> + </createData> + <fillField selector="#fill" userInput="{{SecretData.key2}}+$create.key2$" stepKey="fill"/> + <magentoCLI command="create.key2$" stepKey="cli"/> + </test> +</tests> diff --git a/dev/tests/verification/Tests/SuiteGenerationTest.php b/dev/tests/verification/Tests/SuiteGenerationTest.php index 4a8371692..633146eef 100644 --- a/dev/tests/verification/Tests/SuiteGenerationTest.php +++ b/dev/tests/verification/Tests/SuiteGenerationTest.php @@ -388,6 +388,65 @@ public function testSuiteCommentsGeneration() ); } + /** + * Test hook groups generated during suite generation + */ + public function testSuiteGenerationCreds() + { + $groupName = 'functionalSuiteUsingSecretData'; + + $expectedContents = [ + 'IncludeUsingSecretDataTestCest.php' + ]; + + // Generate the Suite + SuiteGenerator::getInstance()->generateSuite($groupName); + + // Validate log message and add group name for later deletion + TestLoggingUtil::getInstance()->validateMockLogStatement( + 'info', + "suite generated", + ['suite' => $groupName, 'relative_path' => "_generated" . DIRECTORY_SEPARATOR . $groupName] + ); + self::$TEST_GROUPS[] = $groupName; + + // Validate Yaml file updated + $yml = Yaml::parse(file_get_contents(self::CONFIG_YML_FILE)); + $this->assertArrayHasKey($groupName, $yml['groups']); + + $suiteResultBaseDir = self::GENERATE_RESULT_DIR . + DIRECTORY_SEPARATOR . + $groupName . + DIRECTORY_SEPARATOR; + + // Validate tests have been generated + $dirContents = array_diff(scandir($suiteResultBaseDir), ['..', '.']); + + foreach ($expectedContents as $expectedFile) { + $this->assertTrue(in_array($expectedFile, $dirContents)); + } + + //assert group file created and contains correct contents + $groupFile = PROJECT_ROOT . + DIRECTORY_SEPARATOR . + "src" . + DIRECTORY_SEPARATOR . + "Magento" . + DIRECTORY_SEPARATOR . + "FunctionalTestingFramework" . + DIRECTORY_SEPARATOR . + "Group" . + DIRECTORY_SEPARATOR . + $groupName . + ".php"; + + $this->assertTrue(file_exists($groupFile)); + $this->assertFileEquals( + self::RESOURCES_PATH . DIRECTORY_SEPARATOR . $groupName . ".txt", + $groupFile + ); + } + /** * revert any changes made to config.yml * remove _generated directory From daf1e4c1464c5f195774eda215e9dd0f108b6acf Mon Sep 17 00:00:00 2001 From: Ji Lu <jilu1@adobe.com> Date: Fri, 31 Jul 2020 11:37:27 -0500 Subject: [PATCH 28/52] MQE-2158: support using actions from multiple modules in Suites --- ...txt => ActionsInDifferentModulesSuite.txt} | 21 +++++++++++++++---- ...xml => ActionsInDifferentModulesSuite.xml} | 11 ++++++++-- ... IncludeActionsInDifferentModulesTest.xml} | 5 ++++- .../Tests/SuiteGenerationTest.php | 8 +++---- 4 files changed, 34 insertions(+), 11 deletions(-) rename dev/tests/verification/Resources/{functionalSuiteUsingSecretData.txt => ActionsInDifferentModulesSuite.txt} (82%) rename dev/tests/verification/TestModule/Suite/{functionalSuiteUsingSecretData.xml => ActionsInDifferentModulesSuite.xml} (75%) rename dev/tests/verification/TestModule/Test/SampleSuiteTest/{IncludeUsingSecretDataTest.xml => IncludeActionsInDifferentModulesTest.xml} (74%) diff --git a/dev/tests/verification/Resources/functionalSuiteUsingSecretData.txt b/dev/tests/verification/Resources/ActionsInDifferentModulesSuite.txt similarity index 82% rename from dev/tests/verification/Resources/functionalSuiteUsingSecretData.txt rename to dev/tests/verification/Resources/ActionsInDifferentModulesSuite.txt index 1be43a544..d4e1b6eb3 100644 --- a/dev/tests/verification/Resources/functionalSuiteUsingSecretData.txt +++ b/dev/tests/verification/Resources/ActionsInDifferentModulesSuite.txt @@ -21,14 +21,14 @@ use Codeception\Module; * * To use this group extension, include it to "extensions" option of global Codeception config. */ -class functionalSuiteUsingSecretData extends \Codeception\GroupObject +class ActionsInDifferentModulesSuite extends \Codeception\GroupObject { - public static $group = 'functionalSuiteUsingSecretData'; + public static $group = 'ActionsInDifferentModulesSuite'; private $testCount = 1; private $preconditionFailure = null; private $currentTestRun = 0; - private static $HOOK_EXECUTION_INIT = "\n/******** Beginning execution of functionalSuiteUsingSecretData suite %s block ********/\n"; - private static $HOOK_EXECUTION_END = "\n/******** Execution of functionalSuiteUsingSecretData suite %s block complete ********/\n"; + private static $HOOK_EXECUTION_INIT = "\n/******** Beginning execution of ActionsInDifferentModulesSuite suite %s block ********/\n"; + private static $HOOK_EXECUTION_END = "\n/******** Execution of ActionsInDifferentModulesSuite suite %s block complete ********/\n"; /** @var MagentoWebDriver */ private $webDriver; /** @var ModuleContainer */ @@ -83,6 +83,11 @@ class functionalSuiteUsingSecretData extends \Codeception\GroupObject ); $this->getModuleForAction("fillSecretField")->fillSecretField("#fill", $this->getModuleForAction("getSecret")->getSecret("magento/some/secret") . "+" . $this->getModuleForAction("getSecret")->getSecret("magento/some/secret")); // stepKey: fillBefore $this->getModuleForAction("click")->click(PersistedObjectHandler::getInstance()->retrieveEntityField('create2', 'key2', 'suite')); // stepKey: click + print("Entering Action Group [return1] ActionGroupReturningValueActionGroup"); + $grabProducts1Return1 = $this->getModuleForAction("grabMultiple")->grabMultiple("selector"); // stepKey: grabProducts1Return1 + $this->getModuleForAction("assertCount")->assertCount(1, $grabProducts1Return1); // stepKey: assertCountReturn1 + $return1 = $this->getModuleForAction("return")->return($grabProducts1Return1); // stepKey: returnProducts1Return1 + print("Exiting Action Group [return1] ActionGroupReturningValueActionGroup"); } catch (\Exception $exception) { $this->preconditionFailure = $exception->getMessage(); } @@ -133,6 +138,14 @@ class functionalSuiteUsingSecretData extends \Codeception\GroupObject } else { $this->webDriver->_initializeSession(); } + print("Entering Action Group [return2] ExtendedActionGroupReturningValueActionGroup"); + $grabProducts1Return2 = $this->getModuleForAction("grabMultiple")->grabMultiple("selector"); // stepKey: grabProducts1Return2 + $this->getModuleForAction("assertCount")->assertCount(1, $grabProducts1Return2); // stepKey: assertCountReturn2 + $return2 = $this->getModuleForAction("return")->return($grabProducts1Return2); // stepKey: returnProducts1Return2 + $grabProducts2Return2 = $this->getModuleForAction("grabMultiple")->grabMultiple("otherSelector"); // stepKey: grabProducts2Return2 + $this->getModuleForAction("assertCount")->assertCount(2, $grabProducts2Return2); // stepKey: assertSecondCountReturn2 + $return2 = $this->getModuleForAction("return")->return($grabProducts2Return2); // stepKey: returnProducts2Return2 + print("Exiting Action Group [return2] ExtendedActionGroupReturningValueActionGroup"); PersistedObjectHandler::getInstance()->deleteEntity( "create1", "suite" diff --git a/dev/tests/verification/TestModule/Suite/functionalSuiteUsingSecretData.xml b/dev/tests/verification/TestModule/Suite/ActionsInDifferentModulesSuite.xml similarity index 75% rename from dev/tests/verification/TestModule/Suite/functionalSuiteUsingSecretData.xml rename to dev/tests/verification/TestModule/Suite/ActionsInDifferentModulesSuite.xml index fdb9164cb..e3553308d 100644 --- a/dev/tests/verification/TestModule/Suite/functionalSuiteUsingSecretData.xml +++ b/dev/tests/verification/TestModule/Suite/ActionsInDifferentModulesSuite.xml @@ -7,9 +7,9 @@ --> <suites xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="urn:magento:mftf:Suite/etc/suiteSchema.xsd"> - <suite name="functionalSuiteUsingSecretData"> + <suite name="ActionsInDifferentModulesSuite"> <include> - <test name="IncludeUsingSecretDataTest"/> + <test name="IncludeActionsInDifferentModulesTest"/> </include> <before> <magentoCLI command="{{SecretData.key2}}" stepKey="cli"/> @@ -23,8 +23,15 @@ </createData> <fillField selector="#fill" userInput="{{SecretData.key2}}+{{SecretData.key2}}" stepKey="fillBefore"/> <click userInput="$create2.key2$" stepKey="click"/> + <actionGroup ref="ActionGroupReturningValueActionGroup" stepKey="return1"> + <argument name="count" value="1"/> + </actionGroup> </before> <after> + <actionGroup ref="ExtendedActionGroupReturningValueActionGroup" stepKey="return2"> + <argument name="count" value="1"/> + <argument name="otherCount" value="2"/> + </actionGroup> <deleteData createDataKey="create1" stepKey="delete1"/> <deleteData createDataKey="create2" stepKey="delete2"/> <deleteData createDataKey="create3" stepKey="delete3"/> diff --git a/dev/tests/verification/TestModule/Test/SampleSuiteTest/IncludeUsingSecretDataTest.xml b/dev/tests/verification/TestModule/Test/SampleSuiteTest/IncludeActionsInDifferentModulesTest.xml similarity index 74% rename from dev/tests/verification/TestModule/Test/SampleSuiteTest/IncludeUsingSecretDataTest.xml rename to dev/tests/verification/TestModule/Test/SampleSuiteTest/IncludeActionsInDifferentModulesTest.xml index 14360e2f3..1be662bbe 100644 --- a/dev/tests/verification/TestModule/Test/SampleSuiteTest/IncludeUsingSecretDataTest.xml +++ b/dev/tests/verification/TestModule/Test/SampleSuiteTest/IncludeActionsInDifferentModulesTest.xml @@ -7,11 +7,14 @@ --> <tests xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="urn:magento:mftf:Test/etc/testSchema.xsd"> - <test name="IncludeUsingSecretDataTest"> + <test name="IncludeActionsInDifferentModulesTest"> <createData entity="SecretData" stepKey="create"> <field key="someKey">dataHere</field> </createData> <fillField selector="#fill" userInput="{{SecretData.key2}}+$create.key2$" stepKey="fill"/> <magentoCLI command="create.key2$" stepKey="cli"/> + <actionGroup ref="ActionGroupReturningValueActionGroup" stepKey="return"> + <argument name="count" value="3"/> + </actionGroup> </test> </tests> diff --git a/dev/tests/verification/Tests/SuiteGenerationTest.php b/dev/tests/verification/Tests/SuiteGenerationTest.php index 633146eef..f37283871 100644 --- a/dev/tests/verification/Tests/SuiteGenerationTest.php +++ b/dev/tests/verification/Tests/SuiteGenerationTest.php @@ -389,14 +389,14 @@ public function testSuiteCommentsGeneration() } /** - * Test hook groups generated during suite generation + * Test suite generation with actions from different modules */ - public function testSuiteGenerationCreds() + public function testSuiteGenerationActionsInDifferentModules() { - $groupName = 'functionalSuiteUsingSecretData'; + $groupName = 'ActionsInDifferentModulesSuite'; $expectedContents = [ - 'IncludeUsingSecretDataTestCest.php' + 'IncludeActionsInDifferentModulesTestCest.php' ]; // Generate the Suite From 65298651f64258ae8b65f5265bc9867cbb5a2f13 Mon Sep 17 00:00:00 2001 From: soumyau <sunnikri@adobe.com> Date: Mon, 3 Aug 2020 16:51:46 -0500 Subject: [PATCH 29/52] MQE-1974: Report used deprecated metadata in Test (#772) * MQE-1974: Report used deprecated metadata in Test * MQE-1974: Report used deprecated metadata in Test * MQE-1974: Report used deprecated metadata in Test * MQE-1974: Report used deprecated metadata in Test verification tests --- .../Handlers/DataObjectHandlerTest.php | 51 ++++++++++++++ .../OperationDefinitionObjectHandlerTest.php | 67 +++++++++++++++++++ .../Page/Handlers/PageObjectHandlerTest.php | 41 ++++++++++++ .../Handlers/SectionObjectHandlerTest.php | 47 +++++++++++++ .../Util/ActionGroupObjectExtractorTest.php | 26 ++++++- .../Handlers/DataObjectHandler.php | 4 +- .../OperationDefinitionObjectHandler.php | 4 +- .../Objects/OperationDefinitionObject.php | 18 +++++ .../DataGenerator/Persist/CurlHandler.php | 1 + .../Page/Handlers/PageObjectHandler.php | 4 +- .../Page/Handlers/SectionObjectHandler.php | 4 +- .../Test/Util/ActionGroupObjectExtractor.php | 4 +- .../Util/Logger/MftfLogger.php | 10 +-- 13 files changed, 266 insertions(+), 15 deletions(-) diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/DataObjectHandlerTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/DataObjectHandlerTest.php index c9ec92ba8..9f1ca7b5c 100644 --- a/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/DataObjectHandlerTest.php +++ b/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/DataObjectHandlerTest.php @@ -13,12 +13,21 @@ use Magento\FunctionalTestingFramework\ObjectManager; use Magento\FunctionalTestingFramework\ObjectManagerFactory; use tests\unit\Util\MagentoTestCase; +use tests\unit\Util\TestLoggingUtil; /** * Class DataObjectHandlerTest */ class DataObjectHandlerTest extends MagentoTestCase { + /** + * Setup method + */ + public function setUp(): void + { + TestLoggingUtil::getInstance()->setMockLoggingUtil(); + } + // All tests share this array, feel free to add but be careful modifying or removing const PARSER_OUTPUT = [ 'entity' => [ @@ -44,6 +53,22 @@ class DataObjectHandlerTest extends MagentoTestCase ] ]; + const PARSER_OUTPUT_DEPRECATED = [ + 'entity' => [ + 'EntityOne' => [ + 'type' => 'testType', + 'data' => [ + 0 => [ + 'key' => 'testKey', + 'value' => 'testValue' + ] + ], + 'deprecated' => "deprecation message", + 'filename' => "filename.xml" + ], + ] + ]; + const PARSER_OUTPUT_WITH_EXTEND = [ 'entity' => [ 'EntityOne' => [ @@ -134,6 +159,24 @@ public function testGetAllObjects() $this->assertEquals($expected, $actual['EntityOne']); } + /** + * test deprecated data object + */ + public function testDeprecatedDataObject() + { + $this->setUpMockDataObjectHander(self::PARSER_OUTPUT_DEPRECATED); + + // Call the method under test + $actual = DataObjectHandler::getInstance()->getAllObjects(); + + //validate deprecation warning + TestLoggingUtil::getInstance()->validateMockLogStatement( + 'warning', + "DEPRECATION: The data entity 'EntityOne' is deprecated.", + ["fileName" => "filename.xml", "deprecatedMessage" => "deprecation message"] + ); + } + /** * getObject should return the expected data object if it exists */ @@ -269,4 +312,12 @@ private function setUpMockDataObjectHander($entityDataArray) 'getObjectManager' => $mockObjectManager ]); } + + /** + * clean up function runs after all tests + */ + public static function tearDownAfterClass(): void + { + TestLoggingUtil::getInstance()->clearMockLoggingUtil(); + } } diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/OperationDefinitionObjectHandlerTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/OperationDefinitionObjectHandlerTest.php index 9ca10c7f2..570e8d95a 100644 --- a/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/OperationDefinitionObjectHandlerTest.php +++ b/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/OperationDefinitionObjectHandlerTest.php @@ -14,12 +14,21 @@ use Magento\FunctionalTestingFramework\DataGenerator\Handlers\OperationDefinitionObjectHandler; use Magento\FunctionalTestingFramework\DataGenerator\Parsers\OperationDefinitionParser; use tests\unit\Util\MagentoTestCase; +use tests\unit\Util\TestLoggingUtil; /** * Class OperationDefinitionObjectHandlerTest */ class OperationDefinitionObjectHandlerTest extends MagentoTestCase { + /** + * Setup method + */ + public function setUp(): void + { + TestLoggingUtil::getInstance()->setMockLoggingUtil(); + } + public function testGetMultipleObjects() { // Data Variables for Assertions @@ -72,6 +81,56 @@ public function testGetMultipleObjects() $this->assertArrayHasKey($operationType2 . $dataType1, $operations); } + public function testDeprecatedOperation() + { + // Data Variables for Assertions + $dataType1 = "type1"; + $operationType1 = "create"; + + /** + * Parser Output. Just one metadata with 1 field + * operationName + * createType1 + * has field + * key=id, value=integer + */ + $mockData = [OperationDefinitionObjectHandler::ENTITY_OPERATION_ROOT_TAG => [ + "testOperationName" => [ + OperationDefinitionObjectHandler::ENTITY_OPERATION_DATA_TYPE => $dataType1, + OperationDefinitionObjectHandler::ENTITY_OPERATION_TYPE => $operationType1, + OperationDefinitionObjectHandler::ENTITY_OPERATION_AUTH => "auth", + OperationDefinitionObjectHandler::ENTITY_OPERATION_URL => "V1/Type1", + OperationDefinitionObjectHandler::ENTITY_OPERATION_METHOD => "POST", + OperationDefinitionObjectHandler::ENTITY_OPERATION_ENTRY => [ + 0 => [ + OperationDefinitionObjectHandler::ENTITY_OPERATION_ENTRY_KEY => "id", + OperationDefinitionObjectHandler::ENTITY_OPERATION_ENTRY_VALUE => "integer" + ], + ], + OperationDefinitionObjectHandler::OBJ_DEPRECATED => 'deprecation message' + ]]]; + $this->setMockParserOutput($mockData); + + //Perform Assertions + $operationDefinitionManager = OperationDefinitionObjectHandler::getInstance(); + $operations = $operationDefinitionManager->getAllObjects(); + + $this->assertArrayHasKey($operationType1 . $dataType1, $operations); + TestLoggingUtil::getInstance()->validateMockLogStatement( + 'notice', + "NOTICE: 1 metadata operation name violations detected. See mftf.log for details.", + [] + ); + // test run time deprecation notice + $operation = $operationDefinitionManager->getOperationDefinition($operationType1, $dataType1); + $operation->logDeprecated(); + TestLoggingUtil::getInstance()->validateMockLogStatement( + 'warning', + "DEPRECATION: The operation testOperationName is deprecated.", + ['operationType' => 'create', 'deprecatedMessage' => 'deprecation message'] + ); + } + public function testObjectCreation() { // Data Variables for Assertions @@ -379,4 +438,12 @@ private function setMockParserOutput($data) $instance = AspectMock::double(ObjectManager::class, ['create' => $mockOperationParser])->make(); AspectMock::double(ObjectManagerFactory::class, ['getObjectManager' => $instance]); } + + /** + * clean up function runs after all tests + */ + public static function tearDownAfterClass(): void + { + TestLoggingUtil::getInstance()->clearMockLoggingUtil(); + } } diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Page/Handlers/PageObjectHandlerTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Page/Handlers/PageObjectHandlerTest.php index 1a0ba5fc2..c934b680b 100644 --- a/dev/tests/unit/Magento/FunctionalTestFramework/Page/Handlers/PageObjectHandlerTest.php +++ b/dev/tests/unit/Magento/FunctionalTestFramework/Page/Handlers/PageObjectHandlerTest.php @@ -12,9 +12,18 @@ use Magento\FunctionalTestingFramework\Page\Handlers\PageObjectHandler; use Magento\FunctionalTestingFramework\XmlParser\PageParser; use tests\unit\Util\MagentoTestCase; +use tests\unit\Util\TestLoggingUtil; class PageObjectHandlerTest extends MagentoTestCase { + /** + * Setup method + */ + public function setUp(): void + { + TestLoggingUtil::getInstance()->setMockLoggingUtil(); + } + public function testGetPageObject() { $mockData = [ @@ -70,6 +79,30 @@ public function testGetEmptyPage() $this->addToAssertionCount(1); } + public function testDeprecatedPage() + { + $mockData = [ + "testPage1" => [ + "url" => "testURL1", + "module" => "testModule1", + "section" => [ + ], + "area" => "test", + "deprecated" => "deprecation message", + "filename" => "filename.xml" + ]]; + $this->setMockParserOutput($mockData); + + // get pages + $page = PageObjectHandler::getInstance()->getObject('testPage1'); + + TestLoggingUtil::getInstance()->validateMockLogStatement( + 'notice', + "NOTICE: 1 Page name violations detected. See mftf.log for details.", + [] + ); + } + /** * Function used to set mock for parser return and force init method to run between tests. * @@ -86,4 +119,12 @@ private function setMockParserOutput($data) $instance = AspectMock::double(ObjectManager::class, ['get' => $mockSectionParser])->make(); AspectMock::double(ObjectManagerFactory::class, ['getObjectManager' => $instance]); } + + /** + * clean up function runs after all tests + */ + public static function tearDownAfterClass(): void + { + TestLoggingUtil::getInstance()->clearMockLoggingUtil(); + } } diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Page/Handlers/SectionObjectHandlerTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Page/Handlers/SectionObjectHandlerTest.php index b8bcf3dfb..39e26caf9 100644 --- a/dev/tests/unit/Magento/FunctionalTestFramework/Page/Handlers/SectionObjectHandlerTest.php +++ b/dev/tests/unit/Magento/FunctionalTestFramework/Page/Handlers/SectionObjectHandlerTest.php @@ -12,9 +12,18 @@ use Magento\FunctionalTestingFramework\Page\Handlers\SectionObjectHandler; use Magento\FunctionalTestingFramework\XmlParser\SectionParser; use tests\unit\Util\MagentoTestCase; +use tests\unit\Util\TestLoggingUtil; class SectionObjectHandlerTest extends MagentoTestCase { + /** + * Setup method + */ + public function setUp(): void + { + TestLoggingUtil::getInstance()->setMockLoggingUtil(); + } + public function testGetSectionObject() { $mockData = [ @@ -52,6 +61,36 @@ public function testGetSectionObject() $this->assertNull($invalidSection); } + public function testDeprecatedSection() + { + $mockData = [ + "testSection1" => [ + "element" => [ + "testElement" => [ + "type" => "input", + "selector" => "#element", + "deprecated" => "element deprecation message" + ] + ], + "filename" => "filename.xml", + "deprecated" => "section deprecation message" + ] + ]; + + $this->setMockParserOutput($mockData); + + // get sections + $sectionHandler = SectionObjectHandler::getInstance(); + $section = $sectionHandler->getObject("testSection1"); + + //validate deprecation warning + TestLoggingUtil::getInstance()->validateMockLogStatement( + 'notice', + "NOTICE: 1 Section name violations detected. See mftf.log for details.", + [] + ); + } + /** * Set the mock parser return value * @@ -68,4 +107,12 @@ private function setMockParserOutput($data) $instance = AspectMock::double(ObjectManager::class, ["get" => $mockSectionParser])->make(); AspectMock::double(ObjectManagerFactory::class, ["getObjectManager" => $instance]); } + + /** + * clean up function runs after all tests + */ + public static function tearDownAfterClass(): void + { + TestLoggingUtil::getInstance()->clearMockLoggingUtil(); + } } diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Test/Util/ActionGroupObjectExtractorTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Test/Util/ActionGroupObjectExtractorTest.php index 2c9e171d5..1fea7e848 100644 --- a/dev/tests/unit/Magento/FunctionalTestFramework/Test/Util/ActionGroupObjectExtractorTest.php +++ b/dev/tests/unit/Magento/FunctionalTestFramework/Test/Util/ActionGroupObjectExtractorTest.php @@ -34,23 +34,47 @@ public function testEmptyStepKey() $this->testActionGroupObjectExtractor->extractActionGroup($this->createBasicActionObjectArray("")); } + /** + * Tests deprecation message for an action group + */ + public function testDeprecationMessage() + { + $this->testActionGroupObjectExtractor->extractActionGroup( + $this->createBasicActionObjectArray( + "testDeprecatedAction1", + "actionGroup", + "filename1.xml", + "message" + ) + ); + + TestLoggingUtil::getInstance()->validateMockLogStatement( + 'warning', + "DEPRECATION: The action group 'actionGroup' is deprecated.", + ["fileName" => "filename1.xml", "deprecatedMessage" => "message"] + ); + } + /** * Utility function to return mock parser output for testing extraction into ActionObjects. * * @param string $stepKey * @param string $actionGroup * @param string $filename + * @param string $deprecated * @return array */ private function createBasicActionObjectArray( $stepKey = 'testAction1', $actionGroup = "actionGroup", - $filename = "filename.xml" + $filename = "filename.xml", + $deprecated = null ) { $baseArray = [ 'nodeName' => 'actionGroup', 'name' => $actionGroup, 'filename' => $filename, + 'deprecated' => $deprecated, $stepKey => [ "nodeName" => "sampleAction", "stepKey" => $stepKey, diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/DataObjectHandler.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/DataObjectHandler.php index 5a4b42dc4..87dc6a13d 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/DataObjectHandler.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/DataObjectHandler.php @@ -191,8 +191,8 @@ private function processParserOutput($parserOutput) if (array_key_exists(self::OBJ_DEPRECATED, $rawEntity)) { $deprecated = $rawEntity[self::OBJ_DEPRECATED]; LoggingUtil::getInstance()->getLogger(self::class)->deprecation( - $deprecated, - ["dataName" => $filename, "deprecatedEntity" => $deprecated] + "The data entity '{$name}' is deprecated.", + ["fileName" => $filename, "deprecatedMessage" => $deprecated] ); } diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/OperationDefinitionObjectHandler.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/OperationDefinitionObjectHandler.php index 2c8ab7530..eb4651b3e 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/OperationDefinitionObjectHandler.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/OperationDefinitionObjectHandler.php @@ -215,8 +215,8 @@ private function initialize() if ($deprecated !== null) { LoggingUtil::getInstance()->getLogger(self::class)->deprecation( - $deprecated, - ["operationName" => $dataDefName, "deprecatedOperation" => $deprecated] + $message = "The operation {$dataDefName} is deprecated.", + ["operationType" => $operation, "deprecatedMessage" => $deprecated] ); } diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Objects/OperationDefinitionObject.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Objects/OperationDefinitionObject.php index 67fe48300..ef1a7faa5 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Objects/OperationDefinitionObject.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Objects/OperationDefinitionObject.php @@ -6,6 +6,8 @@ namespace Magento\FunctionalTestingFramework\DataGenerator\Objects; +use Magento\FunctionalTestingFramework\Util\Logger\LoggingUtil; + /** * Class OperationDefinitionObject * @SuppressWarnings(PHPMD) @@ -341,4 +343,20 @@ public function addQueryParams() $this->apiUrl = $this->apiUrl . $paramName . "=" . $paramValue; } } + + /** + * Function to log a referenced deprecated operation at runtime. + * + * @return void + */ + public function logDeprecated() + { + if ($this->deprecated != null) { + LoggingUtil::getInstance()->getLogger(self::class)->deprecation( + $message = "The operation {$this->name} is deprecated.", + ["operationType" => $this->operation, "deprecatedMessage" => $this->deprecated], + true + ); + } + } } diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/CurlHandler.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/CurlHandler.php index 362d25e75..89055b83f 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/CurlHandler.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/CurlHandler.php @@ -123,6 +123,7 @@ public function executeRequest($dependentEntities) $returnRegex = $this->operationDefinition->getReturnRegex(); $returnIndex = $this->operationDefinition->getReturnIndex(); $method = $this->operationDefinition->getApiMethod(); + $this->operationDefinition->logDeprecated(); AllureHelper::addAttachmentToCurrentStep($apiUrl, 'API Endpoint'); AllureHelper::addAttachmentToCurrentStep(json_encode($headers, JSON_PRETTY_PRINT), 'Request Headers'); diff --git a/src/Magento/FunctionalTestingFramework/Page/Handlers/PageObjectHandler.php b/src/Magento/FunctionalTestingFramework/Page/Handlers/PageObjectHandler.php index 73695c182..8f4302077 100644 --- a/src/Magento/FunctionalTestingFramework/Page/Handlers/PageObjectHandler.php +++ b/src/Magento/FunctionalTestingFramework/Page/Handlers/PageObjectHandler.php @@ -79,8 +79,8 @@ private function __construct() if ($deprecated !== null) { LoggingUtil::getInstance()->getLogger(self::class)->deprecation( - $deprecated, - ["pageName" => $filename, "deprecatedPage" => $deprecated] + "The page '{$pageName}' is deprecated.", + ["fileName" => $filename, "deprecatedMessage" => $deprecated] ); } diff --git a/src/Magento/FunctionalTestingFramework/Page/Handlers/SectionObjectHandler.php b/src/Magento/FunctionalTestingFramework/Page/Handlers/SectionObjectHandler.php index 5d658bac0..6ac3456fd 100644 --- a/src/Magento/FunctionalTestingFramework/Page/Handlers/SectionObjectHandler.php +++ b/src/Magento/FunctionalTestingFramework/Page/Handlers/SectionObjectHandler.php @@ -89,8 +89,8 @@ private function __construct() $elementDeprecated = $elementData[self::OBJ_DEPRECATED] ?? null; if ($elementDeprecated !== null) { LoggingUtil::getInstance()->getLogger(ElementObject::class)->deprecation( - $elementDeprecated, - ["elementName" => $elementName, "deprecatedElement" => $elementDeprecated] + "The element '{$elementName}' is deprecated.", + ["fileName" => $filename, "deprecatedMessage" => $elementDeprecated] ); } $elements[$elementName] = new ElementObject( diff --git a/src/Magento/FunctionalTestingFramework/Test/Util/ActionGroupObjectExtractor.php b/src/Magento/FunctionalTestingFramework/Test/Util/ActionGroupObjectExtractor.php index 618cddc09..84b960f1b 100644 --- a/src/Magento/FunctionalTestingFramework/Test/Util/ActionGroupObjectExtractor.php +++ b/src/Magento/FunctionalTestingFramework/Test/Util/ActionGroupObjectExtractor.php @@ -65,8 +65,8 @@ public function extractActionGroup($actionGroupData) if (array_key_exists(self::OBJ_DEPRECATED, $actionGroupData)) { $deprecated = $actionGroupData[self::OBJ_DEPRECATED]; LoggingUtil::getInstance()->getLogger(ActionGroupObject::class)->deprecation( - $deprecated, - ["actionGroupName" => $actionGroupData[self::FILENAME], "deprecatedActionGroup" => $deprecated] + "The action group '{$actionGroupData[self::NAME]}' is deprecated.", + ["fileName" => $actionGroupData[self::FILENAME], "deprecatedMessage" => $deprecated] ); } $actionGroupReference = $actionGroupData[self::EXTENDS_ACTION_GROUP] ?? null; diff --git a/src/Magento/FunctionalTestingFramework/Util/Logger/MftfLogger.php b/src/Magento/FunctionalTestingFramework/Util/Logger/MftfLogger.php index 5955235ce..880dd9736 100644 --- a/src/Magento/FunctionalTestingFramework/Util/Logger/MftfLogger.php +++ b/src/Magento/FunctionalTestingFramework/Util/Logger/MftfLogger.php @@ -46,12 +46,14 @@ public function __construct($name, array $handlers = [], array $processors = []) public function deprecation($message, array $context = [], $verbose = false) { $message = "DEPRECATION: " . $message; - // print during test generation - if ($this->phase === MftfApplicationConfig::GENERATION_PHASE && $verbose) { + // print during test generation including metadata + if ((array_key_exists('operationType', $context) || + $this->phase === MftfApplicationConfig::GENERATION_PHASE) && $verbose) { print ($message . json_encode($context) . "\n"); } - // suppress logging during test execution - if ($this->phase !== MftfApplicationConfig::EXECUTION_PHASE) { + // suppress logging during test execution except metadata + if (array_key_exists('operationType', $context) || + $this->phase !== MftfApplicationConfig::EXECUTION_PHASE) { parent::warning($message, $context); } } From 4126761d5b84b1508c2b8e8a5bcc284e702628c7 Mon Sep 17 00:00:00 2001 From: Ji Lu <jilu1@adobe.com> Date: Fri, 7 Aug 2020 08:50:19 -0500 Subject: [PATCH 30/52] MQE-2110: MFTF command to pause test execution --- composer.json | 1 + composer.lock | 551 +++++++++++++++++- .../Test/Handlers/TestObjectHandlerTest.php | 95 ++- docs/configuration.md | 9 + docs/test/actions.md | 2 +- etc/config/.env.example | 3 + .../Console/BaseGenerateCommand.php | 24 + .../Console/RunManifestCommand.php | 9 +- .../Console/RunTestCommand.php | 7 + .../Console/RunTestFailedCommand.php | 4 + .../Console/RunTestGroupCommand.php | 4 + .../Test/Util/TestHookObjectExtractor.php | 9 +- 12 files changed, 709 insertions(+), 9 deletions(-) diff --git a/composer.json b/composer.json index e1a04296f..6f8b0c2c2 100755 --- a/composer.json +++ b/composer.json @@ -24,6 +24,7 @@ "composer/composer": "^1.9", "csharpru/vault-php": "~3.5.3", "csharpru/vault-php-guzzle6-transport": "^2.0", + "hoa/console": "~3.0", "monolog/monolog": "^1.17", "mustache/mustache": "~2.5", "php-webdriver/webdriver": "^1.8.0", diff --git a/composer.lock b/composer.lock index dff4f1006..91fa2c772 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "a0516f6072ced659bf4ed7a486756bb2", + "content-hash": "98733866973fb51e11e316b98c0af563", "packages": [ { "name": "allure-framework/allure-codeception", @@ -1662,6 +1662,555 @@ ], "time": "2019-07-01T23:21:34+00:00" }, + { + "name": "hoa/consistency", + "version": "1.17.05.02", + "source": { + "type": "git", + "url": "https://github.com/hoaproject/Consistency.git", + "reference": "fd7d0adc82410507f332516faf655b6ed22e4c2f" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/hoaproject/Consistency/zipball/fd7d0adc82410507f332516faf655b6ed22e4c2f", + "reference": "fd7d0adc82410507f332516faf655b6ed22e4c2f", + "shasum": "" + }, + "require": { + "hoa/exception": "~1.0", + "php": ">=5.5.0" + }, + "require-dev": { + "hoa/stream": "~1.0", + "hoa/test": "~2.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.x-dev" + } + }, + "autoload": { + "psr-4": { + "Hoa\\Consistency\\": "." + }, + "files": [ + "Prelude.php" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Ivan Enderlin", + "email": "ivan.enderlin@hoa-project.net" + }, + { + "name": "Hoa community", + "homepage": "https://hoa-project.net/" + } + ], + "description": "The Hoa\\Consistency library.", + "homepage": "https://hoa-project.net/", + "keywords": [ + "autoloader", + "callable", + "consistency", + "entity", + "flex", + "keyword", + "library" + ], + "time": "2017-05-02T12:18:12+00:00" + }, + { + "name": "hoa/console", + "version": "3.17.05.02", + "source": { + "type": "git", + "url": "https://github.com/hoaproject/Console.git", + "reference": "e231fd3ea70e6d773576ae78de0bdc1daf331a66" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/hoaproject/Console/zipball/e231fd3ea70e6d773576ae78de0bdc1daf331a66", + "reference": "e231fd3ea70e6d773576ae78de0bdc1daf331a66", + "shasum": "" + }, + "require": { + "hoa/consistency": "~1.0", + "hoa/event": "~1.0", + "hoa/exception": "~1.0", + "hoa/file": "~1.0", + "hoa/protocol": "~1.0", + "hoa/stream": "~1.0", + "hoa/ustring": "~4.0" + }, + "require-dev": { + "hoa/test": "~2.0" + }, + "suggest": { + "ext-pcntl": "To enable hoa://Event/Console/Window:resize.", + "hoa/dispatcher": "To use the console kit.", + "hoa/router": "To use the console kit." + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "3.x-dev" + } + }, + "autoload": { + "psr-4": { + "Hoa\\Console\\": "." + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Ivan Enderlin", + "email": "ivan.enderlin@hoa-project.net" + }, + { + "name": "Hoa community", + "homepage": "https://hoa-project.net/" + } + ], + "description": "The Hoa\\Console library.", + "homepage": "https://hoa-project.net/", + "keywords": [ + "autocompletion", + "chrome", + "cli", + "console", + "cursor", + "getoption", + "library", + "option", + "parser", + "processus", + "readline", + "terminfo", + "tput", + "window" + ], + "time": "2017-05-02T12:26:19+00:00" + }, + { + "name": "hoa/event", + "version": "1.17.01.13", + "source": { + "type": "git", + "url": "https://github.com/hoaproject/Event.git", + "reference": "6c0060dced212ffa3af0e34bb46624f990b29c54" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/hoaproject/Event/zipball/6c0060dced212ffa3af0e34bb46624f990b29c54", + "reference": "6c0060dced212ffa3af0e34bb46624f990b29c54", + "shasum": "" + }, + "require": { + "hoa/consistency": "~1.0", + "hoa/exception": "~1.0" + }, + "require-dev": { + "hoa/test": "~2.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.x-dev" + } + }, + "autoload": { + "psr-4": { + "Hoa\\Event\\": "." + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Ivan Enderlin", + "email": "ivan.enderlin@hoa-project.net" + }, + { + "name": "Hoa community", + "homepage": "https://hoa-project.net/" + } + ], + "description": "The Hoa\\Event library.", + "homepage": "https://hoa-project.net/", + "keywords": [ + "event", + "library", + "listener", + "observer" + ], + "time": "2017-01-13T15:30:50+00:00" + }, + { + "name": "hoa/exception", + "version": "1.17.01.16", + "source": { + "type": "git", + "url": "https://github.com/hoaproject/Exception.git", + "reference": "091727d46420a3d7468ef0595651488bfc3a458f" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/hoaproject/Exception/zipball/091727d46420a3d7468ef0595651488bfc3a458f", + "reference": "091727d46420a3d7468ef0595651488bfc3a458f", + "shasum": "" + }, + "require": { + "hoa/consistency": "~1.0", + "hoa/event": "~1.0" + }, + "require-dev": { + "hoa/test": "~2.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.x-dev" + } + }, + "autoload": { + "psr-4": { + "Hoa\\Exception\\": "." + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Ivan Enderlin", + "email": "ivan.enderlin@hoa-project.net" + }, + { + "name": "Hoa community", + "homepage": "https://hoa-project.net/" + } + ], + "description": "The Hoa\\Exception library.", + "homepage": "https://hoa-project.net/", + "keywords": [ + "exception", + "library" + ], + "time": "2017-01-16T07:53:27+00:00" + }, + { + "name": "hoa/file", + "version": "1.17.07.11", + "source": { + "type": "git", + "url": "https://github.com/hoaproject/File.git", + "reference": "35cb979b779bc54918d2f9a4e02ed6c7a1fa67ca" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/hoaproject/File/zipball/35cb979b779bc54918d2f9a4e02ed6c7a1fa67ca", + "reference": "35cb979b779bc54918d2f9a4e02ed6c7a1fa67ca", + "shasum": "" + }, + "require": { + "hoa/consistency": "~1.0", + "hoa/event": "~1.0", + "hoa/exception": "~1.0", + "hoa/iterator": "~2.0", + "hoa/stream": "~1.0" + }, + "require-dev": { + "hoa/test": "~2.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.x-dev" + } + }, + "autoload": { + "psr-4": { + "Hoa\\File\\": "." + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Ivan Enderlin", + "email": "ivan.enderlin@hoa-project.net" + }, + { + "name": "Hoa community", + "homepage": "https://hoa-project.net/" + } + ], + "description": "The Hoa\\File library.", + "homepage": "https://hoa-project.net/", + "keywords": [ + "Socket", + "directory", + "file", + "finder", + "library", + "link", + "temporary" + ], + "time": "2017-07-11T07:42:15+00:00" + }, + { + "name": "hoa/iterator", + "version": "2.17.01.10", + "source": { + "type": "git", + "url": "https://github.com/hoaproject/Iterator.git", + "reference": "d1120ba09cb4ccd049c86d10058ab94af245f0cc" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/hoaproject/Iterator/zipball/d1120ba09cb4ccd049c86d10058ab94af245f0cc", + "reference": "d1120ba09cb4ccd049c86d10058ab94af245f0cc", + "shasum": "" + }, + "require": { + "hoa/consistency": "~1.0", + "hoa/exception": "~1.0" + }, + "require-dev": { + "hoa/test": "~2.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "2.x-dev" + } + }, + "autoload": { + "psr-4": { + "Hoa\\Iterator\\": "." + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Ivan Enderlin", + "email": "ivan.enderlin@hoa-project.net" + }, + { + "name": "Hoa community", + "homepage": "https://hoa-project.net/" + } + ], + "description": "The Hoa\\Iterator library.", + "homepage": "https://hoa-project.net/", + "keywords": [ + "iterator", + "library" + ], + "time": "2017-01-10T10:34:47+00:00" + }, + { + "name": "hoa/protocol", + "version": "1.17.01.14", + "source": { + "type": "git", + "url": "https://github.com/hoaproject/Protocol.git", + "reference": "5c2cf972151c45f373230da170ea015deecf19e2" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/hoaproject/Protocol/zipball/5c2cf972151c45f373230da170ea015deecf19e2", + "reference": "5c2cf972151c45f373230da170ea015deecf19e2", + "shasum": "" + }, + "require": { + "hoa/consistency": "~1.0", + "hoa/exception": "~1.0" + }, + "require-dev": { + "hoa/test": "~2.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.x-dev" + } + }, + "autoload": { + "psr-4": { + "Hoa\\Protocol\\": "." + }, + "files": [ + "Wrapper.php" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Ivan Enderlin", + "email": "ivan.enderlin@hoa-project.net" + }, + { + "name": "Hoa community", + "homepage": "https://hoa-project.net/" + } + ], + "description": "The Hoa\\Protocol library.", + "homepage": "https://hoa-project.net/", + "keywords": [ + "library", + "protocol", + "resource", + "stream", + "wrapper" + ], + "time": "2017-01-14T12:26:10+00:00" + }, + { + "name": "hoa/stream", + "version": "1.17.02.21", + "source": { + "type": "git", + "url": "https://github.com/hoaproject/Stream.git", + "reference": "3293cfffca2de10525df51436adf88a559151d82" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/hoaproject/Stream/zipball/3293cfffca2de10525df51436adf88a559151d82", + "reference": "3293cfffca2de10525df51436adf88a559151d82", + "shasum": "" + }, + "require": { + "hoa/consistency": "~1.0", + "hoa/event": "~1.0", + "hoa/exception": "~1.0", + "hoa/protocol": "~1.0" + }, + "require-dev": { + "hoa/test": "~2.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.x-dev" + } + }, + "autoload": { + "psr-4": { + "Hoa\\Stream\\": "." + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Ivan Enderlin", + "email": "ivan.enderlin@hoa-project.net" + }, + { + "name": "Hoa community", + "homepage": "https://hoa-project.net/" + } + ], + "description": "The Hoa\\Stream library.", + "homepage": "https://hoa-project.net/", + "keywords": [ + "Context", + "bucket", + "composite", + "filter", + "in", + "library", + "out", + "protocol", + "stream", + "wrapper" + ], + "time": "2017-02-21T16:01:06+00:00" + }, + { + "name": "hoa/ustring", + "version": "4.17.01.16", + "source": { + "type": "git", + "url": "https://github.com/hoaproject/Ustring.git", + "reference": "e6326e2739178799b1fe3fdd92029f9517fa17a0" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/hoaproject/Ustring/zipball/e6326e2739178799b1fe3fdd92029f9517fa17a0", + "reference": "e6326e2739178799b1fe3fdd92029f9517fa17a0", + "shasum": "" + }, + "require": { + "hoa/consistency": "~1.0", + "hoa/exception": "~1.0" + }, + "require-dev": { + "hoa/test": "~2.0" + }, + "suggest": { + "ext-iconv": "ext/iconv must be present (or a third implementation) to use Hoa\\Ustring::transcode().", + "ext-intl": "To get a better Hoa\\Ustring::toAscii() and Hoa\\Ustring::compareTo()." + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "4.x-dev" + } + }, + "autoload": { + "psr-4": { + "Hoa\\Ustring\\": "." + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Ivan Enderlin", + "email": "ivan.enderlin@hoa-project.net" + }, + { + "name": "Hoa community", + "homepage": "https://hoa-project.net/" + } + ], + "description": "The Hoa\\Ustring library.", + "homepage": "https://hoa-project.net/", + "keywords": [ + "library", + "search", + "string", + "unicode" + ], + "time": "2017-01-16T07:08:25+00:00" + }, { "name": "jms/metadata", "version": "1.7.0", diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Test/Handlers/TestObjectHandlerTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Test/Handlers/TestObjectHandlerTest.php index 4db2f21a4..e0fc84d4c 100644 --- a/dev/tests/unit/Magento/FunctionalTestFramework/Test/Handlers/TestObjectHandlerTest.php +++ b/dev/tests/unit/Magento/FunctionalTestFramework/Test/Handlers/TestObjectHandlerTest.php @@ -78,7 +78,7 @@ public function testGetTestObject() $expectedFailedHookObject = new TestHookObject( TestObjectExtractor::TEST_FAILED_HOOK, $testDataArrayBuilder->testName, - [$expectedFailedActionObject] + ["saveScreenshot" => $expectedFailedActionObject] ); $expectedTestActionObject = new ActionObject( @@ -278,6 +278,99 @@ private function setMockParserOutput($data) AspectMock::double(ObjectManagerFactory::class, ['getObjectManager' => $instance]); } + /** + * Validate test object when ENABLE_PAUSE is set to true + * + * @throws \Exception + */ + public function testGetTestObjectWhenEnablePause() + { + // set up mock data + putenv('ENABLE_PAUSE=true'); + $testDataArrayBuilder = new TestDataArrayBuilder(); + $mockData = $testDataArrayBuilder + ->withAnnotations() + ->withFailedHook() + ->withAfterHook() + ->withBeforeHook() + ->withTestActions() + ->build(); + + $resolverMock = new MockModuleResolverBuilder(); + $resolverMock->setup(); + $this->setMockParserOutput($mockData); + + // run object handler method + $toh = TestObjectHandler::getInstance(); + $mockConfig = AspectMock::double(TestObjectHandler::class, ['initTestData' => false]); + $actualTestObject = $toh->getObject($testDataArrayBuilder->testName); + + // perform asserts + $expectedBeforeActionObject = new ActionObject( + $testDataArrayBuilder->testActionBeforeName, + $testDataArrayBuilder->testActionType, + [] + ); + $expectedAfterActionObject = new ActionObject( + $testDataArrayBuilder->testActionAfterName, + $testDataArrayBuilder->testActionType, + [] + ); + $expectedFailedActionObject1 = new ActionObject( + 'saveScreenshot', + 'saveScreenshot', + [] + ); + $expectedFailedActionObject2 = new ActionObject( + 'pauseWhenFailed', + 'pause', + [] + ); + + $expectedBeforeHookObject = new TestHookObject( + TestObjectExtractor::TEST_BEFORE_HOOK, + $testDataArrayBuilder->testName, + ["testActionBefore" => $expectedBeforeActionObject] + ); + $expectedAfterHookObject = new TestHookObject( + TestObjectExtractor::TEST_AFTER_HOOK, + $testDataArrayBuilder->testName, + ["testActionAfter" => $expectedAfterActionObject] + ); + $expectedFailedHookObject = new TestHookObject( + TestObjectExtractor::TEST_FAILED_HOOK, + $testDataArrayBuilder->testName, + [ + "saveScreenshot" => $expectedFailedActionObject1, + "pauseWhenFailed" => $expectedFailedActionObject2, + ] + ); + + $expectedTestActionObject = new ActionObject( + $testDataArrayBuilder->testTestActionName, + $testDataArrayBuilder->testActionType, + [] + ); + $expectedTestObject = new TestObject( + $testDataArrayBuilder->testName, + ["testActionInTest" => $expectedTestActionObject], + [ + 'features' => ['NO MODULE DETECTED'], + 'group' => ['test'], + 'description' => ['test_files' => '<h3>Test files</h3>', 'deprecated' => []] + ], + [ + TestObjectExtractor::TEST_BEFORE_HOOK => $expectedBeforeHookObject, + TestObjectExtractor::TEST_AFTER_HOOK => $expectedAfterHookObject, + TestObjectExtractor::TEST_FAILED_HOOK => $expectedFailedHookObject + ], + null + ); + + $this->assertEquals($expectedTestObject, $actualTestObject); + putenv('ENABLE_PAUSE'); + } + /** * After method functionality * diff --git a/docs/configuration.md b/docs/configuration.md index b16f85d8f..90111a03b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -359,6 +359,15 @@ Global MFTF configuration for the default amount of time (in seconds) that a tes WAIT_TIMEOUT=30 ``` +### ENABLE_PAUSE + +Enables pause of test execution in any point and enter interactive shell where you will be able to try commands in action. +When pause is enabled, MFTF will generate pause() command in _failed() hook so that test will pause execution when failed. + +```conf +ENABLE_PAUSE=true +``` + <!-- Link definitions --> [`MAGENTO_CLI_COMMAND_PATH`]: #magento_cli_command_path diff --git a/docs/test/actions.md b/docs/test/actions.md index 562f6e8ff..a4e0fb44c 100644 --- a/docs/test/actions.md +++ b/docs/test/actions.md @@ -1463,7 +1463,7 @@ Attribute|Type|Use|Description ### pause -See [pause docs on codeception.com](https://codeception.com/docs/02-GettingStarted). +See [pause docs on codeception.com](https://codeception.com/docs/02-GettingStarted#Interactive-Pause). Attribute|Type|Use|Description ---|---|---|--- diff --git a/etc/config/.env.example b/etc/config/.env.example index 53b12036f..64a287adc 100644 --- a/etc/config/.env.example +++ b/etc/config/.env.example @@ -67,6 +67,9 @@ MODULE_ALLOWLIST=Magento_Framework,ConfigurableProductWishlist,ConfigurableProdu #ENABLE_BROWSER_LOG=true BROWSER_LOG_BLOCKLIST=other +#*** Uncomment and set to true to use Codeception's interactive pause functionality +#ENABLE_PAUSE=true + #*** Elastic Search version used for test ***# ELASTICSEARCH_VERSION=7 #*** End of .env ***# diff --git a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php index f3becf935..29e489a63 100644 --- a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php @@ -25,6 +25,13 @@ class BaseGenerateCommand extends Command { const MFTF_NOTICES = "Placeholder text for MFTF notices\n"; + /** + * Enable pause() + * + * @var boolean + */ + private $enablePause = null; + /** * Console output style * @@ -218,4 +225,21 @@ protected function showMftfNotices(OutputInterface $output) $output->writeln(self::MFTF_NOTICES); } } + + /** + * Return if pause() is enabled + * + * @return boolean + */ + protected function pauseEnabled() + { + if (null === $this->enablePause) { + if (getenv('ENABLE_PAUSE') === 'true') { + $this->enablePause = true; + } else { + $this->enablePause = false; + } + } + return $this->enablePause; + } } diff --git a/src/Magento/FunctionalTestingFramework/Console/RunManifestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunManifestCommand.php index 37a90bb77..4e9c92d38 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunManifestCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunManifestCommand.php @@ -108,15 +108,18 @@ protected function execute(InputInterface $input, OutputInterface $output): int */ private function runManifestLine(string $manifestLine, OutputInterface $output) { - $codeceptionCommand = realpath(PROJECT_ROOT . "/vendor/bin/codecept") - . " run functional --verbose --steps " - . $manifestLine; + $codeceptionCommand = realpath(PROJECT_ROOT . "/vendor/bin/codecept") . " run functional --verbose --steps "; + if (getenv('ENABLE_PAUSE') === 'true') { + $codeceptionCommand .= ' --debug'; + } + $codeceptionCommand .= $manifestLine; // run the codecept command in a sub process $process = new Process($codeceptionCommand); $process->setWorkingDirectory(TESTS_BP); $process->setIdleTimeout(600); $process->setTimeout(0); + $process->setInput(STDIN); $subReturnCode = $process->run(function ($type, $buffer) use ($output) { $output->write($buffer); }); diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php index 373256cfc..c96898da8 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php @@ -137,6 +137,9 @@ private function runTests(array $tests, OutputInterface $output) ); } $fullCommand = $codeceptionCommand . $testsDirectory . $testName . ' --verbose --steps'; + if ($this->pauseEnabled()) { + $fullCommand .= ' --debug'; + } $this->returnCode = max($this->returnCode, $this->executeTestCommand($fullCommand, $output)); } } @@ -151,6 +154,9 @@ private function runTests(array $tests, OutputInterface $output) private function runTestsInSuite(array $suitesConfig, OutputInterface $output) { $codeceptionCommand = realpath(PROJECT_ROOT . '/vendor/bin/codecept') . ' run functional --verbose --steps '; + if ($this->pauseEnabled()) { + $codeceptionCommand .= ' --debug'; + } //for tests in suites, run them as a group to run before and after block foreach (array_keys($suitesConfig) as $suite) { $fullCommand = $codeceptionCommand . " -g {$suite}"; @@ -173,6 +179,7 @@ private function executeTestCommand(string $command, OutputInterface $output) $process->setWorkingDirectory(TESTS_BP); $process->setIdleTimeout(600); $process->setTimeout(0); + $process->setInput(STDIN); return $process->run(function ($type, $buffer) use ($output) { $output->write($buffer); }); diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php index c6a2d0b76..96f560f24 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php @@ -119,11 +119,15 @@ protected function execute(InputInterface $input, OutputInterface $output): int foreach ($testManifestList as $testCommand) { $codeceptionCommand = realpath(PROJECT_ROOT . '/vendor/bin/codecept') . ' run functional '; $codeceptionCommand .= $testCommand; + if ($this->pauseEnabled()) { + $codeceptionCommand .= ' --debug'; + } $process = new Process($codeceptionCommand); $process->setWorkingDirectory(TESTS_BP); $process->setIdleTimeout(600); $process->setTimeout(0); + $process->setInput(STDIN); $returnCode = max($returnCode, $process->run( function ($type, $buffer) use ($output) { $output->write($buffer); diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php index 6ea37785d..db3ea2e55 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php @@ -95,6 +95,9 @@ protected function execute(InputInterface $input, OutputInterface $output): int } $commandString = realpath(PROJECT_ROOT . '/vendor/bin/codecept') . ' run functional --verbose --steps'; + if ($this->pauseEnabled()) { + $commandString .= ' --debug'; + } $exitCode = -1; $returnCodes = []; @@ -105,6 +108,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int $process->setWorkingDirectory(TESTS_BP); $process->setIdleTimeout(600); $process->setTimeout(0); + $process->setInput(STDIN); $returnCodes[] = $process->run( function ($type, $buffer) use ($output) { diff --git a/src/Magento/FunctionalTestingFramework/Test/Util/TestHookObjectExtractor.php b/src/Magento/FunctionalTestingFramework/Test/Util/TestHookObjectExtractor.php index c11a6e512..a68f09491 100644 --- a/src/Magento/FunctionalTestingFramework/Test/Util/TestHookObjectExtractor.php +++ b/src/Magento/FunctionalTestingFramework/Test/Util/TestHookObjectExtractor.php @@ -57,19 +57,22 @@ public function extractHook($parentName, $hookType, $testHook) /** * Creates the default failed hook object with a single saveScreenshot action. + * And a pause action when ENABLE_PAUSE is set to true. * * @param string $parentName * @return TestHookObject */ public function createDefaultFailedHook($parentName) { - - $saveScreenshotStep = [new ActionObject("saveScreenshot", "saveScreenshot", [])]; + $defaultSteps['saveScreenshot'] = new ActionObject("saveScreenshot", "saveScreenshot", []); + if (getenv('ENABLE_PAUSE') === 'true') { + $defaultSteps['pauseWhenFailed'] = new ActionObject('pauseWhenFailed', 'pause', []); + } $hook = new TestHookObject( TestObjectExtractor::TEST_FAILED_HOOK, $parentName, - $saveScreenshotStep + $defaultSteps ); return $hook; From 0dcc9fa7da86c03436e9142582a461978fff1bf6 Mon Sep 17 00:00:00 2001 From: Ji Lu <jilu1@adobe.com> Date: Fri, 7 Aug 2020 11:42:16 -0500 Subject: [PATCH 31/52] MQE-2110: MFTF command to pause test execution --- .../FunctionalTestingFramework/Console/RunManifestCommand.php | 4 +++- .../FunctionalTestingFramework/Console/RunTestCommand.php | 4 +++- .../Console/RunTestFailedCommand.php | 4 +++- .../Console/RunTestGroupCommand.php | 4 +++- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/Console/RunManifestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunManifestCommand.php index 4e9c92d38..1709726bb 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunManifestCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunManifestCommand.php @@ -119,7 +119,9 @@ private function runManifestLine(string $manifestLine, OutputInterface $output) $process->setWorkingDirectory(TESTS_BP); $process->setIdleTimeout(600); $process->setTimeout(0); - $process->setInput(STDIN); + if (getenv('ENABLE_PAUSE') === 'true') { + $process->setInput(STDIN); + } $subReturnCode = $process->run(function ($type, $buffer) use ($output) { $output->write($buffer); }); diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php index c96898da8..86352d5c6 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php @@ -179,7 +179,9 @@ private function executeTestCommand(string $command, OutputInterface $output) $process->setWorkingDirectory(TESTS_BP); $process->setIdleTimeout(600); $process->setTimeout(0); - $process->setInput(STDIN); + if ($this->pauseEnabled()) { + $process->setInput(STDIN); + } return $process->run(function ($type, $buffer) use ($output) { $output->write($buffer); }); diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php index 96f560f24..b9012b086 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php @@ -127,7 +127,9 @@ protected function execute(InputInterface $input, OutputInterface $output): int $process->setWorkingDirectory(TESTS_BP); $process->setIdleTimeout(600); $process->setTimeout(0); - $process->setInput(STDIN); + if ($this->pauseEnabled()) { + $process->setInput(STDIN); + } $returnCode = max($returnCode, $process->run( function ($type, $buffer) use ($output) { $output->write($buffer); diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php index db3ea2e55..19dec6d4b 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php @@ -108,7 +108,9 @@ protected function execute(InputInterface $input, OutputInterface $output): int $process->setWorkingDirectory(TESTS_BP); $process->setIdleTimeout(600); $process->setTimeout(0); - $process->setInput(STDIN); + if ($this->pauseEnabled()) { + $process->setInput(STDIN); + } $returnCodes[] = $process->run( function ($type, $buffer) use ($output) { From 20af7fdf70171a0b2de5a2d284e4c12e59c61c8a Mon Sep 17 00:00:00 2001 From: soumyau <sunnikri@adobe.com> Date: Mon, 10 Aug 2020 10:57:58 -0500 Subject: [PATCH 32/52] =?UTF-8?q?MQE-2229:=20Deprecation=20Error=20When=20?= =?UTF-8?q?Deprecated=20ActionGroup=20References=20De=E2=80=A6=20(#775)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * MQE-2229: Deprecation Error When Deprecated ActionGroup References Deprecated Element * MQE-2229: Deprecation Error When Deprecated ActionGroup References Deprecated Element Suppressed check for data entities * MQE-2229: Deprecation Error When Deprecated ActionGroup References Deprecated Element Added unit tests * MQE-2229: Deprecation Error When Deprecated ActionGroup References Deprecated Element Added ObjectHandlerUtil for mocking Handlers * MQE-2229: Deprecation Error When Deprecated ActionGroup References Deprecated Element Verification test --- .../Handlers/DataObjectHandlerTest.php | 44 +-- .../OperationDefinitionObjectHandlerTest.php | 34 +-- .../Handlers/PersistedObjectHandlerTest.php | 41 +-- .../Page/Handlers/PageObjectHandlerTest.php | 24 +- .../Handlers/SectionObjectHandlerTest.php | 22 +- .../DeprecatedEntityUsageCheckTest.php | 251 ++++++++++++++++++ .../Handlers/ActionGroupObjectHandlerTest.php | 31 +-- .../Test/Handlers/TestObjectHandlerTest.php | 30 +-- dev/tests/unit/Util/ObjectHandlerUtil.php | 145 ++++++++++ .../DeprecationCheckActionGroup.xml | 13 + .../DeprecationCheckDeprecatedActionGroup.xml | 13 + .../Data/DeprecationCheckData.xml | 14 + .../Metadata/DeprecationCheckMeta.xml | 18 ++ .../Page/DeprecationCheckPage.xml | 14 + .../Section/DeprecationCheckSection.xml | 16 ++ .../Suite/deprecationCheckSuite.xml | 16 ++ .../Test/DeprecationCheckDeprecatedTest.xml | 17 ++ .../Test/DeprecationCheckTest.xml | 17 ++ .../mftf-deprecated-entity-usage-checks.txt | 23 ++ .../DeprecationStaticCheckTest.php | 92 +++++++ .../DeprecatedEntityUsageCheck.php | 31 ++- .../StaticCheck/StaticChecksList.php | 18 ++ 22 files changed, 739 insertions(+), 185 deletions(-) create mode 100644 dev/tests/unit/Magento/FunctionalTestFramework/StaticCheck/DeprecatedEntityUsageCheckTest.php create mode 100644 dev/tests/unit/Util/ObjectHandlerUtil.php create mode 100644 dev/tests/verification/DeprecationCheckModule/ActionGroup/DeprecationCheckActionGroup.xml create mode 100644 dev/tests/verification/DeprecationCheckModule/ActionGroup/DeprecationCheckDeprecatedActionGroup.xml create mode 100644 dev/tests/verification/DeprecationCheckModule/Data/DeprecationCheckData.xml create mode 100644 dev/tests/verification/DeprecationCheckModule/Metadata/DeprecationCheckMeta.xml create mode 100644 dev/tests/verification/DeprecationCheckModule/Page/DeprecationCheckPage.xml create mode 100644 dev/tests/verification/DeprecationCheckModule/Section/DeprecationCheckSection.xml create mode 100644 dev/tests/verification/DeprecationCheckModule/Suite/deprecationCheckSuite.xml create mode 100644 dev/tests/verification/DeprecationCheckModule/Test/DeprecationCheckDeprecatedTest.xml create mode 100644 dev/tests/verification/DeprecationCheckModule/Test/DeprecationCheckTest.xml create mode 100644 dev/tests/verification/Resources/StaticChecks/mftf-deprecated-entity-usage-checks.txt create mode 100644 dev/tests/verification/Tests/StaticCheck/DeprecationStaticCheckTest.php diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/DataObjectHandlerTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/DataObjectHandlerTest.php index 9f1ca7b5c..745789501 100644 --- a/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/DataObjectHandlerTest.php +++ b/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/DataObjectHandlerTest.php @@ -13,6 +13,7 @@ use Magento\FunctionalTestingFramework\ObjectManager; use Magento\FunctionalTestingFramework\ObjectManagerFactory; use tests\unit\Util\MagentoTestCase; +use tests\unit\Util\ObjectHandlerUtil; use tests\unit\Util\TestLoggingUtil; /** @@ -148,7 +149,7 @@ public function setUp(): void */ public function testGetAllObjects() { - $this->setUpMockDataObjectHander(self::PARSER_OUTPUT); + ObjectHandlerUtil::mockDataObjectHandlerWithData(self::PARSER_OUTPUT); // Call the method under test $actual = DataObjectHandler::getInstance()->getAllObjects(); @@ -164,7 +165,7 @@ public function testGetAllObjects() */ public function testDeprecatedDataObject() { - $this->setUpMockDataObjectHander(self::PARSER_OUTPUT_DEPRECATED); + ObjectHandlerUtil::mockDataObjectHandlerWithData(self::PARSER_OUTPUT_DEPRECATED); // Call the method under test $actual = DataObjectHandler::getInstance()->getAllObjects(); @@ -182,7 +183,7 @@ public function testDeprecatedDataObject() */ public function testGetObject() { - $this->setUpMockDataObjectHander(self::PARSER_OUTPUT); + ObjectHandlerUtil::mockDataObjectHandlerWithData(self::PARSER_OUTPUT); // Call the method under test $actual = DataObjectHandler::getInstance()->getObject('EntityOne'); @@ -197,7 +198,7 @@ public function testGetObject() */ public function testGetObjectNull() { - $this->setUpMockDataObjectHander(self::PARSER_OUTPUT); + ObjectHandlerUtil::mockDataObjectHandlerWithData(self::PARSER_OUTPUT); $actual = DataObjectHandler::getInstance()->getObject('h953u789h0g73t521'); // doesnt exist $this->assertNull($actual); @@ -208,7 +209,7 @@ public function testGetObjectNull() */ public function testGetAllObjectsWithDataExtends() { - $this->setUpMockDataObjectHander(self::PARSER_OUTPUT_WITH_EXTEND); + ObjectHandlerUtil::mockDataObjectHandlerWithData(self::PARSER_OUTPUT_WITH_EXTEND); // Call the method under test $actual = DataObjectHandler::getInstance()->getAllObjects(); @@ -232,7 +233,7 @@ public function testGetAllObjectsWithDataExtends() */ public function testGetObjectWithDataExtends() { - $this->setUpMockDataObjectHander(self::PARSER_OUTPUT_WITH_EXTEND); + ObjectHandlerUtil::mockDataObjectHandlerWithData(self::PARSER_OUTPUT_WITH_EXTEND); // Call the method under test $actual = DataObjectHandler::getInstance()->getObject('EntityTwo'); @@ -255,7 +256,7 @@ public function testGetObjectWithDataExtends() */ public function testGetAllObjectsWithDataExtendsItself() { - $this->setUpMockDataObjectHander(self::PARSER_OUTPUT_WITH_EXTEND_INVALID); + ObjectHandlerUtil::mockDataObjectHandlerWithData(self::PARSER_OUTPUT_WITH_EXTEND_INVALID); $this->expectException(\Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException::class); $this->expectExceptionMessage( @@ -272,7 +273,7 @@ public function testGetAllObjectsWithDataExtendsItself() */ public function testGetObjectWithDataExtendsItself() { - $this->setUpMockDataObjectHander(self::PARSER_OUTPUT_WITH_EXTEND_INVALID); + ObjectHandlerUtil::mockDataObjectHandlerWithData(self::PARSER_OUTPUT_WITH_EXTEND_INVALID); $this->expectException(\Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException::class); $this->expectExceptionMessage( @@ -286,33 +287,6 @@ public function testGetObjectWithDataExtendsItself() ); } - /** - * Set up everything required to mock DataObjectHander::getInstance() - * The first call to getInstance() uses these mocks to emulate the parser, initializing internal state - * according to the PARSER_OUTPUT value - * - * @param array $entityDataArray - */ - private function setUpMockDataObjectHander($entityDataArray) - { - // Clear DataObjectHandler singleton if already set - $property = new \ReflectionProperty(DataObjectHandler::class, "INSTANCE"); - $property->setAccessible(true); - $property->setValue(null); - - $mockDataProfileSchemaParser = AspectMock::double(DataProfileSchemaParser::class, [ - 'readDataProfiles' => $entityDataArray - ])->make(); - - $mockObjectManager = AspectMock::double(ObjectManager::class, [ - 'create' => $mockDataProfileSchemaParser - ])->make(); - - AspectMock::double(ObjectManagerFactory::class, [ - 'getObjectManager' => $mockObjectManager - ]); - } - /** * clean up function runs after all tests */ diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/OperationDefinitionObjectHandlerTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/OperationDefinitionObjectHandlerTest.php index 570e8d95a..35b06a24a 100644 --- a/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/OperationDefinitionObjectHandlerTest.php +++ b/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/OperationDefinitionObjectHandlerTest.php @@ -14,6 +14,7 @@ use Magento\FunctionalTestingFramework\DataGenerator\Handlers\OperationDefinitionObjectHandler; use Magento\FunctionalTestingFramework\DataGenerator\Parsers\OperationDefinitionParser; use tests\unit\Util\MagentoTestCase; +use tests\unit\Util\ObjectHandlerUtil; use tests\unit\Util\TestLoggingUtil; /** @@ -72,7 +73,7 @@ public function testGetMultipleObjects() ], ] ]]]; - $this->setMockParserOutput($mockData); + ObjectHandlerUtil::mockOperationHandlerWithData($mockData); //Perform Assertions $operationDefinitionManager = OperationDefinitionObjectHandler::getInstance(); @@ -109,7 +110,7 @@ public function testDeprecatedOperation() ], OperationDefinitionObjectHandler::OBJ_DEPRECATED => 'deprecation message' ]]]; - $this->setMockParserOutput($mockData); + ObjectHandlerUtil::mockOperationHandlerWithData($mockData); //Perform Assertions $operationDefinitionManager = OperationDefinitionObjectHandler::getInstance(); @@ -239,7 +240,7 @@ public function testObjectCreation() ); // Set up mocked data output - $this->setMockParserOutput($mockData); + ObjectHandlerUtil::mockOperationHandlerWithData($mockData); // Get Operation $operationDefinitionManager = OperationDefinitionObjectHandler::getInstance(); @@ -337,7 +338,7 @@ public function testObjectArrayCreation() ); // Set up mocked data output - $this->setMockParserOutput($mockData); + ObjectHandlerUtil::mockOperationHandlerWithData($mockData); // Get Operation $operationDefinitionManager = OperationDefinitionObjectHandler::getInstance(); @@ -405,7 +406,7 @@ public function testLooseJsonCreation() ); // Set up mocked data output - $this->setMockParserOutput($mockData); + ObjectHandlerUtil::mockOperationHandlerWithData($mockData); // get Operations $operationDefinitionManager = OperationDefinitionObjectHandler::getInstance(); @@ -416,29 +417,6 @@ public function testLooseJsonCreation() $this->assertEquals($array, $operation->getOperationMetadata()[1]); } - /** - * Function used to set mock for parser return and force init method to run between tests. - * - * @param array $data - */ - private function setMockParserOutput($data) - { - // clear Operation object handler value to inject parsed content - $property = new \ReflectionProperty( - OperationDefinitionObjectHandler::class, - 'INSTANCE' - ); - $property->setAccessible(true); - $property->setValue(null); - - $mockOperationParser = AspectMock::double( - OperationDefinitionParser::class, - ["readOperationMetadata" => $data] - )->make(); - $instance = AspectMock::double(ObjectManager::class, ['create' => $mockOperationParser])->make(); - AspectMock::double(ObjectManagerFactory::class, ['getObjectManager' => $instance]); - } - /** * clean up function runs after all tests */ diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/PersistedObjectHandlerTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/PersistedObjectHandlerTest.php index 870e75dcd..7e39fe216 100644 --- a/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/PersistedObjectHandlerTest.php +++ b/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/PersistedObjectHandlerTest.php @@ -16,6 +16,7 @@ use Magento\FunctionalTestingFramework\ObjectManager; use Magento\FunctionalTestingFramework\ObjectManagerFactory; use tests\unit\Util\MagentoTestCase; +use tests\unit\Util\ObjectHandlerUtil; use tests\unit\Util\TestLoggingUtil; /** @@ -84,7 +85,7 @@ public function testCreateSimpleEntity() "; // Mock Classes - $this->mockDataHandlerWithOutput($parserOutput); + ObjectHandlerUtil::mockDataObjectHandlerWithData($parserOutput); $this->mockCurlHandler($jsonResponse); $handler = PersistedObjectHandler::getInstance(); @@ -127,7 +128,7 @@ public function testDeleteSimpleEntity() "; // Mock Classes - $this->mockDataHandlerWithOutput($parserOutput); + ObjectHandlerUtil::mockDataObjectHandlerWithData($parserOutput); $this->mockCurlHandler($jsonResponse); $handler = PersistedObjectHandler::getInstance(); @@ -175,7 +176,7 @@ public function testGetSimpleEntity() "; // Mock Classes - $this->mockDataHandlerWithOutput($parserOutput); + ObjectHandlerUtil::mockDataObjectHandlerWithData($parserOutput); $this->mockCurlHandler($jsonResponse); $handler = PersistedObjectHandler::getInstance(); @@ -235,7 +236,7 @@ public function testUpdateSimpleEntity() "; // Mock Classes - $this->mockDataHandlerWithOutput($parserOutput); + ObjectHandlerUtil::mockDataObjectHandlerWithData($parserOutput); $this->mockCurlHandler($jsonResponse); $handler = PersistedObjectHandler::getInstance(); $handler->createEntity( @@ -322,7 +323,7 @@ public function testRetrieveEntityAcrossScopes() // Mock Classes and Create Entities $handler = PersistedObjectHandler::getInstance(); - $this->mockDataHandlerWithOutput($parserOutputOne); + ObjectHandlerUtil::mockDataObjectHandlerWithData($parserOutputOne); $this->mockCurlHandler($jsonReponseOne); $handler->createEntity( $entityStepKeyOne, @@ -399,7 +400,7 @@ public function testRetrieveEntityValidField($name, $key, $value, $type, $scope, // Mock Classes and Create Entities $handler = PersistedObjectHandler::getInstance(); - $this->mockDataHandlerWithOutput($parserOutputOne); + ObjectHandlerUtil::mockDataObjectHandlerWithData($parserOutputOne); $this->mockCurlHandler($jsonReponseOne); $handler->createEntity($stepKey, $scope, $name); @@ -447,8 +448,7 @@ public function testRetrieveEntityInValidField($name, $key, $value, $type, $scop // Mock Classes and Create Entities $handler = PersistedObjectHandler::getInstance(); - - $this->mockDataHandlerWithOutput($parserOutputOne); + ObjectHandlerUtil::mockDataObjectHandlerWithData($parserOutputOne); $this->mockCurlHandler($jsonReponseOne); $handler->createEntity($stepKey, $scope, $name); @@ -475,31 +475,6 @@ public static function entityDataProvider() ]; } - /** - * Mocks DataObjectHandler to use given output to create - * @param $parserOutput - * @throws \Exception - */ - public function mockDataHandlerWithOutput($parserOutput) - { - // Clear DataObjectHandler singleton if already set - $property = new \ReflectionProperty(DataObjectHandler::class, "INSTANCE"); - $property->setAccessible(true); - $property->setValue(null); - - $mockDataProfileSchemaParser = AspectMock::double(DataProfileSchemaParser::class, [ - 'readDataProfiles' => $parserOutput - ])->make(); - - $mockObjectManager = AspectMock::double(ObjectManager::class, [ - 'create' => $mockDataProfileSchemaParser - ])->make(); - - AspectMock::double(ObjectManagerFactory::class, [ - 'getObjectManager' => $mockObjectManager - ]); - } - public function mockCurlHandler($response) { AspectMock::double(CurlHandler::class, [ diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Page/Handlers/PageObjectHandlerTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Page/Handlers/PageObjectHandlerTest.php index c934b680b..eb305bc4d 100644 --- a/dev/tests/unit/Magento/FunctionalTestFramework/Page/Handlers/PageObjectHandlerTest.php +++ b/dev/tests/unit/Magento/FunctionalTestFramework/Page/Handlers/PageObjectHandlerTest.php @@ -12,6 +12,7 @@ use Magento\FunctionalTestingFramework\Page\Handlers\PageObjectHandler; use Magento\FunctionalTestingFramework\XmlParser\PageParser; use tests\unit\Util\MagentoTestCase; +use tests\unit\Util\ObjectHandlerUtil; use tests\unit\Util\TestLoggingUtil; class PageObjectHandlerTest extends MagentoTestCase @@ -45,7 +46,7 @@ public function testGetPageObject() ], "area" => "test" ]]; - $this->setMockParserOutput($mockData); + ObjectHandlerUtil::mockPageObjectHandlerWithData($mockData); // get pages $pageHandler = PageObjectHandler::getInstance(); @@ -70,7 +71,7 @@ public function testGetEmptyPage() ], "area" => "test" ]]; - $this->setMockParserOutput($mockData); + ObjectHandlerUtil::mockPageObjectHandlerWithData($mockData); // get pages $page = PageObjectHandler::getInstance()->getObject('testPage1'); @@ -91,7 +92,7 @@ public function testDeprecatedPage() "deprecated" => "deprecation message", "filename" => "filename.xml" ]]; - $this->setMockParserOutput($mockData); + ObjectHandlerUtil::mockPageObjectHandlerWithData($mockData); // get pages $page = PageObjectHandler::getInstance()->getObject('testPage1'); @@ -103,23 +104,6 @@ public function testDeprecatedPage() ); } - /** - * Function used to set mock for parser return and force init method to run between tests. - * - * @param array $data - */ - private function setMockParserOutput($data) - { - // clear section object handler value to inject parsed content - $property = new \ReflectionProperty(PageObjectHandler::class, 'INSTANCE'); - $property->setAccessible(true); - $property->setValue(null); - - $mockSectionParser = AspectMock::double(PageParser::class, ["getData" => $data])->make(); - $instance = AspectMock::double(ObjectManager::class, ['get' => $mockSectionParser])->make(); - AspectMock::double(ObjectManagerFactory::class, ['getObjectManager' => $instance]); - } - /** * clean up function runs after all tests */ diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Page/Handlers/SectionObjectHandlerTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Page/Handlers/SectionObjectHandlerTest.php index 39e26caf9..69088944a 100644 --- a/dev/tests/unit/Magento/FunctionalTestFramework/Page/Handlers/SectionObjectHandlerTest.php +++ b/dev/tests/unit/Magento/FunctionalTestFramework/Page/Handlers/SectionObjectHandlerTest.php @@ -12,6 +12,7 @@ use Magento\FunctionalTestingFramework\Page\Handlers\SectionObjectHandler; use Magento\FunctionalTestingFramework\XmlParser\SectionParser; use tests\unit\Util\MagentoTestCase; +use tests\unit\Util\ObjectHandlerUtil; use tests\unit\Util\TestLoggingUtil; class SectionObjectHandlerTest extends MagentoTestCase @@ -46,7 +47,7 @@ public function testGetSectionObject() ] ]; - $this->setMockParserOutput($mockData); + ObjectHandlerUtil::mockSectionObjectHandlerWithData($mockData); // get sections $sectionHandler = SectionObjectHandler::getInstance(); @@ -77,7 +78,7 @@ public function testDeprecatedSection() ] ]; - $this->setMockParserOutput($mockData); + ObjectHandlerUtil::mockSectionObjectHandlerWithData($mockData); // get sections $sectionHandler = SectionObjectHandler::getInstance(); @@ -91,23 +92,6 @@ public function testDeprecatedSection() ); } - /** - * Set the mock parser return value - * - * @param array $data - */ - private function setMockParserOutput($data) - { - // clear section object handler value to inject parsed content - $property = new \ReflectionProperty(SectionObjectHandler::class, "INSTANCE"); - $property->setAccessible(true); - $property->setValue(null); - - $mockSectionParser = AspectMock::double(SectionParser::class, ["getData" => $data])->make(); - $instance = AspectMock::double(ObjectManager::class, ["get" => $mockSectionParser])->make(); - AspectMock::double(ObjectManagerFactory::class, ["getObjectManager" => $instance]); - } - /** * clean up function runs after all tests */ diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/StaticCheck/DeprecatedEntityUsageCheckTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/StaticCheck/DeprecatedEntityUsageCheckTest.php new file mode 100644 index 000000000..dc339c8a3 --- /dev/null +++ b/dev/tests/unit/Magento/FunctionalTestFramework/StaticCheck/DeprecatedEntityUsageCheckTest.php @@ -0,0 +1,251 @@ +<?php +/** + * Copyright © Magento, Inc. All rights reserved. + * See COPYING.txt for license details. + */ + +namespace tests\unit\Magento\FunctionalTestFramework\StaticCheck; + +use AspectMock\Test as AspectMock; +use Magento\FunctionalTestingFramework\DataGenerator\Handlers\OperationDefinitionObjectHandler; +use Magento\FunctionalTestingFramework\DataGenerator\Objects\EntityDataObject; +use Magento\FunctionalTestingFramework\Page\Objects\ElementObject; +use Magento\FunctionalTestingFramework\Page\Objects\PageObject; +use Magento\FunctionalTestingFramework\Page\Objects\SectionObject; +use Magento\FunctionalTestingFramework\StaticCheck\DeprecatedEntityUsageCheck; +use Magento\FunctionalTestingFramework\Test\Objects\TestObject; +use Symfony\Component\Console\Input\InputInterface; +use tests\unit\Util\MagentoTestCase; +use ReflectionClass; +use InvalidArgumentException; +use tests\unit\Util\ObjectHandlerUtil; + +class DeprecatedEntityUsageCheckTest extends MagentoTestCase +{ + /** @var DeprecatedEntityUsageCheck */ + private $staticCheck; + + /** @var ReflectionClass*/ + private $staticCheckClass; + + public function setUp(): void + { + $this->staticCheck = new DeprecatedEntityUsageCheck(); + $this->staticCheckClass = new \ReflectionClass($this->staticCheck); + } + + public function tearDown(): void + { + AspectMock::clean(); + } + + public function testInvalidPathOption() + { + $input = $this->getMockBuilder(InputInterface::class) + ->disableOriginalConstructor() + ->getMock(); + + $input->method('getOption') + ->with('path') + ->willReturn('/invalidPath'); + + $loadAllXmlFiles = $this->staticCheckClass->getMethod('loadAllXMLFiles'); + $loadAllXmlFiles->setAccessible(true); + + $this->expectException(InvalidArgumentException::class); + $loadAllXmlFiles->invoke($this->staticCheck, $input); + } + + public function testViolatingElementReferences() + { + //variables for assertions + $elementName = 'elementOne'; + $sectionName = 'SectionOne'; + $fileName = 'section.xml'; + + $element = new ElementObject($elementName, 'type', '#selector1', null, '41', false, 'deprecated'); + $section = new SectionObject($sectionName, [$element], $fileName); + $elementRef = $sectionName . '.' . $elementName; + $references = [$elementRef => $element, $sectionName => $section]; + $actual = $this->callViolatingReferences($references); + $expected = [ + 'Deprecated Element(s)' => [ + 0 => [ + 'name' => $elementRef, + 'file' => $fileName + ] + ] + ]; + $this->assertEquals($actual, $expected); + } + + public function testViolatingPageReferences() + { + //Page variables for assertions + $pageName = 'Page'; + $fileName = 'page.xml'; + + $page = new PageObject($pageName, '/url.html', 'Test', [], false, "test", $fileName, 'deprecated'); + $references = ['Page' => $page]; + $actual = $this->callViolatingReferences($references); + $expected = [ + 'Deprecated Page(s)' => [ + 0 => [ + 'name' => $pageName, + 'file' => $fileName + ] + ] + ]; + $this->assertEquals($actual, $expected); + } + + public function testViolatingDataReferences() + { + //Data entity variables for assertions + $entityName = 'EntityOne'; + $fileName = 'entity.xml'; + + $entity = new EntityDataObject( + $entityName, + 'testType', + ['testkey' => 'testValue'], + [], + null, + [], + null, + $fileName, + 'deprecated' + ); + $references = [$entityName => $entity]; + $actual = $this->callViolatingReferences($references); + $expected = [ + 'Deprecated Data(s)' => [ + 0 => [ + 'name' => $entityName, + 'file' => $fileName + ] + ] + ]; + $this->assertEquals($actual, $expected); + } + + public function testViolatingTestReferences() + { + // test variables for assertions + $testName = 'Test1'; + $fileName = 'test.xml'; + + $test = new TestObject($testName, [], [], [], $fileName, null, 'deprecated'); + $references = ['Test1' => $test]; + $actual = $this->callViolatingReferences($references); + $expected = [ + 'Deprecated Test(s)' => [ + 0 => [ + 'name' => $testName, + 'file' => $fileName + ] + ] + ]; + $this->assertEquals($actual, $expected); + } + + public function testViolatingMetaDataReferences() + { + // Data Variables for Assertions + $dataType1 = "type1"; + $operationType1 = "create"; + $operationType2 = "update"; + + /** + * Parser Output. + * operationName + * createType1 + * has field + * key=id, value=integer + * updateType1 + * has field + * key=id, value=integer + */ + $mockData = [OperationDefinitionObjectHandler::ENTITY_OPERATION_ROOT_TAG => [ + "testOperationName" => [ + OperationDefinitionObjectHandler::ENTITY_OPERATION_DATA_TYPE => $dataType1, + OperationDefinitionObjectHandler::ENTITY_OPERATION_TYPE => $operationType1, + OperationDefinitionObjectHandler::ENTITY_OPERATION_AUTH => "auth", + OperationDefinitionObjectHandler::ENTITY_OPERATION_URL => "V1/Type1", + OperationDefinitionObjectHandler::ENTITY_OPERATION_METHOD => "POST", + OperationDefinitionObjectHandler::ENTITY_OPERATION_ENTRY => [ + 0 => [ + OperationDefinitionObjectHandler::ENTITY_OPERATION_ENTRY_KEY => "id", + OperationDefinitionObjectHandler::ENTITY_OPERATION_ENTRY_VALUE => "integer" + ], + ], + OperationDefinitionObjectHandler::OBJ_DEPRECATED => 'deprecated' + ],[ + OperationDefinitionObjectHandler::ENTITY_OPERATION_DATA_TYPE => $dataType1, + OperationDefinitionObjectHandler::ENTITY_OPERATION_TYPE => $operationType2, + OperationDefinitionObjectHandler::ENTITY_OPERATION_AUTH => "auth", + OperationDefinitionObjectHandler::ENTITY_OPERATION_URL => "V1/Type1/{id}", + OperationDefinitionObjectHandler::ENTITY_OPERATION_METHOD => "PUT", + OperationDefinitionObjectHandler::ENTITY_OPERATION_ENTRY => [ + 0 => [ + OperationDefinitionObjectHandler::ENTITY_OPERATION_ENTRY_KEY => "id", + OperationDefinitionObjectHandler::ENTITY_OPERATION_ENTRY_VALUE => "integer" + ], + ] + ]]]; + + ObjectHandlerUtil::mockOperationHandlerWithData($mockData); + $dataName = 'dataName1'; + $references = [ + $dataName => [ + $dataType1 => [ + $operationType1, + $operationType2 + ] + ] + ]; + + $expected = [ + '"'.$dataName.'" references deprecated' => [ + 0 => [ + 'name' => $dataType1, + 'file' => 'metadata xml file' + ] + ] + ]; + $property = $this->staticCheckClass->getMethod('findViolatingMetadataReferences'); + $property->setAccessible(true); + $actual = $property->invoke($this->staticCheck, $references); + $this->assertEquals($actual, $expected); + } + + public function testIsDeprecated() + { + // Test Data + $contents = '<tests> + <test name="test" deprecated="true"> + <comment userInput="input1" stepKey="key1"/> + <comment userInput="input2" stepKey="key1"/> + </test> + </tests> + '; + + $property = $this->staticCheckClass->getMethod('isDeprecated'); + $property->setAccessible(true); + $output = $property->invoke($this->staticCheck, $contents); + $this->assertTrue($output); + } + + /** + * Invoke findViolatingReferences + * @param $references + * @return mixed + * @throws \ReflectionException + */ + public function callViolatingReferences($references) + { + $property = $this->staticCheckClass->getMethod('findViolatingReferences'); + $property->setAccessible(true); + return $property->invoke($this->staticCheck, $references); + } +} diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Test/Handlers/ActionGroupObjectHandlerTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Test/Handlers/ActionGroupObjectHandlerTest.php index deb08d783..71c1833ba 100644 --- a/dev/tests/unit/Magento/FunctionalTestFramework/Test/Handlers/ActionGroupObjectHandlerTest.php +++ b/dev/tests/unit/Magento/FunctionalTestFramework/Test/Handlers/ActionGroupObjectHandlerTest.php @@ -15,6 +15,7 @@ use tests\unit\Util\MagentoTestCase; use tests\unit\Util\ActionGroupArrayBuilder; use Magento\FunctionalTestingFramework\Test\Parsers\ActionGroupDataParser; +use tests\unit\Util\ObjectHandlerUtil; class ActionGroupObjectHandlerTest extends MagentoTestCase { @@ -34,7 +35,7 @@ public function testGetTestObjectWithInvalidExtends() ->withFilename() ->withActionObjects() ->build(); - $this->setMockParserOutput(['actionGroups' => $actionGroupOne]); + ObjectHandlerUtil::mockActionGroupObjectHandlerWithData(['actionGroups' => $actionGroupOne]); $handler = ActionGroupObjectHandler::getInstance(); @@ -68,7 +69,14 @@ public function testGetAllTestObjectsWithInvalidExtends() ->withActionObjects() ->build(); - $this->setMockParserOutput(['actionGroups' => array_merge($actionGroupOne, $actionGroupTwo)]); + ObjectHandlerUtil::mockActionGroupObjectHandlerWithData( + [ + 'actionGroups' => array_merge( + $actionGroupOne, + $actionGroupTwo + ) + ] + ); $handler = ActionGroupObjectHandler::getInstance(); @@ -76,23 +84,4 @@ public function testGetAllTestObjectsWithInvalidExtends() $this->expectExceptionMessage("Mftf Action Group can not extend from itself: " . $nameOne); $handler->getAllObjects(); } - - /** - * Function used to set mock for parser return and force init method to run between tests. - * - * @param array $data - * @throws \Exception - */ - private function setMockParserOutput($data) - { - // Clear action group object handler value to inject parsed content - $property = new \ReflectionProperty(ActionGroupObjectHandler::class, 'instance'); - $property->setAccessible(true); - $property->setValue(null); - - $mockDataParser = AspectMock::double(ActionGroupDataParser::class, ['readActionGroupData' => $data])->make(); - $instance = AspectMock::double(ObjectManager::class, ['create' => $mockDataParser]) - ->make(); // bypass the private constructor - AspectMock::double(ObjectManagerFactory::class, ['getObjectManager' => $instance]); - } } diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Test/Handlers/TestObjectHandlerTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Test/Handlers/TestObjectHandlerTest.php index 4db2f21a4..897ba0ea9 100644 --- a/dev/tests/unit/Magento/FunctionalTestFramework/Test/Handlers/TestObjectHandlerTest.php +++ b/dev/tests/unit/Magento/FunctionalTestFramework/Test/Handlers/TestObjectHandlerTest.php @@ -17,6 +17,7 @@ use Magento\FunctionalTestingFramework\Test\Parsers\TestDataParser; use Magento\FunctionalTestingFramework\Test\Util\TestObjectExtractor; use tests\unit\Util\MagentoTestCase; +use tests\unit\Util\ObjectHandlerUtil; use tests\unit\Util\TestDataArrayBuilder; use tests\unit\Util\MockModuleResolverBuilder; @@ -41,7 +42,7 @@ public function testGetTestObject() $resolverMock = new MockModuleResolverBuilder(); $resolverMock->setup(); - $this->setMockParserOutput($mockData); + ObjectHandlerUtil::mockTestObjectHandlerWitData($mockData); // run object handler method $toh = TestObjectHandler::getInstance(); @@ -135,7 +136,7 @@ public function testGetTestsByGroup() $resolverMock = new MockModuleResolverBuilder(); $resolverMock->setup(); - $this->setMockParserOutput(array_merge($includeTest, $excludeTest)); + ObjectHandlerUtil::mockTestObjectHandlerWitData(array_merge($includeTest, $excludeTest)); // execute test method $toh = TestObjectHandler::getInstance(); @@ -184,7 +185,7 @@ public function testGetTestWithModuleName() $resolverMock = new MockModuleResolverBuilder(); $resolverMock->setup(['Vendor_' . $moduleExpected => $filepath]); - $this->setMockParserOutput($mockData); + ObjectHandlerUtil::mockTestObjectHandlerWitData($mockData); // Execute Test Method $toh = TestObjectHandler::getInstance(); $actualTestObject = $toh->getObject($testDataArrayBuilder->testName); @@ -212,7 +213,7 @@ public function testGetTestObjectWithInvalidExtends() ->build(); $resolverMock = new MockModuleResolverBuilder(); $resolverMock->setup(); - $this->setMockParserOutput($testOne); + ObjectHandlerUtil::mockTestObjectHandlerWitData($testOne); $toh = TestObjectHandler::getInstance(); @@ -250,7 +251,7 @@ public function testGetAllTestObjectsWithInvalidExtends() $resolverMock = new MockModuleResolverBuilder(); $resolverMock->setup(); - $this->setMockParserOutput(array_merge($testOne, $testTwo)); + ObjectHandlerUtil::mockTestObjectHandlerWitData(array_merge($testOne, $testTwo)); $toh = TestObjectHandler::getInstance(); @@ -259,25 +260,6 @@ public function testGetAllTestObjectsWithInvalidExtends() $toh->getAllObjects(); } - /** - * Function used to set mock for parser return and force init method to run between tests. - * - * @param array $data - * @throws \Exception - */ - private function setMockParserOutput($data) - { - // clear test object handler value to inject parsed content - $property = new \ReflectionProperty(TestObjectHandler::class, 'testObjectHandler'); - $property->setAccessible(true); - $property->setValue(null); - - $mockDataParser = AspectMock::double(TestDataParser::class, ['readTestData' => $data])->make(); - $instance = AspectMock::double(ObjectManager::class, ['create' => $mockDataParser]) - ->make(); // bypass the private constructor - AspectMock::double(ObjectManagerFactory::class, ['getObjectManager' => $instance]); - } - /** * After method functionality * diff --git a/dev/tests/unit/Util/ObjectHandlerUtil.php b/dev/tests/unit/Util/ObjectHandlerUtil.php new file mode 100644 index 000000000..0e4543f2b --- /dev/null +++ b/dev/tests/unit/Util/ObjectHandlerUtil.php @@ -0,0 +1,145 @@ +<?php +/** + * Copyright © Magento, Inc. All rights reserved. + * See COPYING.txt for license details. + */ +namespace tests\unit\Util; + +use AspectMock\Test as AspectMock; +use Magento\FunctionalTestingFramework\DataGenerator\Handlers\DataObjectHandler; +use Magento\FunctionalTestingFramework\DataGenerator\Handlers\OperationDefinitionObjectHandler; +use Magento\FunctionalTestingFramework\DataGenerator\Parsers\DataProfileSchemaParser; +use Magento\FunctionalTestingFramework\DataGenerator\Parsers\OperationDefinitionParser; +use Magento\FunctionalTestingFramework\ObjectManager; +use Magento\FunctionalTestingFramework\ObjectManagerFactory; +use Magento\FunctionalTestingFramework\Page\Handlers\PageObjectHandler; +use Magento\FunctionalTestingFramework\Page\Handlers\SectionObjectHandler; +use Magento\FunctionalTestingFramework\Test\Handlers\ActionGroupObjectHandler; +use Magento\FunctionalTestingFramework\Test\Handlers\TestObjectHandler; +use Magento\FunctionalTestingFramework\Test\Parsers\ActionGroupDataParser; +use Magento\FunctionalTestingFramework\Test\Parsers\TestDataParser; +use Magento\FunctionalTestingFramework\XmlParser\PageParser; +use Magento\FunctionalTestingFramework\XmlParser\SectionParser; + +class ObjectHandlerUtil +{ + /** + * Set up everything required to mock OperationDefinitionObjectHandler::getInstance() with $data value + * @param array $data + * @throws \Exception + */ + public static function mockOperationHandlerWithData($data) + { + // Clear OperationDefinitionObjectHandler singleton if already set + $property = new \ReflectionProperty( + OperationDefinitionObjectHandler::class, + 'INSTANCE' + ); + $property->setAccessible(true); + $property->setValue(null); + + $mockOperationParser = AspectMock::double( + OperationDefinitionParser::class, + ["readOperationMetadata" => $data] + )->make(); + $instance = AspectMock::double(ObjectManager::class, ['create' => $mockOperationParser])->make(); + AspectMock::double(ObjectManagerFactory::class, ['getObjectManager' => $instance]); + } + + /** + * Set up everything required to mock DataObjectHandler::getInstance() with $data value + * + * @param array $data + */ + public static function mockDataObjectHandlerWithData($data) + { + // Clear DataObjectHandler singleton if already set + $property = new \ReflectionProperty(DataObjectHandler::class, "INSTANCE"); + $property->setAccessible(true); + $property->setValue(null); + + $mockDataProfileSchemaParser = AspectMock::double(DataProfileSchemaParser::class, [ + 'readDataProfiles' => $data + ])->make(); + + $mockObjectManager = AspectMock::double(ObjectManager::class, [ + 'create' => $mockDataProfileSchemaParser + ])->make(); + + AspectMock::double(ObjectManagerFactory::class, [ + 'getObjectManager' => $mockObjectManager + ]); + } + + /** + * Set up everything required to mock PageObjectHandler::getInstance() with $data value + * + * @param array $data + */ + public static function mockPageObjectHandlerWithData($data) + { + // clear section object handler value to inject parsed content + $property = new \ReflectionProperty(PageObjectHandler::class, 'INSTANCE'); + $property->setAccessible(true); + $property->setValue(null); + + $mockSectionParser = AspectMock::double(PageParser::class, ["getData" => $data])->make(); + $instance = AspectMock::double(ObjectManager::class, ['get' => $mockSectionParser])->make(); + AspectMock::double(ObjectManagerFactory::class, ['getObjectManager' => $instance]); + } + + /** + * Set up everything required to mock SectionObjectHandler::getInstance() with $data value + * + * @param array $data + */ + public static function mockSectionObjectHandlerWithData($data) + { + // clear section object handler value to inject parsed content + $property = new \ReflectionProperty(SectionObjectHandler::class, "INSTANCE"); + $property->setAccessible(true); + $property->setValue(null); + + $mockSectionParser = AspectMock::double(SectionParser::class, ["getData" => $data])->make(); + $instance = AspectMock::double(ObjectManager::class, ["get" => $mockSectionParser])->make(); + AspectMock::double(ObjectManagerFactory::class, ["getObjectManager" => $instance]); + } + + /** + * Set up everything required to mock TestObjectHandler::getInstance() with $data value + * + * @param array $data + * @throws \Exception + */ + public static function mockTestObjectHandlerWitData($data) + { + // clear test object handler value to inject parsed content + $property = new \ReflectionProperty(TestObjectHandler::class, 'testObjectHandler'); + $property->setAccessible(true); + $property->setValue(null); + + $mockDataParser = AspectMock::double(TestDataParser::class, ['readTestData' => $data])->make(); + $instance = AspectMock::double(ObjectManager::class, ['create' => $mockDataParser]) + ->make(); // bypass the private constructor + AspectMock::double(ObjectManagerFactory::class, ['getObjectManager' => $instance]); + } + + /** + * Set up everything required to mock ActionGroupObjectHandler::getInstance() with $data value + * + * @param array $data + * @throws \Exception + */ + public static function mockActionGroupObjectHandlerWithData($data) + { + // Clear action group object handler value to inject parsed content + $property = new \ReflectionProperty(ActionGroupObjectHandler::class, 'instance'); + $property->setAccessible(true); + $property->setValue(null); + + $mockDataParser = AspectMock::double(ActionGroupDataParser::class, ['readActionGroupData' => $data])->make(); + $instance = AspectMock::double(ObjectManager::class, ['create' => $mockDataParser]) + ->make(); // bypass the private constructor + AspectMock::double(ObjectManagerFactory::class, ['getObjectManager' => $instance]); + } +} diff --git a/dev/tests/verification/DeprecationCheckModule/ActionGroup/DeprecationCheckActionGroup.xml b/dev/tests/verification/DeprecationCheckModule/ActionGroup/DeprecationCheckActionGroup.xml new file mode 100644 index 000000000..c83dab211 --- /dev/null +++ b/dev/tests/verification/DeprecationCheckModule/ActionGroup/DeprecationCheckActionGroup.xml @@ -0,0 +1,13 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<!-- + /** + * Copyright © Magento, Inc. All rights reserved. + * See COPYING.txt for license details. + */ +--> +<actionGroups xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:noNamespaceSchemaLocation="urn:magento:mftf:Test/etc/actionGroupSchema.xsd"> + <actionGroup name="DeprecationCheckActionGroup"> + <see stepKey="deprecatedSee" userInput="text" selector="{{DeprecationCheckSection.deprecationCheckElement}}" /> + </actionGroup> +</actionGroups> diff --git a/dev/tests/verification/DeprecationCheckModule/ActionGroup/DeprecationCheckDeprecatedActionGroup.xml b/dev/tests/verification/DeprecationCheckModule/ActionGroup/DeprecationCheckDeprecatedActionGroup.xml new file mode 100644 index 000000000..e8412a29e --- /dev/null +++ b/dev/tests/verification/DeprecationCheckModule/ActionGroup/DeprecationCheckDeprecatedActionGroup.xml @@ -0,0 +1,13 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<!-- + /** + * Copyright © Magento, Inc. All rights reserved. + * See COPYING.txt for license details. + */ +--> +<actionGroups xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:noNamespaceSchemaLocation="urn:magento:mftf:Test/etc/actionGroupSchema.xsd"> + <actionGroup name="DeprecationCheckDeprecatedActionGroup" deprecated="deprecated"> + <see stepKey="deprecatedSee" userInput="text" selector="{{DeprecationCheckSection.deprecationCheckElement}}" /> + </actionGroup> +</actionGroups> diff --git a/dev/tests/verification/DeprecationCheckModule/Data/DeprecationCheckData.xml b/dev/tests/verification/DeprecationCheckModule/Data/DeprecationCheckData.xml new file mode 100644 index 000000000..75a6b6678 --- /dev/null +++ b/dev/tests/verification/DeprecationCheckModule/Data/DeprecationCheckData.xml @@ -0,0 +1,14 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + /** + * Copyright © Magento, Inc. All rights reserved. + * See COPYING.txt for license details. + */ +--> + +<entities xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:noNamespaceSchemaLocation="urn:magento:mftf:DataGenerator/etc/dataProfileSchema.xsd"> + <entity name="DeprecationCheckData" type="type1" deprecated="deprecated"> + <data key="field">value</data> + </entity> +</entities> diff --git a/dev/tests/verification/DeprecationCheckModule/Metadata/DeprecationCheckMeta.xml b/dev/tests/verification/DeprecationCheckModule/Metadata/DeprecationCheckMeta.xml new file mode 100644 index 000000000..757c6284b --- /dev/null +++ b/dev/tests/verification/DeprecationCheckModule/Metadata/DeprecationCheckMeta.xml @@ -0,0 +1,18 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + /** + * Copyright © Magento, Inc. All rights reserved. + * See COPYING.txt for license details. + */ +--> + +<operations xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:noNamespaceSchemaLocation="urn:magento:mftf:DataGenerator/etc/dataOperation.xsd"> + <operation name="DeprecationCheckMeta" dataType="type1" type="create" auth="adminFormKey" url="/V1/test" method="POST" deprecated="deprecated"> + <contentType>application/json</contentType> + <object key="category" dataType="type1"> + <field key="key1">value1</field> + <field key="key2">value2</field> + </object> + </operation> +</operations> \ No newline at end of file diff --git a/dev/tests/verification/DeprecationCheckModule/Page/DeprecationCheckPage.xml b/dev/tests/verification/DeprecationCheckModule/Page/DeprecationCheckPage.xml new file mode 100644 index 000000000..dd3f187dc --- /dev/null +++ b/dev/tests/verification/DeprecationCheckModule/Page/DeprecationCheckPage.xml @@ -0,0 +1,14 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + /** + * Copyright © Magento, Inc. All rights reserved. + * See COPYING.txt for license details. + */ +--> + +<pages xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:noNamespaceSchemaLocation="urn:magento:mftf:Page/etc/PageObject.xsd"> + <page name="DeprecationCheckPage" url="/test.html" area="storefront" module="UnknownVendor_DeprecationCheckModule" deprecated="Deprecated page"> + <section name="DeprecationCheckSection"/> + </page> +</pages> diff --git a/dev/tests/verification/DeprecationCheckModule/Section/DeprecationCheckSection.xml b/dev/tests/verification/DeprecationCheckModule/Section/DeprecationCheckSection.xml new file mode 100644 index 000000000..1837e1ef5 --- /dev/null +++ b/dev/tests/verification/DeprecationCheckModule/Section/DeprecationCheckSection.xml @@ -0,0 +1,16 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + /** + * Copyright © Magento, Inc. All rights reserved. + * See COPYING.txt for license details. + */ +--> + +<sections xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:noNamespaceSchemaLocation="urn:magento:mftf:Page/etc/SectionObject.xsd"> + <section name="DeprecationCheckSection" deprecated="deprecated"> + <element name="deprecationCheckElement" type="button" selector="#elementOne" deprecated="deprecated"/> + <element name="elementTwo" type="button" selector="#elementTwo"/> + <element name="elementThree" type="button" selector="#elementThree"/> + </section> +</sections> diff --git a/dev/tests/verification/DeprecationCheckModule/Suite/deprecationCheckSuite.xml b/dev/tests/verification/DeprecationCheckModule/Suite/deprecationCheckSuite.xml new file mode 100644 index 000000000..b89ad99f6 --- /dev/null +++ b/dev/tests/verification/DeprecationCheckModule/Suite/deprecationCheckSuite.xml @@ -0,0 +1,16 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + /** + * Copyright © Magento, Inc. All rights reserved. + * See COPYING.txt for license details. + */ +--> + +<suites xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="urn:magento:mftf:Suite/etc/suiteSchema.xsd"> + <suite name="deprecationCheckSuite"> + <include> + <test name="DeprecationCheckDeprecatedTest"/> + <test name="DeprecationCheckTest"/> + </include> + </suite> +</suites> diff --git a/dev/tests/verification/DeprecationCheckModule/Test/DeprecationCheckDeprecatedTest.xml b/dev/tests/verification/DeprecationCheckModule/Test/DeprecationCheckDeprecatedTest.xml new file mode 100644 index 000000000..1b3ace053 --- /dev/null +++ b/dev/tests/verification/DeprecationCheckModule/Test/DeprecationCheckDeprecatedTest.xml @@ -0,0 +1,17 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + /** + * Copyright © Magento, Inc. All rights reserved. + * See COPYING.txt for license details. + */ +--> + +<tests xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:noNamespaceSchemaLocation="urn:magento:mftf:Test/etc/testSchema.xsd"> + <test name="DeprecationCheckDeprecatedTest" deprecated="deprecated"> + <createData entity="DeprecationCheckData" stepKey="deprecatedCreateData"/> + <actionGroup ref="DeprecationCheckActionGroup" stepKey="deprecationCheckActionGroup" /> + <amOnPage url="{{DeprecationCheckPage.url}}" stepKey="deprecatedAmOnPage" /> + <actionGroup ref="DeprecationCheckDeprecatedActionGroup" stepKey="deprecationCheckDeprecatedActionGroup" /> + </test> +</tests> diff --git a/dev/tests/verification/DeprecationCheckModule/Test/DeprecationCheckTest.xml b/dev/tests/verification/DeprecationCheckModule/Test/DeprecationCheckTest.xml new file mode 100644 index 000000000..1c4fc3dba --- /dev/null +++ b/dev/tests/verification/DeprecationCheckModule/Test/DeprecationCheckTest.xml @@ -0,0 +1,17 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + /** + * Copyright © Magento, Inc. All rights reserved. + * See COPYING.txt for license details. + */ +--> + +<tests xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:noNamespaceSchemaLocation="urn:magento:mftf:Test/etc/testSchema.xsd"> + <test name="DeprecationCheckTest"> + <createData entity="DeprecationCheckData" stepKey="deprecatedCreateData"/> + <actionGroup ref="DeprecationCheckActionGroup" stepKey="deprecationCheckActionGroup" /> + <amOnPage url="{{DeprecationCheckPage.url}}" stepKey="deprecatedAmOnPage" /> + <actionGroup ref="DeprecationCheckDeprecatedActionGroup" stepKey="deprecationCheckDeprecatedActionGroup" /> + </test> +</tests> diff --git a/dev/tests/verification/Resources/StaticChecks/mftf-deprecated-entity-usage-checks.txt b/dev/tests/verification/Resources/StaticChecks/mftf-deprecated-entity-usage-checks.txt new file mode 100644 index 000000000..63e025470 --- /dev/null +++ b/dev/tests/verification/Resources/StaticChecks/mftf-deprecated-entity-usage-checks.txt @@ -0,0 +1,23 @@ + +File "/verification/DeprecationCheckModule/Test/DeprecationCheckTest.xml" contains: + - Deprecated Page(s): + "DeprecationCheckPage" in /verification/DeprecationCheckModule/Page/DeprecationCheckPage.xml + - Deprecated ActionGroup(s): + "DeprecationCheckDeprecatedActionGroup" in /verification/DeprecationCheckModule/ActionGroup/DeprecationCheckDeprecatedActionGroup.xml + - Deprecated Data(s): + "DeprecationCheckData" in /verification/DeprecationCheckModule/Data/DeprecationCheckData.xml + - "DeprecationCheckData" references deprecated: + "type1" in metadata xml file + + +File "/verification/DeprecationCheckModule/ActionGroup/DeprecationCheckActionGroup.xml" contains: + - Deprecated Section(s): + "DeprecationCheckSection" in /verification/DeprecationCheckModule/Section/DeprecationCheckSection.xml + - Deprecated Element(s): + "DeprecationCheckSection.deprecationCheckElement" in /verification/DeprecationCheckModule/Section/DeprecationCheckSection.xml + + +File "/verification/DeprecationCheckModule/Suite/deprecationCheckSuite.xml" contains: + - Deprecated Test(s): + "DeprecationCheckDeprecatedTest" in /verification/DeprecationCheckModule/Test/DeprecationCheckDeprecatedTest.xml + diff --git a/dev/tests/verification/Tests/StaticCheck/DeprecationStaticCheckTest.php b/dev/tests/verification/Tests/StaticCheck/DeprecationStaticCheckTest.php new file mode 100644 index 000000000..bafe586ac --- /dev/null +++ b/dev/tests/verification/Tests/StaticCheck/DeprecationStaticCheckTest.php @@ -0,0 +1,92 @@ +<?php +/** + * Copyright © Magento, Inc. All rights reserved. + * See COPYING.txt for license details. + */ +namespace tests\verification\Tests; + +use AspectMock\Test as AspectMock; +use Magento\FunctionalTestingFramework\StaticCheck\DeprecatedEntityUsageCheck; +use Magento\FunctionalTestingFramework\StaticCheck\StaticChecksList; +use Magento\FunctionalTestingFramework\Util\Filesystem\DirSetupUtil; +use PHPUnit\Framework\TestCase; +use Symfony\Component\Console\Input\InputInterface; + +class DeprecationStaticCheckTest extends TestCase +{ + const STATIC_RESULTS_DIR = TESTS_MODULE_PATH . + DIRECTORY_SEPARATOR . + '_output' . + DIRECTORY_SEPARATOR . + 'static-results'; + + const LOG_FILE = self::STATIC_RESULTS_DIR . + DIRECTORY_SEPARATOR . + DeprecatedEntityUsageCheck::ERROR_LOG_FILENAME . + '.txt'; + + const TEST_MODULE_PATH = TESTS_MODULE_PATH . + DIRECTORY_SEPARATOR . + 'DeprecationCheckModule'. + DIRECTORY_SEPARATOR; + + const RESOURCES_PATH = TESTS_MODULE_PATH . + DIRECTORY_SEPARATOR . + "Resources" . + DIRECTORY_SEPARATOR . + 'StaticChecks'; + + public static function setUpBeforeClass(): void + { + // remove static-results if it exists + if (file_exists(self::STATIC_RESULTS_DIR)) { + DirSetupUtil::rmdirRecursive(self::STATIC_RESULTS_DIR); + } + } + + /** + * test static-check DeprecatedEntityUsageCheck. + * + * @throws \Exception + */ + public function testDeprecatedEntityUsageCheck() + { + $staticCheck = new DeprecatedEntityUsageCheck(); + + $input = $this->mockInputInterface(self::TEST_MODULE_PATH); + AspectMock::double(StaticChecksList::class, ['getErrorFilesPath' => self::STATIC_RESULTS_DIR]); + + /** @var InputInterface $input */ + $staticCheck->execute($input); + + $this->assertTrue(file_exists(self::LOG_FILE)); + $this->assertFileEquals( + self::RESOURCES_PATH. + DIRECTORY_SEPARATOR . + DeprecatedEntityUsageCheck::ERROR_LOG_FILENAME . + ".txt", + self::LOG_FILE + ); + } + + /** + * Sets input interface + * @param $path + * @return \PHPUnit\Framework\MockObject\MockObject + */ + public function mockInputInterface($path) + { + $input = $this->getMockBuilder(InputInterface::class) + ->disableOriginalConstructor() + ->getMock(); + $input->method('getOption') + ->with('path') + ->willReturn($path); + return $input; + } + + public function tearDown(): void + { + DirSetupUtil::rmdirRecursive(self::STATIC_RESULTS_DIR); + } +} diff --git a/src/Magento/FunctionalTestingFramework/StaticCheck/DeprecatedEntityUsageCheck.php b/src/Magento/FunctionalTestingFramework/StaticCheck/DeprecatedEntityUsageCheck.php index 5da16ec6a..547e6a633 100644 --- a/src/Magento/FunctionalTestingFramework/StaticCheck/DeprecatedEntityUsageCheck.php +++ b/src/Magento/FunctionalTestingFramework/StaticCheck/DeprecatedEntityUsageCheck.php @@ -37,6 +37,7 @@ class DeprecatedEntityUsageCheck implements StaticCheckInterface { const EXTENDS_REGEX_PATTERN = '/extends=["\']([^\'"]*)/'; const ACTIONGROUP_REGEX_PATTERN = '/ref=["\']([^\'"]*)/'; + const DEPRECATED_REGEX_PATTERN = '/deprecated=["\']([^\'"]*)/'; const ERROR_LOG_FILENAME = 'mftf-deprecated-entity-usage-checks'; const ERROR_LOG_MESSAGE = 'MFTF Deprecated Entity Usage Check'; @@ -177,7 +178,6 @@ private function loadAllXmlFiles($input) MftfApplicationConfig::LEVEL_DEFAULT, true ); - putenv('CUSTOM_MODULE_PATHS=' . realpath($path)); $modulePaths[] = realpath($path); $includeRootPath = false; } else { @@ -225,6 +225,9 @@ private function findReferenceErrorsInActionFiles($files, $checkTestRef = false) /** @var SplFileInfo $filePath */ foreach ($files as $filePath) { $contents = file_get_contents($filePath); + if ($this->isDeprecated($contents)) { + continue; + } preg_match_all(ActionObject::ACTION_ATTRIBUTE_VARIABLE_REGEX_PATTERN, $contents, $braceReferences); preg_match_all(self::ACTIONGROUP_REGEX_PATTERN, $contents, $actionGroupReferences); preg_match_all(self::EXTENDS_REGEX_PATTERN, $contents, $extendReferences); @@ -316,6 +319,17 @@ private function findReferenceErrorsInActionFiles($files, $checkTestRef = false) return $testErrors; } + /** + * Checks if entity is deprecated in action files. + * @param string $contents + * @return boolean + */ + private function isDeprecated($contents) + { + preg_match_all(self::DEPRECATED_REGEX_PATTERN, $contents, $deprecatedEntity); + return (!empty($deprecatedEntity[1])); + } + /** * Find reference errors in a set of data files * @@ -335,6 +349,11 @@ private function findReferenceErrorsInDataFiles($files) $entities = $domDocument->getElementsByTagName('entity'); foreach ($entities as $entity) { /** @var DOMElement $entity */ + $deprecated = $entity->getAttribute('deprecated'); + // skip check if entity is deprecated + if (!empty($deprecated)) { + continue; + } $entityName = $entity->getAttribute('name'); $metadataType = $entity->getAttribute('type'); $parentEntityName = $entity->getAttribute('extends'); @@ -609,9 +628,9 @@ private function findViolatingReferences($references) $name = $key; list($section,) = explode('.', $key, 2); /** @var SectionObject $references[$section] */ - $file = $references[$section]->getFilename(); + $file = StaticChecksList::getFilePath($references[$section]->getFilename()); } else { - $file = $entity->getFilename(); + $file = StaticChecksList::getFilePath($entity->getFilename()); } $violatingReferences[$this->getSubjectFromClassType($classType)][] = [ 'name' => $name, @@ -633,16 +652,18 @@ private function setErrorOutput($violatingReferences, $path) { $testErrors = []; + $filePath = StaticChecksList::getFilePath($path->getRealPath()); + if (!empty($violatingReferences)) { // Build error output - $errorOutput = "\nFile \"{$path->getRealPath()}\" contains:\n"; + $errorOutput = "\nFile \"{$filePath}\" contains:\n"; foreach ($violatingReferences as $subject => $data) { $errorOutput .= "\t- {$subject}:\n"; foreach ($data as $item) { $errorOutput .= "\t\t\"" . $item['name'] . "\" in " . $item['file'] . "\n"; } } - $testErrors[$path->getRealPath()][] = $errorOutput; + $testErrors[$filePath][] = $errorOutput; } return $testErrors; diff --git a/src/Magento/FunctionalTestingFramework/StaticCheck/StaticChecksList.php b/src/Magento/FunctionalTestingFramework/StaticCheck/StaticChecksList.php index 7cd894e00..ba2139276 100644 --- a/src/Magento/FunctionalTestingFramework/StaticCheck/StaticChecksList.php +++ b/src/Magento/FunctionalTestingFramework/StaticCheck/StaticChecksList.php @@ -69,4 +69,22 @@ public static function getErrorFilesPath() { return self::$errorFilesPath; } + + /** + * Return relative path to files for unit testing purposes. + * @param string $fileNames + * @return string + */ + public static function getFilePath($fileNames) + { + if (!empty($fileNames)) { + $relativeFileNames = ltrim( + str_replace(MAGENTO_BP, '', $fileNames) + ); + if (!empty($relativeFileNames)) { + return $relativeFileNames; + } + } + return $fileNames; + } } From 85d4fa6a8f29c8db4e36b7f97cb136a403154154 Mon Sep 17 00:00:00 2001 From: soumyau <sunnikri@adobe.com> Date: Tue, 11 Aug 2020 14:51:30 -0500 Subject: [PATCH 33/52] MQE-2212: Fix invalid behaviour MAGENTO_BACKEND_BASE_URL (#780) * MQE-2212: Fix invalid behaviour of MAGENTO_BACKEND_BASE_URL * MQE-2212: Fix invalid behaviour MAGENTO_BACKEND_BASE_URL * MQE-2212: Fix invalid behaviour MAGENTO_BACKEND_BASE_URL --- .../Resources/PageReplacementTest.txt | 4 +-- .../Module/MagentoWebDriver.php | 2 +- .../Test/Objects/ActionObject.php | 7 +++- .../Util/MftfGlobals.php | 9 ++++-- .../Util/TestGenerator.php | 32 +++++++++++++++---- 5 files changed, 40 insertions(+), 14 deletions(-) diff --git a/dev/tests/verification/Resources/PageReplacementTest.txt b/dev/tests/verification/Resources/PageReplacementTest.txt index cda119999..6f75ee096 100644 --- a/dev/tests/verification/Resources/PageReplacementTest.txt +++ b/dev/tests/verification/Resources/PageReplacementTest.txt @@ -35,8 +35,8 @@ class PageReplacementTestCest $I->amOnPage("/John/StringLiteral2.html"); // stepKey: twoParamPageStringData $I->amOnPage("/John/" . $I->retrieveEntityField('datakey', 'firstname', 'test') . ".html"); // stepKey: twoParamPageDataPersist $I->amOnPage("/" . $I->retrieveEntityField('datakey', 'firstname', 'test') . "/StringLiteral2.html"); // stepKey: twoParamPagePersistString - $I->amOnPage("/" . getenv("MAGENTO_BACKEND_NAME") . "/backend"); // stepKey: onAdminPage - $I->amOnPage("/" . getenv("MAGENTO_BACKEND_NAME") . "/StringLiteral/page.html"); // stepKey: oneParamAdminPageString + $I->amOnPage((getenv("MAGENTO_BACKEND_BASE_URL") ? rtrim(getenv("MAGENTO_BACKEND_BASE_URL"), "/") : "") . "/" . getenv("MAGENTO_BACKEND_NAME") . "/backend"); // stepKey: onAdminPage + $I->amOnPage((getenv("MAGENTO_BACKEND_BASE_URL") ? rtrim(getenv("MAGENTO_BACKEND_BASE_URL"), "/") : "") . "/" . getenv("MAGENTO_BACKEND_NAME") . "/StringLiteral/page.html"); // stepKey: oneParamAdminPageString $I->amOnUrl("http://myFullUrl.com/"); // stepKey: onExternalPage } } diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php index ef57aa3f1..8886f4493 100644 --- a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php +++ b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php @@ -877,7 +877,7 @@ public function saveScreenshot() */ public function amOnPage($page) { - parent::amOnPage($page); + (0 === strpos($page, 'http')) ? parent::amOnUrl($page) : parent::amOnPage($page); $this->waitForPageLoad(); } diff --git a/src/Magento/FunctionalTestingFramework/Test/Objects/ActionObject.php b/src/Magento/FunctionalTestingFramework/Test/Objects/ActionObject.php index 648471c7a..8d4e7a945 100644 --- a/src/Magento/FunctionalTestingFramework/Test/Objects/ActionObject.php +++ b/src/Magento/FunctionalTestingFramework/Test/Objects/ActionObject.php @@ -726,7 +726,12 @@ private function resolveParameterization($isParameterized, $replacement, $match, $resolvedReplacement = $replacement; } if (get_class($object) == PageObject::class && $object->getArea() == PageObject::ADMIN_AREA) { - $resolvedReplacement = "/{{_ENV.MAGENTO_BACKEND_NAME}}/" . $resolvedReplacement; + $urlSegments = [ + '{{_ENV.MAGENTO_BACKEND_BASE_URL}}', + '{{_ENV.MAGENTO_BACKEND_NAME}}', + $resolvedReplacement + ]; + $resolvedReplacement = implode('/', $urlSegments); } return $resolvedReplacement; } diff --git a/src/Magento/FunctionalTestingFramework/Util/MftfGlobals.php b/src/Magento/FunctionalTestingFramework/Util/MftfGlobals.php index d59df5c11..cb0f4f5ae 100644 --- a/src/Magento/FunctionalTestingFramework/Util/MftfGlobals.php +++ b/src/Magento/FunctionalTestingFramework/Util/MftfGlobals.php @@ -76,12 +76,15 @@ public static function getBackendBaseUrl($withTrailingSeparator = true) { if (!self::$backendBaseUrl) { try { + $backendName = getenv('MAGENTO_BACKEND_NAME'); $bUrl = getenv('MAGENTO_BACKEND_BASE_URL'); - if ($bUrl) { - self::$backendBaseUrl = UrlFormatter::format($bUrl, false); + if ($bUrl && $backendName) { + self::$backendBaseUrl = UrlFormatter::format( + UrlFormatter::format($bUrl) . $backendName, + false + ); } else { $baseUrl = getenv('MAGENTO_BASE_URL'); - $backendName = getenv('MAGENTO_BACKEND_NAME'); if ($baseUrl && $backendName) { self::$backendBaseUrl = UrlFormatter::format( UrlFormatter::format($baseUrl) . $backendName, diff --git a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php index 80056b3b9..82fb2c06a 100644 --- a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php +++ b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php @@ -2030,18 +2030,20 @@ private function resolveRuntimeReference($args, $regex, $func) $newArgs = []; foreach ($args as $key => $arg) { + $newArgs[$key] = $arg; preg_match_all($regex, $arg, $matches); if (!empty($matches[0])) { - $fullMatch = $matches[0][0]; - $refVariable = $matches[1][0]; - unset($matches); - $replacement = "{$func}(\"{$refVariable}\")"; + foreach ($matches[0] as $matchKey => $fullMatch) { + $refVariable = $matches[1][$matchKey]; - $outputArg = $this->processQuoteBreaks($fullMatch, $arg, $replacement); - $newArgs[$key] = $outputArg; + $replacement = $this->getReplacement($func, $refVariable); + + $outputArg = $this->processQuoteBreaks($fullMatch, $newArgs[$key], $replacement); + $newArgs[$key] = $outputArg; + } + unset($matches); continue; } - $newArgs[$key] = $arg; } // override passed in args for use later. @@ -2313,4 +2315,20 @@ private function parseUserInput($userInput) return $this->addUniquenessFunctionCall($userInput); } + + /** + * Supports fallback for BACKEND URL + * + * @param string $func + * @param string $refVariable + * @return string + */ + private function getReplacement($func, $refVariable): string + { + if ($refVariable === 'MAGENTO_BACKEND_BASE_URL') { + return "({$func}(\"{$refVariable}\") ? rtrim({$func}(\"{$refVariable}\"), \"/\") : \"\")"; + } + + return "{$func}(\"{$refVariable}\")"; + } } From 8e045808e778b2cf1028aed6a419e9830402396e Mon Sep 17 00:00:00 2001 From: Ji Lu <jilu1@adobe.com> Date: Thu, 13 Aug 2020 09:34:27 -0500 Subject: [PATCH 34/52] MQE-2110: MFTF command to pause test execution --- .../Console/BaseGenerateCommand.php | 23 +++++++ .../Console/Codecept/CodeceptCommandUtil.php | 66 +++++++++++++++++++ .../Console/CodeceptRunCommand.php | 61 +++++++++++++++++ .../Console/CommandList.php | 1 + .../Console/RunManifestCommand.php | 33 ++++++---- .../Console/RunTestCommand.php | 34 +++++++--- .../Console/RunTestFailedCommand.php | 30 ++++----- .../Console/RunTestGroupCommand.php | 27 ++++---- 8 files changed, 223 insertions(+), 52 deletions(-) create mode 100644 src/Magento/FunctionalTestingFramework/Console/Codecept/CodeceptCommandUtil.php create mode 100644 src/Magento/FunctionalTestingFramework/Console/CodeceptRunCommand.php diff --git a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php index 29e489a63..11ffffdae 100644 --- a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php @@ -14,6 +14,7 @@ use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Input\InputOption; +use Symfony\Component\Console\Input\StringInput; use Symfony\Component\Console\Output\OutputInterface; use Magento\FunctionalTestingFramework\Util\Filesystem\DirSetupUtil; use Magento\FunctionalTestingFramework\Util\TestGenerator; @@ -21,9 +22,16 @@ use Magento\FunctionalTestingFramework\Suite\Handlers\SuiteObjectHandler; use Symfony\Component\Console\Style\SymfonyStyle; +/** + * Class BaseGenerateCommand + * @package Magento\FunctionalTestingFramework\Console + * + * @SuppressWarnings(PHPMD.CouplingBetweenObjects) + */ class BaseGenerateCommand extends Command { const MFTF_NOTICES = "Placeholder text for MFTF notices\n"; + const CODECEPT_RUN_COMMAND = 'codecept:run functional '; /** * Enable pause() @@ -242,4 +250,19 @@ protected function pauseEnabled() } return $this->enablePause; } + + /** + * Runs the bin/mftf codecept:run command and returns exit code + * + * @param string $command + * @param OutputInterface $output + * @return integer + * @throws \Exception + */ + protected function codeceptRunTest(string $command, OutputInterface $output) + { + $input = new StringInput($command); + $command = $this->getApplication()->find('codecept:run'); + return $command->run($input, $output); + } } diff --git a/src/Magento/FunctionalTestingFramework/Console/Codecept/CodeceptCommandUtil.php b/src/Magento/FunctionalTestingFramework/Console/Codecept/CodeceptCommandUtil.php new file mode 100644 index 000000000..3992aa677 --- /dev/null +++ b/src/Magento/FunctionalTestingFramework/Console/Codecept/CodeceptCommandUtil.php @@ -0,0 +1,66 @@ +<?php +/** + * Copyright © Magento, Inc. All rights reserved. + * See COPYING.txt for license details. + */ +declare(strict_types = 1); + +namespace Magento\FunctionalTestingFramework\Console\Codecept; + +use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; +use Magento\FunctionalTestingFramework\Util\Path\FilePathFormatter; +use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Input\ArgvInput; + +class CodeceptCommandUtil +{ + const CODECEPTION_AUTOLOAD_FILE = PROJECT_ROOT . '/vendor/codeception/codeception/autoload.php'; + + /** + * Current working directory + * + * @var string + */ + private $cwd = null; + + /** + * Setup Codeception + * + * @param InputInterface $input + * @return void + */ + public function setup(InputInterface $input) + { + require_once realpath(self::CODECEPTION_AUTOLOAD_FILE); + + $tokens = preg_split('{\\s+}', $input->__toString()); + $tokens[0] = str_replace('codecept:', '', $tokens[0]); + \Closure::bind(function &(ArgvInput $input) use ($tokens) { + return $input->setTokens($tokens); + }, null, ArgvInput::class); + } + + /** + * Save Codeception working directory + * + * @return void + * @throws TestFrameworkException + */ + public function setCodeceptCwd() + { + $this->cwd = getcwd(); + chdir(FilePathFormatter::format(TESTS_BP, false)); + } + + /** + * Restore current working directory + * + * @return void + */ + public function restoreCwd() + { + if ($this->cwd) { + chdir($this->cwd); + } + } +} diff --git a/src/Magento/FunctionalTestingFramework/Console/CodeceptRunCommand.php b/src/Magento/FunctionalTestingFramework/Console/CodeceptRunCommand.php new file mode 100644 index 000000000..311b81e12 --- /dev/null +++ b/src/Magento/FunctionalTestingFramework/Console/CodeceptRunCommand.php @@ -0,0 +1,61 @@ +<?php +/** + * Copyright © Magento, Inc. All rights reserved. + * See COPYING.txt for license details. + */ +declare(strict_types = 1); + +namespace Magento\FunctionalTestingFramework\Console; + +use Codeception\Command\Run; +use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; +use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Output\OutputInterface; +use Magento\FunctionalTestingFramework\Console\Codecept\CodeceptCommandUtil; + +class CodeceptRunCommand extends Run +{ + /** + * Configures the current command + * + * @return void + */ + protected function configure() + { + $this->setName('codecept:run') + ->setDescription( + "Wrapper command to codecept:run. See https://codeception.com/docs/reference/Commands" + ); + + parent::configure(); + } + + /** + * Executes the current command + * + * @param InputInterface $input + * @param OutputInterface $output + * @return integer + * @throws \Exception + */ + public function execute(InputInterface $input, OutputInterface $output): int + { + $commandUtil = new CodeceptCommandUtil(); + $commandUtil->setup($input); + $commandUtil->setCodeceptCwd(); + + try { + $exitCode = parent::execute($input, $output); + } catch (\Exception $e) { + throw new TestFrameworkException( + 'Make sure cest files are generated before running bin/mftf ' + . $this->getName() + . PHP_EOL + . $e->getMessage() + ); + } + + $commandUtil->restoreCwd(); + return $exitCode; + } +} diff --git a/src/Magento/FunctionalTestingFramework/Console/CommandList.php b/src/Magento/FunctionalTestingFramework/Console/CommandList.php index f77c2c576..840e7dd86 100644 --- a/src/Magento/FunctionalTestingFramework/Console/CommandList.php +++ b/src/Magento/FunctionalTestingFramework/Console/CommandList.php @@ -30,6 +30,7 @@ public function __construct(array $commands = []) { $this->commands = [ 'build:project' => new BuildProjectCommand(), + 'codecept:run' => new CodeceptRunCommand(), 'doctor' => new DoctorCommand(), 'generate:suite' => new GenerateSuiteCommand(), 'generate:tests' => new GenerateTestsCommand(), diff --git a/src/Magento/FunctionalTestingFramework/Console/RunManifestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunManifestCommand.php index 1709726bb..76b92c960 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunManifestCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunManifestCommand.php @@ -11,6 +11,7 @@ use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputArgument; use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Input\StringInput; use Symfony\Component\Console\Output\OutputInterface; use Symfony\Component\Process\Process; use Magento\FunctionalTestingFramework\Util\Path\FilePathFormatter; @@ -103,28 +104,32 @@ protected function execute(InputInterface $input, OutputInterface $output): int * @param string $manifestLine * @param OutputInterface $output * @return void + * @throws \Exception * * @SuppressWarnings(PHPMD.UnusedLocalVariable) Need this because of the unused $type variable in the closure */ private function runManifestLine(string $manifestLine, OutputInterface $output) { - $codeceptionCommand = realpath(PROJECT_ROOT . "/vendor/bin/codecept") . " run functional --verbose --steps "; if (getenv('ENABLE_PAUSE') === 'true') { - $codeceptionCommand .= ' --debug'; + $codeceptionCommand = BaseGenerateCommand::CODECEPT_RUN_COMMAND + . '--verbose --steps --debug' . $manifestLine; + $input = new StringInput($codeceptionCommand); + $command = $this->getApplication()->find('codecept:run'); + $subReturnCode = $command->run($input, $output); + } else { + $codeceptionCommand = realpath(PROJECT_ROOT . "/vendor/bin/codecept") + . " run functional --verbose --steps " . $manifestLine; + + // run the codecept command in a sub process + $process = new Process($codeceptionCommand); + $process->setWorkingDirectory(TESTS_BP); + $process->setIdleTimeout(600); + $process->setTimeout(0); + $subReturnCode = $process->run(function ($type, $buffer) use ($output) { + $output->write($buffer); + }); } - $codeceptionCommand .= $manifestLine; - // run the codecept command in a sub process - $process = new Process($codeceptionCommand); - $process->setWorkingDirectory(TESTS_BP); - $process->setIdleTimeout(600); - $process->setTimeout(0); - if (getenv('ENABLE_PAUSE') === 'true') { - $process->setInput(STDIN); - } - $subReturnCode = $process->run(function ($type, $buffer) use ($output) { - $output->write($buffer); - }); $this->returnCode = max($this->returnCode, $subReturnCode); } diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php index 86352d5c6..502ef7c78 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php @@ -119,10 +119,16 @@ protected function execute(InputInterface $input, OutputInterface $output): int * @param OutputInterface $output * @return void * @throws TestFrameworkException + * @throws \Exception */ private function runTests(array $tests, OutputInterface $output) { - $codeceptionCommand = realpath(PROJECT_ROOT . '/vendor/bin/codecept') . ' run functional '; + if ($this->pauseEnabled()) { + $codeceptionCommand = self::CODECEPT_RUN_COMMAND; + } else { + $codeceptionCommand = realpath(PROJECT_ROOT . '/vendor/bin/codecept') . ' run functional '; + } + $testsDirectory = FilePathFormatter::format(TESTS_MODULE_PATH) . TestGenerator::GENERATED_DIR . DIRECTORY_SEPARATOR . @@ -136,11 +142,14 @@ private function runTests(array $tests, OutputInterface $output) $testName . " is not available under " . $testsDirectory ); } - $fullCommand = $codeceptionCommand . $testsDirectory . $testName . ' --verbose --steps'; + if ($this->pauseEnabled()) { - $fullCommand .= ' --debug'; + $fullCommand = $codeceptionCommand . $testsDirectory . $testName . ' --verbose --steps --debug'; + $this->returnCode = max($this->returnCode, $this->codeceptRunTest($fullCommand, $output)); + } else { + $fullCommand = $codeceptionCommand . $testsDirectory . $testName . ' --verbose --steps'; + $this->returnCode = max($this->returnCode, $this->executeTestCommand($fullCommand, $output)); } - $this->returnCode = max($this->returnCode, $this->executeTestCommand($fullCommand, $output)); } } @@ -150,17 +159,24 @@ private function runTests(array $tests, OutputInterface $output) * @param array $suitesConfig * @param OutputInterface $output * @return void + * @throws \Exception */ private function runTestsInSuite(array $suitesConfig, OutputInterface $output) { - $codeceptionCommand = realpath(PROJECT_ROOT . '/vendor/bin/codecept') . ' run functional --verbose --steps '; if ($this->pauseEnabled()) { - $codeceptionCommand .= ' --debug'; + $codeceptionCommand = self::CODECEPT_RUN_COMMAND . '--verbose --steps --debug'; + } else { + $codeceptionCommand = realpath(PROJECT_ROOT . '/vendor/bin/codecept') + . ' run functional --verbose --steps '; } //for tests in suites, run them as a group to run before and after block foreach (array_keys($suitesConfig) as $suite) { $fullCommand = $codeceptionCommand . " -g {$suite}"; - $this->returnCode = max($this->returnCode, $this->executeTestCommand($fullCommand, $output)); + if ($this->pauseEnabled()) { + $this->returnCode = max($this->returnCode, $this->codeceptRunTest($fullCommand, $output)); + } else { + $this->returnCode = max($this->returnCode, $this->executeTestCommand($fullCommand, $output)); + } } } @@ -179,9 +195,7 @@ private function executeTestCommand(string $command, OutputInterface $output) $process->setWorkingDirectory(TESTS_BP); $process->setIdleTimeout(600); $process->setTimeout(0); - if ($this->pauseEnabled()) { - $process->setInput(STDIN); - } + return $process->run(function ($type, $buffer) use ($output) { $output->write($buffer); }); diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php index b9012b086..fa48d0b72 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php @@ -117,24 +117,24 @@ protected function execute(InputInterface $input, OutputInterface $output): int $testManifestList = $this->readTestManifestFile(); $returnCode = 0; foreach ($testManifestList as $testCommand) { - $codeceptionCommand = realpath(PROJECT_ROOT . '/vendor/bin/codecept') . ' run functional '; - $codeceptionCommand .= $testCommand; if ($this->pauseEnabled()) { - $codeceptionCommand .= ' --debug'; + $codeceptionCommand = self::CODECEPT_RUN_COMMAND . $testCommand . ' --debug'; + $this->codeceptRunTest($codeceptionCommand, $output); + } else { + $codeceptionCommand = realpath(PROJECT_ROOT . '/vendor/bin/codecept') . ' run functional '; + $codeceptionCommand .= $testCommand; + + $process = new Process($codeceptionCommand); + $process->setWorkingDirectory(TESTS_BP); + $process->setIdleTimeout(600); + $process->setTimeout(0); + $returnCode = max($returnCode, $process->run( + function ($type, $buffer) use ($output) { + $output->write($buffer); + } + )); } - $process = new Process($codeceptionCommand); - $process->setWorkingDirectory(TESTS_BP); - $process->setIdleTimeout(600); - $process->setTimeout(0); - if ($this->pauseEnabled()) { - $process->setInput(STDIN); - } - $returnCode = max($returnCode, $process->run( - function ($type, $buffer) use ($output) { - $output->write($buffer); - } - )); if (file_exists($this->testsFailedFile)) { $this->failedList = array_merge( $this->failedList, diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php index 19dec6d4b..3047ba277 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php @@ -94,9 +94,10 @@ protected function execute(InputInterface $input, OutputInterface $output): int $command->run(new ArrayInput($args), $output); } - $commandString = realpath(PROJECT_ROOT . '/vendor/bin/codecept') . ' run functional --verbose --steps'; if ($this->pauseEnabled()) { - $commandString .= ' --debug'; + $commandString = self::CODECEPT_RUN_COMMAND . '--verbose --steps --debug'; + } else { + $commandString = realpath(PROJECT_ROOT . '/vendor/bin/codecept') . ' run functional --verbose --steps'; } $exitCode = -1; @@ -104,19 +105,19 @@ protected function execute(InputInterface $input, OutputInterface $output): int foreach ($groups as $group) { $codeceptionCommandString = $commandString . " -g {$group}"; - $process = new Process($codeceptionCommandString); - $process->setWorkingDirectory(TESTS_BP); - $process->setIdleTimeout(600); - $process->setTimeout(0); if ($this->pauseEnabled()) { - $process->setInput(STDIN); + $returnCodes[] = $this->codeceptRunTest($codeceptionCommandString, $output); + } else { + $process = new Process($codeceptionCommandString); + $process->setWorkingDirectory(TESTS_BP); + $process->setIdleTimeout(600); + $process->setTimeout(0); + $returnCodes[] = $process->run( + function ($type, $buffer) use ($output) { + $output->write($buffer); + } + ); } - - $returnCodes[] = $process->run( - function ($type, $buffer) use ($output) { - $output->write($buffer); - } - ); } foreach ($returnCodes as $returnCode) { From e99db412ce7f072e3f83804c61617e1e7c82059f Mon Sep 17 00:00:00 2001 From: Ji Lu <jilu1@adobe.com> Date: Thu, 13 Aug 2020 13:55:17 -0500 Subject: [PATCH 35/52] MQE-2110: MFTF command to pause test execution --- docs/commands/mftf.md | 25 ++++++++ docs/interactive-pause.md | 58 +++++++++++++++++++ .../Console/BaseGenerateCommand.php | 11 ++-- .../Console/CodeceptRunCommand.php | 2 +- .../Console/RunManifestCommand.php | 4 +- .../Console/RunTestCommand.php | 4 +- .../Console/RunTestFailedCommand.php | 4 +- .../Console/RunTestGroupCommand.php | 2 +- .../Module/MagentoWebDriver.php | 3 + 9 files changed, 100 insertions(+), 13 deletions(-) create mode 100644 docs/interactive-pause.md diff --git a/docs/commands/mftf.md b/docs/commands/mftf.md index 3c170e5ca..5f74e5efe 100644 --- a/docs/commands/mftf.md +++ b/docs/commands/mftf.md @@ -559,6 +559,31 @@ To upgrade all test components inside the `Catalog` module: vendor/bin/mftf upgrade:tests /Users/user/magento2/app/code/Magento/Catalog/Test/Mftf/ ``` +### `codecept:run` + +MFTF wrapper command invokes `vendor/bin/codecept run`. This command runs tests in functional suite. It does not generate tests for you. You must do that first. + +#### Usage + +See https://codeception.com/docs/reference/Commands#Run + +```bash +vendor/bin/mftf codecept:run [<suite|test>] --[<option(s)>] +``` + +#### Examples + +```bash +# Run all tests in functional suite +vendor/bin/mftf codecept:run functional +# Run all tests in functional suite with options +vendor/bin/mftf codecept:run functional --verbose --steps --debug +# Run one test +vendor/bin/mftf codecept:run functional Magento/_generated/default/AdminLoginSuccessfulTestCest +# Run all tests in default group +vendor/bin/mftf codecept:run functional --verbose --steps -g default +``` + <!-- LINK DEFINITIONS --> [configuration]: ../configuration.md diff --git a/docs/interactive-pause.md b/docs/interactive-pause.md new file mode 100644 index 000000000..f5fd3597a --- /dev/null +++ b/docs/interactive-pause.md @@ -0,0 +1,58 @@ +# Interactive Pause + +It’s hard to write a complete test at once. You will need to try different commands with different arguments before you find a correct path. + +Since Codeception 3.0 you can pause execution in any point and enter interactive shell where you will be able to try commands in action. + +Now this `Interactive Pause` feature is available in MFTF and all you need to do is to set `ENABLE_PAUSE=true` in `.env`. + +Check it out at [Codeception website][] for documentation and a video to see `Interactive Pause` in action. + +In short, when a test gets to `$I->pause()` step, it stops and shows a console where you can try all available commands with auto-completion, stash commands, and save screenshot, etc. + +## Generation Time + +A `<pause>` action in xml will always be generated into php regardless if `ENABLE_PAUSE=true` is set or not. +However, when `ENABLE_PAUSE=true` is set, an additional`pause()` action will be generated in `_failed()` hook for a test, +so that the test could pause on failure at run time. + +## Execution Time + +To use `Interactive Pause` at run time, there are two types of MFTF commands to use. + +### MFTF Run Commands + +When `ENABLE_PAUSE=true` is set, the following MFTF run commands support `Interactive Pause`. + +```bash +vendor/bin/mftf run:group +``` + +```bash +vendor/bin/mftf run:test +``` + +```bash +vendor/bin/mftf run:manifest +``` + +```bash +vendor/bin/mftf run:failed +``` + +<div class="bs-callout-warning"> +Note: MFTF run command's `--debug` option is different from Codeception `--debug` mode option. +</div> + +### MFTF Codecept Run Command + +You can also use MFTF's wrapper command to run Codeception directly and activate `Interactive Pause` by passing `--debug` option. +You don't need to set `ENABLE_PAUSE=true` for this command. + +```bash +vendor/bin/mftf codecept:run --debug +``` + +## References + +[Codeception website](https://codeception.com/docs/02-GettingStarted#Interactive-Pause) \ No newline at end of file diff --git a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php index 11ffffdae..44c6ceec9 100644 --- a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php @@ -31,7 +31,8 @@ class BaseGenerateCommand extends Command { const MFTF_NOTICES = "Placeholder text for MFTF notices\n"; - const CODECEPT_RUN_COMMAND = 'codecept:run functional '; + const CODECEPT_RUN = 'codecept:run '; + const CODECEPT_RUN_FUNCTIONAL = self::CODECEPT_RUN . 'functional '; /** * Enable pause() @@ -254,15 +255,15 @@ protected function pauseEnabled() /** * Runs the bin/mftf codecept:run command and returns exit code * - * @param string $command + * @param string $commandStr * @param OutputInterface $output * @return integer * @throws \Exception */ - protected function codeceptRunTest(string $command, OutputInterface $output) + protected function codeceptRunTest(string $commandStr, OutputInterface $output) { - $input = new StringInput($command); - $command = $this->getApplication()->find('codecept:run'); + $input = new StringInput($commandStr); + $command = $this->getApplication()->find(self::CODECEPT_RUN); return $command->run($input, $output); } } diff --git a/src/Magento/FunctionalTestingFramework/Console/CodeceptRunCommand.php b/src/Magento/FunctionalTestingFramework/Console/CodeceptRunCommand.php index 311b81e12..de16bad84 100644 --- a/src/Magento/FunctionalTestingFramework/Console/CodeceptRunCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/CodeceptRunCommand.php @@ -24,7 +24,7 @@ protected function configure() { $this->setName('codecept:run') ->setDescription( - "Wrapper command to codecept:run. See https://codeception.com/docs/reference/Commands" + "Wrapper command to vendor/bin/codecept:run. See https://codeception.com/docs/reference/Commands#Run" ); parent::configure(); diff --git a/src/Magento/FunctionalTestingFramework/Console/RunManifestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunManifestCommand.php index 76b92c960..89ae90888 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunManifestCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunManifestCommand.php @@ -111,8 +111,8 @@ protected function execute(InputInterface $input, OutputInterface $output): int private function runManifestLine(string $manifestLine, OutputInterface $output) { if (getenv('ENABLE_PAUSE') === 'true') { - $codeceptionCommand = BaseGenerateCommand::CODECEPT_RUN_COMMAND - . '--verbose --steps --debug' . $manifestLine; + $codeceptionCommand = BaseGenerateCommand::CODECEPT_RUN_FUNCTIONAL + . '--verbose --steps --debug ' . $manifestLine; $input = new StringInput($codeceptionCommand); $command = $this->getApplication()->find('codecept:run'); $subReturnCode = $command->run($input, $output); diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php index 502ef7c78..3d99b6fad 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php @@ -124,7 +124,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int private function runTests(array $tests, OutputInterface $output) { if ($this->pauseEnabled()) { - $codeceptionCommand = self::CODECEPT_RUN_COMMAND; + $codeceptionCommand = self::CODECEPT_RUN_FUNCTIONAL; } else { $codeceptionCommand = realpath(PROJECT_ROOT . '/vendor/bin/codecept') . ' run functional '; } @@ -164,7 +164,7 @@ private function runTests(array $tests, OutputInterface $output) private function runTestsInSuite(array $suitesConfig, OutputInterface $output) { if ($this->pauseEnabled()) { - $codeceptionCommand = self::CODECEPT_RUN_COMMAND . '--verbose --steps --debug'; + $codeceptionCommand = self::CODECEPT_RUN_FUNCTIONAL . '--verbose --steps --debug'; } else { $codeceptionCommand = realpath(PROJECT_ROOT . '/vendor/bin/codecept') . ' run functional --verbose --steps '; diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php index fa48d0b72..87606d53d 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php @@ -118,8 +118,8 @@ protected function execute(InputInterface $input, OutputInterface $output): int $returnCode = 0; foreach ($testManifestList as $testCommand) { if ($this->pauseEnabled()) { - $codeceptionCommand = self::CODECEPT_RUN_COMMAND . $testCommand . ' --debug'; - $this->codeceptRunTest($codeceptionCommand, $output); + $codeceptionCommand = self::CODECEPT_RUN_FUNCTIONAL . $testCommand . ' --debug'; + $returnCode = $this->codeceptRunTest($codeceptionCommand, $output); } else { $codeceptionCommand = realpath(PROJECT_ROOT . '/vendor/bin/codecept') . ' run functional '; $codeceptionCommand .= $testCommand; diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php index 3047ba277..896218efc 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php @@ -95,7 +95,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int } if ($this->pauseEnabled()) { - $commandString = self::CODECEPT_RUN_COMMAND . '--verbose --steps --debug'; + $commandString = self::CODECEPT_RUN_FUNCTIONAL . '--verbose --steps --debug'; } else { $commandString = realpath(PROJECT_ROOT . '/vendor/bin/codecept') . ' run functional --verbose --steps'; } diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php index 8886f4493..859d666de 100644 --- a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php +++ b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php @@ -836,6 +836,9 @@ public function _failed(TestInterface $test, $fail) if ($this->pngReport === null && $this->htmlReport === null) { $this->saveScreenshot(); + if (getenv('ENABLE_PAUSE') === 'true') { + $this->pause(); + } } if ($this->current_test == null) { From b88849237b1f3b000703ea8a0a9183c43bc62b73 Mon Sep 17 00:00:00 2001 From: Ji Lu <jilu1@adobe.com> Date: Fri, 14 Aug 2020 09:06:28 -0500 Subject: [PATCH 36/52] MQE-2110: MFTF command to pause test execution --- docs/interactive-pause.md | 9 +++++++-- docs/test/actions.md | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/interactive-pause.md b/docs/interactive-pause.md index f5fd3597a..e7fc0f704 100644 --- a/docs/interactive-pause.md +++ b/docs/interactive-pause.md @@ -6,7 +6,7 @@ Since Codeception 3.0 you can pause execution in any point and enter interactive Now this `Interactive Pause` feature is available in MFTF and all you need to do is to set `ENABLE_PAUSE=true` in `.env`. -Check it out at [Codeception website][] for documentation and a video to see `Interactive Pause` in action. +Check [pause on codeception.com][] for documentation and a video to see `Interactive Pause` in action. In short, when a test gets to `$I->pause()` step, it stops and shows a console where you can try all available commands with auto-completion, stash commands, and save screenshot, etc. @@ -53,6 +53,11 @@ You don't need to set `ENABLE_PAUSE=true` for this command. vendor/bin/mftf codecept:run --debug ``` +<div class="bs-callout-warning"> +Note: You may want to limit the usage of this Codeception command with arguments and options for `acceptance` only +since it's what is supported by MFTF. You should also replace `acceptance` to `functional` when using this command when referring to Codeception documentation. +</div> + ## References -[Codeception website](https://codeception.com/docs/02-GettingStarted#Interactive-Pause) \ No newline at end of file +[pause on codeception.com](https://codeception.com/docs/02-GettingStarted#Interactive-Pause) \ No newline at end of file diff --git a/docs/test/actions.md b/docs/test/actions.md index a4e0fb44c..647635782 100644 --- a/docs/test/actions.md +++ b/docs/test/actions.md @@ -1463,7 +1463,7 @@ Attribute|Type|Use|Description ### pause -See [pause docs on codeception.com](https://codeception.com/docs/02-GettingStarted#Interactive-Pause). +See usage of `<pause` in [interactive-pause](../interactive-pause.md) and [pause docs on codeception.com](https://codeception.com/docs/02-GettingStarted#Interactive-Pause). Attribute|Type|Use|Description ---|---|---|--- From 1dab5ed0f30b4466207afbcc4439c8c0c9242618 Mon Sep 17 00:00:00 2001 From: Donald Booth <dobooth@adobe.com> Date: Fri, 14 Aug 2020 11:30:31 -0500 Subject: [PATCH 37/52] Grammar and formatting --- docs/commands/mftf.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/commands/mftf.md b/docs/commands/mftf.md index 5f74e5efe..8d1c636f2 100644 --- a/docs/commands/mftf.md +++ b/docs/commands/mftf.md @@ -561,11 +561,11 @@ vendor/bin/mftf upgrade:tests /Users/user/magento2/app/code/Magento/Catalog/Test ### `codecept:run` -MFTF wrapper command invokes `vendor/bin/codecept run`. This command runs tests in functional suite. It does not generate tests for you. You must do that first. +A MFTF wrapper command that invokes `vendor/bin/codecept run`. This command runs tests in functional suite. Tests must be generated before using this command. #### Usage -See https://codeception.com/docs/reference/Commands#Run +See the [Run Command](https://codeception.com/docs/reference/Commands#Run). ```bash vendor/bin/mftf codecept:run [<suite|test>] --[<option(s)>] From 297b2a0c6955cfb503c55b9ba1ec6d1b9010fbda Mon Sep 17 00:00:00 2001 From: Donald Booth <dobooth@adobe.com> Date: Fri, 14 Aug 2020 11:33:02 -0500 Subject: [PATCH 38/52] grammar and formatting --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 90111a03b..f780ac633 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -361,7 +361,7 @@ WAIT_TIMEOUT=30 ### ENABLE_PAUSE -Enables pause of test execution in any point and enter interactive shell where you will be able to try commands in action. +Enables the ability to pause test execution at any point, and enter an interactive shell where you can try commands in action. When pause is enabled, MFTF will generate pause() command in _failed() hook so that test will pause execution when failed. ```conf From 15f62b44e1a9ed9d6a3c09a766bb5edecdacaed2 Mon Sep 17 00:00:00 2001 From: Donald Booth <dobooth@adobe.com> Date: Fri, 14 Aug 2020 11:41:11 -0500 Subject: [PATCH 39/52] Editorial pass --- docs/interactive-pause.md | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/docs/interactive-pause.md b/docs/interactive-pause.md index e7fc0f704..507502ee0 100644 --- a/docs/interactive-pause.md +++ b/docs/interactive-pause.md @@ -1,24 +1,24 @@ # Interactive Pause -It’s hard to write a complete test at once. You will need to try different commands with different arguments before you find a correct path. +It can be difficut to write a successful test on the first attempt. You will need to try different commands, with different arguments, before you find the correct path. -Since Codeception 3.0 you can pause execution in any point and enter interactive shell where you will be able to try commands in action. +Since Codeception 3.0, you can pause execution in any point and enter an interactive shell where you will be able to try commands in action. -Now this `Interactive Pause` feature is available in MFTF and all you need to do is to set `ENABLE_PAUSE=true` in `.env`. +Now this `Interactive Pause` feature is available in MFTF. All you need to do is to set `ENABLE_PAUSE=true` in `.env`. Check [pause on codeception.com][] for documentation and a video to see `Interactive Pause` in action. -In short, when a test gets to `$I->pause()` step, it stops and shows a console where you can try all available commands with auto-completion, stash commands, and save screenshot, etc. +In short, when a test gets to `$I->pause()` step, it stops and shows a console where you can try all available commands with auto-completion, stash commands, save screenshots, etc. ## Generation Time A `<pause>` action in xml will always be generated into php regardless if `ENABLE_PAUSE=true` is set or not. However, when `ENABLE_PAUSE=true` is set, an additional`pause()` action will be generated in `_failed()` hook for a test, -so that the test could pause on failure at run time. +so that the test may pause on failure at run time. ## Execution Time -To use `Interactive Pause` at run time, there are two types of MFTF commands to use. +To use `Interactive Pause` at run time, there are two types of MFTF commands to use: ### MFTF Run Commands @@ -47,17 +47,16 @@ Note: MFTF run command's `--debug` option is different from Codeception `--debug ### MFTF Codecept Run Command You can also use MFTF's wrapper command to run Codeception directly and activate `Interactive Pause` by passing `--debug` option. -You don't need to set `ENABLE_PAUSE=true` for this command. +You do not need to set `ENABLE_PAUSE=true` for this command. ```bash vendor/bin/mftf codecept:run --debug ``` <div class="bs-callout-warning"> -Note: You may want to limit the usage of this Codeception command with arguments and options for `acceptance` only -since it's what is supported by MFTF. You should also replace `acceptance` to `functional` when using this command when referring to Codeception documentation. +You may want to limit the usage of this Codeception command with arguments and options for `acceptance` only, since it is what is supported by MFTF. You should also change `acceptance` to `functional` when using this command when referring to Codeception documentation. </div> ## References -[pause on codeception.com](https://codeception.com/docs/02-GettingStarted#Interactive-Pause) \ No newline at end of file +[pause on codeception.com](https://codeception.com/docs/02-GettingStarted#Interactive-Pause) From 5831ac54e13caab8f5719f2c1f58ce53a8339937 Mon Sep 17 00:00:00 2001 From: Ji Lu <jilu1@adobe.com> Date: Fri, 14 Aug 2020 12:41:50 -0500 Subject: [PATCH 40/52] MQE-2110: MFTF command to pause test execution --- .../Console/BaseGenerateCommand.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php index 44c6ceec9..525711e31 100644 --- a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php @@ -31,8 +31,8 @@ class BaseGenerateCommand extends Command { const MFTF_NOTICES = "Placeholder text for MFTF notices\n"; - const CODECEPT_RUN = 'codecept:run '; - const CODECEPT_RUN_FUNCTIONAL = self::CODECEPT_RUN . 'functional '; + const CODECEPT_RUN = 'codecept:run'; + const CODECEPT_RUN_FUNCTIONAL = self::CODECEPT_RUN . ' functional '; /** * Enable pause() From ba3362b5729ac1037de386e3e76dbf530bc0ea85 Mon Sep 17 00:00:00 2001 From: soumyau <sunnikri@adobe.com> Date: Fri, 14 Aug 2020 14:40:28 -0500 Subject: [PATCH 41/52] MQE-2242: Add static check to flag use of <pause> (#783) * MQE-2242: Add static check to flag use of <pause> * MQE-2242: Add static check to flag use of <pause> --- dev/tests/_bootstrap.php | 2 + dev/tests/util/MftfStaticTestCase.php | 49 ++++ ...tionGroupWithMultiplePausesActionGroup.xml | 18 ++ .../ActionGroupWithNoPauseActionGroup.xml | 15 ++ .../ActionGroupWithPauseActionGroup.xml | 15 ++ .../suiteWithMultiplePauseActionsSuite.xml | 28 +++ .../Suite/suiteWithPauseActionSuite.xml | 27 +++ .../Test/TestWithMultiplePauseActionsTest.xml | 30 +++ .../Test/TestWithPauseActionTest.xml | 23 ++ .../mftf-pause-action-usage-checks.txt | 30 +++ .../DeprecationStaticCheckTest.php | 46 +--- .../PauseActionStaticCheckTest.php | 51 ++++ docs/commands/mftf.md | 11 +- .../Console/StaticChecksCommand.php | 15 +- .../StaticCheck/PauseActionUsageCheck.php | 229 ++++++++++++++++++ .../StaticCheck/StaticChecksList.php | 4 +- .../Util/Script/ScriptUtil.php | 2 +- 17 files changed, 543 insertions(+), 52 deletions(-) create mode 100644 dev/tests/util/MftfStaticTestCase.php create mode 100644 dev/tests/verification/PauseCheckModule/ActionGroup/ActionGroupWithMultiplePausesActionGroup.xml create mode 100644 dev/tests/verification/PauseCheckModule/ActionGroup/ActionGroupWithNoPauseActionGroup.xml create mode 100644 dev/tests/verification/PauseCheckModule/ActionGroup/ActionGroupWithPauseActionGroup.xml create mode 100644 dev/tests/verification/PauseCheckModule/Suite/suiteWithMultiplePauseActionsSuite.xml create mode 100644 dev/tests/verification/PauseCheckModule/Suite/suiteWithPauseActionSuite.xml create mode 100644 dev/tests/verification/PauseCheckModule/Test/TestWithMultiplePauseActionsTest.xml create mode 100644 dev/tests/verification/PauseCheckModule/Test/TestWithPauseActionTest.xml create mode 100644 dev/tests/verification/Resources/StaticChecks/mftf-pause-action-usage-checks.txt create mode 100644 dev/tests/verification/Tests/StaticCheck/PauseActionStaticCheckTest.php create mode 100644 src/Magento/FunctionalTestingFramework/StaticCheck/PauseActionUsageCheck.php diff --git a/dev/tests/_bootstrap.php b/dev/tests/_bootstrap.php index 059441721..9d3750ec6 100644 --- a/dev/tests/_bootstrap.php +++ b/dev/tests/_bootstrap.php @@ -9,9 +9,11 @@ $vendorAutoloadPath = realpath(PROJECT_ROOT . '/vendor/autoload.php'); $mftfTestCasePath = realpath(PROJECT_ROOT . '/dev/tests/util/MftfTestCase.php'); +$mftfStaticTestCasePath = realpath(PROJECT_ROOT . '/dev/tests/util/MftfStaticTestCase.php'); require_once $vendorAutoloadPath; require_once $mftfTestCasePath; +require_once $mftfStaticTestCasePath; // Set up AspectMock $kernel = \AspectMock\Kernel::getInstance(); diff --git a/dev/tests/util/MftfStaticTestCase.php b/dev/tests/util/MftfStaticTestCase.php new file mode 100644 index 000000000..708561031 --- /dev/null +++ b/dev/tests/util/MftfStaticTestCase.php @@ -0,0 +1,49 @@ +<?php +/** + * Copyright © Magento, Inc. All rights reserved. + * See COPYING.txt for license details. + */ + +namespace tests\util; + +use Magento\FunctionalTestingFramework\Util\Filesystem\DirSetupUtil; +use PHPUnit\Framework\TestCase; +use Symfony\Component\Console\Input\InputInterface; + +class MftfStaticTestCase extends TestCase +{ + const STATIC_RESULTS_DIR = TESTS_MODULE_PATH . + DIRECTORY_SEPARATOR . + '_output' . + DIRECTORY_SEPARATOR . + 'static-results'; + + const RESOURCES_PATH = TESTS_MODULE_PATH . + DIRECTORY_SEPARATOR . + "Resources" . + DIRECTORY_SEPARATOR . + 'StaticChecks'; + + /** + * Sets input interface + * @param $path + * @return \PHPUnit\Framework\MockObject\MockObject + */ + public function mockInputInterface($path = null) + { + $input = $this->getMockBuilder(InputInterface::class) + ->disableOriginalConstructor() + ->getMock(); + if ($path) { + $input->method('getOption') + ->with('path') + ->willReturn($path); + } + return $input; + } + + public function tearDown(): void + { + DirSetupUtil::rmdirRecursive(self::STATIC_RESULTS_DIR); + } +} diff --git a/dev/tests/verification/PauseCheckModule/ActionGroup/ActionGroupWithMultiplePausesActionGroup.xml b/dev/tests/verification/PauseCheckModule/ActionGroup/ActionGroupWithMultiplePausesActionGroup.xml new file mode 100644 index 000000000..5e9299631 --- /dev/null +++ b/dev/tests/verification/PauseCheckModule/ActionGroup/ActionGroupWithMultiplePausesActionGroup.xml @@ -0,0 +1,18 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<!-- + /** + * Copyright © Magento, Inc. All rights reserved. + * See COPYING.txt for license details. + */ +--> +<actionGroups xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:noNamespaceSchemaLocation="urn:magento:mftf:Test/etc/actionGroupSchema.xsd"> + <actionGroup name="ActionGroupWithMultiplePausesActionGroup"> + <fillField selector="#foo" userInput="foo" stepKey="fillField1"/> + <pause stepKey="pauseAfterFillField1"/> + <fillField selector="#bar" userInput="bar" stepKey="fillField2"/> + <pause stepKey="pauseAfterFillField2"/> + <fillField selector="#baz" userInput="baz" stepKey="fillField3"/> + <pause stepKey="pauseAfterFillField3"/> + </actionGroup> +</actionGroups> diff --git a/dev/tests/verification/PauseCheckModule/ActionGroup/ActionGroupWithNoPauseActionGroup.xml b/dev/tests/verification/PauseCheckModule/ActionGroup/ActionGroupWithNoPauseActionGroup.xml new file mode 100644 index 000000000..8acae47e8 --- /dev/null +++ b/dev/tests/verification/PauseCheckModule/ActionGroup/ActionGroupWithNoPauseActionGroup.xml @@ -0,0 +1,15 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<!-- + /** + * Copyright © Magento, Inc. All rights reserved. + * See COPYING.txt for license details. + */ +--> +<actionGroups xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:noNamespaceSchemaLocation="urn:magento:mftf:Test/etc/actionGroupSchema.xsd"> + <actionGroup name="ActionGroupWithNoPauseActionGroup"> + <fillField selector="#foo" userInput="foo" stepKey="fillField1"/> + <fillField selector="#bar" userInput="bar" stepKey="fillField2"/> + <fillField selector="#baz" userInput="baz" stepKey="fillField3"/> + </actionGroup> +</actionGroups> diff --git a/dev/tests/verification/PauseCheckModule/ActionGroup/ActionGroupWithPauseActionGroup.xml b/dev/tests/verification/PauseCheckModule/ActionGroup/ActionGroupWithPauseActionGroup.xml new file mode 100644 index 000000000..aaef1befa --- /dev/null +++ b/dev/tests/verification/PauseCheckModule/ActionGroup/ActionGroupWithPauseActionGroup.xml @@ -0,0 +1,15 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<!-- + /** + * Copyright © Magento, Inc. All rights reserved. + * See COPYING.txt for license details. + */ +--> +<actionGroups xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:noNamespaceSchemaLocation="urn:magento:mftf:Test/etc/actionGroupSchema.xsd"> + <actionGroup name="ActionGroupWithPauseActionGroup"> + <fillField selector="#foo" userInput="myData1" stepKey="fillField1"/> + <fillField selector="#bar" userInput="myData2" stepKey="fillField2"/> + <pause stepKey="pauseAfterFillField2"/> + </actionGroup> +</actionGroups> diff --git a/dev/tests/verification/PauseCheckModule/Suite/suiteWithMultiplePauseActionsSuite.xml b/dev/tests/verification/PauseCheckModule/Suite/suiteWithMultiplePauseActionsSuite.xml new file mode 100644 index 000000000..47ff1088b --- /dev/null +++ b/dev/tests/verification/PauseCheckModule/Suite/suiteWithMultiplePauseActionsSuite.xml @@ -0,0 +1,28 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + /** + * Copyright © Magento, Inc. All rights reserved. + * See COPYING.txt for license details. + */ +--> + +<suites xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="urn:magento:mftf:Suite/etc/suiteSchema.xsd"> + <suite name="suiteWithMultiplePauseActionsSuite"> + <include> + <group name="include" /> + </include> + <before> + <amOnPage url="some.url" stepKey="before"/> + <createData entity="SecretData" stepKey="create1"> + <field key="someKey">dataHere</field> + </createData> + <pause stepKey="pauseCreate1"/> + </before> + <after> + <deleteData createDataKey="create1" stepKey="delete1"/> + <deleteData url="deleteThis" stepKey="deleteThis"/> + <fillField selector="#fill" userInput="{{SecretData.key2}}" stepKey="fillAfter"/> + <pause stepKey="pauseFillAfter"/> + </after> + </suite> +</suites> diff --git a/dev/tests/verification/PauseCheckModule/Suite/suiteWithPauseActionSuite.xml b/dev/tests/verification/PauseCheckModule/Suite/suiteWithPauseActionSuite.xml new file mode 100644 index 000000000..71a3b5769 --- /dev/null +++ b/dev/tests/verification/PauseCheckModule/Suite/suiteWithPauseActionSuite.xml @@ -0,0 +1,27 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + /** + * Copyright © Magento, Inc. All rights reserved. + * See COPYING.txt for license details. + */ +--> + +<suites xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="urn:magento:mftf:Suite/etc/suiteSchema.xsd"> + <suite name="suiteWithPauseActionSuite"> + <include> + <group name="include" /> + </include> + <before> + <amOnPage url="some.url" stepKey="before"/> + <createData entity="createThis" stepKey="create"> + <field key="someKey">dataHere</field> + </createData> + <pause stepKey="pauseSuite"/> + <click stepKey="clickWithData" userInput="$create.data$"/> + <fillField selector="#foo" userInput="myData1" stepKey="fillField1"/> + </before> + <after> + <comment userInput="afterBlock" stepKey="afterBlock"/> + </after> + </suite> +</suites> diff --git a/dev/tests/verification/PauseCheckModule/Test/TestWithMultiplePauseActionsTest.xml b/dev/tests/verification/PauseCheckModule/Test/TestWithMultiplePauseActionsTest.xml new file mode 100644 index 000000000..fa47e976c --- /dev/null +++ b/dev/tests/verification/PauseCheckModule/Test/TestWithMultiplePauseActionsTest.xml @@ -0,0 +1,30 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + /** + * Copyright © Magento, Inc. All rights reserved. + * See COPYING.txt for license details. + */ +--> + +<tests xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:noNamespaceSchemaLocation="urn:magento:mftf:Test/etc/testSchema.xsd"> + <test name="TestWithMultiplePauseActionsTest"> + <annotations> + <severity value="CRITICAL"/> + <group value="functional"/> + <features value="Pause check"/> + <stories value="MQE-433"/> + </annotations> + <before> + <amOnPage url="/beforeUrl" stepKey="beforeAmOnPageKey"/> + <pause stepKey="pauseBeforeAmOnPageKey"/> + </before> + <fillField stepKey="step1" selector="#username" userInput="step1"/> + <fillField stepKey="step2" selector="#password" userInput="step2"/> + <pause stepKey="pauseAfterStep2"/> + <after> + <amOnPage url="/afterUrl" stepKey="afterAmOnPageKey"/> + <pause stepKey="pauseAfterAmOnPageKey"/> + </after> + </test> +</tests> diff --git a/dev/tests/verification/PauseCheckModule/Test/TestWithPauseActionTest.xml b/dev/tests/verification/PauseCheckModule/Test/TestWithPauseActionTest.xml new file mode 100644 index 000000000..70d0903b8 --- /dev/null +++ b/dev/tests/verification/PauseCheckModule/Test/TestWithPauseActionTest.xml @@ -0,0 +1,23 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + /** + * Copyright © Magento, Inc. All rights reserved. + * See COPYING.txt for license details. + */ +--> + +<tests xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:noNamespaceSchemaLocation="urn:magento:mftf:Test/etc/testSchema.xsd"> + <test name="TestWithPauseActionTest"> + <annotations> + <severity value="CRITICAL"/> + <group value="functional"/> + <features value="Pause check"/> + <stories value="MQE-433"/> + </annotations> + <amOnPage stepKey="step1" url="/step1"/> + <fillField stepKey="step2" selector="#username" userInput="step2"/> + <fillField stepKey="step3" selector="#password" userInput="step3"/> + <pause stepKey="pauseAfterStep3"/> + </test> +</tests> diff --git a/dev/tests/verification/Resources/StaticChecks/mftf-pause-action-usage-checks.txt b/dev/tests/verification/Resources/StaticChecks/mftf-pause-action-usage-checks.txt new file mode 100644 index 000000000..fd042a24c --- /dev/null +++ b/dev/tests/verification/Resources/StaticChecks/mftf-pause-action-usage-checks.txt @@ -0,0 +1,30 @@ + +File "/verification/PauseCheckModule/ActionGroup/ActionGroupWithMultiplePausesActionGroup.xml" +contains pause action(s): + + ActionGroupWithMultiplePausesActionGroup has pause action at stepKey(s): pauseAfterFillField1, pauseAfterFillField2, pauseAfterFillField3 + +File "/verification/PauseCheckModule/ActionGroup/ActionGroupWithPauseActionGroup.xml" +contains pause action(s): + + ActionGroupWithPauseActionGroup has pause action at stepKey(s): pauseAfterFillField2 + +File "/verification/PauseCheckModule/Test/TestWithMultiplePauseActionsTest.xml" +contains pause action(s): + + TestWithMultiplePauseActionsTest has pause action at stepKey(s): pauseBeforeAmOnPageKey, pauseAfterStep2, pauseAfterAmOnPageKey + +File "/verification/PauseCheckModule/Test/TestWithPauseActionTest.xml" +contains pause action(s): + + TestWithPauseActionTest has pause action at stepKey(s): pauseAfterStep3 + +File "/verification/PauseCheckModule/Suite/suiteWithMultiplePauseActionsSuite.xml" +contains pause action(s): + + suiteWithMultiplePauseActionsSuite has pause action at stepKey(s): pauseCreate1, pauseFillAfter + +File "/verification/PauseCheckModule/Suite/suiteWithPauseActionSuite.xml" +contains pause action(s): + + suiteWithPauseActionSuite has pause action at stepKey(s): pauseSuite diff --git a/dev/tests/verification/Tests/StaticCheck/DeprecationStaticCheckTest.php b/dev/tests/verification/Tests/StaticCheck/DeprecationStaticCheckTest.php index bafe586ac..f6aef65f6 100644 --- a/dev/tests/verification/Tests/StaticCheck/DeprecationStaticCheckTest.php +++ b/dev/tests/verification/Tests/StaticCheck/DeprecationStaticCheckTest.php @@ -8,18 +8,11 @@ use AspectMock\Test as AspectMock; use Magento\FunctionalTestingFramework\StaticCheck\DeprecatedEntityUsageCheck; use Magento\FunctionalTestingFramework\StaticCheck\StaticChecksList; -use Magento\FunctionalTestingFramework\Util\Filesystem\DirSetupUtil; -use PHPUnit\Framework\TestCase; use Symfony\Component\Console\Input\InputInterface; +use tests\util\MftfStaticTestCase; -class DeprecationStaticCheckTest extends TestCase +class DeprecationStaticCheckTest extends MftfStaticTestCase { - const STATIC_RESULTS_DIR = TESTS_MODULE_PATH . - DIRECTORY_SEPARATOR . - '_output' . - DIRECTORY_SEPARATOR . - 'static-results'; - const LOG_FILE = self::STATIC_RESULTS_DIR . DIRECTORY_SEPARATOR . DeprecatedEntityUsageCheck::ERROR_LOG_FILENAME . @@ -30,20 +23,6 @@ class DeprecationStaticCheckTest extends TestCase 'DeprecationCheckModule'. DIRECTORY_SEPARATOR; - const RESOURCES_PATH = TESTS_MODULE_PATH . - DIRECTORY_SEPARATOR . - "Resources" . - DIRECTORY_SEPARATOR . - 'StaticChecks'; - - public static function setUpBeforeClass(): void - { - // remove static-results if it exists - if (file_exists(self::STATIC_RESULTS_DIR)) { - DirSetupUtil::rmdirRecursive(self::STATIC_RESULTS_DIR); - } - } - /** * test static-check DeprecatedEntityUsageCheck. * @@ -68,25 +47,4 @@ public function testDeprecatedEntityUsageCheck() self::LOG_FILE ); } - - /** - * Sets input interface - * @param $path - * @return \PHPUnit\Framework\MockObject\MockObject - */ - public function mockInputInterface($path) - { - $input = $this->getMockBuilder(InputInterface::class) - ->disableOriginalConstructor() - ->getMock(); - $input->method('getOption') - ->with('path') - ->willReturn($path); - return $input; - } - - public function tearDown(): void - { - DirSetupUtil::rmdirRecursive(self::STATIC_RESULTS_DIR); - } } diff --git a/dev/tests/verification/Tests/StaticCheck/PauseActionStaticCheckTest.php b/dev/tests/verification/Tests/StaticCheck/PauseActionStaticCheckTest.php new file mode 100644 index 000000000..7c8f77c2f --- /dev/null +++ b/dev/tests/verification/Tests/StaticCheck/PauseActionStaticCheckTest.php @@ -0,0 +1,51 @@ +<?php +/** + * Copyright © Magento, Inc. All rights reserved. + * See COPYING.txt for license details. + */ +namespace tests\verification\Tests; + +use AspectMock\Test as AspectMock; +use Magento\FunctionalTestingFramework\StaticCheck\PauseActionUsageCheck; +use Magento\FunctionalTestingFramework\StaticCheck\StaticChecksList; +use Symfony\Component\Console\Input\InputInterface; + +use tests\util\MftfStaticTestCase; + +class PauseActionStaticCheckTest extends MftfStaticTestCase +{ + const LOG_FILE = self::STATIC_RESULTS_DIR . + DIRECTORY_SEPARATOR . + PauseActionUsageCheck::ERROR_LOG_FILENAME . + '.txt'; + + const TEST_MODULE_PATH = TESTS_MODULE_PATH . + DIRECTORY_SEPARATOR . + 'PauseCheckModule'. + DIRECTORY_SEPARATOR; + + /** + * test static-check PauseActionUsageCheck. + * + * @throws \Exception + */ + public function testPauseActionUsageCheck() + { + $staticCheck = new PauseActionUsageCheck(); + + $input = $this->mockInputInterface(self::TEST_MODULE_PATH); + AspectMock::double(StaticChecksList::class, ['getErrorFilesPath' => self::STATIC_RESULTS_DIR]); + + /** @var InputInterface $input */ + $staticCheck->execute($input); + + $this->assertTrue(file_exists(self::LOG_FILE)); + $this->assertFileEquals( + self::RESOURCES_PATH. + DIRECTORY_SEPARATOR . + PauseActionUsageCheck::ERROR_LOG_FILENAME . + ".txt", + self::LOG_FILE + ); + } +} diff --git a/docs/commands/mftf.md b/docs/commands/mftf.md index 3c170e5ca..0c5c86285 100644 --- a/docs/commands/mftf.md +++ b/docs/commands/mftf.md @@ -449,7 +449,7 @@ vendor/bin/mftf static-checks [<names>]... | Option | Description | |-----------------------|-----------------------------------------------------------------------------------------------------------| -| `-p, --path` | Path to a MFTF test module to run "deprecatedEntityUsage" static check script. Option is ignored by other static check scripts. +| `-p, --path` | Path to a MFTF test module to run "deprecatedEntityUsage" and "pauseActionUsage" static check scripts. Option is ignored by other static check scripts. #### Examples @@ -479,6 +479,10 @@ vendor/bin/mftf static-checks actionGroupArguments vendor/bin/mftf static-checks deprecatedEntityUsage ``` +```bash +vendor/bin/mftf static-checks pauseActionUsage +``` + ```bash vendor/bin/mftf static-checks annotations ``` @@ -487,6 +491,10 @@ vendor/bin/mftf static-checks annotations vendor/bin/mftf static-checks deprecatedEntityUsage -p path/to/mftf/test/module ``` +```bash +vendor/bin/mftf static-checks pauseActionUsage -p path/to/mftf/test/module +``` + ```bash vendor/bin/mftf static-checks testDependencies actionGroupArguments ``` @@ -499,6 +507,7 @@ vendor/bin/mftf static-checks testDependencies actionGroupArguments |`actionGroupArguments` | Checks that action groups do not have unused arguments.| |`deprecatedEntityUsage`| Checks that deprecated test entities are not being referenced.| |`annotations`| Checks various details of test annotations, such as missing annotations or duplicate annotations.| +|`pauseUsage`| Checks that pause action is not used in action groups, tests or suites.| #### Defining ruleset diff --git a/src/Magento/FunctionalTestingFramework/Console/StaticChecksCommand.php b/src/Magento/FunctionalTestingFramework/Console/StaticChecksCommand.php index fe07dbaaf..806508ef8 100644 --- a/src/Magento/FunctionalTestingFramework/Console/StaticChecksCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/StaticChecksCommand.php @@ -151,12 +151,17 @@ private function validateInput(InputInterface $input) } if ($input->getOption('path')) { - if ( (count($this->staticCheckObjects) !== 1) - || array_keys($this->staticCheckObjects)[0] !== StaticChecksList::DEPRECATED_ENTITY_USAGE_CHECK_NAME ) + if ((count($this->staticCheckObjects) !== 1) + || !in_array( + array_keys($this->staticCheckObjects)[0], + [ + StaticChecksList::DEPRECATED_ENTITY_USAGE_CHECK_NAME, + StaticChecksList::PAUSE_ACTION_USAGE_CHECK_NAME + ] + ) + ) throw new InvalidArgumentException( - '--path option can only be used for "' - . StaticChecksList::DEPRECATED_ENTITY_USAGE_CHECK_NAME - . '".' + '--path option is not supported for the command."' ); } } diff --git a/src/Magento/FunctionalTestingFramework/StaticCheck/PauseActionUsageCheck.php b/src/Magento/FunctionalTestingFramework/StaticCheck/PauseActionUsageCheck.php new file mode 100644 index 000000000..e7d106bf0 --- /dev/null +++ b/src/Magento/FunctionalTestingFramework/StaticCheck/PauseActionUsageCheck.php @@ -0,0 +1,229 @@ +<?php +/** + * Copyright © Magento, Inc. All rights reserved. + * See COPYING.txt for license details. + */ + +namespace Magento\FunctionalTestingFramework\StaticCheck; + +use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig; +use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; +use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Finder\Finder; +use Exception; +use Magento\FunctionalTestingFramework\Util\Script\ScriptUtil; +use Symfony\Component\Finder\SplFileInfo; + +/** + * Class PauseActionUsageCheck + * @package Magento\FunctionalTestingFramework\StaticCheck + */ +class PauseActionUsageCheck implements StaticCheckInterface +{ + const ERROR_LOG_FILENAME = 'mftf-pause-action-usage-checks'; + const ERROR_LOG_MESSAGE = 'MFTF Pause Action Usage Check'; + + /** + * Array containing all errors found after running the execute() function. + * @var array + */ + private $errors = []; + + /** + * String representing the output summary found after running the execute() function. + * @var string + */ + private $output; + + /** + * ScriptUtil instance + * + * @var ScriptUtil + */ + private $scriptUtil; + + /** + * Test xml files to scan + * + * @var Finder|array + */ + private $testXmlFiles = []; + + /** + * Action group xml files to scan + * + * @var Finder|array + */ + private $actionGroupXmlFiles = []; + + /** + * Suite xml files to scan + * + * @var Finder|array + */ + private $suiteXmlFiles = []; + + /** + * Root suite xml files to scan + * + * @var Finder|array + */ + private $rootSuiteXmlFiles = []; + + /** + * Checks usage of pause action in action groups, tests and suites and prints out error to file. + * + * @param InputInterface $input + * @return void + * @throws Exception + */ + public function execute(InputInterface $input) + { + $this->scriptUtil = new ScriptUtil(); + $modulePaths = []; + $includeRootPath = true; + $path = $input->getOption('path'); + if ($path) { + if (!realpath($path)) { + throw new \InvalidArgumentException('Invalid --path option: ' . $path); + } + $modulePaths[] = realpath($path); + $includeRootPath = false; + } else { + $modulePaths = $this->scriptUtil->getAllModulePaths(); + } + + $this->testXmlFiles = $this->scriptUtil->getModuleXmlFilesByScope($modulePaths, 'Test'); + $this->actionGroupXmlFiles = $this->scriptUtil->getModuleXmlFilesByScope($modulePaths, 'ActionGroup'); + $this->suiteXmlFiles = $this->scriptUtil->getModuleXmlFilesByScope($modulePaths, 'Suite'); + if ($includeRootPath) { + $this->rootSuiteXmlFiles = $this->scriptUtil->getRootSuiteXmlFiles(); + } + $this->errors = []; + $this->errors += $this->validatePauseActionUsageInActionGroups($this->actionGroupXmlFiles); + $this->errors += $this->validatePauseActionUsageInTests($this->testXmlFiles); + $this->errors += $this->validatePauseActionUsageInSuites($this->suiteXmlFiles); + $this->errors += $this->validatePauseActionUsageInSuites($this->rootSuiteXmlFiles); + + $this->output = $this->scriptUtil->printErrorsToFile( + $this->errors, + StaticChecksList::getErrorFilesPath() . DIRECTORY_SEPARATOR . self::ERROR_LOG_FILENAME . '.txt', + self::ERROR_LOG_MESSAGE + ); + } + + /** + * Finds usages of pause action in action group files + * @param array $actionGroupXmlFiles + * @return array + */ + private function validatePauseActionUsageInActionGroups($actionGroupXmlFiles) + { + $actionGroupErrors = []; + foreach ($actionGroupXmlFiles as $filePath) { + $domDocument = new \DOMDocument(); + $domDocument->load($filePath); + $actionGroup = $domDocument->getElementsByTagName('actionGroup')->item(0); + $violatingStepKeys = $this->findViolatingPauseStepKeys($actionGroup); + $actionGroupErrors = array_merge($actionGroupErrors, $this->setErrorOutput($violatingStepKeys, $filePath)); + } + return $actionGroupErrors; + } + + /** + * Finds usages of pause action in test files + * @param array $testXmlFiles + * @return array + */ + private function validatePauseActionUsageInTests($testXmlFiles) + { + $testErrors = []; + foreach ($testXmlFiles as $filePath) { + $domDocument = new \DOMDocument(); + $domDocument->load($filePath); + $test = $domDocument->getElementsByTagName('test')->item(0); + $violatingStepKeys = $this->findViolatingPauseStepKeys($test); + $testErrors = array_merge($testErrors, $this->setErrorOutput($violatingStepKeys, $filePath)); + } + return $testErrors; + } + + /** + * Finds usages of pause action in suite files + * @param array $suiteXmlFiles + * @return array + */ + private function validatePauseActionUsageInSuites($suiteXmlFiles) + { + $suiteErrors = []; + foreach ($suiteXmlFiles as $filePath) { + $domDocument = new \DOMDocument(); + $domDocument->load($filePath); + $suite = $domDocument->getElementsByTagName('suite')->item(0); + $violatingStepKeys = $this->findViolatingPauseStepKeys($suite); + $suiteErrors = array_merge($suiteErrors, $this->setErrorOutput($violatingStepKeys, $filePath)); + } + return $suiteErrors; + } + + /** + * Finds violating pause action step keys + * @param \DomNode $entity + * @return array + */ + private function findViolatingPauseStepKeys($entity) + { + $violatingStepKeys = []; + $entityName = $entity->getAttribute('name'); + $references = $entity->getElementsByTagName('pause'); + + foreach ($references as $reference) { + $pauseStepKey = $reference->getAttribute('stepKey'); + $violatingStepKeys[$entityName][] = $pauseStepKey; + } + return $violatingStepKeys; + } + + /** + * Return array containing all errors found after running the execute() function. + * @return array + */ + public function getErrors() + { + return $this->errors; + } + + /** + * Return string of a short human readable result of the check. For example: "No errors found." + * @return string + */ + public function getOutput() + { + return $this->output; + } + + /** + * Build and return error output for pause action usages + * + * @param array $violatingReferences + * @param SplFileInfo $path + * @return mixed + */ + private function setErrorOutput($violatingReferences, $path) + { + $testErrors = []; + + $filePath = StaticChecksList::getFilePath($path->getRealPath()); + + if (!empty($violatingReferences)) { + // Build error output + $errorOutput = "\nFile \"{$filePath}\""; + $errorOutput .= "\ncontains pause action(s):\n\t\t"; + foreach ($violatingReferences as $entityName => $stepKey) { + $errorOutput .= "\n\t {$entityName} has pause action at stepKey(s): " . implode(", ", $stepKey); + } + $testErrors[$filePath][] = $errorOutput; + } + return $testErrors; + } +} diff --git a/src/Magento/FunctionalTestingFramework/StaticCheck/StaticChecksList.php b/src/Magento/FunctionalTestingFramework/StaticCheck/StaticChecksList.php index ba2139276..07c52ce8c 100644 --- a/src/Magento/FunctionalTestingFramework/StaticCheck/StaticChecksList.php +++ b/src/Magento/FunctionalTestingFramework/StaticCheck/StaticChecksList.php @@ -17,6 +17,7 @@ class StaticChecksList implements StaticCheckListInterface { const DEPRECATED_ENTITY_USAGE_CHECK_NAME = 'deprecatedEntityUsage'; + const PAUSE_ACTION_USAGE_CHECK_NAME = 'pauseActionUsage'; const STATIC_RESULTS = 'tests' . DIRECTORY_SEPARATOR .'_output' . DIRECTORY_SEPARATOR . 'static-results'; /** @@ -45,7 +46,8 @@ public function __construct(array $checks = []) 'testDependencies' => new TestDependencyCheck(), 'actionGroupArguments' => new ActionGroupArgumentsCheck(), self::DEPRECATED_ENTITY_USAGE_CHECK_NAME => new DeprecatedEntityUsageCheck(), - 'annotations' => new AnnotationsCheck() + 'annotations' => new AnnotationsCheck(), + self::PAUSE_ACTION_USAGE_CHECK_NAME => new PauseActionUsageCheck() ] + $checks; // Static checks error files directory diff --git a/src/Magento/FunctionalTestingFramework/Util/Script/ScriptUtil.php b/src/Magento/FunctionalTestingFramework/Util/Script/ScriptUtil.php index 14f9c27e0..11afa5cc1 100644 --- a/src/Magento/FunctionalTestingFramework/Util/Script/ScriptUtil.php +++ b/src/Magento/FunctionalTestingFramework/Util/Script/ScriptUtil.php @@ -101,7 +101,7 @@ public function getModuleXmlFilesByScope($modulePaths, $scope) if (!realpath($modulePath . $scopePath)) { continue; } - $finder->files()->followLinks()->in($modulePath . $scopePath)->name("*.xml"); + $finder->files()->followLinks()->in($modulePath . $scopePath)->name("*.xml")->sortByName(); $found = true; } return $found ? $finder->files() : []; From 20f52038ec45000d5472237cc5bc965088a7f7c4 Mon Sep 17 00:00:00 2001 From: Ji Lu <jilu1@adobe.com> Date: Fri, 14 Aug 2020 15:23:04 -0500 Subject: [PATCH 42/52] MQE-2110: MFTF command to pause test execution --- docs/commands/mftf.md | 2 +- docs/interactive-pause.md | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/commands/mftf.md b/docs/commands/mftf.md index 8d1c636f2..523e3e505 100644 --- a/docs/commands/mftf.md +++ b/docs/commands/mftf.md @@ -579,7 +579,7 @@ vendor/bin/mftf codecept:run functional # Run all tests in functional suite with options vendor/bin/mftf codecept:run functional --verbose --steps --debug # Run one test -vendor/bin/mftf codecept:run functional Magento/_generated/default/AdminLoginSuccessfulTestCest +vendor/bin/mftf codecept:run functional Magento/_generated/default/AdminCreateCmsPageTestCest.php --debug # Run all tests in default group vendor/bin/mftf codecept:run functional --verbose --steps -g default ``` diff --git a/docs/interactive-pause.md b/docs/interactive-pause.md index 507502ee0..2b1d2cb10 100644 --- a/docs/interactive-pause.md +++ b/docs/interactive-pause.md @@ -13,7 +13,7 @@ In short, when a test gets to `$I->pause()` step, it stops and shows a console w ## Generation Time A `<pause>` action in xml will always be generated into php regardless if `ENABLE_PAUSE=true` is set or not. -However, when `ENABLE_PAUSE=true` is set, an additional`pause()` action will be generated in `_failed()` hook for a test, +However, when `ENABLE_PAUSE=true` is set, an additional `pause()` action will be generated in `_failed()` hook for a test, so that the test may pause on failure at run time. ## Execution Time @@ -40,10 +40,6 @@ vendor/bin/mftf run:manifest vendor/bin/mftf run:failed ``` -<div class="bs-callout-warning"> -Note: MFTF run command's `--debug` option is different from Codeception `--debug` mode option. -</div> - ### MFTF Codecept Run Command You can also use MFTF's wrapper command to run Codeception directly and activate `Interactive Pause` by passing `--debug` option. @@ -54,7 +50,11 @@ vendor/bin/mftf codecept:run --debug ``` <div class="bs-callout-warning"> -You may want to limit the usage of this Codeception command with arguments and options for `acceptance` only, since it is what is supported by MFTF. You should also change `acceptance` to `functional` when using this command when referring to Codeception documentation. +Note: MFTF run command's `--debug` option is different from Codeception `--debug` mode option. +</div> + +<div class="bs-callout-warning"> +Note: You may want to limit the usage of this Codeception command with arguments and options for `acceptance` only, since it is what is supported by MFTF. You should also change `acceptance` to `functional` when using this command when referring to Codeception documentation. </div> ## References From 2ca6005d1dca71675901b66a54c6988e83093af3 Mon Sep 17 00:00:00 2001 From: Ji Lu <jilu1@adobe.com> Date: Fri, 14 Aug 2020 17:09:24 -0500 Subject: [PATCH 43/52] MQE-2110: MFTF command to pause test execution --- docs/commands/mftf.md | 4 ++++ docs/interactive-pause.md | 45 ++++++++++++++++++++++----------------- 2 files changed, 30 insertions(+), 19 deletions(-) diff --git a/docs/commands/mftf.md b/docs/commands/mftf.md index 523e3e505..c537a9759 100644 --- a/docs/commands/mftf.md +++ b/docs/commands/mftf.md @@ -584,6 +584,10 @@ vendor/bin/mftf codecept:run functional Magento/_generated/default/AdminCreateCm vendor/bin/mftf codecept:run functional --verbose --steps -g default ``` +<div class="bs-callout-warning"> +Note: You may want to limit the usage of this Codeception command with arguments and options for `acceptance` only, since it is what is supported by MFTF. You should also change `acceptance` to `functional` when using this command when referring to Codeception documentation. +</div> + <!-- LINK DEFINITIONS --> [configuration]: ../configuration.md diff --git a/docs/interactive-pause.md b/docs/interactive-pause.md index 2b1d2cb10..85906d2a8 100644 --- a/docs/interactive-pause.md +++ b/docs/interactive-pause.md @@ -4,25 +4,15 @@ It can be difficut to write a successful test on the first attempt. You will nee Since Codeception 3.0, you can pause execution in any point and enter an interactive shell where you will be able to try commands in action. -Now this `Interactive Pause` feature is available in MFTF. All you need to do is to set `ENABLE_PAUSE=true` in `.env`. +Now this `Interactive Pause` feature is available in MFTF. All you need to do is to set `ENABLE_PAUSE` to `true` in `.env`. Check [pause on codeception.com][] for documentation and a video to see `Interactive Pause` in action. In short, when a test gets to `$I->pause()` step, it stops and shows a console where you can try all available commands with auto-completion, stash commands, save screenshots, etc. -## Generation Time +## MFTF Run Commands -A `<pause>` action in xml will always be generated into php regardless if `ENABLE_PAUSE=true` is set or not. -However, when `ENABLE_PAUSE=true` is set, an additional `pause()` action will be generated in `_failed()` hook for a test, -so that the test may pause on failure at run time. - -## Execution Time - -To use `Interactive Pause` at run time, there are two types of MFTF commands to use: - -### MFTF Run Commands - -When `ENABLE_PAUSE=true` is set, the following MFTF run commands support `Interactive Pause`. +The following MFTF run commands support `Interactive Pause` when `ENABLE_PAUSE` is set to `true`. ```bash vendor/bin/mftf run:group @@ -40,10 +30,31 @@ vendor/bin/mftf run:manifest vendor/bin/mftf run:failed ``` -### MFTF Codecept Run Command +### Use `Interactive Pause` During Test Development + +Here is a typical work flow for this use case: + +- Set `ENABLE_PAUSE` to `true` under `.env` +- Add `<pause>` action in a test where you want to stop for investigation +- Run test +- Execution should pause at <pause> action and invoke interactive console +- Try out commands in interactive console +- Resume test execution by pressing `ENTER` + +### Use `Pause` On Test Failure + +When `ENABLE_PAUSE` is set to `true`, MFTF automatically generates `pause()` action in `_failed()` hook for tests and in `_failed()` function in `MagentoWebDriver`. +This allows you to use `pause` to debug test failure for a long running test. The work flow might look like: + +- Set `ENABLE_PAUSE` to `true` under `.env` +- Run test +- Execution pauses and invokes interactive console right after test fails +- Examine and debug on the spot of failure + +## MFTF Codecept Run Command You can also use MFTF's wrapper command to run Codeception directly and activate `Interactive Pause` by passing `--debug` option. -You do not need to set `ENABLE_PAUSE=true` for this command. +You do not need to set `ENABLE_PAUSE=true` for this command if you are not using `Pause on Failure`. ```bash vendor/bin/mftf codecept:run --debug @@ -53,10 +64,6 @@ vendor/bin/mftf codecept:run --debug Note: MFTF run command's `--debug` option is different from Codeception `--debug` mode option. </div> -<div class="bs-callout-warning"> -Note: You may want to limit the usage of this Codeception command with arguments and options for `acceptance` only, since it is what is supported by MFTF. You should also change `acceptance` to `functional` when using this command when referring to Codeception documentation. -</div> - ## References [pause on codeception.com](https://codeception.com/docs/02-GettingStarted#Interactive-Pause) From d6c82cabe930b4729ae53fc1ceff6097534d2364 Mon Sep 17 00:00:00 2001 From: Ji Lu <jilu1@adobe.com> Date: Fri, 14 Aug 2020 17:19:26 -0500 Subject: [PATCH 44/52] MQE-2110: MFTF command to pause test execution --- docs/interactive-pause.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/interactive-pause.md b/docs/interactive-pause.md index 85906d2a8..25e0f7f0e 100644 --- a/docs/interactive-pause.md +++ b/docs/interactive-pause.md @@ -35,7 +35,7 @@ vendor/bin/mftf run:failed Here is a typical work flow for this use case: - Set `ENABLE_PAUSE` to `true` under `.env` -- Add `<pause>` action in a test where you want to stop for investigation +- Add <pause> action in a test where you want to pause execution for debugging - Run test - Execution should pause at <pause> action and invoke interactive console - Try out commands in interactive console @@ -54,7 +54,7 @@ This allows you to use `pause` to debug test failure for a long running test. Th ## MFTF Codecept Run Command You can also use MFTF's wrapper command to run Codeception directly and activate `Interactive Pause` by passing `--debug` option. -You do not need to set `ENABLE_PAUSE=true` for this command if you are not using `Pause on Failure`. +You do not need to set `ENABLE_PAUSE` to `true` for this command if you don't want to pause on test failure. ```bash vendor/bin/mftf codecept:run --debug From a6affde1dc05a57787f44d90427d9d44d4db4ddc Mon Sep 17 00:00:00 2001 From: Ji Lu <jilu1@adobe.com> Date: Fri, 14 Aug 2020 20:50:06 -0500 Subject: [PATCH 45/52] MQE-2110: MFTF command to pause test execution --- docs/commands/mftf.md | 14 +++++++++++++- docs/interactive-pause.md | 8 +++++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/docs/commands/mftf.md b/docs/commands/mftf.md index b3365cfa0..0d74c1efb 100644 --- a/docs/commands/mftf.md +++ b/docs/commands/mftf.md @@ -585,16 +585,28 @@ vendor/bin/mftf codecept:run [<suite|test>] --[<option(s)>] ```bash # Run all tests in functional suite vendor/bin/mftf codecept:run functional +``` + +```bash # Run all tests in functional suite with options vendor/bin/mftf codecept:run functional --verbose --steps --debug +``` + +```bash # Run one test vendor/bin/mftf codecept:run functional Magento/_generated/default/AdminCreateCmsPageTestCest.php --debug +``` + +```bash # Run all tests in default group vendor/bin/mftf codecept:run functional --verbose --steps -g default ``` <div class="bs-callout-warning"> -Note: You may want to limit the usage of this Codeception command with arguments and options for `acceptance` only, since it is what is supported by MFTF. You should also change `acceptance` to `functional` when using this command when referring to Codeception documentation. +<p> +Note: You may want to limit the usage of this Codeception command with arguments and options for "acceptance" only, since it is what's supported by MFTF. +When using this command, you should change "acceptance" to "functional" when referring to Codeception documentation. +</p> </div> <!-- LINK DEFINITIONS --> diff --git a/docs/interactive-pause.md b/docs/interactive-pause.md index 25e0f7f0e..250f19936 100644 --- a/docs/interactive-pause.md +++ b/docs/interactive-pause.md @@ -61,9 +61,11 @@ vendor/bin/mftf codecept:run --debug ``` <div class="bs-callout-warning"> -Note: MFTF run command's `--debug` option is different from Codeception `--debug` mode option. +<p> +Note: MFTF command "--debug" option has different meaning than Codeception command "--debug" mode option. +</p> </div> -## References +<!-- Link definitions --> -[pause on codeception.com](https://codeception.com/docs/02-GettingStarted#Interactive-Pause) +[pause on codeception.com]: https://codeception.com/docs/02-GettingStarted#Interactive-Pause From 97a7e64e4565e46e295f445d452f387eb3db12f5 Mon Sep 17 00:00:00 2001 From: Alex Kolesnyk <kolesnyk@adobe.com> Date: Mon, 17 Aug 2020 14:46:25 -0500 Subject: [PATCH 46/52] MQE-2133: Make tests green for Chrome 84 --- .../Module/MagentoWebDriver.php | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php index 859d666de..3c1d8af50 100644 --- a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php +++ b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php @@ -761,19 +761,25 @@ public function dragAndDrop($source, $target, $xOffset = null, $yOffset = null) $snodes = $this->matchFirstOrFail($this->baseElement, $source); $tnodes = $this->matchFirstOrFail($this->baseElement, $target); $action = new WebDriverActions($this->webDriver); - if ($xOffset !== null || $yOffset !== null) { $targetX = intval($tnodes->getLocation()->getX() + $xOffset); $targetY = intval($tnodes->getLocation()->getY() + $yOffset); - $travelX = intval($targetX - $snodes->getLocation()->getX()); $travelY = intval($targetY - $snodes->getLocation()->getY()); $action->moveToElement($snodes); $action->clickAndHold($snodes); + // Fix Start + $action->moveByOffset(-1,-1); + $action->moveByOffset(1,1); + // Fix End $action->moveByOffset($travelX, $travelY); $action->release()->perform(); } else { $action->clickAndHold($snodes); + // Fix Start + $action->moveByOffset(-1,-1); + $action->moveByOffset(1,1); + // Fix End $action->moveToElement($tnodes); $action->release($tnodes)->perform(); } From 5d4aab2770cd1dfd75ac6f98bde3db8e6da73e6e Mon Sep 17 00:00:00 2001 From: Alex Kolesnyk <kolesnyk@adobe.com> Date: Mon, 17 Aug 2020 15:01:53 -0500 Subject: [PATCH 47/52] MQE-2133: Make tests green for Chrome 84 --- .../Module/MagentoWebDriver.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php index 3c1d8af50..13d44a358 100644 --- a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php +++ b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php @@ -769,16 +769,16 @@ public function dragAndDrop($source, $target, $xOffset = null, $yOffset = null) $action->moveToElement($snodes); $action->clickAndHold($snodes); // Fix Start - $action->moveByOffset(-1,-1); - $action->moveByOffset(1,1); + $action->moveByOffset(-1, -1); + $action->moveByOffset(1, 1); // Fix End $action->moveByOffset($travelX, $travelY); $action->release()->perform(); } else { $action->clickAndHold($snodes); // Fix Start - $action->moveByOffset(-1,-1); - $action->moveByOffset(1,1); + $action->moveByOffset(-1, -1); + $action->moveByOffset(1, 1); // Fix End $action->moveToElement($tnodes); $action->release($tnodes)->perform(); From 9d0317b7202ebcfdc88c67c92d5e96153cf15825 Mon Sep 17 00:00:00 2001 From: Ji Lu <jilu1@adobe.com> Date: Mon, 17 Aug 2020 17:33:05 -0500 Subject: [PATCH 48/52] MQE-2270: Pausing of test during a failure is not intuitive for ENABLE_PAUSE=true --- .../Module/MagentoWebDriver.php | 25 +++++++++++++++++-- .../Test/Objects/ActionObject.php | 1 + .../Test/Util/TestHookObjectExtractor.php | 6 ++++- .../Util/TestGenerator.php | 10 ++++++++ 4 files changed, 39 insertions(+), 3 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php index 13d44a358..00e295606 100644 --- a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php +++ b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php @@ -54,7 +54,9 @@ class MagentoWebDriver extends WebDriver { use AttachmentSupport; - use Pause; + use Pause { + pause as codeceptPause; + } const MAGENTO_CRON_INTERVAL = 60; const MAGENTO_CRON_COMMAND = 'cron:run'; @@ -843,7 +845,7 @@ public function _failed(TestInterface $test, $fail) if ($this->pngReport === null && $this->htmlReport === null) { $this->saveScreenshot(); if (getenv('ENABLE_PAUSE') === 'true') { - $this->pause(); + $this->pause(true); } } @@ -1028,4 +1030,23 @@ public function switchToIFrame($locator = null) $this->webDriver->switchTo()->frame($els[0]); } } + + /** + * Invoke Codeption pause() + * + * @param boolean $pauseOnFail + * @return void + */ + public function pause($pauseOnFail = false) + { + if (!\Codeception\Util\Debug::isEnabled()) { + return; + } + + if ($pauseOnFail) { + print(PHP_EOL . "Failure encountered. Pausing execution..." . PHP_EOL . PHP_EOL); + } + + $this->codeceptPause(); + } } diff --git a/src/Magento/FunctionalTestingFramework/Test/Objects/ActionObject.php b/src/Magento/FunctionalTestingFramework/Test/Objects/ActionObject.php index 8d4e7a945..8e4aa973c 100644 --- a/src/Magento/FunctionalTestingFramework/Test/Objects/ActionObject.php +++ b/src/Magento/FunctionalTestingFramework/Test/Objects/ActionObject.php @@ -77,6 +77,7 @@ class ActionObject const ACTION_TYPE_COMMENT = 'comment'; const ACTION_TYPE_HELPER = 'helper'; const INVISIBLE_STEP_ACTIONS = ['retrieveEntityField', 'getSecret']; + const PAUSE_ACTION_INTERNAL_ATTRIBUTE = 'pauseOnFail'; /** * The unique identifier for the action diff --git a/src/Magento/FunctionalTestingFramework/Test/Util/TestHookObjectExtractor.php b/src/Magento/FunctionalTestingFramework/Test/Util/TestHookObjectExtractor.php index a68f09491..e9c163d31 100644 --- a/src/Magento/FunctionalTestingFramework/Test/Util/TestHookObjectExtractor.php +++ b/src/Magento/FunctionalTestingFramework/Test/Util/TestHookObjectExtractor.php @@ -66,7 +66,11 @@ public function createDefaultFailedHook($parentName) { $defaultSteps['saveScreenshot'] = new ActionObject("saveScreenshot", "saveScreenshot", []); if (getenv('ENABLE_PAUSE') === 'true') { - $defaultSteps['pauseWhenFailed'] = new ActionObject('pauseWhenFailed', 'pause', []); + $defaultSteps['pauseWhenFailed'] = new ActionObject( + 'pauseWhenFailed', + 'pause', + [ActionObject::PAUSE_ACTION_INTERNAL_ATTRIBUTE => true] + ); } $hook = new TestHookObject( diff --git a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php index 82fb2c06a..8caffc565 100644 --- a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php +++ b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php @@ -1441,6 +1441,16 @@ public function generateStepsPhp($actionObjects, $generationScope = TestGenerato $testSteps .= $dateGenerateCode; break; + case "pause": + $pauseAttr = $actionObject->getCustomActionAttributes( + ActionObject::PAUSE_ACTION_INTERNAL_ATTRIBUTE + ); + if ($pauseAttr) { + $testSteps .= sprintf("\t\t$%s->%s(%s);", $actor, $actionObject->getType(), 'true'); + } else { + $testSteps .= sprintf("\t\t$%s->%s();", $actor, $actionObject->getType()); + } + break; case "comment": $input = $input === null ? strtr($value, ['$' => '\$', '{' => '\{', '}' => '\}']) : $input; // Combining userInput from native XML comment and <comment/> action to fall-through 'default' case From cf562e106532553bed328cea5b4fd3b0ce96c3d2 Mon Sep 17 00:00:00 2001 From: Ji Lu <jilu1@adobe.com> Date: Tue, 18 Aug 2020 08:18:08 -0500 Subject: [PATCH 49/52] MQE-2270: Pausing of test during a failure is not intuitive for ENABLE_PAUSE=true --- .../Test/Handlers/TestObjectHandlerTest.php | 23 ++----------------- 1 file changed, 2 insertions(+), 21 deletions(-) diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Test/Handlers/TestObjectHandlerTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Test/Handlers/TestObjectHandlerTest.php index 036656cdd..939dd496e 100644 --- a/dev/tests/unit/Magento/FunctionalTestFramework/Test/Handlers/TestObjectHandlerTest.php +++ b/dev/tests/unit/Magento/FunctionalTestFramework/Test/Handlers/TestObjectHandlerTest.php @@ -260,25 +260,6 @@ public function testGetAllTestObjectsWithInvalidExtends() $toh->getAllObjects(); } - /** - * Function used to set mock for parser return and force init method to run between tests. - * - * @param array $data - * @throws \Exception - */ - private function setMockParserOutput($data) - { - // clear test object handler value to inject parsed content - $property = new \ReflectionProperty(TestObjectHandler::class, 'testObjectHandler'); - $property->setAccessible(true); - $property->setValue(null); - - $mockDataParser = AspectMock::double(TestDataParser::class, ['readTestData' => $data])->make(); - $instance = AspectMock::double(ObjectManager::class, ['create' => $mockDataParser]) - ->make(); // bypass the private constructor - AspectMock::double(ObjectManagerFactory::class, ['getObjectManager' => $instance]); - } - /** * Validate test object when ENABLE_PAUSE is set to true * @@ -299,7 +280,7 @@ public function testGetTestObjectWhenEnablePause() $resolverMock = new MockModuleResolverBuilder(); $resolverMock->setup(); - $this->setMockParserOutput($mockData); + ObjectHandlerUtil::mockTestObjectHandlerWitData($mockData); // run object handler method $toh = TestObjectHandler::getInstance(); @@ -325,7 +306,7 @@ public function testGetTestObjectWhenEnablePause() $expectedFailedActionObject2 = new ActionObject( 'pauseWhenFailed', 'pause', - [] + [ActionObject::PAUSE_ACTION_INTERNAL_ATTRIBUTE => true] ); $expectedBeforeHookObject = new TestHookObject( From 713b3333cbf411c336dc766145e8caa2b61fab38 Mon Sep 17 00:00:00 2001 From: Ji Lu <jilu1@adobe.com> Date: Tue, 18 Aug 2020 12:34:45 -0500 Subject: [PATCH 50/52] MQE-2269: Test manifest run stops at first failure encountered when ENABLE_PAUSE = true --- .../Console/BaseGenerateCommand.php | 1 + .../Console/RunManifestCommand.php | 22 ++++++++++++++----- .../Console/RunTestCommand.php | 15 +++++++++++-- .../Console/RunTestFailedCommand.php | 10 ++++++--- .../Console/RunTestGroupCommand.php | 7 ++++-- 5 files changed, 42 insertions(+), 13 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php index 525711e31..401153c7b 100644 --- a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php @@ -33,6 +33,7 @@ class BaseGenerateCommand extends Command const MFTF_NOTICES = "Placeholder text for MFTF notices\n"; const CODECEPT_RUN = 'codecept:run'; const CODECEPT_RUN_FUNCTIONAL = self::CODECEPT_RUN . ' functional '; + const CODECEPT_RUN_OPTION_NO_EXIT = ' --no-exit '; /** * Enable pause() diff --git a/src/Magento/FunctionalTestingFramework/Console/RunManifestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunManifestCommand.php index 89ae90888..683be1b53 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunManifestCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunManifestCommand.php @@ -81,12 +81,17 @@ protected function execute(InputInterface $input, OutputInterface $output): int // Delete the Codeception failed file just in case it exists from any previous test runs $this->deleteFailedFile(); - foreach ($manifestFile as $manifestLine) { - if (empty($manifestLine)) { + for ($line = 0; $line < count($manifestFile); $line++) { + if (empty($manifestFile[$line])) { continue; } - $this->runManifestLine($manifestLine, $output); + if ($line == count($manifestFile) - 1) { + $this->runManifestLine($manifestFile[$line], $output, true); + } else { + $this->runManifestLine($manifestFile[$line], $output); + } + $this->aggregateFailed(); } @@ -103,18 +108,23 @@ protected function execute(InputInterface $input, OutputInterface $output): int * * @param string $manifestLine * @param OutputInterface $output + * @param boolean $exit * @return void * @throws \Exception * * @SuppressWarnings(PHPMD.UnusedLocalVariable) Need this because of the unused $type variable in the closure */ - private function runManifestLine(string $manifestLine, OutputInterface $output) + private function runManifestLine($manifestLine, $output, $exit = false) { if (getenv('ENABLE_PAUSE') === 'true') { $codeceptionCommand = BaseGenerateCommand::CODECEPT_RUN_FUNCTIONAL - . '--verbose --steps --debug ' . $manifestLine; + . '--verbose --steps --debug '; + if (!$exit) { + $codeceptionCommand .= BaseGenerateCommand::CODECEPT_RUN_OPTION_NO_EXIT; + } + $codeceptionCommand .= $manifestLine; $input = new StringInput($codeceptionCommand); - $command = $this->getApplication()->find('codecept:run'); + $command = $this->getApplication()->find(BaseGenerateCommand::CODECEPT_RUN); $subReturnCode = $command->run($input, $output); } else { $codeceptionCommand = realpath(PROJECT_ROOT . "/vendor/bin/codecept") diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php index 3d99b6fad..6931e8303 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php @@ -135,8 +135,8 @@ private function runTests(array $tests, OutputInterface $output) TestGenerator::DEFAULT_DIR . DIRECTORY_SEPARATOR ; - foreach ($tests as $test) { - $testName = $test . 'Cest.php'; + for ($i = 0; $i < count($tests); $i++) { + $testName = $tests[$i] . 'Cest.php'; if (!realpath($testsDirectory . $testName)) { throw new TestFrameworkException( $testName . " is not available under " . $testsDirectory @@ -145,6 +145,9 @@ private function runTests(array $tests, OutputInterface $output) if ($this->pauseEnabled()) { $fullCommand = $codeceptionCommand . $testsDirectory . $testName . ' --verbose --steps --debug'; + if ($i != count($tests) - 1) { + $fullCommand .= self::CODECEPT_RUN_OPTION_NO_EXIT; + } $this->returnCode = max($this->returnCode, $this->codeceptRunTest($fullCommand, $output)); } else { $fullCommand = $codeceptionCommand . $testsDirectory . $testName . ' --verbose --steps'; @@ -169,10 +172,18 @@ private function runTestsInSuite(array $suitesConfig, OutputInterface $output) $codeceptionCommand = realpath(PROJECT_ROOT . '/vendor/bin/codecept') . ' run functional --verbose --steps '; } + + $count = count($suitesConfig); + $index = 0; //for tests in suites, run them as a group to run before and after block foreach (array_keys($suitesConfig) as $suite) { $fullCommand = $codeceptionCommand . " -g {$suite}"; + + $index += 1; if ($this->pauseEnabled()) { + if ($index != $count) { + $fullCommand .= self::CODECEPT_RUN_OPTION_NO_EXIT; + } $this->returnCode = max($this->returnCode, $this->codeceptRunTest($fullCommand, $output)); } else { $this->returnCode = max($this->returnCode, $this->executeTestCommand($fullCommand, $output)); diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php index 87606d53d..6e6954c70 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php @@ -116,13 +116,16 @@ protected function execute(InputInterface $input, OutputInterface $output): int $testManifestList = $this->readTestManifestFile(); $returnCode = 0; - foreach ($testManifestList as $testCommand) { + for ($i = 0; $i < count($testManifestList); $i++) { if ($this->pauseEnabled()) { - $codeceptionCommand = self::CODECEPT_RUN_FUNCTIONAL . $testCommand . ' --debug'; + $codeceptionCommand = self::CODECEPT_RUN_FUNCTIONAL . $testManifestList[$i] . ' --debug '; + if ($i != count($testManifestList) - 1) { + $codeceptionCommand .= self::CODECEPT_RUN_OPTION_NO_EXIT; + } $returnCode = $this->codeceptRunTest($codeceptionCommand, $output); } else { $codeceptionCommand = realpath(PROJECT_ROOT . '/vendor/bin/codecept') . ' run functional '; - $codeceptionCommand .= $testCommand; + $codeceptionCommand .= $testManifestList[$i]; $process = new Process($codeceptionCommand); $process->setWorkingDirectory(TESTS_BP); @@ -142,6 +145,7 @@ function ($type, $buffer) use ($output) { ); } } + foreach ($this->failedList as $test) { $this->writeFailedTestToFile($test, $this->testsFailedFile); } diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php index 896218efc..53724561e 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php @@ -102,10 +102,13 @@ protected function execute(InputInterface $input, OutputInterface $output): int $exitCode = -1; $returnCodes = []; - foreach ($groups as $group) { - $codeceptionCommandString = $commandString . " -g {$group}"; + for ($i = 0; $i < count($groups); $i++) { + $codeceptionCommandString = $commandString . ' -g ' . $groups[$i]; if ($this->pauseEnabled()) { + if ($i != count($groups) - 1) { + $codeceptionCommandString .= self::CODECEPT_RUN_OPTION_NO_EXIT; + } $returnCodes[] = $this->codeceptRunTest($codeceptionCommandString, $output); } else { $process = new Process($codeceptionCommandString); From 3f285212faa6f224798505d94a12bb8b2697a491 Mon Sep 17 00:00:00 2001 From: Ji Lu <jilu1@adobe.com> Date: Tue, 18 Aug 2020 13:22:09 -0500 Subject: [PATCH 51/52] MQE-2269: Test manifest run stops at first failure encountered when ENABLE_PAUSE = true --- .../FunctionalTestingFramework/Console/RunTestGroupCommand.php | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php index 53724561e..f18b9dc30 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php @@ -52,6 +52,7 @@ protected function configure() * @throws \Exception * * @SuppressWarnings(PHPMD.UnusedLocalVariable) + * @SuppressWarnings(PHPMD.CyclomaticComplexity) */ protected function execute(InputInterface $input, OutputInterface $output): int { From d14fbe59392520d719916f5979eb67341ce3bb61 Mon Sep 17 00:00:00 2001 From: soumyau <sunnikri@adobe.com> Date: Wed, 19 Aug 2020 11:34:51 -0500 Subject: [PATCH 52/52] MQE-2258: CHANGELOG.MD and Composer version bump (#788) * MQE-2258: CHANGELOG.MD and Composer version bump * Grammar and formatting Co-authored-by: Donald Booth <dobooth@adobe.com> --- CHANGELOG.md | 36 ++++++++++++++++++++++++++++++++++++ composer.json | 2 +- composer.lock | 6 ++++-- docs/test/actions.md | 3 ++- 4 files changed, 43 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a24b5f29..fd555ee88 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,41 @@ Magento Functional Testing Framework Changelog ================================================ +3.1.0 +________ + +### Enhancements + +* Customizability + * Introduced the new `return` action that allows action groups to return a value. See the [actions page](./docs/test/actions.md#return) for details. + * Introduced new MFTF command that invokes `vendor/bin/codecept run`. See the [mftf page](./docs/commands/mftf.md#codeceptrun) for details. + +* Usability + * Introduced new action `pause`, to invoke codeception interactive pause for debugging during test execution. See the [Interactive Pause](./docs/interactive-pause.md) page for details. + * Introduced a new `.env` configuration option `ENABLE_PAUSE`, to enable the new pause feature. + +* Maintainability + * Added a new static check that checks for the usage of the `pause` action. See the [command page](./docs/commands/mftf.md#static-checks) for details. + +* Modularity + * MFTF now supports the use of actions from multiple modules within suites. + +* Traceability + * A deprecation notice is now added at test execution time for deprecated metadata usage. + +### Fixes + +* Fixed issue with suite precondition failure for `createData` with required entity. + +### GitHub Issues/Pull requests: + + * [#547](https://github.com/magento/magento2-functional-testing-framework/pull/547) -- Fix invalid behavior of MAGENTO_BACKEND_BASE_URL + * [#742](https://github.com/magento/magento2-functional-testing-framework/pull/742) -- Fix Waits In MagentoPwaWebDriver + * [#750](https://github.com/magento/magento2-functional-testing-framework/pull/750) -- Docs: Outlining the difference between Allure severity levels + * [#683](https://github.com/magento/magento2-functional-testing-framework/pull/683) -- Docs: Renamed sample test name with the correct one + * [#691](https://github.com/magento/magento2-functional-testing-framework/pull/691) -- Docs: Branch name updates + * [#678](https://github.com/magento/magento2-functional-testing-framework/pull/678) -- Docs: Command added to modify the web server rewrites configuration + * [#745](https://github.com/magento/magento2-functional-testing-framework/pull/745) -- Docs: Remove invalid sample test name + 3.0.0 --------- diff --git a/composer.json b/composer.json index 6f8b0c2c2..813ab382b 100755 --- a/composer.json +++ b/composer.json @@ -2,7 +2,7 @@ "name": "magento/magento2-functional-testing-framework", "description": "Magento2 Functional Testing Framework", "type": "library", - "version": "3.0.0", + "version": "3.1.0", "license": "AGPL-3.0", "keywords": ["magento", "automation", "functional", "testing"], "config": { diff --git a/composer.lock b/composer.lock index 91fa2c772..7f7648096 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "98733866973fb51e11e316b98c0af563", + "content-hash": "278e33e2c7d183d0b7689b5a76127d29", "packages": [ { "name": "allure-framework/allure-codeception", @@ -3664,6 +3664,7 @@ "keywords": [ "tokenizer" ], + "abandoned": true, "time": "2020-02-07T06:19:00+00:00" }, { @@ -7469,5 +7470,6 @@ "ext-json": "*", "ext-openssl": "*" }, - "platform-dev": [] + "platform-dev": [], + "plugin-api-version": "1.1.0" } diff --git a/docs/test/actions.md b/docs/test/actions.md index f69ee1d66..42ecde053 100644 --- a/docs/test/actions.md +++ b/docs/test/actions.md @@ -1251,8 +1251,9 @@ Attribute|Type|Use|Description `stepKey`|string|required| A unique identifier of the action. #### Example + ```xml -<!-- Returns value of $grabInputName to the calling +<!-- Returns value of $grabInputName to the calling --> <return value="{$grabInputName}" stepKey="returnInputName"/> ```