diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a3de3fb0..cf798da9f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,11 @@ Magento Functional Testing Framework Changelog ================================================ +2.3.4 +----- +### Fixes +* MagentoWebDriver overrides `parent::_after()` function and remaps to `runAfter()`, necessary to solve compatibility issues in Codeception `2.3.x`. + 2.3.3 ----- ### Fixes diff --git a/bin/mftf b/bin/mftf index d70ee75c3..ad9952e59 100755 --- a/bin/mftf +++ b/bin/mftf @@ -29,7 +29,7 @@ try { try { $application = new Symfony\Component\Console\Application(); $application->setName('Magento Functional Testing Framework CLI'); - $application->setVersion('2.3.0'); + $application->setVersion('2.3.4'); /** @var \Magento\FunctionalTestingFramework\Console\CommandListInterface $commandList */ $commandList = new \Magento\FunctionalTestingFramework\Console\CommandList; foreach ($commandList->getCommands() as $command) { diff --git a/composer.json b/composer.json index 2a9a826fc..c7804dcf9 100755 --- a/composer.json +++ b/composer.json @@ -2,7 +2,7 @@ "name": "magento/magento2-functional-testing-framework", "description": "Magento2 Functional Testing Framework", "type": "library", - "version": "2.3.3", + "version": "2.3.4", "license": "AGPL-3.0", "keywords": ["magento", "automation", "functional", "testing"], "config": { diff --git a/composer.lock b/composer.lock index 72254bfc9..32a599939 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "005cbbfebf014bb331ed0f1540c79475", + "content-hash": "49a38f189328c8b1a79e5787a720ac3c", "packages": [ { "name": "allure-framework/allure-codeception", @@ -349,16 +349,16 @@ }, { "name": "consolidation/config", - "version": "1.0.11", + "version": "1.1.0", "source": { "type": "git", "url": "https://github.com/consolidation/config.git", - "reference": "ede41d946078e97e7a9513aadc3352f1c26817af" + "reference": "c9fc25e9088a708637e18a256321addc0670e578" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/consolidation/config/zipball/ede41d946078e97e7a9513aadc3352f1c26817af", - "reference": "ede41d946078e97e7a9513aadc3352f1c26817af", + "url": "https://api.github.com/repos/consolidation/config/zipball/c9fc25e9088a708637e18a256321addc0670e578", + "reference": "c9fc25e9088a708637e18a256321addc0670e578", "shasum": "" }, "require": { @@ -368,7 +368,7 @@ }, "require-dev": { "g1a/composer-test-scenarios": "^1", - "phpunit/phpunit": "^4", + "phpunit/phpunit": "^5", "satooshi/php-coveralls": "^1.0", "squizlabs/php_codesniffer": "2.*", "symfony/console": "^2.5|^3|^4", @@ -399,7 +399,7 @@ } ], "description": "Provide configuration services for a commandline tool.", - "time": "2018-05-27T01:17:02+00:00" + "time": "2018-08-07T22:57:00+00:00" }, { "name": "consolidation/log", @@ -2555,16 +2555,16 @@ }, { "name": "phpunit/phpunit", - "version": "6.5.10", + "version": "6.5.11", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "5744955af9c0a2de74a5eb5287c50bf025100d39" + "reference": "7bab54cb366076023bbf457a2a0d513332cd40f2" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/5744955af9c0a2de74a5eb5287c50bf025100d39", - "reference": "5744955af9c0a2de74a5eb5287c50bf025100d39", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/7bab54cb366076023bbf457a2a0d513332cd40f2", + "reference": "7bab54cb366076023bbf457a2a0d513332cd40f2", "shasum": "" }, "require": { @@ -2582,7 +2582,7 @@ "phpunit/php-file-iterator": "^1.4.3", "phpunit/php-text-template": "^1.2.1", "phpunit/php-timer": "^1.0.9", - "phpunit/phpunit-mock-objects": "^5.0.8", + "phpunit/phpunit-mock-objects": "^5.0.9", "sebastian/comparator": "^2.1", "sebastian/diff": "^2.0", "sebastian/environment": "^3.1", @@ -2635,20 +2635,20 @@ "testing", "xunit" ], - "time": "2018-08-03T05:27:14+00:00" + "time": "2018-08-07T07:05:35+00:00" }, { "name": "phpunit/phpunit-mock-objects", - "version": "5.0.8", + "version": "5.0.10", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit-mock-objects.git", - "reference": "6f9a3c8bf34188a2b53ce2ae7a126089c53e0a9f" + "reference": "cd1cf05c553ecfec36b170070573e540b67d3f1f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit-mock-objects/zipball/6f9a3c8bf34188a2b53ce2ae7a126089c53e0a9f", - "reference": "6f9a3c8bf34188a2b53ce2ae7a126089c53e0a9f", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit-mock-objects/zipball/cd1cf05c553ecfec36b170070573e540b67d3f1f", + "reference": "cd1cf05c553ecfec36b170070573e540b67d3f1f", "shasum": "" }, "require": { @@ -2661,7 +2661,7 @@ "phpunit/phpunit": "<6.0" }, "require-dev": { - "phpunit/phpunit": "^6.5" + "phpunit/phpunit": "^6.5.11" }, "suggest": { "ext-soap": "*" @@ -2694,7 +2694,7 @@ "mock", "xunit" ], - "time": "2018-07-13T03:27:23+00:00" + "time": "2018-08-09T05:50:03+00:00" }, { "name": "psr/container", @@ -3777,27 +3777,30 @@ }, { "name": "symfony/event-dispatcher", - "version": "v2.8.44", + "version": "v3.4.14", "source": { "type": "git", "url": "https://github.com/symfony/event-dispatcher.git", - "reference": "84ae343f39947aa084426ed1138bb96bf94d1f12" + "reference": "b2e1f19280c09a42dc64c0b72b80fe44dd6e88fb" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/event-dispatcher/zipball/84ae343f39947aa084426ed1138bb96bf94d1f12", - "reference": "84ae343f39947aa084426ed1138bb96bf94d1f12", + "url": "https://api.github.com/repos/symfony/event-dispatcher/zipball/b2e1f19280c09a42dc64c0b72b80fe44dd6e88fb", + "reference": "b2e1f19280c09a42dc64c0b72b80fe44dd6e88fb", "shasum": "" }, "require": { - "php": ">=5.3.9" + "php": "^5.5.9|>=7.0.8" + }, + "conflict": { + "symfony/dependency-injection": "<3.3" }, "require-dev": { "psr/log": "~1.0", - "symfony/config": "^2.0.5|~3.0.0", - "symfony/dependency-injection": "~2.6|~3.0.0", - "symfony/expression-language": "~2.6|~3.0.0", - "symfony/stopwatch": "~2.3|~3.0.0" + "symfony/config": "~2.8|~3.0|~4.0", + "symfony/dependency-injection": "~3.3|~4.0", + "symfony/expression-language": "~2.8|~3.0|~4.0", + "symfony/stopwatch": "~2.8|~3.0|~4.0" }, "suggest": { "symfony/dependency-injection": "", @@ -3806,7 +3809,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-master": "2.8-dev" + "dev-master": "3.4-dev" } }, "autoload": { @@ -3833,7 +3836,7 @@ ], "description": "Symfony EventDispatcher Component", "homepage": "https://symfony.com", - "time": "2018-07-26T09:03:18+00:00" + "time": "2018-07-26T09:06:28+00:00" }, { "name": "symfony/filesystem", @@ -4800,22 +4803,22 @@ }, { "name": "guzzle/guzzle", - "version": "v3.9.3", + "version": "v3.8.1", "source": { "type": "git", - "url": "https://github.com/guzzle/guzzle3.git", - "reference": "0645b70d953bc1c067bbc8d5bc53194706b628d9" + "url": "https://github.com/guzzle/guzzle.git", + "reference": "4de0618a01b34aa1c8c33a3f13f396dcd3882eba" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/guzzle/guzzle3/zipball/0645b70d953bc1c067bbc8d5bc53194706b628d9", - "reference": "0645b70d953bc1c067bbc8d5bc53194706b628d9", + "url": "https://api.github.com/repos/guzzle/guzzle/zipball/4de0618a01b34aa1c8c33a3f13f396dcd3882eba", + "reference": "4de0618a01b34aa1c8c33a3f13f396dcd3882eba", "shasum": "" }, "require": { "ext-curl": "*", "php": ">=5.3.3", - "symfony/event-dispatcher": "~2.1" + "symfony/event-dispatcher": ">=2.1" }, "replace": { "guzzle/batch": "self.version", @@ -4842,21 +4845,18 @@ "guzzle/stream": "self.version" }, "require-dev": { - "doctrine/cache": "~1.3", - "monolog/monolog": "~1.0", + "doctrine/cache": "*", + "monolog/monolog": "1.*", "phpunit/phpunit": "3.7.*", - "psr/log": "~1.0", - "symfony/class-loader": "~2.1", - "zendframework/zend-cache": "2.*,<2.3", - "zendframework/zend-log": "2.*,<2.3" - }, - "suggest": { - "guzzlehttp/guzzle": "Guzzle 5 has moved to a new package name. The package you have installed, Guzzle 3, is deprecated." + "psr/log": "1.0.*", + "symfony/class-loader": "*", + "zendframework/zend-cache": "<2.3", + "zendframework/zend-log": "<2.3" }, "type": "library", "extra": { "branch-alias": { - "dev-master": "3.9-dev" + "dev-master": "3.8-dev" } }, "autoload": { @@ -4880,7 +4880,7 @@ "homepage": "https://github.com/guzzle/guzzle/contributors" } ], - "description": "PHP HTTP client. This library is deprecated in favor of https://packagist.org/packages/guzzlehttp/guzzle", + "description": "Guzzle is a PHP HTTP client library and framework for building RESTful web service clients", "homepage": "http://guzzlephp.org/", "keywords": [ "client", @@ -4892,7 +4892,7 @@ "web service" ], "abandoned": "guzzlehttp/guzzle", - "time": "2015-03-18T18:23:50+00:00" + "time": "2014-01-28T22:29:15+00:00" }, { "name": "jakubledl/dissect", diff --git a/dev/tests/verification/Resources/functionalSuiteHooks.txt b/dev/tests/verification/Resources/functionalSuiteHooks.txt index 8ec37451c..b85f9e578 100644 --- a/dev/tests/verification/Resources/functionalSuiteHooks.txt +++ b/dev/tests/verification/Resources/functionalSuiteHooks.txt @@ -59,11 +59,12 @@ class functionalSuiteHooks extends \Codeception\GroupObject $this->create = new DataPersistenceHandler($createThis, [], $createFields); $this->create->createEntity(); $webDriver->see("John", msq("uniqueData") . "John"); - + // reset configuration and close session $this->getModule('\Magento\FunctionalTestingFramework\Module\MagentoWebDriver')->_resetConfig(); $webDriver->webDriver->close(); $webDriver->webDriver = null; + } catch (\Exception $exception) { $this->preconditionFailure = $exception->getMessage(); } @@ -74,16 +75,37 @@ class functionalSuiteHooks extends \Codeception\GroupObject public function _after(\Codeception\Event\TestEvent $e) { - $this->executePostConditions(); + $this->executePostConditions($e); } - private function executePostConditions() + private function executePostConditions(\Codeception\Event\TestEvent $e) { if ($this->currentTestRun == $this->testCount) { print sprintf(self::$HOOK_EXECUTION_INIT, "after"); try { + // Find out if Test in Suite failed, will cause potential failures in suite after + $cest = $e->getTest(); + + //Access private TestResultObject to find stack and if there are any errors (as opposed to failures) + $testResultObject = call_user_func(\Closure::bind( + function () use ($cest) { + return $cest->getTestResultObject(); + }, + $cest + )); + $errors = $testResultObject->errors(); + + if (!empty($errors)) { + foreach ($errors as $error) { + if ($error->failedTest()->getTestMethod() == $cest->getName()) { + // Do not attempt to run _after if failure was in the _after block + // Try to run _after but catch exceptions to prevent them from overwriting original failure. + print("LAST TEST IN SUITE FAILED, TEST AFTER MAY NOT BE SUCCESSFUL\n"); + } + } + } $webDriver = $this->getModule('\Magento\FunctionalTestingFramework\Module\MagentoWebDriver'); // close any open sessions @@ -97,11 +119,6 @@ class functionalSuiteHooks extends \Codeception\GroupObject $webDriver->amOnPage("some.url"); $webDriver->deleteEntityByUrl("deleteThis"); $webDriver->see("John", msq("uniqueData") . "John"); - - // reset configuration and close session - $this->getModule('\Magento\FunctionalTestingFramework\Module\MagentoWebDriver')->_resetConfig(); - $webDriver->webDriver->close(); - $webDriver->webDriver = null; } catch (\Exception $exception) { print $exception->getMessage(); } diff --git a/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php b/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php index c0bf78f19..d6f365d99 100644 --- a/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php +++ b/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php @@ -8,6 +8,7 @@ use \Codeception\Events; use Magento\FunctionalTestingFramework\Extension\ErrorLogger; +use Magento\FunctionalTestingFramework\Module\MagentoWebDriver; /** * Class TestContextExtension @@ -16,6 +17,9 @@ class TestContextExtension extends \Codeception\Extension { const TEST_PHASE_AFTER = "_after"; + // @codingStandardsIgnoreStart + const MAGENTO_WEB_DRIVER_CLASS = "\Magento\FunctionalTestingFramework\Module\MagentoWebDriver"; + // @codingStandardsIgnoreEnd /** * Codeception Events Mapping to methods @@ -42,7 +46,7 @@ public function testFail(\Codeception\Event\FailEvent $e) $this->runAfterBlock($e, $cest); } } - + /** * Codeception event listener function, triggered on test error. * @param \Codeception\Event\TestEvent $e @@ -61,14 +65,21 @@ function () use ($cest) { )); $errors = $testResultObject->errors(); if (!empty($errors)) { - $stack = $errors[0]->thrownException()->getTrace(); - $context = $this->extractContext($stack, $cest->getTestMethod()); - // Do not attempt to run _after if failure was in the _after block - // Try to run _after but catch exceptions to prevent them from overwriting original failure. - if ($context != TestContextExtension::TEST_PHASE_AFTER) { - $this->runAfterBlock($e, $cest); + foreach ($errors as $error) { + if ($error->failedTest()->getTestMethod() == $cest->getName()) { + $stack = $errors[0]->thrownException()->getTrace(); + $context = $this->extractContext($stack, $cest->getTestMethod()); + // Do not attempt to run _after if failure was in the _after block + // Try to run _after but catch exceptions to prevent them from overwriting original failure. + if ($context != TestContextExtension::TEST_PHASE_AFTER) { + $this->runAfterBlock($e, $cest); + } + continue; + } } } + // Reset Session and Cookies after all Test Runs, workaround due to functional.suite.yml restart: true + $this->getModule(self::MAGENTO_WEB_DRIVER_CLASS)->_runAfter($e->getTest()); } /** diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php index 62907fca4..269c3a60a 100644 --- a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php +++ b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php @@ -125,6 +125,27 @@ public function _resetConfig() $this->config = ConfigSanitizerUtil::sanitizeWebDriverConfig($this->config); } + /** + * Remap parent::_after, called in TestContextExtension + * @param TestInterface $test + * @return void + */ + public function _runAfter(TestInterface $test) + { + parent::_after($test); // TODO: Change the autogenerated stub + } + + /** + * Override parent::_after to do nothing. + * @return void + * @param TestInterface $test + * @SuppressWarnings(PHPMD) + */ + public function _after(TestInterface $test) + { + // DO NOT RESET SESSIONS + } + /** * Returns URL of a host. * diff --git a/src/Magento/FunctionalTestingFramework/Suite/views/SuiteClass.mustache b/src/Magento/FunctionalTestingFramework/Suite/views/SuiteClass.mustache index cee281f36..1cbe5c775 100644 --- a/src/Magento/FunctionalTestingFramework/Suite/views/SuiteClass.mustache +++ b/src/Magento/FunctionalTestingFramework/Suite/views/SuiteClass.mustache @@ -47,6 +47,12 @@ class {{suiteName}} extends \Codeception\GroupObject try { {{> testActions}} + + // reset configuration and close session + $this->getModule('\Magento\FunctionalTestingFramework\Module\MagentoWebDriver')->_resetConfig(); + $webDriver->webDriver->close(); + $webDriver->webDriver = null; + } catch (\Exception $exception) { $this->preconditionFailure = $exception->getMessage(); } @@ -59,16 +65,37 @@ class {{suiteName}} extends \Codeception\GroupObject {{#after}} public function _after(\Codeception\Event\TestEvent $e) { - $this->executePostConditions(); + $this->executePostConditions($e); } - private function executePostConditions() + private function executePostConditions(\Codeception\Event\TestEvent $e) { if ($this->currentTestRun == $this->testCount) { print sprintf(self::$HOOK_EXECUTION_INIT, "after"); try { + // Find out if Test in Suite failed, will cause potential failures in suite after + $cest = $e->getTest(); + + //Access private TestResultObject to find stack and if there are any errors (as opposed to failures) + $testResultObject = call_user_func(\Closure::bind( + function () use ($cest) { + return $cest->getTestResultObject(); + }, + $cest + )); + $errors = $testResultObject->errors(); + + if (!empty($errors)) { + foreach ($errors as $error) { + if ($error->failedTest()->getTestMethod() == $cest->getName()) { + // Do not attempt to run _after if failure was in the _after block + // Try to run _after but catch exceptions to prevent them from overwriting original failure. + print("LAST TEST IN SUITE FAILED, TEST AFTER MAY NOT BE SUCCESSFUL\n"); + } + } + } {{> testActions}} } catch (\Exception $exception) { print $exception->getMessage(); diff --git a/src/Magento/FunctionalTestingFramework/Suite/views/partials/testActions.mustache b/src/Magento/FunctionalTestingFramework/Suite/views/partials/testActions.mustache index 46951aa46..37dbcfbef 100644 --- a/src/Magento/FunctionalTestingFramework/Suite/views/partials/testActions.mustache +++ b/src/Magento/FunctionalTestingFramework/Suite/views/partials/testActions.mustache @@ -11,13 +11,6 @@ if ($webDriver->webDriver != null) { // initialize the webdriver session $webDriver->_initializeSession(); {{/webDriverInit}} -{{#webDriverReset}} - -// reset configuration and close session -$this->getModule('\Magento\FunctionalTestingFramework\Module\MagentoWebDriver')->_resetConfig(); -$webDriver->webDriver->close(); -$webDriver->webDriver = null; -{{/webDriverReset}} {{#action}} {{{action}}} {{/action}}