From f8b594e3f872fcf3c6724178afa57bc713f54041 Mon Sep 17 00:00:00 2001 From: Alex Calandra Date: Wed, 8 Aug 2018 16:59:51 -0500 Subject: [PATCH 01/10] MQE-1181: Fatal error while test running - Added null check for getDriver --- .../Extension/PageReadinessExtension.php | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Magento/FunctionalTestingFramework/Extension/PageReadinessExtension.php b/src/Magento/FunctionalTestingFramework/Extension/PageReadinessExtension.php index 9b9163a0a..1d62a7e7c 100644 --- a/src/Magento/FunctionalTestingFramework/Extension/PageReadinessExtension.php +++ b/src/Magento/FunctionalTestingFramework/Extension/PageReadinessExtension.php @@ -187,7 +187,11 @@ function () use ($metrics) { private function checkForNewPage($step) { try { - $currentUri = $this->getDriver()->_getCurrentUri(); + if (isset($this->getDriver())) { + $currentUri = $this->getDriver()->_getCurrentUri(); + } else { + throw new \Exception(); + } if ($this->uri !== $currentUri) { $this->logDebug( From 06e8605e03db56bfd7b4b7789134d5c5cda4063f Mon Sep 17 00:00:00 2001 From: Alex Calandra Date: Wed, 8 Aug 2018 17:01:48 -0500 Subject: [PATCH 02/10] MQE-1181: Fatal error while test running - Updated null check --- .../Extension/PageReadinessExtension.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Magento/FunctionalTestingFramework/Extension/PageReadinessExtension.php b/src/Magento/FunctionalTestingFramework/Extension/PageReadinessExtension.php index 1d62a7e7c..72997b9cf 100644 --- a/src/Magento/FunctionalTestingFramework/Extension/PageReadinessExtension.php +++ b/src/Magento/FunctionalTestingFramework/Extension/PageReadinessExtension.php @@ -187,7 +187,7 @@ function () use ($metrics) { private function checkForNewPage($step) { try { - if (isset($this->getDriver())) { + if (!empty($this->getDriver())) { $currentUri = $this->getDriver()->_getCurrentUri(); } else { throw new \Exception(); From 4f58adde73e7d3ac4c0938fa1e0b88fe5a30a919 Mon Sep 17 00:00:00 2001 From: Alex Calandra Date: Wed, 8 Aug 2018 17:09:31 -0500 Subject: [PATCH 03/10] MQE-1181: Fatal error while test running - Added null check for error on after step --- .../Extension/TestContextExtension.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php b/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php index c0bf78f19..95dd5d8b4 100644 --- a/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php +++ b/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php @@ -122,6 +122,8 @@ public function afterStep(\Codeception\Event\StepEvent $e) // @codingStandardsIgnoreStart $webDriver = $this->getModule("\Magento\FunctionalTestingFramework\Module\MagentoWebDriver")->webDriver; // @codingStandardsIgnoreEnd - ErrorLogger::getInstance()->logErrors($webDriver, $e); + if (!empty($webDriver)) { + ErrorLogger::getInstance()->logErrors($webDriver, $e); + } } } From 7a77f17db79ff5101a6fc431ec0615b323510002 Mon Sep 17 00:00:00 2001 From: Alex Calandra Date: Thu, 9 Aug 2018 10:55:57 -0500 Subject: [PATCH 04/10] MQE-1181: Fatal error while test running - Removed catches, removed line from functional suite yml --- etc/config/functional.suite.dist.yml | 1 - .../Extension/PageReadinessExtension.php | 6 +----- .../Extension/TestContextExtension.php | 4 +--- 3 files changed, 2 insertions(+), 9 deletions(-) diff --git a/etc/config/functional.suite.dist.yml b/etc/config/functional.suite.dist.yml index da05a13e3..416ce24f7 100644 --- a/etc/config/functional.suite.dist.yml +++ b/etc/config/functional.suite.dist.yml @@ -28,7 +28,6 @@ modules: url: "%MAGENTO_BASE_URL%" backend_name: "%MAGENTO_BACKEND_NAME%" browser: 'chrome' - restart: true window_size: 1280x1024 username: "%MAGENTO_ADMIN_USERNAME%" password: "%MAGENTO_ADMIN_PASSWORD%" diff --git a/src/Magento/FunctionalTestingFramework/Extension/PageReadinessExtension.php b/src/Magento/FunctionalTestingFramework/Extension/PageReadinessExtension.php index 72997b9cf..9b9163a0a 100644 --- a/src/Magento/FunctionalTestingFramework/Extension/PageReadinessExtension.php +++ b/src/Magento/FunctionalTestingFramework/Extension/PageReadinessExtension.php @@ -187,11 +187,7 @@ function () use ($metrics) { private function checkForNewPage($step) { try { - if (!empty($this->getDriver())) { - $currentUri = $this->getDriver()->_getCurrentUri(); - } else { - throw new \Exception(); - } + $currentUri = $this->getDriver()->_getCurrentUri(); if ($this->uri !== $currentUri) { $this->logDebug( diff --git a/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php b/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php index 95dd5d8b4..c0bf78f19 100644 --- a/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php +++ b/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php @@ -122,8 +122,6 @@ public function afterStep(\Codeception\Event\StepEvent $e) // @codingStandardsIgnoreStart $webDriver = $this->getModule("\Magento\FunctionalTestingFramework\Module\MagentoWebDriver")->webDriver; // @codingStandardsIgnoreEnd - if (!empty($webDriver)) { - ErrorLogger::getInstance()->logErrors($webDriver, $e); - } + ErrorLogger::getInstance()->logErrors($webDriver, $e); } } From 462088ec050e61f0fa65ed5995168e6680a3d847 Mon Sep 17 00:00:00 2001 From: Kevin Kozan Date: Thu, 9 Aug 2018 14:13:05 -0500 Subject: [PATCH 05/10] MFTF 2.3.4 - Changelog and version bump - changelog entry for 2.3.4 added - version bump in composer and lock file --- CHANGELOG.md | 5 +++ bin/mftf | 2 +- composer.json | 2 +- composer.lock | 96 +++++++++++++++++++++++++-------------------------- 4 files changed, 55 insertions(+), 50 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a3de3fb0..96109b836 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,11 @@ Magento Functional Testing Framework Changelog ================================================ +2.3.4 +----- +### Fixes +* Config option `restart: true` removed from `etc/config/functional.suite.dist.yml`. + 2.3.3 ----- ### Fixes diff --git a/bin/mftf b/bin/mftf index d70ee75c3..ad9952e59 100755 --- a/bin/mftf +++ b/bin/mftf @@ -29,7 +29,7 @@ try { try { $application = new Symfony\Component\Console\Application(); $application->setName('Magento Functional Testing Framework CLI'); - $application->setVersion('2.3.0'); + $application->setVersion('2.3.4'); /** @var \Magento\FunctionalTestingFramework\Console\CommandListInterface $commandList */ $commandList = new \Magento\FunctionalTestingFramework\Console\CommandList; foreach ($commandList->getCommands() as $command) { diff --git a/composer.json b/composer.json index 2a9a826fc..c7804dcf9 100755 --- a/composer.json +++ b/composer.json @@ -2,7 +2,7 @@ "name": "magento/magento2-functional-testing-framework", "description": "Magento2 Functional Testing Framework", "type": "library", - "version": "2.3.3", + "version": "2.3.4", "license": "AGPL-3.0", "keywords": ["magento", "automation", "functional", "testing"], "config": { diff --git a/composer.lock b/composer.lock index 72254bfc9..32a599939 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "005cbbfebf014bb331ed0f1540c79475", + "content-hash": "49a38f189328c8b1a79e5787a720ac3c", "packages": [ { "name": "allure-framework/allure-codeception", @@ -349,16 +349,16 @@ }, { "name": "consolidation/config", - "version": "1.0.11", + "version": "1.1.0", "source": { "type": "git", "url": "https://github.com/consolidation/config.git", - "reference": "ede41d946078e97e7a9513aadc3352f1c26817af" + "reference": "c9fc25e9088a708637e18a256321addc0670e578" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/consolidation/config/zipball/ede41d946078e97e7a9513aadc3352f1c26817af", - "reference": "ede41d946078e97e7a9513aadc3352f1c26817af", + "url": "https://api.github.com/repos/consolidation/config/zipball/c9fc25e9088a708637e18a256321addc0670e578", + "reference": "c9fc25e9088a708637e18a256321addc0670e578", "shasum": "" }, "require": { @@ -368,7 +368,7 @@ }, "require-dev": { "g1a/composer-test-scenarios": "^1", - "phpunit/phpunit": "^4", + "phpunit/phpunit": "^5", "satooshi/php-coveralls": "^1.0", "squizlabs/php_codesniffer": "2.*", "symfony/console": "^2.5|^3|^4", @@ -399,7 +399,7 @@ } ], "description": "Provide configuration services for a commandline tool.", - "time": "2018-05-27T01:17:02+00:00" + "time": "2018-08-07T22:57:00+00:00" }, { "name": "consolidation/log", @@ -2555,16 +2555,16 @@ }, { "name": "phpunit/phpunit", - "version": "6.5.10", + "version": "6.5.11", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "5744955af9c0a2de74a5eb5287c50bf025100d39" + "reference": "7bab54cb366076023bbf457a2a0d513332cd40f2" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/5744955af9c0a2de74a5eb5287c50bf025100d39", - "reference": "5744955af9c0a2de74a5eb5287c50bf025100d39", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/7bab54cb366076023bbf457a2a0d513332cd40f2", + "reference": "7bab54cb366076023bbf457a2a0d513332cd40f2", "shasum": "" }, "require": { @@ -2582,7 +2582,7 @@ "phpunit/php-file-iterator": "^1.4.3", "phpunit/php-text-template": "^1.2.1", "phpunit/php-timer": "^1.0.9", - "phpunit/phpunit-mock-objects": "^5.0.8", + "phpunit/phpunit-mock-objects": "^5.0.9", "sebastian/comparator": "^2.1", "sebastian/diff": "^2.0", "sebastian/environment": "^3.1", @@ -2635,20 +2635,20 @@ "testing", "xunit" ], - "time": "2018-08-03T05:27:14+00:00" + "time": "2018-08-07T07:05:35+00:00" }, { "name": "phpunit/phpunit-mock-objects", - "version": "5.0.8", + "version": "5.0.10", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit-mock-objects.git", - "reference": "6f9a3c8bf34188a2b53ce2ae7a126089c53e0a9f" + "reference": "cd1cf05c553ecfec36b170070573e540b67d3f1f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit-mock-objects/zipball/6f9a3c8bf34188a2b53ce2ae7a126089c53e0a9f", - "reference": "6f9a3c8bf34188a2b53ce2ae7a126089c53e0a9f", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit-mock-objects/zipball/cd1cf05c553ecfec36b170070573e540b67d3f1f", + "reference": "cd1cf05c553ecfec36b170070573e540b67d3f1f", "shasum": "" }, "require": { @@ -2661,7 +2661,7 @@ "phpunit/phpunit": "<6.0" }, "require-dev": { - "phpunit/phpunit": "^6.5" + "phpunit/phpunit": "^6.5.11" }, "suggest": { "ext-soap": "*" @@ -2694,7 +2694,7 @@ "mock", "xunit" ], - "time": "2018-07-13T03:27:23+00:00" + "time": "2018-08-09T05:50:03+00:00" }, { "name": "psr/container", @@ -3777,27 +3777,30 @@ }, { "name": "symfony/event-dispatcher", - "version": "v2.8.44", + "version": "v3.4.14", "source": { "type": "git", "url": "https://github.com/symfony/event-dispatcher.git", - "reference": "84ae343f39947aa084426ed1138bb96bf94d1f12" + "reference": "b2e1f19280c09a42dc64c0b72b80fe44dd6e88fb" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/event-dispatcher/zipball/84ae343f39947aa084426ed1138bb96bf94d1f12", - "reference": "84ae343f39947aa084426ed1138bb96bf94d1f12", + "url": "https://api.github.com/repos/symfony/event-dispatcher/zipball/b2e1f19280c09a42dc64c0b72b80fe44dd6e88fb", + "reference": "b2e1f19280c09a42dc64c0b72b80fe44dd6e88fb", "shasum": "" }, "require": { - "php": ">=5.3.9" + "php": "^5.5.9|>=7.0.8" + }, + "conflict": { + "symfony/dependency-injection": "<3.3" }, "require-dev": { "psr/log": "~1.0", - "symfony/config": "^2.0.5|~3.0.0", - "symfony/dependency-injection": "~2.6|~3.0.0", - "symfony/expression-language": "~2.6|~3.0.0", - "symfony/stopwatch": "~2.3|~3.0.0" + "symfony/config": "~2.8|~3.0|~4.0", + "symfony/dependency-injection": "~3.3|~4.0", + "symfony/expression-language": "~2.8|~3.0|~4.0", + "symfony/stopwatch": "~2.8|~3.0|~4.0" }, "suggest": { "symfony/dependency-injection": "", @@ -3806,7 +3809,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-master": "2.8-dev" + "dev-master": "3.4-dev" } }, "autoload": { @@ -3833,7 +3836,7 @@ ], "description": "Symfony EventDispatcher Component", "homepage": "https://symfony.com", - "time": "2018-07-26T09:03:18+00:00" + "time": "2018-07-26T09:06:28+00:00" }, { "name": "symfony/filesystem", @@ -4800,22 +4803,22 @@ }, { "name": "guzzle/guzzle", - "version": "v3.9.3", + "version": "v3.8.1", "source": { "type": "git", - "url": "https://github.com/guzzle/guzzle3.git", - "reference": "0645b70d953bc1c067bbc8d5bc53194706b628d9" + "url": "https://github.com/guzzle/guzzle.git", + "reference": "4de0618a01b34aa1c8c33a3f13f396dcd3882eba" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/guzzle/guzzle3/zipball/0645b70d953bc1c067bbc8d5bc53194706b628d9", - "reference": "0645b70d953bc1c067bbc8d5bc53194706b628d9", + "url": "https://api.github.com/repos/guzzle/guzzle/zipball/4de0618a01b34aa1c8c33a3f13f396dcd3882eba", + "reference": "4de0618a01b34aa1c8c33a3f13f396dcd3882eba", "shasum": "" }, "require": { "ext-curl": "*", "php": ">=5.3.3", - "symfony/event-dispatcher": "~2.1" + "symfony/event-dispatcher": ">=2.1" }, "replace": { "guzzle/batch": "self.version", @@ -4842,21 +4845,18 @@ "guzzle/stream": "self.version" }, "require-dev": { - "doctrine/cache": "~1.3", - "monolog/monolog": "~1.0", + "doctrine/cache": "*", + "monolog/monolog": "1.*", "phpunit/phpunit": "3.7.*", - "psr/log": "~1.0", - "symfony/class-loader": "~2.1", - "zendframework/zend-cache": "2.*,<2.3", - "zendframework/zend-log": "2.*,<2.3" - }, - "suggest": { - "guzzlehttp/guzzle": "Guzzle 5 has moved to a new package name. The package you have installed, Guzzle 3, is deprecated." + "psr/log": "1.0.*", + "symfony/class-loader": "*", + "zendframework/zend-cache": "<2.3", + "zendframework/zend-log": "<2.3" }, "type": "library", "extra": { "branch-alias": { - "dev-master": "3.9-dev" + "dev-master": "3.8-dev" } }, "autoload": { @@ -4880,7 +4880,7 @@ "homepage": "https://github.com/guzzle/guzzle/contributors" } ], - "description": "PHP HTTP client. This library is deprecated in favor of https://packagist.org/packages/guzzlehttp/guzzle", + "description": "Guzzle is a PHP HTTP client library and framework for building RESTful web service clients", "homepage": "http://guzzlephp.org/", "keywords": [ "client", @@ -4892,7 +4892,7 @@ "web service" ], "abandoned": "guzzlehttp/guzzle", - "time": "2015-03-18T18:23:50+00:00" + "time": "2014-01-28T22:29:15+00:00" }, { "name": "jakubledl/dissect", From a75cc902b101f7540c52dee413b42404d8b6edc4 Mon Sep 17 00:00:00 2001 From: Kevin Kozan Date: Fri, 10 Aug 2018 09:36:01 -0500 Subject: [PATCH 06/10] MQE-1181: Fatal error while test running - reintroduced functional.suite.dist.yml restart: true - MagentoWebDriver overwrites parent _after function and remaps it (to be called in TestContextExtension --- CHANGELOG.md | 2 +- etc/config/functional.suite.dist.yml | 1 + .../Extension/TestContextExtension.php | 3 +++ .../Module/MagentoWebDriver.php | 19 +++++++++++++++++++ 4 files changed, 24 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 96109b836..cf798da9f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ Magento Functional Testing Framework Changelog 2.3.4 ----- ### Fixes -* Config option `restart: true` removed from `etc/config/functional.suite.dist.yml`. +* MagentoWebDriver overrides `parent::_after()` function and remaps to `runAfter()`, necessary to solve compatibility issues in Codeception `2.3.x`. 2.3.3 ----- diff --git a/etc/config/functional.suite.dist.yml b/etc/config/functional.suite.dist.yml index 416ce24f7..da05a13e3 100644 --- a/etc/config/functional.suite.dist.yml +++ b/etc/config/functional.suite.dist.yml @@ -28,6 +28,7 @@ modules: url: "%MAGENTO_BASE_URL%" backend_name: "%MAGENTO_BACKEND_NAME%" browser: 'chrome' + restart: true window_size: 1280x1024 username: "%MAGENTO_ADMIN_USERNAME%" password: "%MAGENTO_ADMIN_PASSWORD%" diff --git a/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php b/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php index c0bf78f19..06b222424 100644 --- a/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php +++ b/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php @@ -8,6 +8,7 @@ use \Codeception\Events; use Magento\FunctionalTestingFramework\Extension\ErrorLogger; +use Magento\FunctionalTestingFramework\Module\MagentoWebDriver; /** * Class TestContextExtension @@ -89,6 +90,8 @@ function () use ($cest, $I) { null, $cest )); + // Reset Session and Cookies, workaround due to functional.suite.yml restart: true + $this->getModule(MagentoWebDriver::class)->_runAfter($e->getTest()); } catch (\Exception $e) { // Do not rethrow Exception } diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php index 62907fca4..eb7c2bb30 100644 --- a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php +++ b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php @@ -125,6 +125,25 @@ public function _resetConfig() $this->config = ConfigSanitizerUtil::sanitizeWebDriverConfig($this->config); } + /** + * Remap parent::_after, called in TestContextExtension + * @param TestInterface $test + * @return void + */ + public function _runAfter(TestInterface $test) + { + parent::_after($test); // TODO: Change the autogenerated stub + } + + /** + * Override parent::_after to do nothing. + * @return void + */ + public function _after() + { + // DO NOT RESET SESSIONS + } + /** * Returns URL of a host. * From 17f0153871c0b437e32a770e94b2402a12837ce7 Mon Sep 17 00:00:00 2001 From: Kevin Kozan Date: Fri, 10 Aug 2018 09:52:32 -0500 Subject: [PATCH 07/10] MQE-1181: Fatal error while test running - Static check ignore for PHPMD --- .../FunctionalTestingFramework/Module/MagentoWebDriver.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php index eb7c2bb30..269c3a60a 100644 --- a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php +++ b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php @@ -138,8 +138,10 @@ public function _runAfter(TestInterface $test) /** * Override parent::_after to do nothing. * @return void + * @param TestInterface $test + * @SuppressWarnings(PHPMD) */ - public function _after() + public function _after(TestInterface $test) { // DO NOT RESET SESSIONS } From 57882535a697c3cb9f259a396ad6f5bf04d5f15a Mon Sep 17 00:00:00 2001 From: Kevin Kozan Date: Fri, 10 Aug 2018 11:53:07 -0500 Subject: [PATCH 08/10] MQE-1181: Fatal error while test running - Moved reset session to after all test runs - changed suites to not run suite_after if last test was a failure (compatibility issues) --- .../Extension/TestContextExtension.php | 26 ++++++++++++------- .../Suite/views/SuiteClass.mustache | 26 +++++++++++++++++-- 2 files changed, 41 insertions(+), 11 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php b/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php index 06b222424..d6f365d99 100644 --- a/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php +++ b/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php @@ -17,6 +17,9 @@ class TestContextExtension extends \Codeception\Extension { const TEST_PHASE_AFTER = "_after"; + // @codingStandardsIgnoreStart + const MAGENTO_WEB_DRIVER_CLASS = "\Magento\FunctionalTestingFramework\Module\MagentoWebDriver"; + // @codingStandardsIgnoreEnd /** * Codeception Events Mapping to methods @@ -43,7 +46,7 @@ public function testFail(\Codeception\Event\FailEvent $e) $this->runAfterBlock($e, $cest); } } - + /** * Codeception event listener function, triggered on test error. * @param \Codeception\Event\TestEvent $e @@ -62,14 +65,21 @@ function () use ($cest) { )); $errors = $testResultObject->errors(); if (!empty($errors)) { - $stack = $errors[0]->thrownException()->getTrace(); - $context = $this->extractContext($stack, $cest->getTestMethod()); - // Do not attempt to run _after if failure was in the _after block - // Try to run _after but catch exceptions to prevent them from overwriting original failure. - if ($context != TestContextExtension::TEST_PHASE_AFTER) { - $this->runAfterBlock($e, $cest); + foreach ($errors as $error) { + if ($error->failedTest()->getTestMethod() == $cest->getName()) { + $stack = $errors[0]->thrownException()->getTrace(); + $context = $this->extractContext($stack, $cest->getTestMethod()); + // Do not attempt to run _after if failure was in the _after block + // Try to run _after but catch exceptions to prevent them from overwriting original failure. + if ($context != TestContextExtension::TEST_PHASE_AFTER) { + $this->runAfterBlock($e, $cest); + } + continue; + } } } + // Reset Session and Cookies after all Test Runs, workaround due to functional.suite.yml restart: true + $this->getModule(self::MAGENTO_WEB_DRIVER_CLASS)->_runAfter($e->getTest()); } /** @@ -90,8 +100,6 @@ function () use ($cest, $I) { null, $cest )); - // Reset Session and Cookies, workaround due to functional.suite.yml restart: true - $this->getModule(MagentoWebDriver::class)->_runAfter($e->getTest()); } catch (\Exception $e) { // Do not rethrow Exception } diff --git a/src/Magento/FunctionalTestingFramework/Suite/views/SuiteClass.mustache b/src/Magento/FunctionalTestingFramework/Suite/views/SuiteClass.mustache index cee281f36..cac9d28fb 100644 --- a/src/Magento/FunctionalTestingFramework/Suite/views/SuiteClass.mustache +++ b/src/Magento/FunctionalTestingFramework/Suite/views/SuiteClass.mustache @@ -59,16 +59,38 @@ class {{suiteName}} extends \Codeception\GroupObject {{#after}} public function _after(\Codeception\Event\TestEvent $e) { - $this->executePostConditions(); + $this->executePostConditions($e); } - private function executePostConditions() + private function executePostConditions(\Codeception\Event\TestEvent $e) { if ($this->currentTestRun == $this->testCount) { print sprintf(self::$HOOK_EXECUTION_INIT, "after"); try { + // Find out if Test in Suite failed, will cause potential failures in suite after + $cest = $e->getTest(); + + //Access private TestResultObject to find stack and if there are any errors (as opposed to failures) + $testResultObject = call_user_func(\Closure::bind( + function () use ($cest) { + return $cest->getTestResultObject(); + }, + $cest + )); + $errors = $testResultObject->errors(); + + if (!empty($errors)) { + foreach ($errors as $error) { + if ($error->failedTest()->getTestMethod() == $cest->getName()) { + // Do not attempt to run _after if failure was in the _after block + // Try to run _after but catch exceptions to prevent them from overwriting original failure. + \PHPUnit_Framework_Assert::fail("LAST TESTS FAILED IN SUITE, SUITE_AFTER CANNOT BE RUN"); + } + } + } + {{> testActions}} } catch (\Exception $exception) { print $exception->getMessage(); From 2353428fedf1d0883d17df1679cdc83295c8157d Mon Sep 17 00:00:00 2001 From: Kevin Kozan Date: Fri, 10 Aug 2018 12:19:41 -0500 Subject: [PATCH 09/10] MQE-1181: Fatal error while test running - Suite After no longer resets WebDriver, allows Test after to fail naturally (and Suite After to run) --- .../Suite/views/SuiteClass.mustache | 9 +++++++-- .../Suite/views/partials/testActions.mustache | 7 ------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/Suite/views/SuiteClass.mustache b/src/Magento/FunctionalTestingFramework/Suite/views/SuiteClass.mustache index cac9d28fb..1cbe5c775 100644 --- a/src/Magento/FunctionalTestingFramework/Suite/views/SuiteClass.mustache +++ b/src/Magento/FunctionalTestingFramework/Suite/views/SuiteClass.mustache @@ -47,6 +47,12 @@ class {{suiteName}} extends \Codeception\GroupObject try { {{> testActions}} + + // reset configuration and close session + $this->getModule('\Magento\FunctionalTestingFramework\Module\MagentoWebDriver')->_resetConfig(); + $webDriver->webDriver->close(); + $webDriver->webDriver = null; + } catch (\Exception $exception) { $this->preconditionFailure = $exception->getMessage(); } @@ -86,11 +92,10 @@ class {{suiteName}} extends \Codeception\GroupObject if ($error->failedTest()->getTestMethod() == $cest->getName()) { // Do not attempt to run _after if failure was in the _after block // Try to run _after but catch exceptions to prevent them from overwriting original failure. - \PHPUnit_Framework_Assert::fail("LAST TESTS FAILED IN SUITE, SUITE_AFTER CANNOT BE RUN"); + print("LAST TEST IN SUITE FAILED, TEST AFTER MAY NOT BE SUCCESSFUL\n"); } } } - {{> testActions}} } catch (\Exception $exception) { print $exception->getMessage(); diff --git a/src/Magento/FunctionalTestingFramework/Suite/views/partials/testActions.mustache b/src/Magento/FunctionalTestingFramework/Suite/views/partials/testActions.mustache index 46951aa46..37dbcfbef 100644 --- a/src/Magento/FunctionalTestingFramework/Suite/views/partials/testActions.mustache +++ b/src/Magento/FunctionalTestingFramework/Suite/views/partials/testActions.mustache @@ -11,13 +11,6 @@ if ($webDriver->webDriver != null) { // initialize the webdriver session $webDriver->_initializeSession(); {{/webDriverInit}} -{{#webDriverReset}} - -// reset configuration and close session -$this->getModule('\Magento\FunctionalTestingFramework\Module\MagentoWebDriver')->_resetConfig(); -$webDriver->webDriver->close(); -$webDriver->webDriver = null; -{{/webDriverReset}} {{#action}} {{{action}}} {{/action}} From 28a5f178e84c5a1fdb2a7d622dd615eec33986bf Mon Sep 17 00:00:00 2001 From: Kevin Kozan Date: Fri, 10 Aug 2018 13:31:56 -0500 Subject: [PATCH 10/10] MQE-1181: Fatal error while test running - Unit test fixes --- .../Resources/functionalSuiteHooks.txt | 33 ++++++++++++++----- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/dev/tests/verification/Resources/functionalSuiteHooks.txt b/dev/tests/verification/Resources/functionalSuiteHooks.txt index 8ec37451c..b85f9e578 100644 --- a/dev/tests/verification/Resources/functionalSuiteHooks.txt +++ b/dev/tests/verification/Resources/functionalSuiteHooks.txt @@ -59,11 +59,12 @@ class functionalSuiteHooks extends \Codeception\GroupObject $this->create = new DataPersistenceHandler($createThis, [], $createFields); $this->create->createEntity(); $webDriver->see("John", msq("uniqueData") . "John"); - + // reset configuration and close session $this->getModule('\Magento\FunctionalTestingFramework\Module\MagentoWebDriver')->_resetConfig(); $webDriver->webDriver->close(); $webDriver->webDriver = null; + } catch (\Exception $exception) { $this->preconditionFailure = $exception->getMessage(); } @@ -74,16 +75,37 @@ class functionalSuiteHooks extends \Codeception\GroupObject public function _after(\Codeception\Event\TestEvent $e) { - $this->executePostConditions(); + $this->executePostConditions($e); } - private function executePostConditions() + private function executePostConditions(\Codeception\Event\TestEvent $e) { if ($this->currentTestRun == $this->testCount) { print sprintf(self::$HOOK_EXECUTION_INIT, "after"); try { + // Find out if Test in Suite failed, will cause potential failures in suite after + $cest = $e->getTest(); + + //Access private TestResultObject to find stack and if there are any errors (as opposed to failures) + $testResultObject = call_user_func(\Closure::bind( + function () use ($cest) { + return $cest->getTestResultObject(); + }, + $cest + )); + $errors = $testResultObject->errors(); + + if (!empty($errors)) { + foreach ($errors as $error) { + if ($error->failedTest()->getTestMethod() == $cest->getName()) { + // Do not attempt to run _after if failure was in the _after block + // Try to run _after but catch exceptions to prevent them from overwriting original failure. + print("LAST TEST IN SUITE FAILED, TEST AFTER MAY NOT BE SUCCESSFUL\n"); + } + } + } $webDriver = $this->getModule('\Magento\FunctionalTestingFramework\Module\MagentoWebDriver'); // close any open sessions @@ -97,11 +119,6 @@ class functionalSuiteHooks extends \Codeception\GroupObject $webDriver->amOnPage("some.url"); $webDriver->deleteEntityByUrl("deleteThis"); $webDriver->see("John", msq("uniqueData") . "John"); - - // reset configuration and close session - $this->getModule('\Magento\FunctionalTestingFramework\Module\MagentoWebDriver')->_resetConfig(); - $webDriver->webDriver->close(); - $webDriver->webDriver = null; } catch (\Exception $exception) { print $exception->getMessage(); }