Merge pull request magento#104 from magento/MQE-957

MQE-957: MFTF Changes to support DEVOPS-2029

Author: imeron2433

src/Magento/FunctionalTestingFramework/Allure/Adapter/MagentoAllureAdapter.php

@@ -6,6 +6,7 @@
 namespace Magento\FunctionalTestingFramework\Allure\Adapter;
 
 use Magento\FunctionalTestingFramework\Data\Argument\Interpreter\NullType;
+use Magento\FunctionalTestingFramework\Suite\Handlers\SuiteObjectHandler;
 use Yandex\Allure\Adapter\AllureAdapter;
 use Codeception\Event\SuiteEvent;
 
@@ -44,7 +45,7 @@ public function suiteBefore(SuiteEvent $suiteEvent)
 
         if ($this->getGroup() != null) {
             $suite = $suiteEvent->getSuite();
-            $suiteName = ($suite->getName()) . "\\" . $this->getGroup();
+            $suiteName = ($suite->getName()) . "\\" . $this->sanitizeGroupName($this->getGroup());
 
             call_user_func(\Closure::bind(
                 function () use ($suite, $suiteName) {
@@ -64,4 +65,30 @@ function () use ($suite, $suiteName) {
         // call parent function
         parent::suiteBefore($changeSuiteEvent);
     }
+
+    /**
+     * Function which santizes any group names changed by the framework for execution in order to consolidate reporting.
+     *
+     * @param string $group
+     * @return string
+     */
+    private function sanitizeGroupName($group)
+    {
+        $suiteNames = array_keys(SuiteObjectHandler::getInstance()->getAllObjects());
+        $exactMatch = in_array($group, $suiteNames);
+
+        // if this is an existing suite name we dont' need to worry about changing it
+        if ($exactMatch || strpos($group, "_") === false) {
+            return $group;
+        }
+
+        // if we can't find this group in the generated suites we have to assume that the group was split for generation
+        $groupNameSplit = explode("_", $group);
+        array_pop($groupNameSplit);
+        $originalName = implode("_", $groupNameSplit);
+
+        // confirm our original name is one of the existing suite names otherwise just return the original group name
+        $originalName = in_array($originalName, $suiteNames) ? $originalName : $group;
+        return $originalName;
+    }
 }

src/Magento/FunctionalTestingFramework/Suite/views/SuiteClass.mustache

@@ -17,8 +17,9 @@ use Magento\FunctionalTestingFramework\DataGenerator\Persist\DataPersistenceHand
 class {{suiteName}} extends \Codeception\GroupObject
 {
     public static $group = '{{suiteName}}';
-    private static $TEST_COUNT = {{testCount}};
-    private static $CURRENT_TEST_RUN = 0;
+    private $testCount = {{testCount}};
+    private $preconditionFailure = null;
+    private $currentTestRun = 0;
     private static $HOOK_EXECUTION_INIT = "\n/******** Beginning execution of {{suiteName}} suite %s block ********/\n";
     private static $HOOK_EXECUTION_END = "\n/******** Execution of {{suiteName}} suite %s block complete ********/\n";
     {{#var}}
@@ -29,12 +30,26 @@ class {{suiteName}} extends \Codeception\GroupObject
     public function _before(\Codeception\Event\TestEvent $e)
     {
         // increment test count per execution
-        self::$CURRENT_TEST_RUN++;
+        $this->currentTestRun++;
+        $this->executePreConditions();
 
-        if (self::$CURRENT_TEST_RUN == 1) {
+        if ($this->preconditionFailure != null) {
+            //if our preconditions fail, we need to mark all the tests as incomplete.
+            $e->getTest()->getMetadata()->setIncomplete($this->preconditionFailure);
+        }
+    }
+
+
+    private function executePreConditions()
+    {
+        if ($this->currentTestRun == 1) {
             print sprintf(self::$HOOK_EXECUTION_INIT, "before");
 
-            {{> testActions}}
+            try {
+                {{> testActions}}
+            } catch (\Exception $exception) {
+                $this->preconditionFailure = $exception->getMessage();
+            }
 
             print sprintf(self::$HOOK_EXECUTION_END, "before");
         }
@@ -44,10 +59,20 @@ class {{suiteName}} extends \Codeception\GroupObject
     {{#after}}
     public function _after(\Codeception\Event\TestEvent $e)
     {
-        if (self::$CURRENT_TEST_RUN == self::$TEST_COUNT) {
+        $this->executePostConditions();
+    }
+
+
+    private function executePostConditions()
+    {
+        if ($this->currentTestRun == $this->testCount) {
             print sprintf(self::$HOOK_EXECUTION_INIT, "after");
 
-            {{> testActions}}
+            try {
+                {{> testActions}}
+            } catch (\Exception $exception) {
+                print $exception->getMessage();
+            }
 
             print sprintf(self::$HOOK_EXECUTION_END, "after");
         }

src/Magento/FunctionalTestingFramework/Suite/views/partials/testActions.mustache

@@ -10,10 +10,9 @@ if ($webDriver->webDriver != null) {
 
 // initialize the webdriver session
 $webDriver->_initializeSession();
-
-// execute user specified actions
 {{/webDriverInit}}
 {{#webDriverReset}}
+
 // reset configuration and close session
 $this->getModule('\Magento\FunctionalTestingFramework\Module\MagentoWebDriver')->_resetConfig();
 $webDriver->webDriver->close();

src/Magento/FunctionalTestingFramework/Test/Objects/TestObject.php

@@ -132,6 +132,10 @@ public function getAnnotations()
      */
     public function getHooks()
     {
+        // if this test is skipped we do not want any before/after actions to generate as the tests will not run
+        if ($this->isSkipped()) {
+            return [];
+        }
         return $this->hooks;
     }
 
@@ -142,6 +146,11 @@ public function getHooks()
      */
     public function getTestActionCount()
     {
+        // a skipped action results in a single skip being appended to the beginning of the test and no execution
+        if ($this->isSkipped()) {
+            return 1;
+        }
+
         $hookActions = 0;
         if (array_key_exists('before', $this->hooks)) {
             $hookActions += count($this->hooks['before']->getActions());
@@ -152,7 +161,6 @@ public function getTestActionCount()
         }
 
         $testActions = count($this->getOrderedActions());
-
         return $hookActions + $testActions;
     }