Merge remote-tracking branch 'origin/MFTF3.0.0-RC2' into MQE-2077

# Conflicts:
#	etc/di.xml

Author: jilu1

dev/tests/functional/standalone_bootstrap.php

@@ -53,6 +53,9 @@
     defined('WAIT_TIMEOUT') || define('WAIT_TIMEOUT', 30);
     $env->setEnvironmentVariable('WAIT_TIMEOUT', WAIT_TIMEOUT);
 
+    defined('VERBOSE_ARTIFACTS') || define('VERBOSE_ARTIFACTS', false);
+    $env->setEnvironmentVariable('VERBOSE_ARTIFACTS', VERBOSE_ARTIFACTS);
+
     try {
         new DateTimeZone(DEFAULT_TIMEZONE);
     } catch (\Exception $e) {

dev/tests/phpunit.xml

@@ -7,7 +7,7 @@
 
 <phpunit
     xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-    xsi:noNamespaceSchemaLocation="https://schema.phpunit.de/6.3/phpunit.xsd"
+    xsi:noNamespaceSchemaLocation="https://schema.phpunit.de/9.1/phpunit.xsd"
     convertNoticesToExceptions="false"
     bootstrap="_bootstrap.php"
     backupGlobals="false">

dev/tests/unit/Magento/FunctionalTestFramework/Util/Sorter/ParallelGroupSorterTest.php

@@ -101,8 +101,8 @@ public function testSortWithSuites()
         $this->assertCount(5, $actualResult);
 
         $expectedResults =  [
-            1 => ['mockSuite1_0'],
-            2 => ['mockSuite1_1'],
+            1 => ['mockSuite1_0_G'],
+            2 => ['mockSuite1_1_G'],
             3 => ['test3'],
             4 => ['test2','test5', 'test4'],
             5 => ['test1'],

dev/tests/verification/Resources/AssertTest.txt

@@ -65,7 +65,7 @@ class AssertTestCest
 		$I->assertStringNotContainsStringIgnoringCase("apple", "banana", "pass"); // stepKey: assertStringNotContainsStringIgnoringCase
 		$I->assertNotEmpty([1, 2], "pass"); // stepKey: assertNotEmpty1
 		$I->assertNotEmpty($text, "pass"); // stepKey: assertNotEmpty2
-		$I->assertNotEquals(2, 5, "pass", 0); // stepKey: assertNotEquals
+		$I->assertNotEquals(2, 5, "pass"); // stepKey: assertNotEquals
 		$I->assertNotNull("abc", "pass"); // stepKey: assertNotNull1
 		$I->assertNotNull($text, "pass"); // stepKey: assertNotNull2
 		$I->assertNotRegExp("/foo/", "bar", "pass"); // stepKey: assertNotRegExp
@@ -97,7 +97,7 @@ class AssertTestCest
 		$I->assertNotContains("bc", $text, "pass"); // stepKey: assertNotContains2BackwardCompatible
 		$I->assertNotEmpty([1, 2], "pass"); // stepKey: assertNotEmpty1BackwardCompatible
 		$I->assertNotEmpty($text, "pass"); // stepKey: assertNotEmpty2BackwardCompatible
-		$I->assertNotEquals(2, 5, "pass", 0); // stepKey: assertNotEqualsBackwardCompatible
+		$I->assertNotEquals(2, 5, "pass"); // stepKey: assertNotEqualsBackwardCompatible
 		$I->assertNotNull("abc", "pass"); // stepKey: assertNotNull1BackwardCompatible
 		$I->assertNotNull($text, "pass"); // stepKey: assertNotNull2BackwardCompatible
 		$I->assertNotRegExp("/foo/", "bar", "pass"); // stepKey: assertNotRegExpBackwardCompatible
@@ -145,5 +145,11 @@ class AssertTestCest
 		$I->assertElementContainsAttribute("#username", "value", $I->retrieveEntityField('createData1', 'firstname', 'test')); // stepKey: assertElementContainsAttribute8
 		$I->comment("assert entity resolution");
 		$I->assertEquals("John", "Doe", "pass"); // stepKey: assertEqualsEntity
+		$I->assertEqualsWithDelta(10.0000, 10.0000, 1, "pass"); // stepKey: a1
+		$I->assertNotEqualsWithDelta(10.0000, 12.0000, 1, "pass"); // stepKey: a2
+		$I->assertEqualsCanonicalizing(["4", "2", "1", "3"], ["1", "2", "3", "4"], "pass"); // stepKey: a3
+		$I->assertNotEqualsCanonicalizing(["5", "8", "7", "9"], ["1", "2", "3", "4"], "pass"); // stepKey: a4
+		$I->assertEqualsIgnoringCase("Cat", "cat", "pass"); // stepKey: a5
+		$I->assertNotEqualsIgnoringCase("Cat", "Dog", "pass"); // stepKey: a6
 	}
 }

dev/tests/verification/TestModule/Test/AssertTest.xml

@@ -113,7 +113,7 @@
         <assertNotEmpty stepKey="assertNotEmpty2" message="pass">
             <actualResult type="variable">text</actualResult>
         </assertNotEmpty>
-        <assertNotEquals stepKey="assertNotEquals" message="pass" delta="">
+        <assertNotEquals stepKey="assertNotEquals" message="pass">
             <expectedResult type="int">2</expectedResult>
             <actualResult type="int">5</actualResult>
         </assertNotEquals>
@@ -227,7 +227,7 @@
         <assertNotEmpty stepKey="assertNotEmpty2BackwardCompatible" message="pass">
             <actualResult type="variable">text</actualResult>
         </assertNotEmpty>
-        <assertNotEquals stepKey="assertNotEqualsBackwardCompatible" message="pass" delta="">
+        <assertNotEquals stepKey="assertNotEqualsBackwardCompatible" message="pass">
             <actualResult type="int">5</actualResult>
             <expectedResult type="int">2</expectedResult>
         </assertNotEquals>
@@ -375,5 +375,30 @@
             <expectedResult type="string">{{simpleData.firstname}}</expectedResult>
             <actualResult type="string">{{simpleData.lastname}}</actualResult>
         </assertEquals>
+
+        <assertEqualsWithDelta stepKey="a1" message="pass" delta="1">
+            <expectedResult type="const">10.0000</expectedResult>
+            <actualResult type="const">10.0000</actualResult>
+        </assertEqualsWithDelta>
+        <assertNotEqualsWithDelta stepKey="a2" message="pass" delta="1">
+            <expectedResult type="const">10.0000</expectedResult>
+            <actualResult type="const">12.0000</actualResult>
+        </assertNotEqualsWithDelta>
+        <assertEqualsCanonicalizing stepKey="a3" message="pass">
+            <expectedResult type="array">[4, 2, 1, 3]</expectedResult>
+            <actualResult type="array">[1, 2, 3, 4]</actualResult>
+        </assertEqualsCanonicalizing>
+        <assertNotEqualsCanonicalizing stepKey="a4" message="pass">
+            <expectedResult type="array">[5, 8, 7, 9]</expectedResult>
+            <actualResult type="array">[1, 2, 3, 4]</actualResult>
+        </assertNotEqualsCanonicalizing>
+        <assertEqualsIgnoringCase stepKey="a5" message="pass">
+            <expectedResult type="string">Cat</expectedResult>
+            <actualResult type="string">cat</actualResult>
+        </assertEqualsIgnoringCase>
+        <assertNotEqualsIgnoringCase stepKey="a6" message="pass">
+            <expectedResult type="string">Cat</expectedResult>
+            <actualResult type="string">Dog</actualResult>
+        </assertNotEqualsIgnoringCase>
     </test>
 </tests>

dev/tests/verification/Tests/SuiteGenerationTest.php

@@ -118,10 +118,10 @@ public function testSuiteGenerationParallel()
         $groupName = 'functionalSuite1';
 
         $expectedGroups = [
-            'functionalSuite1_0',
-            'functionalSuite1_1',
-            'functionalSuite1_2',
-            'functionalSuite1_3'
+            'functionalSuite1_0_G',
+            'functionalSuite1_1_G',
+            'functionalSuite1_2_G',
+            'functionalSuite1_3_G'
         ];
 
         $expectedContents = [

docs/configuration.md

@@ -299,6 +299,20 @@ Example:
 CREDENTIAL_AWS_SECRETS_MANAGER_PROFILE=default
 ```
 
+### VERBOSE_ARTIFACTS
+
+Determines if passed tests should still have all their Allure artifacts. These artifacts include `.txt` attachments for `dontSee` actions and `createData` actions.
+
+If enabled, all tests will have all of their normal Allure artifacts.
+
+If disabled, passed tests will have their Allure artifacts trimmed. Failed tests will still contain all their artifacts.
+
+This is set `false` by default.
+
+```conf
+VERBOSE_ARTIFACTS=true
+```
+
 ### ENABLE_BROWSER_LOG
 
 Enables addition of browser logs to Allure steps

docs/test/assertions.md

@@ -167,6 +167,17 @@ Attribute|Type|Use|Description
 
 See [assertEquals docs on codeception.com](http://codeception.com/docs/modules/Asserts#assertEquals).
 
+Attribute|Type|Use|Description
+---|---|---|---
+`message`|string|optional|Text of informational message about a cause of failure.
+`stepKey`|string|required| A unique identifier of the text step.
+`before`|string|optional| `stepKey` of action that must be executed next.
+`after`|string|optional| `stepKey` of the preceding action.
+
+### assertEqualsWithDelta
+
+See [assertEqualsWithDelta docs on codeception.com](http://codeception.com/docs/modules/Asserts#assertEqualsWithDelta).
+
 Attribute|Type|Use|Description
 ---|---|---|---
 `delta`|string|optional|
@@ -175,6 +186,28 @@ Attribute|Type|Use|Description
 `before`|string|optional| `stepKey` of action that must be executed next.
 `after`|string|optional| `stepKey` of the preceding action.
 
+### assertEqualsCanonicalizing
+
+See [assertEqualsCanonicalizing docs on codeception.com](http://codeception.com/docs/modules/Asserts#assertEqualsCanonicalizing).
+
+Attribute|Type|Use|Description
+---|---|---|---
+`message`|string|optional|Text of informational message about a cause of failure.
+`stepKey`|string|required| A unique identifier of the text step.
+`before`|string|optional| `stepKey` of action that must be executed next.
+`after`|string|optional| `stepKey` of the preceding action.
+
+### assertEqualsIgnoringCase
+
+See [assertEqualsIgnoringCase docs on codeception.com](http://codeception.com/docs/modules/Asserts#assertEqualsIgnoringCase).
+
+Attribute|Type|Use|Description
+---|---|---|---
+`message`|string|optional|Text of informational message about a cause of failure.
+`stepKey`|string|required| A unique identifier of the text step.
+`before`|string|optional| `stepKey` of action that must be executed next.
+`after`|string|optional| `stepKey` of the preceding action.
+
 ### assertFalse
 
 See [assertFalse docs on codeception.com](http://codeception.com/docs/modules/Asserts#assertFalse).
@@ -344,6 +377,17 @@ Attribute|Type|Use|Description
 
 See [assertNotEquals docs on codeception.com](http://codeception.com/docs/modules/Asserts#assertNotEquals).
 
+Attribute|Type|Use|Description
+---|---|---|---
+`message`|string|optional|Text of informational message about a cause of failure.
+`stepKey`|string|required| A unique identifier of the text step.
+`before`|string|optional| `stepKey` of action that must be executed next.
+`after`|string|optional| `stepKey` of the preceding action.
+
+### assertNotEqualsWithDelta
+
+See [assertNotEqualsWithDelta docs on codeception.com](http://codeception.com/docs/modules/Asserts#assertNotEqualsWithDelta).
+
 Attribute|Type|Use|Description
 ---|---|---|---
 `delta`|string|optional|
@@ -352,6 +396,28 @@ Attribute|Type|Use|Description
 `before`|string|optional| `stepKey` of action that must be executed next.
 `after`|string|optional| `stepKey` of the preceding action.
 
+### assertNotEqualsCanonicalizing
+
+See [assertNotEqualsCanonicalizing docs on codeception.com](http://codeception.com/docs/modules/Asserts#assertNotEqualsCanonicalizing).
+
+Attribute|Type|Use|Description
+---|---|---|---
+`message`|string|optional|Text of informational message about a cause of failure.
+`stepKey`|string|required| A unique identifier of the text step.
+`before`|string|optional| `stepKey` of action that must be executed next.
+`after`|string|optional| `stepKey` of the preceding action.
+
+### assertNotEqualsIgnoringCase
+
+See [assertNotEqualsIgnoringCase docs on codeception.com](http://codeception.com/docs/modules/Asserts#assertNotEqualsIgnoringCase).
+
+Attribute|Type|Use|Description
+---|---|---|---
+`message`|string|optional|Text of informational message about a cause of failure.
+`stepKey`|string|required| A unique identifier of the text step.
+`before`|string|optional| `stepKey` of action that must be executed next.
+`after`|string|optional| `stepKey` of the preceding action.
+
 ### assertNotInstanceOf
 
 See [assertNotInstanceOf docs on codeception.com](http://codeception.com/docs/modules/Asserts#assertNotInstanceOf).

etc/config/.env.example

@@ -59,6 +59,9 @@ MODULE_WHITELIST=Magento_Framework,ConfigurableProductWishlist,ConfigurableProdu
 #*** Default timeout for wait actions
 #WAIT_TIMEOUT=30
 
+#*** Uncomment and set to enable all tests, regardless of passing status, to have all their Allure artifacts.
+#VERBOSE_ARTIFACTS=true
+
 #*** Uncomment and set to enable browser log entries on actions in Allure. Blacklist is used to filter logs of a specific "source"
 #ENABLE_BROWSER_LOG=true
 #BROWSER_LOG_BLACKLIST=other

etc/di.xml

@@ -8,7 +8,7 @@
 
 <!-- Entity value gets replaced in Dom.php before reading $xml -->
 <!DOCTYPE config [
-        <!ENTITY commonTestActions "acceptPopup|actionGroup|amOnPage|amOnUrl|amOnSubdomain|appendField|assertArrayIsSortasserted|assertElementContainsAttribute|attachFile|cancelPopup|checkOption|clearField|click|clickWithLeftButton|clickWithRightButton|closeAdminNotification|closeTab|comment|conditionalClick|createData|deleteData|updateData|getData|dontSee|dontSeeJsError|dontSeeCheckboxIsChecked|dontSeeCookie|dontSeeCurrentUrlEquals|dontSeeCurrentUrlMatches|dontSeeElement|dontSeeElementInDOM|dontSeeInCurrentUrl|dontSeeInField|dontSeeInFormFields|dontSeeInPageSource|dontSeeInSource|dontSeeInTitle|dontSeeLink|dontSeeOptionIsSelected|doubleClick|dragAndDrop|entity|executeJS|fillField|formatMoney|generateDate|grabAttributeFrom|grabCookie|grabFromCurrentUrl|grabMultiple|grabPageSource|grabTextFrom|grabValueFrom|loadSessionSnapshot|loginAsAdmin|magentoCLI|magentoCron|makeScreenshot|maximizeWindow|moveBack|moveForward|moveMouseOver|mSetLocale|mResetLocale|openNewTab|pause|parseFloat|pressKey|reloadPage|resetCookie|submitForm|resizeWindow|saveSessionSnapshot|scrollTo|scrollToTopOfPage|searchAndMultiSelectOption|see|seeCheckboxIsChecked|seeCookie|seeCurrentUrlEquals|seeCurrentUrlMatches|seeElement|seeElementInDOM|seeInCurrentUrl|seeInField|seeInFormFields|seeInPageSource|seeInPopup|seeInSource|seeInTitle|seeLink|seeNumberOfElements|seeOptionIsSelected|selectOption|setCookie|submitForm|switchToIFrame|switchToNextTab|switchToPreviousTab|switchToWindow|typeInPopup|uncheckOption|unselectOption|wait|waitForAjaxLoad|waitForElement|waitForElementChange|waitForElementNotVisible|waitForElementVisible|waitForPwaElementNotVisible|waitForPwaElementVisible|waitForJS|waitForLoadingMaskToDisappear|waitForPageLoad|waitForText|assertArrayHasKey|assertArrayNotHasKey|assertContains|assertStringContainsString|assertStringContainsStringIgnoringCase|assertCount|assertEmpty|assertEquals|assertFalse|assertFileExists|assertFileNotExists|assertGreaterOrEquals|assertGreaterThan|assertGreaterThanOrEqual|assertInstanceOf|assertIsEmpty|assertLessOrEquals|assertLessThan|assertLessThanOrEqual|assertNotContains|assertStringNotContainsString|assertStringNotContainsStringIgnoringCase|assertNotEmpty|assertNotEquals|assertNotInstanceOf|assertNotNull|assertNotRegExp|assertNotSame|assertNull|assertRegExp|assertSame|assertStringStartsNotWith|assertStringStartsWith|assertTrue|expectException|fail|dontSeeFullUrlEquals|dontSee|dontSeeFullUrlMatches|dontSeeInFullUrl|seeFullUrlEquals|seeFullUrlMatches|seeInFullUrl|grabFromFullUrl|helper">
+        <!ENTITY commonTestActions "acceptPopup|actionGroup|amOnPage|amOnUrl|amOnSubdomain|appendField|assertArrayIsSortasserted|assertElementContainsAttribute|attachFile|cancelPopup|checkOption|clearField|click|clickWithLeftButton|clickWithRightButton|closeAdminNotification|closeTab|comment|conditionalClick|createData|deleteData|updateData|getData|dontSee|dontSeeJsError|dontSeeCheckboxIsChecked|dontSeeCookie|dontSeeCurrentUrlEquals|dontSeeCurrentUrlMatches|dontSeeElement|dontSeeElementInDOM|dontSeeInCurrentUrl|dontSeeInField|dontSeeInFormFields|dontSeeInPageSource|dontSeeInSource|dontSeeInTitle|dontSeeLink|dontSeeOptionIsSelected|doubleClick|dragAndDrop|entity|executeJS|fillField|formatMoney|generateDate|grabAttributeFrom|grabCookie|grabFromCurrentUrl|grabMultiple|grabPageSource|grabTextFrom|grabValueFrom|loadSessionSnapshot|loginAsAdmin|magentoCLI|magentoCron|makeScreenshot|maximizeWindow|moveBack|moveForward|moveMouseOver|mSetLocale|mResetLocale|openNewTab|pause|parseFloat|pressKey|reloadPage|resetCookie|submitForm|resizeWindow|saveSessionSnapshot|scrollTo|scrollToTopOfPage|searchAndMultiSelectOption|see|seeCheckboxIsChecked|seeCookie|seeCurrentUrlEquals|seeCurrentUrlMatches|seeElement|seeElementInDOM|seeInCurrentUrl|seeInField|seeInFormFields|seeInPageSource|seeInPopup|seeInSource|seeInTitle|seeLink|seeNumberOfElements|seeOptionIsSelected|selectOption|setCookie|submitForm|switchToIFrame|switchToNextTab|switchToPreviousTab|switchToWindow|typeInPopup|uncheckOption|unselectOption|wait|waitForAjaxLoad|waitForElement|waitForElementChange|waitForElementNotVisible|waitForElementVisible|waitForPwaElementNotVisible|waitForPwaElementVisible|waitForJS|waitForLoadingMaskToDisappear|waitForPageLoad|waitForText|assertArrayHasKey|assertArrayNotHasKey|assertContains|assertStringContainsString|assertStringContainsStringIgnoringCase|assertCount|assertEmpty|assertEquals|assertFalse|assertFileExists|assertFileNotExists|assertGreaterOrEquals|assertGreaterThan|assertGreaterThanOrEqual|assertInstanceOf|assertIsEmpty|assertLessOrEquals|assertLessThan|assertLessThanOrEqual|assertNotContains|assertStringNotContainsString|assertStringNotContainsStringIgnoringCase|assertNotEmpty|assertNotEquals|assertNotInstanceOf|assertNotNull|assertNotRegExp|assertNotSame|assertNull|assertRegExp|assertSame|assertStringStartsNotWith|assertStringStartsWith|assertTrue|expectException|fail|dontSeeFullUrlEquals|dontSee|dontSeeFullUrlMatches|dontSeeInFullUrl|seeFullUrlEquals|seeFullUrlMatches|seeInFullUrl|grabFromFullUrl|helper|assertEqualsWithDelta|assertEqualsCanonicalizing|assertEqualsIgnoringCase|assertNotEqualsWithDelta|assertNotEqualsCanonicalizing|assertNotEqualsIgnoringCase">
 ]>
 
 <config xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="../src/Magento/FunctionalTestingFramework/ObjectManager/etc/config.xsd">

src/Magento/FunctionalTestingFramework/Allure/Adapter/MagentoAllureAdapter.php

@@ -12,6 +12,8 @@
 use Magento\FunctionalTestingFramework\Test\Objects\ActionGroupObject;
 use Magento\FunctionalTestingFramework\Test\Objects\ActionObject;
 use Magento\FunctionalTestingFramework\Util\TestGenerator;
+use Yandex\Allure\Adapter\Model\Failure;
+use Yandex\Allure\Adapter\Model\Provider;
 use Yandex\Allure\Adapter\Model\Status;
 use Yandex\Allure\Adapter\Model\Step;
 use Yandex\Allure\Adapter\Allure;
@@ -119,6 +121,7 @@ private function sanitizeGroupName($group)
         // if we can't find this group in the generated suites we have to assume that the group was split for generation
         $groupNameSplit = explode("_", $group);
         array_pop($groupNameSplit);
+        array_pop($groupNameSplit);
         $originalName = implode("_", $groupNameSplit);
 
         // confirm our original name is one of the existing suite names otherwise just return the original group name
@@ -256,13 +259,16 @@ public function testError(FailEvent $failEvent)
      */
     public function testEnd(TestEvent $testEvent)
     {
+        // Peek top of testCaseStorage to check of failure
+        $testFailed = $this->getLifecycle()->getTestCaseStorage()->get()->getFailure();
         // Pops top of stepStorage, need to add it back in after processing
         $rootStep = $this->getLifecycle()->getStepStorage()->pollLast();
         $formattedSteps = [];
         $actionGroupStepContainer = null;
 
         $actionGroupStepKey = null;
         foreach ($rootStep->getSteps() as $step) {
+            $this->removeAttachments($step, $testFailed);
             $stepKey = str_replace($actionGroupStepKey, '', $step->getName());
             if ($stepKey !== '[]' && $stepKey !== null) {
                 $step->setName($stepKey);
@@ -379,4 +385,21 @@ private function retrieveStepKey($stepLine)
 
         return $stepKey;
     }
+
+    /**
+     * Removes attachments from step depending on MFTF configuration
+     * @param Step    $step
+     * @param Failure $testFailed
+     * @return void
+     */
+    private function removeAttachments($step, $testFailed)
+    {
+        //Remove Attachments if verbose flag is not true AND test did not fail
+        if (getenv('VERBOSE_ARTIFACTS') !== true && $testFailed === null) {
+            foreach ($step->getAttachments() as $index => $attachment) {
+                $step->removeAttachment($index);
+                unlink(Provider::getOutputDirectory() . DIRECTORY_SEPARATOR . $attachment->getSource());
+            }
+        }
+    }
 }