Merge branch 'develop' into MQE-683

Author: KevinBKozan

CHANGELOG.md

@@ -1,5 +1,10 @@
 Magento Functional Testing Framework Changelog
 ================================================
+2.6.3
+-----
+
+### New Feature
+* `--filter` option was added to `bin/mftf generate:tests` command. For more details please go to https://devdocs.magento.com/mftf/docs/commands/mftf.html#generatetests
 
 2.6.2
 -----

composer.json

@@ -2,7 +2,7 @@
     "name": "magento/magento2-functional-testing-framework",
     "description": "Magento2 Functional Testing Framework",
     "type": "library",
-    "version": "2.6.2",
+    "version": "3.0.0",
     "license": "AGPL-3.0",
     "keywords": ["magento", "automation", "functional", "testing"],
     "config": {

composer.lock

@@ -15,7 +15,7 @@
             <stories value="[Deprecated] MFTF DevDocs available"/>
             <title value="[Deprecated] Magento Functional Testing Framework Documentation is available."/>
             <description value="[Deprecated] Magento Functional Testing Framework Documentation is available."/>
-            <severity value="CRITICAL"/>
+            <severity value="MINOR"/>
             <group value="mftf"/>
         </annotations>
 

dev/tests/functional/tests/MFTF/DevDocs/Test/DeprecatedDevDocsTest.xml

@@ -65,7 +65,7 @@ public function testGetSuiteObject()
             ->withTestActions()
             ->build();
 
-        $mockTestData = ['tests' => array_merge($mockSimpleTest, $mockGroup1Test1, $mockGroup1Test2, $mockGroup2Test1)];
+        $mockTestData = array_merge($mockSimpleTest, $mockGroup1Test1, $mockGroup1Test2, $mockGroup2Test1);
         $this->setMockTestAndSuiteParserOutput($mockTestData, $mockData);
 
         // parse and retrieve suite object with mocked data

dev/tests/unit/Magento/FunctionalTestFramework/Suite/Handlers/SuiteObjectHandlerTest.php

@@ -69,7 +69,7 @@ public function testGenerateSuite()
             ->withTestActions()
             ->build();
 
-        $mockTestData = ['tests' => array_merge($mockSimpleTest)];
+        $mockTestData = array_merge($mockSimpleTest);
         $this->setMockTestAndSuiteParserOutput($mockTestData, $mockData);
 
         // parse and generate suite object with mocked data
@@ -105,7 +105,7 @@ public function testGenerateAllSuites()
             ->withTestActions()
             ->build();
 
-        $mockTestData = ['tests' => array_merge($mockSimpleTest)];
+        $mockTestData = array_merge($mockSimpleTest);
         $this->setMockTestAndSuiteParserOutput($mockTestData, $mockData);
 
         // parse and retrieve suite object with mocked data
@@ -172,7 +172,7 @@ public function testInvalidSuiteTestPair()
             ->withAnnotations(['group' => [['value' => 'group2']]])
             ->withTestActions()
             ->build();
-        $mockTestData = ['tests' => array_merge($mockSimpleTest, $mockSimpleTest2)];
+        $mockTestData = array_merge($mockSimpleTest, $mockSimpleTest2);
         $this->setMockTestAndSuiteParserOutput($mockTestData, $mockSuiteData);
 
         // Make invalid manifest
@@ -196,7 +196,7 @@ public function testNonExistentSuiteTestPair()
             ->withAnnotations(['group' => [['value' => 'group1']]])
             ->withTestActions()
             ->build();
-        $mockTestData = ['tests' => array_merge($mockSimpleTest)];
+        $mockTestData = array_merge($mockSimpleTest);
         $this->setMockTestAndSuiteParserOutput($mockTestData, []);
 
         // Make invalid manifest

dev/tests/unit/Magento/FunctionalTestFramework/Suite/SuiteGeneratorTest.php

@@ -41,7 +41,7 @@ public function testGetTestObject()
 
         $resolverMock = new MockModuleResolverBuilder();
         $resolverMock->setup();
-        $this->setMockParserOutput(['tests' => $mockData]);
+        $this->setMockParserOutput($mockData);
 
         // run object handler method
         $toh = TestObjectHandler::getInstance();
@@ -135,7 +135,7 @@ public function testGetTestsByGroup()
 
         $resolverMock = new MockModuleResolverBuilder();
         $resolverMock->setup();
-        $this->setMockParserOutput(['tests' => array_merge($includeTest, $excludeTest)]);
+        $this->setMockParserOutput(array_merge($includeTest, $excludeTest));
 
         // execute test method
         $toh = TestObjectHandler::getInstance();
@@ -184,7 +184,7 @@ public function testGetTestWithModuleName()
         $resolverMock = new MockModuleResolverBuilder();
         $resolverMock->setup(['Vendor_' . $moduleExpected => $filepath]);
 
-        $this->setMockParserOutput(['tests' => $mockData]);
+        $this->setMockParserOutput($mockData);
         // Execute Test Method
         $toh = TestObjectHandler::getInstance();
         $actualTestObject = $toh->getObject($testDataArrayBuilder->testName);
@@ -212,7 +212,7 @@ public function testGetTestObjectWithInvalidExtends()
             ->build();
         $resolverMock = new MockModuleResolverBuilder();
         $resolverMock->setup();
-        $this->setMockParserOutput(['tests' => $testOne]);
+        $this->setMockParserOutput($testOne);
 
         $toh = TestObjectHandler::getInstance();
 
@@ -250,7 +250,7 @@ public function testGetAllTestObjectsWithInvalidExtends()
 
         $resolverMock = new MockModuleResolverBuilder();
         $resolverMock->setup();
-        $this->setMockParserOutput(['tests' => array_merge($testOne, $testTwo)]);
+        $this->setMockParserOutput(array_merge($testOne, $testTwo));
 
         $toh = TestObjectHandler::getInstance();
 

dev/tests/unit/Magento/FunctionalTestFramework/Test/Handlers/TestObjectHandlerTest.php

@@ -67,7 +67,7 @@ public function testGenerateExtendedTest()
             ->withTestReference("simpleTest")
             ->build();
 
-        $mockTestData = ['tests' => array_merge($mockSimpleTest, $mockExtendedTest)];
+        $mockTestData = array_merge($mockSimpleTest, $mockExtendedTest);
         $this->setMockTestOutput($mockTestData);
 
         // parse and generate test object with mocked data
@@ -112,7 +112,7 @@ public function testGenerateExtendedWithHooks()
             ->withTestReference("simpleTest")
             ->build();
 
-        $mockTestData = ['tests' => array_merge($mockSimpleTest, $mockExtendedTest)];
+        $mockTestData = array_merge($mockSimpleTest, $mockExtendedTest);
         $this->setMockTestOutput($mockTestData);
 
         // parse and generate test object with mocked data
@@ -143,7 +143,7 @@ public function testExtendedTestNoParent()
             ->withTestReference("simpleTest")
             ->build();
 
-        $mockTestData = ['tests' => array_merge($mockExtendedTest)];
+        $mockTestData = array_merge($mockExtendedTest);
         $this->setMockTestOutput($mockTestData);
 
         // parse and generate test object with mocked data
@@ -182,7 +182,7 @@ public function testExtendingExtendedTest()
             ->withTestReference("simpleTest")
             ->build();
 
-        $mockTestData = ['tests' => array_merge($mockParentTest, $mockSimpleTest, $mockExtendedTest)];
+        $mockTestData = array_merge($mockParentTest, $mockSimpleTest, $mockExtendedTest);
         $this->setMockTestOutput($mockTestData);
 
         $this->expectExceptionMessage("Cannot extend a test that already extends another test. Test: simpleTest");

dev/tests/unit/Magento/FunctionalTestFramework/Test/Util/ObjectExtensionUtilTest.php

@@ -146,20 +146,6 @@ function ($arg) {
                 'Test' . DIRECTORY_SEPARATOR .'Mftf'
             ]
         );
-        $mockResolver->verifyInvoked(
-            'globRelevantPaths',
-            [
-                $magentoBaseCodePath
-                    . DIRECTORY_SEPARATOR . "dev"
-                    . DIRECTORY_SEPARATOR . "tests"
-                    . DIRECTORY_SEPARATOR . "acceptance"
-                    . DIRECTORY_SEPARATOR . "tests"
-                    . DIRECTORY_SEPARATOR . "functional"
-                    . DIRECTORY_SEPARATOR . "Magento"
-                    . DIRECTORY_SEPARATOR . "FunctionalTest"
-                , ''
-            ]
-        );
     }
 
     /**

dev/tests/unit/Magento/FunctionalTestFramework/Util/ModuleResolverTest.php

@@ -8,6 +8,7 @@
 
 use AspectMock\Test as AspectMock;
 
+use Magento\FunctionalTestingFramework\Filter\FilterList;
 use Magento\FunctionalTestingFramework\Test\Objects\ActionObject;
 use Magento\FunctionalTestingFramework\Test\Objects\TestHookObject;
 use Magento\FunctionalTestingFramework\Test\Objects\TestObject;
@@ -17,6 +18,16 @@
 
 class TestGeneratorTest extends MagentoTestCase
 {
+    /**
+     * After method functionality
+     *
+     * @return void
+     */
+    public function tearDown()
+    {
+        AspectMock::clean();
+    }
+
     /**
      * Basic test to check exceptions for incorrect entities.
      *
@@ -99,4 +110,54 @@ public function testAllowSkipped()
         $this->assertContains($actionInput, $output);
         $this->assertContains($beforeActionInput, $output);
     }
+
+    /**
+     * Tests that TestGenerator createAllTestFiles correctly filters based on severity
+     *
+     * @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException
+     */
+    public function testFilter()
+    {
+        // Mock filters for TestGenerator
+        AspectMock::double(
+            MftfApplicationConfig::class,
+            ['getFilterList' => new FilterList(['severity' => ["CRITICAL"]])]
+        );
+
+        $actionInput = 'fakeInput';
+        $actionObject = new ActionObject('fakeAction', 'comment', [
+            'userInput' => $actionInput
+        ]);
+
+        $annotation1 = ['severity' => ['CRITICAL']];
+        $annotation2 = ['severity' => ['MINOR']];
+        $test1 = new TestObject(
+            "test1",
+            ["fakeAction" => $actionObject],
+            $annotation1,
+            [],
+            "filename"
+        );
+        $test2 = new TestObject(
+            "test2",
+            ["fakeAction" => $actionObject],
+            $annotation2,
+            [],
+            "filename"
+        );
+        AspectMock::double(TestGenerator::class, ['loadAllTestObjects' => ["sampleTest" => $test1, "test2" => $test2]]);
+
+        // Mock createCestFile to return name of tests that testGenerator tried to create
+        $generatedTests = [];
+        AspectMock::double(TestGenerator::class, ['createCestFile' => function ($arg1, $arg2) use (&$generatedTests) {
+            $generatedTests[$arg2] = true;
+        }]);
+
+        $testGeneratorObject = TestGenerator::getInstance("", ["sampleTest" => $test1, "test2" => $test2]);
+        $testGeneratorObject->createAllTestFiles(null, []);
+
+        // Ensure Test1 was Generated but not Test 2
+        $this->assertArrayHasKey('test1Cest', $generatedTests);
+        $this->assertArrayNotHasKey('test2Cest', $generatedTests);
+    }
 }

dev/tests/unit/Magento/FunctionalTestFramework/Util/TestGeneratorTest.php

@@ -159,6 +159,7 @@ vendor/bin/mftf generate:tests [option] [<test name>] [<test name>] [--remove]
 | Option | Description|
 | ---| --- |
 | `--config=[<default> or <singleRun> or <parallel>]` | Creates a single manifest file with a list of all tests. The default location is `tests/functional/Magento/FunctionalTest/_generated/testManifest.txt`.<br/> You can split the list into multiple groups using `--config=parallel`; the groups will be generated in `_generated/groups/` like `_generated/groups/group1.txt, group2.txt, ...`.<br/> Available values: `default` (default), `singleRun`(same as `default`), and `parallel`.<br/> Example: `generate:tests --config=parallel`. |
+| `--filter` | Option to filter tests to be generated.<br/>Template: '<filterName>:<filterValue>'.<br/>Existing filter types: severity.<br/>Existing severity values: BLOCKER, CRITICAL, MAJOR, AVERAGE, MINOR.<br/>Example: --filter=severity:CRITICAL|
 | `--force` | Forces test generation, regardless of the module merge order defined in the Magento instance. Example: `generate:tests --force`. |
 | `-i,--time` | Set time in minutes to determine the group size when `--config=parallel` is used. The __default value__ is `10`. Example: `generate:tests --config=parallel --time=15`|
 | `--tests` | Defines the test configuration as a JSON string.|

docs/commands/mftf.md

@@ -85,6 +85,16 @@ The following example shows the usage of `grabValueFrom` in testing, where the r
 <fillField selector=".functionalTestSelector" userInput="{$grabStepKey}" stepKey="fillFieldKey1"/>
 ```
 
+The following is an example of the `Magento/Catalog/Test/Mftf/ActionGroup/AssertDiscountsPercentageOfProductsActionGroup.xml` test:
+
+```xml
+<grabValueFrom selector="{{AdminProductFormAdvancedPricingSection.productTierPricePercentageValuePriceInput('0')}}" stepKey="grabProductTierPriceInput"/>
+<assertEquals stepKey="assertProductTierPriceInput">
+    <expectedResult type="string">{{amount}}</expectedResult>
+    <actualResult type="string">$grabProductTierPriceInput</actualResult>
+</assertEquals>
+```
+
 ## Hard-coded data input
 
 The data to operate against can be included as literals in a test. Hard-coded data input can be useful in assertions.

docs/data.md

@@ -2,11 +2,11 @@
 
 <div class="bs-callout bs-callout-info" markdown="1">
 [Find your MFTF version][] of the MFTF.
-The latest Magento 2.3.x release supports MFTF 2.4.5.
+The latest Magento 2.3.x release supports MFTF 2.5.3.
 The latest Magento 2.2.x release supports MFTF 2.4.5.
 </div>
 
-## Prepare environment  {#prepare-environment}
+## Prepare environment {#prepare-environment}
 
 Make sure that you have the following software installed and configured on your development environment:
 
@@ -53,11 +53,11 @@ Install the Magento application.
 composer install
 ```
 
-## Prepare Magento  {#prepare-magento}
+## Prepare Magento {#prepare-magento}
 
 Configure the following settings in Magento as described below.
 
-### WYSIWYG settings    {#wysiwyg-settings}
+### WYSIWYG settings {#wysiwyg-settings}
 
 A Selenium web driver cannot enter data to fields with WYSIWYG.
 
@@ -74,6 +74,12 @@ or via command line:
 bin/magento config:set cms/wysiwyg/enabled disabled
 ```
 
+Clean the cache after changing the configuration values:
+
+```bash
+bin/magento cache:clean config full_page
+```
+
 <div class="bs-callout bs-callout-tip">
 When you want to test the WYSIWYG functionality, re-enable WYSIWYG in your test suite.
 </div>
@@ -97,15 +103,21 @@ bin/magento config:set admin/security/admin_account_sharing 1
 bin/magento config:set admin/security/use_form_key 0
 ```
 
+Clean the cache after changing the configuration values:
+
+```bash
+bin/magento cache:clean config full_page
+```
+
 ### Webserver configuration {#web-server-configuration}
 
 The MFTF does not support executing CLI commands if your web server points to `<MAGE_ROOT_DIR>/pub` directory as recommended in the [Installation Guide][Installation Guide docroot]. For the MFTF to execute the CLI commands, the web server must point to the Magento root directory.
 
 ### Nginx settings {#nginx-settings}
 
-If Nginx Web server is used on your development environment then **Use Web Server Rewrites** setting in **Stores** > Settings > **Configuration** > **Web** > **Search Engine Optimization** must be set to **Yes**.
+If the Nginx Web server is used on your development environment, then **Use Web Server Rewrites** setting in **Stores** > Settings > **Configuration** > **General** > **Web** > **Search Engine Optimization** must be set to **Yes**.
 
-To be able to run Magento command line commands in tests add the following location block to Nginx configuration file:
+To be able to run Magento command line commands in tests, add the following location block to the Nginx configuration file in the Magento root directory:
 
 ```conf
 location ~* ^/dev/tests/acceptance/utils($|/) {