Merge pull request magento#605 from magento/MQE-2008

MQE-2008: Filter test generation and execution by severity

Author: okolesnyk

CHANGELOG.md

@@ -1,5 +1,10 @@
 Magento Functional Testing Framework Changelog
 ================================================
+2.6.3
+-----
+
+### New Feature
+* `--filter` option was added to `bin/mftf generate:tests` command. For more details please go to https://devdocs.magento.com/mftf/docs/commands/mftf.html#generatetests
 
 2.6.2
 -----

composer.json

@@ -2,7 +2,7 @@
     "name": "magento/magento2-functional-testing-framework",
     "description": "Magento2 Functional Testing Framework",
     "type": "library",
-    "version": "2.6.2",
+    "version": "2.6.3",
     "license": "AGPL-3.0",
     "keywords": ["magento", "automation", "functional", "testing"],
     "config": {

composer.lock

@@ -15,7 +15,7 @@
             <stories value="[Deprecated] MFTF DevDocs available"/>
             <title value="[Deprecated] Magento Functional Testing Framework Documentation is available."/>
             <description value="[Deprecated] Magento Functional Testing Framework Documentation is available."/>
-            <severity value="CRITICAL"/>
+            <severity value="MINOR"/>
             <group value="mftf"/>
         </annotations>
 

dev/tests/functional/tests/MFTF/DevDocs/Test/DeprecatedDevDocsTest.xml

@@ -65,7 +65,7 @@ public function testGetSuiteObject()
             ->withTestActions()
             ->build();
 
-        $mockTestData = ['tests' => array_merge($mockSimpleTest, $mockGroup1Test1, $mockGroup1Test2, $mockGroup2Test1)];
+        $mockTestData = array_merge($mockSimpleTest, $mockGroup1Test1, $mockGroup1Test2, $mockGroup2Test1);
         $this->setMockTestAndSuiteParserOutput($mockTestData, $mockData);
 
         // parse and retrieve suite object with mocked data

dev/tests/unit/Magento/FunctionalTestFramework/Suite/Handlers/SuiteObjectHandlerTest.php

@@ -69,7 +69,7 @@ public function testGenerateSuite()
             ->withTestActions()
             ->build();
 
-        $mockTestData = ['tests' => array_merge($mockSimpleTest)];
+        $mockTestData = array_merge($mockSimpleTest);
         $this->setMockTestAndSuiteParserOutput($mockTestData, $mockData);
 
         // parse and generate suite object with mocked data
@@ -105,7 +105,7 @@ public function testGenerateAllSuites()
             ->withTestActions()
             ->build();
 
-        $mockTestData = ['tests' => array_merge($mockSimpleTest)];
+        $mockTestData = array_merge($mockSimpleTest);
         $this->setMockTestAndSuiteParserOutput($mockTestData, $mockData);
 
         // parse and retrieve suite object with mocked data
@@ -172,7 +172,7 @@ public function testInvalidSuiteTestPair()
             ->withAnnotations(['group' => [['value' => 'group2']]])
             ->withTestActions()
             ->build();
-        $mockTestData = ['tests' => array_merge($mockSimpleTest, $mockSimpleTest2)];
+        $mockTestData = array_merge($mockSimpleTest, $mockSimpleTest2);
         $this->setMockTestAndSuiteParserOutput($mockTestData, $mockSuiteData);
 
         // Make invalid manifest
@@ -196,7 +196,7 @@ public function testNonExistentSuiteTestPair()
             ->withAnnotations(['group' => [['value' => 'group1']]])
             ->withTestActions()
             ->build();
-        $mockTestData = ['tests' => array_merge($mockSimpleTest)];
+        $mockTestData = array_merge($mockSimpleTest);
         $this->setMockTestAndSuiteParserOutput($mockTestData, []);
 
         // Make invalid manifest

dev/tests/unit/Magento/FunctionalTestFramework/Suite/SuiteGeneratorTest.php

@@ -41,7 +41,7 @@ public function testGetTestObject()
 
         $resolverMock = new MockModuleResolverBuilder();
         $resolverMock->setup();
-        $this->setMockParserOutput(['tests' => $mockData]);
+        $this->setMockParserOutput($mockData);
 
         // run object handler method
         $toh = TestObjectHandler::getInstance();
@@ -135,7 +135,7 @@ public function testGetTestsByGroup()
 
         $resolverMock = new MockModuleResolverBuilder();
         $resolverMock->setup();
-        $this->setMockParserOutput(['tests' => array_merge($includeTest, $excludeTest)]);
+        $this->setMockParserOutput(array_merge($includeTest, $excludeTest));
 
         // execute test method
         $toh = TestObjectHandler::getInstance();
@@ -184,7 +184,7 @@ public function testGetTestWithModuleName()
         $resolverMock = new MockModuleResolverBuilder();
         $resolverMock->setup(['Vendor_' . $moduleExpected => $filepath]);
 
-        $this->setMockParserOutput(['tests' => $mockData]);
+        $this->setMockParserOutput($mockData);
         // Execute Test Method
         $toh = TestObjectHandler::getInstance();
         $actualTestObject = $toh->getObject($testDataArrayBuilder->testName);
@@ -212,7 +212,7 @@ public function testGetTestObjectWithInvalidExtends()
             ->build();
         $resolverMock = new MockModuleResolverBuilder();
         $resolverMock->setup();
-        $this->setMockParserOutput(['tests' => $testOne]);
+        $this->setMockParserOutput($testOne);
 
         $toh = TestObjectHandler::getInstance();
 
@@ -250,7 +250,7 @@ public function testGetAllTestObjectsWithInvalidExtends()
 
         $resolverMock = new MockModuleResolverBuilder();
         $resolverMock->setup();
-        $this->setMockParserOutput(['tests' => array_merge($testOne, $testTwo)]);
+        $this->setMockParserOutput(array_merge($testOne, $testTwo));
 
         $toh = TestObjectHandler::getInstance();
 

dev/tests/unit/Magento/FunctionalTestFramework/Test/Handlers/TestObjectHandlerTest.php

@@ -67,7 +67,7 @@ public function testGenerateExtendedTest()
             ->withTestReference("simpleTest")
             ->build();
 
-        $mockTestData = ['tests' => array_merge($mockSimpleTest, $mockExtendedTest)];
+        $mockTestData = array_merge($mockSimpleTest, $mockExtendedTest);
         $this->setMockTestOutput($mockTestData);
 
         // parse and generate test object with mocked data
@@ -112,7 +112,7 @@ public function testGenerateExtendedWithHooks()
             ->withTestReference("simpleTest")
             ->build();
 
-        $mockTestData = ['tests' => array_merge($mockSimpleTest, $mockExtendedTest)];
+        $mockTestData = array_merge($mockSimpleTest, $mockExtendedTest);
         $this->setMockTestOutput($mockTestData);
 
         // parse and generate test object with mocked data
@@ -143,7 +143,7 @@ public function testExtendedTestNoParent()
             ->withTestReference("simpleTest")
             ->build();
 
-        $mockTestData = ['tests' => array_merge($mockExtendedTest)];
+        $mockTestData = array_merge($mockExtendedTest);
         $this->setMockTestOutput($mockTestData);
 
         // parse and generate test object with mocked data
@@ -182,7 +182,7 @@ public function testExtendingExtendedTest()
             ->withTestReference("simpleTest")
             ->build();
 
-        $mockTestData = ['tests' => array_merge($mockParentTest, $mockSimpleTest, $mockExtendedTest)];
+        $mockTestData = array_merge($mockParentTest, $mockSimpleTest, $mockExtendedTest);
         $this->setMockTestOutput($mockTestData);
 
         $this->expectExceptionMessage("Cannot extend a test that already extends another test. Test: simpleTest");

dev/tests/unit/Magento/FunctionalTestFramework/Test/Util/ObjectExtensionUtilTest.php

@@ -8,6 +8,7 @@
 
 use AspectMock\Test as AspectMock;
 
+use Magento\FunctionalTestingFramework\Filter\FilterList;
 use Magento\FunctionalTestingFramework\Test\Objects\ActionObject;
 use Magento\FunctionalTestingFramework\Test\Objects\TestHookObject;
 use Magento\FunctionalTestingFramework\Test\Objects\TestObject;
@@ -17,6 +18,16 @@
 
 class TestGeneratorTest extends MagentoTestCase
 {
+    /**
+     * After method functionality
+     *
+     * @return void
+     */
+    public function tearDown()
+    {
+        AspectMock::clean();
+    }
+
     /**
      * Basic test to check exceptions for incorrect entities.
      *
@@ -99,4 +110,54 @@ public function testAllowSkipped()
         $this->assertContains($actionInput, $output);
         $this->assertContains($beforeActionInput, $output);
     }
+
+    /**
+     * Tests that TestGenerator createAllTestFiles correctly filters based on severity
+     *
+     * @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException
+     */
+    public function testFilter()
+    {
+        // Mock filters for TestGenerator
+        AspectMock::double(
+            MftfApplicationConfig::class,
+            ['getFilterList' => new FilterList(['severity' => ["CRITICAL"]])]
+        );
+
+        $actionInput = 'fakeInput';
+        $actionObject = new ActionObject('fakeAction', 'comment', [
+            'userInput' => $actionInput
+        ]);
+
+        $annotation1 = ['severity' => ['CRITICAL']];
+        $annotation2 = ['severity' => ['MINOR']];
+        $test1 = new TestObject(
+            "test1",
+            ["fakeAction" => $actionObject],
+            $annotation1,
+            [],
+            "filename"
+        );
+        $test2 = new TestObject(
+            "test2",
+            ["fakeAction" => $actionObject],
+            $annotation2,
+            [],
+            "filename"
+        );
+        AspectMock::double(TestGenerator::class, ['loadAllTestObjects' => ["sampleTest" => $test1, "test2" => $test2]]);
+
+        // Mock createCestFile to return name of tests that testGenerator tried to create
+        $generatedTests = [];
+        AspectMock::double(TestGenerator::class, ['createCestFile' => function ($arg1, $arg2) use (&$generatedTests) {
+            $generatedTests[$arg2] = true;
+        }]);
+
+        $testGeneratorObject = TestGenerator::getInstance("", ["sampleTest" => $test1, "test2" => $test2]);
+        $testGeneratorObject->createAllTestFiles(null, []);
+
+        // Ensure Test1 was Generated but not Test 2
+        $this->assertArrayHasKey('test1Cest', $generatedTests);
+        $this->assertArrayNotHasKey('test2Cest', $generatedTests);
+    }
 }

dev/tests/unit/Magento/FunctionalTestFramework/Util/TestGeneratorTest.php

@@ -159,6 +159,7 @@ vendor/bin/mftf generate:tests [option] [<test name>] [<test name>] [--remove]
 | Option | Description|
 | ---| --- |
 | `--config=[<default> or <singleRun> or <parallel>]` | Creates a single manifest file with a list of all tests. The default location is `tests/functional/Magento/FunctionalTest/_generated/testManifest.txt`.<br/> You can split the list into multiple groups using `--config=parallel`; the groups will be generated in `_generated/groups/` like `_generated/groups/group1.txt, group2.txt, ...`.<br/> Available values: `default` (default), `singleRun`(same as `default`), and `parallel`.<br/> Example: `generate:tests --config=parallel`. |
+| `--filter` | Option to filter tests to be generated.<br/>Template: '<filterName>:<filterValue>'.<br/>Existing filter types: severity.<br/>Existing severity values: BLOCKER, CRITICAL, MAJOR, AVERAGE, MINOR.<br/>Example: --filter=severity:CRITICAL|
 | `--force` | Forces test generation, regardless of the module merge order defined in the Magento instance. Example: `generate:tests --force`. |
 | `-i,--time` | Set time in minutes to determine the group size when `--config=parallel` is used. The __default value__ is `10`. Example: `generate:tests --config=parallel --time=15`|
 | `--tests` | Defines the test configuration as a JSON string.|

docs/commands/mftf.md

@@ -6,6 +6,7 @@
 namespace Magento\FunctionalTestingFramework\Config;
 
 use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException;
+use Magento\FunctionalTestingFramework\Filter\FilterList;
 
 class MftfApplicationConfig
 {
@@ -25,6 +26,13 @@ class MftfApplicationConfig
     const LEVEL_NONE = "none";
     const MFTF_DEBUG_LEVEL = [self::LEVEL_DEFAULT, self::LEVEL_DEVELOPER, self::LEVEL_NONE];
 
+    /**
+     * Contains object with test filters.
+     *
+     * @var FilterList
+     */
+    private $filterList;
+
     /**
      * Determines whether the user has specified a force option for generation
      *
@@ -74,14 +82,16 @@ class MftfApplicationConfig
      * @param boolean $verboseEnabled
      * @param string  $debugLevel
      * @param boolean $allowSkipped
+     * @param array   $filters
      * @throws TestFrameworkException
      */
     private function __construct(
         $forceGenerate = false,
         $phase = self::EXECUTION_PHASE,
         $verboseEnabled = null,
         $debugLevel = self::LEVEL_NONE,
-        $allowSkipped = false
+        $allowSkipped = false,
+        $filters = []
     ) {
         $this->forceGenerate = $forceGenerate;
 
@@ -101,6 +111,7 @@ private function __construct(
                 $this->debugLevel = self::LEVEL_DEVELOPER;
         }
         $this->allowSkipped = $allowSkipped;
+        $this->filterList = new FilterList($filters);
     }
 
     /**
@@ -112,6 +123,7 @@ private function __construct(
      * @param boolean $verboseEnabled
      * @param string  $debugLevel
      * @param boolean $allowSkipped
+     * @param array   $filters
      * @return void
      * @throws TestFrameworkException
      */
@@ -120,11 +132,19 @@ public static function create(
         $phase = self::EXECUTION_PHASE,
         $verboseEnabled = null,
         $debugLevel = self::LEVEL_NONE,
-        $allowSkipped = false
+        $allowSkipped = false,
+        $filters = []
     ) {
         if (self::$MFTF_APPLICATION_CONTEXT == null) {
             self::$MFTF_APPLICATION_CONTEXT =
-                new MftfApplicationConfig($forceGenerate, $phase, $verboseEnabled, $debugLevel, $allowSkipped);
+                new MftfApplicationConfig(
+                    $forceGenerate,
+                    $phase,
+                    $verboseEnabled,
+                    $debugLevel,
+                    $allowSkipped,
+                    $filters
+                );
         }
     }
 
@@ -196,4 +216,14 @@ public function getPhase()
     {
         return $this->phase;
     }
+
+    /**
+     * Returns a class with registered filter list.
+     *
+     * @return FilterList
+     */
+    public function getFilterList()
+    {
+        return $this->filterList;
+    }
 }

src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php

@@ -35,7 +35,7 @@ class BaseGenerateCommand extends Command
      *
      * @var SymfonyStyle
      */
-    private $ioStyle = null;
+    protected $ioStyle = null;
 
     /**
      * Configures the base command.