From fd1d845f38e963fdb42d3322540e203155c9ba77 Mon Sep 17 00:00:00 2001
From: Alex Kolesnyk <kolesnyk@adobe.com>
Date: Tue, 25 Feb 2020 16:23:21 -0600
Subject: [PATCH 1/9] MQE-2008: Filter test generation and execution by
 severity

---
 .../DevDocs/Test/DeprecatedDevDocsTest.xml    |  2 +-
 .../Suite/Handlers/SuiteObjectHandlerTest.php |  2 +-
 .../Suite/SuiteGeneratorTest.php              |  8 +-
 .../Test/Handlers/TestObjectHandlerTest.php   | 10 +--
 .../Test/Util/ObjectExtensionUtilTest.php     |  8 +-
 .../Config/MftfApplicationConfig.php          | 36 ++++++++-
 .../Console/GenerateTestsCommand.php          | 74 ++++++++++++++-----
 .../Filter/FilterInterface.php                | 27 +++++++
 .../Filter/FilterList.php                     | 57 ++++++++++++++
 .../Filter/Test/Severity.php                  | 69 +++++++++++++++++
 .../Suite/Objects/SuiteObject.php             | 10 ++-
 .../Test/Handlers/TestObjectHandler.php       |  4 +-
 .../Test/Parsers/TestDataParser.php           | 33 ++++++++-
 .../Util/Logger/MftfLogger.php                | 19 +++++
 14 files changed, 315 insertions(+), 44 deletions(-)
 create mode 100644 src/Magento/FunctionalTestingFramework/Filter/FilterInterface.php
 create mode 100644 src/Magento/FunctionalTestingFramework/Filter/FilterList.php
 create mode 100644 src/Magento/FunctionalTestingFramework/Filter/Test/Severity.php

diff --git a/dev/tests/functional/tests/MFTF/DevDocs/Test/DeprecatedDevDocsTest.xml b/dev/tests/functional/tests/MFTF/DevDocs/Test/DeprecatedDevDocsTest.xml
index bf0078df4..c93dc8011 100644
--- a/dev/tests/functional/tests/MFTF/DevDocs/Test/DeprecatedDevDocsTest.xml
+++ b/dev/tests/functional/tests/MFTF/DevDocs/Test/DeprecatedDevDocsTest.xml
@@ -15,7 +15,7 @@
             <stories value="[Deprecated] MFTF DevDocs available"/>
             <title value="[Deprecated] Magento Functional Testing Framework Documentation is available."/>
             <description value="[Deprecated] Magento Functional Testing Framework Documentation is available."/>
-            <severity value="CRITICAL"/>
+            <severity value="MINOR"/>
             <group value="mftf"/>
         </annotations>
 
diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Suite/Handlers/SuiteObjectHandlerTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Suite/Handlers/SuiteObjectHandlerTest.php
index 5bf29507c..e45127f4c 100644
--- a/dev/tests/unit/Magento/FunctionalTestFramework/Suite/Handlers/SuiteObjectHandlerTest.php
+++ b/dev/tests/unit/Magento/FunctionalTestFramework/Suite/Handlers/SuiteObjectHandlerTest.php
@@ -65,7 +65,7 @@ public function testGetSuiteObject()
             ->withTestActions()
             ->build();
 
-        $mockTestData = ['tests' => array_merge($mockSimpleTest, $mockGroup1Test1, $mockGroup1Test2, $mockGroup2Test1)];
+        $mockTestData = array_merge($mockSimpleTest, $mockGroup1Test1, $mockGroup1Test2, $mockGroup2Test1);
         $this->setMockTestAndSuiteParserOutput($mockTestData, $mockData);
 
         // parse and retrieve suite object with mocked data
diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Suite/SuiteGeneratorTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Suite/SuiteGeneratorTest.php
index e3f820ede..842be4f88 100644
--- a/dev/tests/unit/Magento/FunctionalTestFramework/Suite/SuiteGeneratorTest.php
+++ b/dev/tests/unit/Magento/FunctionalTestFramework/Suite/SuiteGeneratorTest.php
@@ -69,7 +69,7 @@ public function testGenerateSuite()
             ->withTestActions()
             ->build();
 
-        $mockTestData = ['tests' => array_merge($mockSimpleTest)];
+        $mockTestData = array_merge($mockSimpleTest);
         $this->setMockTestAndSuiteParserOutput($mockTestData, $mockData);
 
         // parse and generate suite object with mocked data
@@ -105,7 +105,7 @@ public function testGenerateAllSuites()
             ->withTestActions()
             ->build();
 
-        $mockTestData = ['tests' => array_merge($mockSimpleTest)];
+        $mockTestData = array_merge($mockSimpleTest);
         $this->setMockTestAndSuiteParserOutput($mockTestData, $mockData);
 
         // parse and retrieve suite object with mocked data
@@ -172,7 +172,7 @@ public function testInvalidSuiteTestPair()
             ->withAnnotations(['group' => [['value' => 'group2']]])
             ->withTestActions()
             ->build();
-        $mockTestData = ['tests' => array_merge($mockSimpleTest, $mockSimpleTest2)];
+        $mockTestData = array_merge($mockSimpleTest, $mockSimpleTest2);
         $this->setMockTestAndSuiteParserOutput($mockTestData, $mockSuiteData);
 
         // Make invalid manifest
@@ -196,7 +196,7 @@ public function testNonExistentSuiteTestPair()
             ->withAnnotations(['group' => [['value' => 'group1']]])
             ->withTestActions()
             ->build();
-        $mockTestData = ['tests' => array_merge($mockSimpleTest)];
+        $mockTestData = array_merge($mockSimpleTest);
         $this->setMockTestAndSuiteParserOutput($mockTestData, []);
 
         // Make invalid manifest
diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Test/Handlers/TestObjectHandlerTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Test/Handlers/TestObjectHandlerTest.php
index 67f6f2b43..a4504ab07 100644
--- a/dev/tests/unit/Magento/FunctionalTestFramework/Test/Handlers/TestObjectHandlerTest.php
+++ b/dev/tests/unit/Magento/FunctionalTestFramework/Test/Handlers/TestObjectHandlerTest.php
@@ -41,7 +41,7 @@ public function testGetTestObject()
 
         $resolverMock = new MockModuleResolverBuilder();
         $resolverMock->setup();
-        $this->setMockParserOutput(['tests' => $mockData]);
+        $this->setMockParserOutput($mockData);
 
         // run object handler method
         $toh = TestObjectHandler::getInstance();
@@ -135,7 +135,7 @@ public function testGetTestsByGroup()
 
         $resolverMock = new MockModuleResolverBuilder();
         $resolverMock->setup();
-        $this->setMockParserOutput(['tests' => array_merge($includeTest, $excludeTest)]);
+        $this->setMockParserOutput(array_merge($includeTest, $excludeTest));
 
         // execute test method
         $toh = TestObjectHandler::getInstance();
@@ -184,7 +184,7 @@ public function testGetTestWithModuleName()
         $resolverMock = new MockModuleResolverBuilder();
         $resolverMock->setup(['Vendor_' . $moduleExpected => $filepath]);
 
-        $this->setMockParserOutput(['tests' => $mockData]);
+        $this->setMockParserOutput($mockData);
         // Execute Test Method
         $toh = TestObjectHandler::getInstance();
         $actualTestObject = $toh->getObject($testDataArrayBuilder->testName);
@@ -212,7 +212,7 @@ public function testGetTestObjectWithInvalidExtends()
             ->build();
         $resolverMock = new MockModuleResolverBuilder();
         $resolverMock->setup();
-        $this->setMockParserOutput(['tests' => $testOne]);
+        $this->setMockParserOutput($testOne);
 
         $toh = TestObjectHandler::getInstance();
 
@@ -250,7 +250,7 @@ public function testGetAllTestObjectsWithInvalidExtends()
 
         $resolverMock = new MockModuleResolverBuilder();
         $resolverMock->setup();
-        $this->setMockParserOutput(['tests' => array_merge($testOne, $testTwo)]);
+        $this->setMockParserOutput(array_merge($testOne, $testTwo));
 
         $toh = TestObjectHandler::getInstance();
 
diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Test/Util/ObjectExtensionUtilTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Test/Util/ObjectExtensionUtilTest.php
index d86185b83..9982b8040 100644
--- a/dev/tests/unit/Magento/FunctionalTestFramework/Test/Util/ObjectExtensionUtilTest.php
+++ b/dev/tests/unit/Magento/FunctionalTestFramework/Test/Util/ObjectExtensionUtilTest.php
@@ -67,7 +67,7 @@ public function testGenerateExtendedTest()
             ->withTestReference("simpleTest")
             ->build();
 
-        $mockTestData = ['tests' => array_merge($mockSimpleTest, $mockExtendedTest)];
+        $mockTestData = array_merge($mockSimpleTest, $mockExtendedTest);
         $this->setMockTestOutput($mockTestData);
 
         // parse and generate test object with mocked data
@@ -112,7 +112,7 @@ public function testGenerateExtendedWithHooks()
             ->withTestReference("simpleTest")
             ->build();
 
-        $mockTestData = ['tests' => array_merge($mockSimpleTest, $mockExtendedTest)];
+        $mockTestData = array_merge($mockSimpleTest, $mockExtendedTest);
         $this->setMockTestOutput($mockTestData);
 
         // parse and generate test object with mocked data
@@ -143,7 +143,7 @@ public function testExtendedTestNoParent()
             ->withTestReference("simpleTest")
             ->build();
 
-        $mockTestData = ['tests' => array_merge($mockExtendedTest)];
+        $mockTestData = array_merge($mockExtendedTest);
         $this->setMockTestOutput($mockTestData);
 
         // parse and generate test object with mocked data
@@ -182,7 +182,7 @@ public function testExtendingExtendedTest()
             ->withTestReference("simpleTest")
             ->build();
 
-        $mockTestData = ['tests' => array_merge($mockParentTest, $mockSimpleTest, $mockExtendedTest)];
+        $mockTestData = array_merge($mockParentTest, $mockSimpleTest, $mockExtendedTest);
         $this->setMockTestOutput($mockTestData);
 
         $this->expectExceptionMessage("Cannot extend a test that already extends another test. Test: simpleTest");
diff --git a/src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php b/src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php
index 80db27de0..4b215cb4f 100644
--- a/src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php
+++ b/src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php
@@ -6,6 +6,7 @@
 namespace Magento\FunctionalTestingFramework\Config;
 
 use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException;
+use Magento\FunctionalTestingFramework\Filter\FilterList;
 
 class MftfApplicationConfig
 {
@@ -25,6 +26,13 @@ class MftfApplicationConfig
     const LEVEL_NONE = "none";
     const MFTF_DEBUG_LEVEL = [self::LEVEL_DEFAULT, self::LEVEL_DEVELOPER, self::LEVEL_NONE];
 
+    /**
+     * Contains object with test filters.
+     *
+     * @var FilterList
+     */
+    private $filterList;
+
     /**
      * Determines whether the user has specified a force option for generation
      *
@@ -74,6 +82,7 @@ class MftfApplicationConfig
      * @param boolean $verboseEnabled
      * @param string  $debugLevel
      * @param boolean $allowSkipped
+     * @param array   $filters
      * @throws TestFrameworkException
      */
     private function __construct(
@@ -81,7 +90,8 @@ private function __construct(
         $phase = self::EXECUTION_PHASE,
         $verboseEnabled = null,
         $debugLevel = self::LEVEL_NONE,
-        $allowSkipped = false
+        $allowSkipped = false,
+        $filters = []
     ) {
         $this->forceGenerate = $forceGenerate;
 
@@ -101,6 +111,7 @@ private function __construct(
                 $this->debugLevel = self::LEVEL_DEVELOPER;
         }
         $this->allowSkipped = $allowSkipped;
+        $this->filterList = new FilterList($filters);
     }
 
     /**
@@ -112,6 +123,7 @@ private function __construct(
      * @param boolean $verboseEnabled
      * @param string  $debugLevel
      * @param boolean $allowSkipped
+     * @param array   $filters
      * @return void
      * @throws TestFrameworkException
      */
@@ -120,11 +132,19 @@ public static function create(
         $phase = self::EXECUTION_PHASE,
         $verboseEnabled = null,
         $debugLevel = self::LEVEL_NONE,
-        $allowSkipped = false
+        $allowSkipped = false,
+        $filters = []
     ) {
         if (self::$MFTF_APPLICATION_CONTEXT == null) {
             self::$MFTF_APPLICATION_CONTEXT =
-                new MftfApplicationConfig($forceGenerate, $phase, $verboseEnabled, $debugLevel, $allowSkipped);
+                new MftfApplicationConfig(
+                    $forceGenerate,
+                    $phase,
+                    $verboseEnabled,
+                    $debugLevel,
+                    $allowSkipped,
+                    $filters
+                );
         }
     }
 
@@ -196,4 +216,14 @@ public function getPhase()
     {
         return $this->phase;
     }
+
+    /**
+     * Returns a class with registered filter list.
+     *
+     * @return FilterList
+     */
+    public function getFilterList()
+    {
+        return $this->filterList;
+    }
 }
diff --git a/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php b/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php
index b9378edc1..f43488c0b 100644
--- a/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php
+++ b/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php
@@ -18,7 +18,11 @@
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Output\OutputInterface;
+use Symfony\Component\Console\Style\SymfonyStyle;
 
+/**
+ * @SuppressWarnings(PHPMD)
+ */
 class GenerateTestsCommand extends BaseGenerateCommand
 {
     /**
@@ -30,6 +34,7 @@ protected function configure()
     {
         $this->setName('generate:tests')
             ->setDescription('Run validation and generate all test files and suites based on xml declarations')
+            ->addUsage('AdminLoginTest')
             ->addArgument(
                 'name',
                 InputArgument::OPTIONAL | InputArgument::IS_ARRAY,
@@ -46,6 +51,15 @@ protected function configure()
                 't',
                 InputOption::VALUE_REQUIRED,
                 'A parameter accepting a JSON string used to determine the test configuration'
+            )->addOption(
+                'filter',
+                null,
+                InputOption::VALUE_IS_ARRAY | InputOption::VALUE_OPTIONAL,
+                'Option to filter tests to be generated.' . PHP_EOL
+                . '<info>Template:</info> <filterName>:<filterValue>' . PHP_EOL
+                . '<info>Existing filter types:</info> severity.' . PHP_EOL
+                . '<info>Existing severity values:</info> BLOCKER, CRITICAL, MAJOR, AVERAGE, MINOR.' . PHP_EOL
+                . '<info>Example:</info> --filter=severity:CRITICAL' . PHP_EOL
             );
 
         parent::configure();
@@ -56,13 +70,14 @@ protected function configure()
      *
      * @param InputInterface  $input
      * @param OutputInterface $output
-     * @return void
+     * @return void|integer
      * @throws TestFrameworkException
      * @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException
      * @throws \Magento\FunctionalTestingFramework\Exceptions\XmlException
      */
     protected function execute(InputInterface $input, OutputInterface $output)
     {
+        $ioStyle = new SymfonyStyle($input, $output);
         $tests = $input->getArgument('name');
         $config = $input->getOption('config');
         $json = $input->getOption('tests'); // for backward compatibility
@@ -72,15 +87,25 @@ protected function execute(InputInterface $input, OutputInterface $output)
         $remove = $input->getOption('remove');
         $verbose = $output->isVerbose();
         $allowSkipped = $input->getOption('allow-skipped');
-
+        $filters = $input->getOption('filter');
+        foreach ($filters as $filter) {
+            list($filterType, $filterValue) = explode(':', $filter);
+            $filterList[$filterType][] = $filterValue;
+        }
         // Set application configuration so we can references the user options in our framework
-        MftfApplicationConfig::create(
-            $force,
-            MftfApplicationConfig::GENERATION_PHASE,
-            $verbose,
-            $debug,
-            $allowSkipped
-        );
+        try {
+            MftfApplicationConfig::create(
+                $force,
+                MftfApplicationConfig::GENERATION_PHASE,
+                $verbose,
+                $debug,
+                $allowSkipped,
+                $filterList ?? []
+            );
+        } catch (\Exception $exception) {
+            $ioStyle->error("Test generation failed." . PHP_EOL . $exception->getMessage());
+            return 1;
+        }
 
         $this->setOutputStyle($input, $output);
         $this->showMftfNotices($output);
@@ -105,20 +130,31 @@ protected function execute(InputInterface $input, OutputInterface $output)
                 ($debug !== MftfApplicationConfig::LEVEL_NONE));
         }
 
-        $testConfiguration = $this->createTestConfiguration($json, $tests);
+        try {
+            $testConfiguration = $this->createTestConfiguration($json, $tests);
 
-        // create our manifest file here
-        $testManifest = TestManifestFactory::makeManifest($config, $testConfiguration['suites']);
-        TestGenerator::getInstance(null, $testConfiguration['tests'])->createAllTestFiles($testManifest);
+            // create our manifest file here
+            $testManifest = TestManifestFactory::makeManifest($config, $testConfiguration['suites']);
 
-        if ($config == 'parallel') {
-            /** @var ParallelTestManifest $testManifest */
-            $testManifest->createTestGroups($time);
-        }
+            TestGenerator::getInstance(null, $testConfiguration['tests'])->createAllTestFiles($testManifest);
+
+            if ($config == 'parallel') {
+                /** @var ParallelTestManifest $testManifest */
+                $testManifest->createTestGroups($time);
+            }
+
+            SuiteGenerator::getInstance()->generateAllSuites($testManifest);
 
-        SuiteGenerator::getInstance()->generateAllSuites($testManifest);
+            $testManifest->generate();
+        } catch (\Exception $e) {
+            $message = $e->getMessage() . PHP_EOL;
+            $message .= !empty($filters) ? 'Filter(s): ' . implode(', ', $filters) . PHP_EOL : '';
+            $message .= !empty($tests) ? 'Test name(s): ' . implode(', ', $tests) . PHP_EOL : '';
+            $message .= !empty($json) && empty($tests) ? 'Test configuration: ' . $json . PHP_EOL : '';
+            $ioStyle->note($message);
 
-        $testManifest->generate();
+            return 1;
+        }
 
         $output->writeln("Generate Tests Command Run");
     }
diff --git a/src/Magento/FunctionalTestingFramework/Filter/FilterInterface.php b/src/Magento/FunctionalTestingFramework/Filter/FilterInterface.php
new file mode 100644
index 000000000..bb89ab3c2
--- /dev/null
+++ b/src/Magento/FunctionalTestingFramework/Filter/FilterInterface.php
@@ -0,0 +1,27 @@
+<?php
+/**
+ * Copyright © Magento, Inc. All rights reserved.
+ * See COPYING.txt for license details.
+ */
+
+declare(strict_types=1);
+
+namespace Magento\FunctionalTestingFramework\Filter;
+
+/**
+ * Interface for future test filters
+ * @api
+ */
+interface FilterInterface
+{
+    /**
+     * @param array $filterValues
+     */
+    public function __construct(array $filterValues = []);
+
+    /**
+     * @param array $tests
+     * @return void
+     */
+    public function filter(array &$tests);
+}
diff --git a/src/Magento/FunctionalTestingFramework/Filter/FilterList.php b/src/Magento/FunctionalTestingFramework/Filter/FilterList.php
new file mode 100644
index 000000000..35ff32edd
--- /dev/null
+++ b/src/Magento/FunctionalTestingFramework/Filter/FilterList.php
@@ -0,0 +1,57 @@
+<?php
+/**
+ * Copyright © Magento, Inc. All rights reserved.
+ * See COPYING.txt for license details.
+ */
+
+declare(strict_types=1);
+
+namespace Magento\FunctionalTestingFramework\Filter;
+
+use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException;
+
+/**
+ * Class FilterList has a list of filters.
+ */
+class FilterList
+{
+    /**
+     * List of filters
+     * @var \Magento\FunctionalTestingFramework\Filter\FilterInterface[]
+     */
+    private $filters = [];
+
+    /**
+     * Constructor for Filter list.
+     *
+     * @param array $filters
+     * @throws \Exception
+     */
+    public function __construct(array $filters = [])
+    {
+        foreach ($filters as $filterType => $filterValue) {
+            $className = "Magento\FunctionalTestingFramework\Filter\Test\\" . ucfirst($filterType);
+            if (!class_exists($className)) {
+                throw new TestFrameworkException("Filter type '" . $filterType . "' do not exist.");
+            }
+            $this->filters[$filterType] = new $className($filterValue);
+        }
+    }
+
+    /**
+     * @return array
+     */
+    public function getFilters(): array
+    {
+        return $this->filters;
+    }
+
+    /**
+     * @param string $filterType
+     * @return \Magento\FunctionalTestingFramework\Filter\FilterInterface
+     */
+    public function getFilter(string $filterType): FilterInterface
+    {
+        return $this->filters[$filterType];
+    }
+}
diff --git a/src/Magento/FunctionalTestingFramework/Filter/Test/Severity.php b/src/Magento/FunctionalTestingFramework/Filter/Test/Severity.php
new file mode 100644
index 000000000..cfcb82de1
--- /dev/null
+++ b/src/Magento/FunctionalTestingFramework/Filter/Test/Severity.php
@@ -0,0 +1,69 @@
+<?php
+/**
+ * Copyright © Magento, Inc. All rights reserved.
+ * See COPYING.txt for license details.
+ */
+
+declare(strict_types=1);
+
+namespace Magento\FunctionalTestingFramework\Filter\Test;
+
+use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException;
+use Magento\FunctionalTestingFramework\Filter\FilterInterface;
+
+/**
+ * Class Severity
+ */
+class Severity implements FilterInterface
+{
+    const ANNOTATION_TAG = 'severity';
+    const SEVERITY_VALUES = [
+        "BLOCKER",
+        "CRITICAL",
+        "MAJOR",
+        "AVERAGE",
+        "MINOR",
+    ];
+
+    /**
+     * @var array
+     */
+    private $filterValues = [];
+
+    /**
+     * Severity constructor.
+     *
+     * @param array $filterValues
+     * @throws TestFrameworkException
+     */
+    public function __construct(array $filterValues = [])
+    {
+        if (array_diff($filterValues, self::SEVERITY_VALUES) ==! []) {
+            throw new TestFrameworkException(
+                'Not existing severity specified.' . PHP_EOL
+                . 'Possible values: '. implode(', ', self::SEVERITY_VALUES) . '.' . PHP_EOL
+                . 'Provided values: ' . implode(', ', $filterValues) . '.'  . PHP_EOL
+            );
+        }
+        $this->filterValues = $filterValues;
+    }
+
+    /**
+     * Filter tests by severity.
+     *
+     * @param array $tests
+     * @return void
+     */
+    public function filter(array &$tests)
+    {
+        foreach ($tests as $testName => $test) {
+            if (is_array($test) && !empty($test['annotations'][self::ANNOTATION_TAG])) {
+                foreach ($test['annotations'][self::ANNOTATION_TAG] as $severity) {
+                    if (!in_array($severity['value'], $this->filterValues, true)) {
+                        unset($tests[$testName]);
+                    }
+                }
+            }
+        }
+    }
+}
diff --git a/src/Magento/FunctionalTestingFramework/Suite/Objects/SuiteObject.php b/src/Magento/FunctionalTestingFramework/Suite/Objects/SuiteObject.php
index d7f6bd18a..e10c58681 100644
--- a/src/Magento/FunctionalTestingFramework/Suite/Objects/SuiteObject.php
+++ b/src/Magento/FunctionalTestingFramework/Suite/Objects/SuiteObject.php
@@ -7,6 +7,7 @@
 
 use Magento\FunctionalTestingFramework\Test\Objects\TestHookObject;
 use Magento\FunctionalTestingFramework\Test\Objects\TestObject;
+use Magento\FunctionalTestingFramework\Util\Logger\LoggingUtil;
 
 /**
  * Class SuiteObject
@@ -84,6 +85,7 @@ public function getTests()
      * @param TestObject[] $includeTests
      * @param TestObject[] $excludeTests
      * @return TestObject[]
+     * @throws \Exception
      */
     private function resolveTests($includeTests, $excludeTests)
     {
@@ -96,10 +98,10 @@ private function resolveTests($includeTests, $excludeTests)
         }
 
         if (empty($finalTestList)) {
-            trigger_error(
-                "Current suite configuration for " .
-                $this->name . " contains no tests.",
-                E_USER_WARNING
+            LoggingUtil::getInstance()->getLogger(self::class)->notification(
+                "Current suite configuration for " . $this->name . " contains no tests." . PHP_EOL,
+                [],
+                true
             );
         }
 
diff --git a/src/Magento/FunctionalTestingFramework/Test/Handlers/TestObjectHandler.php b/src/Magento/FunctionalTestingFramework/Test/Handlers/TestObjectHandler.php
index a0d490df5..d5ed43806 100644
--- a/src/Magento/FunctionalTestingFramework/Test/Handlers/TestObjectHandler.php
+++ b/src/Magento/FunctionalTestingFramework/Test/Handlers/TestObjectHandler.php
@@ -16,6 +16,8 @@
 use Magento\FunctionalTestingFramework\Test\Util\ObjectExtensionUtil;
 use Magento\FunctionalTestingFramework\Test\Util\TestObjectExtractor;
 use Magento\FunctionalTestingFramework\Test\Util\AnnotationExtractor;
+use Magento\FunctionalTestingFramework\Util\Logger\LoggingUtil;
+use PHP_CodeSniffer\Tokenizers\PHP;
 
 /**
  * Class TestObjectHandler
@@ -140,7 +142,7 @@ private function initTestData()
         }
 
         $exceptionCollector = new ExceptionCollector();
-        foreach ($parsedTestArray[TestObjectHandler::XML_ROOT] as $testName => $testData) {
+        foreach ($parsedTestArray as $testName => $testData) {
             if (!is_array($testData)) {
                 continue;
             }
diff --git a/src/Magento/FunctionalTestingFramework/Test/Parsers/TestDataParser.php b/src/Magento/FunctionalTestingFramework/Test/Parsers/TestDataParser.php
index 18044c1af..04baca734 100644
--- a/src/Magento/FunctionalTestingFramework/Test/Parsers/TestDataParser.php
+++ b/src/Magento/FunctionalTestingFramework/Test/Parsers/TestDataParser.php
@@ -7,29 +7,58 @@
 namespace Magento\FunctionalTestingFramework\Test\Parsers;
 
 use Magento\FunctionalTestingFramework\Config\DataInterface;
+use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig;
+use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException;
+use Magento\FunctionalTestingFramework\Filter\FilterInterface;
+use Magento\FunctionalTestingFramework\Util\Logger\LoggingUtil;
 
 /**
  * Class TestDataParser
  */
 class TestDataParser
 {
+    /**
+     * @var DataInterface
+     */
+    private $testData;
+
+    /**
+     * @var \Magento\FunctionalTestingFramework\Filter\FilterList
+     */
+    private $filterList;
+
     /**
      * TestDataParser constructor.
      *
      * @param DataInterface $testData
+     * @throws \Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException
      */
     public function __construct(DataInterface $testData)
     {
-        $this->testData = $testData;
+        $this->testData = array_filter($testData->get('tests'), function ($value) {
+            return is_array($value);
+        });
+
+        $this->filterList = MftfApplicationConfig::getConfig()->getFilterList();
     }
 
     /**
      * Returns an array of data based on *Test.xml files
      *
      * @return array
+     * @throws TestFrameworkException
      */
     public function readTestData()
     {
-        return $this->testData->get();
+        /** @var FilterInterface $filter */
+        foreach ($this->filterList->getFilters() as $filter) {
+            $filter->filter($this->testData);
+        }
+
+        if (empty($this->testData)) {
+            throw new TestFrameworkException("No tests found.");
+        }
+
+        return $this->testData;
     }
 }
diff --git a/src/Magento/FunctionalTestingFramework/Util/Logger/MftfLogger.php b/src/Magento/FunctionalTestingFramework/Util/Logger/MftfLogger.php
index 3983a755a..0e8c1e17f 100644
--- a/src/Magento/FunctionalTestingFramework/Util/Logger/MftfLogger.php
+++ b/src/Magento/FunctionalTestingFramework/Util/Logger/MftfLogger.php
@@ -49,4 +49,23 @@ public function criticalFailure($message, array $context = [], $verbose = false)
         }
         parent::critical($message, $context);
     }
+
+    /**
+     * Adds a log record at the NOTICE level.
+     *
+     * @param string  $message
+     * @param array   $context
+     * @param boolean $verbose
+     * @return void
+     * @throws \Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException
+     */
+    public function notification($message, array $context = [], $verbose = false)
+    {
+        $message = "NOTICE: " . $message;
+        // Suppress print during unit testing
+        if (MftfApplicationConfig::getConfig()->getPhase() !== MftfApplicationConfig::UNIT_TEST_PHASE && $verbose) {
+            print ($message . implode("\n", $context) . "\n");
+        }
+        parent::notice($message, $context);
+    }
 }

From 269a4e506be9a821d37b7f3c893ccdff33c68150 Mon Sep 17 00:00:00 2001
From: Alex Kolesnyk <kolesnyk@adobe.com>
Date: Wed, 26 Feb 2020 10:19:31 -0600
Subject: [PATCH 2/9] MQE-2008: Filter test generation and execution by
 severity

---
 docs/commands/mftf.md                                           | 1 +
 .../Test/Handlers/TestObjectHandler.php                         | 2 --
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/docs/commands/mftf.md b/docs/commands/mftf.md
index b2c36dae4..e0394dc12 100644
--- a/docs/commands/mftf.md
+++ b/docs/commands/mftf.md
@@ -159,6 +159,7 @@ vendor/bin/mftf generate:tests [option] [<test name>] [<test name>] [--remove]
 | Option | Description|
 | ---| --- |
 | `--config=[<default> or <singleRun> or <parallel>]` | Creates a single manifest file with a list of all tests. The default location is `tests/functional/Magento/FunctionalTest/_generated/testManifest.txt`.<br/> You can split the list into multiple groups using `--config=parallel`; the groups will be generated in `_generated/groups/` like `_generated/groups/group1.txt, group2.txt, ...`.<br/> Available values: `default` (default), `singleRun`(same as `default`), and `parallel`.<br/> Example: `generate:tests --config=parallel`. |
+| `--filter` | Option to filter tests to be generated.<br/>Template: '<filterName>:<filterValue>'.<br/>Existing filter types: severity.<br/>Existing severity values: BLOCKER, CRITICAL, MAJOR, AVERAGE, MINOR.<br/>Example: --filter=severity:CRITICAL|
 | `--force` | Forces test generation, regardless of the module merge order defined in the Magento instance. Example: `generate:tests --force`. |
 | `-i,--time` | Set time in minutes to determine the group size when `--config=parallel` is used. The __default value__ is `10`. Example: `generate:tests --config=parallel --time=15`|
 | `--tests` | Defines the test configuration as a JSON string.|
diff --git a/src/Magento/FunctionalTestingFramework/Test/Handlers/TestObjectHandler.php b/src/Magento/FunctionalTestingFramework/Test/Handlers/TestObjectHandler.php
index d5ed43806..18d3cd184 100644
--- a/src/Magento/FunctionalTestingFramework/Test/Handlers/TestObjectHandler.php
+++ b/src/Magento/FunctionalTestingFramework/Test/Handlers/TestObjectHandler.php
@@ -24,8 +24,6 @@
  */
 class TestObjectHandler implements ObjectHandlerInterface
 {
-    const XML_ROOT = 'tests';
-
     /**
      * Test Object Handler
      *

From 7758075324312085720017f98fe61af467ff544b Mon Sep 17 00:00:00 2001
From: Alex Kolesnyk <kolesnyk@adobe.com>
Date: Wed, 26 Feb 2020 16:50:49 -0600
Subject: [PATCH 3/9] MQE-2008: Filter test generation and execution by
 severity

---
 .../Filter/Test/Severity.php                     | 13 +++++++------
 .../Suite/Objects/SuiteObject.php                | 12 ++++++------
 .../Suite/SuiteGenerator.php                     |  6 ++++++
 .../Test/Handlers/TestObjectHandler.php          |  2 ++
 .../Test/Parsers/TestDataParser.php              | 16 ----------------
 .../Util/Manifest/DefaultTestManifest.php        |  3 +++
 .../Util/Manifest/ParallelTestManifest.php       |  2 +-
 .../Util/TestGenerator.php                       | 10 ++++++++++
 8 files changed, 35 insertions(+), 29 deletions(-)

diff --git a/src/Magento/FunctionalTestingFramework/Filter/Test/Severity.php b/src/Magento/FunctionalTestingFramework/Filter/Test/Severity.php
index cfcb82de1..9c3aa0e7a 100644
--- a/src/Magento/FunctionalTestingFramework/Filter/Test/Severity.php
+++ b/src/Magento/FunctionalTestingFramework/Filter/Test/Severity.php
@@ -10,6 +10,7 @@
 
 use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException;
 use Magento\FunctionalTestingFramework\Filter\FilterInterface;
+use Magento\FunctionalTestingFramework\Test\Objects\TestObject;
 
 /**
  * Class Severity
@@ -51,17 +52,17 @@ public function __construct(array $filterValues = [])
     /**
      * Filter tests by severity.
      *
-     * @param array $tests
+     * @param TestObject[] $tests
      * @return void
      */
     public function filter(array &$tests)
     {
+        /** @var TestObject $test */
         foreach ($tests as $testName => $test) {
-            if (is_array($test) && !empty($test['annotations'][self::ANNOTATION_TAG])) {
-                foreach ($test['annotations'][self::ANNOTATION_TAG] as $severity) {
-                    if (!in_array($severity['value'], $this->filterValues, true)) {
-                        unset($tests[$testName]);
-                    }
+            $severities = $test->getAnnotationByName(self::ANNOTATION_TAG);
+            foreach ($severities as $severity) {
+                if (!in_array($severity, $this->filterValues, true)) {
+                    unset($tests[$testName]);
                 }
             }
         }
diff --git a/src/Magento/FunctionalTestingFramework/Suite/Objects/SuiteObject.php b/src/Magento/FunctionalTestingFramework/Suite/Objects/SuiteObject.php
index e10c58681..7168d9556 100644
--- a/src/Magento/FunctionalTestingFramework/Suite/Objects/SuiteObject.php
+++ b/src/Magento/FunctionalTestingFramework/Suite/Objects/SuiteObject.php
@@ -5,6 +5,8 @@
  */
 namespace Magento\FunctionalTestingFramework\Suite\Objects;
 
+use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig;
+use Magento\FunctionalTestingFramework\Filter\FilterInterface;
 use Magento\FunctionalTestingFramework\Test\Objects\TestHookObject;
 use Magento\FunctionalTestingFramework\Test\Objects\TestObject;
 use Magento\FunctionalTestingFramework\Util\Logger\LoggingUtil;
@@ -97,12 +99,10 @@ private function resolveTests($includeTests, $excludeTests)
             unset($finalTestList[$testName]);
         }
 
-        if (empty($finalTestList)) {
-            LoggingUtil::getInstance()->getLogger(self::class)->notification(
-                "Current suite configuration for " . $this->name . " contains no tests." . PHP_EOL,
-                [],
-                true
-            );
+        $filters = MftfApplicationConfig::getConfig()->getFilterList()->getFilters();
+        /** @var FilterInterface $filter */
+        foreach ($filters as $filter) {
+            $filter->filter($finalTestList);
         }
 
         return $finalTestList;
diff --git a/src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php b/src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php
index e3da10497..7aec4f21f 100644
--- a/src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php
+++ b/src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php
@@ -95,6 +95,12 @@ public function generateAllSuites($testManifest)
         $suites = $testManifest->getSuiteConfig();
 
         foreach ($suites as $suiteName => $suiteContent) {
+            if (empty($suiteContent)) {
+                LoggingUtil::getInstance()->getLogger(self::class)->notification(
+                    "Suite '" . $suiteName . "' contains no tests and won't be generated." . PHP_EOL, [], true
+                );
+                continue;
+            }
             $firstElement = array_values($suiteContent)[0];
 
             // if the first element is a string we know that we simply have an array of tests
diff --git a/src/Magento/FunctionalTestingFramework/Test/Handlers/TestObjectHandler.php b/src/Magento/FunctionalTestingFramework/Test/Handlers/TestObjectHandler.php
index 18d3cd184..d5ed43806 100644
--- a/src/Magento/FunctionalTestingFramework/Test/Handlers/TestObjectHandler.php
+++ b/src/Magento/FunctionalTestingFramework/Test/Handlers/TestObjectHandler.php
@@ -24,6 +24,8 @@
  */
 class TestObjectHandler implements ObjectHandlerInterface
 {
+    const XML_ROOT = 'tests';
+
     /**
      * Test Object Handler
      *
diff --git a/src/Magento/FunctionalTestingFramework/Test/Parsers/TestDataParser.php b/src/Magento/FunctionalTestingFramework/Test/Parsers/TestDataParser.php
index 04baca734..dc3489de1 100644
--- a/src/Magento/FunctionalTestingFramework/Test/Parsers/TestDataParser.php
+++ b/src/Magento/FunctionalTestingFramework/Test/Parsers/TestDataParser.php
@@ -22,11 +22,6 @@ class TestDataParser
      */
     private $testData;
 
-    /**
-     * @var \Magento\FunctionalTestingFramework\Filter\FilterList
-     */
-    private $filterList;
-
     /**
      * TestDataParser constructor.
      *
@@ -38,8 +33,6 @@ public function __construct(DataInterface $testData)
         $this->testData = array_filter($testData->get('tests'), function ($value) {
             return is_array($value);
         });
-
-        $this->filterList = MftfApplicationConfig::getConfig()->getFilterList();
     }
 
     /**
@@ -50,15 +43,6 @@ public function __construct(DataInterface $testData)
      */
     public function readTestData()
     {
-        /** @var FilterInterface $filter */
-        foreach ($this->filterList->getFilters() as $filter) {
-            $filter->filter($this->testData);
-        }
-
-        if (empty($this->testData)) {
-            throw new TestFrameworkException("No tests found.");
-        }
-
         return $this->testData;
     }
 }
diff --git a/src/Magento/FunctionalTestingFramework/Util/Manifest/DefaultTestManifest.php b/src/Magento/FunctionalTestingFramework/Util/Manifest/DefaultTestManifest.php
index eb4f79db2..4cbef3168 100644
--- a/src/Magento/FunctionalTestingFramework/Util/Manifest/DefaultTestManifest.php
+++ b/src/Magento/FunctionalTestingFramework/Util/Manifest/DefaultTestManifest.php
@@ -86,6 +86,9 @@ public function generate()
     protected function generateSuiteEntries($fileResource)
     {
         foreach ($this->getSuiteConfig() as $suiteName => $tests) {
+            if (count($tests) === 0) {
+                continue;
+            }
             $line = "-g {$suiteName}";
             fwrite($fileResource, $line . PHP_EOL);
         }
diff --git a/src/Magento/FunctionalTestingFramework/Util/Manifest/ParallelTestManifest.php b/src/Magento/FunctionalTestingFramework/Util/Manifest/ParallelTestManifest.php
index 9b12cfd00..201e002a2 100644
--- a/src/Magento/FunctionalTestingFramework/Util/Manifest/ParallelTestManifest.php
+++ b/src/Magento/FunctionalTestingFramework/Util/Manifest/ParallelTestManifest.php
@@ -132,7 +132,7 @@ private function generateGroupFile($testGroup, $nodeNumber, $suites)
             $fileResource = fopen($this->dirPath . DIRECTORY_SEPARATOR . "group{$nodeNumber}.txt", 'a');
 
             $line = null;
-            if (array_key_exists($entryName, $suites)) {
+            if (!empty($suites[$entryName])) {
                 $line = "-g {$entryName}";
             } else {
                 $line = $this->relativeDirPath . DIRECTORY_SEPARATOR . $entryName . '.php';
diff --git a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php
index 2ec565c44..d68010e7d 100644
--- a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php
+++ b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php
@@ -11,6 +11,7 @@
 use Magento\FunctionalTestingFramework\DataGenerator\Objects\EntityDataObject;
 use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException;
 use Magento\FunctionalTestingFramework\Exceptions\TestReferenceException;
+use Magento\FunctionalTestingFramework\Filter\FilterInterface;
 use Magento\FunctionalTestingFramework\Suite\Handlers\SuiteObjectHandler;
 use Magento\FunctionalTestingFramework\Test\Handlers\ActionGroupObjectHandler;
 use Magento\FunctionalTestingFramework\Test\Handlers\TestObjectHandler;
@@ -171,6 +172,9 @@ private function loadAllTestObjects($testsToIgnore)
     {
         if ($this->tests === null || empty($this->tests)) {
             $testObjects = TestObjectHandler::getInstance()->getAllObjects();
+
+
+
             return array_diff_key($testObjects, $testsToIgnore);
         }
 
@@ -288,6 +292,12 @@ private function assembleAllTestPhp($testManifest, array $testsToIgnore)
         /** @var TestObject[] $testObjects */
         $testObjects = $this->loadAllTestObjects($testsToIgnore);
         $cestPhpArray = [];
+        $filters = MftfApplicationConfig::getConfig()->getFilterList()->getFilters();
+        /** @var FilterInterface $filter */
+        foreach ($filters as $filter) {
+            $filter->filter($testObjects);
+        }
+
 
         foreach ($testObjects as $test) {
             // Do not generate test if it is an extended test and parent does not exist

From a9f5bba4197a442b137f94cb77b8b4da4951afae Mon Sep 17 00:00:00 2001
From: Alex Kolesnyk <kolesnyk@adobe.com>
Date: Wed, 26 Feb 2020 21:53:15 -0600
Subject: [PATCH 4/9] MQE-2008: Filter test generation and execution by
 severity

---
 .../Filter/Test/Severity.php                  | 26 +++++++++----------
 .../Suite/SuiteGenerator.php                  |  4 ++-
 .../Util/TestGenerator.php                    |  4 ---
 3 files changed, 15 insertions(+), 19 deletions(-)

diff --git a/src/Magento/FunctionalTestingFramework/Filter/Test/Severity.php b/src/Magento/FunctionalTestingFramework/Filter/Test/Severity.php
index 9c3aa0e7a..e88c4af7b 100644
--- a/src/Magento/FunctionalTestingFramework/Filter/Test/Severity.php
+++ b/src/Magento/FunctionalTestingFramework/Filter/Test/Severity.php
@@ -11,6 +11,7 @@
 use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException;
 use Magento\FunctionalTestingFramework\Filter\FilterInterface;
 use Magento\FunctionalTestingFramework\Test\Objects\TestObject;
+use Magento\FunctionalTestingFramework\Test\Util\AnnotationExtractor;
 
 /**
  * Class Severity
@@ -18,13 +19,6 @@
 class Severity implements FilterInterface
 {
     const ANNOTATION_TAG = 'severity';
-    const SEVERITY_VALUES = [
-        "BLOCKER",
-        "CRITICAL",
-        "MAJOR",
-        "AVERAGE",
-        "MINOR",
-    ];
 
     /**
      * @var array
@@ -39,14 +33,18 @@ class Severity implements FilterInterface
      */
     public function __construct(array $filterValues = [])
     {
-        if (array_diff($filterValues, self::SEVERITY_VALUES) ==! []) {
-            throw new TestFrameworkException(
-                'Not existing severity specified.' . PHP_EOL
-                . 'Possible values: '. implode(', ', self::SEVERITY_VALUES) . '.' . PHP_EOL
-                . 'Provided values: ' . implode(', ', $filterValues) . '.'  . PHP_EOL
-            );
+        $severityValues = AnnotationExtractor::MAGENTO_TO_ALLURE_SEVERITY_MAP;
+
+        foreach ($filterValues as $filterValue) {
+            if (!isset($severityValues[$filterValue])) {
+                throw new TestFrameworkException(
+                    'Not existing severity specified.' . PHP_EOL
+                    . 'Possible values: '. implode(', ', array_keys($severityValues)) . '.' . PHP_EOL
+                    . 'Provided values: ' . implode(', ', $filterValues) . '.'  . PHP_EOL
+                );
+            }
+            $this->filterValues[] = $severityValues[$filterValue];
         }
-        $this->filterValues = $filterValues;
     }
 
     /**
diff --git a/src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php b/src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php
index 7aec4f21f..b40b48ae7 100644
--- a/src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php
+++ b/src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php
@@ -97,7 +97,9 @@ public function generateAllSuites($testManifest)
         foreach ($suites as $suiteName => $suiteContent) {
             if (empty($suiteContent)) {
                 LoggingUtil::getInstance()->getLogger(self::class)->notification(
-                    "Suite '" . $suiteName . "' contains no tests and won't be generated." . PHP_EOL, [], true
+                    "Suite '" . $suiteName . "' contains no tests and won't be generated." . PHP_EOL,
+                    [],
+                    true
                 );
                 continue;
             }
diff --git a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php
index d68010e7d..2e7d265fa 100644
--- a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php
+++ b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php
@@ -172,9 +172,6 @@ private function loadAllTestObjects($testsToIgnore)
     {
         if ($this->tests === null || empty($this->tests)) {
             $testObjects = TestObjectHandler::getInstance()->getAllObjects();
-
-
-
             return array_diff_key($testObjects, $testsToIgnore);
         }
 
@@ -298,7 +295,6 @@ private function assembleAllTestPhp($testManifest, array $testsToIgnore)
             $filter->filter($testObjects);
         }
 
-
         foreach ($testObjects as $test) {
             // Do not generate test if it is an extended test and parent does not exist
             if ($test->isSkipped() && !empty($test->getParentName())) {

From 1c7b03b63e249e6990c5b28a7675b94b757f9c4f Mon Sep 17 00:00:00 2001
From: Kevin Kozan <kkozan@adobe.com>
Date: Thu, 27 Feb 2020 10:17:01 -0600
Subject: [PATCH 5/9] MQE-2008: Filter test generation and execution by
 severity

- Unit test
---
 .../Util/TestGeneratorTest.php                | 60 +++++++++++++++++++
 1 file changed, 60 insertions(+)

diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Util/TestGeneratorTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Util/TestGeneratorTest.php
index 8f8eee749..ad51d3681 100644
--- a/dev/tests/unit/Magento/FunctionalTestFramework/Util/TestGeneratorTest.php
+++ b/dev/tests/unit/Magento/FunctionalTestFramework/Util/TestGeneratorTest.php
@@ -8,6 +8,7 @@
 
 use AspectMock\Test as AspectMock;
 
+use Magento\FunctionalTestingFramework\Filter\FilterList;
 use Magento\FunctionalTestingFramework\Test\Objects\ActionObject;
 use Magento\FunctionalTestingFramework\Test\Objects\TestHookObject;
 use Magento\FunctionalTestingFramework\Test\Objects\TestObject;
@@ -17,6 +18,16 @@
 
 class TestGeneratorTest extends MagentoTestCase
 {
+    /**
+     * After method functionality
+     *
+     * @return void
+     */
+    public function tearDown()
+    {
+        AspectMock::clean();
+    }
+
     /**
      * Basic test to check exceptions for incorrect entities.
      *
@@ -99,4 +110,53 @@ public function testAllowSkipped()
         $this->assertContains($actionInput, $output);
         $this->assertContains($beforeActionInput, $output);
     }
+
+    /**
+     * Tests that TestGenerator createAllTestFiles correctly filters based on severity
+     *
+     * @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException
+     */
+    public function testFilter()
+    {
+        // Mock filters for TestGenerator
+        AspectMock::double(MftfApplicationConfig::class, ['getFilterList' => new FilterList(['severity' => ["CRITICAL"]])]);
+
+        $actionInput = 'fakeInput';
+        $actionObject = new ActionObject('fakeAction', 'comment', [
+            'userInput' => $actionInput
+        ]);
+
+        $annotation1 = ['severity' => ['CRITICAL']];
+        $annotation2 = ['severity' => ['MINOR']];
+        $test1 = new TestObject(
+            "test1",
+            ["fakeAction" => $actionObject],
+            $annotation1,
+            [],
+            "filename"
+        );
+        $test2 = new TestObject(
+            "test2",
+            ["fakeAction" => $actionObject],
+            $annotation2,
+            [],
+            "filename"
+        );
+        AspectMock::double(TestGenerator::class, ['loadAllTestObjects' => ["sampleTest" => $test1, "test2" => $test2]]);
+
+        // Mock createCestFile to return name of tests that testGenerator tried to create
+        $generatedTests = [];
+        AspectMock::double(TestGenerator::class, ['createCestFile' => function ($arg1, $arg2) use (&$generatedTests){
+            $generatedTests[$arg2] = true;
+        }]);
+
+        $testGeneratorObject = TestGenerator::getInstance("", ["sampleTest" => $test1, "test2" => $test2]);
+        $testGeneratorObject->createAllTestFiles(null, []);
+
+        // Ensure Test1 was Generated but not Test 2
+        $this->assertArrayHasKey('test1Cest', $generatedTests);
+        $this->assertArrayNotHasKey('test2Cest', $generatedTests);
+
+    }
+
 }

From afc5551f8d024c6a4cb1b8fe0d9dbb1a1f85bc11 Mon Sep 17 00:00:00 2001
From: Kevin Kozan <kkozan@adobe.com>
Date: Thu, 27 Feb 2020 10:19:37 -0600
Subject: [PATCH 6/9] MQE-2008: Filter test generation and execution by
 severity

- Static Fix
---
 .../Magento/FunctionalTestFramework/Util/TestGeneratorTest.php  | 2 --
 1 file changed, 2 deletions(-)

diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Util/TestGeneratorTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Util/TestGeneratorTest.php
index ad51d3681..cd930a997 100644
--- a/dev/tests/unit/Magento/FunctionalTestFramework/Util/TestGeneratorTest.php
+++ b/dev/tests/unit/Magento/FunctionalTestFramework/Util/TestGeneratorTest.php
@@ -156,7 +156,5 @@ public function testFilter()
         // Ensure Test1 was Generated but not Test 2
         $this->assertArrayHasKey('test1Cest', $generatedTests);
         $this->assertArrayNotHasKey('test2Cest', $generatedTests);
-
     }
-
 }

From a6ec117400c80c0aff10961029a3d2957e343ae1 Mon Sep 17 00:00:00 2001
From: Alex Kolesnyk <kolesnyk@adobe.com>
Date: Thu, 27 Feb 2020 11:00:03 -0600
Subject: [PATCH 7/9] MQE-2008: Filter test generation and execution by
 severity

---
 .../FunctionalTestFramework/Util/TestGeneratorTest.php     | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Util/TestGeneratorTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Util/TestGeneratorTest.php
index cd930a997..e11b0f9b2 100644
--- a/dev/tests/unit/Magento/FunctionalTestFramework/Util/TestGeneratorTest.php
+++ b/dev/tests/unit/Magento/FunctionalTestFramework/Util/TestGeneratorTest.php
@@ -119,7 +119,10 @@ public function testAllowSkipped()
     public function testFilter()
     {
         // Mock filters for TestGenerator
-        AspectMock::double(MftfApplicationConfig::class, ['getFilterList' => new FilterList(['severity' => ["CRITICAL"]])]);
+        AspectMock::double(
+            MftfApplicationConfig::class,
+            ['getFilterList' => new FilterList(['severity' => ["CRITICAL"]])]
+        );
 
         $actionInput = 'fakeInput';
         $actionObject = new ActionObject('fakeAction', 'comment', [
@@ -146,7 +149,7 @@ public function testFilter()
 
         // Mock createCestFile to return name of tests that testGenerator tried to create
         $generatedTests = [];
-        AspectMock::double(TestGenerator::class, ['createCestFile' => function ($arg1, $arg2) use (&$generatedTests){
+        AspectMock::double(TestGenerator::class, ['createCestFile' => function ($arg1, $arg2) use (&$generatedTests) {
             $generatedTests[$arg2] = true;
         }]);
 

From 75f5224674f629b44fe47b24f5536ed9c030c0d5 Mon Sep 17 00:00:00 2001
From: Alex Kolesnyk <kolesnyk@adobe.com>
Date: Thu, 27 Feb 2020 11:40:49 -0600
Subject: [PATCH 8/9] MQE-2008: Filter test generation and execution by
 severity

---
 .../Console/BaseGenerateCommand.php                         | 2 +-
 .../Console/GenerateTestsCommand.php                        | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php
index 3433f23a8..273632586 100644
--- a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php
+++ b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php
@@ -35,7 +35,7 @@ class BaseGenerateCommand extends Command
      *
      * @var SymfonyStyle
      */
-    private $ioStyle = null;
+    protected $ioStyle = null;
 
     /**
      * Configures the base command.
diff --git a/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php b/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php
index f43488c0b..3ca53f8f6 100644
--- a/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php
+++ b/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php
@@ -77,7 +77,7 @@ protected function configure()
      */
     protected function execute(InputInterface $input, OutputInterface $output)
     {
-        $ioStyle = new SymfonyStyle($input, $output);
+        $this->setOutputStyle($input, $output);
         $tests = $input->getArgument('name');
         $config = $input->getOption('config');
         $json = $input->getOption('tests'); // for backward compatibility
@@ -103,7 +103,7 @@ protected function execute(InputInterface $input, OutputInterface $output)
                 $filterList ?? []
             );
         } catch (\Exception $exception) {
-            $ioStyle->error("Test generation failed." . PHP_EOL . $exception->getMessage());
+            $this->ioStyle->error("Test generation failed." . PHP_EOL . $exception->getMessage());
             return 1;
         }
 
@@ -151,7 +151,7 @@ protected function execute(InputInterface $input, OutputInterface $output)
             $message .= !empty($filters) ? 'Filter(s): ' . implode(', ', $filters) . PHP_EOL : '';
             $message .= !empty($tests) ? 'Test name(s): ' . implode(', ', $tests) . PHP_EOL : '';
             $message .= !empty($json) && empty($tests) ? 'Test configuration: ' . $json . PHP_EOL : '';
-            $ioStyle->note($message);
+            $this->ioStyle->note($message);
 
             return 1;
         }

From c215ab0ac08e6a86884e6cda5406fe128b0aa0b9 Mon Sep 17 00:00:00 2001
From: Alex Kolesnyk <kolesnyk@adobe.com>
Date: Thu, 27 Feb 2020 12:28:47 -0600
Subject: [PATCH 9/9] MQE-2008: Filter test generation and execution by
 severity

- update composer.json and CHANGELOG.md
---
 CHANGELOG.md  | 5 +++++
 composer.json | 2 +-
 composer.lock | 2 +-
 3 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4792fa8ae..a41360787 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,10 @@
 Magento Functional Testing Framework Changelog
 ================================================
+2.6.3
+-----
+
+### New Feature
+* `--filter` option was added to `bin/mftf generate:tests` command. For more details please go to https://devdocs.magento.com/mftf/docs/commands/mftf.html#generatetests
 
 2.6.2
 -----
diff --git a/composer.json b/composer.json
index ed4656a37..0ff50a2e8 100755
--- a/composer.json
+++ b/composer.json
@@ -2,7 +2,7 @@
     "name": "magento/magento2-functional-testing-framework",
     "description": "Magento2 Functional Testing Framework",
     "type": "library",
-    "version": "2.6.2",
+    "version": "2.6.3",
     "license": "AGPL-3.0",
     "keywords": ["magento", "automation", "functional", "testing"],
     "config": {
diff --git a/composer.lock b/composer.lock
index caa95e992..6f370593b 100644
--- a/composer.lock
+++ b/composer.lock
@@ -4,7 +4,7 @@
         "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
         "This file is @generated automatically"
     ],
-    "content-hash": "d9ea4056a8f4501c3f2766e09edce40d",
+    "content-hash": "83e4e17679bff5fdd472c246dca8ac48",
     "packages": [
         {
             "name": "allure-framework/allure-codeception",