diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4792fa8ae..a41360787 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,10 @@
Magento Functional Testing Framework Changelog
================================================
+2.6.3
+-----
+
+### New Feature
+* `--filter` option was added to `bin/mftf generate:tests` command. For more details please go to https://devdocs.magento.com/mftf/docs/commands/mftf.html#generatetests
2.6.2
-----
diff --git a/composer.json b/composer.json
index ed4656a37..0ff50a2e8 100755
--- a/composer.json
+++ b/composer.json
@@ -2,7 +2,7 @@
"name": "magento/magento2-functional-testing-framework",
"description": "Magento2 Functional Testing Framework",
"type": "library",
- "version": "2.6.2",
+ "version": "2.6.3",
"license": "AGPL-3.0",
"keywords": ["magento", "automation", "functional", "testing"],
"config": {
diff --git a/composer.lock b/composer.lock
index caa95e992..6f370593b 100644
--- a/composer.lock
+++ b/composer.lock
@@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
- "content-hash": "d9ea4056a8f4501c3f2766e09edce40d",
+ "content-hash": "83e4e17679bff5fdd472c246dca8ac48",
"packages": [
{
"name": "allure-framework/allure-codeception",
diff --git a/dev/tests/functional/tests/MFTF/DevDocs/Test/DeprecatedDevDocsTest.xml b/dev/tests/functional/tests/MFTF/DevDocs/Test/DeprecatedDevDocsTest.xml
index bf0078df4..c93dc8011 100644
--- a/dev/tests/functional/tests/MFTF/DevDocs/Test/DeprecatedDevDocsTest.xml
+++ b/dev/tests/functional/tests/MFTF/DevDocs/Test/DeprecatedDevDocsTest.xml
@@ -15,7 +15,7 @@
-
+
diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Suite/Handlers/SuiteObjectHandlerTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Suite/Handlers/SuiteObjectHandlerTest.php
index 5bf29507c..e45127f4c 100644
--- a/dev/tests/unit/Magento/FunctionalTestFramework/Suite/Handlers/SuiteObjectHandlerTest.php
+++ b/dev/tests/unit/Magento/FunctionalTestFramework/Suite/Handlers/SuiteObjectHandlerTest.php
@@ -65,7 +65,7 @@ public function testGetSuiteObject()
->withTestActions()
->build();
- $mockTestData = ['tests' => array_merge($mockSimpleTest, $mockGroup1Test1, $mockGroup1Test2, $mockGroup2Test1)];
+ $mockTestData = array_merge($mockSimpleTest, $mockGroup1Test1, $mockGroup1Test2, $mockGroup2Test1);
$this->setMockTestAndSuiteParserOutput($mockTestData, $mockData);
// parse and retrieve suite object with mocked data
diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Suite/SuiteGeneratorTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Suite/SuiteGeneratorTest.php
index e3f820ede..842be4f88 100644
--- a/dev/tests/unit/Magento/FunctionalTestFramework/Suite/SuiteGeneratorTest.php
+++ b/dev/tests/unit/Magento/FunctionalTestFramework/Suite/SuiteGeneratorTest.php
@@ -69,7 +69,7 @@ public function testGenerateSuite()
->withTestActions()
->build();
- $mockTestData = ['tests' => array_merge($mockSimpleTest)];
+ $mockTestData = array_merge($mockSimpleTest);
$this->setMockTestAndSuiteParserOutput($mockTestData, $mockData);
// parse and generate suite object with mocked data
@@ -105,7 +105,7 @@ public function testGenerateAllSuites()
->withTestActions()
->build();
- $mockTestData = ['tests' => array_merge($mockSimpleTest)];
+ $mockTestData = array_merge($mockSimpleTest);
$this->setMockTestAndSuiteParserOutput($mockTestData, $mockData);
// parse and retrieve suite object with mocked data
@@ -172,7 +172,7 @@ public function testInvalidSuiteTestPair()
->withAnnotations(['group' => [['value' => 'group2']]])
->withTestActions()
->build();
- $mockTestData = ['tests' => array_merge($mockSimpleTest, $mockSimpleTest2)];
+ $mockTestData = array_merge($mockSimpleTest, $mockSimpleTest2);
$this->setMockTestAndSuiteParserOutput($mockTestData, $mockSuiteData);
// Make invalid manifest
@@ -196,7 +196,7 @@ public function testNonExistentSuiteTestPair()
->withAnnotations(['group' => [['value' => 'group1']]])
->withTestActions()
->build();
- $mockTestData = ['tests' => array_merge($mockSimpleTest)];
+ $mockTestData = array_merge($mockSimpleTest);
$this->setMockTestAndSuiteParserOutput($mockTestData, []);
// Make invalid manifest
diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Test/Handlers/TestObjectHandlerTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Test/Handlers/TestObjectHandlerTest.php
index 67f6f2b43..a4504ab07 100644
--- a/dev/tests/unit/Magento/FunctionalTestFramework/Test/Handlers/TestObjectHandlerTest.php
+++ b/dev/tests/unit/Magento/FunctionalTestFramework/Test/Handlers/TestObjectHandlerTest.php
@@ -41,7 +41,7 @@ public function testGetTestObject()
$resolverMock = new MockModuleResolverBuilder();
$resolverMock->setup();
- $this->setMockParserOutput(['tests' => $mockData]);
+ $this->setMockParserOutput($mockData);
// run object handler method
$toh = TestObjectHandler::getInstance();
@@ -135,7 +135,7 @@ public function testGetTestsByGroup()
$resolverMock = new MockModuleResolverBuilder();
$resolverMock->setup();
- $this->setMockParserOutput(['tests' => array_merge($includeTest, $excludeTest)]);
+ $this->setMockParserOutput(array_merge($includeTest, $excludeTest));
// execute test method
$toh = TestObjectHandler::getInstance();
@@ -184,7 +184,7 @@ public function testGetTestWithModuleName()
$resolverMock = new MockModuleResolverBuilder();
$resolverMock->setup(['Vendor_' . $moduleExpected => $filepath]);
- $this->setMockParserOutput(['tests' => $mockData]);
+ $this->setMockParserOutput($mockData);
// Execute Test Method
$toh = TestObjectHandler::getInstance();
$actualTestObject = $toh->getObject($testDataArrayBuilder->testName);
@@ -212,7 +212,7 @@ public function testGetTestObjectWithInvalidExtends()
->build();
$resolverMock = new MockModuleResolverBuilder();
$resolverMock->setup();
- $this->setMockParserOutput(['tests' => $testOne]);
+ $this->setMockParserOutput($testOne);
$toh = TestObjectHandler::getInstance();
@@ -250,7 +250,7 @@ public function testGetAllTestObjectsWithInvalidExtends()
$resolverMock = new MockModuleResolverBuilder();
$resolverMock->setup();
- $this->setMockParserOutput(['tests' => array_merge($testOne, $testTwo)]);
+ $this->setMockParserOutput(array_merge($testOne, $testTwo));
$toh = TestObjectHandler::getInstance();
diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Test/Util/ObjectExtensionUtilTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Test/Util/ObjectExtensionUtilTest.php
index d86185b83..9982b8040 100644
--- a/dev/tests/unit/Magento/FunctionalTestFramework/Test/Util/ObjectExtensionUtilTest.php
+++ b/dev/tests/unit/Magento/FunctionalTestFramework/Test/Util/ObjectExtensionUtilTest.php
@@ -67,7 +67,7 @@ public function testGenerateExtendedTest()
->withTestReference("simpleTest")
->build();
- $mockTestData = ['tests' => array_merge($mockSimpleTest, $mockExtendedTest)];
+ $mockTestData = array_merge($mockSimpleTest, $mockExtendedTest);
$this->setMockTestOutput($mockTestData);
// parse and generate test object with mocked data
@@ -112,7 +112,7 @@ public function testGenerateExtendedWithHooks()
->withTestReference("simpleTest")
->build();
- $mockTestData = ['tests' => array_merge($mockSimpleTest, $mockExtendedTest)];
+ $mockTestData = array_merge($mockSimpleTest, $mockExtendedTest);
$this->setMockTestOutput($mockTestData);
// parse and generate test object with mocked data
@@ -143,7 +143,7 @@ public function testExtendedTestNoParent()
->withTestReference("simpleTest")
->build();
- $mockTestData = ['tests' => array_merge($mockExtendedTest)];
+ $mockTestData = array_merge($mockExtendedTest);
$this->setMockTestOutput($mockTestData);
// parse and generate test object with mocked data
@@ -182,7 +182,7 @@ public function testExtendingExtendedTest()
->withTestReference("simpleTest")
->build();
- $mockTestData = ['tests' => array_merge($mockParentTest, $mockSimpleTest, $mockExtendedTest)];
+ $mockTestData = array_merge($mockParentTest, $mockSimpleTest, $mockExtendedTest);
$this->setMockTestOutput($mockTestData);
$this->expectExceptionMessage("Cannot extend a test that already extends another test. Test: simpleTest");
diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Util/TestGeneratorTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Util/TestGeneratorTest.php
index 8f8eee749..e11b0f9b2 100644
--- a/dev/tests/unit/Magento/FunctionalTestFramework/Util/TestGeneratorTest.php
+++ b/dev/tests/unit/Magento/FunctionalTestFramework/Util/TestGeneratorTest.php
@@ -8,6 +8,7 @@
use AspectMock\Test as AspectMock;
+use Magento\FunctionalTestingFramework\Filter\FilterList;
use Magento\FunctionalTestingFramework\Test\Objects\ActionObject;
use Magento\FunctionalTestingFramework\Test\Objects\TestHookObject;
use Magento\FunctionalTestingFramework\Test\Objects\TestObject;
@@ -17,6 +18,16 @@
class TestGeneratorTest extends MagentoTestCase
{
+ /**
+ * After method functionality
+ *
+ * @return void
+ */
+ public function tearDown()
+ {
+ AspectMock::clean();
+ }
+
/**
* Basic test to check exceptions for incorrect entities.
*
@@ -99,4 +110,54 @@ public function testAllowSkipped()
$this->assertContains($actionInput, $output);
$this->assertContains($beforeActionInput, $output);
}
+
+ /**
+ * Tests that TestGenerator createAllTestFiles correctly filters based on severity
+ *
+ * @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException
+ */
+ public function testFilter()
+ {
+ // Mock filters for TestGenerator
+ AspectMock::double(
+ MftfApplicationConfig::class,
+ ['getFilterList' => new FilterList(['severity' => ["CRITICAL"]])]
+ );
+
+ $actionInput = 'fakeInput';
+ $actionObject = new ActionObject('fakeAction', 'comment', [
+ 'userInput' => $actionInput
+ ]);
+
+ $annotation1 = ['severity' => ['CRITICAL']];
+ $annotation2 = ['severity' => ['MINOR']];
+ $test1 = new TestObject(
+ "test1",
+ ["fakeAction" => $actionObject],
+ $annotation1,
+ [],
+ "filename"
+ );
+ $test2 = new TestObject(
+ "test2",
+ ["fakeAction" => $actionObject],
+ $annotation2,
+ [],
+ "filename"
+ );
+ AspectMock::double(TestGenerator::class, ['loadAllTestObjects' => ["sampleTest" => $test1, "test2" => $test2]]);
+
+ // Mock createCestFile to return name of tests that testGenerator tried to create
+ $generatedTests = [];
+ AspectMock::double(TestGenerator::class, ['createCestFile' => function ($arg1, $arg2) use (&$generatedTests) {
+ $generatedTests[$arg2] = true;
+ }]);
+
+ $testGeneratorObject = TestGenerator::getInstance("", ["sampleTest" => $test1, "test2" => $test2]);
+ $testGeneratorObject->createAllTestFiles(null, []);
+
+ // Ensure Test1 was Generated but not Test 2
+ $this->assertArrayHasKey('test1Cest', $generatedTests);
+ $this->assertArrayNotHasKey('test2Cest', $generatedTests);
+ }
}
diff --git a/docs/commands/mftf.md b/docs/commands/mftf.md
index b2c36dae4..e0394dc12 100644
--- a/docs/commands/mftf.md
+++ b/docs/commands/mftf.md
@@ -159,6 +159,7 @@ vendor/bin/mftf generate:tests [option] [] [] [--remove]
| Option | Description|
| ---| --- |
| `--config=[ or or ]` | Creates a single manifest file with a list of all tests. The default location is `tests/functional/Magento/FunctionalTest/_generated/testManifest.txt`.
You can split the list into multiple groups using `--config=parallel`; the groups will be generated in `_generated/groups/` like `_generated/groups/group1.txt, group2.txt, ...`.
Available values: `default` (default), `singleRun`(same as `default`), and `parallel`.
Example: `generate:tests --config=parallel`. |
+| `--filter` | Option to filter tests to be generated.
Template: ':'.
Existing filter types: severity.
Existing severity values: BLOCKER, CRITICAL, MAJOR, AVERAGE, MINOR.
Example: --filter=severity:CRITICAL|
| `--force` | Forces test generation, regardless of the module merge order defined in the Magento instance. Example: `generate:tests --force`. |
| `-i,--time` | Set time in minutes to determine the group size when `--config=parallel` is used. The __default value__ is `10`. Example: `generate:tests --config=parallel --time=15`|
| `--tests` | Defines the test configuration as a JSON string.|
diff --git a/src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php b/src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php
index 80db27de0..4b215cb4f 100644
--- a/src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php
+++ b/src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php
@@ -6,6 +6,7 @@
namespace Magento\FunctionalTestingFramework\Config;
use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException;
+use Magento\FunctionalTestingFramework\Filter\FilterList;
class MftfApplicationConfig
{
@@ -25,6 +26,13 @@ class MftfApplicationConfig
const LEVEL_NONE = "none";
const MFTF_DEBUG_LEVEL = [self::LEVEL_DEFAULT, self::LEVEL_DEVELOPER, self::LEVEL_NONE];
+ /**
+ * Contains object with test filters.
+ *
+ * @var FilterList
+ */
+ private $filterList;
+
/**
* Determines whether the user has specified a force option for generation
*
@@ -74,6 +82,7 @@ class MftfApplicationConfig
* @param boolean $verboseEnabled
* @param string $debugLevel
* @param boolean $allowSkipped
+ * @param array $filters
* @throws TestFrameworkException
*/
private function __construct(
@@ -81,7 +90,8 @@ private function __construct(
$phase = self::EXECUTION_PHASE,
$verboseEnabled = null,
$debugLevel = self::LEVEL_NONE,
- $allowSkipped = false
+ $allowSkipped = false,
+ $filters = []
) {
$this->forceGenerate = $forceGenerate;
@@ -101,6 +111,7 @@ private function __construct(
$this->debugLevel = self::LEVEL_DEVELOPER;
}
$this->allowSkipped = $allowSkipped;
+ $this->filterList = new FilterList($filters);
}
/**
@@ -112,6 +123,7 @@ private function __construct(
* @param boolean $verboseEnabled
* @param string $debugLevel
* @param boolean $allowSkipped
+ * @param array $filters
* @return void
* @throws TestFrameworkException
*/
@@ -120,11 +132,19 @@ public static function create(
$phase = self::EXECUTION_PHASE,
$verboseEnabled = null,
$debugLevel = self::LEVEL_NONE,
- $allowSkipped = false
+ $allowSkipped = false,
+ $filters = []
) {
if (self::$MFTF_APPLICATION_CONTEXT == null) {
self::$MFTF_APPLICATION_CONTEXT =
- new MftfApplicationConfig($forceGenerate, $phase, $verboseEnabled, $debugLevel, $allowSkipped);
+ new MftfApplicationConfig(
+ $forceGenerate,
+ $phase,
+ $verboseEnabled,
+ $debugLevel,
+ $allowSkipped,
+ $filters
+ );
}
}
@@ -196,4 +216,14 @@ public function getPhase()
{
return $this->phase;
}
+
+ /**
+ * Returns a class with registered filter list.
+ *
+ * @return FilterList
+ */
+ public function getFilterList()
+ {
+ return $this->filterList;
+ }
}
diff --git a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php
index 3433f23a8..273632586 100644
--- a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php
+++ b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php
@@ -35,7 +35,7 @@ class BaseGenerateCommand extends Command
*
* @var SymfonyStyle
*/
- private $ioStyle = null;
+ protected $ioStyle = null;
/**
* Configures the base command.
diff --git a/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php b/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php
index b9378edc1..3ca53f8f6 100644
--- a/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php
+++ b/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php
@@ -18,7 +18,11 @@
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
+use Symfony\Component\Console\Style\SymfonyStyle;
+/**
+ * @SuppressWarnings(PHPMD)
+ */
class GenerateTestsCommand extends BaseGenerateCommand
{
/**
@@ -30,6 +34,7 @@ protected function configure()
{
$this->setName('generate:tests')
->setDescription('Run validation and generate all test files and suites based on xml declarations')
+ ->addUsage('AdminLoginTest')
->addArgument(
'name',
InputArgument::OPTIONAL | InputArgument::IS_ARRAY,
@@ -46,6 +51,15 @@ protected function configure()
't',
InputOption::VALUE_REQUIRED,
'A parameter accepting a JSON string used to determine the test configuration'
+ )->addOption(
+ 'filter',
+ null,
+ InputOption::VALUE_IS_ARRAY | InputOption::VALUE_OPTIONAL,
+ 'Option to filter tests to be generated.' . PHP_EOL
+ . 'Template: :' . PHP_EOL
+ . 'Existing filter types: severity.' . PHP_EOL
+ . 'Existing severity values: BLOCKER, CRITICAL, MAJOR, AVERAGE, MINOR.' . PHP_EOL
+ . 'Example: --filter=severity:CRITICAL' . PHP_EOL
);
parent::configure();
@@ -56,13 +70,14 @@ protected function configure()
*
* @param InputInterface $input
* @param OutputInterface $output
- * @return void
+ * @return void|integer
* @throws TestFrameworkException
* @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException
* @throws \Magento\FunctionalTestingFramework\Exceptions\XmlException
*/
protected function execute(InputInterface $input, OutputInterface $output)
{
+ $this->setOutputStyle($input, $output);
$tests = $input->getArgument('name');
$config = $input->getOption('config');
$json = $input->getOption('tests'); // for backward compatibility
@@ -72,15 +87,25 @@ protected function execute(InputInterface $input, OutputInterface $output)
$remove = $input->getOption('remove');
$verbose = $output->isVerbose();
$allowSkipped = $input->getOption('allow-skipped');
-
+ $filters = $input->getOption('filter');
+ foreach ($filters as $filter) {
+ list($filterType, $filterValue) = explode(':', $filter);
+ $filterList[$filterType][] = $filterValue;
+ }
// Set application configuration so we can references the user options in our framework
- MftfApplicationConfig::create(
- $force,
- MftfApplicationConfig::GENERATION_PHASE,
- $verbose,
- $debug,
- $allowSkipped
- );
+ try {
+ MftfApplicationConfig::create(
+ $force,
+ MftfApplicationConfig::GENERATION_PHASE,
+ $verbose,
+ $debug,
+ $allowSkipped,
+ $filterList ?? []
+ );
+ } catch (\Exception $exception) {
+ $this->ioStyle->error("Test generation failed." . PHP_EOL . $exception->getMessage());
+ return 1;
+ }
$this->setOutputStyle($input, $output);
$this->showMftfNotices($output);
@@ -105,20 +130,31 @@ protected function execute(InputInterface $input, OutputInterface $output)
($debug !== MftfApplicationConfig::LEVEL_NONE));
}
- $testConfiguration = $this->createTestConfiguration($json, $tests);
+ try {
+ $testConfiguration = $this->createTestConfiguration($json, $tests);
- // create our manifest file here
- $testManifest = TestManifestFactory::makeManifest($config, $testConfiguration['suites']);
- TestGenerator::getInstance(null, $testConfiguration['tests'])->createAllTestFiles($testManifest);
+ // create our manifest file here
+ $testManifest = TestManifestFactory::makeManifest($config, $testConfiguration['suites']);
- if ($config == 'parallel') {
- /** @var ParallelTestManifest $testManifest */
- $testManifest->createTestGroups($time);
- }
+ TestGenerator::getInstance(null, $testConfiguration['tests'])->createAllTestFiles($testManifest);
- SuiteGenerator::getInstance()->generateAllSuites($testManifest);
+ if ($config == 'parallel') {
+ /** @var ParallelTestManifest $testManifest */
+ $testManifest->createTestGroups($time);
+ }
- $testManifest->generate();
+ SuiteGenerator::getInstance()->generateAllSuites($testManifest);
+
+ $testManifest->generate();
+ } catch (\Exception $e) {
+ $message = $e->getMessage() . PHP_EOL;
+ $message .= !empty($filters) ? 'Filter(s): ' . implode(', ', $filters) . PHP_EOL : '';
+ $message .= !empty($tests) ? 'Test name(s): ' . implode(', ', $tests) . PHP_EOL : '';
+ $message .= !empty($json) && empty($tests) ? 'Test configuration: ' . $json . PHP_EOL : '';
+ $this->ioStyle->note($message);
+
+ return 1;
+ }
$output->writeln("Generate Tests Command Run");
}
diff --git a/src/Magento/FunctionalTestingFramework/Filter/FilterInterface.php b/src/Magento/FunctionalTestingFramework/Filter/FilterInterface.php
new file mode 100644
index 000000000..bb89ab3c2
--- /dev/null
+++ b/src/Magento/FunctionalTestingFramework/Filter/FilterInterface.php
@@ -0,0 +1,27 @@
+ $filterValue) {
+ $className = "Magento\FunctionalTestingFramework\Filter\Test\\" . ucfirst($filterType);
+ if (!class_exists($className)) {
+ throw new TestFrameworkException("Filter type '" . $filterType . "' do not exist.");
+ }
+ $this->filters[$filterType] = new $className($filterValue);
+ }
+ }
+
+ /**
+ * @return array
+ */
+ public function getFilters(): array
+ {
+ return $this->filters;
+ }
+
+ /**
+ * @param string $filterType
+ * @return \Magento\FunctionalTestingFramework\Filter\FilterInterface
+ */
+ public function getFilter(string $filterType): FilterInterface
+ {
+ return $this->filters[$filterType];
+ }
+}
diff --git a/src/Magento/FunctionalTestingFramework/Filter/Test/Severity.php b/src/Magento/FunctionalTestingFramework/Filter/Test/Severity.php
new file mode 100644
index 000000000..e88c4af7b
--- /dev/null
+++ b/src/Magento/FunctionalTestingFramework/Filter/Test/Severity.php
@@ -0,0 +1,68 @@
+filterValues[] = $severityValues[$filterValue];
+ }
+ }
+
+ /**
+ * Filter tests by severity.
+ *
+ * @param TestObject[] $tests
+ * @return void
+ */
+ public function filter(array &$tests)
+ {
+ /** @var TestObject $test */
+ foreach ($tests as $testName => $test) {
+ $severities = $test->getAnnotationByName(self::ANNOTATION_TAG);
+ foreach ($severities as $severity) {
+ if (!in_array($severity, $this->filterValues, true)) {
+ unset($tests[$testName]);
+ }
+ }
+ }
+ }
+}
diff --git a/src/Magento/FunctionalTestingFramework/Suite/Objects/SuiteObject.php b/src/Magento/FunctionalTestingFramework/Suite/Objects/SuiteObject.php
index d7f6bd18a..7168d9556 100644
--- a/src/Magento/FunctionalTestingFramework/Suite/Objects/SuiteObject.php
+++ b/src/Magento/FunctionalTestingFramework/Suite/Objects/SuiteObject.php
@@ -5,8 +5,11 @@
*/
namespace Magento\FunctionalTestingFramework\Suite\Objects;
+use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig;
+use Magento\FunctionalTestingFramework\Filter\FilterInterface;
use Magento\FunctionalTestingFramework\Test\Objects\TestHookObject;
use Magento\FunctionalTestingFramework\Test\Objects\TestObject;
+use Magento\FunctionalTestingFramework\Util\Logger\LoggingUtil;
/**
* Class SuiteObject
@@ -84,6 +87,7 @@ public function getTests()
* @param TestObject[] $includeTests
* @param TestObject[] $excludeTests
* @return TestObject[]
+ * @throws \Exception
*/
private function resolveTests($includeTests, $excludeTests)
{
@@ -95,12 +99,10 @@ private function resolveTests($includeTests, $excludeTests)
unset($finalTestList[$testName]);
}
- if (empty($finalTestList)) {
- trigger_error(
- "Current suite configuration for " .
- $this->name . " contains no tests.",
- E_USER_WARNING
- );
+ $filters = MftfApplicationConfig::getConfig()->getFilterList()->getFilters();
+ /** @var FilterInterface $filter */
+ foreach ($filters as $filter) {
+ $filter->filter($finalTestList);
}
return $finalTestList;
diff --git a/src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php b/src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php
index e3da10497..b40b48ae7 100644
--- a/src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php
+++ b/src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php
@@ -95,6 +95,14 @@ public function generateAllSuites($testManifest)
$suites = $testManifest->getSuiteConfig();
foreach ($suites as $suiteName => $suiteContent) {
+ if (empty($suiteContent)) {
+ LoggingUtil::getInstance()->getLogger(self::class)->notification(
+ "Suite '" . $suiteName . "' contains no tests and won't be generated." . PHP_EOL,
+ [],
+ true
+ );
+ continue;
+ }
$firstElement = array_values($suiteContent)[0];
// if the first element is a string we know that we simply have an array of tests
diff --git a/src/Magento/FunctionalTestingFramework/Test/Handlers/TestObjectHandler.php b/src/Magento/FunctionalTestingFramework/Test/Handlers/TestObjectHandler.php
index a0d490df5..d5ed43806 100644
--- a/src/Magento/FunctionalTestingFramework/Test/Handlers/TestObjectHandler.php
+++ b/src/Magento/FunctionalTestingFramework/Test/Handlers/TestObjectHandler.php
@@ -16,6 +16,8 @@
use Magento\FunctionalTestingFramework\Test\Util\ObjectExtensionUtil;
use Magento\FunctionalTestingFramework\Test\Util\TestObjectExtractor;
use Magento\FunctionalTestingFramework\Test\Util\AnnotationExtractor;
+use Magento\FunctionalTestingFramework\Util\Logger\LoggingUtil;
+use PHP_CodeSniffer\Tokenizers\PHP;
/**
* Class TestObjectHandler
@@ -140,7 +142,7 @@ private function initTestData()
}
$exceptionCollector = new ExceptionCollector();
- foreach ($parsedTestArray[TestObjectHandler::XML_ROOT] as $testName => $testData) {
+ foreach ($parsedTestArray as $testName => $testData) {
if (!is_array($testData)) {
continue;
}
diff --git a/src/Magento/FunctionalTestingFramework/Test/Parsers/TestDataParser.php b/src/Magento/FunctionalTestingFramework/Test/Parsers/TestDataParser.php
index 18044c1af..dc3489de1 100644
--- a/src/Magento/FunctionalTestingFramework/Test/Parsers/TestDataParser.php
+++ b/src/Magento/FunctionalTestingFramework/Test/Parsers/TestDataParser.php
@@ -7,29 +7,42 @@
namespace Magento\FunctionalTestingFramework\Test\Parsers;
use Magento\FunctionalTestingFramework\Config\DataInterface;
+use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig;
+use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException;
+use Magento\FunctionalTestingFramework\Filter\FilterInterface;
+use Magento\FunctionalTestingFramework\Util\Logger\LoggingUtil;
/**
* Class TestDataParser
*/
class TestDataParser
{
+ /**
+ * @var DataInterface
+ */
+ private $testData;
+
/**
* TestDataParser constructor.
*
* @param DataInterface $testData
+ * @throws \Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException
*/
public function __construct(DataInterface $testData)
{
- $this->testData = $testData;
+ $this->testData = array_filter($testData->get('tests'), function ($value) {
+ return is_array($value);
+ });
}
/**
* Returns an array of data based on *Test.xml files
*
* @return array
+ * @throws TestFrameworkException
*/
public function readTestData()
{
- return $this->testData->get();
+ return $this->testData;
}
}
diff --git a/src/Magento/FunctionalTestingFramework/Util/Logger/MftfLogger.php b/src/Magento/FunctionalTestingFramework/Util/Logger/MftfLogger.php
index 3983a755a..0e8c1e17f 100644
--- a/src/Magento/FunctionalTestingFramework/Util/Logger/MftfLogger.php
+++ b/src/Magento/FunctionalTestingFramework/Util/Logger/MftfLogger.php
@@ -49,4 +49,23 @@ public function criticalFailure($message, array $context = [], $verbose = false)
}
parent::critical($message, $context);
}
+
+ /**
+ * Adds a log record at the NOTICE level.
+ *
+ * @param string $message
+ * @param array $context
+ * @param boolean $verbose
+ * @return void
+ * @throws \Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException
+ */
+ public function notification($message, array $context = [], $verbose = false)
+ {
+ $message = "NOTICE: " . $message;
+ // Suppress print during unit testing
+ if (MftfApplicationConfig::getConfig()->getPhase() !== MftfApplicationConfig::UNIT_TEST_PHASE && $verbose) {
+ print ($message . implode("\n", $context) . "\n");
+ }
+ parent::notice($message, $context);
+ }
}
diff --git a/src/Magento/FunctionalTestingFramework/Util/Manifest/DefaultTestManifest.php b/src/Magento/FunctionalTestingFramework/Util/Manifest/DefaultTestManifest.php
index eb4f79db2..4cbef3168 100644
--- a/src/Magento/FunctionalTestingFramework/Util/Manifest/DefaultTestManifest.php
+++ b/src/Magento/FunctionalTestingFramework/Util/Manifest/DefaultTestManifest.php
@@ -86,6 +86,9 @@ public function generate()
protected function generateSuiteEntries($fileResource)
{
foreach ($this->getSuiteConfig() as $suiteName => $tests) {
+ if (count($tests) === 0) {
+ continue;
+ }
$line = "-g {$suiteName}";
fwrite($fileResource, $line . PHP_EOL);
}
diff --git a/src/Magento/FunctionalTestingFramework/Util/Manifest/ParallelTestManifest.php b/src/Magento/FunctionalTestingFramework/Util/Manifest/ParallelTestManifest.php
index 9b12cfd00..201e002a2 100644
--- a/src/Magento/FunctionalTestingFramework/Util/Manifest/ParallelTestManifest.php
+++ b/src/Magento/FunctionalTestingFramework/Util/Manifest/ParallelTestManifest.php
@@ -132,7 +132,7 @@ private function generateGroupFile($testGroup, $nodeNumber, $suites)
$fileResource = fopen($this->dirPath . DIRECTORY_SEPARATOR . "group{$nodeNumber}.txt", 'a');
$line = null;
- if (array_key_exists($entryName, $suites)) {
+ if (!empty($suites[$entryName])) {
$line = "-g {$entryName}";
} else {
$line = $this->relativeDirPath . DIRECTORY_SEPARATOR . $entryName . '.php';
diff --git a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php
index 2ec565c44..2e7d265fa 100644
--- a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php
+++ b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php
@@ -11,6 +11,7 @@
use Magento\FunctionalTestingFramework\DataGenerator\Objects\EntityDataObject;
use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException;
use Magento\FunctionalTestingFramework\Exceptions\TestReferenceException;
+use Magento\FunctionalTestingFramework\Filter\FilterInterface;
use Magento\FunctionalTestingFramework\Suite\Handlers\SuiteObjectHandler;
use Magento\FunctionalTestingFramework\Test\Handlers\ActionGroupObjectHandler;
use Magento\FunctionalTestingFramework\Test\Handlers\TestObjectHandler;
@@ -288,6 +289,11 @@ private function assembleAllTestPhp($testManifest, array $testsToIgnore)
/** @var TestObject[] $testObjects */
$testObjects = $this->loadAllTestObjects($testsToIgnore);
$cestPhpArray = [];
+ $filters = MftfApplicationConfig::getConfig()->getFilterList()->getFilters();
+ /** @var FilterInterface $filter */
+ foreach ($filters as $filter) {
+ $filter->filter($testObjects);
+ }
foreach ($testObjects as $test) {
// Do not generate test if it is an extended test and parent does not exist