Skip to content

Commit fd1d845

Browse files
okolesnykokolesnyk
committed
MQE-2008: Filter test generation and execution by severity
1 parent 8b33258 commit fd1d845

File tree

14 files changed

+315
-44
lines changed

14 files changed

+315
-44
lines changed

‎dev/tests/functional/tests/MFTF/DevDocs/Test/DeprecatedDevDocsTest.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
<stories value="[Deprecated] MFTF DevDocs available"/>
1616
<title value="[Deprecated] Magento Functional Testing Framework Documentation is available."/>
1717
<description value="[Deprecated] Magento Functional Testing Framework Documentation is available."/>
18-
<severity value="CRITICAL"/>
18+
<severity value="MINOR"/>
1919
<group value="mftf"/>
2020
</annotations>
2121

‎dev/tests/unit/Magento/FunctionalTestFramework/Suite/Handlers/SuiteObjectHandlerTest.php

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ public function testGetSuiteObject()
6565
->withTestActions()
6666
->build();
6767

68-
$mockTestData = ['tests' => array_merge($mockSimpleTest, $mockGroup1Test1, $mockGroup1Test2, $mockGroup2Test1)];
68+
$mockTestData = array_merge($mockSimpleTest, $mockGroup1Test1, $mockGroup1Test2, $mockGroup2Test1);
6969
$this->setMockTestAndSuiteParserOutput($mockTestData, $mockData);
7070

7171
// parse and retrieve suite object with mocked data

‎dev/tests/unit/Magento/FunctionalTestFramework/Suite/SuiteGeneratorTest.php

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ public function testGenerateSuite()
6969
->withTestActions()
7070
->build();
7171

72-
$mockTestData = ['tests' => array_merge($mockSimpleTest)];
72+
$mockTestData = array_merge($mockSimpleTest);
7373
$this->setMockTestAndSuiteParserOutput($mockTestData, $mockData);
7474

7575
// parse and generate suite object with mocked data
@@ -105,7 +105,7 @@ public function testGenerateAllSuites()
105105
->withTestActions()
106106
->build();
107107

108-
$mockTestData = ['tests' => array_merge($mockSimpleTest)];
108+
$mockTestData = array_merge($mockSimpleTest);
109109
$this->setMockTestAndSuiteParserOutput($mockTestData, $mockData);
110110

111111
// parse and retrieve suite object with mocked data
@@ -172,7 +172,7 @@ public function testInvalidSuiteTestPair()
172172
->withAnnotations(['group' => [['value' => 'group2']]])
173173
->withTestActions()
174174
->build();
175-
$mockTestData = ['tests' => array_merge($mockSimpleTest, $mockSimpleTest2)];
175+
$mockTestData = array_merge($mockSimpleTest, $mockSimpleTest2);
176176
$this->setMockTestAndSuiteParserOutput($mockTestData, $mockSuiteData);
177177

178178
// Make invalid manifest
@@ -196,7 +196,7 @@ public function testNonExistentSuiteTestPair()
196196
->withAnnotations(['group' => [['value' => 'group1']]])
197197
->withTestActions()
198198
->build();
199-
$mockTestData = ['tests' => array_merge($mockSimpleTest)];
199+
$mockTestData = array_merge($mockSimpleTest);
200200
$this->setMockTestAndSuiteParserOutput($mockTestData, []);
201201

202202
// Make invalid manifest

‎dev/tests/unit/Magento/FunctionalTestFramework/Test/Handlers/TestObjectHandlerTest.php

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ public function testGetTestObject()
4141

4242
$resolverMock = new MockModuleResolverBuilder();
4343
$resolverMock->setup();
44-
$this->setMockParserOutput(['tests' => $mockData]);
44+
$this->setMockParserOutput($mockData);
4545

4646
// run object handler method
4747
$toh = TestObjectHandler::getInstance();
@@ -135,7 +135,7 @@ public function testGetTestsByGroup()
135135

136136
$resolverMock = new MockModuleResolverBuilder();
137137
$resolverMock->setup();
138-
$this->setMockParserOutput(['tests' => array_merge($includeTest, $excludeTest)]);
138+
$this->setMockParserOutput(array_merge($includeTest, $excludeTest));
139139

140140
// execute test method
141141
$toh = TestObjectHandler::getInstance();
@@ -184,7 +184,7 @@ public function testGetTestWithModuleName()
184184
$resolverMock = new MockModuleResolverBuilder();
185185
$resolverMock->setup(['Vendor_' . $moduleExpected => $filepath]);
186186

187-
$this->setMockParserOutput(['tests' => $mockData]);
187+
$this->setMockParserOutput($mockData);
188188
// Execute Test Method
189189
$toh = TestObjectHandler::getInstance();
190190
$actualTestObject = $toh->getObject($testDataArrayBuilder->testName);
@@ -212,7 +212,7 @@ public function testGetTestObjectWithInvalidExtends()
212212
->build();
213213
$resolverMock = new MockModuleResolverBuilder();
214214
$resolverMock->setup();
215-
$this->setMockParserOutput(['tests' => $testOne]);
215+
$this->setMockParserOutput($testOne);
216216

217217
$toh = TestObjectHandler::getInstance();
218218

@@ -250,7 +250,7 @@ public function testGetAllTestObjectsWithInvalidExtends()
250250

251251
$resolverMock = new MockModuleResolverBuilder();
252252
$resolverMock->setup();
253-
$this->setMockParserOutput(['tests' => array_merge($testOne, $testTwo)]);
253+
$this->setMockParserOutput(array_merge($testOne, $testTwo));
254254

255255
$toh = TestObjectHandler::getInstance();
256256

‎dev/tests/unit/Magento/FunctionalTestFramework/Test/Util/ObjectExtensionUtilTest.php

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ public function testGenerateExtendedTest()
6767
->withTestReference("simpleTest")
6868
->build();
6969

70-
$mockTestData = ['tests' => array_merge($mockSimpleTest, $mockExtendedTest)];
70+
$mockTestData = array_merge($mockSimpleTest, $mockExtendedTest);
7171
$this->setMockTestOutput($mockTestData);
7272

7373
// parse and generate test object with mocked data
@@ -112,7 +112,7 @@ public function testGenerateExtendedWithHooks()
112112
->withTestReference("simpleTest")
113113
->build();
114114

115-
$mockTestData = ['tests' => array_merge($mockSimpleTest, $mockExtendedTest)];
115+
$mockTestData = array_merge($mockSimpleTest, $mockExtendedTest);
116116
$this->setMockTestOutput($mockTestData);
117117

118118
// parse and generate test object with mocked data
@@ -143,7 +143,7 @@ public function testExtendedTestNoParent()
143143
->withTestReference("simpleTest")
144144
->build();
145145

146-
$mockTestData = ['tests' => array_merge($mockExtendedTest)];
146+
$mockTestData = array_merge($mockExtendedTest);
147147
$this->setMockTestOutput($mockTestData);
148148

149149
// parse and generate test object with mocked data
@@ -182,7 +182,7 @@ public function testExtendingExtendedTest()
182182
->withTestReference("simpleTest")
183183
->build();
184184

185-
$mockTestData = ['tests' => array_merge($mockParentTest, $mockSimpleTest, $mockExtendedTest)];
185+
$mockTestData = array_merge($mockParentTest, $mockSimpleTest, $mockExtendedTest);
186186
$this->setMockTestOutput($mockTestData);
187187

188188
$this->expectExceptionMessage("Cannot extend a test that already extends another test. Test: simpleTest");

‎src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php

Lines changed: 33 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
namespace Magento\FunctionalTestingFramework\Config;
77

88
use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException;
9+
use Magento\FunctionalTestingFramework\Filter\FilterList;
910

1011
class MftfApplicationConfig
1112
{
@@ -25,6 +26,13 @@ class MftfApplicationConfig
2526
const LEVEL_NONE = "none";
2627
const MFTF_DEBUG_LEVEL = [self::LEVEL_DEFAULT, self::LEVEL_DEVELOPER, self::LEVEL_NONE];
2728

29+
/**
30+
* Contains object with test filters.
31+
*
32+
* @var FilterList
33+
*/
34+
private $filterList;
35+
2836
/**
2937
* Determines whether the user has specified a force option for generation
3038
*
@@ -74,14 +82,16 @@ class MftfApplicationConfig
7482
* @param boolean $verboseEnabled
7583
* @param string $debugLevel
7684
* @param boolean $allowSkipped
85+
* @param array $filters
7786
* @throws TestFrameworkException
7887
*/
7988
private function __construct(
8089
$forceGenerate = false,
8190
$phase = self::EXECUTION_PHASE,
8291
$verboseEnabled = null,
8392
$debugLevel = self::LEVEL_NONE,
84-
$allowSkipped = false
93+
$allowSkipped = false,
94+
$filters = []
8595
) {
8696
$this->forceGenerate = $forceGenerate;
8797

@@ -101,6 +111,7 @@ private function __construct(
101111
$this->debugLevel = self::LEVEL_DEVELOPER;
102112
}
103113
$this->allowSkipped = $allowSkipped;
114+
$this->filterList = new FilterList($filters);
104115
}
105116

106117
/**
@@ -112,6 +123,7 @@ private function __construct(
112123
* @param boolean $verboseEnabled
113124
* @param string $debugLevel
114125
* @param boolean $allowSkipped
126+
* @param array $filters
115127
* @return void
116128
* @throws TestFrameworkException
117129
*/
@@ -120,11 +132,19 @@ public static function create(
120132
$phase = self::EXECUTION_PHASE,
121133
$verboseEnabled = null,
122134
$debugLevel = self::LEVEL_NONE,
123-
$allowSkipped = false
135+
$allowSkipped = false,
136+
$filters = []
124137
) {
125138
if (self::$MFTF_APPLICATION_CONTEXT == null) {
126139
self::$MFTF_APPLICATION_CONTEXT =
127-
new MftfApplicationConfig($forceGenerate, $phase, $verboseEnabled, $debugLevel, $allowSkipped);
140+
new MftfApplicationConfig(
141+
$forceGenerate,
142+
$phase,
143+
$verboseEnabled,
144+
$debugLevel,
145+
$allowSkipped,
146+
$filters
147+
);
128148
}
129149
}
130150

@@ -196,4 +216,14 @@ public function getPhase()
196216
{
197217
return $this->phase;
198218
}
219+
220+
/**
221+
* Returns a class with registered filter list.
222+
*
223+
* @return FilterList
224+
*/
225+
public function getFilterList()
226+
{
227+
return $this->filterList;
228+
}
199229
}

‎src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php

Lines changed: 55 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,11 @@
1818
use Symfony\Component\Console\Input\InputInterface;
1919
use Symfony\Component\Console\Input\InputOption;
2020
use Symfony\Component\Console\Output\OutputInterface;
21+
use Symfony\Component\Console\Style\SymfonyStyle;
2122

23+
/**
24+
* @SuppressWarnings(PHPMD)
25+
*/
2226
class GenerateTestsCommand extends BaseGenerateCommand
2327
{
2428
/**
@@ -30,6 +34,7 @@ protected function configure()
3034
{
3135
$this->setName('generate:tests')
3236
->setDescription('Run validation and generate all test files and suites based on xml declarations')
37+
->addUsage('AdminLoginTest')
3338
->addArgument(
3439
'name',
3540
InputArgument::OPTIONAL | InputArgument::IS_ARRAY,
@@ -46,6 +51,15 @@ protected function configure()
4651
't',
4752
InputOption::VALUE_REQUIRED,
4853
'A parameter accepting a JSON string used to determine the test configuration'
54+
)->addOption(
55+
'filter',
56+
null,
57+
InputOption::VALUE_IS_ARRAY | InputOption::VALUE_OPTIONAL,
58+
'Option to filter tests to be generated.' . PHP_EOL
59+
. '<info>Template:</info> <filterName>:<filterValue>' . PHP_EOL
60+
. '<info>Existing filter types:</info> severity.' . PHP_EOL
61+
. '<info>Existing severity values:</info> BLOCKER, CRITICAL, MAJOR, AVERAGE, MINOR.' . PHP_EOL
62+
. '<info>Example:</info> --filter=severity:CRITICAL' . PHP_EOL
4963
);
5064

5165
parent::configure();
@@ -56,13 +70,14 @@ protected function configure()
5670
*
5771
* @param InputInterface $input
5872
* @param OutputInterface $output
59-
* @return void
73+
* @return void|integer
6074
* @throws TestFrameworkException
6175
* @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException
6276
* @throws \Magento\FunctionalTestingFramework\Exceptions\XmlException
6377
*/
6478
protected function execute(InputInterface $input, OutputInterface $output)
6579
{
80+
$ioStyle = new SymfonyStyle($input, $output);
6681
$tests = $input->getArgument('name');
6782
$config = $input->getOption('config');
6883
$json = $input->getOption('tests'); // for backward compatibility
@@ -72,15 +87,25 @@ protected function execute(InputInterface $input, OutputInterface $output)
7287
$remove = $input->getOption('remove');
7388
$verbose = $output->isVerbose();
7489
$allowSkipped = $input->getOption('allow-skipped');
75-
90+
$filters = $input->getOption('filter');
91+
foreach ($filters as $filter) {
92+
list($filterType, $filterValue) = explode(':', $filter);
93+
$filterList[$filterType][] = $filterValue;
94+
}
7695
// Set application configuration so we can references the user options in our framework
77-
MftfApplicationConfig::create(
78-
$force,
79-
MftfApplicationConfig::GENERATION_PHASE,
80-
$verbose,
81-
$debug,
82-
$allowSkipped
83-
);
96+
try {
97+
MftfApplicationConfig::create(
98+
$force,
99+
MftfApplicationConfig::GENERATION_PHASE,
100+
$verbose,
101+
$debug,
102+
$allowSkipped,
103+
$filterList ?? []
104+
);
105+
} catch (\Exception $exception) {
106+
$ioStyle->error("Test generation failed." . PHP_EOL . $exception->getMessage());
107+
return 1;
108+
}
84109

85110
$this->setOutputStyle($input, $output);
86111
$this->showMftfNotices($output);
@@ -105,20 +130,31 @@ protected function execute(InputInterface $input, OutputInterface $output)
105130
($debug !== MftfApplicationConfig::LEVEL_NONE));
106131
}
107132

108-
$testConfiguration = $this->createTestConfiguration($json, $tests);
133+
try {
134+
$testConfiguration = $this->createTestConfiguration($json, $tests);
109135

110-
// create our manifest file here
111-
$testManifest = TestManifestFactory::makeManifest($config, $testConfiguration['suites']);
112-
TestGenerator::getInstance(null, $testConfiguration['tests'])->createAllTestFiles($testManifest);
136+
// create our manifest file here
137+
$testManifest = TestManifestFactory::makeManifest($config, $testConfiguration['suites']);
113138

114-
if ($config == 'parallel') {
115-
/** @var ParallelTestManifest $testManifest */
116-
$testManifest->createTestGroups($time);
117-
}
139+
TestGenerator::getInstance(null, $testConfiguration['tests'])->createAllTestFiles($testManifest);
140+
141+
if ($config == 'parallel') {
142+
/** @var ParallelTestManifest $testManifest */
143+
$testManifest->createTestGroups($time);
144+
}
145+
146+
SuiteGenerator::getInstance()->generateAllSuites($testManifest);
118147

119-
SuiteGenerator::getInstance()->generateAllSuites($testManifest);
148+
$testManifest->generate();
149+
} catch (\Exception $e) {
150+
$message = $e->getMessage() . PHP_EOL;
151+
$message .= !empty($filters) ? 'Filter(s): ' . implode(', ', $filters) . PHP_EOL : '';
152+
$message .= !empty($tests) ? 'Test name(s): ' . implode(', ', $tests) . PHP_EOL : '';
153+
$message .= !empty($json) && empty($tests) ? 'Test configuration: ' . $json . PHP_EOL : '';
154+
$ioStyle->note($message);
120155

121-
$testManifest->generate();
156+
return 1;
157+
}
122158

123159
$output->writeln("Generate Tests Command Run");
124160
}

‎src/Magento/FunctionalTestingFramework/Filter/FilterInterface.php

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
<?php
2+
/**
3+
* Copyright © Magento, Inc. All rights reserved.
4+
* See COPYING.txt for license details.
5+
*/
6+
7+
declare(strict_types=1);
8+
9+
namespace Magento\FunctionalTestingFramework\Filter;
10+
11+
/**
12+
* Interface for future test filters
13+
* @api
14+
*/
15+
interface FilterInterface
16+
{
17+
/**
18+
* @param array $filterValues
19+
*/
20+
public function __construct(array $filterValues = []);
21+
22+
/**
23+
* @param array $tests
24+
* @return void
25+
*/
26+
public function filter(array &$tests);
27+
}

0 commit comments

Comments
 (0)