diff --git a/CHANGELOG.md b/CHANGELOG.md index 4792fa8ae..a41360787 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ Magento Functional Testing Framework Changelog ================================================ +2.6.3 +----- + +### New Feature +* `--filter` option was added to `bin/mftf generate:tests` command. For more details please go to https://devdocs.magento.com/mftf/docs/commands/mftf.html#generatetests 2.6.2 ----- diff --git a/composer.json b/composer.json index ed4656a37..0ff50a2e8 100755 --- a/composer.json +++ b/composer.json @@ -2,7 +2,7 @@ "name": "magento/magento2-functional-testing-framework", "description": "Magento2 Functional Testing Framework", "type": "library", - "version": "2.6.2", + "version": "2.6.3", "license": "AGPL-3.0", "keywords": ["magento", "automation", "functional", "testing"], "config": { diff --git a/composer.lock b/composer.lock index caa95e992..6f370593b 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "d9ea4056a8f4501c3f2766e09edce40d", + "content-hash": "83e4e17679bff5fdd472c246dca8ac48", "packages": [ { "name": "allure-framework/allure-codeception", diff --git a/dev/tests/functional/tests/MFTF/DevDocs/Test/DeprecatedDevDocsTest.xml b/dev/tests/functional/tests/MFTF/DevDocs/Test/DeprecatedDevDocsTest.xml index bf0078df4..c93dc8011 100644 --- a/dev/tests/functional/tests/MFTF/DevDocs/Test/DeprecatedDevDocsTest.xml +++ b/dev/tests/functional/tests/MFTF/DevDocs/Test/DeprecatedDevDocsTest.xml @@ -15,7 +15,7 @@ <description value="[Deprecated] Magento Functional Testing Framework Documentation is available."/> - <severity value="CRITICAL"/> + <severity value="MINOR"/> <group value="mftf"/> </annotations> diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Suite/Handlers/SuiteObjectHandlerTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Suite/Handlers/SuiteObjectHandlerTest.php index 5bf29507c..e45127f4c 100644 --- a/dev/tests/unit/Magento/FunctionalTestFramework/Suite/Handlers/SuiteObjectHandlerTest.php +++ b/dev/tests/unit/Magento/FunctionalTestFramework/Suite/Handlers/SuiteObjectHandlerTest.php @@ -65,7 +65,7 @@ public function testGetSuiteObject() ->withTestActions() ->build(); - $mockTestData = ['tests' => array_merge($mockSimpleTest, $mockGroup1Test1, $mockGroup1Test2, $mockGroup2Test1)]; + $mockTestData = array_merge($mockSimpleTest, $mockGroup1Test1, $mockGroup1Test2, $mockGroup2Test1); $this->setMockTestAndSuiteParserOutput($mockTestData, $mockData); // parse and retrieve suite object with mocked data diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Suite/SuiteGeneratorTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Suite/SuiteGeneratorTest.php index e3f820ede..842be4f88 100644 --- a/dev/tests/unit/Magento/FunctionalTestFramework/Suite/SuiteGeneratorTest.php +++ b/dev/tests/unit/Magento/FunctionalTestFramework/Suite/SuiteGeneratorTest.php @@ -69,7 +69,7 @@ public function testGenerateSuite() ->withTestActions() ->build(); - $mockTestData = ['tests' => array_merge($mockSimpleTest)]; + $mockTestData = array_merge($mockSimpleTest); $this->setMockTestAndSuiteParserOutput($mockTestData, $mockData); // parse and generate suite object with mocked data @@ -105,7 +105,7 @@ public function testGenerateAllSuites() ->withTestActions() ->build(); - $mockTestData = ['tests' => array_merge($mockSimpleTest)]; + $mockTestData = array_merge($mockSimpleTest); $this->setMockTestAndSuiteParserOutput($mockTestData, $mockData); // parse and retrieve suite object with mocked data @@ -172,7 +172,7 @@ public function testInvalidSuiteTestPair() ->withAnnotations(['group' => [['value' => 'group2']]]) ->withTestActions() ->build(); - $mockTestData = ['tests' => array_merge($mockSimpleTest, $mockSimpleTest2)]; + $mockTestData = array_merge($mockSimpleTest, $mockSimpleTest2); $this->setMockTestAndSuiteParserOutput($mockTestData, $mockSuiteData); // Make invalid manifest @@ -196,7 +196,7 @@ public function testNonExistentSuiteTestPair() ->withAnnotations(['group' => [['value' => 'group1']]]) ->withTestActions() ->build(); - $mockTestData = ['tests' => array_merge($mockSimpleTest)]; + $mockTestData = array_merge($mockSimpleTest); $this->setMockTestAndSuiteParserOutput($mockTestData, []); // Make invalid manifest diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Test/Handlers/TestObjectHandlerTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Test/Handlers/TestObjectHandlerTest.php index 67f6f2b43..a4504ab07 100644 --- a/dev/tests/unit/Magento/FunctionalTestFramework/Test/Handlers/TestObjectHandlerTest.php +++ b/dev/tests/unit/Magento/FunctionalTestFramework/Test/Handlers/TestObjectHandlerTest.php @@ -41,7 +41,7 @@ public function testGetTestObject() $resolverMock = new MockModuleResolverBuilder(); $resolverMock->setup(); - $this->setMockParserOutput(['tests' => $mockData]); + $this->setMockParserOutput($mockData); // run object handler method $toh = TestObjectHandler::getInstance(); @@ -135,7 +135,7 @@ public function testGetTestsByGroup() $resolverMock = new MockModuleResolverBuilder(); $resolverMock->setup(); - $this->setMockParserOutput(['tests' => array_merge($includeTest, $excludeTest)]); + $this->setMockParserOutput(array_merge($includeTest, $excludeTest)); // execute test method $toh = TestObjectHandler::getInstance(); @@ -184,7 +184,7 @@ public function testGetTestWithModuleName() $resolverMock = new MockModuleResolverBuilder(); $resolverMock->setup(['Vendor_' . $moduleExpected => $filepath]); - $this->setMockParserOutput(['tests' => $mockData]); + $this->setMockParserOutput($mockData); // Execute Test Method $toh = TestObjectHandler::getInstance(); $actualTestObject = $toh->getObject($testDataArrayBuilder->testName); @@ -212,7 +212,7 @@ public function testGetTestObjectWithInvalidExtends() ->build(); $resolverMock = new MockModuleResolverBuilder(); $resolverMock->setup(); - $this->setMockParserOutput(['tests' => $testOne]); + $this->setMockParserOutput($testOne); $toh = TestObjectHandler::getInstance(); @@ -250,7 +250,7 @@ public function testGetAllTestObjectsWithInvalidExtends() $resolverMock = new MockModuleResolverBuilder(); $resolverMock->setup(); - $this->setMockParserOutput(['tests' => array_merge($testOne, $testTwo)]); + $this->setMockParserOutput(array_merge($testOne, $testTwo)); $toh = TestObjectHandler::getInstance(); diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Test/Util/ObjectExtensionUtilTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Test/Util/ObjectExtensionUtilTest.php index d86185b83..9982b8040 100644 --- a/dev/tests/unit/Magento/FunctionalTestFramework/Test/Util/ObjectExtensionUtilTest.php +++ b/dev/tests/unit/Magento/FunctionalTestFramework/Test/Util/ObjectExtensionUtilTest.php @@ -67,7 +67,7 @@ public function testGenerateExtendedTest() ->withTestReference("simpleTest") ->build(); - $mockTestData = ['tests' => array_merge($mockSimpleTest, $mockExtendedTest)]; + $mockTestData = array_merge($mockSimpleTest, $mockExtendedTest); $this->setMockTestOutput($mockTestData); // parse and generate test object with mocked data @@ -112,7 +112,7 @@ public function testGenerateExtendedWithHooks() ->withTestReference("simpleTest") ->build(); - $mockTestData = ['tests' => array_merge($mockSimpleTest, $mockExtendedTest)]; + $mockTestData = array_merge($mockSimpleTest, $mockExtendedTest); $this->setMockTestOutput($mockTestData); // parse and generate test object with mocked data @@ -143,7 +143,7 @@ public function testExtendedTestNoParent() ->withTestReference("simpleTest") ->build(); - $mockTestData = ['tests' => array_merge($mockExtendedTest)]; + $mockTestData = array_merge($mockExtendedTest); $this->setMockTestOutput($mockTestData); // parse and generate test object with mocked data @@ -182,7 +182,7 @@ public function testExtendingExtendedTest() ->withTestReference("simpleTest") ->build(); - $mockTestData = ['tests' => array_merge($mockParentTest, $mockSimpleTest, $mockExtendedTest)]; + $mockTestData = array_merge($mockParentTest, $mockSimpleTest, $mockExtendedTest); $this->setMockTestOutput($mockTestData); $this->expectExceptionMessage("Cannot extend a test that already extends another test. Test: simpleTest"); diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Util/TestGeneratorTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Util/TestGeneratorTest.php index 8f8eee749..e11b0f9b2 100644 --- a/dev/tests/unit/Magento/FunctionalTestFramework/Util/TestGeneratorTest.php +++ b/dev/tests/unit/Magento/FunctionalTestFramework/Util/TestGeneratorTest.php @@ -8,6 +8,7 @@ use AspectMock\Test as AspectMock; +use Magento\FunctionalTestingFramework\Filter\FilterList; use Magento\FunctionalTestingFramework\Test\Objects\ActionObject; use Magento\FunctionalTestingFramework\Test\Objects\TestHookObject; use Magento\FunctionalTestingFramework\Test\Objects\TestObject; @@ -17,6 +18,16 @@ class TestGeneratorTest extends MagentoTestCase { + /** + * After method functionality + * + * @return void + */ + public function tearDown() + { + AspectMock::clean(); + } + /** * Basic test to check exceptions for incorrect entities. * @@ -99,4 +110,54 @@ public function testAllowSkipped() $this->assertContains($actionInput, $output); $this->assertContains($beforeActionInput, $output); } + + /** + * Tests that TestGenerator createAllTestFiles correctly filters based on severity + * + * @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException + */ + public function testFilter() + { + // Mock filters for TestGenerator + AspectMock::double( + MftfApplicationConfig::class, + ['getFilterList' => new FilterList(['severity' => ["CRITICAL"]])] + ); + + $actionInput = 'fakeInput'; + $actionObject = new ActionObject('fakeAction', 'comment', [ + 'userInput' => $actionInput + ]); + + $annotation1 = ['severity' => ['CRITICAL']]; + $annotation2 = ['severity' => ['MINOR']]; + $test1 = new TestObject( + "test1", + ["fakeAction" => $actionObject], + $annotation1, + [], + "filename" + ); + $test2 = new TestObject( + "test2", + ["fakeAction" => $actionObject], + $annotation2, + [], + "filename" + ); + AspectMock::double(TestGenerator::class, ['loadAllTestObjects' => ["sampleTest" => $test1, "test2" => $test2]]); + + // Mock createCestFile to return name of tests that testGenerator tried to create + $generatedTests = []; + AspectMock::double(TestGenerator::class, ['createCestFile' => function ($arg1, $arg2) use (&$generatedTests) { + $generatedTests[$arg2] = true; + }]); + + $testGeneratorObject = TestGenerator::getInstance("", ["sampleTest" => $test1, "test2" => $test2]); + $testGeneratorObject->createAllTestFiles(null, []); + + // Ensure Test1 was Generated but not Test 2 + $this->assertArrayHasKey('test1Cest', $generatedTests); + $this->assertArrayNotHasKey('test2Cest', $generatedTests); + } } diff --git a/docs/commands/mftf.md b/docs/commands/mftf.md index b2c36dae4..e0394dc12 100644 --- a/docs/commands/mftf.md +++ b/docs/commands/mftf.md @@ -159,6 +159,7 @@ vendor/bin/mftf generate:tests [option] [<test name>] [<test name>] [--remove] | Option | Description| | ---| --- | | `--config=[<default> or <singleRun> or <parallel>]` | Creates a single manifest file with a list of all tests. The default location is `tests/functional/Magento/FunctionalTest/_generated/testManifest.txt`.<br/> You can split the list into multiple groups using `--config=parallel`; the groups will be generated in `_generated/groups/` like `_generated/groups/group1.txt, group2.txt, ...`.<br/> Available values: `default` (default), `singleRun`(same as `default`), and `parallel`.<br/> Example: `generate:tests --config=parallel`. | +| `--filter` | Option to filter tests to be generated.<br/>Template: '<filterName>:<filterValue>'.<br/>Existing filter types: severity.<br/>Existing severity values: BLOCKER, CRITICAL, MAJOR, AVERAGE, MINOR.<br/>Example: --filter=severity:CRITICAL| | `--force` | Forces test generation, regardless of the module merge order defined in the Magento instance. Example: `generate:tests --force`. | | `-i,--time` | Set time in minutes to determine the group size when `--config=parallel` is used. The __default value__ is `10`. Example: `generate:tests --config=parallel --time=15`| | `--tests` | Defines the test configuration as a JSON string.| diff --git a/src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php b/src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php index 80db27de0..4b215cb4f 100644 --- a/src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php +++ b/src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php @@ -6,6 +6,7 @@ namespace Magento\FunctionalTestingFramework\Config; use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; +use Magento\FunctionalTestingFramework\Filter\FilterList; class MftfApplicationConfig { @@ -25,6 +26,13 @@ class MftfApplicationConfig const LEVEL_NONE = "none"; const MFTF_DEBUG_LEVEL = [self::LEVEL_DEFAULT, self::LEVEL_DEVELOPER, self::LEVEL_NONE]; + /** + * Contains object with test filters. + * + * @var FilterList + */ + private $filterList; + /** * Determines whether the user has specified a force option for generation * @@ -74,6 +82,7 @@ class MftfApplicationConfig * @param boolean $verboseEnabled * @param string $debugLevel * @param boolean $allowSkipped + * @param array $filters * @throws TestFrameworkException */ private function __construct( @@ -81,7 +90,8 @@ private function __construct( $phase = self::EXECUTION_PHASE, $verboseEnabled = null, $debugLevel = self::LEVEL_NONE, - $allowSkipped = false + $allowSkipped = false, + $filters = [] ) { $this->forceGenerate = $forceGenerate; @@ -101,6 +111,7 @@ private function __construct( $this->debugLevel = self::LEVEL_DEVELOPER; } $this->allowSkipped = $allowSkipped; + $this->filterList = new FilterList($filters); } /** @@ -112,6 +123,7 @@ private function __construct( * @param boolean $verboseEnabled * @param string $debugLevel * @param boolean $allowSkipped + * @param array $filters * @return void * @throws TestFrameworkException */ @@ -120,11 +132,19 @@ public static function create( $phase = self::EXECUTION_PHASE, $verboseEnabled = null, $debugLevel = self::LEVEL_NONE, - $allowSkipped = false + $allowSkipped = false, + $filters = [] ) { if (self::$MFTF_APPLICATION_CONTEXT == null) { self::$MFTF_APPLICATION_CONTEXT = - new MftfApplicationConfig($forceGenerate, $phase, $verboseEnabled, $debugLevel, $allowSkipped); + new MftfApplicationConfig( + $forceGenerate, + $phase, + $verboseEnabled, + $debugLevel, + $allowSkipped, + $filters + ); } } @@ -196,4 +216,14 @@ public function getPhase() { return $this->phase; } + + /** + * Returns a class with registered filter list. + * + * @return FilterList + */ + public function getFilterList() + { + return $this->filterList; + } } diff --git a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php index 3433f23a8..273632586 100644 --- a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php @@ -35,7 +35,7 @@ class BaseGenerateCommand extends Command * * @var SymfonyStyle */ - private $ioStyle = null; + protected $ioStyle = null; /** * Configures the base command. diff --git a/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php b/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php index b9378edc1..3ca53f8f6 100644 --- a/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php @@ -18,7 +18,11 @@ use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Input\InputOption; use Symfony\Component\Console\Output\OutputInterface; +use Symfony\Component\Console\Style\SymfonyStyle; +/** + * @SuppressWarnings(PHPMD) + */ class GenerateTestsCommand extends BaseGenerateCommand { /** @@ -30,6 +34,7 @@ protected function configure() { $this->setName('generate:tests') ->setDescription('Run validation and generate all test files and suites based on xml declarations') + ->addUsage('AdminLoginTest') ->addArgument( 'name', InputArgument::OPTIONAL | InputArgument::IS_ARRAY, @@ -46,6 +51,15 @@ protected function configure() 't', InputOption::VALUE_REQUIRED, 'A parameter accepting a JSON string used to determine the test configuration' + )->addOption( + 'filter', + null, + InputOption::VALUE_IS_ARRAY | InputOption::VALUE_OPTIONAL, + 'Option to filter tests to be generated.' . PHP_EOL + . '<info>Template:</info> <filterName>:<filterValue>' . PHP_EOL + . '<info>Existing filter types:</info> severity.' . PHP_EOL + . '<info>Existing severity values:</info> BLOCKER, CRITICAL, MAJOR, AVERAGE, MINOR.' . PHP_EOL + . '<info>Example:</info> --filter=severity:CRITICAL' . PHP_EOL ); parent::configure(); @@ -56,13 +70,14 @@ protected function configure() * * @param InputInterface $input * @param OutputInterface $output - * @return void + * @return void|integer * @throws TestFrameworkException * @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException * @throws \Magento\FunctionalTestingFramework\Exceptions\XmlException */ protected function execute(InputInterface $input, OutputInterface $output) { + $this->setOutputStyle($input, $output); $tests = $input->getArgument('name'); $config = $input->getOption('config'); $json = $input->getOption('tests'); // for backward compatibility @@ -72,15 +87,25 @@ protected function execute(InputInterface $input, OutputInterface $output) $remove = $input->getOption('remove'); $verbose = $output->isVerbose(); $allowSkipped = $input->getOption('allow-skipped'); - + $filters = $input->getOption('filter'); + foreach ($filters as $filter) { + list($filterType, $filterValue) = explode(':', $filter); + $filterList[$filterType][] = $filterValue; + } // Set application configuration so we can references the user options in our framework - MftfApplicationConfig::create( - $force, - MftfApplicationConfig::GENERATION_PHASE, - $verbose, - $debug, - $allowSkipped - ); + try { + MftfApplicationConfig::create( + $force, + MftfApplicationConfig::GENERATION_PHASE, + $verbose, + $debug, + $allowSkipped, + $filterList ?? [] + ); + } catch (\Exception $exception) { + $this->ioStyle->error("Test generation failed." . PHP_EOL . $exception->getMessage()); + return 1; + } $this->setOutputStyle($input, $output); $this->showMftfNotices($output); @@ -105,20 +130,31 @@ protected function execute(InputInterface $input, OutputInterface $output) ($debug !== MftfApplicationConfig::LEVEL_NONE)); } - $testConfiguration = $this->createTestConfiguration($json, $tests); + try { + $testConfiguration = $this->createTestConfiguration($json, $tests); - // create our manifest file here - $testManifest = TestManifestFactory::makeManifest($config, $testConfiguration['suites']); - TestGenerator::getInstance(null, $testConfiguration['tests'])->createAllTestFiles($testManifest); + // create our manifest file here + $testManifest = TestManifestFactory::makeManifest($config, $testConfiguration['suites']); - if ($config == 'parallel') { - /** @var ParallelTestManifest $testManifest */ - $testManifest->createTestGroups($time); - } + TestGenerator::getInstance(null, $testConfiguration['tests'])->createAllTestFiles($testManifest); - SuiteGenerator::getInstance()->generateAllSuites($testManifest); + if ($config == 'parallel') { + /** @var ParallelTestManifest $testManifest */ + $testManifest->createTestGroups($time); + } - $testManifest->generate(); + SuiteGenerator::getInstance()->generateAllSuites($testManifest); + + $testManifest->generate(); + } catch (\Exception $e) { + $message = $e->getMessage() . PHP_EOL; + $message .= !empty($filters) ? 'Filter(s): ' . implode(', ', $filters) . PHP_EOL : ''; + $message .= !empty($tests) ? 'Test name(s): ' . implode(', ', $tests) . PHP_EOL : ''; + $message .= !empty($json) && empty($tests) ? 'Test configuration: ' . $json . PHP_EOL : ''; + $this->ioStyle->note($message); + + return 1; + } $output->writeln("Generate Tests Command Run"); } diff --git a/src/Magento/FunctionalTestingFramework/Filter/FilterInterface.php b/src/Magento/FunctionalTestingFramework/Filter/FilterInterface.php new file mode 100644 index 000000000..bb89ab3c2 --- /dev/null +++ b/src/Magento/FunctionalTestingFramework/Filter/FilterInterface.php @@ -0,0 +1,27 @@ +<?php +/** + * Copyright © Magento, Inc. All rights reserved. + * See COPYING.txt for license details. + */ + +declare(strict_types=1); + +namespace Magento\FunctionalTestingFramework\Filter; + +/** + * Interface for future test filters + * @api + */ +interface FilterInterface +{ + /** + * @param array $filterValues + */ + public function __construct(array $filterValues = []); + + /** + * @param array $tests + * @return void + */ + public function filter(array &$tests); +} diff --git a/src/Magento/FunctionalTestingFramework/Filter/FilterList.php b/src/Magento/FunctionalTestingFramework/Filter/FilterList.php new file mode 100644 index 000000000..35ff32edd --- /dev/null +++ b/src/Magento/FunctionalTestingFramework/Filter/FilterList.php @@ -0,0 +1,57 @@ +<?php +/** + * Copyright © Magento, Inc. All rights reserved. + * See COPYING.txt for license details. + */ + +declare(strict_types=1); + +namespace Magento\FunctionalTestingFramework\Filter; + +use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; + +/** + * Class FilterList has a list of filters. + */ +class FilterList +{ + /** + * List of filters + * @var \Magento\FunctionalTestingFramework\Filter\FilterInterface[] + */ + private $filters = []; + + /** + * Constructor for Filter list. + * + * @param array $filters + * @throws \Exception + */ + public function __construct(array $filters = []) + { + foreach ($filters as $filterType => $filterValue) { + $className = "Magento\FunctionalTestingFramework\Filter\Test\\" . ucfirst($filterType); + if (!class_exists($className)) { + throw new TestFrameworkException("Filter type '" . $filterType . "' do not exist."); + } + $this->filters[$filterType] = new $className($filterValue); + } + } + + /** + * @return array + */ + public function getFilters(): array + { + return $this->filters; + } + + /** + * @param string $filterType + * @return \Magento\FunctionalTestingFramework\Filter\FilterInterface + */ + public function getFilter(string $filterType): FilterInterface + { + return $this->filters[$filterType]; + } +} diff --git a/src/Magento/FunctionalTestingFramework/Filter/Test/Severity.php b/src/Magento/FunctionalTestingFramework/Filter/Test/Severity.php new file mode 100644 index 000000000..e88c4af7b --- /dev/null +++ b/src/Magento/FunctionalTestingFramework/Filter/Test/Severity.php @@ -0,0 +1,68 @@ +<?php +/** + * Copyright © Magento, Inc. All rights reserved. + * See COPYING.txt for license details. + */ + +declare(strict_types=1); + +namespace Magento\FunctionalTestingFramework\Filter\Test; + +use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; +use Magento\FunctionalTestingFramework\Filter\FilterInterface; +use Magento\FunctionalTestingFramework\Test\Objects\TestObject; +use Magento\FunctionalTestingFramework\Test\Util\AnnotationExtractor; + +/** + * Class Severity + */ +class Severity implements FilterInterface +{ + const ANNOTATION_TAG = 'severity'; + + /** + * @var array + */ + private $filterValues = []; + + /** + * Severity constructor. + * + * @param array $filterValues + * @throws TestFrameworkException + */ + public function __construct(array $filterValues = []) + { + $severityValues = AnnotationExtractor::MAGENTO_TO_ALLURE_SEVERITY_MAP; + + foreach ($filterValues as $filterValue) { + if (!isset($severityValues[$filterValue])) { + throw new TestFrameworkException( + 'Not existing severity specified.' . PHP_EOL + . 'Possible values: '. implode(', ', array_keys($severityValues)) . '.' . PHP_EOL + . 'Provided values: ' . implode(', ', $filterValues) . '.' . PHP_EOL + ); + } + $this->filterValues[] = $severityValues[$filterValue]; + } + } + + /** + * Filter tests by severity. + * + * @param TestObject[] $tests + * @return void + */ + public function filter(array &$tests) + { + /** @var TestObject $test */ + foreach ($tests as $testName => $test) { + $severities = $test->getAnnotationByName(self::ANNOTATION_TAG); + foreach ($severities as $severity) { + if (!in_array($severity, $this->filterValues, true)) { + unset($tests[$testName]); + } + } + } + } +} diff --git a/src/Magento/FunctionalTestingFramework/Suite/Objects/SuiteObject.php b/src/Magento/FunctionalTestingFramework/Suite/Objects/SuiteObject.php index d7f6bd18a..7168d9556 100644 --- a/src/Magento/FunctionalTestingFramework/Suite/Objects/SuiteObject.php +++ b/src/Magento/FunctionalTestingFramework/Suite/Objects/SuiteObject.php @@ -5,8 +5,11 @@ */ namespace Magento\FunctionalTestingFramework\Suite\Objects; +use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig; +use Magento\FunctionalTestingFramework\Filter\FilterInterface; use Magento\FunctionalTestingFramework\Test\Objects\TestHookObject; use Magento\FunctionalTestingFramework\Test\Objects\TestObject; +use Magento\FunctionalTestingFramework\Util\Logger\LoggingUtil; /** * Class SuiteObject @@ -84,6 +87,7 @@ public function getTests() * @param TestObject[] $includeTests * @param TestObject[] $excludeTests * @return TestObject[] + * @throws \Exception */ private function resolveTests($includeTests, $excludeTests) { @@ -95,12 +99,10 @@ private function resolveTests($includeTests, $excludeTests) unset($finalTestList[$testName]); } - if (empty($finalTestList)) { - trigger_error( - "Current suite configuration for " . - $this->name . " contains no tests.", - E_USER_WARNING - ); + $filters = MftfApplicationConfig::getConfig()->getFilterList()->getFilters(); + /** @var FilterInterface $filter */ + foreach ($filters as $filter) { + $filter->filter($finalTestList); } return $finalTestList; diff --git a/src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php b/src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php index e3da10497..b40b48ae7 100644 --- a/src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php +++ b/src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php @@ -95,6 +95,14 @@ public function generateAllSuites($testManifest) $suites = $testManifest->getSuiteConfig(); foreach ($suites as $suiteName => $suiteContent) { + if (empty($suiteContent)) { + LoggingUtil::getInstance()->getLogger(self::class)->notification( + "Suite '" . $suiteName . "' contains no tests and won't be generated." . PHP_EOL, + [], + true + ); + continue; + } $firstElement = array_values($suiteContent)[0]; // if the first element is a string we know that we simply have an array of tests diff --git a/src/Magento/FunctionalTestingFramework/Test/Handlers/TestObjectHandler.php b/src/Magento/FunctionalTestingFramework/Test/Handlers/TestObjectHandler.php index a0d490df5..d5ed43806 100644 --- a/src/Magento/FunctionalTestingFramework/Test/Handlers/TestObjectHandler.php +++ b/src/Magento/FunctionalTestingFramework/Test/Handlers/TestObjectHandler.php @@ -16,6 +16,8 @@ use Magento\FunctionalTestingFramework\Test\Util\ObjectExtensionUtil; use Magento\FunctionalTestingFramework\Test\Util\TestObjectExtractor; use Magento\FunctionalTestingFramework\Test\Util\AnnotationExtractor; +use Magento\FunctionalTestingFramework\Util\Logger\LoggingUtil; +use PHP_CodeSniffer\Tokenizers\PHP; /** * Class TestObjectHandler @@ -140,7 +142,7 @@ private function initTestData() } $exceptionCollector = new ExceptionCollector(); - foreach ($parsedTestArray[TestObjectHandler::XML_ROOT] as $testName => $testData) { + foreach ($parsedTestArray as $testName => $testData) { if (!is_array($testData)) { continue; } diff --git a/src/Magento/FunctionalTestingFramework/Test/Parsers/TestDataParser.php b/src/Magento/FunctionalTestingFramework/Test/Parsers/TestDataParser.php index 18044c1af..dc3489de1 100644 --- a/src/Magento/FunctionalTestingFramework/Test/Parsers/TestDataParser.php +++ b/src/Magento/FunctionalTestingFramework/Test/Parsers/TestDataParser.php @@ -7,29 +7,42 @@ namespace Magento\FunctionalTestingFramework\Test\Parsers; use Magento\FunctionalTestingFramework\Config\DataInterface; +use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig; +use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; +use Magento\FunctionalTestingFramework\Filter\FilterInterface; +use Magento\FunctionalTestingFramework\Util\Logger\LoggingUtil; /** * Class TestDataParser */ class TestDataParser { + /** + * @var DataInterface + */ + private $testData; + /** * TestDataParser constructor. * * @param DataInterface $testData + * @throws \Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException */ public function __construct(DataInterface $testData) { - $this->testData = $testData; + $this->testData = array_filter($testData->get('tests'), function ($value) { + return is_array($value); + }); } /** * Returns an array of data based on *Test.xml files * * @return array + * @throws TestFrameworkException */ public function readTestData() { - return $this->testData->get(); + return $this->testData; } } diff --git a/src/Magento/FunctionalTestingFramework/Util/Logger/MftfLogger.php b/src/Magento/FunctionalTestingFramework/Util/Logger/MftfLogger.php index 3983a755a..0e8c1e17f 100644 --- a/src/Magento/FunctionalTestingFramework/Util/Logger/MftfLogger.php +++ b/src/Magento/FunctionalTestingFramework/Util/Logger/MftfLogger.php @@ -49,4 +49,23 @@ public function criticalFailure($message, array $context = [], $verbose = false) } parent::critical($message, $context); } + + /** + * Adds a log record at the NOTICE level. + * + * @param string $message + * @param array $context + * @param boolean $verbose + * @return void + * @throws \Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException + */ + public function notification($message, array $context = [], $verbose = false) + { + $message = "NOTICE: " . $message; + // Suppress print during unit testing + if (MftfApplicationConfig::getConfig()->getPhase() !== MftfApplicationConfig::UNIT_TEST_PHASE && $verbose) { + print ($message . implode("\n", $context) . "\n"); + } + parent::notice($message, $context); + } } diff --git a/src/Magento/FunctionalTestingFramework/Util/Manifest/DefaultTestManifest.php b/src/Magento/FunctionalTestingFramework/Util/Manifest/DefaultTestManifest.php index eb4f79db2..4cbef3168 100644 --- a/src/Magento/FunctionalTestingFramework/Util/Manifest/DefaultTestManifest.php +++ b/src/Magento/FunctionalTestingFramework/Util/Manifest/DefaultTestManifest.php @@ -86,6 +86,9 @@ public function generate() protected function generateSuiteEntries($fileResource) { foreach ($this->getSuiteConfig() as $suiteName => $tests) { + if (count($tests) === 0) { + continue; + } $line = "-g {$suiteName}"; fwrite($fileResource, $line . PHP_EOL); } diff --git a/src/Magento/FunctionalTestingFramework/Util/Manifest/ParallelTestManifest.php b/src/Magento/FunctionalTestingFramework/Util/Manifest/ParallelTestManifest.php index 9b12cfd00..201e002a2 100644 --- a/src/Magento/FunctionalTestingFramework/Util/Manifest/ParallelTestManifest.php +++ b/src/Magento/FunctionalTestingFramework/Util/Manifest/ParallelTestManifest.php @@ -132,7 +132,7 @@ private function generateGroupFile($testGroup, $nodeNumber, $suites) $fileResource = fopen($this->dirPath . DIRECTORY_SEPARATOR . "group{$nodeNumber}.txt", 'a'); $line = null; - if (array_key_exists($entryName, $suites)) { + if (!empty($suites[$entryName])) { $line = "-g {$entryName}"; } else { $line = $this->relativeDirPath . DIRECTORY_SEPARATOR . $entryName . '.php'; diff --git a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php index 2ec565c44..2e7d265fa 100644 --- a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php +++ b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php @@ -11,6 +11,7 @@ use Magento\FunctionalTestingFramework\DataGenerator\Objects\EntityDataObject; use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; use Magento\FunctionalTestingFramework\Exceptions\TestReferenceException; +use Magento\FunctionalTestingFramework\Filter\FilterInterface; use Magento\FunctionalTestingFramework\Suite\Handlers\SuiteObjectHandler; use Magento\FunctionalTestingFramework\Test\Handlers\ActionGroupObjectHandler; use Magento\FunctionalTestingFramework\Test\Handlers\TestObjectHandler; @@ -288,6 +289,11 @@ private function assembleAllTestPhp($testManifest, array $testsToIgnore) /** @var TestObject[] $testObjects */ $testObjects = $this->loadAllTestObjects($testsToIgnore); $cestPhpArray = []; + $filters = MftfApplicationConfig::getConfig()->getFilterList()->getFilters(); + /** @var FilterInterface $filter */ + foreach ($filters as $filter) { + $filter->filter($testObjects); + } foreach ($testObjects as $test) { // Do not generate test if it is an extended test and parent does not exist