Skip to content

Commit 686afb0

Browse files
jilu1jilu1
authored
Merge branch 'develop' into MQE-2082
2 parents 05a620f + cd13059 commit 686afb0

File tree

8 files changed

+281
-17
lines changed

8 files changed

+281
-17
lines changed

‎.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,3 +19,4 @@ dev/tests/mftf.log
1919
dev/tests/docs/*
2020
dev/tests/_output
2121
dev/tests/functional.suite.yml
22+
mftf-annotations-static-check.txt

‎docs/commands/mftf.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -477,6 +477,10 @@ vendor/bin/mftf static-checks actionGroupArguments
477477
vendor/bin/mftf static-checks deprecatedEntityUsage
478478
```
479479

480+
```bash
481+
vendor/bin/mftf static-checks annotations
482+
```
483+
480484
```bash
481485
vendor/bin/mftf static-checks deprecatedEntityUsage -p path/to/mftf/test/module
482486
```
@@ -492,6 +496,7 @@ vendor/bin/mftf static-checks testDependencies actionGroupArguments
492496
|`testDependencies` | Checks that test dependencies do not violate Magento module's composer dependencies.|
493497
|`actionGroupArguments` | Checks that action groups do not have unused arguments.|
494498
|`deprecatedEntityUsage`| Checks that deprecated test entities are not being referenced.|
499+
|`annotations`| Checks various details of test annotations, such as missing annotations or duplicate annotations.|
495500
496501
#### Defining ruleset
497502

‎src/Magento/FunctionalTestingFramework/StaticCheck/AnnotationsCheck.php

Lines changed: 246 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,246 @@
1+
<?php
2+
/**
3+
* Copyright © Magento, Inc. All rights reserved.
4+
* See COPYING.txt for license details.
5+
*/
6+
7+
namespace Magento\FunctionalTestingFramework\StaticCheck;
8+
9+
use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig;
10+
use Magento\FunctionalTestingFramework\Test\Handlers\TestObjectHandler;
11+
use Magento\FunctionalTestingFramework\Test\Objects\TestObject;
12+
use Magento\FunctionalTestingFramework\Util\Script\ScriptUtil;
13+
use Symfony\Component\Console\Input\InputInterface;
14+
use Exception;
15+
16+
/**
17+
* Class AnnotationsCheck
18+
* @package Magento\FunctionalTestingFramework\StaticCheck
19+
*/
20+
class AnnotationsCheck implements StaticCheckInterface
21+
{
22+
const ERROR_LOG_FILENAME = 'mftf-annotations-static-check';
23+
const ERROR_LOG_MESSAGE = 'MFTF Annotations Static Check';
24+
25+
/**
26+
* Array containing all errors found after running the execute() function.
27+
* @var array
28+
*/
29+
private $errors = [];
30+
31+
/**
32+
* String representing the output summary found after running the execute() function.
33+
* @var string
34+
*/
35+
private $output;
36+
37+
/**
38+
* Array containing
39+
* key = Story appended to Title
40+
* value = test names that have that pair
41+
* @var array
42+
*/
43+
private $storiesTitlePairs = [];
44+
45+
/**
46+
* Array containing
47+
* key = testCaseId appended to Title
48+
* value = test names that have that pair
49+
* @var array
50+
*/
51+
private $testCaseIdTitlePairs = [];
52+
53+
/**
54+
* Validates test annotations
55+
*
56+
* @param InputInterface $input
57+
* @return void
58+
* @throws Exception
59+
*/
60+
public function execute(InputInterface $input)
61+
{
62+
// Set MFTF to the UNIT_TEST_PHASE to mute the default DEPRECATION warnings from the TestObjectHandler.
63+
MftfApplicationConfig::create(
64+
true,
65+
MftfApplicationConfig::UNIT_TEST_PHASE,
66+
false,
67+
MftfApplicationConfig::LEVEL_DEFAULT,
68+
true
69+
);
70+
$allTests = TestObjectHandler::getInstance(false)->getAllObjects();
71+
72+
foreach ($allTests as $test) {
73+
$this->validateRequiredAnnotations($test);
74+
$this->validateSkipIssueId($test);
75+
$this->aggregateStoriesTitlePairs($test);
76+
$this->aggregateTestCaseIdTitlePairs($test);
77+
}
78+
79+
$this->validateStoriesTitlePairs();
80+
$this->validateTestCaseIdTitlePairs();
81+
82+
$scriptUtil = new ScriptUtil();
83+
$this->output = $scriptUtil->printErrorsToFile(
84+
$this->errors,
85+
self::ERROR_LOG_FILENAME,
86+
self::ERROR_LOG_MESSAGE
87+
);
88+
}
89+
90+
/**
91+
* Return array containing all errors found after running the execute() function.
92+
* @return array
93+
*/
94+
public function getErrors()
95+
{
96+
return $this->errors;
97+
}
98+
99+
/**
100+
* Return string of a short human readable result of the check. For example: "No Dependency errors found."
101+
* @return string
102+
*/
103+
public function getOutput()
104+
{
105+
return $this->output;
106+
}
107+
108+
/**
109+
* Validates that the test has the following annotations:
110+
* stories
111+
* title
112+
* description
113+
* severity
114+
*
115+
* @param TestObject $test
116+
* @return void
117+
*/
118+
private function validateRequiredAnnotations($test)
119+
{
120+
$annotations = $test->getAnnotations();
121+
$missing = [];
122+
123+
$stories = $annotations['stories'] ?? null;
124+
if ($stories === null) {
125+
$missing[] = "stories";
126+
}
127+
128+
$title = $annotations['title'] ?? null;
129+
if ($title === null) {
130+
$missing[] = "title";
131+
}
132+
133+
$description = $annotations['description']['main'] ?? null;
134+
if ($description === null) {
135+
$missing[] = "description";
136+
}
137+
138+
$severity = $annotations['severity'] ?? null;
139+
if ($severity === null) {
140+
$missing[] = "severity";
141+
}
142+
143+
$allMissing = join(", ", $missing);
144+
if (strlen($allMissing) > 0) {
145+
$this->errors[][] = "Test {$test->getName()} is missing the required annotations: " . $allMissing;
146+
}
147+
}
148+
149+
/**
150+
* Validates that if the test is skipped, that it has an issueId value.
151+
*
152+
* @param TestObject $test
153+
* @return void
154+
*/
155+
private function validateSkipIssueId($test)
156+
{
157+
$annotations = $test->getAnnotations();
158+
159+
$skip = $annotations['skip'] ?? null;
160+
if ($skip !== null) {
161+
$issueId = $skip[0] ?? null;
162+
if ($issueId === null || strlen($issueId) == 0) {
163+
$this->errors[][] = "Test {$test->getName()} is skipped but the issueId is empty.";
164+
}
165+
}
166+
}
167+
168+
/**
169+
* Add the key = "stories appended to title", value = test name, to the class variable.
170+
*
171+
* @param TestObject $test
172+
* @return void
173+
*/
174+
private function aggregateStoriesTitlePairs($test)
175+
{
176+
$annotations = $test->getAnnotations();
177+
$stories = $annotations['stories'][0] ?? null;
178+
$title = $this->getTestTitleWithoutPrefix($test);
179+
if ($stories !== null && $title !== null) {
180+
$this->storiesTitlePairs[$stories . $title][] = $test->getName();
181+
}
182+
}
183+
184+
/**
185+
* Add the key = "testCaseId appended to title", value = test name, to the class variable.
186+
*
187+
* @param TestObject $test
188+
* @return void
189+
*/
190+
private function aggregateTestCaseIdTitlePairs($test)
191+
{
192+
$annotations = $test->getAnnotations();
193+
$testCaseId = $annotations['testCaseId'][0] ?? null;
194+
$title = $this->getTestTitleWithoutPrefix($test);
195+
if ($testCaseId !== null && $title !== null) {
196+
$this->testCaseIdTitlePairs[$testCaseId . $title][] = $test->getName();
197+
}
198+
}
199+
200+
/**
201+
* Strip away the testCaseId prefix that was automatically added to the test title
202+
* so that way we have just the raw title from the XML file.
203+
*
204+
* @param TestObject $test
205+
* @return string|null
206+
*/
207+
private function getTestTitleWithoutPrefix($test)
208+
{
209+
$annotations = $test->getAnnotations();
210+
$title = $annotations['title'][0] ?? null;
211+
if ($title === null) {
212+
return null;
213+
} else {
214+
$testCaseId = $annotations['testCaseId'][0] ?? "[NO TESTCASEID]";
215+
return substr($title, strlen($testCaseId . ": "));
216+
}
217+
}
218+
219+
/**
220+
* Adds an error if any story+title pairs are used by more than one test.
221+
*
222+
* @return void
223+
*/
224+
private function validateStoriesTitlePairs()
225+
{
226+
foreach ($this->storiesTitlePairs as $pair) {
227+
if (sizeof($pair) > 1) {
228+
$this->errors[][] = "Stories + title combination must be unique: " . join(", ", $pair);
229+
}
230+
}
231+
}
232+
233+
/**
234+
* Adds an error if any testCaseId+title pairs are used by more than one test.
235+
*
236+
* @return void
237+
*/
238+
private function validateTestCaseIdTitlePairs()
239+
{
240+
foreach ($this->testCaseIdTitlePairs as $pair) {
241+
if (sizeof($pair) > 1) {
242+
$this->errors[][] = "testCaseId + title combination must be unique: " . join(", ", $pair);
243+
}
244+
}
245+
}
246+
}

‎src/Magento/FunctionalTestingFramework/StaticCheck/StaticChecksList.php

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ public function __construct(array $checks = [])
3333
'testDependencies' => new TestDependencyCheck(),
3434
'actionGroupArguments' => new ActionGroupArgumentsCheck(),
3535
self::DEPRECATED_ENTITY_USAGE_CHECK_NAME => new DeprecatedEntityUsageCheck(),
36+
'annotations' => new AnnotationsCheck()
3637
] + $checks;
3738
}
3839

‎src/Magento/FunctionalTestingFramework/Test/Handlers/TestObjectHandler.php

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -53,11 +53,11 @@ class TestObjectHandler implements ObjectHandlerInterface
5353
* @return TestObjectHandler
5454
* @throws XmlException
5555
*/
56-
public static function getInstance()
56+
public static function getInstance($validateAnnotations = true)
5757
{
5858
if (!self::$testObjectHandler) {
5959
self::$testObjectHandler = new TestObjectHandler();
60-
self::$testObjectHandler->initTestData();
60+
self::$testObjectHandler->initTestData($validateAnnotations);
6161
}
6262

6363
return self::$testObjectHandler;
@@ -129,7 +129,7 @@ public function getTestsByGroup($groupName)
129129
* @SuppressWarnings(PHPMD.UnusedPrivateMethod)
130130
* @throws XmlException
131131
*/
132-
private function initTestData()
132+
private function initTestData($validateAnnotations = true)
133133
{
134134
$testDataParser = ObjectManagerFactory::getObjectManager()->create(TestDataParser::class);
135135
$parsedTestArray = $testDataParser->readTestData();
@@ -150,15 +150,17 @@ private function initTestData()
150150
continue;
151151
}
152152
try {
153-
$this->tests[$testName] = $testObjectExtractor->extractTestData($testData);
153+
$this->tests[$testName] = $testObjectExtractor->extractTestData($testData, $validateAnnotations);
154154
} catch (XmlException $exception) {
155155
$exceptionCollector->addError(self::class, $exception->getMessage());
156156
}
157157
}
158158
$exceptionCollector->throwException();
159159
$testNameValidator->summarize(NameValidationUtil::TEST_NAME);
160-
$testObjectExtractor->getAnnotationExtractor()->validateStoryTitleUniqueness();
161-
$testObjectExtractor->getAnnotationExtractor()->validateTestCaseIdTitleUniqueness();
160+
if ($validateAnnotations) {
161+
$testObjectExtractor->getAnnotationExtractor()->validateStoryTitleUniqueness();
162+
$testObjectExtractor->getAnnotationExtractor()->validateTestCaseIdTitleUniqueness();
163+
}
162164
}
163165

164166
/**

‎src/Magento/FunctionalTestingFramework/Test/Util/ActionGroupAnnotationExtractor.php

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,13 @@ class ActionGroupAnnotationExtractor extends AnnotationExtractor
1717
* This method trims away irrelevant tags and returns annotations used in the array passed. The annotations
1818
* can be found in both Tests and their child element tests.
1919
*
20-
* @param array $testAnnotations
21-
* @param string $filename
20+
* @param array $testAnnotations
21+
* @param string $filename
22+
* @param boolean $validateAnnotations
2223
* @return array
2324
* @throws \Exception
2425
*/
25-
public function extractAnnotations($testAnnotations, $filename)
26+
public function extractAnnotations($testAnnotations, $filename, $validateAnnotations = true)
2627
{
2728
$annotationObjects = [];
2829
$annotations = $this->stripDescriptorTags($testAnnotations, parent::NODE_NAME);

‎src/Magento/FunctionalTestingFramework/Test/Util/AnnotationExtractor.php

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -55,12 +55,14 @@ public function __construct()
5555
* This method trims away irrelevant tags and returns annotations used in the array passed. The annotations
5656
* can be found in both Tests and their child element tests.
5757
*
58-
* @param array $testAnnotations
59-
* @param string $filename
58+
* @param array $testAnnotations
59+
* @param string $filename
60+
* @param boolean $validateAnnotations
6061
* @return array
6162
* @throws XmlException
63+
* @SuppressWarnings(PHPMD.CyclomaticComplexity)
6264
*/
63-
public function extractAnnotations($testAnnotations, $filename)
65+
public function extractAnnotations($testAnnotations, $filename, $validateAnnotations = true)
6466
{
6567
$annotationObjects = [];
6668
$annotations = $this->stripDescriptorTags($testAnnotations, self::NODE_NAME);
@@ -82,7 +84,9 @@ public function extractAnnotations($testAnnotations, $filename)
8284

8385
if ($annotationKey == "skip") {
8486
$annotationData = $annotationData['issueId'];
85-
$this->validateSkippedIssues($annotationData, $filename);
87+
if ($validateAnnotations) {
88+
$this->validateSkippedIssues($annotationData, $filename);
89+
}
8690
}
8791

8892
foreach ($annotationData as $annotationValue) {
@@ -100,7 +104,9 @@ public function extractAnnotations($testAnnotations, $filename)
100104
}
101105

102106
$this->addTestCaseIdToTitle($annotationObjects, $filename);
103-
$this->validateMissingAnnotations($annotationObjects, $filename);
107+
if ($validateAnnotations) {
108+
$this->validateMissingAnnotations($annotationObjects, $filename);
109+
}
104110
$this->addStoryTitleToMap($annotationObjects, $filename);
105111

106112
return $annotationObjects;

0 commit comments

Comments
 (0)