Skip to content

Commit ac49cbe

Browse files
tomreecetomreece
authored
Merge pull request magento#437 from magento/MQE-1626
MQE-1626: bin/mftf run:manifest
2 parents ad0e4aa + c2b9c25 commit ac49cbe

File tree

3 files changed

+185
-0
lines changed

3 files changed

+185
-0
lines changed

‎docs/commands/mftf.md

+32
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,14 @@ vendor/bin/mftf run:test LoginAsAdminTest LoginAsCustomerTest -r
5858

5959
This command cleans up the previously generated tests; generates and runs the `LoginAsAdminTest` and `LoginAsCustomerTest` tests.
6060

61+
### Generate and run a testManifest.txt file
62+
63+
```bash
64+
vendor/bin/mftf run:manifest path/to/your/testManifest.txt
65+
```
66+
67+
This command runs all tests specified in a `testManifest.txt` file. When you generate tests, a `testManifest.txt` file is also generated for you. You can pass this file directly to the `run:manifest` command and it will execute all listed tests. You can also create your own file in the same format to execute a subset of tests. Note: This command does not generate tests.
68+
6169
### Generate and run previously failed tests
6270

6371
```bash
@@ -338,6 +346,30 @@ Generate the `LoginCustomerTest` and `StorefrontCreateCustomerTest` tests from X
338346
vendor/bin/mftf run:test LoginCustomerTest StorefrontCreateCustomerTest
339347
```
340348

349+
### `run:manifest`
350+
351+
Runs a testManifest.txt file.
352+
353+
This command runs all tests specified in a testManifest.xml file. It does not generate tests for you. You must do that as first.
354+
355+
#### Usage
356+
357+
```bash
358+
vendor/bin/mftf run:manifest path/to/your/testManifest.txt
359+
```
360+
361+
#### Example testManifest.xml file
362+
363+
Each line should contain either: one test path or one group (-g) reference.
364+
365+
```
366+
tests/functional/tests/MFTF/_generated/default/AdminLoginTestCest.php
367+
-g PaypalTestSuite
368+
tests/functional/tests/MFTF/_generated/default/SomeOtherTestCest.php
369+
tests/functional/tests/MFTF/_generated/default/ThirdTestCest.php
370+
-g SomeOtherSuite
371+
```
372+
341373
### `run:failed`
342374

343375
Regenerates and reruns tests that previously failed.

‎src/Magento/FunctionalTestingFramework/Console/CommandList.php

+1
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ public function __construct(array $commands = [])
3737
'run:test' => new RunTestCommand(),
3838
'run:group' => new RunTestGroupCommand(),
3939
'run:failed' => new RunTestFailedCommand(),
40+
'run:manifest' => new RunManifestCommand(),
4041
'setup:env' => new SetupEnvCommand(),
4142
'upgrade:tests' => new UpgradeTestsCommand(),
4243
'generate:docs' => new GenerateDocsCommand(),

‎src/Magento/FunctionalTestingFramework/Console/RunManifestCommand.php

+152
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
<?php
2+
/**
3+
* Copyright © Magento, Inc. All rights reserved.
4+
* See COPYING.txt for license details.
5+
*/
6+
declare(strict_types = 1);
7+
8+
namespace Magento\FunctionalTestingFramework\Console;
9+
10+
use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException;
11+
use Symfony\Component\Console\Command\Command;
12+
use Symfony\Component\Console\Input\InputArgument;
13+
use Symfony\Component\Console\Input\InputInterface;
14+
use Symfony\Component\Console\Output\OutputInterface;
15+
use Symfony\Component\Process\Process;
16+
17+
class RunManifestCommand extends Command
18+
{
19+
/**
20+
* The return code. Determined by all tests that run.
21+
*
22+
* @var integer
23+
*/
24+
private $returnCode = 0;
25+
26+
/**
27+
* A list of tests that failed.
28+
* Eg: "tests/functional/tests/MFTF/_generated/default/AdminLoginTestCest.php:AdminLoginTest"
29+
*
30+
* @var string[]
31+
*/
32+
private $failedTests = [];
33+
34+
/**
35+
* Configure the run:manifest command.
36+
*
37+
* @return void
38+
*/
39+
protected function configure()
40+
{
41+
$this->setName("run:manifest")
42+
->setDescription("runs a manifest file")
43+
->addArgument("path", InputArgument::REQUIRED, "path to a manifest file");
44+
}
45+
46+
/**
47+
* Executes the run:manifest command.
48+
*
49+
* @param InputInterface $input
50+
* @param OutputInterface $output
51+
* @throws TestFrameworkException
52+
* @return integer
53+
*/
54+
protected function execute(InputInterface $input, OutputInterface $output): int
55+
{
56+
$path = $input->getArgument("path");
57+
58+
if (!file_exists($path)) {
59+
throw new TestFrameworkException("Could not find file $path. Check the path and try again.");
60+
}
61+
62+
$manifestFile = file($path, FILE_IGNORE_NEW_LINES);
63+
64+
// Delete the Codeception failed file just in case it exists from any previous test runs
65+
$this->deleteFailedFile();
66+
67+
foreach ($manifestFile as $manifestLine) {
68+
if (empty($manifestLine)) {
69+
continue;
70+
}
71+
72+
$this->runManifestLine($manifestLine, $output);
73+
$this->aggregateFailed();
74+
}
75+
76+
if (!empty($this->failedTests)) {
77+
$this->deleteFailedFile();
78+
$this->writeFailedFile();
79+
}
80+
81+
return $this->returnCode;
82+
}
83+
84+
/**
85+
* Runs a test (or group) line from the manifest file
86+
*
87+
* @param string $manifestLine
88+
* @param OutputInterface $output
89+
* @return void
90+
*
91+
* @SuppressWarnings(PHPMD.UnusedLocalVariable) Need this because of the unused $type variable in the closure
92+
*/
93+
private function runManifestLine(string $manifestLine, OutputInterface $output)
94+
{
95+
$codeceptionCommand = realpath(PROJECT_ROOT . "/vendor/bin/codecept")
96+
. " run functional --verbose --steps "
97+
. $manifestLine;
98+
99+
// run the codecept command in a sub process
100+
$process = new Process($codeceptionCommand);
101+
$process->setWorkingDirectory(TESTS_BP);
102+
$process->setIdleTimeout(600);
103+
$process->setTimeout(0);
104+
$subReturnCode = $process->run(function ($type, $buffer) use ($output) {
105+
$output->write($buffer);
106+
});
107+
$this->returnCode = max($this->returnCode, $subReturnCode);
108+
}
109+
110+
/**
111+
* Keeps track of any tests that failed while running the manifest file.
112+
*
113+
* Each codecept command executions overwrites the failed file. Since we are running multiple codecept commands,
114+
* we need to hold on to any failures in order to write a final failed file containing all tests.
115+
*
116+
* @return void
117+
*/
118+
private function aggregateFailed()
119+
{
120+
if (file_exists(RunTestFailedCommand::TESTS_FAILED_FILE)) {
121+
$currentFile = file(RunTestFailedCommand::TESTS_FAILED_FILE, FILE_IGNORE_NEW_LINES);
122+
$this->failedTests = array_merge(
123+
$this->failedTests,
124+
$currentFile
125+
);
126+
}
127+
}
128+
129+
/**
130+
* Delete the Codeception failed file.
131+
*
132+
* @return void
133+
*/
134+
private function deleteFailedFile()
135+
{
136+
if (file_exists(RunTestFailedCommand::TESTS_FAILED_FILE)) {
137+
unlink(RunTestFailedCommand::TESTS_FAILED_FILE);
138+
}
139+
}
140+
141+
/**
142+
* Writes any tests that failed to the Codeception failed file.
143+
*
144+
* @return void
145+
*/
146+
private function writeFailedFile()
147+
{
148+
foreach ($this->failedTests as $test) {
149+
file_put_contents(RunTestFailedCommand::TESTS_FAILED_FILE, $test . "\n", FILE_APPEND);
150+
}
151+
}
152+
}

0 commit comments

Comments
 (0)