Merge pull request #74 from magento/MQE-778

MQE-778: robo generate:tests generates all tests and suites by default

Author: imeron2433

dev/tests/verification/Tests/SuiteGenerationTest.php

@@ -17,13 +17,6 @@ class SuiteGenerationTest extends MftfTestCase
     const RESOURCES_DIR = TESTS_BP . DIRECTORY_SEPARATOR . 'verification' . DIRECTORY_SEPARATOR . 'Resources';
     const CONFIG_YML_FILE = FW_BP . DIRECTORY_SEPARATOR . SuiteGenerator::YAML_CODECEPTION_CONFIG_FILENAME;
 
-    const MANIFEST_RESULTS = [
-        'IncludeTest2Cest.php',
-        'additionalTestCest.php',
-        'IncludeTestCest.php',
-        'additionalIncludeTest2Cest.php'
-    ];
-
     /**
      * Flag to track existence of config.yml file
      *
@@ -48,6 +41,11 @@ public static function setUpBeforeClass()
             return;
         }
 
+        // destroy manifest file if it exists
+        if (file_exists(self::getManifestFilePath())) {
+            unlink(self::getManifestFilePath());
+        }
+
         $configYml = fopen(self::CONFIG_YML_FILE, "w");
         fclose($configYml);
     }
@@ -59,6 +57,13 @@ public function testSuiteGeneration1()
     {
          $groupName = 'functionalSuite1';
 
+        $expectedContents = [
+           'additionalTestCest.php',
+           'additionalIncludeTest2Cest.php',
+           'IncludeTest2Cest.php',
+           'IncludeTestCest.php'
+        ];
+
         // Generate the Suite
         SuiteGenerator::getInstance()->generateSuite($groupName);
 
@@ -79,23 +84,10 @@ public function testSuiteGeneration1()
             $groupName .
             DIRECTORY_SEPARATOR;
 
-        // Validate test manifest contents
-        $actualManifest = dirname($suiteResultBaseDir). DIRECTORY_SEPARATOR . 'testManifest.txt';
-        $actualTestReferences = explode(PHP_EOL, file_get_contents($actualManifest));
-
-        for ($i = 0; $i < count($actualTestReferences); $i++) {
-            if (empty($actualTestReferences[$i])) {
-                continue;
-            }
-
-            $this->assertStringEndsWith(self::MANIFEST_RESULTS[$i], $actualTestReferences[$i]);
-            $this->assertNotFalse(strpos($actualTestReferences[$i], $groupName));
-        }
-
-        // Validate expected php files exist
-        foreach (self::MANIFEST_RESULTS as $expectedTestReference) {
-            $cestName = explode(":", $expectedTestReference, 2);
-            $this->assertFileExists($suiteResultBaseDir . $cestName[0]);
+        // Validate tests have been generated
+        $dirContents = array_diff(scandir($suiteResultBaseDir), ['..', '.']);
+        foreach ($expectedContents as $expectedFile) {
+            $this->assertTrue(in_array($expectedFile, $dirContents));
         }
     }
 
@@ -117,4 +109,20 @@ public static function tearDownAfterClass()
 
         unlink(self::CONFIG_YML_FILE);
     }
+
+    /**
+     * Getter for manifest file path
+     *
+     * @return string
+     */
+    private static function getManifestFilePath()
+    {
+        return TESTS_BP .
+            DIRECTORY_SEPARATOR .
+            "verification" .
+            DIRECTORY_SEPARATOR .
+            "_generated" .
+            DIRECTORY_SEPARATOR .
+            'testManifest.txt';
+    }
 }

src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php

@@ -32,6 +32,13 @@ class SuiteGenerator
      */
     private static $SUITE_GENERATOR_INSTANCE;
 
+    /**
+     * Boolean to track whether we have already cleared the yaml file.
+     *
+     * @var bool
+     */
+    private $ymlFileCleared = false;
+
     /**
      * Group Class Generator initialized in constructor.
      *
@@ -61,14 +68,35 @@ public static function getInstance()
         return self::$SUITE_GENERATOR_INSTANCE;
     }
 
+    /**
+     * Function which takes all suite configurations and generates to appropriate directory, updating yml configuration
+     * as needed. Returns an array of all tests generated keyed by test name.
+     *
+     * @param string $config
+     * @return array
+     */
+    public function generateAllSuites($config)
+    {
+        $testsReferencedInSuites = [];
+        $suites = SuiteObjectHandler::getInstance()->getAllObjects();
+        foreach ($suites as $suite) {
+            /** @var SuiteObject $suite */
+            $testsReferencedInSuites = array_merge($testsReferencedInSuites, $suite->getTests());
+            $this->generateSuite($suite->getName(), $config);
+        }
+
+        return $testsReferencedInSuites;
+    }
+
     /**
      * Function which takes a suite name and generates corresponding dir, test files, group class, and updates
      * yml configuration for group run.
      *
      * @param string $suiteName
+     * @param string $config
      * @return void
      */
-    public function generateSuite($suiteName)
+    public function generateSuite($suiteName, $config = null)
     {
         /**@var SuiteObject $suite **/
         $suite = SuiteObjectHandler::getInstance()->getObject($suiteName);
@@ -77,7 +105,7 @@ public function generateSuite($suiteName)
         $groupNamespace = null;
 
         DirSetupUtil::createGroupDir($fullPath);
-        $this->generateRelevantGroupTests($suiteName, $suite->getTests());
+        $this->generateRelevantGroupTests($suiteName, $suite->getTests(), $config);
 
         if ($suite->requiresGroupFile()) {
             // if the suite requires a group file, generate it and set the namespace
@@ -117,29 +145,69 @@ private function appendEntriesToConfig($suiteName, $suitePath, $groupNamespace)
             $ymlArray[self::YAML_GROUPS_TAG]= [];
         }
 
-        $ymlArray[self::YAML_GROUPS_TAG][$suiteName] = [$relativeSuitePath];
+        $ymlArray = $this->clearPreviousSessionConfigEntries($ymlArray);
 
-        if ($groupNamespace &&
-            !in_array($groupNamespace, $ymlArray[self::YAML_EXTENSIONS_TAG][self::YAML_ENABLED_TAG])) {
+        if ($groupNamespace) {
             $ymlArray[self::YAML_EXTENSIONS_TAG][self::YAML_ENABLED_TAG][] = $groupNamespace;
         }
+        $ymlArray[self::YAML_GROUPS_TAG][$suiteName] = [$relativeSuitePath];
 
         $ymlText = self::YAML_COPYRIGHT_TEXT . Yaml::dump($ymlArray, 10);
         file_put_contents($configYmlFile, $ymlText);
     }
 
+    /**
+     * Function which takes the current config.yml array and clears any previous configuration for suite group object
+     * files.
+     *
+     * @param array $ymlArray
+     * @return array
+     */
+    private function clearPreviousSessionConfigEntries($ymlArray)
+    {
+        if ($this->ymlFileCleared) {
+            return $ymlArray;
+        }
+
+        $newYmlArray = $ymlArray;
+        // if the yaml entries haven't already been cleared
+
+        if (array_key_exists(self::YAML_EXTENSIONS_TAG, $ymlArray)) {
+            foreach ($ymlArray[self::YAML_EXTENSIONS_TAG][self::YAML_ENABLED_TAG] as $key => $entry) {
+                if (preg_match('/(Group\\\\.*)/', $entry)) {
+                    unset($newYmlArray[self::YAML_EXTENSIONS_TAG][self::YAML_ENABLED_TAG][$key]);
+                }
+
+            }
+
+            // needed for proper yml file generation based on indices
+            $newYmlArray[self::YAML_EXTENSIONS_TAG][self::YAML_ENABLED_TAG] =
+                array_values($newYmlArray[self::YAML_EXTENSIONS_TAG][self::YAML_ENABLED_TAG]);
+
+        }
+
+        if (array_key_exists(self::YAML_GROUPS_TAG, $newYmlArray)) {
+            unset($newYmlArray[self::YAML_GROUPS_TAG]);
+        }
+
+        $this->ymlFileCleared = true;
+
+        return $newYmlArray;
+    }
+
     /**
      * Function which takes a string which is the desired output directory (under _generated) and an array of tests
      * relevant to the suite to be generated. The function takes this information and creates a new instance of the test
      * generator which is then called to create all the test files for the suite.
      *
      * @param string $path
      * @param array $tests
+     * @param string $config
      * @return void
      */
-    private function generateRelevantGroupTests($path, $tests)
+    private function generateRelevantGroupTests($path, $tests, $config)
     {
         $testGenerator = TestGenerator::getInstance($path, $tests);
-        $testGenerator->createAllTestFiles();
+        $testGenerator->createAllTestFiles($config);
     }
 }

src/Magento/FunctionalTestingFramework/Suite/Util/SuiteObjectExtractor.php

@@ -44,6 +44,12 @@ public function parseSuiteDataIntoObjects($parsedSuiteData)
     {
         $suiteObjects = [];
         $testHookObjectExtractor = new TestHookObjectExtractor();
+
+        // make sure there are suites defined before trying to parse as objects.
+        if (!array_key_exists(self::SUITE_ROOT_TAG, $parsedSuiteData)) {
+            return $suiteObjects;
+        }
+
         foreach ($parsedSuiteData[self::SUITE_ROOT_TAG] as $parsedSuite) {
             if (!is_array($parsedSuite)) {
                 // skip non array items parsed from suite (suite objects will always be arrays)

src/Magento/FunctionalTestingFramework/Util/Filesystem/DirSetupUtil.php

@@ -11,6 +11,13 @@
 
 class DirSetupUtil
 {
+    /**
+     * Array which will track any previously cleared directories, to prevent any unintended removal.
+     *
+     * @var array
+     */
+    private static $DIR_CONTEXT = [];
+
     /**
      * Method used to clean export dir if needed and create new empty export dir.
      *
@@ -19,11 +26,17 @@ class DirSetupUtil
      */
     public static function createGroupDir($fullPath)
     {
+        // make sure we haven't already cleaned up this directory at any point before deletion
+        if (in_array($fullPath, self::$DIR_CONTEXT)) {
+            return;
+        }
+
         if (file_exists($fullPath)) {
             self::rmDirRecursive($fullPath);
         }
 
         mkdir($fullPath, 0777, true);
+        self::$DIR_CONTEXT[] = $fullPath;
     }
 
     /**

src/Magento/FunctionalTestingFramework/Util/Manifest/DefaultTestManifest.php

@@ -7,6 +7,7 @@
 namespace Magento\FunctionalTestingFramework\Util\Manifest;
 
 use Magento\FunctionalTestingFramework\Test\Objects\TestObject;
+use Magento\FunctionalTestingFramework\Util\Filesystem\DirSetupUtil;
 
 class DefaultTestManifest extends BaseTestManifest
 {
@@ -19,6 +20,13 @@ class DefaultTestManifest extends BaseTestManifest
      */
     protected $manifestPath;
 
+    /**
+     * A static array to track which test manifests have been cleared to prevent overwriting during generation.
+     *
+     * @var array
+     */
+    private static $CLEARED_MANIFESTS = [];
+
     /**
      * An array containing all test names for output.
      *
@@ -34,8 +42,9 @@ class DefaultTestManifest extends BaseTestManifest
     public function __construct($manifestPath, $testPath)
     {
         $this->manifestPath = $manifestPath . DIRECTORY_SEPARATOR . 'testManifest.txt';
+        $this->cleanManifest($this->manifestPath);
         parent::__construct($testPath, self::DEFAULT_CONFIG);
-        $fileResource = fopen($this->manifestPath, 'w');
+        $fileResource = fopen($this->manifestPath, 'a');
         fclose($fileResource);
     }
 
@@ -67,4 +76,26 @@ public function generate($nodes = null)
 
         fclose($fileResource);
     }
+
+    /**
+     * Function which checks the path for an existing test manifest and clears if the file has not already been cleared
+     * during current runtime.
+     *
+     * @param string $path
+     * @return void
+     */
+    private function cleanManifest($path)
+    {
+        // if we have already cleared the file then simply return
+        if (in_array($path, self::$CLEARED_MANIFESTS)) {
+            return;
+        }
+
+        // if the file exists remove
+        if (file_exists($path)) {
+            unlink($path);
+        }
+
+        self::$CLEARED_MANIFESTS[] = $path;
+    }
 }

src/Magento/FunctionalTestingFramework/Util/Manifest/SingleRunTestManifest.php

@@ -19,7 +19,7 @@ public function __construct($manifestPath, $testPath)
     {
         parent::__construct($manifestPath, $testPath);
         $this->runTypeConfig = self::SINGLE_RUN_CONFIG;
-        $fileResource = fopen($this->manifestPath, 'w');
+        $fileResource = fopen($this->manifestPath, 'a');
         fclose($fileResource);
     }
 

src/Magento/FunctionalTestingFramework/Util/TestGenerator.php

@@ -135,11 +135,12 @@ private function createCestFile($testPhp, $filename)
      *
      * @param string $runConfig
      * @param int $nodes
+     * @param TestObject[] $testsToIgnore
      * @return void
      * @throws TestReferenceException
      * @throws \Exception
      */
-    public function createAllTestFiles($runConfig = null, $nodes = null)
+    public function createAllTestFiles($runConfig = null, $nodes = null, $testsToIgnore = [])
     {
         DirSetupUtil::createGroupDir($this->exportDirectory);
 
@@ -149,8 +150,8 @@ public function createAllTestFiles($runConfig = null, $nodes = null)
             $this->exportDirectory,
             $runConfig
         );
-        $testPhpArray = $this->assembleAllTestPhp($testManifest, $nodes);
 
+        $testPhpArray = $this->assembleAllTestPhp($testManifest, $nodes, $testsToIgnore);
         foreach ($testPhpArray as $testPhpFile) {
             $this->createCestFile($testPhpFile[1], $testPhpFile[0]);
         }
@@ -196,14 +197,16 @@ private function assembleTestPhp($testObject)
      *
      * @param BaseTestManifest $testManifest
      * @param int $nodes
+     * @param TestObject[] $testsToIgnore
      * @return array
      * @throws TestReferenceException
      * @throws \Exception
      */
-    private function assembleAllTestPhp($testManifest, $nodes)
+    private function assembleAllTestPhp($testManifest, $nodes, $testsToIgnore)
     {
         /** @var TestObject[] $testObjects */
         $testObjects = $this->loadAllTestObjects();
+        $testObjects = array_diff_key($testObjects, $testsToIgnore);
         $cestPhpArray = [];
 
         foreach ($testObjects as $test) {