From db50f2089a788399173bc2b1d9d22c2ee31a9cd4 Mon Sep 17 00:00:00 2001 From: Kevin Kozan Date: Mon, 1 Nov 2021 09:36:22 -0500 Subject: [PATCH 1/3] MQE-2974 Created covtodb script --- composer.json | 3 +- composer.lock | 77 ++++++++++++++++++++++----- covToDB.php | 144 ++++++++++++++++++++++++++++++++++++++++++++++++++ db.class.php | 6 +++ 4 files changed, 215 insertions(+), 15 deletions(-) create mode 100644 covToDB.php create mode 100644 db.class.php diff --git a/composer.json b/composer.json index 471ea68..79005ef 100644 --- a/composer.json +++ b/composer.json @@ -1,6 +1,7 @@ { "name": "magento/code-coverage-diff-tool", "require": { - "phpunit/phpcov": "8.2.0" + "phpunit/phpcov": "8.2.0", + "sergeytsalkov/meekrodb": "*" } } diff --git a/composer.lock b/composer.lock index e38fc70..d056a01 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "2597cf257beefaa49d92fb23637004c4", + "content-hash": "5b5cec149f881be9c84fd92ddf07e526", "packages": [ { "name": "doctrine/instantiator", @@ -355,16 +355,16 @@ }, { "name": "phpdocumentor/reflection-docblock", - "version": "5.2.2", + "version": "5.3.0", "source": { "type": "git", "url": "https://github.com/phpDocumentor/ReflectionDocBlock.git", - "reference": "069a785b2141f5bcf49f3e353548dc1cce6df556" + "reference": "622548b623e81ca6d78b721c5e029f4ce664f170" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpDocumentor/ReflectionDocBlock/zipball/069a785b2141f5bcf49f3e353548dc1cce6df556", - "reference": "069a785b2141f5bcf49f3e353548dc1cce6df556", + "url": "https://api.github.com/repos/phpDocumentor/ReflectionDocBlock/zipball/622548b623e81ca6d78b721c5e029f4ce664f170", + "reference": "622548b623e81ca6d78b721c5e029f4ce664f170", "shasum": "" }, "require": { @@ -375,7 +375,8 @@ "webmozart/assert": "^1.9.1" }, "require-dev": { - "mockery/mockery": "~1.3.2" + "mockery/mockery": "~1.3.2", + "psalm/phar": "^4.8" }, "type": "library", "extra": { @@ -405,22 +406,22 @@ "description": "With this component, a library can provide support for annotations via DocBlocks or otherwise retrieve information that is embedded in a DocBlock.", "support": { "issues": "https://github.com/phpDocumentor/ReflectionDocBlock/issues", - "source": "https://github.com/phpDocumentor/ReflectionDocBlock/tree/master" + "source": "https://github.com/phpDocumentor/ReflectionDocBlock/tree/5.3.0" }, - "time": "2020-09-03T19:13:55+00:00" + "time": "2021-10-19T17:43:47+00:00" }, { "name": "phpdocumentor/type-resolver", - "version": "1.5.0", + "version": "1.5.1", "source": { "type": "git", "url": "https://github.com/phpDocumentor/TypeResolver.git", - "reference": "30f38bffc6f24293dadd1823936372dfa9e86e2f" + "reference": "a12f7e301eb7258bb68acd89d4aefa05c2906cae" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpDocumentor/TypeResolver/zipball/30f38bffc6f24293dadd1823936372dfa9e86e2f", - "reference": "30f38bffc6f24293dadd1823936372dfa9e86e2f", + "url": "https://api.github.com/repos/phpDocumentor/TypeResolver/zipball/a12f7e301eb7258bb68acd89d4aefa05c2906cae", + "reference": "a12f7e301eb7258bb68acd89d4aefa05c2906cae", "shasum": "" }, "require": { @@ -455,9 +456,9 @@ "description": "A PSR-5 based resolver of Class names, Types and Structural Element Names", "support": { "issues": "https://github.com/phpDocumentor/TypeResolver/issues", - "source": "https://github.com/phpDocumentor/TypeResolver/tree/1.5.0" + "source": "https://github.com/phpDocumentor/TypeResolver/tree/1.5.1" }, - "time": "2021-09-17T15:28:14+00:00" + "time": "2021-10-02T14:08:47+00:00" }, { "name": "phpspec/prophecy", @@ -1973,6 +1974,54 @@ ], "time": "2020-09-28T06:39:44+00:00" }, + { + "name": "sergeytsalkov/meekrodb", + "version": "v2.4", + "source": { + "type": "git", + "url": "https://github.com/SergeyTsalkov/meekrodb.git", + "reference": "e30c240d54bc81f58c58507a9ed768032eb494a5" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/SergeyTsalkov/meekrodb/zipball/e30c240d54bc81f58c58507a9ed768032eb494a5", + "reference": "e30c240d54bc81f58c58507a9ed768032eb494a5", + "shasum": "" + }, + "require": { + "php": ">=5.3.0" + }, + "type": "library", + "autoload": { + "classmap": [ + "db.class.php" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "LGPL-3.0" + ], + "authors": [ + { + "name": "Sergey Tsalkov", + "email": "stsalkov@gmail.com" + } + ], + "description": "The Simple PHP/MySQL Library", + "homepage": "http://www.meekro.com", + "keywords": [ + "database", + "mysql", + "mysqli", + "pdo" + ], + "support": { + "email": "support@meekro.com", + "issues": "https://github.com/SergeyTsalkov/meekrodb/issues", + "source": "https://github.com/SergeyTsalkov/meekrodb/tree/v2.4" + }, + "time": "2021-06-29T04:31:23+00:00" + }, { "name": "symfony/polyfill-ctype", "version": "v1.23.0", diff --git a/covToDB.php b/covToDB.php new file mode 100644 index 0000000..8a4d718 --- /dev/null +++ b/covToDB.php @@ -0,0 +1,144 @@ + $runID + ]); + $cachedCCRUNID = DB::insertId(); + } else { + $cachedCCRUNID = $run[0]['ccrunid']; + } + unset($run); + + //CACHE LOCAL IDS FOR EASY USE + $cachedTests = []; + $tests = DB::query("SELECT * FROM CC_TESTS WHERE ccrunid=%s AND testtype=%s", $cachedCCRUNID, $testType); + foreach ($tests as $test) { + $cachedTests[$test['testname']] = $test['testid']; + } + unset($tests); + $cachedFiles = []; + $files = DB::query("SELECT * FROM CC_FILES WHERE ccrunid=%s", $cachedCCRUNID); + foreach ($files as $file) { + $cachedFiles[$file['filepath']] = $file['fileid']; + } + unset($files); + + //ITERATE THROUGH FILES + foreach (scandir($coveragePath) as $file) { + printf("Reading ($currentFile/$fileCount)\r"); + $currentFile += 1; + if (pathinfo($file)['extension'] !== 'cov') { + continue; + } + $fileCoverage = readCoverage($coveragePath . DIRECTORY_SEPARATOR . $file); + //TEST NAME INSERT + foreach ($fileCoverage->getTests() as $testname => $content) { + if (isset($cachedTests[$testname])) { + continue; + } +// $existingTest = DB::query( +// "SELECT * FROM CC_TESTS WHERE ccrunid=%s AND testtype=%s AND testname=%s", +// $cachedCCRUNID, $testType, $testname +// ); +// if (!empty($existingTest)) { +// continue; +// } + DB::insert("CC_TESTS", [ + "testtype" => $testType, + "testname" => $testname, + "ccrunid" => $cachedCCRUNID + ]); + $cachedTests[$testname] = DB::insertId(); + } + + //DO FILE + LINE INSERTS + foreach ($fileCoverage->getData(true)->lineCoverage() as $testFile => $content) { + if (!isset($cachedFiles[$testFile])) { + DB::insert("CC_FILES", [ + "filepath" => $testFile, + "ccrunid" => $cachedCCRUNID + ]); + $cachedFiles[$testFile] = DB::insertId(); + } +// $existingFile = DB::query( +// "SELECT * FROM CC_FILES WHERE ccrunid=%s AND filepath=%s", +// $cachedCCRUNID, $testFile +// ); +// if (empty($existingFile)) { +// DB::insert("CC_FILES", [ +// "filepath" => $testFile, +// "ccrunid" => $cachedCCRUNID +// ]); +// $cachedFiles[$testFile] = DB::insertId(); +// } + foreach ($content as $lineNumber => $tests) { + foreach ($tests as $test) { + $existingLine = DB::query( + "SELECT * FROM CC_LINES WHERE linenumber=%s AND testid=%s AND fileid=%s AND ccrunid=%s", + $lineNumber, $cachedTests[$test], $cachedFiles[$testFile], $cachedCCRUNID + ); + if (empty($existingLine)) { + DB::insert("CC_LINES", [ + "linenumber" => $lineNumber, + "testid" => $cachedTests[$test], + "fileid" => $cachedFiles[$testFile], + "ccrunid" => $cachedCCRUNID + ]); + } + } + } + } + printf("Inserted from file {$file}\n"); + } + printf("Inserted from all files in {$coveragePath}\n"); +} + +/** + * Reads a single .cov file into memory. + */ +function readCoverage($coveragePath) { + if (!is_file($coveragePath)) { + return null; + } + $file = include($coveragePath); + return $file; +} + +if (!array_key_exists(3, $argv)) { + printf("This script requires 3 parameters to run:\nInputDirectory RunID TestType"); +} else { + main(filter_var($argv[1], FILTER_SANITIZE_STRING),filter_var($argv[2], FILTER_SANITIZE_STRING),filter_var($argv[3], FILTER_SANITIZE_STRING)); +} diff --git a/db.class.php b/db.class.php new file mode 100644 index 0000000..ac665b4 --- /dev/null +++ b/db.class.php @@ -0,0 +1,6 @@ + Date: Tue, 2 Nov 2021 08:55:30 -0500 Subject: [PATCH 2/3] full local cache version --- covToDB.php | 55 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 23 deletions(-) diff --git a/covToDB.php b/covToDB.php index 8a4d718..ec7fa1f 100644 --- a/covToDB.php +++ b/covToDB.php @@ -30,6 +30,7 @@ function readCoverageFromFolder($coveragePath, $runID, $testType) { printf("Reading coverage...\n"); require_once __DIR__ . DIRECTORY_SEPARATOR . 'db.class.php'; + $start_time = microtime(TRUE); //INPUT RUN IF NOT PRESENT $run = DB::query("SELECT * FROM CC_RUN WHERE runid=%s", $runID); if (empty($run)) { @@ -64,18 +65,23 @@ function readCoverageFromFolder($coveragePath, $runID, $testType) { continue; } $fileCoverage = readCoverage($coveragePath . DIRECTORY_SEPARATOR . $file); + //TEST NAME INSERT foreach ($fileCoverage->getTests() as $testname => $content) { + //found in initial cache if (isset($cachedTests[$testname])) { continue; } -// $existingTest = DB::query( -// "SELECT * FROM CC_TESTS WHERE ccrunid=%s AND testtype=%s AND testname=%s", -// $cachedCCRUNID, $testType, $testname -// ); -// if (!empty($existingTest)) { -// continue; -// } + $existingTest = DB::query( + "SELECT * FROM CC_TESTS WHERE ccrunid=%s AND testtype=%s AND testname=%s", + $cachedCCRUNID, $testType, $testname + ); + //not found in initial cache but found in DB + if (!empty($existingTest)) { + $cachedTests[$existingTest[0]['testname']] = $existingTest[0]['testid']; + continue; + } + //actual new record DB::insert("CC_TESTS", [ "testtype" => $testType, "testname" => $testname, @@ -86,24 +92,24 @@ function readCoverageFromFolder($coveragePath, $runID, $testType) { //DO FILE + LINE INSERTS foreach ($fileCoverage->getData(true)->lineCoverage() as $testFile => $content) { + // not found in initial cache if (!isset($cachedFiles[$testFile])) { - DB::insert("CC_FILES", [ - "filepath" => $testFile, - "ccrunid" => $cachedCCRUNID - ]); - $cachedFiles[$testFile] = DB::insertId(); + $existingFile = DB::query( + "SELECT * FROM CC_FILES WHERE ccrunid=%s AND filepath=%s", + $cachedCCRUNID, $testFile + ); + // found in DB + if (!empty($existingFile)) { + $cachedFiles[$existingFile[0]['filepath']] = $existingFile[0]['fileid']; + } else { + // actual new record + DB::insert("CC_FILES", [ + "filepath" => $testFile, + "ccrunid" => $cachedCCRUNID + ]); + $cachedFiles[$testFile] = DB::insertId(); + } } -// $existingFile = DB::query( -// "SELECT * FROM CC_FILES WHERE ccrunid=%s AND filepath=%s", -// $cachedCCRUNID, $testFile -// ); -// if (empty($existingFile)) { -// DB::insert("CC_FILES", [ -// "filepath" => $testFile, -// "ccrunid" => $cachedCCRUNID -// ]); -// $cachedFiles[$testFile] = DB::insertId(); -// } foreach ($content as $lineNumber => $tests) { foreach ($tests as $test) { $existingLine = DB::query( @@ -123,7 +129,10 @@ function readCoverageFromFolder($coveragePath, $runID, $testType) { } printf("Inserted from file {$file}\n"); } + $endtime = microtime(TRUE); + $runtime = $endtime-$start_time; printf("Inserted from all files in {$coveragePath}\n"); + printf("Execution took {$runtime} seconds.\n"); } /** From 1b01959427d109193fedb78e75a6f84d5a48775b Mon Sep 17 00:00:00 2001 From: Kevin Kozan Date: Tue, 9 Nov 2021 11:16:38 -0600 Subject: [PATCH 3/3] parallelized execution update --- covToDB-Directory.php | 62 +++++++++++++++++++ covToDB-Single.php | 139 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 201 insertions(+) create mode 100644 covToDB-Directory.php create mode 100644 covToDB-Single.php diff --git a/covToDB-Directory.php b/covToDB-Directory.php new file mode 100644 index 0000000..3cc4a10 --- /dev/null +++ b/covToDB-Directory.php @@ -0,0 +1,62 @@ + $runID + ]); + $cachedCCRUNID = DB::insertId(); + } else { + $cachedCCRUNID = $run[0]['ccrunid']; + } + unset($run); + + //CACHE LOCAL IDS FOR EASY USE + $cachedTests = []; + $tests = DB::query("SELECT * FROM CC_TESTS WHERE ccrunid=%s AND testtype=%s", $cachedCCRUNID, $testType); + foreach ($tests as $test) { + $cachedTests[$test['testname']] = $test['testid']; + } + unset($tests); + $cachedFiles = []; + $files = DB::query("SELECT * FROM CC_FILES WHERE ccrunid=%s", $cachedCCRUNID); + foreach ($files as $file) { + $cachedFiles[$file['filepath']] = $file['fileid']; + } + unset($files); + + //ITERATE THROUGH FILES + printf("Reading {$filepath}\r"); + $fileCoverage = readCoverage($filepath); + + //TEST NAME INSERT + foreach ($fileCoverage->getTests() as $testname => $content) { + //found in initial cache + if (isset($cachedTests[$testname])) { + continue; + } + $existingTest = DB::query( + "SELECT * FROM CC_TESTS WHERE ccrunid=%s AND testtype=%s AND testname=%s", + $cachedCCRUNID, $testType, $testname + ); + //not found in initial cache but found in DB + if (!empty($existingTest)) { + $cachedTests[$existingTest[0]['testname']] = $existingTest[0]['testid']; + continue; + } + //actual new record + DB::insert("CC_TESTS", [ + "testtype" => $testType, + "testname" => $testname, + "ccrunid" => $cachedCCRUNID + ]); + $cachedTests[$testname] = DB::insertId(); + } + + //DO FILE + LINE INSERTS + foreach ($fileCoverage->getData(true)->lineCoverage() as $testFile => $content) { + // not found in initial cache + if (!isset($cachedFiles[$testFile])) { + $existingFile = DB::query( + "SELECT * FROM CC_FILES WHERE ccrunid=%s AND filepath=%s", + $cachedCCRUNID, $testFile + ); + // found in DB + if (!empty($existingFile)) { + $cachedFiles[$existingFile[0]['filepath']] = $existingFile[0]['fileid']; + } else { + // actual new record + DB::insert("CC_FILES", [ + "filepath" => $testFile, + "ccrunid" => $cachedCCRUNID + ]); + $cachedFiles[$testFile] = DB::insertId(); + } + } + foreach ($content as $lineNumber => $tests) { + foreach ($tests as $test) { + $existingLine = DB::query( + "SELECT * FROM CC_LINES WHERE linenumber=%s AND testid=%s AND fileid=%s AND ccrunid=%s", + $lineNumber, $cachedTests[$test], $cachedFiles[$testFile], $cachedCCRUNID + ); + if (empty($existingLine)) { + DB::insert("CC_LINES", [ + "linenumber" => $lineNumber, + "testid" => $cachedTests[$test], + "fileid" => $cachedFiles[$testFile], + "ccrunid" => $cachedCCRUNID + ]); + } + } + } + } + printf("Inserted from file {$filepath}\n"); + + $endtime = microtime(TRUE); + $runtime = $endtime-$start_time; + printf("Execution took {$runtime} seconds.\n"); +} + +/** + * Reads a single .cov file into memory. + */ +function readCoverage($coveragePath) { + if (!is_file($coveragePath)) { + return null; + } + $file = include($coveragePath); + return $file; +} + +if (!array_key_exists(3, $argv)) { + printf("This script requires 3 parameters to run:\nFilePath RunID TestType"); +} else { + main(filter_var($argv[1], FILTER_SANITIZE_STRING),filter_var($argv[2], FILTER_SANITIZE_STRING),filter_var($argv[3], FILTER_SANITIZE_STRING)); +}