MQE-3267: trim invalid utf-8 characters from response

Author: jilu1

composer.json

@@ -15,6 +15,7 @@
         "ext-intl": "*",
         "ext-json": "*",
         "ext-openssl": "*",
+        "ext-iconv": "*",
         "allure-framework/allure-codeception": "^1.4",
         "aws/aws-sdk-php": "^3.132",
         "codeception/codeception": "^4.1",

composer.lock

@@ -584,7 +584,8 @@ public function magentoCLI($command, $timeout = null, $arguments = null)
         $response = $executor->read();
         $executor->close();
 
-        return $response;
+        $response = trim($this->safeUtf8Conversion($response));
+        return $response != "" ? $response : "CLI did not return output.";
     }
 
     /**
@@ -1059,4 +1060,29 @@ public function pause($pauseOnFail = false)
 
         $this->codeceptPause();
     }
+
+    /**
+     * Return UTF-8 encoding string with control/invisible characters removed, return original string on error.
+     *
+     * @param string $input
+     * @return string
+     */
+    private function safeUtf8Conversion(string $input): string
+    {
+        // Convert $input string to UTF-8 encoding
+        $convInput = iconv("ISO-8859-1", "UTF-8//IGNORE", $input);
+        if ($convInput !== false) {
+            // Remove invisible control characters, unused code points and replacement character
+            // so that they don't break xml test results for Allure
+            $cleanInput = preg_replace('/[^\PC\s]|\x{FFFD}/u', '', $convInput);
+            if ($cleanInput !== null) {
+                return $cleanInput;
+            } else {
+                $err = preg_last_error_msg();
+                print("MagentoCLI response preg_replace() with error $err.\n");
+            }
+        }
+
+        return $input;
+    }
 }