Merge pull request #4314 from boegel/run_systemtools

implement `fail_on_error`/`in_dry_run`/`output_file` options + enable caching for `run` function, and switch from `run_cmd` to `run` function in systemtools

Author: ocaisa

easybuild/tools/run.py

@@ -77,6 +77,36 @@
 RunResult = namedtuple('RunResult', ('output', 'exit_code', 'stderr'))
 
 
+def run_cmd_cache(func):
+    """Function decorator to cache (and retrieve cached) results of running commands."""
+    cache = {}
+
+    @functools.wraps(func)
+    def cache_aware_func(cmd, *args, **kwargs):
+        """Retrieve cached result of selected commands, or run specified and collect & cache result."""
+        # cache key is combination of command and input provided via stdin ('stdin' for run, 'inp' for run_cmd)
+        key = (cmd, kwargs.get('stdin', None) or kwargs.get('inp', None))
+        # fetch from cache if available, cache it if it's not, but only on cmd strings
+        if isinstance(cmd, str) and key in cache:
+            _log.debug("Using cached value for command '%s': %s", cmd, cache[key])
+            return cache[key]
+        else:
+            res = func(cmd, *args, **kwargs)
+            if cmd in CACHED_COMMANDS:
+                cache[key] = res
+            return res
+
+    # expose clear/update methods of cache to wrapped function
+    cache_aware_func.clear_cache = cache.clear
+    cache_aware_func.update_cache = cache.update
+
+    return cache_aware_func
+
+
+run_cache = run_cmd_cache
+
+
+@run_cache
 def run(cmd, fail_on_error=True, split_stderr=False, stdin=None,
         hidden=False, in_dry_run=False, work_dir=None, shell=True,
         output_file=False, stream_output=False, asynchronous=False,
@@ -90,7 +120,7 @@ def run(cmd, fail_on_error=True, split_stderr=False, stdin=None,
     :param hidden: do not show command in terminal output (when using --trace, or with --extended-dry-run / -x)
     :param in_dry_run: also run command in dry run mode
     :param work_dir: working directory to run command in (current working directory if None)
-    :param shell: execute command through a shell (enabled by default)
+    :param shell: execute command through bash shell (enabled by default)
     :param output_file: collect command output in temporary output file
     :param stream_output: stream command output to stdout
     :param asynchronous: run command asynchronously
@@ -104,7 +134,7 @@ def run(cmd, fail_on_error=True, split_stderr=False, stdin=None,
     """
 
     # temporarily raise a NotImplementedError until all options are implemented
-    if any((not fail_on_error, split_stderr, in_dry_run, work_dir, output_file, stream_output, asynchronous)):
+    if any((split_stderr, work_dir, stream_output, asynchronous)):
         raise NotImplementedError
 
     if qa_patterns or qa_wait_patterns:
@@ -117,19 +147,24 @@ def run(cmd, fail_on_error=True, split_stderr=False, stdin=None,
     else:
         raise EasyBuildError(f"Unknown command type ('{type(cmd)}'): {cmd}")
 
-    silent = build_option('silent')
-
     if work_dir is None:
         work_dir = os.getcwd()
 
-    # output file for command output (only used if output_file is enabled)
-    cmd_out_fp = None
+    # temporary output file for command output, if requested
+    if output_file or not hidden:
+        # collect output of running command in temporary log file, if desired
+        fd, cmd_out_fp = tempfile.mkstemp(suffix='.log', prefix='easybuild-run-')
+        os.close(fd)
+        _log.info(f'run_cmd: Output of "{cmd}" will be logged to {cmd_out_fp}')
+    else:
+        cmd_out_fp = None
 
     # early exit in 'dry run' mode, after printing the command that would be run (unless 'hidden' is enabled)
-    if build_option('extended_dry_run'):
+    if not in_dry_run and build_option('extended_dry_run'):
         if not hidden:
-            msg = f"  running command \"%{cmd_msg}s\"\n"
-            msg += f"  (in %{work_dir})"
+            silent = build_option('silent')
+            msg = f"  running command \"{cmd_msg}s\"\n"
+            msg += f"  (in {work_dir})"
             dry_run_msg(msg, silent=silent)
 
         return RunResult(output='', exit_code=0, stderr=None)
@@ -142,14 +177,23 @@ def run(cmd, fail_on_error=True, split_stderr=False, stdin=None,
         # 'input' value fed to subprocess.run must be a byte sequence
         stdin = stdin.encode()
 
+    # use bash as shell instead of the default /bin/sh used by subprocess.run
+    # (which could be dash instead of bash, like on Ubuntu, see https://wiki.ubuntu.com/DashAsBinSh)
+    if shell:
+        executable = '/bin/bash'
+    else:
+        # stick to None (default value) when not running command via a shell
+        executable = None
+
     _log.info(f"Running command '{cmd_msg}' in {work_dir}")
-    proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, input=stdin, shell=shell)
+    proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=fail_on_error,
+                          input=stdin, shell=shell, executable=executable)
 
     # return output as a regular string rather than a byte sequence (and non-UTF-8 characters get stripped out)
     output = proc.stdout.decode('utf-8', 'ignore')
 
     res = RunResult(output=output, exit_code=proc.returncode, stderr=None)
-    _log.info(f"Command '{cmd_msg}' exited with exit code {res.exit_code} and output:\n%{res.output}")
+    _log.info(f"Command '{cmd_msg}' exited with exit code {res.exit_code} and output:\n{res.output}")
 
     if not hidden:
         time_since_start = time_str_since(start_time)
@@ -185,32 +229,6 @@ def cmd_trace_msg(cmd, start_time, work_dir, stdin, cmd_out_fp):
     trace_msg('\n'.join(lines))
 
 
-def run_cmd_cache(func):
-    """Function decorator to cache (and retrieve cached) results of running commands."""
-    cache = {}
-
-    @functools.wraps(func)
-    def cache_aware_func(cmd, *args, **kwargs):
-        """Retrieve cached result of selected commands, or run specified and collect & cache result."""
-        # cache key is combination of command and input provided via stdin
-        key = (cmd, kwargs.get('inp', None))
-        # fetch from cache if available, cache it if it's not, but only on cmd strings
-        if isinstance(cmd, str) and key in cache:
-            _log.debug("Using cached value for command '%s': %s", cmd, cache[key])
-            return cache[key]
-        else:
-            res = func(cmd, *args, **kwargs)
-            if cmd in CACHED_COMMANDS:
-                cache[key] = res
-            return res
-
-    # expose clear/update methods of cache to wrapped function
-    cache_aware_func.clear_cache = cache.clear
-    cache_aware_func.update_cache = cache.update
-
-    return cache_aware_func
-
-
 def get_output_from_process(proc, read_size=None, asynchronous=False):
     """
     Get output from running process (that was opened with subprocess.Popen).