Merge pull request easybuilders#4468 from boegel/from_commit

add support for `--from-commit` and `--include-easyblocks-from-commit`

Author: branfosj

easybuild/framework/easyconfig/tools.py

@@ -60,8 +60,9 @@
 from easybuild.tools.filetools import find_easyconfigs, is_patch_file, locate_files
 from easybuild.tools.filetools import read_file, resolve_path, which, write_file
 from easybuild.tools.github import GITHUB_EASYCONFIGS_REPO
-from easybuild.tools.github import det_pr_labels, det_pr_title, download_repo, fetch_easyconfigs_from_pr, fetch_pr_data
-from easybuild.tools.github import fetch_files_from_pr
+from easybuild.tools.github import det_pr_labels, det_pr_title, download_repo, fetch_easyconfigs_from_commit
+from easybuild.tools.github import fetch_easyconfigs_from_pr, fetch_pr_data
+from easybuild.tools.github import fetch_files_from_commit, fetch_files_from_pr
 from easybuild.tools.multidiff import multidiff
 from easybuild.tools.py2vs3 import OrderedDict
 from easybuild.tools.toolchain.toolchain import is_system_toolchain
@@ -310,7 +311,7 @@ def get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR, robot_path=None):
     return paths
 
 
-def alt_easyconfig_paths(tmpdir, tweaked_ecs=False, from_prs=None, review_pr=None):
+def alt_easyconfig_paths(tmpdir, tweaked_ecs=False, from_prs=None, from_commit=None, review_pr=None):
     """Obtain alternative paths for easyconfig files."""
 
     # paths where tweaked easyconfigs will be placed, easyconfigs listed on the command line take priority and will be
@@ -321,18 +322,20 @@ def alt_easyconfig_paths(tmpdir, tweaked_ecs=False, from_prs=None, review_pr=Non
         tweaked_ecs_paths = (os.path.join(tmpdir, 'tweaked_easyconfigs'),
                              os.path.join(tmpdir, 'tweaked_dep_easyconfigs'))
 
-    # paths where files touched in PRs will be downloaded to,
-    # which are picked up via 'pr_paths' build option in fetch_files_from_pr
-    pr_paths = []
+    # paths where files touched in commit/PRs will be downloaded to,
+    # which are picked up via 'extra_ec_paths' build option in fetch_files_from_pr
+    extra_ec_paths = []
     if from_prs:
-        pr_paths = from_prs[:]
-    if review_pr and review_pr not in pr_paths:
-        pr_paths.append(review_pr)
+        extra_ec_paths = from_prs[:]
+    if review_pr and review_pr not in extra_ec_paths:
+        extra_ec_paths.append(review_pr)
+    if extra_ec_paths:
+        extra_ec_paths = [os.path.join(tmpdir, 'files_pr%s' % pr) for pr in extra_ec_paths]
 
-    if pr_paths:
-        pr_paths = [os.path.join(tmpdir, 'files_pr%s' % pr) for pr in pr_paths]
+    if from_commit:
+        extra_ec_paths.append(os.path.join(tmpdir, 'files_commit_' + from_commit))
 
-    return tweaked_ecs_paths, pr_paths
+    return tweaked_ecs_paths, extra_ec_paths
 
 
 def det_easyconfig_paths(orig_paths):
@@ -346,27 +349,31 @@ def det_easyconfig_paths(orig_paths):
     except ValueError:
         raise EasyBuildError("Argument to --from-pr must be a comma separated list of PR #s.")
 
+    from_commit = build_option('from_commit')
     robot_path = build_option('robot_path')
 
     # list of specified easyconfig files
     ec_files = orig_paths[:]
 
+    commit_files, pr_files = [], []
     if from_prs:
-        pr_files = []
         for pr in from_prs:
-            # path to where easyconfig files should be downloaded is determined via 'pr_paths' build option,
-            # which corresponds to the list of PR paths returned by alt_easyconfig_paths
+            # path to where easyconfig files should be downloaded is determined
+            # via 'extra_ec_paths' build options,
+            # which corresponds to the list of commit/PR paths returned by alt_easyconfig_paths
             pr_files.extend(fetch_easyconfigs_from_pr(pr))
-
-        if ec_files:
-            # replace paths for specified easyconfigs that are touched in PR
-            for i, ec_file in enumerate(ec_files):
-                for pr_file in pr_files:
-                    if ec_file == os.path.basename(pr_file):
-                        ec_files[i] = pr_file
-        else:
-            # if no easyconfigs are specified, use all the ones touched in the PR
-            ec_files = [path for path in pr_files if path.endswith('.eb')]
+    elif from_commit:
+        commit_files = fetch_easyconfigs_from_commit(from_commit, files=ec_files)
+
+    if ec_files:
+        # replace paths for specified easyconfigs that are touched in commit/PRs
+        for i, ec_file in enumerate(ec_files):
+            for file in commit_files + pr_files:
+                if ec_file == os.path.basename(file):
+                    ec_files[i] = file
+    else:
+        # if no easyconfigs are specified, use all the ones touched in the commit/PRs
+        ec_files = [path for path in commit_files + pr_files if path.endswith('.eb')]
 
     filter_ecs = build_option('filter_ecs')
     if filter_ecs:
@@ -784,7 +791,7 @@ def avail_easyblocks():
     return easyblocks
 
 
-def det_copy_ec_specs(orig_paths, from_pr):
+def det_copy_ec_specs(orig_paths, from_pr=None, from_commit=None):
     """Determine list of paths + target directory for --copy-ec."""
 
     if from_pr is not None and not isinstance(from_pr, list):
@@ -848,4 +855,41 @@ def det_copy_ec_specs(orig_paths, from_pr):
             elif pr_matches:
                 raise EasyBuildError("Found multiple paths for %s in PR: %s", filename, pr_matches)
 
+    # consider --from-commit (only if --from-pr was not used)
+    elif from_commit:
+        tmpdir = os.path.join(tempfile.gettempdir(), 'fetch_files_from_commit_%s' % from_commit)
+        commit_paths = fetch_files_from_commit(from_commit, path=tmpdir)
+
+        # assume that files need to be copied to current working directory for now
+        target_path = os.getcwd()
+
+        if orig_paths:
+            last_path = orig_paths[-1]
+
+            # check files touched by commit and see if the target directory for --copy-ec
+            # corresponds to the name of one of these files;
+            # if so we should copy the specified file(s) to the current working directory,
+            # since interpreting the last argument as target location is very unlikely to be correct in this case
+            commit_filenames = [os.path.basename(p) for p in commit_paths]
+            if last_path in commit_filenames:
+                paths = orig_paths[:]
+            else:
+                target_path = last_path
+                # exclude last argument that is used as target location
+                paths = orig_paths[:-1]
+
+        # if list of files to copy is empty at this point,
+        # we simply copy *all* files touched by the PR
+        if not paths:
+            paths = commit_paths
+
+        # replace path for files touched by commit (no need to worry about others)
+        for idx, path in enumerate(paths):
+            filename = os.path.basename(path)
+            commit_matches = [x for x in commit_paths if os.path.basename(x) == filename]
+            if len(commit_matches) == 1:
+                paths[idx] = commit_matches[0]
+            elif commit_matches:
+                raise EasyBuildError("Found multiple paths for %s in commit: %s", filename, commit_matches)
+
     return paths, target_path

easybuild/main.py

@@ -329,7 +329,7 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session
 
     if options.copy_ec:
         # figure out list of files to copy + target location (taking into account --from-pr)
-        eb_args, target_path = det_copy_ec_specs(eb_args, from_pr_list)
+        eb_args, target_path = det_copy_ec_specs(eb_args, from_pr=from_pr_list, from_commit=options.from_commit)
 
     categorized_paths = categorize_files_by_type(eb_args)
 
@@ -765,8 +765,14 @@ def prepare_main(args=None, logfile=None, testing=None):
 
 
 if __name__ == "__main__":
-    init_session_state, eb_go, cfg_settings = prepare_main()
+    # take into account that EasyBuildError may be raised when parsing the EasyBuild configuration
+    try:
+        init_session_state, eb_go, cfg_settings = prepare_main()
+    except EasyBuildError as err:
+        print_error(err.msg)
+
     hooks = load_hooks(eb_go.options.hooks)
+
     try:
         main(prepared_cfg_data=(init_session_state, eb_go, cfg_settings))
     except EasyBuildError as err:

easybuild/tools/config.py

@@ -220,6 +220,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
         'filter_env_vars',
         'filter_rpath_sanity_libs',
         'force_download',
+        'from_commit',
         'git_working_dirs_path',
         'github_user',
         'github_org',
@@ -229,6 +230,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
         'http_header_fields_urlpat',
         'hooks',
         'ignore_dirs',
+        'include_easyblocks_from_commit',
         'insecure_download',
         'job_backend_config',
         'job_cores',
@@ -403,7 +405,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
         'build_specs',
         'command_line',
         'external_modules_metadata',
-        'pr_paths',
+        'extra_ec_paths',
         'robot_path',
         'valid_module_classes',
         'valid_stops',