Prefer $EBPYTHONPREFIXES over $PYTHONPATH

easybuild/tools/config.py

@@ -331,6 +331,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
         'modules_tool_version_check',
         'mpi_tests',
         'pre_create_installdir',
+        'replace_pythonpath',
         'show_progress_bar',
         'trace',
     ],

easybuild/tools/module_generator.py

@@ -42,6 +42,7 @@
 from contextlib import contextmanager
 from easybuild.tools import LooseVersion
 from textwrap import wrap
+from typing import List, Union, Sequence
 
 from easybuild.base import fancylogger
 from easybuild.tools.build_log import EasyBuildError, print_warning
@@ -205,63 +206,79 @@ def get_modules_path(self, fake=False, mod_path_suffix=None):
 
         return os.path.join(mod_path, mod_path_suffix)
 
-    def _filter_paths(self, key, paths):
+    def _filter_paths(self, key: str, paths: List[str]) -> List[str]:
         """Filter out paths already added to key and return the remaining ones"""
+        assert isinstance(paths, list)
+
         if self.added_paths_per_key is None:
             # For compatibility this is only a warning for now and we don't filter any paths
             print_warning('Module creation has not been started. Call start_module_creation first!')
             return paths
 
         added_paths = self.added_paths_per_key.setdefault(key, set())
-        # paths can be a string
-        if isinstance(paths, str):
-            if paths in added_paths:
-                filtered_paths = None
-            else:
-                added_paths.add(paths)
-                filtered_paths = paths
-        else:
-            # Coerce any iterable/generator into a list
-            if not isinstance(paths, list):
-                paths = list(paths)
-            filtered_paths = [x for x in paths if x not in added_paths and not added_paths.add(x)]
+        filtered_paths = [x for x in paths if x not in added_paths and not added_paths.add(x)]
         if filtered_paths != paths:
             removed_paths = paths if filtered_paths is None else [x for x in paths if x not in filtered_paths]
             print_warning("Suppressed adding the following path(s) to $%s of the module as they were already added: %s",
-                          key, removed_paths,
+                          key, ', '.join(removed_paths),
                           log=self.log)
-            if not filtered_paths:
-                filtered_paths = None
         return filtered_paths
 
-    def append_paths(self, key, paths, allow_abs=False, expand_relpaths=True):
+    def append_paths(self, key: str, paths: Union[str, Sequence[str]], allow_abs=False, expand_relpaths=True) -> str:
         """
         Generate append-path statements for the given list of paths.
 
         :param key: environment variable to append paths to
-        :param paths: list of paths to append
+        :param paths: single or list of paths to append
         :param allow_abs: allow providing of absolute paths
         :param expand_relpaths: expand relative paths into absolute paths (by prefixing install dir)
         """
-        paths = self._filter_paths(key, paths)
-        if paths is None:
-            return ''
         return self.update_paths(key, paths, prepend=False, allow_abs=allow_abs, expand_relpaths=expand_relpaths)
 
-    def prepend_paths(self, key, paths, allow_abs=False, expand_relpaths=True):
+    def prepend_paths(self, key: str, paths: Union[str, Sequence[str]], allow_abs=False, expand_relpaths=True) -> str:
         """
         Generate prepend-path statements for the given list of paths.
 
         :param key: environment variable to append paths to
-        :param paths: list of paths to append
+        :param paths: single or list of paths to prepend
         :param allow_abs: allow providing of absolute paths
         :param expand_relpaths: expand relative paths into absolute paths (by prefixing install dir)
         """
-        paths = self._filter_paths(key, paths)
-        if paths is None:
-            return ''
         return self.update_paths(key, paths, prepend=True, allow_abs=allow_abs, expand_relpaths=expand_relpaths)
 
+    def update_paths(self, key: str, paths: Union[str, Sequence[str]], prepend=True, allow_abs=False,
+                     expand_relpaths=True) -> str:
+        """
+        Generate append/prepend-path statements for the given list of paths.
+
+        :param key: environment variable to append paths to
+        :param paths: single or list of paths to add
+        :param allow_abs: allow providing of absolute paths
+        :param expand_relpaths: expand relative paths into absolute paths (by prefixing install dir)
+        """
+        if isinstance(paths, str):
+            self.log.debug("Wrapping %s into a list before using it for %s", paths, key)
+            paths = [paths]
+        elif not isinstance(paths, list):
+            paths = list(paths)  # coerce any iterator into a list
+
+        if key == 'PYTHONPATH':
+            python_paths = [path for path in paths if re.match(r'lib(64)?/python\d+\.\d+/site-packages', path)]
+            if len(python_paths) > 1:
+                raise EasyBuildError('Found multiple paths for PYTHONPATH: ' + ', '.join(python_paths))
+
+            # Special condition for PYTHONPATHs that match the standard pattern,
+            # replace with EBPYTHONPREFIXES which is added to python sys path at runtime via sitecustomize.py
+            if python_paths and build_option('replace_pythonpath'):
+                python_path = python_paths[0]
+                self.log.info("Replaced PYTHONPATH %s with EBPYTHONPREFIXES", python_path)
+                ret = self._update_paths('EBPYTHONPREFIXES', [''], prepend, allow_abs, expand_relpaths)
+                paths = [path for path in paths if path != python_path]
+                if paths:
+                    ret += self._update_paths(key, paths, prepend, allow_abs, expand_relpaths)
+                return ret
+        return self._update_paths(key, paths, prepend, allow_abs, expand_relpaths)
+
     def _modulerc_check_module_version(self, module_version):
         """
         Check value type & contents of specified module-version spec.
@@ -551,12 +568,12 @@ def unload_module(self, mod_name):
         """
         raise NotImplementedError
 
-    def update_paths(self, key, paths, prepend=True, allow_abs=False, expand_relpaths=True):
+    def _update_paths(self, key: str, paths: List[str], prepend=True, allow_abs=False, expand_relpaths=True) -> str:
         """
         Generate prepend-path or append-path statements for the given list of paths.
 
         :param key: environment variable to prepend/append paths to
-        :param paths: list of paths to prepend
+        :param paths: list of paths to add
         :param prepend: whether to prepend (True) or append (False) paths
         :param allow_abs: allow providing of absolute paths
         :param expand_relpaths: expand relative paths into absolute paths (by prefixing install dir)
@@ -955,16 +972,18 @@ def msg_on_unload(self, msg):
         print_cmd = "puts stderr %s" % quote_str(msg, tcl=True)
         return '\n'.join(['', self.conditional_statement("module-info mode unload", print_cmd, indent=False)])
 
-    def update_paths(self, key, paths, prepend=True, allow_abs=False, expand_relpaths=True):
+    def _update_paths(self, key: str, paths: list, prepend=True, allow_abs=False, expand_relpaths=True) -> str:
         """
         Generate prepend-path or append-path statements for the given list of paths.
 
         :param key: environment variable to prepend/append paths to
-        :param paths: list of paths to prepend
+        :param paths: list of paths to add
         :param prepend: whether to prepend (True) or append (False) paths
         :param allow_abs: allow providing of absolute paths
         :param expand_relpaths: expand relative paths into absolute paths (by prefixing install dir)
         """
+        paths = self._filter_paths(key, paths)
+
         if prepend:
             update_type = 'prepend'
         else:
@@ -974,10 +993,6 @@ def update_paths(self, key, paths, prepend=True, allow_abs=False, expand_relpath
             self.log.info("Not including statement to %s environment variable $%s, as specified", update_type, key)
             return ''
 
-        if isinstance(paths, str):
-            self.log.debug("Wrapping %s into a list before using it to %s path %s", paths, update_type, key)
-            paths = [paths]
-
         abspaths = []
         for path in paths:
             if os.path.isabs(path) and not allow_abs:
@@ -1426,16 +1441,18 @@ def modulerc(self, module_version=None, filepath=None, modulerc_txt=None):
         return super(ModuleGeneratorLua, self).modulerc(module_version=module_version, filepath=filepath,
                                                         modulerc_txt=modulerc_txt)
 
-    def update_paths(self, key, paths, prepend=True, allow_abs=False, expand_relpaths=True):
+    def _update_paths(self, key: str, paths: List[str], prepend=True, allow_abs=False, expand_relpaths=True) -> str:
         """
         Generate prepend_path or append_path statements for the given list of paths
 
         :param key: environment variable to prepend/append paths to
-        :param paths: list of paths to prepend/append
+        :param paths: list of paths to add
         :param prepend: whether to prepend (True) or append (False) paths
         :param allow_abs: allow providing of absolute paths
         :param expand_relpaths: expand relative paths into absolute paths (by prefixing install dir)
         """
+        paths = self._filter_paths(key, paths)
+
         if prepend:
             update_type = 'prepend'
         else:
@@ -1445,10 +1462,6 @@ def update_paths(self, key, paths, prepend=True, allow_abs=False, expand_relpath
             self.log.info("Not including statement to %s environment variable $%s, as specified", update_type, key)
             return ''
 
-        if isinstance(paths, str):
-            self.log.debug("Wrapping %s into a list before using it to %s path %s", update_type, paths, key)
-            paths = [paths]
-
         abspaths = []
         for path in paths:
             if os.path.isabs(path):

easybuild/tools/options.py

@@ -489,6 +489,8 @@ def override_options(self):
             'remove-ghost-install-dirs': ("Remove ghost installation directories when --force or --rebuild is used, "
                                           "rather than just warning about them",
                                           None, 'store_true', False),
+            'replace-pythonpath': ("Replaces PYTHONPATH with EBPYTHONPREFIXES in modules when possible",
+                                   None, 'store_true', True),
             'required-linked-shared-libs': ("Comma-separated list of shared libraries (names, file names, or paths) "
                                             "which must be linked in all installed binaries/libraries",
                                             'strlist', 'extend', None),

test/framework/module_generator.py

@@ -694,6 +694,9 @@ def append_paths(*args, **kwargs):
             res = append_paths('key', ['1234@example.com'], expand_relpaths=False)
             self.assertEqual("append-path\tkey\t\t1234@example.com\n", res)
 
+            expected = "append-path\tEBPYTHONPREFIXES\t\t$root\nappend-path\tPYTHONPATH\t\t$root/foo\n"
+            res = append_paths('PYTHONPATH', ['lib/python3.12/site-packages', 'foo'])
+            self.assertEqual(expected, res)
         else:
             expected = ''.join([
                 'append_path("key", pathJoin(root, "path1"))\n',
@@ -714,6 +717,10 @@ def append_paths(*args, **kwargs):
             res = append_paths('key', ['1234@example.com'], expand_relpaths=False)
             self.assertEqual('append_path("key", "1234@example.com")\n', res)
 
+            expected = 'append_path("EBPYTHONPREFIXES", root)\nappend_path("PYTHONPATH", pathJoin(root, "foo"))\n'
+            res = append_paths('PYTHONPATH', ['lib/python3.12/site-packages', 'foo'])
+            self.assertEqual(expected, res)
+
         self.assertErrorRegex(EasyBuildError, "Absolute path %s/foo passed to update_paths "
                                               "which only expects relative paths." % self.modgen.app.installdir,
                               append_paths, "key2", ["bar", "%s/foo" % self.modgen.app.installdir])