Merge branch 'develop' into python_O

Author: boegel

easybuild/framework/easyblock.py

@@ -155,16 +155,18 @@
     'exts_list': [[], 'List with extensions added to the base installation', EXTENSIONS],
 
     # MODULES easyconfig parameters
+    'modaliases': [{}, "Aliases to be defined in module file", MODULES],
     'modextrapaths': [{}, "Extra paths to be prepended in module file", MODULES],
     'modextravars': [{}, "Extra environment variables to be added to module file", MODULES],
     'modloadmsg': [{}, "Message that should be printed when generated module is loaded", MODULES],
     'modluafooter': ["", "Footer to include in generated module file (Lua syntax)", MODULES],
+    'modaltsoftname': [None, "Module name to use (rather than using software name", MODULES],
     'modtclfooter': ["", "Footer to include in generated module file (Tcl syntax)", MODULES],
-    'modaliases': [{}, "Aliases to be defined in module file", MODULES],
     'moduleclass': ['base', 'Module class to be used for this software', MODULES],
     'moduleforceunload': [False, 'Force unload of all modules when loading the extension', MODULES],
     'moduleloadnoconflict': [False, "Don't check for conflicts, unload other versions instead ", MODULES],
     'include_modpath_extensions': [True, "Include $MODULEPATH extensions specified by module naming scheme.", MODULES],
+    'recursive_module_unload': [False, 'Recursive unload of all dependencies when unloading module', MODULES],
 
     # OTHER easyconfig parameters
     'buildstats': [None, "A list of dicts with build statistics", OTHER],

easybuild/framework/easyconfig/default.py

@@ -109,7 +109,8 @@ class EasyConfig(object):
     Class which handles loading, reading, validation of easyconfigs
     """
 
-    def __init__(self, path, extra_options=None, build_specs=None, validate=True, hidden=None, rawtxt=None):
+    def __init__(self, path, extra_options=None, build_specs=None, validate=True, hidden=None, rawtxt=None,
+                 auto_convert_value_types=True):
         """
         initialize an easyconfig.
         @param path: path to easyconfig file to be parsed (ignored if rawtxt is specified)
@@ -118,6 +119,8 @@ def __init__(self, path, extra_options=None, build_specs=None, validate=True, hi
         @param validate: indicates whether validation should be performed (note: combined with 'validate' build option)
         @param hidden: indicate whether corresponding module file should be installed hidden ('.'-prefixed)
         @param rawtxt: raw contents of easyconfig file
+        @param auto_convert_value_types: indicates wether types of easyconfig values should be automatically converted
+                                         in case they are wrong
         """
         self.template_values = None
         self.enable_templating = True  # a boolean to control templating
@@ -184,7 +187,8 @@ def __init__(self, path, extra_options=None, build_specs=None, validate=True, hi
 
         # parse easyconfig file
         self.build_specs = build_specs
-        self.parser = EasyConfigParser(filename=self.path, rawcontent=self.rawtxt)
+        self.parser = EasyConfigParser(filename=self.path, rawcontent=self.rawtxt,
+                                       auto_convert_value_types=auto_convert_value_types)
         self.parse()
 
         # handle allowed system dependencies
@@ -214,6 +218,7 @@ def __init__(self, path, extra_options=None, build_specs=None, validate=True, hi
         self.short_mod_name = mns.det_short_module_name(self)
         self.mod_subdir = mns.det_module_subdir(self)
 
+        self.software_license = None
 
     def copy(self):
         """
@@ -335,16 +340,17 @@ def validate(self, check_osdeps=True):
 
     def validate_license(self):
         """Validate the license"""
-        lic = self._config['software_license'][0]
+        lic = self['software_license']
         if lic is None:
             # when mandatory, remove this possibility
             if 'software_license' in self.mandatory:
-                raise EasyBuildError("License is mandatory, but 'software_license' is undefined")
-        elif not isinstance(lic, License):
-            raise EasyBuildError('License %s has to be a License subclass instance, found classname %s.',
-                                 lic, lic.__class__.__name__)
-        elif not lic.name in EASYCONFIG_LICENSES_DICT:
-            raise EasyBuildError('Invalid license %s (classname: %s).', lic.name, lic.__class__.__name__)
+                raise EasyBuildError("Software license is mandatory, but 'software_license' is undefined")
+        elif lic in EASYCONFIG_LICENSES_DICT:
+            # create License instance
+            self.software_license = EASYCONFIG_LICENSES_DICT[lic]()
+        else:
+            known_licenses = ', '.join(sorted(EASYCONFIG_LICENSES_DICT.keys()))
+            raise EasyBuildError("Invalid license %s (known licenses: %s)", lic, known_licenses)
 
         # TODO, when GROUP_SOURCE and/or GROUP_BINARY is True
         #  check the owner of source / binary (must match 'group' parameter from easyconfig)
@@ -478,7 +484,21 @@ def toolchain(self):
         returns the Toolchain used
         """
         if self._toolchain is None:
-            self._toolchain = get_toolchain(self['toolchain'], self['toolchainopts'], mns=ActiveMNS())
+            # provide list of (direct) toolchain dependencies (name & version), if easyconfig can be found for toolchain
+            tcdeps = None
+            tcname, tcversion = self['toolchain']['name'], self['toolchain']['version']
+            if tcname != DUMMY_TOOLCHAIN_NAME:
+                tc_ecfile = robot_find_easyconfig(tcname, tcversion)
+                if tc_ecfile is None:
+                    self.log.debug("No easyconfig found for toolchain %s version %s, can't determine dependencies",
+                                   tcname, tcversion)
+                else:
+                    self.log.debug("Found easyconfig for toolchain %s version %s: %s", tcname, tcversion, tc_ecfile)
+                    tc_ec = process_easyconfig(tc_ecfile)[0]
+                    tcdeps = tc_ec['dependencies']
+                    self.log.debug("Toolchain dependencies based on easyconfig: %s", tcdeps)
+
+            self._toolchain = get_toolchain(self['toolchain'], self['toolchainopts'], mns=ActiveMNS(), tcdeps=tcdeps)
             tc_dict = self._toolchain.as_dict()
             self.log.debug("Initialized toolchain: %s (opts: %s)" % (tc_dict, self['toolchainopts']))
         return self._toolchain
@@ -1008,28 +1028,33 @@ def create_paths(path, name, version):
 
 def robot_find_easyconfig(name, version):
     """
-    Find an easyconfig for module in path
+    Find an easyconfig for module in path, returns (absolute) path to easyconfig file (or None, if none is found).
     """
     key = (name, version)
     if key in _easyconfig_files_cache:
         _log.debug("Obtained easyconfig path from cache for %s: %s" % (key, _easyconfig_files_cache[key]))
         return _easyconfig_files_cache[key]
+
     paths = build_option('robot_path')
-    if not paths:
-        raise EasyBuildError("No robot path specified, which is required when looking for easyconfigs (use --robot)")
-    if not isinstance(paths, (list, tuple)):
+    if paths is None:
+        paths = []
+    elif not isinstance(paths, (list, tuple)):
         paths = [paths]
-    # candidate easyconfig paths
+
+    res = None
     for path in paths:
         easyconfigs_paths = create_paths(path, name, version)
         for easyconfig_path in easyconfigs_paths:
             _log.debug("Checking easyconfig path %s" % easyconfig_path)
             if os.path.isfile(easyconfig_path):
                 _log.debug("Found easyconfig file for name %s, version %s at %s" % (name, version, easyconfig_path))
                 _easyconfig_files_cache[key] = os.path.abspath(easyconfig_path)
-                return _easyconfig_files_cache[key]
+                res = _easyconfig_files_cache[key]
+                break
+        if res:
+            break
 
-    return None
+    return res
 
 
 class ActiveMNS(object):
@@ -1091,7 +1116,22 @@ def _det_module_name_with(self, mns_method, ec, force_visible=False):
             - string representing module name has length > 0
             - module name only contains printable characters (string.printable, except carriage-control chars)
         """
-        mod_name = mns_method(self.check_ec_type(ec))
+        ec = self.check_ec_type(ec)
+
+        # replace software name with desired replacement (if specified)
+        orig_name = None
+        if ec.get('modaltsoftname', None):
+            orig_name = ec['name']
+            ec['name'] = ec['modaltsoftname']
+            self.log.info("Replaced software name '%s' with '%s' when determining module name", orig_name, ec['name'])
+        else:
+            self.log.debug("No alternative software name specified to determine module name with")
+
+        mod_name = mns_method(ec)
+
+        # restore original software name if it was tampered with
+        if orig_name is not None:
+            ec['name'] = orig_name
 
         if not is_valid_module_name(mod_name):
             raise EasyBuildError("%s is not a valid module name", str(mod_name))
@@ -1134,7 +1174,7 @@ def det_short_module_name(self, ec, force_visible=False):
         self.log.debug("Obtained valid short module name %s" % mod_name)
 
         # sanity check: obtained module name should pass the 'is_short_modname_for' check
-        if not self.is_short_modname_for(mod_name, ec['name']):
+        if not self.is_short_modname_for(mod_name, ec.get('modaltsoftname', None) or ec['name']):
             raise EasyBuildError("is_short_modname_for('%s', '%s') for active module naming scheme returns False",
                                  mod_name, ec['name'])
         return mod_name

easybuild/framework/easyconfig/easyconfig.py

@@ -370,8 +370,8 @@ def retrieve_blocks_in_spec(spec, only_blocks, silent=False):
         ec_format_version = FORMAT_DEFAULT_VERSION
     _log.debug("retrieve_blocks_in_spec: derived easyconfig format version: %s" % ec_format_version)
 
-    # blocks in easyconfigs are only supported in format versions prior to 2.0
-    if pieces and ec_format_version < EasyVersion('2.0'):
+    # blocks in easyconfigs are only supported in easyconfig format 1.0
+    if pieces and ec_format_version == EasyVersion('1.0'):
         # make a map of blocks
         blocks = []
         while pieces:

easybuild/framework/easyconfig/format/one.py

@@ -46,11 +46,10 @@
 
 def build_easyconfig_constants_dict():
     """Make a dictionary with all constants that can be used"""
-    # sanity check
     all_consts = [
         ('TEMPLATE_CONSTANTS', dict([(x[0], x[1]) for x in TEMPLATE_CONSTANTS])),
         ('EASYCONFIG_CONSTANTS', dict([(key, val[0]) for key, val in EASYCONFIG_CONSTANTS.items()])),
-        ('EASYCONFIG_LICENSES', EASYCONFIG_LICENSES_DICT),
+        ('EASYCONFIG_LICENSES', dict([(klass().name, name) for name, klass in EASYCONFIG_LICENSES_DICT.items()])),
     ]
     err = []
     const_dict = {}

easybuild/framework/easyconfig/format/pyheaderconfigobj.py

@@ -0,0 +1,143 @@
+# #
+# Copyright 2013-2015 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en),
+# the Hercules foundation (http://www.herculesstichting.be/in_English)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# http://github.com/hpcugent/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild.  If not, see <http://www.gnu.org/licenses/>.
+# #
+"""
+YAML easyconfig format (.yeb)
+Useful: http://www.yaml.org/spec/1.2/spec.html
+
+@author: Caroline De Brouwer (Ghent University)
+@author: Kenneth Hoste (Ghent University)
+"""
+import os
+from vsc.utils import fancylogger
+
+from easybuild.framework.easyconfig.format.format import INDENT_4SPACES, EasyConfigFormat
+from easybuild.framework.easyconfig.format.pyheaderconfigobj import build_easyconfig_constants_dict
+from easybuild.framework.easyconfig.format.pyheaderconfigobj import build_easyconfig_variables_dict
+from easybuild.tools.build_log import EasyBuildError
+from easybuild.tools.filetools import read_file
+from easybuild.tools.utilities import only_if_module_is_available, quote_str
+
+
+_log = fancylogger.getLogger('easyconfig.format.yeb', fname=False)
+
+
+YAML_DIR = r'%YAML'
+YAML_SEP = '---'
+YEB_FORMAT_EXTENSION = '.yeb'
+
+
+def yaml_join(loader, node):
+    """
+    defines custom YAML join function.
+    see http://stackoverflow.com/questions/5484016/how-can-i-do-string-concatenation-or-string-replacement-in-yaml/23212524#23212524
+    @param loader: the YAML Loader
+    @param node: the YAML (sequence) node
+    """
+    seq = loader.construct_sequence(node)
+    return ''.join([str(i) for i in seq])
+
+
+try:
+    import yaml
+    # register the tag handlers
+    yaml.add_constructor('!join', yaml_join)
+except ImportError:
+    pass
+
+
+class FormatYeb(EasyConfigFormat):
+    """Support for easyconfig YAML format"""
+    USABLE = True
+
+    def __init__(self):
+        """FormatYeb constructor"""
+        super(FormatYeb, self).__init__()
+        self.log.experimental("Parsing .yeb easyconfigs")
+
+    def validate(self):
+        """Format validation"""
+        _log.info(".yeb format validation isn't implemented (yet) - validation always passes")
+        return True
+
+    def get_config_dict(self):
+        """
+        Return parsed easyconfig as a dictionary, based on specified arguments.
+        """
+        return self.parsed_yeb
+
+    @only_if_module_is_available('yaml')
+    def parse(self, txt):
+        """
+        Process YAML file
+        """
+        txt = self._inject_constants_dict(txt)
+        self.parsed_yeb = yaml.load(txt)
+
+    def _inject_constants_dict(self, txt):
+        """Inject constants so they are resolved when actually parsing the YAML text."""
+        constants_dict = build_easyconfig_constants_dict()
+
+        lines = txt.splitlines()
+
+        # extract possible YAML header, for example
+        # %YAML 1.2
+        # ---
+        yaml_header = []
+        for i, line in enumerate(lines):
+            if line.startswith(YAML_DIR):
+                if lines[i+1].startswith(YAML_SEP):
+                    yaml_header.extend([lines.pop(i), lines.pop(i)])
+
+        injected_constants = ['__CONSTANTS__: ']
+        for key, value in constants_dict.items():
+            injected_constants.append('%s- &%s %s' % (INDENT_4SPACES, key, quote_str(value)))
+
+        full_txt = '\n'.join(yaml_header + injected_constants + lines)
+
+        return full_txt
+
+    def dump(self, ecfg, default_values, templ_const, templ_val):
+        """Dump parsed easyconfig in .yeb format"""
+        raise NotImplementedError("Dumping of .yeb easyconfigs not supported yet")
+
+    def extract_comments(self, txt):
+        """Extract comments from easyconfig file"""
+        self.log.debug("Not extracting comments from .yeb easyconfigs")
+
+
+def is_yeb_format(filename, rawcontent):
+    """
+    Determine whether easyconfig is in .yeb format.
+    If filename is None, rawcontent will be used to check the format.
+    """
+    isyeb = False
+    if filename:
+        isyeb = os.path.splitext(filename)[-1] == YEB_FORMAT_EXTENSION
+    else:
+        # if one line like 'name: ' is found, this must be YAML format
+        for line in rawcontent.splitlines():
+            if line.startswith('name: '):
+                isyeb = True
+    return isyeb