MRG, ENH: Remove 15-char limit for FIF (#8574)

* ENH: Remove 15-char limit for FIF

* FIX: Public API

Author: larsoner

doc/changes/latest.inc

@@ -36,6 +36,8 @@ Enhancements
 
 - Update the ``notebook`` 3d backend to use ``ipyvtk_simple`` for a better integration within ``Jupyter`` (:gh:`8503` by `Guillaume Favelier`_)
 
+- Remove the 15-character limitation for channel names when writing to FIF format. If you need the old 15-character names, you can use something like ``raw.rename_channels({n: n[:13] for n in raw.ch_names}, allow_duplicates=True)``, by `Eric Larson`_ (:gh:`8346`)
+
 - Add toggle-all button to :class:`mne.Report` HTML and ``width`` argument to :meth:`mne.Report.add_bem_to_section` (:gh:`8723` by `Eric Larson`_)
 
 - Add infant template MRI dataset downloader :func:`mne.datasets.fetch_infant_template` (:gh:`8738` by `Eric Larson`_ and `Christian O'Reilly`_)

mne/channels/channels.py

@@ -25,12 +25,14 @@
                      _get_stim_channel, _check_fname)
 from ..io.compensator import get_current_comp
 from ..io.constants import FIFF
-from ..io.meas_info import anonymize_info, Info, MontageMixin, create_info
+from ..io.meas_info import (anonymize_info, Info, MontageMixin, create_info,
+                            _rename_comps)
 from ..io.pick import (channel_type, pick_info, pick_types, _picks_by_type,
                        _check_excludes_includes, _contains_ch_type,
                        channel_indices_by_type, pick_channels, _picks_to_idx,
                        _get_channel_types, get_channel_type_constants,
                        _pick_data_channels)
+from ..io.tag import _rename_list
 from ..io.write import DATE_NONE
 from ..io._digitization import _get_data_as_dict_from_dig
 
@@ -471,13 +473,14 @@ def set_channel_types(self, mapping, verbose=None):
             warn(msg.format(", ".join(sorted(names)), *this_change))
         return self
 
-    @fill_doc
-    def rename_channels(self, mapping):
+    @verbose
+    def rename_channels(self, mapping, allow_duplicates=False, verbose=None):
         """Rename channels.
 
         Parameters
         ----------
-        %(rename_channels_mapping)s
+        %(rename_channels_mapping_duplicates)s
+        %(verbose_meth)s
 
         Returns
         -------
@@ -494,7 +497,7 @@ def rename_channels(self, mapping):
         from ..io import BaseRaw
 
         ch_names_orig = list(self.info['ch_names'])
-        rename_channels(self.info, mapping)
+        rename_channels(self.info, mapping, allow_duplicates)
 
         # Update self._orig_units for Raw
         if isinstance(self, BaseRaw) and self._orig_units is not None:
@@ -1150,17 +1153,16 @@ def interpolate_bads(self, reset_bads=True, mode='accurate',
         return self
 
 
-@fill_doc
-def rename_channels(info, mapping):
+@verbose
+def rename_channels(info, mapping, allow_duplicates=False, verbose=None):
     """Rename channels.
 
-    .. warning::  The channel names must have at most 15 characters
-
     Parameters
     ----------
     info : dict
         Measurement info to modify.
-    %(rename_channels_mapping)s
+    %(rename_channels_mapping_duplicates)s
+    %(verbose)s
     """
     _validate_type(info, Info, 'info')
     info._check_consistency()
@@ -1187,12 +1189,6 @@ def rename_channels(info, mapping):
     for new_name in new_names:
         _validate_type(new_name[1], 'str', 'New channel mappings')
 
-    bad_new_names = [name for _, name in new_names if len(name) > 15]
-    if len(bad_new_names):
-        raise ValueError('Channel names cannot be longer than 15 '
-                         'characters. These channel names are not '
-                         'valid : %s' % new_names)
-
     # do the remapping locally
     for c_ind, new_name in new_names:
         for bi, bad in enumerate(bads):
@@ -1201,13 +1197,21 @@ def rename_channels(info, mapping):
         ch_names[c_ind] = new_name
 
     # check that all the channel names are unique
-    if len(ch_names) != len(np.unique(ch_names)):
+    if len(ch_names) != len(np.unique(ch_names)) and not allow_duplicates:
         raise ValueError('New channel names are not unique, renaming failed')
 
     # do the remapping in info
     info['bads'] = bads
+    ch_names_mapping = dict()
     for ch, ch_name in zip(info['chs'], ch_names):
+        ch_names_mapping[ch['ch_name']] = ch_name
         ch['ch_name'] = ch_name
+    # .get b/c fwd info omits it
+    _rename_comps(info.get('comps', []), ch_names_mapping)
+    if 'projs' in info:  # fwd might omit it
+        for proj in info['projs']:
+            proj['data']['col_names'][:] = \
+                _rename_list(proj['data']['col_names'], ch_names_mapping)
     info._update_redundant()
     info._check_consistency()
 

mne/channels/montage.py

@@ -195,12 +195,12 @@ def plot(self, scale_factor=20, show_names=True, kind='topomap', show=True,
                             sphere=sphere)
 
     @fill_doc
-    def rename_channels(self, mapping):
+    def rename_channels(self, mapping, allow_duplicates=False):
         """Rename the channels.
 
         Parameters
         ----------
-        %(rename_channels_mapping)s
+        %(rename_channels_mapping_duplicates)s
 
         Returns
         -------
@@ -209,7 +209,7 @@ def rename_channels(self, mapping):
         """
         from .channels import rename_channels
         temp_info = create_info(list(self._get_ch_pos()), 1000., 'eeg')
-        rename_channels(temp_info, mapping)
+        rename_channels(temp_info, mapping, allow_duplicates)
         self.ch_names = temp_info['ch_names']
 
     def save(self, fname):

mne/channels/tests/test_channels.py

@@ -83,10 +83,6 @@ def test_rename_channels():
     # Test bad input
     pytest.raises(ValueError, rename_channels, info, 1.)
     pytest.raises(ValueError, rename_channels, info, 1.)
-    # Test name too long (channel names must be less than 15 characters)
-    A16 = 'A' * 16
-    mapping = {'MEG 2641': A16}
-    pytest.raises(ValueError, rename_channels, info, mapping)
 
     # Test successful changes
     # Test ch_name and ch_names are changed

mne/cov.py

@@ -24,7 +24,7 @@
                       _DATA_CH_TYPES_SPLIT)
 
 from .io.constants import FIFF
-from .io.meas_info import read_bad_channels, create_info
+from .io.meas_info import _read_bad_channels, create_info
 from .io.tag import find_tag
 from .io.tree import dir_tree_find
 from .io.write import (start_block, end_block, write_int, write_name_list,
@@ -2022,7 +2022,7 @@ def _read_cov(fid, node, cov_kind, limited=False, verbose=None):
             projs = _read_proj(fid, this)
 
             #   Read the bad channel list
-            bads = read_bad_channels(fid, this)
+            bads = _read_bad_channels(fid, this, None)
 
             #   Put it together
             assert dim == len(data)

mne/evoked.py

@@ -32,7 +32,8 @@
 from .io.tag import read_tag
 from .io.tree import dir_tree_find
 from .io.pick import pick_types, _picks_to_idx, _FNIRS_CH_TYPES_SPLIT
-from .io.meas_info import read_meas_info, write_meas_info
+from .io.meas_info import (read_meas_info, write_meas_info,
+                           _read_extended_ch_info, _rename_list)
 from .io.proj import ProjMixin
 from .io.write import (start_file, start_block, end_file, end_block,
                        write_int, write_string, write_float_matrix,
@@ -1080,7 +1081,9 @@ def _read_evoked(fname, condition=None, kind='average', allow_maxshield=False):
                 raise ValueError('Number of channels and number of '
                                  'channel definitions are different')
 
+            ch_names_mapping = _read_extended_ch_info(chs, my_evoked, fid)
             info['chs'] = chs
+            info['bads'][:] = _rename_list(info['bads'], ch_names_mapping)
             logger.info('    Found channel information in evoked data. '
                         'nchan = %d' % nchan)
             if sfreq > 0:

mne/forward/forward.py

@@ -25,11 +25,13 @@
 from ..io.tag import find_tag, read_tag
 from ..io.matrix import (_read_named_matrix, _transpose_named_matrix,
                          write_named_matrix)
-from ..io.meas_info import read_bad_channels, write_info
+from ..io.meas_info import (_read_bad_channels, write_info, _write_ch_infos,
+                            _read_extended_ch_info, _make_ch_names_mapping,
+                            _rename_list)
 from ..io.pick import (pick_channels_forward, pick_info, pick_channels,
                        pick_types)
 from ..io.write import (write_int, start_block, end_block,
-                        write_coord_trans, write_ch_info, write_name_list,
+                        write_coord_trans, write_name_list,
                         write_string, start_file, end_file, write_id)
 from ..io.base import BaseRaw
 from ..evoked import Evoked, EvokedArray
@@ -286,14 +288,14 @@ def _read_forward_meas_info(tree, fid):
     info['meas_id'] = tag.data if tag is not None else None
 
     # Add channel information
-    chs = list()
+    info['chs'] = chs = list()
     for k in range(parent_meg['nent']):
         kind = parent_meg['directory'][k].kind
         pos = parent_meg['directory'][k].pos
         if kind == FIFF.FIFF_CH_INFO:
             tag = read_tag(fid, pos)
             chs.append(tag.data)
-    info['chs'] = chs
+    ch_names_mapping = _read_extended_ch_info(chs, parent_meg, fid)
     info._update_redundant()
 
     #   Get the MRI <-> head coordinate transformation
@@ -322,7 +324,8 @@ def _read_forward_meas_info(tree, fid):
     else:
         raise ValueError('MEG/head coordinate transformation not found')
 
-    info['bads'] = read_bad_channels(fid, parent_meg)
+    info['bads'] = _read_bad_channels(
+        fid, parent_meg, ch_names_mapping=ch_names_mapping)
     # clean up our bad list, old versions could have non-existent bads
     info['bads'] = [bad for bad in info['bads'] if bad in info['ch_names']]
 
@@ -918,18 +921,17 @@ def write_forward_meas_info(fid, info):
         raise ValueError('Head<-->sensor transform not found')
     write_coord_trans(fid, meg_head_t)
 
+    ch_names_mapping = dict()
     if 'chs' in info:
         #  Channel information
+        ch_names_mapping = _make_ch_names_mapping(info['chs'])
         write_int(fid, FIFF.FIFF_NCHAN, len(info['chs']))
-        for k, c in enumerate(info['chs']):
-            #   Scan numbers may have been messed up
-            c = deepcopy(c)
-            c['scanno'] = k + 1
-            write_ch_info(fid, c)
+        _write_ch_infos(fid, info['chs'], False, ch_names_mapping)
     if 'bads' in info and len(info['bads']) > 0:
         #   Bad channels
+        bads = _rename_list(info['bads'], ch_names_mapping)
         start_block(fid, FIFF.FIFFB_MNE_BAD_CHANNELS)
-        write_name_list(fid, FIFF.FIFF_MNE_CH_NAME_LIST, info['bads'])
+        write_name_list(fid, FIFF.FIFF_MNE_CH_NAME_LIST, bads)
         end_block(fid, FIFF.FIFFB_MNE_BAD_CHANNELS)
 
     end_block(fid, FIFF.FIFFB_MNE_PARENT_MEAS_FILE)

mne/io/array/tests/test_array.py

@@ -28,11 +28,15 @@ def test_long_names():
     info = create_info(['a' * 15 + 'b', 'a' * 16], 1000., verbose='error')
     data = np.empty((2, 1000))
     raw = RawArray(data, info)
+    assert raw.ch_names == ['a' * 15 + 'b', 'a' * 16]
+    # and a way to get the old behavior
+    raw.rename_channels({k: k[:13] for k in raw.ch_names},
+                        allow_duplicates=True, verbose='error')
     assert raw.ch_names == ['a' * 13 + '-0', 'a' * 13 + '-1']
     info = create_info(['a' * 16] * 11, 1000., verbose='error')
     data = np.empty((11, 1000))
     raw = RawArray(data, info)
-    assert raw.ch_names == ['a' * 12 + '-%s' % ii for ii in range(11)]
+    assert raw.ch_names == ['a' * 16 + '-%s' % ii for ii in range(11)]
 
 
 def test_array_copy():

mne/io/constants.py

@@ -31,6 +31,7 @@
 FIFF.FIFFB_HPI_COIL           = 110  # Data acquired from one HPI coil
 FIFF.FIFFB_PROJECT            = 111
 FIFF.FIFFB_CONTINUOUS_DATA    = 112
+FIFF.FIFFB_CH_INFO            = 113  # Extra channel information
 FIFF.FIFFB_VOID               = 114
 FIFF.FIFFB_EVENTS             = 115
 FIFF.FIFFB_INDEX              = 116
@@ -156,6 +157,23 @@
 FIFF.FIFF_HPI_COIL_NO            = 245   # Coil number listed by HPI measurement
 FIFF.FIFF_HPI_COILS_USED         = 246   # List of coils finally used when the transformation was computed
 FIFF.FIFF_HPI_DIGITIZATION_ORDER = 247   # Which Isotrak digitization point corresponds to each of the coils energized
+
+
+#
+# Tags used for storing channel info
+#
+FIFF.FIFF_CH_SCAN_NO             = 250   # Channel scan number. Corresponds to fiffChInfoRec.scanNo field
+FIFF.FIFF_CH_LOGICAL_NO          = 251   # Channel logical number. Corresponds to fiffChInfoRec.logNo field
+FIFF.FIFF_CH_KIND                = 252   # Channel type. Corresponds to fiffChInfoRec.kind field"
+FIFF.FIFF_CH_RANGE               = 253   # Conversion from recorded number to (possibly virtual) voltage at the output"
+FIFF.FIFF_CH_CAL                 = 254   # Calibration coefficient from output voltage to some real units
+FIFF.FIFF_CH_LOC                 = 255   # Channel loc
+FIFF.FIFF_CH_UNIT                = 256   # Unit of the data
+FIFF.FIFF_CH_UNIT_MUL            = 257   # Unit multiplier exponent
+FIFF.FIFF_CH_DACQ_NAME           = 258   # Name of the channel in the data acquisition system. Corresponds to fiffChInfoRec.name.
+FIFF.FIFF_CH_COIL_TYPE           = 350   # Coil type in coil_def.dat
+FIFF.FIFF_CH_COORD_FRAME         = 351   # Coordinate frame (integer)
+
 #
 # Pointers
 #

mne/io/ctf_comp.py

@@ -74,6 +74,32 @@ def read_ctf_comp(fid, node, chs, verbose=None):
     compdata : list
         The compensation data
     """
+    return _read_ctf_comp(fid, node, chs, None)
+
+
+def _read_ctf_comp(fid, node, chs, ch_names_mapping):
+    """Read the CTF software compensation data from the given node.
+
+    Parameters
+    ----------
+    fid : file
+        The file descriptor.
+    node : dict
+        The node in the FIF tree.
+    chs : list
+        The list of channels from info['chs'] to match with
+        compensators that are read.
+    ch_names_mapping : dict | None
+        The channel renaming to use.
+    %(verbose)s
+
+    Returns
+    -------
+    compdata : list
+        The compensation data
+    """
+    from .meas_info import _rename_comps
+    ch_names_mapping = dict() if ch_names_mapping is None else ch_names_mapping
     compdata = []
     comps = dir_tree_find(node, FIFF.FIFFB_MNE_CTF_COMP_DATA)
 
@@ -105,6 +131,7 @@ def read_ctf_comp(fid, node, chs, verbose=None):
 
         one['save_calibrated'] = bool(calibrated)
         one['data'] = mat
+        _rename_comps([one], ch_names_mapping)
         if not calibrated:
             #   Calibrate...
             _calibrate_comp(one, chs, mat['row_names'], mat['col_names'])