fmatoolbox 0.1.8__tar.gz → 0.2.1__tar.gz

{fmatoolbox-0.1.8/src/fmatoolbox.egg-info → fmatoolbox-0.2.1}/PKG-INFO

@@ -1,25 +1,25 @@
 Metadata-Version: 2.4
 Name: fmatoolbox
-Version: 0.1.8
+Version: 0.2.1
 Summary: Python port of FMAToolbox
 Author: Pietro Bozzo
 License-Expression: GPL-3.0-or-later
 Project-URL: Homepage, https://github.com/pietro-bozzo/FMAToolboxPython
 Project-URL: Repository, https://github.com/pietro-bozzo/FMAToolboxPython
 Project-URL: Issues, https://github.com/pietro-bozzo/FMAToolboxPython/issues
-Requires-Python: >=3.11
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: numpy
+Requires-Dist: loky
 Requires-Dist: numba
-Requires-Dist: scipy
+Requires-Dist: numpy
 Requires-Dist: matplotlib
+Requires-Dist: scipy
 Requires-Dist: statsmodels
-Requires-Dist: loky
 Provides-Extra: assemblies
+Requires-Dist: joblib; extra == "assemblies"
 Requires-Dist: scikit-learn; extra == "assemblies"
 Requires-Dist: scikit-image; extra == "assemblies"
-Requires-Dist: joblib; extra == "assemblies"
 Provides-Extra: hierarch
 Requires-Dist: hierarch; extra == "hierarch"
 Requires-Dist: pandas; extra == "hierarch"

{fmatoolbox-0.1.8 → fmatoolbox-0.2.1}/pyproject.toml

@@ -4,27 +4,27 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "fmatoolbox"
-version = "0.1.8"
+version = "0.2.1"
 description = "Python port of FMAToolbox"
 authors = [{name = "Pietro Bozzo"}]
 readme = "README.md"
 license = "GPL-3.0-or-later"
 
-requires-python = ">=3.11"
+requires-python = ">=3.10"
 dependencies = [
-    "numpy",
+    "loky",
     "numba",
-    "scipy",
+    "numpy",
     "matplotlib",
+    "scipy",
     "statsmodels",
-    "loky",
 ]
 
 [project.optional-dependencies]
 assemblies = [
+    "joblib",
     "scikit-learn",
     "scikit-image",
-    "joblib",
 ]
 hierarch = [
     "hierarch",

{fmatoolbox-0.1.8 → fmatoolbox-0.2.1}/src/fmatoolbox/__init__.py

@@ -1,5 +1,6 @@
 from . import analysis
 from . import data
+from . import exceptions
 from . import general
 from . import intervals
 from . import plotting

{fmatoolbox-0.1.8 → fmatoolbox-0.2.1}/src/fmatoolbox/analysis.py

@@ -1,6 +1,6 @@
-''' Specialized analyses for FMAToolbox '''
+""" Specialized analyses for FMAToolbox """
 
-import fmatoolbox.general
+import fmatoolbox.intervals
 import numpy as np
 import scipy as sp
 import statsmodels.stats.multitest
@@ -553,7 +553,7 @@ def cellAssembliesICA(spikes, window:float=None, when=None, drop_mix:bool=False)
    time = raster[:,0]
    n = raster[:,1:] # discard time column
    if when is not None:
-       time, valid = fmatoolbox.general.restrict(time,when,s_ind=True)
+       time, valid = fmatoolbox.intervals.restrict(time,when,s_ind=True)
        n = n[valid]
 
    # remove units which never spiked to avoid cov error

{fmatoolbox-0.1.8 → fmatoolbox-0.2.1}/src/fmatoolbox/data.py

@@ -1,4 +1,4 @@
-''' Functions to load data for a recording session '''
+""" Functions to load data for a recording session """
 
 import pathlib
 import scipy.io
@@ -242,80 +242,130 @@ def saveRegionSpikes(file:str|PathLike[str], spikes):
     return
 
 
-def loadEventFile(filename:str|PathLike[str], compact:bool=False):
-    # load events from a .evt file
-    #
-    # arguments:
-    #     filename    file, .evt file, each line must have format 'beginning of basename_event1_1'
-    #     compact     bool = false, if true, return events as compact dictionary
-    #
-    # output:
-    #     events      dict, keys are either 'times' and 'descriptions' or event names if 'compact' = True
+def loadEventFile(filename:str|PathLike[str], compact:bool=None):
+    """load an event file, accepted formats:
+    - .npy: each column of the array is returned as a value in `events` (single event)
+    - .npz: each array of the archive is returned as an event in `events`
+    - .mat: DESCRIBE
+    - .evt: text file where each line must have format 'beginning of basename_event1_1' or 'end of basename_event1_1'
+    - other extensions: interpreted as text files, as produced by ``saveEvent``
 
-    with open(filename,'r') as f:
-        lines = f.read().splitlines()
+    Args:
+        filename:   file to read
+        compact:    if True, for .npz files, return a single event instead of a dict of events, for .evt files, return a
+                    compact dictionary; ignored otherwise, defaults to True
 
-    # extract first non-whitespace token
-    times = [line.split()[0] for line in lines]
-    times = np.array([float(t) / 1000.0 for t in times]) # convert to seconds
+    Returns:
+        events:     either a single event (a dict) or a dict of events (a dict of dict)
+                    in general, an event is a dict of type {'name': 1d np array}
+    """
 
-    # remove first token and following whitespace
-    descriptions = []
-    for line in lines:
-        parts = line.split(maxsplit=1)
-        descriptions.append(parts[1] if len(parts) > 1 else '')
-
-    if compact:
-        # group by events, description is of type 'beginning of basename_event1' or 'beginning of basename_event1_1'
-        events = {} #collections.defaultdict(lambda: collections.defaultdict(list))
-        for t, d in zip(times,descriptions):
-            if " of " not in d:
-                raise fmatoolbox.exceptions.FileFormatError(f"Unexpected format: '{d}'")
-            phase, full_id = d.split(" of ",1)
-            phase = phase.strip()
-            full_id = full_id.strip()
-            # remove trailing 'bis' and '_1'
-            if full_id.endswith('bis'):
-                full_id = full_id[:-3]
-            parts = full_id.split('_')
-            try:
-                if parts[-1].isdigit():
-                    part = parts[-2]
-                else:
-                    part = parts[-1]
-            except:
-                raise ValueError(f"Invalid event ID format: '{full_id}'")
-            event_id = part
-            # concatenate values
-            if event_id in events:
-                if phase in events[event_id]:
-                    events[event_id][phase].append(t)
-                else:
-                    events[event_id][phase] = [t]
+    if compact is None: compact = True
+    def event_dict(data, columns=None):
+        if columns is None: columns = ['col0'] if data.ndim == 1 else [f'col{i}' for i in range(data.shape[1])]
+        return {columns[0]: data} if data.ndim == 1 else {columns[i]: data[:,i] for i in range(data.shape[1])}
+
+    filename = pathlib.Path(filename)
+    format = filename.suffix
+    match format:
+
+        # 1. npy file
+        case '.npy':
+            events = event_dict(np.load(filename))
+
+        # 2. npz file
+        case '.npz':
+            with np.load(filename) as data:
+                events = dict(data) if compact else {d: event_dict(data[d]) for d in data}
+
+        # 3. MATLAB's mat file
+        case '.mat':
+            name = filename.name.removesuffix(".events.mat").rsplit(".",1)[-1]
+            events = {name: scipy.io.loadmat(filename, simplify_cells=True)[name]}
+
+        # 4. evt file (retrocompatibility)
+        case '.evt':
+            with open(filename,'r') as f:
+                lines = f.read().splitlines()
+
+            # extract first non-whitespace token
+            times = [line.split()[0] for line in lines]
+            times = np.array([float(t) / 1000.0 for t in times]) # convert to seconds
+
+            # remove first token and following whitespace
+            descriptions = []
+            for line in lines:
+                parts = line.split(maxsplit=1)
+                descriptions.append(parts[1] if len(parts) > 1 else '')
+
+            if compact:
+                # group by events, description is of type 'beginning of basename_event1' or 'beginning of basename_event1_1'
+                events = {}
+                for t, d in zip(times,descriptions):
+                    if " of " not in d:
+                        raise fmatoolbox.exceptions.FileFormatError(f"Unexpected format: '{d}'")
+                    phase, full_id = d.split(" of ",1)
+                    phase = phase.strip()
+                    full_id = full_id.strip()
+                    # remove trailing 'bis' and '_1'
+                    if full_id.endswith('bis'):
+                        full_id = full_id[:-3]
+                    parts = full_id.split('_')
+                    try:
+                        if parts[-1].isdigit():
+                            part = parts[-2]
+                        else:
+                            part = parts[-1]
+                    except:
+                        raise ValueError(f"Invalid event ID format: '{full_id}'")
+                    event_id = part
+                    # concatenate values
+                    if event_id in events:
+                        if phase in events[event_id]:
+                            events[event_id][phase].append(t)
+                        else:
+                            events[event_id][phase] = [t]
+                    else:
+                        events[event_id] = {phase : [t]}
             else:
-                events[event_id] = {phase : [t]}
-    else:
-        events = { 'time': times, 'description': descriptions }
+                events = {'time':times, 'description':descriptions}
+
+        # 5. text file
+        case _:
+            data = np.loadtxt(filename, comments='%', delimiter=',', ndmin=2)
+            # try reading column names from header
+            columns = [f'col{i}' for i in range(data.shape[1])]
+            with filename.open('r') as f:
+                for line in f:
+                    line = line.strip()
+                    if not line.startswith('%'):
+                        break
+                    line = line.removeprefix('%').strip()
+                    if line.startswith('columns:'):
+                        for i, c in enumerate(line.removeprefix('columns:').split(',')):
+                            columns[i] = c.strip()
+            events = {columns[i]: data[:,i] for i in range(data.shape[1])}
 
     return events
 
 
 def loadEvents(session:str|PathLike[str], extra:str|list[str]):
-    # load event files for a session
-    #
-    # arguments:
-    #     session      path to session's .xml file, event files must be in session folder
-    #     extra        (:) string, extensions of other event files named <basename>.extra[i] to load,
-    #                  can be 'subdir/extension' to load '.../basename/subdir/basename.extension'
-    #
-    # output:
-    #     events       dict, keys are event names
-    #     cat_names    (:) string, names of events contained in <basename>.cat.evt
+    """load event files for a session
+
+    Args:
+        session:    path to session's .xml file, event files must be in session folder
+        extra:      (:) string, extensions of other event files named <basename>.extra[i] to load,
+                    can be 'subdir/extension' to load '.../basename/subdir/basename.extension'
+
+    Returns:
+        events:     dict, keys are event names
+        cat_names:  (:) string, names of events contained in <basename>.cat.evt
+    """
 
     session = pathlib.Path(session)
     file_root = session.parent
     # accept single string input
-    if isinstance(extra,str):
+    if isinstance(extra, str):
         extra = [extra]
 
     # load all *.evt files
@@ -323,13 +373,13 @@ def loadEvents(session:str|PathLike[str], extra:str|list[str]):
     cat_names = []
     for file in file_root.glob("*.evt"):
         try:
-            events.update(loadEventFile(file,compact=True))
+            events.update(loadEventFile(file, compact=True))
         except fmatoolbox.exceptions.FileFormatError:
             pass
         if str(file)[-8:] == '.cat.evt':
             cat_names = list(events.keys())
 
-    # load other file types
+    # load other file types         TO DO: add npy support, use loadEventFile for mat files as well
     for extension in extra:
         # handle subfolders, e.g., 'subfolder/name'
         e = pathlib.Path(extension).name
@@ -337,20 +387,71 @@ def loadEvents(session:str|PathLike[str], extra:str|list[str]):
         this_froot = file_root / p / session.stem
         if this_froot.with_suffix(f'.{e}.npz').exists():
             # load .npz file
-            data = np.load(this_froot.with_suffix(f'.{e}.npz'))
-            for d in data:
-                events[d] = {f'col{i}': data[d][:,i] for i in range(data[d].shape[1])}
+            _e = loadEventFile(this_froot.with_suffix(f'.{e}.npz'), compact=False)
+            events.update(_e)
         elif this_froot.with_suffix(f'.{e}.events.mat').exists():
             # load .events.mat file
             events[e] = scipy.io.loadmat(this_froot.with_suffix(f'.{e}.events.mat'),simplify_cells=True)[e]
         else:
             # load text file
-            data = np.loadtxt(this_froot.with_suffix(f'.{e}'),comments='%',delimiter=',',ndmin=2)
-            events[e] = {f'col{i}': data[:,i] for i in range(data.shape[1])}
+            events[e] = loadEventFile(this_froot.with_suffix(f'.{e}'))
 
     return events, cat_names
 
 
+def saveEvent(session:str|PathLike[str], format:str, data, columns=None, name=None):
+    """save an event in standard FMAT format
+
+    Args:
+        session:    path to session's .xml file, along which the event file will be created
+        format:     format of event file, parsed as <subdirs>/<name>.<format>, where <subdirs> and <name> are optional, e.g.,
+                    - 'sleep.npy' gives name='sleep', format='npy'
+                    - 'units/firing.wake' gives subdirs='units', name='firing', format='wake'
+                    accepted formats are:
+                    - 'npy': DESCRIBE
+                    - 'npz': DESCRIBE
+                    - a different str, to save the event as a text file named <basename>.<format>, prepending a metadata header
+                      in this case, `data` must be an array whose columns represent different quantities; if 1d, interpreted as one column
+        data:       array to save; for npz format, must be a dict of {'name': array}
+        columns:    names of columns of `data`, mandatory for text formats, ignored otherwise
+        name:       event name used to construct the output file name, ignored for text formats, mandatory otherwise
+
+    Examples:
+        # save <basename>.sleep in text format containing two events: 'onset' and 'offset'
+        saveEvent(session, 'sleep', [[1,2]], columns=['onset','offset'])
+
+        # save the same as <basename>.sleep.npz
+        saveEvent(session, 'npz', {'onset':[1,2], 'offset':[1.5,3]}, name='sleep')
+        # or
+        saveEvent(session, 'sleep.npz', {'onset':[1,2], 'offset':[1.5,3]})
+
+        # save the same file in subdirectory 'new' within the session
+        saveEvent(session, 'new/sleep.npz', {'onset':[1,2], 'offset':[1.5,3]})
+    """
+
+    # parse format
+    subdirs, _, filename = format.rpartition('/')
+    name, dot, format = filename.rpartition('.')
+    if not dot:
+        name, format = '', filename
+    session = pathlib.Path(session)
+    froot = session.parent / subdirs / session.stem
+
+    if format == 'npy' or format == 'npz':
+        if not name:
+            raise ValueError(f"'name' must be specified for {format} format")
+        fname = froot.with_suffix(f'.{name}.{format}')
+        np.save(fname, data) if format == 'npy' else np.savez(fname, **data)
+
+    else:
+        if columns is None:
+            raise ValueError("'columns' must be specified for text formats")
+        fname = froot.with_suffix(f'.{format}')
+        np.savetxt(fname, data, delimiter=',', header=f" columns: {', '.join(columns)}", comments="%")
+
+    return
+
+
 def loadSpikeWaveforms(session:str|PathLike[str]):
 
     file_root = pathlib.Path(session).with_suffix('')
@@ -474,29 +575,6 @@ def makeAnatFile(session:str|PathLike[str]):
     return
 
 
-def saveEvent(session:str|PathLike[str], format:str, data, columns):
-    """save an event in standard FMAT format
-
-    Args:
-        session:    path to session's .xml file, along which the event file will be created
-        format:     format of event file, either:
-                    - 'npy': to implement
-                    - 'npz': to implement
-                    - a different str, to save the event as a text file named <basename>.<format>, prepending a metadata header
-                      in this case, `data` must be an array whose columns represent different quantities; if 1d, interpreted as one column
-        data:       array to save
-        columns:    names of columns of `data`
-    """
-
-    if extension == 'npy' or extension == 'npz':
-        print('implement!')
-    else:
-        fname = pathlib.Path(session).with_suffix(f'.{extension}')
-        np.savetxt(fname, data, delimiter=',', header=f" columns: {', '.join(columns)}", comments="%")
-
-    return
-
-
 def printXML(session:str|PathLike[str]):
     """simple utility to print contents of a session's .xml file"""
 

fmatoolbox-0.2.1/src/fmatoolbox/general.py

@@ -0,0 +1,179 @@
+""" General-purpose data processing for FMAToolbox """
+
+import numpy as np
+import fmatoolbox.intervals
+
+
+def unwhere(indices,shape=None):
+    """perform the inverse operation of ``numpy.where``
+
+    arguments:
+        indices    ndarray of int, output of `numpy.where`
+        shape      tuple = None, shape of the output array
+
+    output:
+        mask       ndarray of bool, original input to `numpy.where`
+    """
+
+    if shape is None:
+        shape = tuple(i.max() + 1 if len(i) else 0 for i in indices)
+    mask = np.zeros(shape,dtype=bool)
+    mask[indices] = True
+    return mask
+
+
+def shuffleEvents(events, offset:float=None, n:int=None, group=None, fast:bool=None, intervals=None):
+    '''
+    shuffle point-process events (i.e., time stamps) preserving their inter-event interval
+
+    arguments:
+        events       (s,) | (s,f) float, samples to shuffle; s: # samples, f: # features, if 2d, first column contains time stamps
+        offset       float = 0 s, reference time, necessary when events start at a time != 0, e.g., for a portion of a recording
+                     in [1000 s, 3000 s]; must be smaller than first event time, ignored if 'intervals' are provided
+        n            int = 1, number of independent repetitions of the shuffling procedure, changes output shape
+        group        (s,) int, grouping variable to shuffle events separately per group, adds output 'group'
+        fast         bool = False, if True, 'events' must be time sorted (to save computation time)
+        intervals    (:,2) float = None, rows are [start, stop] times of intervals to restrict shuffle into; if 1d, reshaped to (1,2)
+
+    output:
+        shuffled     (:,:,) float, shuffled events, shape depends on 'n':
+                     - same as 'events' if 'n' = 1
+                     - (s,f,n), i.e., with an extra dimension for 'n' shuffling repetitions, otherwise
+        group        (:,) int, optional, grouping variable for shuffled events, only returned if 'group' is not None, shape depends on 'n':
+                     - (s,) if 'n' = 1
+                     - (s,n), otherwise
+    '''
+
+    # default values
+    events = np.asarray(events)
+    n_dim = events.ndim
+    if offset is None: offset = 0
+    if n is None: n = 1
+    if group is not None: group = np.asarray(group,dtype=int)
+    if intervals is not None:
+        events, valid = fmatoolbox.intervals.restrict(events,intervals,shift=True,s_ind=True)
+        offset = 0
+        if group is not None:
+            group = group[valid]
+
+    # zero-repetitions case
+    if n == 0:
+        if group is not None:
+            return np.zeros(events.shape+(0,)), np.zeros((events.shape[0],0))
+        return np.zeros(events.shape+(0,))
+
+    # make 'events' 3d by appending singleton dimensions
+    events = events.reshape(*events.shape, *(1,) * (3 - n_dim)) # (events, features, 1)
+
+    # sort by time
+    if not fast:
+        order = events[:,0,0].argsort(axis=0)
+        events = events[order]
+        if group is not None:
+            group = group[order]
+
+    # repeat events 'n' times
+    events = np.tile(events,(1,1,n)) # (events, features, shuffle repetitions)
+
+    # core f to shuffle events of a group
+    rng = np.random.default_rng()
+    def shuffleGroup(e):
+        # shuffle inter-event intervals (IEI)
+        iei = np.diff(e[:,0,:],prepend=offset,axis=0)
+        iei_shape = iei.shape
+        permutation_idx = np.argsort(rng.random(iei_shape),axis=0)
+        iei = iei[permutation_idx,np.arange(iei_shape[1])]
+        # reconstruct shuffled samples
+        shuffled = offset + np.cumsum(iei,axis=0) # shuffled times (events, shuffle repetitions)
+        features = e[permutation_idx[:,None,:], np.arange(1,e.shape[1])[None,:,None], np.arange(iei_shape[1])[None,None,:]]
+        shuffled = np.concatenate((shuffled[:,None],features),axis=1) # append shuffled features
+        return shuffled
+
+    # perform shuffling
+    if group is None:
+        shuffled = shuffleGroup(events)
+    else:
+        shuffled = []
+        unique_groups, g_count = np.unique(group,return_counts=True)
+        for g in unique_groups:
+            shuffled.append(shuffleGroup(events[group==g]))
+        # sort by time
+        shuffled = np.concatenate(shuffled,axis=0)
+        order = shuffled[:,0,:].argsort(axis=0)
+        shuffled = shuffled[order[:,None,:], np.arange(shuffled.shape[1])[None,:,None], np.arange(shuffled.shape[2])[None,None,:]]
+        # build 'group' for output
+        group = np.repeat(unique_groups,g_count) # use unique output to sort
+        group = group[order] # sort as 'shuffled'
+
+    # unshift events back into intervals
+    if intervals is not None:
+        for i in range(n):
+            shuffled[:,:,i] = fmatoolbox.intervals.unshift(shuffled[:,:,i],intervals)
+
+    # restore shape
+    if n_dim == 1 and n == 1:
+        shuffled = shuffled.ravel()
+    elif n == 1:
+        shuffled = shuffled[:,:,0]
+
+    if group is not None:
+        if n_dim == 1 and n == 1:
+            group = group.ravel()
+        elif n == 1:
+            group = group[:,0]
+        return shuffled, group
+
+    return shuffled
+
+
+def circularShift(x, shift, axis:int=None):
+    """independent circular shift of each slice of 'x' along a given axis
+
+    arguments:
+        x        ndarray to shift
+        shift    int | (:,) int, shift amount per slice; if int, the same shift is applied to each slice
+        axis     int = 0, axis to roll along; 0 means columns of 'x' will be rolled
+    """
+
+    if axis is None: axis = 0
+    shift = np.array(shift,ndmin=1)
+
+    n, m = x.shape[:2]
+    if axis == 0:
+        if shift.size == 1:
+            shift = np.repeat(shift,m,axis=0)
+        shifted = x[(np.arange(n)[:,None]-shift) % n, np.arange(m)]
+    elif axis == 1:
+        if shift.size == 1:
+            shift = np.repeat(shift,n,axis=0)
+        shifted = x[np.arange(n)[:,None], (np.arange(m)-shift[:,None]) % m]
+    else:
+        raise ValueError('not implemented!')
+
+    return shifted
+
+
+def jitter(samples,dt,low=None):
+    """add a jitter drawn from a uniform distribution to each sample's time stamp
+
+    arguments:
+        samples    (s,) | (s,f) numeric, samples to jitter; s: # samples, f: # features, if 2d, first column contains time stamps
+        dt         float, higher bound of the uniform distribution
+        low        float = -dt, lower bound of the uniform distribution
+
+    output:
+        samples    numeric, jittered samples (only time stamps are changed)
+    """
+
+    if low is None: low = -dt
+
+    samples = np.asarray(samples,dtype=float)
+    n_dim = samples.ndim
+    rng = np.random.default_rng()
+
+    if n_dim < 2:
+        samples += rng.uniform(low=low,high=dt,size=samples.shape)
+    elif n_dim == 2:
+        samples[:,0,...] += rng.uniform(low=low,high=dt,size=samples.shape[:1] + samples.shape[2:])
+
+    return samples