labdata 0.1.7__tar.gz → 0.1.9__tar.gz

{labdata-0.1.7 → labdata-0.1.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: labdata
-Version: 0.1.7
+Version: 0.1.9
 Summary: Package to manage data in experimental neuroscience labs
 Author-email: Joao Couto <jpcouto@gmail.com>
 Project-URL: Homepage, https://github.com/jcouto/labdata

labdata-0.1.9/labdata/__init__.py

@@ -0,0 +1,15 @@
+VERSION = "0.1.9"
+
+from .utils import *
+from .copy import copy_to_upload_server
+from .s3 import copy_to_s3
+
+_project = (prefs.get('database', {}) or {}).get('database.project', None)
+if 'LABDATA_DATABASE_PROJECT' in os.environ.keys():
+    if len(os.environ['LABDATA_DATABASE_PROJECT']):
+        _project = os.environ['LABDATA_DATABASE_PROJECT']
+
+plugins = load_plugins(project=_project)
+for _name, _mod in plugins.items():
+    globals()[_name] = _mod
+

{labdata-0.1.7 → labdata-0.1.9}/labdata/cli.py

@@ -2,6 +2,40 @@ from .utils import *
 import argparse
 
 class CLI_parser():
+    '''Command-line interface dispatcher for labdata.
+
+    Each public method corresponds to a ``labdata <command>`` sub-command.
+    Instantiating the class parses ``sys.argv`` and calls the matching method.
+
+    Sub-commands
+    ------------
+    dashboard
+        Launch the Streamlit dashboard or a Dash explorer.
+    subjects
+        List subjects, optionally filtered by user, name, or sex.
+    sessions
+        List sessions and datasets for one or more subjects.
+    get
+        Download dataset files from S3 to local storage.
+    put
+        Copy raw data files to the upload server (triggers an UploadJob).
+    clean
+        Delete local files that are confirmed uploaded (checksums verified).
+    run
+        Queue or launch an analysis compute job.
+    task
+        Execute a single ``ComputeTask`` by job ID (called by workers).
+    task_reset
+        Reset a failed/stuck compute task so it can be retried.
+    upload
+        Process pending ``UploadJob`` entries (server-side, admin use).
+    logpipe
+        Pipe stdout into a ``ComputeTask.task_log`` entry.
+    run_container
+        Launch a local Apptainer container interactively.
+    build_container
+        Build Apptainer ``.sif`` images from definition files.
+    '''
     def __init__(self):
         parser = argparse.ArgumentParser(
             description = f'{tcolor["y"]("labdata")} - tools to manage data in an experimental neuroscience lab',
@@ -35,7 +69,7 @@ Server commands (don't run on experimental computers):
             ''')
         
         parser.add_argument('command',
-                            help= 'type: labdata2 <command> -h for help')
+                            help= 'type: labdata <command> -h for help')
 
         args = parser.parse_args(sys.argv[1:2])
         command = args.command.replace('-','_') # can use - in command
@@ -46,9 +80,45 @@ Server commands (don't run on experimental computers):
         getattr(self,command)()  # Runs the following parser
 
     def dashboard(self):
+        '''Launch the labdata web dashboard or an interactive data explorer.
+
+        Without flags, opens the main Streamlit dashboard (metadata insertion,
+        session browser, admin panel).  With ``--spike-sorting`` or
+        ``--cell-segmentation`` opens a Dash-based interactive explorer instead.
+
+        Usage
+        -----
+        ::
+
+            labdata dashboard
+            labdata dashboard -p <project>
+            labdata dashboard --spike-sorting -f <subject_filter>
+            labdata dashboard --cell-segmentation -f <subject_filter>
+            labdata dashboard --admin
+            labdata dashboard --console
+
+        Options
+        -------
+        -f, --filter-name
+            Filter subjects by name substring.
+        -u, --user
+            Filter by lab member username.
+        -p, --project
+            Project name (overrides preference file).
+        --spike-sorting
+            Open the Dash spike-sorting explorer instead of Streamlit.
+        --cell-segmentation
+            Open the Dash cell-segmentation explorer instead of Streamlit.
+        --admin
+            Include the admin tab (storage stats, project switcher).
+        --console
+            Include the IPython console tab.
+        --debug
+            Enable Dash debug mode for error tracebacks.
+        '''
         parser = argparse.ArgumentParser(
-            description = 'Open the interactive explorer or graphical interface.',
-            usage = '''labdata dashboard -f <FILTER>''')
+            description='Open the interactive explorer or graphical interface.',
+            usage='labdata dashboard -f <FILTER>')
         
         parser.add_argument('-f','--filter-name',
                             default = None,
@@ -91,20 +161,51 @@ Server commands (don't run on experimental computers):
                                       open_browser = True)
         else:
             import subprocess as sub
+            import os
             main_webpage_path = Path(__file__).parent / 'dashboard' / 'index.py'
-            command = f'streamlit run {str(main_webpage_path)}'
+            command = f'streamlit run --server.showEmailPrompt=False --browser.gatherUsageStats=False {str(main_webpage_path)}'
+            if 'LABDATA_VERBOSE' in os.environ.keys():
+                print(f'Running streamlit dashboard: {command}')
             if not args.project is None:
                 command += f' project={args.project}'
             if args.console:
                 command += ' console=1'
             if args.admin:
                 command += ' admin=1'
-            sub.run(command.split(' '))
+            sub.run(command,shell=True)
     
     def subjects(self):
+        '''List subjects registered in the database.
+
+        Prints subjects grouped by responsible lab member, with optional
+        filters and an optional column showing total raw data size.
+
+        Usage
+        -----
+        ::
+
+            labdata subjects
+            labdata subjects -u <user>
+            labdata subjects -f <name_substring>
+            labdata subjects -s M
+            labdata subjects --include-size
+
+        Options
+        -------
+        -u, --user
+            Filter by lab member username.
+        -f, --filter-name
+            Filter by subject name substring (SQL ``LIKE`` match).
+        -s, --filter-sex
+            Filter by sex (``M``, ``F``, or ``U``).
+        -p, --project
+            Project name.
+        --include-size
+            Show total raw data size (TB) and file count per subject.
+        '''
         parser = argparse.ArgumentParser(
-            description = 'List sessions and datatypes',
-            usage = '''labdata subject -u <USER> -f <FILTER>''')
+            description='List sessions and datatypes',
+            usage='labdata subject -u <USER> -f <FILTER>')
 
         project = _get_project()
         parser.add_argument('-p','--project',
@@ -178,9 +279,32 @@ Server commands (don't run on experimental computers):
             print('')
 
     def sessions(self):
+        '''List sessions and datasets for one or more subjects.
+
+        Prints sessions grouped by subject, showing dataset type, name, and
+        (optionally) size.  For ephys datasets, also shows how many probes have
+        been sorted.
+
+        Usage
+        -----
+        ::
+
+            labdata sessions -a <subject>
+            labdata sessions -a <subject1> <subject2>
+            labdata sessions -a <subject> --include-size
+
+        Options
+        -------
+        -a, --subject
+            One or more subject names.
+        -p, --project
+            Project name.
+        --include-size
+            Show total raw data size (TB/GB) and file count.
+        '''
         parser = argparse.ArgumentParser(
-            description = 'List sessions and datatypes',
-            usage = '''labdata sessions -a <SUBJECT>''')
+            description='List sessions and datatypes',
+            usage='labdata sessions -a <SUBJECT>')
         parser = self._add_default_arguments(parser,1, include_project = True)
         parser.add_argument('--include-size', action = 'store_true',
                             default = False)
@@ -228,9 +352,35 @@ Server commands (don't run on experimental computers):
                         print(f'\t\t {pre} {t["dataset_type"]} - {t["dataset_name"]} {extra}')
     
     def get(self):
+        '''Download dataset files from S3 to local storage.
+
+        Resolves the matching ``File`` entries via ``Dataset.DataFiles`` for
+        the requested subjects/sessions/datatypes and calls ``File.get()``
+        to download any files not already present locally.
+
+        Usage
+        -----
+        ::
+
+            labdata get -a <subject>
+            labdata get -a <subject> -s <session>
+            labdata get -a <subject> -s <session> -d ephys
+            labdata get -a <subject1> <subject2> -s <session>
+
+        Options
+        -------
+        -a, --subject
+            One or more subject names.
+        -s, --session
+            One or more session names.
+        -d, --datatype
+            Filter by dataset type (e.g. ``ephys``, ``twophoton``).
+        -p, --project
+            Project name.
+        '''
         parser = argparse.ArgumentParser(
-            description = 'Download data from one or multiple sessions',
-            usage = '''labdata sessions -a <SUBJECTS> -s <SESSIONS>''')
+            description='Download data from one or multiple sessions',
+            usage='labdata get -a <SUBJECTS> -s <SESSIONS>')
         parser = self._add_default_arguments(parser,3, include_project = True)
         
         # TODO: Add an argument to include files that match a pattern..
@@ -268,9 +418,45 @@ Server commands (don't run on experimental computers):
         (schema.File() & (schema.Dataset.DataFiles() & keys).proj()).get()
         
     def put(self):
+        '''Copy raw data files to the upload server and create an UploadJob.
+
+        Opens a Qt file-copy widget that checksums source files, copies them to
+        the designated upload server path, and registers an ``UploadJob`` in the
+        database for the requested upload rule.  **This does not upload to the
+        cloud** — that happens when ``labdata upload`` is later run on the server.
+
+        Usage
+        -----
+        ::
+
+            labdata put <path/to/data/folder>
+            labdata put <path/to/data/folder> -r ephys -a <subject> -s <session>
+            labdata put <file1> <file2> -r twophoton
+
+        Options
+        -------
+        filepaths
+            One or more local file paths or a single folder (all files inside
+            the folder are selected recursively).
+        -r, --rule
+            Upload rule to apply on the server (e.g. ``ephys``, ``two_photon``,
+            ``miniscope``, ``fixed_brain``).  Default is ``none`` (copy only).
+        -a, --subject
+            Subject name to associate with the upload job.
+        -s, --session
+            Session name to associate with the upload job.
+        -t, --datatype-name
+            Dataset type name.
+        -p, --project
+            Project name.
+        --overwrite
+            Overwrite files that already exist on the server.
+        --ask
+            Prompt for confirmation before copying each file.
+        '''
         parser = argparse.ArgumentParser(
-            description = 'Copies data to the server to be uploaded [THIS DOES NOT UPLOAD TO THE CLOUD]',
-            usage = '''labdata put -a <SUBJECT> -s <SESSION>''')
+            description='Copies data to the server to be uploaded [THIS DOES NOT UPLOAD TO THE CLOUD]',
+            usage='labdata put -a <SUBJECT> -s <SESSION>')
         from .rules import rulesmap
         rules_help  = ','.join([tcolor['g'](r) for r in rulesmap.keys()])
         parser = self._add_default_arguments(parser,include_project = True)
@@ -322,9 +508,34 @@ Server commands (don't run on experimental computers):
             app.exit()
             #sys.exit(app.exec_())
     def clean(self):
+        '''Delete local copies of files that are confirmed uploaded to S3.
+
+        Compares MD5 checksums of local files against the ``File`` table.
+        Files that match are deleted; files not yet confirmed in the database
+        are kept.  Use ``--dry-run`` to preview what would be deleted.
+
+        Usage
+        -----
+        ::
+
+            labdata clean
+            labdata clean --dry-run
+            labdata clean -f ephys
+            labdata clean --only-processed
+
+        Options
+        -------
+        -f, --filter
+            Only consider files whose path contains this string (repeatable).
+        --dry-run
+            Print what would be deleted without actually deleting.
+        --only-processed
+            Only delete files listed in ``ProcessedFile`` (original raw files
+            that were transformed by an upload rule).
+        '''
         parser = argparse.ArgumentParser(
-            description = 'Releases local storage space.',
-            usage = '''labdata clean -f "ephys"''')
+            description='Releases local storage space.',
+            usage='labdata clean -f "ephys"')
         parser.add_argument('-f','--filter',action = 'store',default = [], type = str, nargs = '+')
         parser.add_argument('--dry-run',action = 'store_true', default = False)
         parser.add_argument('--only-processed',action = 'store_true', default = False)
@@ -335,9 +546,57 @@ Server commands (don't run on experimental computers):
         print(deleted,kept)
 
     def run(self):
+        '''Queue or launch an analysis compute job.
+
+        Creates ``ComputeTask`` entries for the matching datasets and submits
+        them to the selected execution target (local Apptainer, SLURM, or EC2).
+        If ``-j`` is given, re-submits an existing job without creating a new one.
+
+        Usage
+        -----
+        ::
+
+            labdata run <analysis> -a <subject> -s <session>
+            labdata run <analysis> -a <subject> -t slurm
+            labdata run <analysis> -a <subject> -t slurm -- <analysis-specific-args>
+            labdata run <analysis> --queue
+            labdata run <analysis> -j <job_id>
+
+        Options
+        -------
+        analysis
+            Name of the registered analysis (e.g. ``spks``, ``suite2p``,
+            ``caiman``, ``deeplabcut``, ``populate``).
+        -a, --subject
+            One or more subject names.
+        -s, --session
+            One or more session names.
+        -d, --datatype
+            Filter by dataset type.
+        -j, --job
+            Re-use an existing ``ComputeTask`` job ID instead of creating one.
+        -t, --target
+            Execution target: ``local``, ``slurm``, or a named remote from
+            the preference file.  Defaults to ``prefs["compute"]["default_target"]``.
+        -p, --project
+            Project name.
+        --force-submit
+            Submit even if an identical task command already exists.
+        --multisession
+            Combine all matched datasets into a single multi-session job.
+        --keep-files
+            Preserve intermediate files after the job completes.
+        --queue
+            Print the current job queue for all targets and exit.
+        --delete-computes
+            Delete all ``ComputeTask`` entries for this analysis (careful).
+        --
+            Everything after ``--`` is forwarded as secondary arguments to the
+            analysis class (e.g. algorithm flags, parameter set selection).
+        '''
         parser = argparse.ArgumentParser(
-            description = 'Allocates or runs an analysis',
-            usage = f'''labdata run <ANALYSIS> -a <SUBJECT> -s <SESSION>
+            description='Allocates or runs an analysis',
+            usage=f'''labdata run <ANALYSIS> -a <SUBJECT> -s <SESSION>
             
 Available analysis are 
     {tcolor['g'](', '.join(prefs["compute"]["analysis"].keys()))}''')
@@ -414,9 +673,37 @@ Available analysis are
         run_analysis(target,jobids, obj, project = args.project)
 
     def task_reset(self):
+        '''Reset a failed or stuck compute task so it can be retried.
+
+        Sets ``task_waiting=1`` and ``task_status="WAITING"`` for the specified
+        job(s) so they will be picked up by the next ``labdata task`` call.
+        With ``--resubmit``, immediately re-launches the job on the same or a
+        specified target.
+
+        Usage
+        -----
+        ::
+
+            labdata task_reset <job_id>
+            labdata task_reset <job_id1> <job_id2>
+            labdata task_reset <job_id> --resubmit
+            labdata task_reset <job_id> --resubmit -t slurm
+            labdata task_reset --clear-all
+
+        Options
+        -------
+        job_id
+            One or more ``ComputeTask.job_id`` values.
+        -t, --target
+            Execution target for ``--resubmit`` (default: from job record).
+        --resubmit
+            After resetting, immediately re-launch the job.
+        --clear-all
+            Delete **all** compute tasks (requires confirmation).
+        '''
         parser = argparse.ArgumentParser(
-            description = 'Reset a task in the ComputeTask so the job can be ran again',
-            usage = '''labdata task_reset <JOB_ID> ''')
+            description='Reset a task in the ComputeTask so the job can be ran again',
+            usage='labdata task_reset <JOB_ID>')
         parser.add_argument('job_id', action = 'store', type = int, nargs='+')
         parser.add_argument('-t','--target', action = 'store', default = None, type = str)
         parser.add_argument('--resubmit', action = 'store_true', default = False)
@@ -450,9 +737,32 @@ Available analysis are
                 run_analysis(target,[job_id],obj)
                     
     def task(self):
+        '''Execute a single ``ComputeTask`` by job ID.
+
+        This is the command run by worker nodes (inside an Apptainer container
+        or on a SLURM job) to actually perform the analysis.  It instantiates
+        the correct ``BaseCompute`` subclass for the job and calls ``compute()``.
+
+        You normally do not call this directly — use ``labdata run`` to queue
+        and submit jobs instead.
+
+        Usage
+        -----
+        ::
+
+            labdata task <job_id>
+            labdata task <job_id> -p <project>
+
+        Options
+        -------
+        job_id
+            ``ComputeTask.job_id`` to execute.
+        -p, --project
+            Project name (used when the worker has no preference file project set).
+        '''
         parser = argparse.ArgumentParser(
-            description = 'Runs a ComputeTask',
-            usage = '''labdata task <JOB_ID> ''')
+            description='Runs a ComputeTask',
+            usage='labdata task <JOB_ID>')
         parser.add_argument('job_id',action = 'store',default = None,type = int)
         project = _get_project()
         parser.add_argument('-p','--project',
@@ -468,9 +778,42 @@ Available analysis are
             task.compute()
 
     def upload(self):
+        '''Process pending ``UploadJob`` entries (server-side / admin command).
+
+        Fetches waiting upload jobs assigned to this host, applies the
+        appropriate upload rule (compression + S3 upload), and updates the
+        database.  Run this on the upload server, not on experimental computers.
+
+        Usage
+        -----
+        ::
+
+            labdata upload
+            labdata upload <job_id>
+            labdata upload --queue
+            labdata upload --reset-failed
+            labdata upload --all-hosts -n 4
+
+        Options
+        -------
+        job_id
+            Specific ``UploadJob.job_id`` values to process (optional).
+            When omitted, all waiting jobs for this host are processed.
+        -p, --project
+            Project name to filter jobs.
+        -n, --n-jobs
+            Number of parallel upload workers.  Default is ``DEFAULT_N_JOBS``.
+        --queue
+            Print the current upload queue (working, failed, waiting, recent
+            completed jobs) and exit without processing.
+        --all-hosts
+            Process jobs assigned to any host, not just this machine.
+        --reset-failed
+            Reset all ``FAILED`` jobs to ``WAITING`` before processing.
+        '''
         parser = argparse.ArgumentParser(
-            description = 'Runs an UploadTask',
-            usage = '''labdata upload <JOB_ID> (optional) ''')
+            description='Runs an UploadTask',
+            usage='labdata upload <JOB_ID> (optional)')
         parser.add_argument('job_id',action = 'store',default = [], type = int, nargs = '*')
         parser.add_argument('--all-hosts',action = 'store_true',default = False)
         parser.add_argument('--reset-failed',action = 'store_true',default = False)
@@ -546,9 +889,34 @@ Available analysis are
             tasks = process_upload_jobs(key)
 
     def logpipe(self):
+        '''Pipe stdin into a ``ComputeTask.task_log`` entry.
+
+        Reads lines from stdin (intended to be piped from an analysis process)
+        and periodically flushes them to the ``task_log`` field of the given
+        ``ComputeTask``.  This lets container jobs write their stdout to the
+        database for remote monitoring.
+
+        Typical usage (inside an sbatch script)::
+
+            labdata task <job_id> 2>&1 | labdata logpipe <job_id>
+
+        Usage
+        -----
+        ::
+
+            labdata logpipe <job_id>
+            labdata logpipe <job_id> -i 10
+
+        Options
+        -------
+        job_id
+            ``ComputeTask.job_id`` to write logs to.
+        -i, --refresh-period
+            How often (seconds) to flush the log to the database.  Default 5.
+        '''
         parser = argparse.ArgumentParser(
-            description = 'Sends the stdout to a log',
-            usage = '''labdata logpipe <JOB_ID> ''')
+            description='Sends the stdout to a log',
+            usage='labdata logpipe <JOB_ID>')
         parser.add_argument('job_id',action = 'store',default = None,type = int)
         parser.add_argument('-i','--refresh-period',action = 'store',default = 5., type = float)
         args = parser.parse_args(sys.argv[2:])
@@ -569,7 +937,8 @@ Available analysis are
                     return
                 else:
                     print(f'Appending stdout to ComputeTask [{job_id}] ')
-        def handle_line(line,log, tic):
+        def handle_line(line, log, tic):
+            '''Append a line to the running log buffer and periodically flush to the DB.'''
             if not line is None:
                 if log is None:
                     log = line
@@ -739,5 +1108,15 @@ def _get_project():
             project = str(os.environ['LABDATA_DATABASE_PROJECT'])
     return project
 
+def main_deprecated():
+    import sys
+    print("Warning: 'labdata2' is deprecated, use 'labdata' instead.", file=sys.stderr)
+    main()
+
 def main():
+    '''Entry point for the ``labdata`` command-line tool.
+
+    Instantiates ``CLI_parser``, which reads ``sys.argv`` and dispatches to
+    the appropriate sub-command method.
+    '''
     CLI_parser()

{labdata-0.1.7 → labdata-0.1.9}/labdata/compute/caiman.py

@@ -2,21 +2,27 @@ from ..utils import *
 from .utils import BaseCompute
 from .suite2p import Suite2pCompute
 class CaimanCompute(Suite2pCompute):
+    '''Cell segmentation compute class using CaImAn.
+
+    Works for both one-photon (miniscope) and two-photon datasets.  Runs
+    inside the ``labdata-caiman`` Apptainer container (CPU-only).
+
+    The compute pipeline:
+
+    1. Identify files and determine dataset type (``Miniscope`` or ``TwoPhoton``).
+    2. Copy the ``zarr.zip`` stack to the scratch directory.
+    3. Run CaImAn CNMF/CNMF-E with the specified parameters.
+    4. Delete memory-mapped intermediate files.
+    5. Upload the HDF5 results file to S3 and insert into ``CellSegmentation``.
+
+    Parameters are stored in ``CellSegmentationParams`` (shared with
+    ``Suite2pCompute`` using the same part tables).
+    '''
     container = 'labdata-caiman'
     cuda = False
     name = 'caiman'
     url = 'http://github.com/flatironinstitute/CaImAn'
-    def __init__(self,job_id, project = None, allow_s3 = None, **kwargs):
-        '''
-        This class runs Caiman on a dataset, which can be used for both 1p and 2p data. The ComputeTask will:
-        
-        1. **File Identification and Dataset Type Check**: Identify the files and determine the type of dataset (Miniscope or TwoPhoton).
-        2. **File Copy to Scratch**: Copy only the necessary file(s) to a scratch folder for processing.
-        3. **Caiman Execution**: Execute Caiman on the copied file or folder. Using the parameters specified.
-        4. **Cleanup and Result Integration**: Delete the memory-mapped files generated during processing and integrate the results into the CellSegmentation table.
-
-        This class includes a handler for the CLI.
-        '''
+    def __init__(self, job_id, project=None, allow_s3=None, **kwargs):
         super(Suite2pCompute,self).__init__(job_id, project = project, allow_s3 = allow_s3) # takes BaseCompute init
         self.file_filters = ['.zarr.zip'] # this only runs on zarr.zip for the moment.
         # default parameters
@@ -359,7 +365,23 @@ class CaimanCompute(Suite2pCompute):
             print(f'[{self.name} job] Kept the temporary folder {temporary_folder}.')
         self.schema.CellSegmentationMetrics().populate(dkey, display_progress = True)
         
-def setup_cluster(n_cpus,cluster = None):
+def setup_cluster(n_cpus, cluster=None):
+    '''Start (or restart) a CaImAn multiprocessing cluster.
+
+    If ``cluster`` is not ``None``, the existing cluster is stopped first.
+
+    Parameters
+    ----------
+    n_cpus : int
+        Number of parallel worker processes.
+    cluster : object or None, optional
+        Existing CaImAn cluster object to stop before starting a new one.
+
+    Returns
+    -------
+    object
+        The newly created CaImAn cluster (``dview``).
+    '''
     from caiman import stop_server, cluster
     #%% start a cluster for parallel processing (if a cluster already exists it will be closed and a new session will be opened)
     if not cluster is None:  # 'locals' contains list of current local variables