labdata 0.1.22__tar.gz → 0.1.23__tar.gz

{labdata-0.1.22 → labdata-0.1.23}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: labdata
-Version: 0.1.22
+Version: 0.1.23
 Summary: Package to manage data in experimental neuroscience labs
 Author-email: Joao Couto <jpcouto@gmail.com>
 Project-URL: Homepage, https://github.com/jcouto/labdata

{labdata-0.1.22 → labdata-0.1.23}/labdata/__init__.py

@@ -1,4 +1,4 @@
-VERSION = "0.1.22"
+VERSION = "0.1.23"
 
 from .utils import *
 from .copy import copy_to_upload_server

{labdata-0.1.22 → labdata-0.1.23}/labdata/cli.py

@@ -615,7 +615,9 @@ Administration commands (don't run on experimental computers):
             whose output goes to ``task_log`` and ``~/.labdata/logs``,
             instead of in the terminal.
         --queue
-            Print the current job queue for all targets and exit.
+            Print the scheduler queue of each target, then the ``ComputeTask``
+            table by status with the last line of each task log, and exit.
+            With an analysis name, only the tasks of that analysis.
         --delete-computes
             Delete all ``ComputeTask`` entries for this analysis (careful).
         --
@@ -666,6 +668,8 @@ Available analysis are
             targets = np.unique([prefs['compute']['default_target']]+
                                 list(prefs['compute']['remotes'].keys()))
             for q in targets:
+                if q == 'local':  # no scheduler queue, the tasks are in the table below
+                    continue
                 queue = check_queue(target = q)
                 if queue is None:
                     continue
@@ -678,7 +682,14 @@ Available analysis are
                         cc = tcolor['g']
                     else:
                         cc = tcolor['r']
-                    print(f"\t {j['jobid']} {j['name']} {cc('running')} time: {tcolor['y'](j['time'])}\n\t\tpartition: {j['partition']} node: {j['nodelist(reason)']} ")
+                    print(f"\t {j['jobid']} {j['name']} {cc(j['state'].lower())} time: {tcolor['y'](j['time'])}\n\t\tpartition: {j['partition']} node: {j['nodelist(reason)']} ")
+            # the ComputeTask table, optionally for one analysis: labdata run <analysis> --queue
+            qparser = argparse.ArgumentParser(add_help = False)
+            qparser.add_argument('analysis', nargs = '?', default = None)
+            qparser.add_argument('-p','--project', default = _get_project())
+            qargs, _ = qparser.parse_known_args(argum)
+            from .utils import load_project_schema
+            self._print_compute_tasks(load_project_schema(qargs.project), analysis = qargs.analysis)
             return
         args = parser.parse_args(argum)
         if '--delete-computes' in argum:
@@ -2051,7 +2062,54 @@ examples:
                                 type = str,
                                 help= f'Select project: {project}')
         return parser
-        
+
+    def _print_compute_tasks(self, schema, analysis = None, limit = 10):
+        '''Print the ComputeTask table by status, with the last line of each task log.
+
+        Shows the most recent ``limit`` tasks of each status and the count of the rest.
+        '''
+        tasks = schema.ComputeTask()
+        if not analysis is None:
+            tasks = tasks & dict(task_name = analysis)
+        title = 'ComputeTask table' + (f' [{analysis}]' if not analysis is None else '')
+        print(tcolor['m'](f"++++++++++++++++++++ {title} ++++++++++++++++++++"))
+        if not len(tasks):
+            print('\t no compute tasks.')
+            return
+        statuses = [('WORKING', tcolor['c']), ('WAITING', tcolor['y']), ('ON SERVER', tcolor['b']),
+                    ('FAILED', tcolor['r']), ('CANCELED', tcolor['o']), ('COMPLETED', tcolor['g'])]
+        def _brief(log, width = 100):
+            if log is None:
+                return ''
+            if isinstance(log, bytes):
+                log = log.decode('utf-8', errors = 'replace')
+            lines = [l.strip() for l in str(log).splitlines() if len(l.strip())]
+            line = ' '.join(lines[-1].split()) if len(lines) else ''
+            return line if len(line) <= width else line[:width-3] + '...'
+        def _time(t):
+            return t.strftime('%Y-%m-%d %H:%M') if not t is None else ''
+        for status, cc in statuses:
+            sel = tasks & f'task_status LIKE "%{status}%"'
+            n = len(sel)
+            if not n:
+                continue
+            print(cc(f"{status} ({n})"))
+            rows = sel.fetch('job_id', 'task_name', 'task_target', 'task_host', 'subject_name',
+                             'session_name', 'task_starttime', 'task_endtime', 'task_log',
+                             as_dict = True, order_by = 'job_id DESC', limit = limit)
+            for r in rows:
+                where = '/'.join([str(r[k]) for k in ('subject_name', 'session_name') if not r[k] is None])
+                when = _time(r['task_endtime'] if status in ('COMPLETED', 'FAILED', 'CANCELED') and
+                             not r['task_endtime'] is None else r['task_starttime'])
+                print(f"\t {r['job_id']} {r['task_name']} {where} "
+                      f"target: {r['task_target']} host: {r['task_host']}" +
+                      (f" {tcolor['y'](when)}" if len(when) else ''))
+                log = _brief(r['task_log'])
+                if len(log):
+                    print(f"\t\t{log}")
+            if n > limit:
+                print(f"\t ... and {n - limit} more")
+
     def _get_default_arg(self,argument,cli_arg = 'submit', default = None):
         # checks if there is a default in the options
         if not f'{cli_arg}_defaults' in labdata_preferences.keys():

{labdata-0.1.22 → labdata-0.1.23}/labdata/compute/ephys.py

@@ -35,7 +35,7 @@ class SpksCompute(BaseCompute):
                                low_pass = 300.,
                                high_pass = 13000.)
         self.parameter_keys = ['motion_correction','low_pass','high_pass','thresholds','remove_cross_duplicates',
-                               'waveforms_from_input','dredge']
+                               'waveforms_from_input','dredge','remove_channels_without_signal']
         # the parameters that go on the SpikeSortingParams
         self.use_hdf5 = True  # flag to use h5py or zarr format for the waveforms.
         self.parameter_set_num = None # identifier in SpikeSortingParams
@@ -131,9 +131,15 @@ class SpksCompute(BaseCompute):
                                 action='store', default = None, nargs=2, type = float,
                                 help = "Interval to sort in seconds: start end")
             parser.add_argument('--dredge',
-                                action='store_true', default = False, 
+                                action='store_true', default = False,
                                 help = "Motion correction using dredge")
-            
+            parser.add_argument('--remove-channels-without-signal',
+                                action='store', default = None, nargs = '*', type = str,
+                                choices = ['amplitude','correlation','coherence'],
+                                help = "Leave out the channels without signal before sorting and the common average reference. "
+                                "Methods: amplitude (low MAD in the sorting band), correlation and coherence (low correlation or phase coherence "
+                                "with the channels within 300 um on the same shank, LFP band). Without methods: amplitude coherence.")
+
             args = parser.parse_args(arguments[1:])
             if 'ks2.5' in  args.method: # defaults for ks2.5
                 self.parameters = dict(algorithm_name = 'spks_kilosort2.5',
@@ -163,7 +169,12 @@ class SpksCompute(BaseCompute):
                 self.parameters['waveforms_from_input'] = False 
             if args.dredge:
                 # default is to extract from the original input file
-                self.parameters['dredge'] = True 
+                self.parameters['dredge'] = True
+            if not args.remove_channels_without_signal is None:
+                methods = args.remove_channels_without_signal
+                if not len(methods):
+                    methods = ['amplitude','coherence']
+                self.parameters['remove_channels_without_signal'] = sorted(set(methods))
             print(args)
             if not args.thresholds is None:
                 self.parameters['thresholds'] = args.thresholds
@@ -294,6 +305,13 @@ class SpksCompute(BaseCompute):
             dredge_motion_correction = False
             if 'dredge' in self.parameters.keys():
                 dredge_motion_correction = self.parameters['dredge']
+            preprocessing = dict()
+            if self.parameters.get('remove_channels_without_signal', None):
+                # channels without signal are left out before the filters and the common average reference
+                from spks.raw import default_filter_pipeline_par
+                preprocessing['filter_pipeline_par'] = [dict(function = 'remove_channels_without_signal',
+                                                             methods = self.parameters['remove_channels_without_signal'])] + [
+                                                                 dict(d) for d in default_filter_pipeline_par]
             # run kilosort using spks for preprocessing.
             if self.parameters['algorithm_name'] == 'spks_kilosort2.5':      
                 results_folder = run_kilosort(version = '2.5',sessionfiles = probepath,
@@ -303,7 +321,8 @@ class SpksCompute(BaseCompute):
                                               thresholds = self.parameters['thresholds'],
                                               lowpass = self.parameters['low_pass'],
                                               highpass = self.parameters['high_pass'],
-                                              dredge_motion_correction = dredge_motion_correction)
+                                              dredge_motion_correction = dredge_motion_correction,
+                                              **preprocessing)
             elif self.parameters['algorithm_name'] == 'spks_kilosort3.0':      
                 results_folder = run_kilosort(version = '3.0',
                                               sessionfiles = probepath,
@@ -313,7 +332,8 @@ class SpksCompute(BaseCompute):
                                               thresholds = self.parameters['thresholds'],
                                               lowpass = self.parameters['low_pass'],
                                               highpass = self.parameters['high_pass'],
-                                              dredge_motion_correction = dredge_motion_correction)
+                                              dredge_motion_correction = dredge_motion_correction,
+                                              **preprocessing)
             elif self.parameters['algorithm_name'] == 'spks_kilosort4.0':      
                 results_folder = run_kilosort(version = '4.0',
                                               sessionfiles = probepath,
@@ -323,7 +343,8 @@ class SpksCompute(BaseCompute):
                                               thresholds = self.parameters['thresholds'],
                                               lowpass = self.parameters['low_pass'],
                                               highpass = self.parameters['high_pass'],
-                                              dredge_motion_correction = dredge_motion_correction)
+                                              dredge_motion_correction = dredge_motion_correction,
+                                              **preprocessing)
             elif self.parameters['algorithm_name'] == 'spks_mountainsort5':
                 raise(NotImplemented(f"[{self.name} job] - Algorithm {self.parameters['algorithm_name']} not implemented."))
             else:
@@ -416,6 +437,10 @@ class SpksCompute(BaseCompute):
             del clu
         clu = Clusters(results_folder, get_waveforms = False, get_metrics = False)
         clu.compute_template_amplitudes_and_depths()
+        # channels of the sorting in the numbering of the recorded file (channels can be left out before sorting)
+        sorting_channel_indices = clu.channel_map.flatten()
+        if not clu.metadata is None and 'original_channels' in clu.metadata.keys():
+            sorting_channel_indices = np.asarray(clu.metadata['original_channels']).flatten()[sorting_channel_indices]
         base_keys = []
         ssdict = [] # sortings
         udict = [] # unit
@@ -439,7 +464,7 @@ class SpksCompute(BaseCompute):
                                n_detected_spikes = len(spikes),
                                sorting_datetime = datetime.fromtimestamp(
                                    Path(results_folder).stat().st_ctime),
-                               sorting_channel_indices = clu.channel_map.flatten(),
+                               sorting_channel_indices = sorting_channel_indices,
                                sorting_channel_coords = clu.channel_positions))
             udict.append([]) # list of lists
             for iclu in np.unique(clusters):
@@ -569,10 +594,14 @@ class SpksCompute(BaseCompute):
             dat = map_binary(binaryfile, nchannels = nchannels)
             nsamples = int(clu.sampling_rate*2)
             offset_samples = int(np.random.uniform(offsets[0]+nsamples, offsets[1]-nsamples-1))
+            segment = np.array(dat[offset_samples : offset_samples + nsamples])
+            # statistics in microvolt with the gains of the waveforms, UnitMetrics reads the noise from here
+            from ..rules.ephys import channel_statistics
             self.schema.SpikeSorting.Segment.insert1(dict(base_key,
                                             segment_num = 1,
                                             offset_samples = offset_samples - offsets[0],
-                                            segment = np.array(dat[offset_samples : offset_samples + nsamples])))
+                                            segment = segment,
+                                            **channel_statistics(segment.astype(np.float32)*clu.channel_gains)))
             del dat
         self.set_job_status(job_log = f'Completed {base_key}')
         from labdata.schema import UnitMetrics
@@ -632,7 +661,9 @@ def select_random_waveforms(unit_dict, wpre=45, wpost=45, duration=None, nmax_wa
 
     Adds a ``waveform_indices`` array to each unit dict, drawing up to
     ``nmax_waves`` spikes from the first quarter and last quarter of the
-    recording.  Spikes too close to the file boundaries (within ``wpre`` or
+    recording.  A unit with no spikes in either quarter gets up to
+    ``2*nmax_waves`` spikes from the whole recording.  Spikes too close to
+    the file boundaries (within ``wpre`` or
     ``wpost`` samples) are excluded.
 
     Parameters
@@ -668,6 +699,13 @@ def select_random_waveforms(unit_dict, wpre=45, wpost=45, duration=None, nmax_wa
             sel += [t for t in np.random.choice(s_end, nmax_waves, replace=True)]
         else:
             sel += [t for t in s_end]
+        if not len(sel):
+            # the unit fires only in the middle of the recording, take the spikes from anywhere
+            s_all = s[(s>(wpre+2)) & (s<(duration-2*wpost))]
+            if len(s_all)>2*nmax_waves:
+                sel = [t for t in np.random.choice(s_all, 2*nmax_waves, replace=True)]
+            else:
+                sel = [t for t in s_all]
         u['waveform_indices'] = np.sort(np.array(sel).flatten()) # add this to the  
     return unit_dict
 

{labdata-0.1.22 → labdata-0.1.23}/labdata/compute/utils.py

@@ -598,16 +598,21 @@ class BaseCompute():
                                         job_waiting = 0)
 
                     def cleanup_function(job_id=self.job_id):
-                        '''Put the task back in the queue (CANCELLED, waiting) if the process stops while it works.'''
-                        status = (self.schema.ComputeTask() & dict(job_id = job_id)).fetch(as_dict = True)[0]
-                        if status['task_status'] in ['WORKING']:
-                            # printed, so labdata logpipe keeps the reason in task_log
-                            print(f'[task {job_id}] INTERRUPTED on {prefs["hostname"]} (the process was stopped); '
-                                  'the task is back in the queue.', flush = True)
-                            self.schema.ComputeTask.update1(dict(job_id = job_id,
-                                                                 task_status = 'CANCELLED',
-                                                                 task_waiting = 1,
-                                                                 task_endtime = datetime.now()))
+                        '''Put the task back in the queue (CANCELLED, waiting) if the process stops while it works.
+
+                        Runs from a signal handler, maybe while the task waits on a query: it writes on a
+                        connection of its own, and to the output directly (labdata logpipe keeps the line).'''
+                        from ..rules.utils import _new_connection
+                        try:
+                            conn = _new_connection()
+                            conn.query(f'''UPDATE {self.schema.ComputeTask.full_table_name}
+                                           SET task_status = "CANCELLED", task_waiting = 1, task_endtime = NOW()
+                                           WHERE job_id = %s AND task_status = "WORKING"''', args = (job_id,))
+                            conn.close()
+                            os.write(1, (f'[task {job_id}] INTERRUPTED on {prefs["hostname"]} (the process was stopped); '
+                                         'the task is back in the queue.\n').encode())
+                        except Exception as err:
+                            os.write(2, f'[task {job_id}] Could not put the task back in the queue: {err}\n'.encode())
 
                     self.cleanup_function = cleanup_function
                     self.register_safe_exit()

{labdata-0.1.22 → labdata-0.1.23}/labdata/dashboard/compute.py

@@ -382,7 +382,7 @@ def _queue_view(schema):
         status_field='task_status',
         target_field='task_target',
         cols=['job_id', 'task_name', 'task_status', 'task_host', 'task_target',
-              'task_waiting', 'task_log', 'subject_name', 'session_name',
+              'task_waiting', 'task_cmd', 'task_log', 'subject_name', 'session_name',
               'task_starttime', 'task_endtime'],
         delete_key='compute',
         resubmit_fn=lambda job_ids, target_override=None: _resubmit_jobs(schema, job_ids, project, target_override=target_override),

{labdata-0.1.22 → labdata-0.1.23}/labdata/rules/ephys.py

@@ -278,13 +278,25 @@ def ephys_noise_statistics_from_file(filepath, channel_indices, gain,
                  channel_mad = np.zeros((len(channel_indices),len(dd))),
                  channel_max = np.zeros((len(channel_indices),len(dd))),
                  channel_min = np.zeros((len(channel_indices),len(dd))))
-    from scipy.stats import median_abs_deviation
     for i,d in enumerate(dd):
-        res['channel_mad'][:,i] = median_abs_deviation(d[:,channel_indices],axis = 0) 
-        res['channel_max'][:,i] = np.max(d[:,channel_indices],axis = 0) 
-        res['channel_min'][:,i] = np.min(d[:,channel_indices],axis = 0) 
-        res['channel_median'][:,i] = np.median(d[:,channel_indices],axis = 0) 
-        res['channel_peak_to_peak'][:,i] = res['channel_max'][:,i]-res['channel_min'][:,i]
+        for k,v in channel_statistics(d[:,channel_indices]).items():
+            res[k][:,i] = v
+    return res
+
+def channel_statistics(data):
+    '''Return the peak-to-peak, median, MAD, max and min of each channel.
+
+    ``data`` is samples by channels, usually in microvolt.  The MAD is the
+    unscaled median absolute deviation; divide it by 0.6745 for the standard
+    deviation of Gaussian noise.  Each value of the returned dict has one entry
+    per channel.
+    '''
+    from scipy.stats import median_abs_deviation
+    res = dict(channel_mad = median_abs_deviation(data,axis = 0),
+               channel_max = np.max(data,axis = 0),
+               channel_min = np.min(data,axis = 0),
+               channel_median = np.median(data,axis = 0))
+    res['channel_peak_to_peak'] = res['channel_max']-res['channel_min']
     return res
 
 def extract_events_from_nidq(paths):

{labdata-0.1.22 → labdata-0.1.23}/labdata/rules/utils.py

@@ -116,21 +116,27 @@ class UploadRule():
             self._release_interrupt_handler()
 
     def _interrupted(self):
-        '''Mark the job FAILED when the process stops while it is WORKING.'''
+        '''Mark the job FAILED when the process stops while it is WORKING.
+
+        This runs from a signal handler, maybe while the job waits on a query:
+        it writes on a connection of its own and to the output file directly,
+        since the job's connection and ``print`` may be in use.
+        '''
         from ..schema import UploadJob
+        if self._stage_stop is not None:
+            self._stage_stop.set()   # no more progress lines
+        host = self.prefs['hostname']
         try:
-            job = (UploadJob() & dict(job_id = self.job_id)).fetch(as_dict = True)
-            if len(job) and job[0]['job_status'] == 'WORKING':
-                last = (job[0]['job_log'] or '')[:300]
-                if self._stage_stop is not None:
-                    self._stage_stop.set()   # no more progress lines
-                self._stage = None
-                self.set_job_status(job_status = 'FAILED', job_endtime = datetime.now(),
-                                    job_log = f'INTERRUPTED on {self.prefs["hostname"]} '
-                                              f'(the process was stopped) during: {last}')
-                print(f'Job {self.job_id} was interrupted; marked FAILED.', flush = True)
+            conn = _new_connection()
+            conn.query(f'''UPDATE {UploadJob.full_table_name}
+                           SET job_status = "FAILED", job_waiting = 0, job_endtime = NOW(), job_host = %s,
+                               job_log = LEFT(CONCAT("INTERRUPTED on ", %s, " (the process was stopped) during: ",
+                                                     COALESCE(job_log, "")), 500)
+                           WHERE job_id = %s AND job_status = "WORKING"''', args = (host, host, self.job_id))
+            conn.close()
+            os.write(1, f'Job {self.job_id}: interrupted; marked FAILED if it was working.\n'.encode())
         except Exception as err:
-            print(f'Could not mark job {self.job_id} as interrupted: {err}', flush = True)
+            os.write(2, f'Could not mark job {self.job_id} as interrupted: {err}\n'.encode())
 
     def _hold_interrupt_handler(self):
         import safe_exit