labdata 0.1.16__tar.gz → 0.1.18__tar.gz

{labdata-0.1.16 → labdata-0.1.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: labdata
-Version: 0.1.16
+Version: 0.1.18
 Summary: Package to manage data in experimental neuroscience labs
 Author-email: Joao Couto <jpcouto@gmail.com>
 Project-URL: Homepage, https://github.com/jcouto/labdata

{labdata-0.1.16 → labdata-0.1.18}/labdata/__init__.py

@@ -1,4 +1,4 @@
-VERSION = "0.1.16"
+VERSION = "0.1.18"
 
 from .utils import *
 from .copy import copy_to_upload_server

{labdata-0.1.16 → labdata-0.1.18}/labdata/cli.py

@@ -625,16 +625,21 @@ Available analysis are
             return
         from .compute import parse_analysis, run_analysis
         # parse analysis will check if the analysis is defined
-        jobids,obj = parse_analysis(analysis = args.analysis,
-                                    job_id = args.job, 
-                                    subject = args.subject,
-                                    session = args.session,
-                                    datatype = args.datatype,
-                                    secondary_args = secondary_args,
-                                    multisession = args.multisession,
-                                    force_submit = args.force_submit,
-                                    full_command = ' '.join(sys.argv[1:]),
-                                    project = args.project)
+        try:
+            jobids,obj = parse_analysis(analysis = args.analysis,
+                                        job_id = args.job, 
+                                        subject = args.subject,
+                                        session = args.session,
+                                        datatype = args.datatype,
+                                        secondary_args = secondary_args,
+                                        multisession = args.multisession,
+                                        force_submit = args.force_submit,
+                                        full_command = ' '.join(sys.argv[1:]),
+                                        project = args.project)
+        except (ValueError, NotImplementedError) as err:
+            # what the request asked for is not there: say what, and stop
+            print(f'{tcolor["r"]("Nothing to run:")} {err}')
+            sys.exit(1)
         if not len(jobids):
             print('Nothing to run.')
             return

{labdata-0.1.16 → labdata-0.1.18}/labdata/compute/pose.py

@@ -159,7 +159,19 @@ class DeeplabcutCompute(BaseCompute):
             raise(NotImplementedError('Specifying no session is not yet implemented'))
         datasets = []
         for k in keys:
-            datasets += (self.schema.DatasetVideo()& k).fetch(as_dict = True)
+            videos = (self.schema.DatasetVideo() & k).fetch(as_dict = True)
+            if not len(videos):
+                raise ValueError(f'Session {k["session_name"]} of {k["subject_name"]} has no video '
+                                 '(nothing in DatasetVideo); videos are registered when the '
+                                 'session is uploaded.')
+            video_name = self.parameters.get('video_name')
+            if video_name is not None:
+                names = sorted({v['video_name'] for v in videos})
+                videos = [v for v in videos if v['video_name'] == video_name]
+                if not len(videos):
+                    raise ValueError(f'Session {k["session_name"]} of {k["subject_name"]} has no video '
+                                     f'named {video_name}; it has {", ".join(names)}.')
+            datasets += videos
         datasets = [dict(subject_name = d['subject_name'],
                          session_name = d['session_name'],
                          dataset_name = d['dataset_name']) for d in datasets]

{labdata-0.1.16 → labdata-0.1.18}/labdata/compute/utils.py

@@ -1,13 +1,36 @@
 from ..utils import *
 import traceback
 
-def load_analysis_object(analysis):
+def available_analyses(project=None):
+    '''The compute classes of the preference file that are loaded, by analysis name.
+
+    An entry whose class is not there, because it comes from a plugin that is
+    not loaded, or is not a compute class at all, is left out.  With
+    ``project`` the project's plugins are loaded first.
+    '''
+    import labdata
+    if project is not None:
+        load_project_schema(project)
+    analyses = {}
+    for name, path in prefs['compute']['analysis'].items():
+        try:
+            obj = eval(path)
+        except (AttributeError, NameError):
+            continue
+        if isinstance(obj, type) and issubclass(obj, BaseCompute):
+            analyses[name] = obj
+    return analyses
+
+
+def load_analysis_object(analysis, project=None):
     '''Load a compute class by its analysis name from user preferences.
 
     Parameters
     ----------
     analysis : str
         Key in ``prefs['compute']['analysis']`` (e.g. ``'spks'``, ``'suite2p'``).
+    project : str or None, optional
+        Project whose plugins to load before looking for the class.
 
     Returns
     -------
@@ -17,23 +40,22 @@ def load_analysis_object(analysis):
     Raises
     ------
     SystemExit
-        If the analysis name is not found in preferences, prints a help message
-        listing available analyses and exits.
+        If the analysis is not among the loaded ones, prints a help message
+        listing the analyses that are and exits.
     '''
-    if not analysis in prefs['compute']['analysis'].keys():
+    analyses = available_analyses(project)
+    if analysis not in analyses:
+        path = prefs['compute']['analysis'].get(analysis)
+        where = f' for project {project}' if project else ''
+        because = (f' ({path} is not loaded{where}, or is not a compute class)' if path else '')
         print(f'''
-Could not find {tcolor['r'](analysis)} analysis.
-
-    The analysis are {tcolor['g'](','.join(prefs["compute"]["analysis"].keys()))}
-
+Could not find {tcolor['r'](analysis)} analysis{because}.
+    The analyses are {tcolor['g'](','.join(analyses.keys()))}
 Add the analysis to the "{tcolor['y']("compute")}" section of the preference file {tcolor['y']("{analysis_name:analysis_object}")}.
-
 ''')
-        sys.exit()        
-        raise ValueError('Add the analysis to the "compute" section of the preference file {analysis_name:analysis_object}.\n\n')
-        
-    import labdata
-    return eval(prefs['compute']['analysis'][analysis])
+        sys.exit(1)
+    return analyses[analysis]
+
 
 def handle_compute(job_id, project=None):
     '''Instantiate and return the correct compute object for a queued job.
@@ -61,7 +83,7 @@ def handle_compute(job_id, project=None):
     jobinfo = jobinfo.iloc[0]
     if jobinfo.task_waiting == 0:
         print(f'Task {job_id} is running on {jobinfo.task_host}')
-    obj = load_analysis_object(jobinfo.task_name)(jobinfo.job_id,project)
+    obj = load_analysis_object(jobinfo.task_name, project=project)(jobinfo.job_id,project)
     return obj
 
 def parse_analysis(analysis, job_id=None,
@@ -115,7 +137,7 @@ def parse_analysis(analysis, job_id=None,
     obj : BaseCompute
         Instantiated compute object.
     '''
-    obj = load_analysis_object(analysis)(job_id, project=project)
+    obj = load_analysis_object(analysis, project=project)(job_id, project=project)
     obj.secondary_parse(secondary_args, parameter_number)
     if obj.job_id is None:
         # then we have to create jobs and assign
@@ -123,6 +145,9 @@ def parse_analysis(analysis, job_id=None,
             schema = load_project_schema(project)
         else:
             schema = obj.schema
+        # a subject or session that is not in the project is the usual reason
+        # for finding nothing; say so, with the closest names, before looking
+        check_subject_and_session(schema, subject, session, project = project)
          # first check if there is a task that has already been submitted with the exact same command.
          # this has a caveat: if the order of the arguments is switched, it wont work..
         if not full_command is None:
@@ -135,6 +160,12 @@ def parse_analysis(analysis, job_id=None,
                 return [], None
         if not subject is None or not session is None:
             datasets = obj.find_datasets(subject_name = subject, session_name = session)
+            if datasets is not None and not len(datasets):
+                asked = ' '.join(f'{name} {", ".join(_as_names(v))}' for name, v in
+                                 (('subject', subject), ('session', session)) if _as_names(v))
+                print(tcolor['r'](f'No dataset for {analysis} matches {asked}.')
+                      + _datasets_on_record(schema, subject, session))
+                return [], obj
         else:
             datasets = None
         job_ids = obj.place_tasks_in_queue(datasets,
@@ -144,6 +175,60 @@ def parse_analysis(analysis, job_id=None,
         # now we have the job ids, need to figure out how to launch the jobs
         return job_ids, obj # returns the job ids and the task
 
+def _as_names(value):
+    '''The names in a command line argument: a string, a list of strings, or None.'''
+    if value is None:
+        return []
+    if isinstance(value, str):
+        value = [value]
+    return [str(v) for v in value if str(v).strip()]
+
+
+def _datasets_on_record(schema, subject, session):
+    '''One line saying what datasets the requested sessions do have, or nothing.'''
+    keys = [dict(subject_name = a, session_name = s) for a in _as_names(subject) for s in _as_names(session)]
+    if not keys:
+        return ''
+    rows = (schema.Dataset() & keys).fetch('session_name', 'dataset_name', 'dataset_type', as_dict = True)
+    if not rows:
+        return ' The session has no dataset on record.'
+    have = sorted({f'{r["dataset_name"]} ({r["dataset_type"] or "no type"})' for r in rows})
+    return f' On record: {", ".join(have)}.'
+
+
+def check_subject_and_session(schema, subject=None, session=None, project=None):
+    '''Raise a ValueError naming what is not in the project, before a compute looks for datasets.
+
+    A subject that is not in ``Subject`` is reported with the closest names
+    on record; a session that the subject does not have is reported with the
+    subject's latest sessions.  Nothing happens when neither is given.
+    '''
+    import difflib
+    where = f'project {project}' if project else 'the project'
+    subjects = _as_names(subject)
+    sessions = _as_names(session)
+    if not subjects and not sessions:
+        return
+    known = list(schema.Subject().fetch('subject_name'))
+    for name in subjects:
+        if name not in known:
+            close = difflib.get_close_matches(name, known, n = 5, cutoff = 0.6)
+            hint = f' Close names: {", ".join(close)}.' if close else ''
+            raise ValueError(f'Subject {name} is not in {where}.{hint}')
+    if sessions and not subjects:
+        raise ValueError('A session needs its subject: pass the subject name as well.')
+    for name in subjects:
+        have = list((schema.Session() & dict(subject_name = name)).fetch(
+            'session_name', order_by = 'session_datetime DESC'))
+        for ses in sessions:
+            if ses not in have:
+                close = difflib.get_close_matches(ses, have, n = 3, cutoff = 0.6)
+                hint = (f' Close names: {", ".join(close)}.' if close else
+                        f' Its latest sessions: {", ".join(have[:5])}.' if have else
+                        ' It has no sessions.')
+                raise ValueError(f'Subject {name} has no session {ses} in {where}.{hint}')
+
+
 def check_archived(jobid, check_local=False, schema=None):
     '''Check whether files assigned to a compute job are in S3 Glacier.
 
@@ -591,7 +676,9 @@ class BaseCompute():
                     idx += list(filter(lambda x: not x is None,[i if f in s else None for i,s in enumerate(
                         files.file_path.values)]))
                 if len(idx) == 0:
-                    raise ValueError(f'Could not find valid Dataset.DataFiles for {dataset}')
+                    found = ', '.join(Path(f).name for f in files.file_path.values[:8]) if len(files) else 'no files at all'
+                    raise ValueError(f'{self.name} needs a file of type {", ".join(self.file_filters)} '
+                                     f'and dataset {dataset} has {found}.')
                 files = files.iloc[idx]
                 if type(dataset) is dict:
                     key = dict(dataset,task_name = self.name)

{labdata-0.1.16 → labdata-0.1.18}/labdata/dashboard/admin.py

@@ -3,6 +3,7 @@ from datetime import datetime, timedelta
 
 import pandas as pd
 import streamlit as st
+from writes import log
 import plotly.express as px
 
 
@@ -285,8 +286,10 @@ def _add_user_dialog(project, user):
             add_user_to_project(project, user)
             get_project_users.clear()
             st.session_state['admin_msg'] = ('success', f'Added {user} to {project}.')
+            log(f'added user {user} to project {project}')
         except Exception as e:
             st.session_state['admin_msg'] = ('error', str(e))
+            log(f'could not add user {user} to project {project}: {e}')
         st.rerun()
     if col2.button("Cancel", key="add_cancel"):
         st.rerun()
@@ -304,8 +307,10 @@ def _remove_user_dialog(project, user):
             remove_user_from_project(project, user)
             get_project_users.clear()
             st.session_state['admin_msg'] = ('success', f'Removed {user} from {project}.')
+            log(f'removed user {user} from project {project}')
         except Exception as e:
             st.session_state['admin_msg'] = ('error', str(e))
+            log(f'could not remove user {user} from project {project}: {e}')
         st.rerun()
     if col2.button("Cancel", key="remove_cancel"):
         st.rerun()

{labdata-0.1.16 → labdata-0.1.18}/labdata/dashboard/compute.py

@@ -1,6 +1,7 @@
 import pandas as pd
 import numpy as np
 import streamlit as st
+from writes import write, show_flash, log
 
 STATES = ['COMPLETED', 'WORKING', 'WAITING', 'FAILED', 'CANCELED','ON SERVER']
 STATE_COLORS = ['#01D186', '#48CAE4', '#F3A21C', '#FF5733', '#F3A21C','#F3A21C']
@@ -50,6 +51,21 @@ def _resubmit_jobs(schema, job_ids, project, target_override=None):
     return errors
 
 
+def _fetch_rows(table):
+    '''Fetch all rows of a job table, reconnecting once if the first attempt fails.
+
+    The dashboard shares one database connection between browser sessions, and
+    the client can be left half closed (a stale socket, a packet sequence
+    error, or ``'NoneType' object has no attribute 'read'`` from pymysql).
+    Such failures clear on a fresh connection, so retry once before giving up.
+    '''
+    try:
+        return table.fetch(as_dict=True)
+    except Exception:
+        table.connection.connect()
+        return table.fetch(as_dict=True)
+
+
 def _jobs_panel(table, status_field, target_field, cols, delete_key,
                 resubmit_fn=None, allow_delete_completed=False, on_server_fn=None):
 
@@ -59,20 +75,13 @@ def _jobs_panel(table, status_field, target_field, cols, delete_key,
     if st.session_state.get(f'{delete_key}_resubmit_errors'):
         for e in st.session_state.pop(f'{delete_key}_resubmit_errors'):
             st.error(e)
+    show_flash(f'{delete_key}_msg')
 
     try:
-        rows = table.fetch(as_dict=True)
+        rows = _fetch_rows(table)
     except Exception as e:
-        if 'Packet sequence' in str(e) or 'sequence number wrong' in str(e):
-            try:
-                table.connection.connect()
-                rows = table.fetch(as_dict=True)
-            except Exception as e2:
-                st.error(f'Database connection error: {e2}')
-                return
-        else:
-            st.error(f'Error fetching jobs: {e}')
-            return
+        st.error(f'Could not fetch the jobs: {e}')
+        return
     if not rows:
         st.info('There are no jobs in the **ComputeTask** table.')
         return
@@ -95,9 +104,14 @@ def _jobs_panel(table, status_field, target_field, cols, delete_key,
             st.warning(f'Delete all {n} COMPLETED job(s)? This cannot be undone.')
             col_yes, col_no, _ = st.columns([1, 1, 4])
             if col_yes.button('Yes, delete', type='primary', key=f'{delete_key}_del_completed_yes'):
-                (table & f'{status_field} = "COMPLETED"').delete(safemode=False)
+                ok, _ = write(table, 'delete the completed jobs',
+                              lambda: (table & f'{status_field} = "COMPLETED"').delete(
+                                  safemode=False, transaction=False),
+                              success=f'Deleted {n} completed job(s).',
+                              flash_key=f'{delete_key}_msg')
                 st.session_state.pop(f'{delete_key}_del_completed_pending', None)
-                st.rerun()
+                if ok:
+                    st.rerun()
             if col_no.button('Cancel', key=f'{delete_key}_del_completed_no'):
                 st.session_state.pop(f'{delete_key}_del_completed_pending', None)
                 st.rerun()
@@ -175,9 +189,14 @@ def _jobs_panel(table, status_field, target_field, cols, delete_key,
         st.warning(f'Delete all {len(jobs_to_delete)} filtered job(s)? This cannot be undone.')
         col_yes, col_no, _ = st.columns([1, 1, 4])
         if col_yes.button('Yes, delete', type='primary', key=f'{delete_key}_del_filtered_yes'):
-            (table & [dict(job_id=j) for j in jobs_to_delete]).delete(safemode=False)
+            ok, _ = write(table, 'delete the filtered jobs',
+                          lambda: (table & [dict(job_id=j) for j in jobs_to_delete]).delete(
+                              safemode=False, transaction=False),
+                          success=f'Deleted {len(jobs_to_delete)} job(s).',
+                          flash_key=f'{delete_key}_msg')
             st.session_state.pop(f'{delete_key}_del_filtered_pending', None)
-            st.rerun()
+            if ok:
+                st.rerun()
         if col_no.button('Cancel', key=f'{delete_key}_del_filtered_no'):
             st.session_state.pop(f'{delete_key}_del_filtered_pending', None)
             st.rerun()
@@ -221,10 +240,14 @@ def _jobs_panel(table, status_field, target_field, cols, delete_key,
             st.warning(f'Are you sure you want to delete {len(jobs_to_delete)} job(s)? This cannot be undone and scheduled jobs might fail.')
             col_yes, col_no, _ = st.columns([1, 1, 4])
             if col_yes.button('Yes, delete', type='primary', key=f'{delete_key}_yes'):
-                (table & [dict(job_id=j) for j in jobs_to_delete]).delete(safemode=False)
+                ok, _ = write(table, 'delete the selected jobs',
+                              lambda: (table & [dict(job_id=j) for j in jobs_to_delete]).delete(
+                                  safemode=False, transaction=False),
+                              success=f'Deleted {len(jobs_to_delete)} job(s).',
+                              flash_key=f'{delete_key}_msg')
                 st.session_state.pop(f'{delete_key}_pending', None)
-                st.success(f'Deleted {len(jobs_to_delete)} job(s).')
-                st.rerun()
+                if ok:
+                    st.rerun()
             if col_no.button('Cancel', key=f'{delete_key}_no'):
                 st.session_state.pop(f'{delete_key}_pending', None)
                 st.rerun()
@@ -245,6 +268,13 @@ def _jobs_panel(table, status_field, target_field, cols, delete_key,
                 st.session_state.pop(f'{delete_key}_resubmit_pending', None)
                 if errors:
                     st.session_state[f'{delete_key}_resubmit_errors'] = errors
+                    log(f'could not re-submit {len(errors)} of {len(jobs_to_resubmit)} job(s): '
+                        + '; '.join(str(e) for e in errors))
+                else:
+                    st.session_state[f'{delete_key}_msg'] = (
+                        f'Re-submitted {len(jobs_to_resubmit)} job(s).')
+                    log(f're-submitted job(s) {", ".join(str(j) for j in jobs_to_resubmit)}'
+                        + (f' to {override}' if override else ''))
                 st.rerun()
             if col_no.button('Cancel', key=f'{delete_key}_resubmit_no'):
                 st.session_state.pop(f'{delete_key}_resubmit_pending', None)
@@ -255,9 +285,12 @@ def _jobs_panel(table, status_field, target_field, cols, delete_key,
             st.warning(f'Reset {len(jobs_to_reset)} job(s) to **ON SERVER** / waiting?')
             col_yes, col_no, _ = st.columns([1, 1, 4])
             if col_yes.button('Yes, reset', type='primary', key=f'{delete_key}_on_server_yes'):
-                on_server_fn(jobs_to_reset)
+                ok, _ = write(table, 'reset the jobs', lambda: on_server_fn(jobs_to_reset),
+                              success=f'Reset {len(jobs_to_reset)} job(s) to ON SERVER.',
+                              flash_key=f'{delete_key}_msg')
                 st.session_state.pop(f'{delete_key}_on_server_pending', None)
-                st.rerun()
+                if ok:
+                    st.rerun()
             if col_no.button('Cancel', key=f'{delete_key}_on_server_no'):
                 st.session_state.pop(f'{delete_key}_on_server_pending', None)
                 st.rerun()