labdata 0.1.16__tar.gz → 0.1.17__tar.gz

{labdata-0.1.16 → labdata-0.1.17}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: labdata
-Version: 0.1.16
+Version: 0.1.17
 Summary: Package to manage data in experimental neuroscience labs
 Author-email: Joao Couto <jpcouto@gmail.com>
 Project-URL: Homepage, https://github.com/jcouto/labdata

{labdata-0.1.16 → labdata-0.1.17}/labdata/__init__.py

@@ -1,4 +1,4 @@
-VERSION = "0.1.16"
+VERSION = "0.1.17"
 
 from .utils import *
 from .copy import copy_to_upload_server

{labdata-0.1.16 → labdata-0.1.17}/labdata/dashboard/compute.py

@@ -1,6 +1,7 @@
 import pandas as pd
 import numpy as np
 import streamlit as st
+from writes import write, show_flash
 
 STATES = ['COMPLETED', 'WORKING', 'WAITING', 'FAILED', 'CANCELED','ON SERVER']
 STATE_COLORS = ['#01D186', '#48CAE4', '#F3A21C', '#FF5733', '#F3A21C','#F3A21C']
@@ -50,6 +51,21 @@ def _resubmit_jobs(schema, job_ids, project, target_override=None):
     return errors
 
 
+def _fetch_rows(table):
+    '''Fetch all rows of a job table, reconnecting once if the first attempt fails.
+
+    The dashboard shares one database connection between browser sessions, and
+    the client can be left half closed (a stale socket, a packet sequence
+    error, or ``'NoneType' object has no attribute 'read'`` from pymysql).
+    Such failures clear on a fresh connection, so retry once before giving up.
+    '''
+    try:
+        return table.fetch(as_dict=True)
+    except Exception:
+        table.connection.connect()
+        return table.fetch(as_dict=True)
+
+
 def _jobs_panel(table, status_field, target_field, cols, delete_key,
                 resubmit_fn=None, allow_delete_completed=False, on_server_fn=None):
 
@@ -59,20 +75,13 @@ def _jobs_panel(table, status_field, target_field, cols, delete_key,
     if st.session_state.get(f'{delete_key}_resubmit_errors'):
         for e in st.session_state.pop(f'{delete_key}_resubmit_errors'):
             st.error(e)
+    show_flash(f'{delete_key}_msg')
 
     try:
-        rows = table.fetch(as_dict=True)
+        rows = _fetch_rows(table)
     except Exception as e:
-        if 'Packet sequence' in str(e) or 'sequence number wrong' in str(e):
-            try:
-                table.connection.connect()
-                rows = table.fetch(as_dict=True)
-            except Exception as e2:
-                st.error(f'Database connection error: {e2}')
-                return
-        else:
-            st.error(f'Error fetching jobs: {e}')
-            return
+        st.error(f'Could not fetch the jobs: {e}')
+        return
     if not rows:
         st.info('There are no jobs in the **ComputeTask** table.')
         return
@@ -95,9 +104,14 @@ def _jobs_panel(table, status_field, target_field, cols, delete_key,
             st.warning(f'Delete all {n} COMPLETED job(s)? This cannot be undone.')
             col_yes, col_no, _ = st.columns([1, 1, 4])
             if col_yes.button('Yes, delete', type='primary', key=f'{delete_key}_del_completed_yes'):
-                (table & f'{status_field} = "COMPLETED"').delete(safemode=False)
+                ok, _ = write(table, 'delete the completed jobs',
+                              lambda: (table & f'{status_field} = "COMPLETED"').delete(
+                                  safemode=False, transaction=False),
+                              success=f'Deleted {n} completed job(s).',
+                              flash_key=f'{delete_key}_msg')
                 st.session_state.pop(f'{delete_key}_del_completed_pending', None)
-                st.rerun()
+                if ok:
+                    st.rerun()
             if col_no.button('Cancel', key=f'{delete_key}_del_completed_no'):
                 st.session_state.pop(f'{delete_key}_del_completed_pending', None)
                 st.rerun()
@@ -175,9 +189,14 @@ def _jobs_panel(table, status_field, target_field, cols, delete_key,
         st.warning(f'Delete all {len(jobs_to_delete)} filtered job(s)? This cannot be undone.')
         col_yes, col_no, _ = st.columns([1, 1, 4])
         if col_yes.button('Yes, delete', type='primary', key=f'{delete_key}_del_filtered_yes'):
-            (table & [dict(job_id=j) for j in jobs_to_delete]).delete(safemode=False)
+            ok, _ = write(table, 'delete the filtered jobs',
+                          lambda: (table & [dict(job_id=j) for j in jobs_to_delete]).delete(
+                              safemode=False, transaction=False),
+                          success=f'Deleted {len(jobs_to_delete)} job(s).',
+                          flash_key=f'{delete_key}_msg')
             st.session_state.pop(f'{delete_key}_del_filtered_pending', None)
-            st.rerun()
+            if ok:
+                st.rerun()
         if col_no.button('Cancel', key=f'{delete_key}_del_filtered_no'):
             st.session_state.pop(f'{delete_key}_del_filtered_pending', None)
             st.rerun()
@@ -221,10 +240,14 @@ def _jobs_panel(table, status_field, target_field, cols, delete_key,
             st.warning(f'Are you sure you want to delete {len(jobs_to_delete)} job(s)? This cannot be undone and scheduled jobs might fail.')
             col_yes, col_no, _ = st.columns([1, 1, 4])
             if col_yes.button('Yes, delete', type='primary', key=f'{delete_key}_yes'):
-                (table & [dict(job_id=j) for j in jobs_to_delete]).delete(safemode=False)
+                ok, _ = write(table, 'delete the selected jobs',
+                              lambda: (table & [dict(job_id=j) for j in jobs_to_delete]).delete(
+                                  safemode=False, transaction=False),
+                              success=f'Deleted {len(jobs_to_delete)} job(s).',
+                              flash_key=f'{delete_key}_msg')
                 st.session_state.pop(f'{delete_key}_pending', None)
-                st.success(f'Deleted {len(jobs_to_delete)} job(s).')
-                st.rerun()
+                if ok:
+                    st.rerun()
             if col_no.button('Cancel', key=f'{delete_key}_no'):
                 st.session_state.pop(f'{delete_key}_pending', None)
                 st.rerun()
@@ -245,6 +268,9 @@ def _jobs_panel(table, status_field, target_field, cols, delete_key,
                 st.session_state.pop(f'{delete_key}_resubmit_pending', None)
                 if errors:
                     st.session_state[f'{delete_key}_resubmit_errors'] = errors
+                else:
+                    st.session_state[f'{delete_key}_msg'] = (
+                        f'Re-submitted {len(jobs_to_resubmit)} job(s).')
                 st.rerun()
             if col_no.button('Cancel', key=f'{delete_key}_resubmit_no'):
                 st.session_state.pop(f'{delete_key}_resubmit_pending', None)
@@ -255,9 +281,12 @@ def _jobs_panel(table, status_field, target_field, cols, delete_key,
             st.warning(f'Reset {len(jobs_to_reset)} job(s) to **ON SERVER** / waiting?')
             col_yes, col_no, _ = st.columns([1, 1, 4])
             if col_yes.button('Yes, reset', type='primary', key=f'{delete_key}_on_server_yes'):
-                on_server_fn(jobs_to_reset)
+                ok, _ = write(table, 'reset the jobs', lambda: on_server_fn(jobs_to_reset),
+                              success=f'Reset {len(jobs_to_reset)} job(s) to ON SERVER.',
+                              flash_key=f'{delete_key}_msg')
                 st.session_state.pop(f'{delete_key}_on_server_pending', None)
-                st.rerun()
+                if ok:
+                    st.rerun()
             if col_no.button('Cancel', key=f'{delete_key}_on_server_no'):
                 st.session_state.pop(f'{delete_key}_on_server_pending', None)
                 st.rerun()

{labdata-0.1.16 → labdata-0.1.17}/labdata/dashboard/index.py

@@ -60,6 +60,8 @@ def intro_tab(user_name=None, schema=None):
         '''Fetch all strain names.'''
         return list(schema.Strain().fetch('strain_name'))
 
+    from writes import write, show_flash
+    show_flash('subjects_msg')
     list_tab, add_tab, edit_tab, weighing_tab = st.tabs(['Subjects/Sessions', 'New', 'Update', 'Weighing'])
 
     with list_tab:
@@ -164,10 +166,18 @@ def intro_tab(user_name=None, schema=None):
             insert_dict['subject_sex'] = 'U' if sex == 'Unknown' else sex
             insert_dict['strain_name'] = st.selectbox('Strain', available_strains)
             if st.form_submit_button('Add Subject', type='primary'):
-                schema.Subject().insert1(insert_dict)
-                get_subjects.clear()
-                st.success(f'Added subject {insert_dict["subject_name"]}.')
-                st.rerun()
+                if not (insert_dict['subject_name'] or '').strip():
+                    st.error('Give the subject an id.')
+                elif not insert_dict['user_name']:
+                    st.error('Pick a user name.')
+                else:
+                    ok, _ = write(schema.Subject, f'add subject {insert_dict["subject_name"]}',
+                                  lambda: schema.Subject().insert1(insert_dict),
+                                  success=f'Added subject {insert_dict["subject_name"]}.',
+                                  flash_key='subjects_msg')
+                    if ok:
+                        get_subjects.clear()
+                        st.rerun()
 
     with edit_tab:
         users = get_users()
@@ -186,10 +196,13 @@ def intro_tab(user_name=None, schema=None):
                 edit_dict['strain_name'] = st.selectbox('Strain', available_strains,
                                                          index=available_strains.index(edit_dict['strain_name']))
                 if st.form_submit_button('Save changes', type='primary'):
-                    schema.Subject().update1(edit_dict)
-                    get_subjects.clear()
-                    st.success(f'Updated {selected_subject}.')
-                    st.rerun()
+                    ok, _ = write(schema.Subject, f'update {selected_subject}',
+                                  lambda: schema.Subject().update1(edit_dict),
+                                  success=f'Updated {selected_subject}.',
+                                  flash_key='subjects_msg')
+                    if ok:
+                        get_subjects.clear()
+                        st.rerun()
 
     with weighing_tab:
         @st.cache_data
@@ -222,13 +235,14 @@ def intro_tab(user_name=None, schema=None):
                 w_weight = st.number_input('Weight (g)', min_value=0.0, step=0.1, format='%.1f')
                 if st.form_submit_button('Add', type='primary'):
                     w_dt = datetime.datetime.combine(w_date, w_time)
-                    try:
-                        schema.Weighing().insert1(dict(subject_name=w_subject, weighing_datetime=w_dt, weight=w_weight))
+                    ok, _ = write(schema.Weighing, f'add the weight of {w_subject}',
+                                  lambda: schema.Weighing().insert1(dict(
+                                      subject_name=w_subject, weighing_datetime=w_dt, weight=w_weight)),
+                                  success=f'Added {w_weight:.1f} g for {w_subject} at {w_dt}.',
+                                  flash_key='subjects_msg')
+                    if ok:
                         get_weighings.clear()
-                        st.success(f'Added {w_weight:.1f} g for {w_subject} at {w_dt}.')
                         st.rerun()
-                    except Exception as e:
-                        st.error(str(e))
 
         with plot_col:
             weights_df = get_weighings(w_subject)
@@ -267,16 +281,17 @@ def intro_tab(user_name=None, schema=None):
                                 new_weight = st.number_input('New weight (g)', value=float(sel_row['weight']),
                                                              min_value=0.0, step=0.1, format='%.1f')
                                 if st.form_submit_button('Update', type='primary'):
-                                    try:
-                                        schema.Weighing().update1(dict(
-                                            subject_name=w_subject,
-                                            weighing_datetime=sel_dt_exact.to_pydatetime(),
-                                            weight=new_weight))
+                                    ok, _ = write(schema.Weighing, f'update the weight of {w_subject}',
+                                                  lambda: schema.Weighing().update1(dict(
+                                                      subject_name=w_subject,
+                                                      weighing_datetime=sel_dt_exact.to_pydatetime(),
+                                                      weight=new_weight)),
+                                                  success=(f'Updated the weight of {w_subject} at '
+                                                           f'{sel_dt_exact:%Y-%m-%d %H:%M} to {new_weight:.1f} g.'),
+                                                  flash_key='subjects_msg')
+                                    if ok:
                                         get_weighings.clear()
-                                        st.success(f'Updated to {new_weight:.1f} g.')
                                         st.rerun()
-                                    except Exception as e:
-                                        st.error(str(e))
                         except Exception:
                             pass
 
@@ -329,6 +344,36 @@ if project is None:
             project = k.replace('project=','')
 
 import time as _time
+
+
+def _is_db_client_error(exc):
+    '''True for a pymysql error or any exception raised from inside pymysql.
+
+    The dashboard shares one client connection between browser sessions; a half
+    closed client raises AttributeError from its socket reader rather than a
+    database error, and a fresh connection clears it.'''
+    import traceback
+    import pymysql
+    if isinstance(exc, pymysql.err.Error) or isinstance(getattr(exc, '__cause__', None), pymysql.err.Error):
+        return True
+    return any('pymysql' in f.filename for f in traceback.extract_tb(exc.__traceback__))
+
+
+def _reconnect():
+    '''Drop the shared client connection and open a new one.'''
+    import datajoint as dj
+    try:
+        conn = dj.conn()
+        if conn._conn is not None:
+            try:
+                conn._conn.close()
+            except Exception:
+                pass
+        conn.connect()
+    except Exception:
+        pass
+
+
 _schema_err = None
 for _attempt in range(5):
     try:
@@ -342,6 +387,13 @@ for _attempt in range(5):
         _schema_err = _e
         load_schema.clear()
         _time.sleep(0.3 * (_attempt + 1))
+    except Exception as _e:
+        if not _is_db_client_error(_e):
+            raise
+        _schema_err = _e
+        load_schema.clear()
+        _reconnect()
+        _time.sleep(0.3 * (_attempt + 1))
 if _schema_err is not None:
     st.error(f'Database connection error after 5 attempts: {_schema_err}. Please refresh the page.')
     st.stop()
@@ -433,20 +485,13 @@ tab_name = st.session_state['active_tab']
 try:
     page_names_to_funcs[tab_name]()
 except Exception as _e:
-    import pymysql, datajoint as dj
-    def _is_mysql_err(exc):
-        return isinstance(exc, pymysql.err.Error) or (
-            hasattr(exc, '__cause__') and isinstance(exc.__cause__, pymysql.err.Error))
-    if _is_mysql_err(_e):
-        try:
-            conn = dj.conn()
-            if conn._conn is not None:
-                try:
-                    conn._conn.close()
-                except Exception:
-                    pass
-            conn.connect()
-        except Exception:
-            pass
+    # a stale client connection: reconnect and draw again, at most twice in a row
+    _retries = st.session_state.get('_db_reconnects', 0)
+    if _is_db_client_error(_e) and _retries < 2:
+        st.session_state['_db_reconnects'] = _retries + 1
+        _reconnect()
         st.rerun()
+    st.session_state.pop('_db_reconnects', None)
     raise
+else:
+    st.session_state.pop('_db_reconnects', None)

{labdata-0.1.16 → labdata-0.1.17}/labdata/dashboard/notes.py

@@ -1,6 +1,7 @@
 import pandas as pd
 import streamlit as st
 from datetime import datetime, date, time as dtime
+from writes import write
 
 
 def notes_tab(schema=None):
@@ -161,11 +162,17 @@ def notes_tab(schema=None):
                     note_dt = datetime.combine(note_date, note_time)
                     dataset_key = dict(subject_name=subject_name, session_name=session_name,
                                        dataset_name=dataset_name)
-                    schema.Note.insert1(dict(notetaker=notetaker, note_datetime=note_dt, notes=note_text))
-                    schema.Dataset.update1({**dataset_key, 'notetaker': notetaker, 'note_datetime': note_dt})
-                    get_note_counts.clear()
-                    get_subject_notes.clear()
-                    st.success(f'Note {dataset_key} added successfully.')
+
+                    def _add_note():
+                        # the note and the link from the dataset land together
+                        schema.Note.insert1(dict(notetaker=notetaker, note_datetime=note_dt, notes=note_text))
+                        schema.Dataset.update1({**dataset_key, 'notetaker': notetaker, 'note_datetime': note_dt})
+
+                    ok, _ = write(schema.Note, 'add the note', _add_note,
+                                  success=f'Added the note to {subject_name} / {session_name} / {dataset_name}.')
+                    if ok:
+                        get_note_counts.clear()
+                        get_subject_notes.clear()
 
     with update_tab:
         selected = st.session_state.get('selected_note', {})
@@ -188,9 +195,12 @@ def notes_tab(schema=None):
                     if not new_text.strip():
                         st.error('Note text cannot be empty.')
                     else:
-                        schema.Note.update1(dict(notetaker=notetaker_upd, note_datetime=note_dt_upd,
-                                                 notes=new_text))
-                        get_note_counts.clear()
-                        get_subject_notes.clear()
-                        st.session_state.pop('selected_note', None)
-                        st.success(f'Note {dict(notetaker=notetaker_upd, note_datetime=note_dt_upd)} updated.')
+                        ok, _ = write(schema.Note, 'update the note',
+                                      lambda: schema.Note.update1(dict(notetaker=notetaker_upd,
+                                                                       note_datetime=note_dt_upd,
+                                                                       notes=new_text)),
+                                      success=f'Updated the note by {notetaker_upd} from {note_dt_upd}.')
+                        if ok:
+                            get_note_counts.clear()
+                            get_subject_notes.clear()
+                            st.session_state.pop('selected_note', None)