labdata 0.1.9__tar.gz → 0.1.11__tar.gz

{labdata-0.1.9 → labdata-0.1.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: labdata
-Version: 0.1.9
+Version: 0.1.11
 Summary: Package to manage data in experimental neuroscience labs
 Author-email: Joao Couto <jpcouto@gmail.com>
 Project-URL: Homepage, https://github.com/jcouto/labdata

{labdata-0.1.9 → labdata-0.1.11}/labdata/__init__.py

@@ -1,4 +1,4 @@
-VERSION = "0.1.9"
+VERSION = "0.1.11"
 
 from .utils import *
 from .copy import copy_to_upload_server

{labdata-0.1.9 → labdata-0.1.11}/labdata/compute/pose.py

@@ -63,7 +63,10 @@ class DeeplabcutCompute(BaseCompute):
         if self.parameters['mode'] == 'train':
             for i,r in parameters.iterrows():
                 # go through every parameter and label_set
-                if (self.model_parameters == json.loads(r.parameters_dict) and 
+                flag = False
+                if type(r.parameters_dict) in [str]:
+                    flag = (self.model_parameters == json.loads(r.parameters_dict))
+                if (flag and 
                     self.parameters['model_num'] is None and 
                     self.parameters['pose_label_set_num'] == r['pose_label_set_num']):
                     model_num = r.model_num

labdata-0.1.11/labdata/dashboard/docstrings.py

@@ -0,0 +1,265 @@
+"""Docstring viewer tab — browse DataJoint table docstrings, definitions and column comments."""
+import re
+import inspect
+import pandas as pd
+import streamlit as st
+
+from schema_graph import (
+    _enumerate_tables,
+    _get_tier,
+    _get_schema_type,
+    _get_category,
+)
+
+
+@st.cache_data(ttl=600)
+def _collect_tables(_schema):
+    """Walk the schema and collect per-table metadata for browsing.
+
+    Parameters
+    ----------
+    _schema : module
+        Loaded project schema module (underscore prefix skips Streamlit hashing).
+
+    Returns
+    -------
+    pandas.DataFrame
+        One row per master and part table with columns ``key``, ``schema``,
+        ``tier``, ``category`` and ``full_table_name``.
+    """
+    masters, parts_map = _enumerate_tables(_schema)
+    rows = []
+
+    def _add(key, cls):
+        try:
+            full_name = cls.full_table_name
+        except Exception:
+            return
+        sql_name = full_name.split('.')[-1].strip('`')
+        rows.append(dict(
+            key=key,
+            schema=_get_schema_type(full_name, _schema),
+            tier=_get_tier(sql_name),
+            category=_get_category(key),
+            full_table_name=full_name,
+        ))
+
+    for key, cls in masters.items():
+        _add(key, cls)
+        for part_key, part_cls in parts_map.get(key, {}).items():
+            _add(part_key, part_cls)
+
+    df = pd.DataFrame(rows)
+    if len(df):
+        df = df.sort_values(['schema', 'category', 'key']).reset_index(drop=True)
+    return df
+
+
+def _resolve_table(schema, key):
+    """Resolve a master or ``Master.Part`` key to its table class."""
+    if '.' in key:
+        master_name, part_name = key.split('.', 1)
+        master = getattr(schema, master_name, None)
+        return getattr(master, part_name, None) if master is not None else None
+    return getattr(schema, key, None)
+
+
+def _class_docstring(cls):
+    """Return the authored class docstring, without datajoint's appended definition.
+
+    On ``declare`` DataJoint appends ``\\nTable definition:\\n\\n<definition>`` to
+    ``__doc__`` (datajoint/schemas.py).  We strip that here so it isn't rendered
+    twice and so the leftover column-0 line doesn't defeat ``cleandoc``'s dedent
+    (which would render the real docstring as a code block).
+    """
+    raw = (cls.__doc__ or '').split('\nTable definition:\n\n', 1)[0]
+    return inspect.cleandoc(raw).strip()
+
+
+def _rst_to_md(text):
+    """Convert the few reStructuredText constructs we see to markdown."""
+    return re.sub(r'``([^`]+)``', r'`\1`', text)         # ``literal`` -> `literal`
+
+
+def _render_docstring(doc):
+    """Render a numpydoc/RST docstring as markdown for ``st.markdown``.
+
+    Uses the ``numpydoc`` parser so every section (Parameters, Returns,
+    Raises, Notes, See Also, …) renders consistently and prose is never
+    swallowed into an indented code block.  Falls back to a plain dedent if
+    ``numpydoc`` is unavailable or the parse fails.
+    """
+    try:
+        from numpydoc.docscrape import NumpyDocString
+        nd = NumpyDocString(doc)
+    except Exception:
+        return _rst_to_md(inspect.cleandoc(doc))
+
+    parts = []
+    summary = '\n'.join(nd['Summary']).strip()
+    if summary:
+        parts.append(summary)
+    extended = '\n'.join(nd['Extended Summary']).strip()
+    if extended:
+        parts.append(extended)
+
+    for section in ('Parameters', 'Returns', 'Yields', 'Raises', 'Attributes'):
+        items = nd[section]
+        if not items:
+            continue
+        bullets = []
+        for it in items:
+            head = f'**{it.name}**' if it.name else ''
+            if it.type:
+                head += f' — *{it.type}*' if head else f'*{it.type}*'
+            desc = ' '.join(l.strip() for l in it.desc).strip()
+            bullets.append(f'- {head}' + (f': {desc}' if desc else ''))
+        parts.append(f'**{section}**\n\n' + '\n'.join(bullets))
+
+    see_also = nd['See Also']
+    if see_also:
+        bullets = []
+        for refs, desc in see_also:
+            names = ', '.join(f'`{n}`' for n, _ in refs)
+            desc = ' '.join(d.strip() for d in desc).strip()
+            bullets.append(f'- {names}' + (f': {desc}' if desc else ''))
+        parts.append('**See Also**\n\n' + '\n'.join(bullets))
+
+    for section in ('Notes', 'References', 'Examples'):
+        body = '\n'.join(nd[section]).strip()
+        if body:
+            parts.append(f'**{section}**\n\n{body}')
+
+    return _rst_to_md('\n\n'.join(parts))
+
+
+def _methods_table(cls):
+    """Return [(name, signature, doc)] for methods defined on the table.
+
+    Only methods implemented in the project (not inherited from DataJoint base
+    classes) are included — e.g. ``File.get``.
+    """
+    rows = []
+    for name in sorted(dir(cls)):
+        if name.startswith('_'):
+            continue
+        try:
+            attr = inspect.getattr_static(cls, name)
+        except Exception:
+            continue
+        func = attr.__func__ if isinstance(attr, (classmethod, staticmethod)) else attr
+        if not (inspect.isfunction(func) or inspect.ismethod(func)):
+            continue
+        if (getattr(func, '__module__', '') or '').startswith('datajoint'):
+            continue                                     # skip dj-provided methods
+        try:
+            sig = str(inspect.signature(func))
+        except (ValueError, TypeError):
+            sig = '(...)'
+        rows.append((name, sig, inspect.getdoc(func) or ''))
+    return rows
+
+
+def _attribute_table(cls):
+    """Return a DataFrame of (column, key, type, null, default, comment) or None."""
+    try:
+        attrs = cls.heading.attributes
+    except Exception:
+        return None
+    rows = []
+    for name, a in attrs.items():
+        rows.append(dict(
+            column=name,
+            key='🔑' if a.in_key else '',
+            type=a.type,
+            null='✓' if a.nullable else '',
+            default='' if a.default is None else str(a.default),
+            comment=a.comment or '',
+        ))
+    return pd.DataFrame(rows) if rows else None
+
+
+def docstrings_tab(schema=None):
+    """Streamlit tab: browse table docstrings, DataJoint definitions and column comments.
+
+    Parameters
+    ----------
+    schema : module or None, optional
+        Project schema module.  Loaded from session state when ``None``.
+    """
+    if schema is None:
+        schema = st.session_state.get('schema')
+    if schema is None:
+        st.error('No schema available.')
+        return
+
+    st.write('## Table reference')
+
+    tables = _collect_tables(schema)
+    if not len(tables):
+        st.info('No tables found in the loaded schema.')
+        return
+
+    filt_col, cat_col, search_col = st.columns([1, 1, 2])
+    with filt_col:
+        schemas = ['(all)'] + sorted(tables['schema'].unique())
+        sel_schema = st.selectbox('Schema', schemas, key='docstr_schema')
+    with cat_col:
+        cats = ['(all)'] + sorted(tables['category'].unique())
+        sel_cat = st.selectbox('Category', cats, key='docstr_cat')
+    with search_col:
+        query = st.text_input('Search', key='docstr_search',
+                              placeholder='Filter table names…').strip().lower()
+
+    view = tables
+    if sel_schema != '(all)':
+        view = view[view['schema'] == sel_schema]
+    if sel_cat != '(all)':
+        view = view[view['category'] == sel_cat]
+    if query:
+        view = view[view['key'].str.lower().str.contains(query)]
+
+    if not len(view):
+        st.info('No tables match the current filters.')
+        return
+
+    sel_key = st.selectbox(
+        f'Table ({len(view)})',
+        list(view['key']),
+        key='docstr_table',
+    )
+
+    cls = _resolve_table(schema, sel_key)
+    if cls is None:
+        st.error(f'Could not resolve table `{sel_key}`.')
+        return
+
+    meta = tables[tables['key'] == sel_key].iloc[0]
+    st.caption(f'`{meta["full_table_name"]}` — {meta["schema"]} schema · '
+               f'{meta["tier"]} · {meta["category"]}')
+
+    doc = _class_docstring(cls)
+    if doc:
+        st.markdown(_render_docstring(doc))
+    else:
+        st.caption('_No class docstring._')
+
+    definition = getattr(cls, 'definition', None)
+    if definition:
+        st.write('### Definition')
+        st.code(inspect.cleandoc(definition), language='text')
+
+    attr_df = _attribute_table(cls)
+    if attr_df is not None:
+        st.write('### Columns')
+        st.dataframe(attr_df, hide_index=True, width='stretch')
+
+    methods = _methods_table(cls)
+    if methods:
+        st.write('### Methods')
+        for name, sig, mdoc in methods:
+            st.markdown(f'**`{name}{sig}`**')
+            if mdoc:
+                st.markdown(_render_docstring(mdoc))
+            else:
+                st.caption('_No docstring._')

{labdata-0.1.9 → labdata-0.1.11}/labdata/dashboard/index.py

@@ -294,6 +294,7 @@ from segmentation import segmentation_tab
 from video import video_tab
 from decisiontask import decisionmaking_tab
 from compute import compute_tab
+from docstrings import docstrings_tab
 
 @st.cache_resource
 def load_schema(project):
@@ -365,16 +366,17 @@ if show_admin:
             get_schema_sizes(schema)
             get_project_users(schema)
             st.session_state['project'] = selected
-            st.session_state.pop('sidebar_tab', None)
+            for _k in ('active_tab', 'main_nav', 'ref_nav'):
+                st.session_state.pop(_k, None)
             st.rerun()
 
 page_names_to_funcs = {}
 
 if show_admin:
     from admin import admin_tab
-    from schema_graph import schema_graph_tab
     page_names_to_funcs["**Admin**"] = partial(admin_tab, schema=schema)
-    page_names_to_funcs["**Tables graph**"] = partial(schema_graph_tab, schema=schema)
+
+from schema_graph import schema_graph_tab
 
 page_names_to_funcs.update({
     "**Subjects**": partial(intro_tab, schema = schema),
@@ -395,16 +397,34 @@ for p in plugins.keys():
     if hasattr(plugins[p],'dashboard_function'):
         page_names_to_funcs[plugins[p].dashboard_name] = partial(plugins[p].dashboard_function, schema = schema)
 
+# Reference tabs live at the bottom of the selector, below the docs link.
+page_names_to_funcs["**Table reference**"] = partial(docstrings_tab, schema = schema)
+page_names_to_funcs["**Tables graph**"] = partial(schema_graph_tab, schema = schema)
+
+# Split the selector into the main tabs and the reference tabs (bottom group).
+_ref_names = [n for n in ("**Table reference**", "**Tables graph**") if n in page_names_to_funcs]
+_main_names = [n for n in page_names_to_funcs if n not in _ref_names]
+
+if st.session_state.get('active_tab') not in page_names_to_funcs:
+    st.session_state['active_tab'] = _main_names[0]
+    st.session_state['main_nav'] = _main_names[0]
+    st.session_state['ref_nav'] = None
+
+def _nav_main():
+    if st.session_state.get('main_nav') is not None:
+        st.session_state['active_tab'] = st.session_state['main_nav']
+        st.session_state['ref_nav'] = None
+
+def _nav_ref():
+    if st.session_state.get('ref_nav') is not None:
+        st.session_state['active_tab'] = st.session_state['ref_nav']
+        st.session_state['main_nav'] = None
+
+st.sidebar.radio("``labdata dashboard``", _main_names, key='main_nav', on_change=_nav_main)
+st.sidebar.radio("Reference · [📖 Documentation](https://jcouto.github.io/labdata-docs/)",
+                 _ref_names, key='ref_nav', index=None, on_change=_nav_ref)
 
-_tab_options = list(page_names_to_funcs.keys())
-_tab_default = st.session_state.get('sidebar_tab', _tab_options[0])
-if _tab_default not in _tab_options:
-    _tab_default = _tab_options[0]
-tab_name = st.sidebar.radio(
-    "``labdata dashboard``",
-    _tab_options,
-    index=_tab_options.index(_tab_default),
-    key='sidebar_tab')
+tab_name = st.session_state['active_tab']
 
 try:
     page_names_to_funcs[tab_name]()

{labdata-0.1.9 → labdata-0.1.11}/labdata/dashboard/schema_graph.py

@@ -30,11 +30,11 @@ _CATEGORY_RULES = [
                   'choice', 'running']),
 ]
 # Horizontal band center per category (pixels); spacing between bands
-_CAT_SPACING = 1200
+_CAT_SPACING = 700
 _CATEGORY_X = {name: i * _CAT_SPACING for i, (name, _) in enumerate(_CATEGORY_RULES)}
 _CATEGORY_X['other'] = len(_CATEGORY_RULES) * _CAT_SPACING
-_LEVEL_SPACING = 350   # vertical pixels between topological levels
-_NODE_JITTER = 280     # horizontal spread when multiple nodes share category+level
+_LEVEL_SPACING = 220   # vertical pixels between topological levels
+_NODE_JITTER = 170     # horizontal spread when multiple nodes share category+level
 
 
 def _get_category(key):
@@ -95,8 +95,24 @@ def _enumerate_tables(schema_module):
     return masters, parts
 
 
-# Cached graph data 
+@st.cache_data(ttl=600)
+def _approx_row_count(full_table_name):
+    """Fast (approximate) row count from information_schema.
+
+    Gets the estimate from ``TABLE_ROWS`` which is faster than ``SELECT COUNT(*)``.
+    Returns the estimate, or ``None`` if it can't be determined.
+    """
+    try:
+        db, tbl = (p.strip('`') for p in full_table_name.split('.'))
+        res = dj.conn().query(
+            "SELECT table_rows FROM information_schema.tables "
+            "WHERE table_schema=%s AND table_name=%s", (db, tbl)).fetchone()
+        return int(res[0]) if res and res[0] is not None else None
+    except Exception:
+        return None
 
+
+# Cached graph data
 @st.cache_data(ttl=600)
 def build_graph_data(_schema, included_schemas, included_tiers):
     """
@@ -195,6 +211,16 @@ def build_graph_data(_schema, included_schemas, included_tiers):
             x_offset = (i - (n - 1) / 2) * _NODE_JITTER
             positions[k] = (cx + x_offset, cy)
 
+    # Mean-center coordinates around the origin so the graph renders centered
+    # (all-positive x otherwise puts it to the right and vis.js auto-fit slides
+    # it to center on every rerun, which reads as a jump on click).
+    if positions:
+        xs = [p[0] for p in positions.values()]
+        ys = [p[1] for p in positions.values()]
+        ox = (min(xs) + max(xs)) / 2
+        oy = (min(ys) + max(ys)) / 2
+        positions = {k: (x - ox, y - oy) for k, (x, y) in positions.items()}
+
     nodes = [
         {**node_meta[n], 'key': n, 'level': levels.get(n, 0),
          'x': positions[n][0], 'y': positions[n][1]}
@@ -219,7 +245,7 @@ def build_graph_data(_schema, included_schemas, included_tiers):
                 continue
             n_parts = len(part_dict)
             px = master_x + (idx - (n_parts - 1) / 2) * _NODE_JITTER
-            py = (master_lv + 1) * _LEVEL_SPACING
+            py = positions[master_key][1] + _LEVEL_SPACING
             pn.append(dict(
                 key=part_key,
                 display=part_key.split('.')[-1],
@@ -242,13 +268,7 @@ def build_graph_data(_schema, included_schemas, included_tiers):
 # Tab entry point
 
 def schema_graph_tab(schema, **kwargs):
-    '''Streamlit tab: interactive DataJoint schema dependency graph.
-
-    Renders a physics-enabled force-directed graph (via ``streamlit-agraph``)
-    where nodes are tables colour-coded by category (lab, subject, ephys,
-    imaging, behaviour…) and edges show foreign-key dependencies.  Nodes can
-    be filtered by schema type (Manual, Computed, Imported, Lookup) and by
-    category.
+    '''Streamlit tab: interactive datajoint schema dependency graph.
 
     Parameters
     ----------
@@ -341,7 +361,9 @@ def schema_graph_tab(schema, **kwargs):
             color='#555555', arrows='to',
         ))
 
-    # Config
+    # Config.  Seed coordinates are mean-centered (see build_graph_data) so the
+    # graph starts centered and vis.js's auto-fit has nothing to slide on a
+    # click-triggered rerun (that was the "jump from right to center").
     config = Config(
         width='100%',
         height=500,
@@ -349,6 +371,17 @@ def schema_graph_tab(schema, **kwargs):
         physics=True,
         hierarchical=False,
     )
+    # barnesHut force tuning: with physics on, these (not the seed spacing)
+    # determine the final layout density.  Weaker repulsion + stronger central
+    # gravity + shorter springs => a more compact cluster.
+    config.physics['barnesHut'] = {
+        'gravitationalConstant': -800,
+        'centralGravity': 0.5,
+        'springLength': 90,
+        'springConstant': 0.05,
+        'damping': 0.3,
+        'avoidOverlap': 0.2,
+    }
 
     # Render
     clicked = agraph(nodes=agraph_nodes, edges=agraph_edges, config=config)
@@ -381,10 +414,9 @@ def schema_graph_tab(schema, **kwargs):
                         if cls:
                             break
                 if cls:
-                    try:
-                        st.metric('Rows', len(cls()))
-                    except Exception:
-                        pass
+                    approx = _approx_row_count(node['full_table_name'])
+                    if approx is not None:
+                        st.metric('Rows (approx.)', f'~{approx:,}')
 
 
     st.caption(

{labdata-0.1.9 → labdata-0.1.11}/labdata/dashboard/sorting.py

@@ -20,9 +20,10 @@ def _fetch_unit_data(_schema, subject_name, session_name, dataset_name,
                probe_num=probe_num)
     unit_keys = [dict(unit_id=u) for u in unit_ids_tuple]
 
-    # spike times + amplitudes from SpikeSorting.Unit
+    # spike times + amplitudes + positions from SpikeSorting.Unit
     unit_rows = (_schema.SpikeSorting.Unit & key & unit_keys).fetch(
-        'unit_id', 'spike_times', 'spike_amplitudes', as_dict=True)
+        'unit_id', 'spike_times', 'spike_amplitudes', 'spike_positions',
+        as_dict=True)
 
     # sampling-rate / sync conversion (matches get_spike_times logic)
     try:
@@ -39,6 +40,7 @@ def _fetch_unit_data(_schema, subject_name, session_name, dataset_name,
 
     spike_times = {}
     spike_amplitudes = {}
+    spike_positions = {}
     for r in unit_rows:
         arr = np.asarray(r['spike_times'], dtype=np.float64)
         if interp is not None:
@@ -49,6 +51,11 @@ def _fetch_unit_data(_schema, subject_name, session_name, dataset_name,
         if r['spike_amplitudes'] is not None:
             spike_amplitudes[int(r['unit_id'])] = np.asarray(
                 r['spike_amplitudes'], dtype=np.float64)
+        if r['spike_positions'] is not None:
+            pos = np.asarray(r['spike_positions'], dtype=np.float64)
+            # column 1 is depth along the probe (µm)
+            spike_positions[int(r['unit_id'])] = (
+                pos[:, 1] if pos.ndim == 2 and pos.shape[1] > 1 else pos.ravel())
 
     # waveforms
     wf_rows = (_schema.SpikeSorting.Waveforms & key & unit_keys).fetch(
@@ -65,7 +72,7 @@ def _fetch_unit_data(_schema, subject_name, session_name, dataset_name,
         (_schema.SpikeSorting & key).fetch1('sorting_channel_coords'),
         dtype=np.float64)
 
-    return spike_times, spike_amplitudes, waveforms, chmap
+    return spike_times, spike_amplitudes, spike_positions, waveforms, chmap
 
 
 def _xcorr(x, y, maxlag):
@@ -206,8 +213,11 @@ def _unit_panel(schema, plot_df, sel_key, unit_id_col, amp_col,
             _tdata = _tdata[_tdata['_pass'] == 'pass']
         _sort_cols = (['shank'] if 'shank' in _tdata.columns else []) + ['_depth']
         _tdata = (_tdata.sort_values(_sort_cols)
-                        .drop(columns=['_pass', '_depth'])
+                        .drop(columns=['_depth'])
                         .reset_index(drop=True))
+        # positional mask of rows that fail the SUA criteria (greyed out below)
+        _fail_mask = (_tdata['_pass'] == 'fail').values
+        _tdata = _tdata.drop(columns=['_pass'])
 
         # Cursor: index of the currently active unit in _tdata (for n/b navigation)
         _cur_key = (f'srt_tbl_cursor_{sel_key["subject_name"]}_{sel_key["session_name"]}_'
@@ -260,6 +270,8 @@ def _unit_panel(schema, plot_df, sel_key, unit_id_col, amp_col,
             if unit_id_col and unit_id_col in row.index:
                 if int(row[unit_id_col]) in _selected_ids:
                     return ['background-color: #f5a623; color: #000000'] * len(row)
+            if row.name < len(_fail_mask) and _fail_mask[row.name]:
+                return ['background-color: #f0f0f0; color: #999999'] * len(row)
             return [''] * len(row)
 
         _tbl_key = (f'srt_unit_table_{sel_key["subject_name"]}_{sel_key["session_name"]}_'
@@ -428,6 +440,7 @@ def _unit_panel(schema, plot_df, sel_key, unit_id_col, amp_col,
         # all_spike_times / all_spike_amps keyed by (prb_str, unit_id_int)
         all_spike_times = {}
         all_spike_amps  = {}
+        all_spike_pos   = {}
         all_waveforms   = []   # each dict has unit_id + probe keys
         chmap_by_probe  = {}
 
@@ -437,7 +450,7 @@ def _unit_panel(schema, plot_df, sel_key, unit_id_col, amp_col,
             except ValueError:
                 probe_num_val = prb
             try:
-                sp_map, amp_map, wf_data, chmap = _fetch_unit_data(
+                sp_map, amp_map, pos_map, wf_data, chmap = _fetch_unit_data(
                     schema,
                     sel_key['subject_name'], sel_key['session_name'],
                     sel_key['dataset_name'], sel_key['parameter_set_num'],
@@ -449,6 +462,8 @@ def _unit_panel(schema, plot_df, sel_key, unit_id_col, amp_col,
                 all_spike_times[(prb, uid)] = v
             for uid, v in amp_map.items():
                 all_spike_amps[(prb, uid)] = v
+            for uid, v in pos_map.items():
+                all_spike_pos[(prb, uid)] = v
             for w in wf_data:
                 all_waveforms.append(dict(w, probe=prb))
             chmap_by_probe[prb] = chmap
@@ -481,6 +496,71 @@ def _unit_panel(schema, plot_df, sel_key, unit_id_col, amp_col,
                                        value=0.1, step=0.01, format='%.3f',
                                        key='srt_wf_scale')
 
+        # toggle + spike-count selector for the amplitude/depth over-time plots
+        dc1, dc2 = st.columns([1, 2], vertical_alignment='center')
+        show_drift = dc1.toggle(
+            'Spikes over time', key='srt_show_drift',
+            help='Plot spike amplitude and depth over time for the selected units.')
+        drift_n = dc2.number_input(
+            'Number of spikes to plot', min_value=100, max_value=200000,
+            value=5000, step=100, key='srt_drift_nspikes',
+            help='A random subset of this many spikes per unit is plotted.')
+
+        if show_drift and ordered_units:
+            _drift_rows = []
+            _rng = np.random.default_rng(0)
+            for prb, uid in ordered_units:
+                t = all_spike_times.get((prb, uid))
+                if t is None or not len(t):
+                    continue
+                n = len(t)
+                idx = (np.sort(_rng.choice(n, int(drift_n), replace=False))
+                       if n > drift_n else np.arange(n))
+                a = all_spike_amps.get((prb, uid))
+                p = all_spike_pos.get((prb, uid))
+                _drift_rows.append(pd.DataFrame({
+                    'time_s': np.asarray(t)[idx],
+                    'unit': _ulabel(prb, uid),
+                    'amplitude': (np.asarray(a)[idx]
+                                  if a is not None and len(a) >= n
+                                  else np.full(len(idx), np.nan)),
+                    'depth': (np.asarray(p)[idx]
+                              if p is not None and len(p) >= n
+                              else np.full(len(idx), np.nan)),
+                }))
+            if _drift_rows:
+                drift_df = pd.concat(_drift_rows, ignore_index=True)
+                alt.data_transformers.disable_max_rows()
+                _t_dom = [float(drift_df['time_s'].min()), float(drift_df['time_s'].max())]
+                _drift_base = alt.Chart(drift_df).mark_circle(size=10, opacity=0.4).encode(
+                    x=alt.X('time_s:Q', title='Time (s)', axis=_tax,
+                            scale=alt.Scale(domain=_t_dom)),
+                    color=alt.Color('unit:N', scale=unit_color_scale, title='Unit'),
+                    tooltip=['unit:N',
+                             alt.Tooltip('time_s:Q', format='.1f', title='Time (s)'),
+                             alt.Tooltip('amplitude:Q', format='.0f', title='Amplitude'),
+                             alt.Tooltip('depth:Q', format='.0f', title='Depth (µm)')])
+                _dcol1, _dcol2 = st.columns(2)
+                if drift_df['amplitude'].notna().any():
+                    _dcol1.altair_chart(
+                        _drift_base.encode(
+                            y=alt.Y('amplitude:Q', title='Amplitude', axis=_tax))
+                        .add_params(alt.selection_interval(bind='scales',
+                                                           name='drift_amp_zoom'))
+                        .properties(height=280, title='Amplitude over time'),
+                        width='stretch')
+                if drift_df['depth'].notna().any():
+                    _d_dom = [float(drift_df['depth'].min()) - 50,
+                              float(drift_df['depth'].max()) + 50]
+                    _dcol2.altair_chart(
+                        _drift_base.encode(
+                            y=alt.Y('depth:Q', title='Depth (µm)', axis=_tax,
+                                    scale=alt.Scale(domain=_d_dom)))
+                        .add_params(alt.selection_interval(bind='scales',
+                                                           name='drift_depth_zoom'))
+                        .properties(height=280, title='Depth over time'),
+                        width='stretch')
+
         col_left, col_right = st.columns([3, 2])
 
         with col_left:
@@ -815,23 +895,39 @@ def sorting_tab(schema=None):
     default_param = _get_default_param(schema)
     default_params = [default_param] if default_param and default_param in param_options else []
 
-    tab_sessions, tab_explorer, tab_units = st.tabs(['Sessions', 'Unit explorer', 'Unit counts'])
-
-    # Tab 1: Sessions 
-    with tab_sessions:
-        sel_params_1 = st.multiselect(
+    # Global parameter-set and subject selection (apply to every tab). Apply any
+    # pending change requested by a chart click before the widgets are instantiated.
+    if '_srt_pending_param' in st.session_state:
+        st.session_state['srt_param_sel'] = st.session_state.pop('_srt_pending_param')
+    if '_srt_pending_subjects' in st.session_state:
+        st.session_state['srt_subjects'] = st.session_state.pop('_srt_pending_subjects')
+
+    _all_sorted = _get_all_sorting_sessions(schema)
+    subject_options = (sorted(_all_sorted['subject_name'].unique().tolist())
+                       if not _all_sorted.empty else [])
+
+    _gp, _gs = st.columns(2)
+    with _gp:
+        st.session_state.setdefault('srt_param_sel', default_params)
+        sel_params = st.multiselect(
             'Parameter sets',
             options=list(param_options.keys()),
             format_func=lambda k: param_options[k],
-            default=default_params,
-            key='srt_param_sel_sessions',
+            key='srt_param_sel',
         )
-        selected_params_1 = sel_params_1 if sel_params_1 else list(param_options.keys())
+    with _gs:
+        selected_subjects = st.multiselect(
+            'Subjects', options=subject_options, key='srt_subjects')
+    selected_params = sel_params if sel_params else list(param_options.keys())
+
+    tab_sessions, tab_explorer, tab_units = st.tabs(['Sessions', 'Unit explorer', 'Unit counts'])
 
+    # Tab 1: Sessions
+    with tab_sessions:
         # histogram: session counts per subject
         counts_df = _get_session_counts(schema)
         if not counts_df.empty:
-            counts_filt = counts_df[counts_df['parameter_set_num'].isin(selected_params_1)]
+            counts_filt = counts_df[counts_df['parameter_set_num'].isin(selected_params)]
             if not counts_filt.empty:
                 param_sets_h = sorted(counts_filt['parameter_set_num'].unique().tolist())
                 bar_color_scale = alt.Scale(
@@ -856,17 +952,22 @@ def sorting_tab(schema=None):
                 bar_pts = (bar_event.selection or {}).get('bar_sel', [])
                 if bar_pts and bar_pts[0].get('subject_name'):
                     bar_pt = bar_pts[0]
-                    bar_subject = bar_pt.get('subject_name')
                     bar_param = str(bar_pt.get('parameter_set_num', ''))
-                    st.session_state['srt_subjects'] = [bar_subject]
-                    if bar_param in param_options:
-                        st.session_state['srt_param_sel_units'] = [bar_param]
+                    # act only once per distinct selection, otherwise the sticky
+                    # chart selection keeps overriding manual param/subject changes
+                    _sig = ('bar', bar_pt.get('subject_name'), bar_param)
+                    if st.session_state.get('_srt_last_click') != _sig:
+                        st.session_state['_srt_last_click'] = _sig
+                        st.session_state['_srt_pending_subjects'] = [bar_pt.get('subject_name')]
+                        if bar_param in param_options:
+                            st.session_state['_srt_pending_param'] = [bar_param]
+                        st.rerun()
 
         # all-sessions scatter
         all_sessions_df = _get_all_sorting_sessions(schema)
         if not all_sessions_df.empty:
             scatter_df = all_sessions_df[
-                all_sessions_df['parameter_set_num'].isin(selected_params_1)
+                all_sessions_df['parameter_set_num'].isin(selected_params)
             ].reset_index(drop=True)
             if scatter_df.empty:
                 scatter_df = all_sessions_df.copy()
@@ -902,16 +1003,23 @@ def sorting_tab(schema=None):
             if scat_pts and scat_pts[0].get('subject_name'):
                 pt = scat_pts[0]
                 param_str = str(pt.get('parameter_set_num', ''))
-                st.session_state['srt_selected_key'] = {
-                    'subject_name':      pt.get('subject_name'),
-                    'session_name':      pt.get('session_name'),
-                    'dataset_name':      pt.get('dataset_name'),
-                    'parameter_set_num': param_str,
-                }
-                # sync tab 2: subject + param set
-                st.session_state['srt_subjects'] = [pt.get('subject_name')]
-                if param_str in param_options:
-                    st.session_state['srt_param_sel_units'] = [param_str]
+                # act only once per distinct selection, otherwise the sticky
+                # chart selection keeps overriding manual param/subject changes
+                _sig = ('scat', pt.get('subject_name'), pt.get('session_name'),
+                        pt.get('dataset_name'), param_str)
+                if st.session_state.get('_srt_last_click') != _sig:
+                    st.session_state['_srt_last_click'] = _sig
+                    st.session_state['srt_selected_key'] = {
+                        'subject_name':      pt.get('subject_name'),
+                        'session_name':      pt.get('session_name'),
+                        'dataset_name':      pt.get('dataset_name'),
+                        'parameter_set_num': param_str,
+                    }
+                    # sync global subject + param set
+                    st.session_state['_srt_pending_subjects'] = [pt.get('subject_name')]
+                    if param_str in param_options:
+                        st.session_state['_srt_pending_param'] = [param_str]
+                    st.rerun()
 
     # Tab 2: Unit explorer
     with tab_explorer:
@@ -935,12 +1043,19 @@ def sorting_tab(schema=None):
 
         if _exp_unit_criteria_id is not None:
             sua_df = _get_all_sua_counts(schema, _exp_unit_criteria_id)
+            # restrict to the globally selected parameter set(s)
+            sua_df = sua_df[sua_df['parameter_set_num'].isin(selected_params)]
             if not sua_df.empty:
                 sel_key_now = st.session_state.get('srt_selected_key', {})
                 _sel_subj = sel_key_now.get('subject_name')
-                if _sel_subj and _sel_subj in sua_df['subject_name'].values:
+                # follow the global Subjects selector: show every selected subject
+                # (each gets its own colour); fall back to the clicked session's
+                # subject when nothing is selected.
+                if selected_subjects:
+                    sua_df = sua_df[sua_df['subject_name'].isin(selected_subjects)]
+                elif _sel_subj and _sel_subj in sua_df['subject_name'].values:
                     sua_df = sua_df[sua_df['subject_name'] == _sel_subj]
-                # One row per session: sum SUA across all parameter sets and probes.
+                # One row per session: sum SUA across the selected parameter sets and probes.
                 # Retain the parameter_set_num with the highest SUA for click-to-load.
                 _best_param = (sua_df.sort_values('sua', ascending=False)
                                .drop_duplicates(['subject_name', 'session_name', 'dataset_name'])
@@ -1071,24 +1186,6 @@ def sorting_tab(schema=None):
 
     # Tab 3: Unit counts
     with tab_units:
-        sessions_all = schema.SpikeSorting().fetch(format='frame').reset_index()
-
-        st.session_state.setdefault('srt_param_sel_units', default_params)
-        sel_params_2 = st.multiselect(
-            'Parameter sets',
-            options=list(param_options.keys()),
-            format_func=lambda k: param_options[k],
-            key='srt_param_sel_units',
-        )
-        selected_params_2 = sel_params_2 if sel_params_2 else list(param_options.keys())
-
-        sess2 = sessions_all[sessions_all['parameter_set_num'].isin(
-            [int(p) if p.isdigit() else p for p in selected_params_2]
-        )]
-        subjects2 = np.unique(sess2['subject_name'].values) if len(sess2) else []
-
-        selected_subjects2 = st.multiselect('Subject:', subjects2, key='srt_subjects')
-
         criteria_ids2 = schema.UnitCountCriteria().fetch(as_dict=True)
         if not criteria_ids2:
             st.write('Add a UnitCountCriteria to the table: example '
@@ -1106,10 +1203,10 @@ def sorting_tab(schema=None):
             unit_criteria2 = (int(selection2.split('->')[0].strip(' []'))
                               if '->' in selection2 else None)
 
-            if selected_subjects2 and unit_criteria2 is not None:
+            if selected_subjects and unit_criteria2 is not None:
                 units2 = []
-                for p in selected_params_2:
-                    for s in selected_subjects2:
+                for p in selected_params:
+                    for s in selected_subjects:
                         units2.append(_get_unit_counts(schema, s, p, unit_criteria2))
                 units2 = [df for df in units2 if not df.empty]
                 if units2:

{labdata-0.1.9 → labdata-0.1.11}/labdata/schema/ephys.py

@@ -697,6 +697,30 @@ class UnitCount(dj.Computed):
         keys = [dict(a,unit_criteria_id = key['unit_criteria_id']) for i,a in allu.iterrows()]
         self.Unit.insert(keys, ignore_extra_fields = True) # add to the Unit part table
 
+
+def _apply_unit_criteria(unitmetrics,criteria):
+    parameters = [p.strip(' ') for p in criteria.split('&')]
+    operators = {'<': np.less, '>': np.greater, '<=':np.less_equal, '>=':np.greater_equal, '==': np.equal, '!=':np.not_equal}
+    idx = []
+    unitmetrics = unitmetrics.replace({None: np.nan})
+    for p in parameters:
+        for o in operators.keys():
+            if o in p:
+                n = [t.strip(' ') for t in p.split(o)]
+                try:
+                    n[1] = float(n[1])
+                except:
+                    print(f'Could not convert {n[1]} to float.')
+                    pass
+                if '|' in n[0]: # then take the absolute value
+                    n[0] = n[0].strip('|')
+                    idx.append(operators[o](np.abs(unitmetrics[n[0]].values),n[1]))
+                else:
+                    idx.append(operators[o](unitmetrics[n[0]].values,n[1]))
+    for i in idx:
+        idx[0] = np.logical_and(idx[0],i)
+    return idx[0]
+
 from .histology import Atlas
 
 @analysisschema

{labdata-0.1.9 → labdata-0.1.11}/labdata/schema/onephoton.py

@@ -24,7 +24,7 @@ class Widefield(dj.Imported):
     width                  : int                 # width of each frame
     height                 : int                 # height of each frame
     frame_rate             : double              # frame rate
-    [nullable] -> File                           # path to the stack
+    -> [nullable] File                           # path to the stack
     magnification = NULL   : double              # magnification
     objective_angle = NULL : double              # angle
     objective = NULL       : varchar(32)         # objective

{labdata-0.1.9 → labdata-0.1.11}/labdata.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: labdata
-Version: 0.1.9
+Version: 0.1.11
 Summary: Package to manage data in experimental neuroscience labs
 Author-email: Joao Couto <jpcouto@gmail.com>
 Project-URL: Homepage, https://github.com/jcouto/labdata

{labdata-0.1.9 → labdata-0.1.11}/labdata.egg-info/SOURCES.txt

@@ -30,6 +30,7 @@ labdata/dashboard/ai_utils.py
 labdata/dashboard/compute.py
 labdata/dashboard/console.py
 labdata/dashboard/decisiontask.py
+labdata/dashboard/docstrings.py
 labdata/dashboard/index.py
 labdata/dashboard/notes.py
 labdata/dashboard/procedures.py