labdata 0.1.3__tar.gz → 0.1.6__tar.gz

{labdata-0.1.3 → labdata-0.1.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: labdata
-Version: 0.1.3
+Version: 0.1.6
 Summary: Package to manage data in experimental neuroscience labs
 Author-email: Joao Couto <jpcouto@gmail.com>
 Project-URL: Homepage, https://github.com/jcouto/labdata
@@ -27,6 +27,8 @@ Requires-Dist: streamlit>1.54; extra == "dashboard"
 Requires-Dist: streamlit-bokeh; extra == "dashboard"
 Requires-Dist: scipy; extra == "dashboard"
 Requires-Dist: scikit-image; extra == "dashboard"
+Requires-Dist: networkx; extra == "dashboard"
+Requires-Dist: streamlit-agraph; extra == "dashboard"
 Dynamic: license-file
 
 ## labdata

{labdata-0.1.3 → labdata-0.1.6}/labdata/__init__.py

@@ -1,4 +1,4 @@
-VERSION = "0.1.3"
+VERSION = "0.1.6"
 
 from .utils import *
 from .copy import copy_to_upload_server

{labdata-0.1.3 → labdata-0.1.6}/labdata/cli.py

@@ -62,6 +62,8 @@ Server commands (don't run on experimental computers):
         parser.add_argument('--debug' ,action = 'store_true', default = False, help = 'To debug the dash (dashboard) errors.')
         parser.add_argument('--console', action = 'store_true', default = False,
                             help = 'Include the IPython console tab in the dashboard.')
+        parser.add_argument('--admin', action = 'store_true', default = False,
+                            help = 'Launch dashboard in admin mode (shows admin tab with storage stats and project switcher).')
         project = _get_project()
         parser.add_argument('-p','--project',
                             default = project,
@@ -95,6 +97,8 @@ Server commands (don't run on experimental computers):
                 command += f' project={args.project}'
             if args.console:
                 command += ' console=1'
+            if args.admin:
+                command += ' admin=1'
             sub.run(command.split(' '))
     
     def subjects(self):

{labdata-0.1.3 → labdata-0.1.6}/labdata/compute/caiman.py

@@ -61,7 +61,7 @@ class CaimanCompute(Suite2pCompute):
                                 help = "Expected halfwidth of the neurons in pixels")
             parser.add_argument('--gsig-filt',
                                 action='store', default=None, type = int, nargs = 2,
-                                help = "Filter for 1photon data")
+                                help = "Filter for 1photon data, will highlight the somas for motion correction (defaults to [6, 6] for 1p data).")
             parser.add_argument('--nb',
                                 action='store', default=0, type = int,
                                 help = "number of background components (rank) if positive, set to 0 for CNMFE")
@@ -123,6 +123,7 @@ class CaimanCompute(Suite2pCompute):
                           p = int(args.p),
                           gSig = [int(a) for a in args.gsig], 
                           gSiz = [int(a) for a in 2*np.array(args.gsig) + 1],
+                          gSig_filt = args.gsig_filt,
                           merge_thr = float(args.merge_thr),      
                           rf = int(args.rf),                    
                           stride = int(args.stride),              
@@ -154,8 +155,9 @@ class CaimanCompute(Suite2pCompute):
         dset = (self.schema.Miniscope() & self.dataset_key)
         cnmfparams = {
             'motion_correct' : True,
+            'gSig_filt' : [6,6],                   # 1p needs this for motion correction (spatial filter to register somas)
             'method_init': 'corr_pnr',             # use this for 1 photon
-            'K': None,                             # for 1p                                
+            'K': None,                             # for 1p use None                            
             'nb': 0,                               # number of background components (rank) if positive, set to 0 for CNMFE
             'nb_patch': 0,
             'low_rank_background': None,           # for 1p
@@ -165,7 +167,6 @@ class CaimanCompute(Suite2pCompute):
             'center_psf': True,                    # True for 1p
             'only_init': True,                     # set it to True to run CNMF-E
             'method_deconvolution': 'oasis'}       # could use 'cvxpy' alternatively
-
         if not len(dset):
             dset = (self.schema.TwoPhoton() & self.dataset_key)
             # add the 2p parameters here and do the processing per plane.
@@ -179,25 +180,43 @@ class CaimanCompute(Suite2pCompute):
         parameters = (self.schema.CellSegmentationParams & f'parameter_set_num = {self.parameter_set_num}').fetch1()
 
         params = json.loads(parameters['parameters_dict'])
+        excluded = ['detrendWin','denoise_dff','quantileMin']
         for k in params.keys():
-            cnmfparams[k] = params[k]
-
+            if not k in excluded:
+                cnmfparams[k] = params[k]
+            
         import logging
         logger = logging.getLogger('caiman')
-        # Set to logging.INFO if you want much output, potentially much more output
+        # Set to logging.INFO if you want more, not very useful output..
         logger.setLevel(logging.WARNING)
         handler = logging.StreamHandler()
         logger.addHandler(handler)
         
         import time
         from caiman.source_extraction.cnmf.params import CNMFParams
-        parameters = CNMFParams(params_dict = dict({k:cnmfparams[k] for k in ['motion_correct','pw_rigid']},
+        kk = ['motion_correct','pw_rigid','max_shifts','gSig_filt']
+        if not is_two_photon: # for miniscope make sure the registration has gSig_filt
+            if 'gSig_filt' in params.keys():
+                if params['gSig_filt'] is None: # that disables it so use the default
+                    cnmfparams['gSig_filt'] = [6,6]
+                    print('[caiman] Using default gSig_filt: (6,6) because this is 1photon.')
+            else: # add it
+                cnmfparams['gSig_filt'] = [6,6]
+                print('[caiman] Forcing default gSig_filt: (6,6) because this is 1photon.')
+
+        parameters = CNMFParams(params_dict = dict({k:cnmfparams[k] for k in kk},
                                                    fr = frame_rate))
         
         os.environ['CAIMAN_DATA'] = f'{temporary_folder}'
 
         paths = export_to_tiff(dat,temporary_folder,
                                crop_region = self.parameters['roi'])
+        if not self.keep_intermediate:
+            if not self.files_existed:
+                print('[caiman] Freeing up space by deleting raw data that was downloaded for this compute task.')
+                for f in (self.schema.File & dset).get():
+                    os.unlink(f)
+
         pmotion = parameters.get_group('motion')
 
         n_cpus = DEFAULT_N_JOBS
@@ -209,7 +228,7 @@ class CaimanCompute(Suite2pCompute):
         mot_correct.motion_correct(save_movie=True)
         print(f'Done with motion correction in {(time.time() - tstart)/60.} min.')
         
-        [os.unlink(f) for f in paths]
+        [os.unlink(f) for f in paths] # clear raw converted tiff
         fname_mc = mot_correct.fname_tot_els if cnmfparams['pw_rigid'] else mot_correct.fname_tot_rig
         if pmotion['pw_rigid']:
             bord_px = np.ceil(np.maximum(np.max(np.abs(mot_correct.x_shifts_els)),
@@ -222,6 +241,7 @@ class CaimanCompute(Suite2pCompute):
         fname_new = save_memmap(fname_mc, base_name='memmap_', order='C',
                                 border_to_0 = bord_px)
         [os.unlink(f) for f in fname_mc]
+        [os.unlink(f) for f in temporary_folder.rglob("*chunk_*order_F*")]; # clear motion correction temporary
         
         from caiman import load_memmap
         Yr, dims, T = load_memmap(fname_new)
@@ -335,9 +355,6 @@ class CaimanCompute(Suite2pCompute):
             print(f'[{self.name} job] Removing the temporary folder.')
             import shutil
             shutil.rmtree(temporary_folder)
-            if not self.files_existed:
-                for f in (self.schema.File & dset).get():
-                    os.unlink(f)
         else:
             print(f'[{self.name} job] Kept the temporary folder {temporary_folder}.')
         self.schema.CellSegmentationMetrics().populate(dkey, display_progress = True)

labdata-0.1.6/labdata/dashboard/admin.py

@@ -0,0 +1,385 @@
+"""Admin dashboard tab; file storage statistics per project and users. Adds users to projects"""
+import pandas as pd
+import streamlit as st
+import plotly.express as px
+
+
+def _query_project(conn, base_db, project_name):
+    """
+    Use mysql here because we don't want to be loading all projects but need to access Dataset.DataFiles.
+    
+    Run aggregation queries for one project via the database connection.
+    Returns (total_bytes, n_files, dtype_rows) where dtype_rows is a list of
+    {'project', 'dataset_type', 'total_bytes'} dicts.
+    """
+    project_db = f"{base_db}_{project_name}"
+
+    # Total raw data size for the project
+    total_bytes, n_files = 0, 0
+    try:
+        cur = conn.query(f"""
+            SELECT COALESCE(SUM(f.file_size), 0), COUNT(*)
+            FROM `{base_db}`.`file` f
+            INNER JOIN `{project_db}`.`dataset__data_files` df
+                ON f.file_path = df.file_path AND f.storage = df.storage
+        """)
+        row = cur.fetchone()
+        if row:
+            total_bytes = float(row[0] or 0)
+            n_files = int(row[1] or 0)
+    except Exception:
+        pass
+
+    # Size per dataset type
+    dtype_rows = []
+    try:
+        cur = conn.query(f"""
+            SELECT COALESCE(d.dataset_type, 'unknown'),
+                   COALESCE(SUM(f.file_size), 0)
+            FROM `{base_db}`.`file` f
+            INNER JOIN `{project_db}`.`dataset__data_files` df
+                ON f.file_path = df.file_path AND f.storage = df.storage
+            INNER JOIN `{project_db}`.`dataset` d
+                ON df.subject_name = d.subject_name
+               AND df.session_name = d.session_name
+               AND df.dataset_name = d.dataset_name
+            GROUP BY d.dataset_type
+        """)
+        for dtype, size in cur.fetchall():
+            dtype_rows.append({
+                'project': project_name,
+                'dataset_type': dtype,
+                'total_bytes': float(size or 0),
+            })
+    except Exception:
+        pass
+
+    return total_bytes, n_files, dtype_rows
+
+
+@st.cache_data(ttl=3600)
+def get_storage_stats(_schema):
+    """
+    Query file sizes across all projects using the database connection.
+    
+    Returns (summary_df, dtype_df).
+      summary_df: project, n_files, total_bytes, size_tb
+      dtype_df:   project, dataset_type, total_bytes, size_tb
+    """
+    base_db = _schema.globalschema.database
+    conn = _schema.globalschema.connection
+    projects = list(_schema.Project.fetch('project_name'))
+
+    summary_rows, dtype_rows = [], []
+    for project_name in projects:
+        total_bytes, n_files, dtype = _query_project(conn, base_db, project_name)
+        summary_rows.append({
+            'project': project_name,
+            'n_files': n_files,
+            'total_bytes': total_bytes,
+            'size_tb': total_bytes / 1e12,
+        })
+        dtype_rows.extend(dtype)
+
+    summary_df = pd.DataFrame(summary_rows).sort_values('total_bytes', ascending=False)
+    dtype_df = pd.DataFrame(dtype_rows) if dtype_rows else pd.DataFrame(
+        columns=['project', 'dataset_type', 'total_bytes'])
+    if not dtype_df.empty:
+        dtype_df['size_tb'] = dtype_df['total_bytes'] / 1e12
+
+    return summary_df, dtype_df
+
+
+@st.cache_data(ttl=3600)
+def get_schema_sizes(_schema):
+    """
+    Query information_schema for the on-disk size of every database that
+    belongs to this lab installation (prefix = globalschema.database).
+    Returns a DataFrame with columns: project, schema_type, size_tb.
+    Schema types are derived from the suffix after the project name:
+      (none)       'data'
+      _computed    'computed'
+      anything else → that suffix (e.g. a per-user schema)
+    The global schema (no project suffix) is labelled project='(global)', type='global'.
+    """
+    base_db = _schema.globalschema.database
+    conn = _schema.globalschema.connection
+    projects = set(_schema.Project.fetch('project_name'))
+
+    escaped = base_db.replace('_', r'\_')
+    try:
+        cur = conn.query(
+            f"SELECT table_schema,"
+            f" COALESCE(SUM(data_length + index_length), 0) AS size_bytes"
+            f" FROM information_schema.tables"
+            f" WHERE table_schema LIKE '{escaped}%%' ESCAPE '\\\\'"
+            f" GROUP BY table_schema"
+        )
+        rows = cur.fetchall()
+    except Exception as e:
+        return pd.DataFrame({'_error': [str(e)], 'project': [''], 'schema_type': [''], 'size_tb': [0.0]})
+
+    records = []
+    for schema_name, size_bytes in rows:
+        if schema_name == base_db:
+            project, schema_type = '(global)', 'global'
+        else:
+            suffix = schema_name[len(base_db) + 1:]   # strip "base_db_"
+            # identify which project this belongs to
+            matched_project = None
+            for p in sorted(projects, key=len, reverse=True):
+                if suffix == p or suffix.startswith(p + '_'):
+                    matched_project = p
+                    break
+            if matched_project is None:
+                continue
+            remainder = suffix[len(matched_project):]  # '' or '_computed' or '_user'
+            schema_type = remainder.lstrip('_') or 'data'
+            project = matched_project
+
+        records.append({
+            'project': project,
+            'schema_type': schema_type,
+            'size_tb': float(size_bytes) / 1e12,
+        })
+
+    return pd.DataFrame(records)
+
+
+@st.cache_data(ttl=300)
+def get_project_users(_schema):
+    """
+    One row per project with description and comma-separated user list.
+    Shows all projects, even those with no assigned users.
+    """
+    try:
+        projects = pd.DataFrame(
+            _schema.Project.fetch('project_name', 'project_description', as_dict=True))
+        users = pd.DataFrame(
+            _schema.Project.User().fetch('project_name', 'user_name', as_dict=True))
+        if projects.empty:
+            return pd.DataFrame()
+        if users.empty:
+            projects['Users'] = ''
+        else:
+            user_summary = (
+                users.groupby('project_name')['user_name']
+                     .apply(lambda u: ', '.join(sorted(u)))
+                     .reset_index()
+                     .rename(columns={'user_name': 'Users'})
+            )
+            projects = projects.merge(user_summary, on='project_name', how='left')
+            projects['Users'] = projects['Users'].fillna('')
+        return (projects
+                .rename(columns={'project_name': 'Project',
+                                 'project_description': 'Description'})
+                [['Project', 'Description', 'Users']])
+    except Exception:
+        return pd.DataFrame()
+
+
+@st.dialog("Add user to project")
+def _add_user_dialog(project, user):
+    st.write(f"Add **{user}** to project **{project}**?")
+    col1, col2 = st.columns(2)
+    if col1.button("Confirm", type="primary", key="add_confirm"):
+        from labdata.export import add_user_to_project
+        try:
+            add_user_to_project(project, user)
+            get_project_users.clear()
+            st.session_state['admin_msg'] = ('success', f'Added {user} to {project}.')
+        except Exception as e:
+            st.session_state['admin_msg'] = ('error', str(e))
+        st.rerun()
+    if col2.button("Cancel", key="add_cancel"):
+        st.rerun()
+
+
+@st.dialog("Remove user from project")
+def _remove_user_dialog(project, user):
+    st.warning(
+        f"Remove **{user}** from project **{project}**? "
+        f"This will revoke their database permissions for this project.")
+    col1, col2 = st.columns(2)
+    if col1.button("Confirm", type="primary", key="remove_confirm"):
+        from labdata.export import remove_user_from_project
+        try:
+            remove_user_from_project(project, user)
+            get_project_users.clear()
+            st.session_state['admin_msg'] = ('success', f'Removed {user} from {project}.')
+        except Exception as e:
+            st.session_state['admin_msg'] = ('error', str(e))
+        st.rerun()
+    if col2.button("Cancel", key="remove_cancel"):
+        st.rerun()
+
+
+def admin_tab(schema, **kwargs):
+    st.header("``labdata`` administration and data overview ")
+
+    with st.spinner("Querying storage stats across all projects..."):
+        summary_df, dtype_df = get_storage_stats(schema)
+        schema_df = get_schema_sizes(schema)
+
+    _axis_style = dict(
+        showline=True, linecolor='black', linewidth=1.5,
+        ticks='outside', tickcolor='black', tickfont=dict(color='black'),
+        title_font=dict(color='black'),
+    )
+    _layout_base = dict(
+        paper_bgcolor='white', plot_bgcolor='white',
+        font=dict(color='black'),
+        hoverlabel=dict(font=dict(color='black')),
+        xaxis=_axis_style,
+        yaxis=dict(**_axis_style, gridcolor='rgba(0,0,0,0.08)'),
+    )
+
+    if summary_df.empty:
+        st.warning("No file data found.")
+    else:
+        # total size and file count 
+        col_size, col_files = st.columns(2)
+
+        with col_size:
+            st.subheader("Raw file size per project")
+            fig = px.bar(
+                summary_df,
+                x='project', y='size_tb',
+                labels={'project': 'Project', 'size_tb': 'Size (TB)'},
+                text=summary_df['size_tb'].apply(lambda v: f'{v:.3f} TB'),
+            )
+            fig.update_traces(textposition='outside',
+                              marker_color='rgba(112,112,112,0.7)',
+                              marker_line_color='black', marker_line_width=1.5)
+            fig.update_layout(yaxis_title='Size (TB)', xaxis_title='Project',
+                              showlegend=False, **_layout_base)
+            st.plotly_chart(fig, width='stretch')
+
+        with col_files:
+            st.subheader("Number of raw files per project")
+
+            fig_n = px.bar(
+                summary_df,
+                x='project', y='n_files',
+                labels={'project': 'Project', 'n_files': 'Number of Files'},
+                text='n_files',
+            )
+            fig_n.update_traces(textposition='outside',
+                                marker_color='rgba(112,112,112,0.7)',
+                                marker_line_color='black', marker_line_width=1.5)
+            fig_n.update_layout(yaxis_title='Number of Files', xaxis_title='Project',
+                                showlegend=False, **_layout_base)
+            st.plotly_chart(fig_n, width='stretch')
+
+        # dataset type and  schema size
+        col_dtype, col_schema = st.columns(2)
+
+        with col_dtype:
+            if not dtype_df.empty:
+                st.subheader("Raw file size per dataset type")
+                fig2 = px.bar(
+                    dtype_df,
+                    x='project', y='size_tb', color='dataset_type',
+                    barmode='stack',
+                    labels={'project': 'Project', 'size_tb': 'Size (TB)',
+                            'dataset_type': 'Dataset Type'},
+                )
+                fig2.update_layout(
+                    yaxis_title='Size (TB)', xaxis_title='Project',
+                    legend_font=dict(color='black'), legend_title_font=dict(color='black'),
+                    **_layout_base,
+                )
+                st.plotly_chart(fig2, width='stretch')
+
+        with col_schema:
+            st.subheader("Database size per project")
+            if '_error' in schema_df.columns:
+                st.warning(f"Could not query schema sizes: {schema_df['_error'].iloc[0]}")
+            elif schema_df.empty:
+                st.info("No schema size data found.")
+            else:
+                fig3 = px.bar(
+                    schema_df,
+                    x='project', y='size_tb', color='schema_type',
+                    barmode='group',
+                    labels={'project': 'Project', 'size_tb': 'Size (TB)',
+                            'schema_type': 'Schema'},
+                )
+                fig3.update_layout(
+                    yaxis_title='Size (TB)', xaxis_title='Project',
+                    legend_font=dict(color='black'), legend_title_font=dict(color='black'),
+                    **_layout_base,
+                )
+                st.plotly_chart(fig3, width='stretch')
+
+    col_users, col_add = st.columns(2)
+
+    with col_users:
+        st.subheader("Users and project assignments")
+        users_df = get_project_users(schema)
+        if users_df.empty:
+            st.info("No user/project associations found.")
+        else:
+            st.dataframe(
+                users_df.style.map(lambda _: 'font-weight: bold', subset=['Project']),
+                hide_index=True, width='stretch')
+
+    with col_add:
+        # show feedback message from the last dialog action
+        if 'admin_msg' in st.session_state:
+            level, text = st.session_state.pop('admin_msg')
+            if level == 'success':
+                st.success(text)
+            else:
+                st.error(text)
+
+        try:
+            all_projects = list(schema.Project.fetch('project_name'))
+            all_users = list(schema.LabMember.fetch('user_name'))
+        except Exception:
+            all_projects, all_users = [], []
+
+        if all_projects and all_users:
+            # add user 
+            st.subheader("Add user to project")
+            target_project = st.selectbox('Project', options=all_projects,
+                                          key='add_project_sel')
+            target_user = st.selectbox('User', options=all_users,
+                                       key='add_user_sel')
+            if st.button('Add User', type='primary', key='add_user_btn'):
+                _add_user_dialog(target_project, target_user)
+
+            # remove user 
+            st.markdown("""<style>
+[data-testid="stVerticalBlockBorderWrapper"] {
+    border: 2px solid red !important;
+    border-radius: 6px;
+}
+</style>""", unsafe_allow_html=True)
+
+            with st.container(border=True):
+                st.subheader("Remove user from project")
+                remove_project = st.selectbox(
+                    'Project', options=all_projects, key='remove_project_sel')
+                try:
+                    project_users = list(
+                        (schema.Project.User() & f'project_name = "{remove_project}"')
+                        .fetch('user_name'))
+                except Exception:
+                    project_users = []
+
+                if project_users:
+                    remove_user = st.selectbox(
+                        'User', options=project_users, key='remove_user_sel')
+                    if st.button('Remove user', type='primary', key='remove_user_btn'):
+                        _remove_user_dialog(remove_project, remove_user)
+                else:
+                    st.info("No users assigned to this project.")
+        else:
+            st.info("No projects or users available.")
+
+    if st.button("Refresh stats"):
+        get_storage_stats.clear()
+        get_schema_sizes.clear()
+        get_project_users.clear()
+        st.rerun()