LCNE-patchseq-analysis 0.1.0__tar.gz → 0.3.0__tar.gz

{lcne_patchseq_analysis-0.1.0 → lcne_patchseq_analysis-0.3.0}/.flake8

@@ -4,6 +4,7 @@ exclude =
     __pycache__,
     build,
     .venv,
-    venv
+    venv,
+    code/
 max-complexity = 10
 max-line-length = 100

{lcne_patchseq_analysis-0.1.0 → lcne_patchseq_analysis-0.3.0}/.gitignore

@@ -140,4 +140,5 @@ dmypy.json
 
 .vscode
 metadata.yml
-data
+data
+LIMS_credentials.json

{lcne_patchseq_analysis-0.1.0 → lcne_patchseq_analysis-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: LCNE-patchseq-analysis
-Version: 0.1.0
+Version: 0.3.0
 Summary: Generated from aind-library-template
 Author: Allen Institute for Neural Dynamics
 Author-email: Han Hou <han.hou@alleninstitute.org>
@@ -11,7 +11,6 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: numpy
 Requires-Dist: pandas
-Requires-Dist: openpyxl
 Requires-Dist: matplotlib
 Provides-Extra: dev
 Requires-Dist: black; extra == "dev"
@@ -21,13 +20,23 @@ Requires-Dist: interrogate; extra == "dev"
 Requires-Dist: isort; extra == "dev"
 Requires-Dist: Sphinx; extra == "dev"
 Requires-Dist: furo; extra == "dev"
+Provides-Extra: pipeline
+Requires-Dist: black; extra == "pipeline"
+Requires-Dist: coverage; extra == "pipeline"
+Requires-Dist: flake8; extra == "pipeline"
+Requires-Dist: interrogate; extra == "pipeline"
+Requires-Dist: isort; extra == "pipeline"
+Requires-Dist: Sphinx; extra == "pipeline"
+Requires-Dist: furo; extra == "pipeline"
+Requires-Dist: openpyxl; extra == "pipeline"
+Requires-Dist: pg8000; extra == "pipeline"
 
 # LCNE-patchseq-analysis
 
 [![License](https://img.shields.io/badge/license-MIT-brightgreen)](LICENSE)
 ![Code Style](https://img.shields.io/badge/code%20style-black-black)
 [![semantic-release: angular](https://img.shields.io/badge/semantic--release-angular-e10079?logo=semantic-release)](https://github.com/semantic-release/semantic-release)
-![Interrogate](https://img.shields.io/badge/interrogate-100.0%25-brightgreen)
+![Interrogate](https://img.shields.io/badge/interrogate-80.0%25-yellow)
 ![Coverage](https://img.shields.io/badge/coverage-100%25-brightgreen?logo=codecov)
 ![Python](https://img.shields.io/badge/python->=3.9-blue?logo=python)
 

{lcne_patchseq_analysis-0.1.0 → lcne_patchseq_analysis-0.3.0}/README.md

@@ -3,7 +3,7 @@
 [![License](https://img.shields.io/badge/license-MIT-brightgreen)](LICENSE)
 ![Code Style](https://img.shields.io/badge/code%20style-black-black)
 [![semantic-release: angular](https://img.shields.io/badge/semantic--release-angular-e10079?logo=semantic-release)](https://github.com/semantic-release/semantic-release)
-![Interrogate](https://img.shields.io/badge/interrogate-100.0%25-brightgreen)
+![Interrogate](https://img.shields.io/badge/interrogate-80.0%25-yellow)
 ![Coverage](https://img.shields.io/badge/coverage-100%25-brightgreen?logo=codecov)
 ![Python](https://img.shields.io/badge/python->=3.9-blue?logo=python)
 

{lcne_patchseq_analysis-0.1.0 → lcne_patchseq_analysis-0.3.0}/docs/source/conf.py

@@ -1,4 +1,5 @@
 """Configuration file for the Sphinx documentation builder."""
+
 #
 # For the full list of built-in configuration values, see the documentation:
 # https://www.sphinx-doc.org/en/master/usage/configuration.html

lcne_patchseq_analysis-0.3.0/notebook/demo.ipynb

@@ -0,0 +1,194 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "The autoreload extension is already loaded. To reload it, use:\n",
+      "  %reload_ext autoreload\n"
+     ]
+    }
+   ],
+   "source": [
+    "%load_ext autoreload\n",
+    "%autoreload 2\n",
+    "import logging\n",
+    "logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from LCNE_patchseq_analysis.data_util.metadata import read_brian_spreadsheet, cross_check_metadata"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "##  Load patchseq metadata and perform cross check"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO: Reading metadata from C:\\Users\\han.hou\\Downloads\\IVSCC_LC_summary.xlsx...\n",
+      "INFO: Querying and adding LIMS data...\n",
+      "INFO: Merged LIMS to spreadsheet, total 390 rows: 146 in both, 125 in spreadsheet only, 119 in LIMS only.\n",
+      "\n",
+      "INFO: \n",
+      "INFO: --------------------------------------------------\n",
+      "INFO: Cross-checking metadata between tab_xyz and master tables...\n",
+      "INFO: Source columns: ['x_tab_xyz', 'y_tab_xyz', 'z_tab_xyz', 'Annotated structure_tab_xyz', 'notes_tab_xyz']\n",
+      "INFO: Master columns: ['x_tab_master', 'y_tab_master', 'z_tab_master', 'Annotated structure_tab_master', 'notes_tab_master']\n",
+      "WARNING: Found 7 inconsistencies between x_tab_xyz and x_tab_master:\n",
+      "WARNING:       Date                      jem-id_cell_specimen  x_tab_master   x_tab_xyz\n",
+      "2023-09-01 Dbh-Cre_KH212;RCL-H2B-GFP-692026.10.10.02  10534.982420 10151.01953\n",
+      "2023-08-20 Dbh-Cre_KH212;RCL-H2B-GFP-692023.08.06.01  10541.875980 10702.28320\n",
+      "2023-08-20 Dbh-Cre_KH212;RCL-H2B-GFP-692023.08.06.02  10702.283200 10761.00195\n",
+      "2023-06-02 Dbh-Cre_KH212;RCL-H2B-GFP-676766.10.06.03  10521.757810 10541.87598\n",
+      "2023-03-15                   C57BL6J-665266.11.06.03  10451.809570 10534.98242\n",
+      "2023-01-20     Ndnf-IRES2-dgCre;Ai14-659663.11.06.03  10391.497070 10521.75781\n",
+      "2023-01-20     Ndnf-IRES2-dgCre;Ai14-659663.11.06.04   9531.198242 10451.80957\n",
+      "WARNING: \n",
+      "WARNING: Found 7 inconsistencies between y_tab_xyz and y_tab_master:\n",
+      "WARNING:       Date                      jem-id_cell_specimen  y_tab_master   y_tab_xyz\n",
+      "2023-09-01 Dbh-Cre_KH212;RCL-H2B-GFP-692026.10.10.02   4183.531250 3701.974609\n",
+      "2023-08-20 Dbh-Cre_KH212;RCL-H2B-GFP-692023.08.06.01   4110.681641 3840.954834\n",
+      "2023-08-20 Dbh-Cre_KH212;RCL-H2B-GFP-692023.08.06.02   3840.954834 4288.832031\n",
+      "2023-06-02 Dbh-Cre_KH212;RCL-H2B-GFP-676766.10.06.03   4256.657715 4110.681641\n",
+      "2023-03-15                   C57BL6J-665266.11.06.03   4402.110352 4183.531250\n",
+      "2023-01-20     Ndnf-IRES2-dgCre;Ai14-659663.11.06.03   4161.165039 4256.657715\n",
+      "2023-01-20     Ndnf-IRES2-dgCre;Ai14-659663.11.06.04   2449.594727 4402.110352\n",
+      "WARNING: \n",
+      "WARNING: Found 5 inconsistencies between z_tab_xyz and z_tab_master:\n",
+      "WARNING:       Date                      jem-id_cell_specimen  z_tab_master  z_tab_xyz\n",
+      "2023-09-01 Dbh-Cre_KH212;RCL-H2B-GFP-692026.10.10.02        4984.0     4824.0\n",
+      "2023-08-20 Dbh-Cre_KH212;RCL-H2B-GFP-692023.08.06.01        5034.0     4727.0\n",
+      "2023-06-02 Dbh-Cre_KH212;RCL-H2B-GFP-676766.10.06.03        4889.0     5034.0\n",
+      "2023-03-15                   C57BL6J-665266.11.06.03        4889.0     4984.0\n",
+      "2023-01-20     Ndnf-IRES2-dgCre;Ai14-659663.11.06.04        4265.0     4889.0\n",
+      "WARNING: \n",
+      "INFO: All good between Annotated structure_tab_xyz and Annotated structure_tab_master!\n",
+      "INFO: All good between notes_tab_xyz and notes_tab_master!\n",
+      "INFO: \n",
+      "INFO: --------------------------------------------------\n",
+      "INFO: Cross-checking metadata between tab_ephys_fx and master tables...\n",
+      "INFO: Source columns: ['failed_electrode_0_tab_ephys_fx', 'failed_no_seal_tab_ephys_fx', 'failed_bad_rs_tab_ephys_fx']\n",
+      "INFO: Master columns: ['failed_electrode_0_tab_master', 'failed_no_seal_tab_master', 'failed_bad_rs_tab_master']\n",
+      "INFO: All good between failed_electrode_0_tab_ephys_fx and failed_electrode_0_tab_master!\n",
+      "WARNING: Found 6 inconsistencies between failed_no_seal_tab_ephys_fx and failed_no_seal_tab_master:\n",
+      "WARNING:       Date                            jem-id_cell_specimen  failed_no_seal_tab_master  failed_no_seal_tab_ephys_fx\n",
+      "2024-04-03       Dbh-Cre_KH212;RCL-H2B-GFP-724916.11.06.02                        1.0                          0.0\n",
+      "2023-04-19           Slc17a6-IRES-Cre;Ai14-670829.11.06.02                        1.0                          0.0\n",
+      "2022-11-17           Slc17a6-IRES-Cre;Ai14-651168.10.06.03                        1.0                          0.0\n",
+      "2022-11-15 Dbh-Cre_KH212;RCL-Sun1sfGFP-neo-650884.09.06.05                        1.0                          0.0\n",
+      "2022-11-02             Rbp4-Cre_KL100;Ai14-650443.10.06.02                        1.0                          0.0\n",
+      "2022-10-27                         C57BL6J-647687.09.06.01                        1.0                          0.0\n",
+      "WARNING: \n",
+      "WARNING: Found 2 inconsistencies between failed_bad_rs_tab_ephys_fx and failed_bad_rs_tab_master:\n",
+      "WARNING:       Date    jem-id_cell_specimen  failed_bad_rs_tab_master  failed_bad_rs_tab_ephys_fx\n",
+      "2024-03-20 C57BL6J-722426.10.06.01                       1.0                         0.0\n",
+      "2024-03-20 C57BL6J-722426.10.06.03                       1.0                         0.0\n",
+      "WARNING: \n",
+      "INFO: \n",
+      "INFO: --------------------------------------------------\n",
+      "INFO: Cross-checking metadata between lims and master tables...\n",
+      "INFO: Source columns: ['cell_specimen_id_lims', 'ephys_roi_id_lims', 'ephys_qc_lims', 'storage_directory_lims']\n",
+      "INFO: Master columns: ['cell_specimen_id_tab_master', 'ephys_roi_id_tab_master', 'ephys_qc_tab_master', 'storage_directory_tab_master']\n",
+      "INFO: All good between cell_specimen_id_lims and cell_specimen_id_tab_master!\n",
+      "INFO: All good between ephys_roi_id_lims and ephys_roi_id_tab_master!\n",
+      "INFO: All good between ephys_qc_lims and ephys_qc_tab_master!\n",
+      "INFO: All good between storage_directory_lims and storage_directory_tab_master!\n"
+     ]
+    }
+   ],
+   "source": [
+    "dfs = read_brian_spreadsheet()\n",
+    "for source in [\"tab_xyz\", \"tab_ephys_fx\", \"lims\"]:\n",
+    "    df_inconsistencies = cross_check_metadata(dfs[\"df_merged\"], source)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### ❌ Oh no! These inconsistencies must be caused by manually copying and pasting across the tabs!!!"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Quick overview using pygwalker"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!pip install pygwalker --quiet"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "NameError",
+     "evalue": "name 'df' is not defined",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
+      "Cell \u001b[1;32mIn[15], line 2\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mpygwalker\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mas\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mpyg\u001b[39;00m\n\u001b[1;32m----> 2\u001b[0m walker \u001b[38;5;241m=\u001b[39m pyg\u001b[38;5;241m.\u001b[39mwalk(\u001b[43mdf\u001b[49m)\n\u001b[0;32m      3\u001b[0m walker\n",
+      "\u001b[1;31mNameError\u001b[0m: name 'df' is not defined"
+     ]
+    }
+   ],
+   "source": [
+    "import pygwalker as pyg\n",
+    "walker = pyg.walk(df)\n",
+    "walker"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "patchseq_pipeline",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.21"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

{lcne_patchseq_analysis-0.1.0 → lcne_patchseq_analysis-0.3.0}/pyproject.toml

@@ -20,7 +20,6 @@ dynamic = ["version"]
 dependencies = [
     'numpy',
     'pandas',
-    'openpyxl',
     'matplotlib',
 ]
 
@@ -35,6 +34,18 @@ dev = [
     'furo'
 ]
 
+pipeline = [
+    'black',
+    'coverage',
+    'flake8',
+    'interrogate',
+    'isort',
+    'Sphinx',
+    'furo',
+    'openpyxl',
+    'pg8000',
+]
+
 [tool.setuptools.packages.find]
 where = ["src"]
 
@@ -57,6 +68,7 @@ exclude = '''
     | _build
     | build
     | dist
+    | code
   )/
   | .gitignore
 )

lcne_patchseq_analysis-0.3.0/src/LCNE_patchseq_analysis/__init__.py

@@ -0,0 +1,3 @@
+"""Init package"""
+
+__version__ = "0.3.0"

lcne_patchseq_analysis-0.3.0/src/LCNE_patchseq_analysis/data_util/__init__.py

@@ -0,0 +1 @@
+"""Utils for accessing data"""

lcne_patchseq_analysis-0.3.0/src/LCNE_patchseq_analysis/data_util/ephys.py

@@ -0,0 +1,123 @@
+"""Ephys-related data utils"""
+
+import concurrent.futures
+import logging
+import os
+import subprocess
+
+import pandas as pd
+from tqdm import tqdm
+
+from LCNE_patchseq_analysis.data_util.metadata import read_brian_spreadsheet
+
+logger = logging.getLogger(__name__)
+
+s3_bucket = "s3://aind-scratch-data/aind-patchseq-data/raw"
+
+
+def sync_directory(local_dir, destination, if_copy=False):
+    """
+    Sync the local directory with the given S3 destination using aws s3 sync.
+    Returns a status string based on the command output.
+    """
+    try:
+        if if_copy:
+            # Run aws s3 cp command and capture the output
+            result = subprocess.run(
+                ["aws", "s3", "cp", local_dir, destination], capture_output=True, text=True
+            )
+        else:
+            # Run aws s3 sync command and capture the output
+            result = subprocess.run(
+                ["aws", "s3", "sync", local_dir, destination], capture_output=True, text=True
+            )
+        output = result.stdout + result.stderr
+
+        # Check output: if "upload:" appears, files were sent;
+        # otherwise, assume that nothing needed uploading.
+        if "upload:" in output:
+            logger.info(f"Uploaded {local_dir} to {destination}!")
+            return "successfully uploaded"
+        else:
+            logger.info(output)
+            logger.info(f"Already exists, skip {local_dir}.")
+            return "already exists, skip"
+    except Exception as e:
+        return f"error during sync: {e}"
+
+
+def upload_one(row, s3_bucket):
+    """Process a single row: normalize the path, check existence,
+    and perform (or simulate) the sync.
+    """
+    # Check if the storage_directory_combined value is null.
+    if pd.isnull(row["storage_directory_combined"]):
+        logger.info("The path is null")
+        status = "the path is null"
+        path = None
+    else:
+        # Normalize the path and prepend a backslash.
+        path = "\\" + os.path.normpath(row["storage_directory_combined"])
+        roi_name = os.path.basename(path)
+
+        # Check if the local path exists.
+        if not os.path.exists(path):
+            logger.info(f"Cannot find the path: {path}")
+            status = "cannot find the path"
+        else:
+            logger.info(f"Syncing {path} to {s3_bucket}/{roi_name}...")
+            status = sync_directory(path, s3_bucket + "/" + roi_name)
+    return {"storage_directory": path, "status": status}
+
+
+def upload_raw_from_isilon_to_s3_batch(df, s3_bucket=s3_bucket, max_workers=10):
+    """Upload raw data from Isilon to S3, using the metadata dataframe in parallel."""
+    results = []
+
+    # Create a thread pool to process rows in parallel.
+    with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
+        # Submit each row for processing.
+        futures = [executor.submit(upload_one, row, s3_bucket) for idx, row in df.iterrows()]
+
+        # Collect the results as they complete.
+        for future in tqdm(
+            concurrent.futures.as_completed(futures), total=len(futures), desc="Uploading..."
+        ):
+            results.append(future.result())
+
+    logger.info(f"Uploaded {len(results)} files to {s3_bucket} in parallel...")
+    logger.info(
+        f'Successful uploads: {len([r for r in results if r["status"] == "successfully uploaded"])}'
+    )
+    logger.info(f'Skiped: {len([r for r in results if r["status"] == "already exists, skip"])}')
+    logger.info(
+        f'Error during sync: {len([r for r in results if r["status"] == "error during sync"])}'
+    )
+    logger.info(
+        "Cannot find on Isilon: "
+        f'{len([r for r in results if r["status"] == "cannot find the path"])}'
+    )
+    logger.info(f'Null path: {len([r for r in results if r["status"] == "the path is null"])}')
+
+    return pd.DataFrame(results)
+
+
+def trigger_patchseq_upload(metadata_path=os.path.expanduser(R"~\Downloads\IVSCC_LC_summary.xlsx")):
+    # Generate a list of isilon paths
+    dfs = read_brian_spreadsheet(file_path=metadata_path, add_lims=True)
+    df_merged = dfs["df_merged"]
+
+    # Upload raw data
+    upload_raw_from_isilon_to_s3_batch(df_merged, s3_bucket=s3_bucket, max_workers=10)
+
+    # Also save df_merged as csv and upload to s3
+    df_merged.to_csv("df_metadata_merged.csv", index=False)
+    sync_directory("df_metadata_merged.csv", s3_bucket + "/df_metadata_merged.csv", if_copy=True)
+
+
+if __name__ == "__main__":
+
+    # Set logger level
+    logging.basicConfig(level=logging.INFO)
+
+    trigger_patchseq_upload(os.path.expanduser(R"~\Downloads\IVSCC_LC_summary.xlsx"))

lcne_patchseq_analysis-0.3.0/src/LCNE_patchseq_analysis/data_util/lims.py

@@ -0,0 +1,78 @@
+"""Utilities for querying the LIMS database.
+
+From Brian
+"""
+
+import json
+
+import pandas as pd  # pandas will be needed to work in a dataframe
+import pg8000  # pg8000 access SQL databases
+
+# code from Agata
+# these are nice functions to open LIMS, make a query and then close LIMS after
+
+
+def _connect(user, host, database, password, port):
+    conn = pg8000.connect(user=user, host=host, database=database, password=password, port=port)
+    return conn, conn.cursor()
+
+
+def _select(cursor, query):
+    cursor.execute(query)
+    columns = [d[0] for d in cursor.description]
+    return [dict(zip(columns, c)) for c in cursor.fetchall()]
+
+
+def limsquery(query, user, host, database, password, port):
+    """A function that takes a string containing a SQL query, connects to the LIMS database
+    and outputs the result."""
+    conn, cursor = _connect(user, host, database, password, port)
+    try:
+        results = _select(cursor, query)
+    finally:
+        cursor.close()
+        conn.close()
+    return results
+
+
+# this last function will take our query results and put them in a dataframe
+# so that they are easy to work with
+def get_lims_dataframe(query):
+    """Return a dataframe with lims query"""
+
+    # Get credentials from json
+    with open("LIMS_credentials.json") as f:
+        credentials = json.load(f)
+
+    result = limsquery(query, **credentials)
+    try:
+        data_df = pd.DataFrame(data=result, columns=result[0].keys())
+    except IndexError:
+        print("Could not find results for your query.")
+        data_df = pd.DataFrame()
+    return data_df
+
+
+# Query for LCNE patchseq experiments
+def get_lims_LCNE_patchseq():
+    lims_query = """
+        SELECT
+            s.id AS specimen_id,
+            s.name AS specimen_name,
+            proj.code,
+            err.id AS ephys_roi_id,
+            err.workflow_state AS Ephys_QC,
+            s.patched_cell_container,
+            err.storage_directory
+        FROM ephys_roi_results AS err
+        JOIN specimens AS s ON s.ephys_roi_result_id = err.id
+        JOIN projects AS proj ON s.project_id = proj.id
+        WHERE proj.code = 'mIVSCC-MET-R01_LC';
+    """
+    lims_df = get_lims_dataframe(lims_query)
+    return lims_df
+
+
+if __name__ == "__main__":
+    lims_df = get_lims_LCNE_patchseq()
+    print(lims_df.head())