LCNE-patchseq-analysis 0.1.0__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {lcne_patchseq_analysis-0.1.0 → lcne_patchseq_analysis-0.2.0}/.flake8 +2 -1
- {lcne_patchseq_analysis-0.1.0 → lcne_patchseq_analysis-0.2.0}/PKG-INFO +12 -3
- {lcne_patchseq_analysis-0.1.0 → lcne_patchseq_analysis-0.2.0}/README.md +1 -1
- {lcne_patchseq_analysis-0.1.0 → lcne_patchseq_analysis-0.2.0}/docs/source/conf.py +1 -0
- lcne_patchseq_analysis-0.2.0/notebook/demo.ipynb +291 -0
- {lcne_patchseq_analysis-0.1.0 → lcne_patchseq_analysis-0.2.0}/pyproject.toml +13 -1
- lcne_patchseq_analysis-0.2.0/src/LCNE_patchseq_analysis/__init__.py +2 -0
- lcne_patchseq_analysis-0.2.0/src/LCNE_patchseq_analysis/data_util/__init__.py +1 -0
- lcne_patchseq_analysis-0.2.0/src/LCNE_patchseq_analysis/data_util/ephys.py +1 -0
- lcne_patchseq_analysis-0.2.0/src/LCNE_patchseq_analysis/data_util/lims.py +73 -0
- lcne_patchseq_analysis-0.2.0/src/LCNE_patchseq_analysis/data_util/metadata.py +129 -0
- {lcne_patchseq_analysis-0.1.0 → lcne_patchseq_analysis-0.2.0}/src/LCNE_patchseq_analysis.egg-info/PKG-INFO +12 -3
- {lcne_patchseq_analysis-0.1.0 → lcne_patchseq_analysis-0.2.0}/src/LCNE_patchseq_analysis.egg-info/SOURCES.txt +1 -0
- {lcne_patchseq_analysis-0.1.0 → lcne_patchseq_analysis-0.2.0}/src/LCNE_patchseq_analysis.egg-info/requires.txt +11 -1
- lcne_patchseq_analysis-0.1.0/notebook/demo.ipynb +0 -705
- lcne_patchseq_analysis-0.1.0/src/LCNE_patchseq_analysis/__init__.py +0 -2
- lcne_patchseq_analysis-0.1.0/src/LCNE_patchseq_analysis/data_util/__init__.py +0 -2
- lcne_patchseq_analysis-0.1.0/src/LCNE_patchseq_analysis/data_util/ephys.py +0 -1
- lcne_patchseq_analysis-0.1.0/src/LCNE_patchseq_analysis/data_util/metadata.py +0 -62
- {lcne_patchseq_analysis-0.1.0 → lcne_patchseq_analysis-0.2.0}/.github/ISSUE_TEMPLATE/bug_report.md +0 -0
- {lcne_patchseq_analysis-0.1.0 → lcne_patchseq_analysis-0.2.0}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
- {lcne_patchseq_analysis-0.1.0 → lcne_patchseq_analysis-0.2.0}/.github/ISSUE_TEMPLATE/user-story.md +0 -0
- {lcne_patchseq_analysis-0.1.0 → lcne_patchseq_analysis-0.2.0}/.github/workflows/init.yml +0 -0
- {lcne_patchseq_analysis-0.1.0 → lcne_patchseq_analysis-0.2.0}/.github/workflows/tag_and_publish.yml +0 -0
- {lcne_patchseq_analysis-0.1.0 → lcne_patchseq_analysis-0.2.0}/.github/workflows/test_and_lint.yml +0 -0
- {lcne_patchseq_analysis-0.1.0 → lcne_patchseq_analysis-0.2.0}/.gitignore +0 -0
- {lcne_patchseq_analysis-0.1.0 → lcne_patchseq_analysis-0.2.0}/LICENSE +0 -0
- {lcne_patchseq_analysis-0.1.0 → lcne_patchseq_analysis-0.2.0}/docs/Makefile +0 -0
- {lcne_patchseq_analysis-0.1.0 → lcne_patchseq_analysis-0.2.0}/docs/make.bat +0 -0
- {lcne_patchseq_analysis-0.1.0 → lcne_patchseq_analysis-0.2.0}/docs/source/_static/dark-logo.svg +0 -0
- {lcne_patchseq_analysis-0.1.0 → lcne_patchseq_analysis-0.2.0}/docs/source/_static/favicon.ico +0 -0
- {lcne_patchseq_analysis-0.1.0 → lcne_patchseq_analysis-0.2.0}/docs/source/_static/light-logo.svg +0 -0
- {lcne_patchseq_analysis-0.1.0 → lcne_patchseq_analysis-0.2.0}/docs/source/index.rst +0 -0
- {lcne_patchseq_analysis-0.1.0 → lcne_patchseq_analysis-0.2.0}/environment/Dockerfile +0 -0
- {lcne_patchseq_analysis-0.1.0 → lcne_patchseq_analysis-0.2.0}/environment/postInstall +0 -0
- {lcne_patchseq_analysis-0.1.0 → lcne_patchseq_analysis-0.2.0}/setup.cfg +0 -0
- {lcne_patchseq_analysis-0.1.0 → lcne_patchseq_analysis-0.2.0}/setup.py +0 -0
- {lcne_patchseq_analysis-0.1.0 → lcne_patchseq_analysis-0.2.0}/src/LCNE_patchseq_analysis.egg-info/dependency_links.txt +0 -0
- {lcne_patchseq_analysis-0.1.0 → lcne_patchseq_analysis-0.2.0}/src/LCNE_patchseq_analysis.egg-info/top_level.txt +0 -0
- {lcne_patchseq_analysis-0.1.0 → lcne_patchseq_analysis-0.2.0}/tests/__init__.py +0 -0
- {lcne_patchseq_analysis-0.1.0 → lcne_patchseq_analysis-0.2.0}/tests/test_example.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.2
|
|
2
2
|
Name: LCNE-patchseq-analysis
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: Generated from aind-library-template
|
|
5
5
|
Author: Allen Institute for Neural Dynamics
|
|
6
6
|
Author-email: Han Hou <han.hou@alleninstitute.org>
|
|
@@ -11,7 +11,6 @@ Description-Content-Type: text/markdown
|
|
|
11
11
|
License-File: LICENSE
|
|
12
12
|
Requires-Dist: numpy
|
|
13
13
|
Requires-Dist: pandas
|
|
14
|
-
Requires-Dist: openpyxl
|
|
15
14
|
Requires-Dist: matplotlib
|
|
16
15
|
Provides-Extra: dev
|
|
17
16
|
Requires-Dist: black; extra == "dev"
|
|
@@ -21,13 +20,23 @@ Requires-Dist: interrogate; extra == "dev"
|
|
|
21
20
|
Requires-Dist: isort; extra == "dev"
|
|
22
21
|
Requires-Dist: Sphinx; extra == "dev"
|
|
23
22
|
Requires-Dist: furo; extra == "dev"
|
|
23
|
+
Provides-Extra: pipeline
|
|
24
|
+
Requires-Dist: black; extra == "pipeline"
|
|
25
|
+
Requires-Dist: coverage; extra == "pipeline"
|
|
26
|
+
Requires-Dist: flake8; extra == "pipeline"
|
|
27
|
+
Requires-Dist: interrogate; extra == "pipeline"
|
|
28
|
+
Requires-Dist: isort; extra == "pipeline"
|
|
29
|
+
Requires-Dist: Sphinx; extra == "pipeline"
|
|
30
|
+
Requires-Dist: furo; extra == "pipeline"
|
|
31
|
+
Requires-Dist: openpyxl; extra == "pipeline"
|
|
32
|
+
Requires-Dist: pg8000; extra == "pipeline"
|
|
24
33
|
|
|
25
34
|
# LCNE-patchseq-analysis
|
|
26
35
|
|
|
27
36
|
[](LICENSE)
|
|
28
37
|
|
|
29
38
|
[](https://github.com/semantic-release/semantic-release)
|
|
30
|
-
|
|
31
40
|
|
|
32
41
|
|
|
33
42
|
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
[](LICENSE)
|
|
4
4
|
|
|
5
5
|
[](https://github.com/semantic-release/semantic-release)
|
|
6
|
-
|
|
7
7
|
|
|
8
8
|
|
|
9
9
|
|
|
@@ -0,0 +1,291 @@
|
|
|
1
|
+
{
|
|
2
|
+
"cells": [
|
|
3
|
+
{
|
|
4
|
+
"cell_type": "code",
|
|
5
|
+
"execution_count": 4,
|
|
6
|
+
"metadata": {},
|
|
7
|
+
"outputs": [],
|
|
8
|
+
"source": [
|
|
9
|
+
"%load_ext autoreload\n",
|
|
10
|
+
"%autoreload 2"
|
|
11
|
+
]
|
|
12
|
+
},
|
|
13
|
+
{
|
|
14
|
+
"cell_type": "code",
|
|
15
|
+
"execution_count": 62,
|
|
16
|
+
"metadata": {},
|
|
17
|
+
"outputs": [],
|
|
18
|
+
"source": [
|
|
19
|
+
"from LCNE_patchseq_analysis.data_util.metadata import read_brian_spreadsheet, cross_check_metadata"
|
|
20
|
+
]
|
|
21
|
+
},
|
|
22
|
+
{
|
|
23
|
+
"cell_type": "code",
|
|
24
|
+
"execution_count": 63,
|
|
25
|
+
"metadata": {},
|
|
26
|
+
"outputs": [],
|
|
27
|
+
"source": [
|
|
28
|
+
"dfs = read_brian_spreadsheet(add_lims=True)\n",
|
|
29
|
+
"df = dfs[\"df_all\"]"
|
|
30
|
+
]
|
|
31
|
+
},
|
|
32
|
+
{
|
|
33
|
+
"cell_type": "markdown",
|
|
34
|
+
"metadata": {},
|
|
35
|
+
"source": [
|
|
36
|
+
"## Cross tab sanity check"
|
|
37
|
+
]
|
|
38
|
+
},
|
|
39
|
+
{
|
|
40
|
+
"cell_type": "markdown",
|
|
41
|
+
"metadata": {},
|
|
42
|
+
"source": [
|
|
43
|
+
"Check overlapped columns across tabs"
|
|
44
|
+
]
|
|
45
|
+
},
|
|
46
|
+
{
|
|
47
|
+
"cell_type": "code",
|
|
48
|
+
"execution_count": 64,
|
|
49
|
+
"metadata": {},
|
|
50
|
+
"outputs": [
|
|
51
|
+
{
|
|
52
|
+
"name": "stdout",
|
|
53
|
+
"output_type": "stream",
|
|
54
|
+
"text": [
|
|
55
|
+
"Found 9 inconsistencies between tab_xyz and master tables:\n",
|
|
56
|
+
" Date jem-id_cell_specimen x_tab_master \\\n",
|
|
57
|
+
"165 2023-09-01 Dbh-Cre_KH212;RCL-H2B-GFP-692026.10.10.02 10534.982420 \n",
|
|
58
|
+
"166 2023-08-25 Dbh-Cre_KH212;RCL-H2B-GFP-692022.09.06.01 NaN \n",
|
|
59
|
+
"167 2023-08-20 Dbh-Cre_KH212;RCL-H2B-GFP-692023.08.06.01 10541.875980 \n",
|
|
60
|
+
"168 2023-08-20 Dbh-Cre_KH212;RCL-H2B-GFP-692023.08.06.02 10702.283200 \n",
|
|
61
|
+
"170 2023-06-02 Dbh-Cre_KH212;RCL-H2B-GFP-676766.10.06.03 10521.757810 \n",
|
|
62
|
+
"202 2023-03-15 C57BL6J-665266.11.06.03 10451.809570 \n",
|
|
63
|
+
"217 2023-01-20 Ndnf-IRES2-dgCre;Ai14-659663.11.06.03 10391.497070 \n",
|
|
64
|
+
"219 2023-01-20 Ndnf-IRES2-dgCre;Ai14-659663.11.06.04 9531.198242 \n",
|
|
65
|
+
"220 2023-01-20 Ndnf-IRES2-dgCre;Ai14-659663.11.06.01 NaN \n",
|
|
66
|
+
"\n",
|
|
67
|
+
" y_tab_master z_tab_master Annotated structure_tab_master \\\n",
|
|
68
|
+
"165 4183.531250 4984.0 PAG \n",
|
|
69
|
+
"166 NaN NaN SCiw \n",
|
|
70
|
+
"167 4110.681641 5034.0 PB \n",
|
|
71
|
+
"168 3840.954834 4727.0 LC \n",
|
|
72
|
+
"170 4256.657715 4889.0 LDT \n",
|
|
73
|
+
"202 4402.110352 4889.0 LDT \n",
|
|
74
|
+
"217 4161.165039 4889.0 PCG \n",
|
|
75
|
+
"219 2449.594727 4265.0 PCG \n",
|
|
76
|
+
"220 NaN NaN LDT \n",
|
|
77
|
+
"\n",
|
|
78
|
+
" notes_tab_master x_tab_xyz y_tab_xyz z_tab_xyz \\\n",
|
|
79
|
+
"165 NaN 10151.019530 3701.974609 4824.0 \n",
|
|
80
|
+
"166 NaN 9531.198242 2449.594727 4265.0 \n",
|
|
81
|
+
"167 NaN 10702.283200 3840.954834 4727.0 \n",
|
|
82
|
+
"168 NaN 10761.001950 4288.832031 4727.0 \n",
|
|
83
|
+
"170 NaN 10541.875980 4110.681641 5034.0 \n",
|
|
84
|
+
"202 NaN 10534.982420 4183.531250 4984.0 \n",
|
|
85
|
+
"217 NaN 10521.757810 4256.657715 4889.0 \n",
|
|
86
|
+
"219 NaN 10451.809570 4402.110352 4889.0 \n",
|
|
87
|
+
"220 NaN 10391.497070 4161.165039 4889.0 \n",
|
|
88
|
+
"\n",
|
|
89
|
+
" Annotated structure_tab_xyz notes_tab_xyz \n",
|
|
90
|
+
"165 PAG NaN \n",
|
|
91
|
+
"166 SCiw NaN \n",
|
|
92
|
+
"167 PB NaN \n",
|
|
93
|
+
"168 LC NaN \n",
|
|
94
|
+
"170 LDT NaN \n",
|
|
95
|
+
"202 LDT NaN \n",
|
|
96
|
+
"217 PCG NaN \n",
|
|
97
|
+
"219 PCG NaN \n",
|
|
98
|
+
"220 LDT NaN \n",
|
|
99
|
+
"\n",
|
|
100
|
+
"\n",
|
|
101
|
+
"Found 103 inconsistencies between tab_ephys_fx and master tables:\n",
|
|
102
|
+
" Date jem-id_cell_specimen \\\n",
|
|
103
|
+
"0 2025-02-06 C57BL6J-785653.03.02.02 \n",
|
|
104
|
+
"1 2025-02-06 C57BL6J-785653.04.02.02 \n",
|
|
105
|
+
"2 2025-02-06 C57BL6J-785653.03.02.01 \n",
|
|
106
|
+
"3 2025-02-06 C57BL6J-785653.04.02.01 \n",
|
|
107
|
+
"4 2025-02-05 C57BL6J-785652.03.02.02 \n",
|
|
108
|
+
".. ... ... \n",
|
|
109
|
+
"187 2023-04-19 Slc17a6-IRES-Cre;Ai14-670829.11.06.02 \n",
|
|
110
|
+
"243 2022-11-17 Slc17a6-IRES-Cre;Ai14-651168.10.06.03 \n",
|
|
111
|
+
"251 2022-11-15 Dbh-Cre_KH212;RCL-Sun1sfGFP-neo-650884.09.06.05 \n",
|
|
112
|
+
"257 2022-11-02 Rbp4-Cre_KL100;Ai14-650443.10.06.02 \n",
|
|
113
|
+
"258 2022-10-27 C57BL6J-647687.09.06.01 \n",
|
|
114
|
+
"\n",
|
|
115
|
+
" failed_electrode_0_tab_master failed_no_seal_tab_master \\\n",
|
|
116
|
+
"0 NaN NaN \n",
|
|
117
|
+
"1 NaN NaN \n",
|
|
118
|
+
"2 NaN NaN \n",
|
|
119
|
+
"3 NaN NaN \n",
|
|
120
|
+
"4 NaN NaN \n",
|
|
121
|
+
".. ... ... \n",
|
|
122
|
+
"187 0.0 1.0 \n",
|
|
123
|
+
"243 0.0 1.0 \n",
|
|
124
|
+
"251 0.0 1.0 \n",
|
|
125
|
+
"257 0.0 1.0 \n",
|
|
126
|
+
"258 0.0 1.0 \n",
|
|
127
|
+
"\n",
|
|
128
|
+
" failed_bad_rs_tab_master failed_electrode_0_tab_ephys_fx \\\n",
|
|
129
|
+
"0 NaN 0.0 \n",
|
|
130
|
+
"1 NaN 0.0 \n",
|
|
131
|
+
"2 NaN 0.0 \n",
|
|
132
|
+
"3 NaN 0.0 \n",
|
|
133
|
+
"4 NaN 0.0 \n",
|
|
134
|
+
".. ... ... \n",
|
|
135
|
+
"187 0.0 0.0 \n",
|
|
136
|
+
"243 0.0 0.0 \n",
|
|
137
|
+
"251 0.0 0.0 \n",
|
|
138
|
+
"257 0.0 0.0 \n",
|
|
139
|
+
"258 0.0 0.0 \n",
|
|
140
|
+
"\n",
|
|
141
|
+
" failed_no_seal_tab_ephys_fx failed_bad_rs_tab_ephys_fx \n",
|
|
142
|
+
"0 0.0 0.0 \n",
|
|
143
|
+
"1 0.0 0.0 \n",
|
|
144
|
+
"2 0.0 0.0 \n",
|
|
145
|
+
"3 0.0 0.0 \n",
|
|
146
|
+
"4 0.0 0.0 \n",
|
|
147
|
+
".. ... ... \n",
|
|
148
|
+
"187 0.0 0.0 \n",
|
|
149
|
+
"243 0.0 0.0 \n",
|
|
150
|
+
"251 0.0 0.0 \n",
|
|
151
|
+
"257 0.0 0.0 \n",
|
|
152
|
+
"258 0.0 0.0 \n",
|
|
153
|
+
"\n",
|
|
154
|
+
"[103 rows x 8 columns]\n",
|
|
155
|
+
"\n",
|
|
156
|
+
"\n",
|
|
157
|
+
"Found 15 inconsistencies between lims and master tables:\n",
|
|
158
|
+
" Date jem-id_cell_specimen ephys_roi_id_tab_master \\\n",
|
|
159
|
+
"0 2025-02-06 C57BL6J-785653.03.02.02 1418804349 \n",
|
|
160
|
+
"1 2025-02-06 C57BL6J-785653.04.02.02 1418799012 \n",
|
|
161
|
+
"2 2025-02-06 C57BL6J-785653.03.02.01 1418797120 \n",
|
|
162
|
+
"3 2025-02-06 C57BL6J-785653.04.02.01 1418784590 \n",
|
|
163
|
+
"4 2025-02-05 C57BL6J-785652.03.02.02 1418553949 \n",
|
|
164
|
+
"5 2025-02-05 C57BL6J-785652.03.02.01 1418549638 \n",
|
|
165
|
+
"6 2025-02-05 C57BL6J-785652.03.01.01 1418547172 \n",
|
|
166
|
+
"7 2025-02-05 C57BL6J-785652.04.02.01 1418555572 \n",
|
|
167
|
+
"8 2025-02-05 C57BL6J-785652.04.02.02 1418561975 \n",
|
|
168
|
+
"9 2025-01-30 Dbh-Cre-KI;Ai65-780952.04.02.01 1417392272 \n",
|
|
169
|
+
"10 2025-01-30 Dbh-Cre-KI;Ai65-780952.03.01.01 1417382638 \n",
|
|
170
|
+
"11 2025-01-30 Dbh-Cre-KI;Ai65-780952.04.01.02 1417380803 \n",
|
|
171
|
+
"12 2025-01-30 Dbh-Cre-KI;Ai65-780952.03.02.01 1417375160 \n",
|
|
172
|
+
"13 2025-01-30 Dbh-Cre-KI;Ai65-780952.04.01.01 1417373093 \n",
|
|
173
|
+
"14 2025-01-29 Dbh-Cre-KI;Ai65-780955.03.01.01 1417138763 \n",
|
|
174
|
+
"\n",
|
|
175
|
+
" ephys_qc_tab_master storage_directory_tab_master ephys_roi_id_lims \\\n",
|
|
176
|
+
"0 auto_passed NaN 1.418804e+09 \n",
|
|
177
|
+
"1 auto_passed NaN 1.418799e+09 \n",
|
|
178
|
+
"2 auto_passed NaN 1.418797e+09 \n",
|
|
179
|
+
"3 auto_passed NaN 1.418785e+09 \n",
|
|
180
|
+
"4 auto_passed NaN 1.418554e+09 \n",
|
|
181
|
+
"5 auto_passed NaN 1.418550e+09 \n",
|
|
182
|
+
"6 auto_passed NaN 1.418547e+09 \n",
|
|
183
|
+
"7 auto_passed NaN 1.418556e+09 \n",
|
|
184
|
+
"8 auto_passed NaN 1.418562e+09 \n",
|
|
185
|
+
"9 auto_passed NaN 1.417392e+09 \n",
|
|
186
|
+
"10 auto_failed NaN 1.417383e+09 \n",
|
|
187
|
+
"11 auto_passed NaN 1.417381e+09 \n",
|
|
188
|
+
"12 auto_passed NaN 1.417375e+09 \n",
|
|
189
|
+
"13 auto_passed NaN 1.417373e+09 \n",
|
|
190
|
+
"14 auto_passed NaN 1.417139e+09 \n",
|
|
191
|
+
"\n",
|
|
192
|
+
" ephys_qc_lims storage_directory_lims \n",
|
|
193
|
+
"0 auto_passed /allen/programs/celltypes/production/mousecell... \n",
|
|
194
|
+
"1 auto_passed /allen/programs/celltypes/production/mousecell... \n",
|
|
195
|
+
"2 auto_passed /allen/programs/celltypes/production/mousecell... \n",
|
|
196
|
+
"3 auto_passed /allen/programs/celltypes/production/mousecell... \n",
|
|
197
|
+
"4 auto_passed /allen/programs/celltypes/production/mousecell... \n",
|
|
198
|
+
"5 auto_passed /allen/programs/celltypes/production/mousecell... \n",
|
|
199
|
+
"6 auto_passed /allen/programs/celltypes/production/mousecell... \n",
|
|
200
|
+
"7 auto_passed /allen/programs/celltypes/production/mousecell... \n",
|
|
201
|
+
"8 auto_passed /allen/programs/celltypes/production/mousecell... \n",
|
|
202
|
+
"9 auto_passed /allen/programs/celltypes/production/mousecell... \n",
|
|
203
|
+
"10 auto_failed /allen/programs/celltypes/production/mousecell... \n",
|
|
204
|
+
"11 auto_passed /allen/programs/celltypes/production/mousecell... \n",
|
|
205
|
+
"12 auto_passed /allen/programs/celltypes/production/mousecell... \n",
|
|
206
|
+
"13 auto_passed /allen/programs/celltypes/production/mousecell... \n",
|
|
207
|
+
"14 auto_passed /allen/programs/celltypes/production/mousecell... \n",
|
|
208
|
+
"\n",
|
|
209
|
+
"\n"
|
|
210
|
+
]
|
|
211
|
+
}
|
|
212
|
+
],
|
|
213
|
+
"source": [
|
|
214
|
+
"dfs = read_brian_spreadsheet()\n",
|
|
215
|
+
"for source in [\"tab_xyz\", \"tab_ephys_fx\", \"lims\"]:\n",
|
|
216
|
+
" df_inconsistencies = cross_check_metadata(dfs[\"df_all\"], source)\n",
|
|
217
|
+
" \n",
|
|
218
|
+
" if len(df_inconsistencies) == 0:\n",
|
|
219
|
+
" print(\"All good!\")\n",
|
|
220
|
+
" continue\n",
|
|
221
|
+
" \n",
|
|
222
|
+
" print(f\"Found {len(df_inconsistencies)} inconsistencies between {source} and master tables:\")\n",
|
|
223
|
+
" print(df_inconsistencies)\n",
|
|
224
|
+
" print(\"\\n\")"
|
|
225
|
+
]
|
|
226
|
+
},
|
|
227
|
+
{
|
|
228
|
+
"cell_type": "markdown",
|
|
229
|
+
"metadata": {},
|
|
230
|
+
"source": [
|
|
231
|
+
"### ❌ Oh no! These inconsistencies must be caused by manually copying and pasting across the tabs!!!"
|
|
232
|
+
]
|
|
233
|
+
},
|
|
234
|
+
{
|
|
235
|
+
"cell_type": "markdown",
|
|
236
|
+
"metadata": {},
|
|
237
|
+
"source": [
|
|
238
|
+
"## Quick overview using pygwalker"
|
|
239
|
+
]
|
|
240
|
+
},
|
|
241
|
+
{
|
|
242
|
+
"cell_type": "code",
|
|
243
|
+
"execution_count": null,
|
|
244
|
+
"metadata": {},
|
|
245
|
+
"outputs": [
|
|
246
|
+
{
|
|
247
|
+
"name": "stdout",
|
|
248
|
+
"output_type": "stream",
|
|
249
|
+
"text": [
|
|
250
|
+
"\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\n"
|
|
251
|
+
]
|
|
252
|
+
}
|
|
253
|
+
],
|
|
254
|
+
"source": [
|
|
255
|
+
"!pip install pygwalker --quiet"
|
|
256
|
+
]
|
|
257
|
+
},
|
|
258
|
+
{
|
|
259
|
+
"cell_type": "code",
|
|
260
|
+
"execution_count": null,
|
|
261
|
+
"metadata": {},
|
|
262
|
+
"outputs": [],
|
|
263
|
+
"source": [
|
|
264
|
+
"import pygwalker as pyg\n",
|
|
265
|
+
"walker = pyg.walk(df)\n",
|
|
266
|
+
"walker"
|
|
267
|
+
]
|
|
268
|
+
}
|
|
269
|
+
],
|
|
270
|
+
"metadata": {
|
|
271
|
+
"kernelspec": {
|
|
272
|
+
"display_name": "patchseq_pipeline",
|
|
273
|
+
"language": "python",
|
|
274
|
+
"name": "python3"
|
|
275
|
+
},
|
|
276
|
+
"language_info": {
|
|
277
|
+
"codemirror_mode": {
|
|
278
|
+
"name": "ipython",
|
|
279
|
+
"version": 3
|
|
280
|
+
},
|
|
281
|
+
"file_extension": ".py",
|
|
282
|
+
"mimetype": "text/x-python",
|
|
283
|
+
"name": "python",
|
|
284
|
+
"nbconvert_exporter": "python",
|
|
285
|
+
"pygments_lexer": "ipython3",
|
|
286
|
+
"version": "3.9.21"
|
|
287
|
+
}
|
|
288
|
+
},
|
|
289
|
+
"nbformat": 4,
|
|
290
|
+
"nbformat_minor": 2
|
|
291
|
+
}
|
|
@@ -20,7 +20,6 @@ dynamic = ["version"]
|
|
|
20
20
|
dependencies = [
|
|
21
21
|
'numpy',
|
|
22
22
|
'pandas',
|
|
23
|
-
'openpyxl',
|
|
24
23
|
'matplotlib',
|
|
25
24
|
]
|
|
26
25
|
|
|
@@ -35,6 +34,18 @@ dev = [
|
|
|
35
34
|
'furo'
|
|
36
35
|
]
|
|
37
36
|
|
|
37
|
+
pipeline = [
|
|
38
|
+
'black',
|
|
39
|
+
'coverage',
|
|
40
|
+
'flake8',
|
|
41
|
+
'interrogate',
|
|
42
|
+
'isort',
|
|
43
|
+
'Sphinx',
|
|
44
|
+
'furo',
|
|
45
|
+
'openpyxl',
|
|
46
|
+
'pg8000',
|
|
47
|
+
]
|
|
48
|
+
|
|
38
49
|
[tool.setuptools.packages.find]
|
|
39
50
|
where = ["src"]
|
|
40
51
|
|
|
@@ -57,6 +68,7 @@ exclude = '''
|
|
|
57
68
|
| _build
|
|
58
69
|
| build
|
|
59
70
|
| dist
|
|
71
|
+
| code
|
|
60
72
|
)/
|
|
61
73
|
| .gitignore
|
|
62
74
|
)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Utils for accessing data"""
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Get ephys data"""
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
"""Utilities for querying the LIMS database.
|
|
2
|
+
|
|
3
|
+
From Brian
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
import pandas as pd # pandas will be needed to work in a dataframe
|
|
7
|
+
import pg8000 # pg8000 access SQL databases
|
|
8
|
+
|
|
9
|
+
# code from Agata
|
|
10
|
+
# these are nice functions to open LIMS, make a query and then close LIMS after
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _connect(user="limsreader", host="limsdb2", database="lims2", password="limsro", port=5432):
|
|
14
|
+
conn = pg8000.connect(user=user, host=host, database=database, password=password, port=port)
|
|
15
|
+
return conn, conn.cursor()
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _select(cursor, query):
|
|
19
|
+
cursor.execute(query)
|
|
20
|
+
columns = [d[0] for d in cursor.description]
|
|
21
|
+
return [dict(zip(columns, c)) for c in cursor.fetchall()]
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def limsquery(
|
|
25
|
+
query, user="limsreader", host="limsdb2", database="lims2", password="limsro", port=5432
|
|
26
|
+
):
|
|
27
|
+
"""A function that takes a string containing a SQL query, connects to the LIMS database
|
|
28
|
+
and outputs the result."""
|
|
29
|
+
conn, cursor = _connect(user, host, database, password, port)
|
|
30
|
+
try:
|
|
31
|
+
results = _select(cursor, query)
|
|
32
|
+
finally:
|
|
33
|
+
cursor.close()
|
|
34
|
+
conn.close()
|
|
35
|
+
return results
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
# this last function will take our query results and put them in a dataframe
|
|
39
|
+
# so that they are easy to work with
|
|
40
|
+
def get_lims_dataframe(query):
|
|
41
|
+
"""Return a dataframe with lims query"""
|
|
42
|
+
result = limsquery(query)
|
|
43
|
+
try:
|
|
44
|
+
data_df = pd.DataFrame(data=result, columns=result[0].keys())
|
|
45
|
+
except IndexError:
|
|
46
|
+
print("Could not find results for your query.")
|
|
47
|
+
data_df = pd.DataFrame()
|
|
48
|
+
return data_df
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
# Query for LCNE patchseq experiments
|
|
52
|
+
def get_lims_LCNE_patchseq():
|
|
53
|
+
lims_query = """
|
|
54
|
+
SELECT
|
|
55
|
+
s.id AS specimen_id,
|
|
56
|
+
s.name AS specimen_name,
|
|
57
|
+
proj.code,
|
|
58
|
+
err.id AS ephys_roi_id,
|
|
59
|
+
err.workflow_state AS Ephys_QC,
|
|
60
|
+
s.patched_cell_container,
|
|
61
|
+
err.storage_directory
|
|
62
|
+
FROM ephys_roi_results AS err
|
|
63
|
+
JOIN specimens AS s ON s.ephys_roi_result_id = err.id
|
|
64
|
+
JOIN projects AS proj ON s.project_id = proj.id
|
|
65
|
+
WHERE proj.code = 'mIVSCC-MET-R01_LC';
|
|
66
|
+
"""
|
|
67
|
+
lims_df = get_lims_dataframe(lims_query)
|
|
68
|
+
return lims_df
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
if __name__ == "__main__":
|
|
72
|
+
lims_df = get_lims_LCNE_patchseq()
|
|
73
|
+
print(lims_df.head())
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
"""Get metadata"""
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
import os
|
|
5
|
+
|
|
6
|
+
import pandas as pd
|
|
7
|
+
|
|
8
|
+
from LCNE_patchseq_analysis.data_util.lims import get_lims_LCNE_patchseq
|
|
9
|
+
|
|
10
|
+
metadata_path = os.path.expanduser(R"~\Downloads\IVSCC_LC_summary.xlsx")
|
|
11
|
+
logger = logging.getLogger(__name__)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def read_brian_spreadsheet(file_path=metadata_path, add_lims=True):
|
|
15
|
+
"""Read metadata, cell xyz coordinates, and ephys features from Brian's spreadsheet
|
|
16
|
+
|
|
17
|
+
Assuming IVSCC_LC_summary.xlsx is downloaded at file_path
|
|
18
|
+
|
|
19
|
+
Args:
|
|
20
|
+
file_path (str): Path to the metadata spreadsheet
|
|
21
|
+
add_lims (bool): Whether to add LIMS data
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
if not os.path.exists(file_path):
|
|
25
|
+
raise FileNotFoundError(f"File not found at {file_path}")
|
|
26
|
+
|
|
27
|
+
logger.info(f"Reading metadata from {file_path}...")
|
|
28
|
+
tab_names = pd.ExcelFile(file_path).sheet_names
|
|
29
|
+
|
|
30
|
+
# Get the master table
|
|
31
|
+
tab_master = [name for name in tab_names if "updated" in name.lower()][0]
|
|
32
|
+
df_master = pd.read_excel(file_path, sheet_name=tab_master)
|
|
33
|
+
|
|
34
|
+
# Get xyz coordinates
|
|
35
|
+
tab_xyz = [name for name in tab_names if "xyz" in name.lower()][0]
|
|
36
|
+
df_xyz = pd.read_excel(file_path, sheet_name=tab_xyz)
|
|
37
|
+
|
|
38
|
+
# Get ephys features
|
|
39
|
+
tab_ephys_fx = [name for name in tab_names if "ephys_fx" in name.lower()][0]
|
|
40
|
+
df_ephys_fx = pd.read_excel(file_path, sheet_name=tab_ephys_fx)
|
|
41
|
+
|
|
42
|
+
# Merge the tables
|
|
43
|
+
df_all = (
|
|
44
|
+
df_master.merge(
|
|
45
|
+
df_xyz.rename(
|
|
46
|
+
columns={
|
|
47
|
+
"specimen_name": "jem-id_cell_specimen",
|
|
48
|
+
"structure_acronym": "Annotated structure",
|
|
49
|
+
}
|
|
50
|
+
),
|
|
51
|
+
on="jem-id_cell_specimen",
|
|
52
|
+
how="outer",
|
|
53
|
+
suffixes=("_tab_master", "_tab_xyz"),
|
|
54
|
+
)
|
|
55
|
+
.merge(
|
|
56
|
+
df_ephys_fx.rename(
|
|
57
|
+
columns={
|
|
58
|
+
"failed_seal": "failed_no_seal",
|
|
59
|
+
"failed_input_access_resistance": "failed_bad_rs",
|
|
60
|
+
}
|
|
61
|
+
),
|
|
62
|
+
on="cell_specimen_id",
|
|
63
|
+
how="outer",
|
|
64
|
+
suffixes=("_tab_master", "_tab_ephys_fx"),
|
|
65
|
+
)
|
|
66
|
+
.sort_values("Date", ascending=False)
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
if add_lims:
|
|
70
|
+
logger.info("Querying and adding LIMS data...")
|
|
71
|
+
df_lims = get_lims_LCNE_patchseq()
|
|
72
|
+
df_all = df_all.merge(
|
|
73
|
+
df_lims,
|
|
74
|
+
left_on="jem-id_cell_specimen",
|
|
75
|
+
right_on="specimen_name",
|
|
76
|
+
how="left",
|
|
77
|
+
suffixes=("_tab_master", "_lims"),
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
return {
|
|
81
|
+
"df_all": df_all,
|
|
82
|
+
"df_master": df_master,
|
|
83
|
+
"df_xyz": df_xyz,
|
|
84
|
+
"df_ephys_fx": df_ephys_fx,
|
|
85
|
+
**({"df_lims": df_lims} if add_lims else {}),
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def cross_check_metadata(df, source):
|
|
90
|
+
"""Cross-check metadata between source and master tables
|
|
91
|
+
|
|
92
|
+
source in ["tab_xyz", "tab_ephys_fx", "lims]
|
|
93
|
+
"""
|
|
94
|
+
source_columns = [col for col in df.columns if source in col]
|
|
95
|
+
master_columns = [col.replace(source, "tab_master") for col in source_columns]
|
|
96
|
+
|
|
97
|
+
logger.info(f"Cross-checking metadata between {source} and master tables...")
|
|
98
|
+
logger.info(f"Source columns: {source_columns}")
|
|
99
|
+
logger.info(f"Master columns: {master_columns}")
|
|
100
|
+
|
|
101
|
+
# Find out inconsistencies between source and master, if both of them are not null
|
|
102
|
+
df_inconsistencies = df.loc[
|
|
103
|
+
(
|
|
104
|
+
df[source_columns].notnull()
|
|
105
|
+
& df[source_columns].notnull()
|
|
106
|
+
& (df[source_columns].to_numpy() != df[master_columns].to_numpy())
|
|
107
|
+
).any(axis=1),
|
|
108
|
+
["Date", "jem-id_cell_specimen"] + master_columns + source_columns,
|
|
109
|
+
]
|
|
110
|
+
|
|
111
|
+
return df_inconsistencies
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
if __name__ == "__main__":
|
|
115
|
+
logging.basicConfig(level=logging.INFO)
|
|
116
|
+
|
|
117
|
+
dfs = read_brian_spreadsheet()
|
|
118
|
+
for source in ["tab_xyz", "tab_ephys_fx", "lims"]:
|
|
119
|
+
df_inconsistencies = cross_check_metadata(dfs["df_all"], source)
|
|
120
|
+
|
|
121
|
+
if len(df_inconsistencies) == 0:
|
|
122
|
+
print("All good!")
|
|
123
|
+
continue
|
|
124
|
+
|
|
125
|
+
print(
|
|
126
|
+
f"Found {len(df_inconsistencies)} inconsistencies between {source} and master tables:"
|
|
127
|
+
)
|
|
128
|
+
print(df_inconsistencies)
|
|
129
|
+
print("\n")
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.2
|
|
2
2
|
Name: LCNE-patchseq-analysis
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: Generated from aind-library-template
|
|
5
5
|
Author: Allen Institute for Neural Dynamics
|
|
6
6
|
Author-email: Han Hou <han.hou@alleninstitute.org>
|
|
@@ -11,7 +11,6 @@ Description-Content-Type: text/markdown
|
|
|
11
11
|
License-File: LICENSE
|
|
12
12
|
Requires-Dist: numpy
|
|
13
13
|
Requires-Dist: pandas
|
|
14
|
-
Requires-Dist: openpyxl
|
|
15
14
|
Requires-Dist: matplotlib
|
|
16
15
|
Provides-Extra: dev
|
|
17
16
|
Requires-Dist: black; extra == "dev"
|
|
@@ -21,13 +20,23 @@ Requires-Dist: interrogate; extra == "dev"
|
|
|
21
20
|
Requires-Dist: isort; extra == "dev"
|
|
22
21
|
Requires-Dist: Sphinx; extra == "dev"
|
|
23
22
|
Requires-Dist: furo; extra == "dev"
|
|
23
|
+
Provides-Extra: pipeline
|
|
24
|
+
Requires-Dist: black; extra == "pipeline"
|
|
25
|
+
Requires-Dist: coverage; extra == "pipeline"
|
|
26
|
+
Requires-Dist: flake8; extra == "pipeline"
|
|
27
|
+
Requires-Dist: interrogate; extra == "pipeline"
|
|
28
|
+
Requires-Dist: isort; extra == "pipeline"
|
|
29
|
+
Requires-Dist: Sphinx; extra == "pipeline"
|
|
30
|
+
Requires-Dist: furo; extra == "pipeline"
|
|
31
|
+
Requires-Dist: openpyxl; extra == "pipeline"
|
|
32
|
+
Requires-Dist: pg8000; extra == "pipeline"
|
|
24
33
|
|
|
25
34
|
# LCNE-patchseq-analysis
|
|
26
35
|
|
|
27
36
|
[](LICENSE)
|
|
28
37
|
|
|
29
38
|
[](https://github.com/semantic-release/semantic-release)
|
|
30
|
-
|
|
31
40
|
|
|
32
41
|
|
|
33
42
|
|
|
@@ -28,6 +28,7 @@ src/LCNE_patchseq_analysis.egg-info/requires.txt
|
|
|
28
28
|
src/LCNE_patchseq_analysis.egg-info/top_level.txt
|
|
29
29
|
src/LCNE_patchseq_analysis/data_util/__init__.py
|
|
30
30
|
src/LCNE_patchseq_analysis/data_util/ephys.py
|
|
31
|
+
src/LCNE_patchseq_analysis/data_util/lims.py
|
|
31
32
|
src/LCNE_patchseq_analysis/data_util/metadata.py
|
|
32
33
|
tests/__init__.py
|
|
33
34
|
tests/test_example.py
|