pymaude 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pymaude-0.2.0/.github/workflows/pypi.yml +70 -0
- pymaude-0.2.0/.gitignore +13 -0
- pymaude-0.2.0/LICENSE +21 -0
- pymaude-0.2.0/PKG-INFO +21 -0
- pymaude-0.2.0/example.ipynb +546 -0
- pymaude-0.2.0/publication/archive.py +23 -0
- pymaude-0.2.0/publication/benchmark_init_db.py +39 -0
- pymaude-0.2.0/publication/benchmark_query.py +0 -0
- pymaude-0.2.0/publication/validation.py +0 -0
- pymaude-0.2.0/pyproject.toml +36 -0
- pymaude-0.2.0/setup.cfg +4 -0
- pymaude-0.2.0/src/pymaude/__init__.py +14 -0
- pymaude-0.2.0/src/pymaude/database.py +1100 -0
- pymaude-0.2.0/src/pymaude/metadata.py +54 -0
- pymaude-0.2.0/src/pymaude.egg-info/PKG-INFO +21 -0
- pymaude-0.2.0/src/pymaude.egg-info/SOURCES.txt +21 -0
- pymaude-0.2.0/src/pymaude.egg-info/dependency_links.txt +1 -0
- pymaude-0.2.0/src/pymaude.egg-info/requires.txt +8 -0
- pymaude-0.2.0/src/pymaude.egg-info/top_level.txt +1 -0
- pymaude-0.2.0/tests/__init__.py +0 -0
- pymaude-0.2.0/tests/conftest.py +97 -0
- pymaude-0.2.0/tests/test_database.py +242 -0
- pymaude-0.2.0/tests/test_search.py +126 -0
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
# This workflow will upload a Python Package to PyPI when a release is created
|
|
2
|
+
# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries
|
|
3
|
+
|
|
4
|
+
# This workflow uses actions that are not certified by GitHub.
|
|
5
|
+
# They are provided by a third-party and are governed by
|
|
6
|
+
# separate terms of service, privacy policy, and support
|
|
7
|
+
# documentation.
|
|
8
|
+
|
|
9
|
+
name: Upload Python Package
|
|
10
|
+
|
|
11
|
+
on:
|
|
12
|
+
release:
|
|
13
|
+
types: [published]
|
|
14
|
+
|
|
15
|
+
permissions:
|
|
16
|
+
contents: read
|
|
17
|
+
|
|
18
|
+
jobs:
|
|
19
|
+
release-build:
|
|
20
|
+
runs-on: ubuntu-latest
|
|
21
|
+
|
|
22
|
+
steps:
|
|
23
|
+
- uses: actions/checkout@v4
|
|
24
|
+
|
|
25
|
+
- uses: actions/setup-python@v5
|
|
26
|
+
with:
|
|
27
|
+
python-version: "3.x"
|
|
28
|
+
|
|
29
|
+
- name: Build release distributions
|
|
30
|
+
run: |
|
|
31
|
+
# NOTE: put your own distribution build steps here.
|
|
32
|
+
python -m pip install build
|
|
33
|
+
python -m build
|
|
34
|
+
|
|
35
|
+
- name: Upload distributions
|
|
36
|
+
uses: actions/upload-artifact@v4
|
|
37
|
+
with:
|
|
38
|
+
name: release-dists
|
|
39
|
+
path: dist/
|
|
40
|
+
|
|
41
|
+
pypi-publish:
|
|
42
|
+
runs-on: ubuntu-latest
|
|
43
|
+
needs:
|
|
44
|
+
- release-build
|
|
45
|
+
permissions:
|
|
46
|
+
# IMPORTANT: this permission is mandatory for trusted publishing
|
|
47
|
+
id-token: write
|
|
48
|
+
|
|
49
|
+
# Dedicated environments with protections for publishing are strongly recommended.
|
|
50
|
+
# For more information, see: https://docs.github.com/en/actions/deployment/targeting-different-environments/using-environments-for-deployment#deployment-protection-rules
|
|
51
|
+
environment:
|
|
52
|
+
name: pypi
|
|
53
|
+
# OPTIONAL: uncomment and update to include your PyPI project URL in the deployment status:
|
|
54
|
+
# url: https://pypi.org/p/YOURPROJECT
|
|
55
|
+
#
|
|
56
|
+
# ALTERNATIVE: if your GitHub Release name is the PyPI project version string
|
|
57
|
+
# ALTERNATIVE: exactly, uncomment the following line instead:
|
|
58
|
+
url: https://pypi.org/project/PyMAUDE/${{ github.event.release.name }}
|
|
59
|
+
|
|
60
|
+
steps:
|
|
61
|
+
- name: Retrieve release distributions
|
|
62
|
+
uses: actions/download-artifact@v4
|
|
63
|
+
with:
|
|
64
|
+
name: release-dists
|
|
65
|
+
path: dist/
|
|
66
|
+
|
|
67
|
+
- name: Publish release distributions to PyPI
|
|
68
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
69
|
+
with:
|
|
70
|
+
packages-dir: dist/
|
pymaude-0.2.0/.gitignore
ADDED
pymaude-0.2.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Jacob Schwartz
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
pymaude-0.2.0/PKG-INFO
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: pymaude
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: PyMAUDE: FDA MAUDE adverse event database interface
|
|
5
|
+
Author-email: Jacob Schwartz <jaschwa@umich.edu>
|
|
6
|
+
License: MIT
|
|
7
|
+
Classifier: Development Status :: 4 - Beta
|
|
8
|
+
Classifier: Intended Audience :: Science/Research
|
|
9
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Requires-Python: >=3.9
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
License-File: LICENSE
|
|
14
|
+
Requires-Dist: duckdb>=0.10.0
|
|
15
|
+
Requires-Dist: pandas>=1.3.0
|
|
16
|
+
Requires-Dist: requests>=2.25.0
|
|
17
|
+
Requires-Dist: pyyaml>=5.1
|
|
18
|
+
Provides-Extra: dev
|
|
19
|
+
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
20
|
+
Requires-Dist: pytest-cov; extra == "dev"
|
|
21
|
+
Dynamic: license-file
|
|
@@ -0,0 +1,546 @@
|
|
|
1
|
+
{
|
|
2
|
+
"cells": [
|
|
3
|
+
{
|
|
4
|
+
"cell_type": "markdown",
|
|
5
|
+
"id": "fac46bca",
|
|
6
|
+
"metadata": {},
|
|
7
|
+
"source": "# PyMAUDE — Example Notebook\n\n**PyMAUDE** is a Python library for accessing and analyzing FDA MAUDE (Manufacturer and User Facility Device Experience) adverse event data via a fast [DuckDB](https://duckdb.org/) backend.\n\nThis notebook downloads real FDA MAUDE data and demonstrates all major library capabilities using venous stent and rotational thrombectomy devices as worked examples.\n\n> **First run:** `add_years(..., download=True)` downloads zip files from the FDA FTP server (~few hundred MB for 3 years of device/master/text). Subsequent runs use the local cache and skip unchanged files via SHA-256 checksum.\n\n---\n## Contents\n1. [Setup](#1-setup)\n2. [Download & load data](#2-load)\n3. [Inspect the database](#3-inspect)\n4. [Exact-field queries](#4-exact)\n5. [Substring search](#5-search)\n6. [Grouped search across device classes](#6-grouped)\n7. [Event narratives](#7-narratives)\n8. [Enrich with patient outcomes](#8-patient)\n9. [Enrich with device problem codes](#9-problems)\n10. [Trend analysis by year](#10-trends)\n11. [Raw SQL](#11-sql)\n12. [Archive a snapshot for publication](#12-archive)"
|
|
8
|
+
},
|
|
9
|
+
{
|
|
10
|
+
"cell_type": "markdown",
|
|
11
|
+
"id": "c51af603",
|
|
12
|
+
"metadata": {},
|
|
13
|
+
"source": [
|
|
14
|
+
"---\n",
|
|
15
|
+
"## 1. Setup <a id=\"1-setup\"></a>"
|
|
16
|
+
]
|
|
17
|
+
},
|
|
18
|
+
{
|
|
19
|
+
"cell_type": "code",
|
|
20
|
+
"execution_count": 1,
|
|
21
|
+
"id": "2cb7914b",
|
|
22
|
+
"metadata": {},
|
|
23
|
+
"outputs": [],
|
|
24
|
+
"source": [
|
|
25
|
+
"from pymaude import MaudeDatabase\n",
|
|
26
|
+
"\n",
|
|
27
|
+
"DB_PATH = './maude.duckdb' # persistent DuckDB file\n",
|
|
28
|
+
"DATA_DIR = './maude_data' # downloaded zip/txt files live here\n",
|
|
29
|
+
"YEARS = '2024-2026' # adjust to taste; more years = more data"
|
|
30
|
+
]
|
|
31
|
+
},
|
|
32
|
+
{
|
|
33
|
+
"cell_type": "markdown",
|
|
34
|
+
"id": "7b7d402f",
|
|
35
|
+
"metadata": {},
|
|
36
|
+
"source": "---\n## 2. Download & load data <a id=\"2-load\"></a>\n\nPass `download=True` to fetch from the FDA FTP area. Files are cached locally — re-running this cell only downloads files whose SHA-256 checksum has changed since the last run."
|
|
37
|
+
},
|
|
38
|
+
{
|
|
39
|
+
"cell_type": "code",
|
|
40
|
+
"execution_count": null,
|
|
41
|
+
"id": "033387c3",
|
|
42
|
+
"metadata": {},
|
|
43
|
+
"outputs": [],
|
|
44
|
+
"source": "db = MaudeDatabase(DB_PATH, data_dir=DATA_DIR, verbose=True, memory_limit='2GB')\n\ndb.add_years(\n YEARS,\n # tables=['master', 'device', 'text', 'patient', 'problem'],\n # tables=['master', 'device', 'text', 'problem'],\n tables=['master', 'problem'],\n download=True\n)"
|
|
45
|
+
},
|
|
46
|
+
{
|
|
47
|
+
"cell_type": "markdown",
|
|
48
|
+
"id": "31bc09f7",
|
|
49
|
+
"metadata": {},
|
|
50
|
+
"source": "To pull in the latest monthly FDA updates at any point — this is also the safest way to refresh cumulative tables (`master`, `patient`, `problem`), since it automatically re-requests every year they already have loaded:\n```python\ndb.update(download=True)\n```"
|
|
51
|
+
},
|
|
52
|
+
{
|
|
53
|
+
"cell_type": "markdown",
|
|
54
|
+
"id": "e2ab5aa5",
|
|
55
|
+
"metadata": {},
|
|
56
|
+
"source": "Call `add_years()` again at any time to extend coverage or add more tables — here we add `device`, `text`, and `patient` outcomes alongside the same year range.\n\nNote: `master` and `problem` are cumulative tables — they're fully replaced on every reload rather than updated year-by-year, so `add_years()` rejects a request that doesn't cover every year they already have loaded (it would otherwise silently drop the missing years). That's why this passes the full `YEARS` range again rather than just `'2026'` — pass `force_partial=True` if you really do want to narrow an existing cumulative table's coverage."
|
|
57
|
+
},
|
|
58
|
+
{
|
|
59
|
+
"cell_type": "code",
|
|
60
|
+
"execution_count": null,
|
|
61
|
+
"id": "fb16afa6",
|
|
62
|
+
"metadata": {},
|
|
63
|
+
"outputs": [],
|
|
64
|
+
"source": "db.add_years(YEARS, download=True)"
|
|
65
|
+
},
|
|
66
|
+
{
|
|
67
|
+
"cell_type": "markdown",
|
|
68
|
+
"id": "2cac8f0a",
|
|
69
|
+
"metadata": {},
|
|
70
|
+
"source": [
|
|
71
|
+
"---\n",
|
|
72
|
+
"## 3. Inspect the database <a id=\"3-inspect\"></a>"
|
|
73
|
+
]
|
|
74
|
+
},
|
|
75
|
+
{
|
|
76
|
+
"cell_type": "code",
|
|
77
|
+
"execution_count": null,
|
|
78
|
+
"id": "ceb0edfe",
|
|
79
|
+
"metadata": {},
|
|
80
|
+
"outputs": [],
|
|
81
|
+
"source": [
|
|
82
|
+
"db.info()"
|
|
83
|
+
]
|
|
84
|
+
},
|
|
85
|
+
{
|
|
86
|
+
"cell_type": "markdown",
|
|
87
|
+
"id": "03522383",
|
|
88
|
+
"metadata": {},
|
|
89
|
+
"source": [
|
|
90
|
+
"---\n",
|
|
91
|
+
"## 4. Exact-field queries <a id=\"4-exact\"></a>\n",
|
|
92
|
+
"\n",
|
|
93
|
+
"`query_device()` does **exact, case-insensitive** matching. Combine any of `brand_name`, `generic_name`, `manufacturer_name`, `product_code`, `start_date`, `end_date`."
|
|
94
|
+
]
|
|
95
|
+
},
|
|
96
|
+
{
|
|
97
|
+
"cell_type": "code",
|
|
98
|
+
"execution_count": null,
|
|
99
|
+
"id": "d36aad1d",
|
|
100
|
+
"metadata": {},
|
|
101
|
+
"outputs": [],
|
|
102
|
+
"source": [
|
|
103
|
+
"# All events for a specific product code (NIQ = venous stents)\n",
|
|
104
|
+
"niq = db.query_device(product_code='NIQ')\n",
|
|
105
|
+
"print(f'NIQ (venous stent) events loaded: {len(niq):,}')\n",
|
|
106
|
+
"niq[['MDR_REPORT_KEY', 'BRAND_NAME', 'GENERIC_NAME', 'MANUFACTURER_D_NAME', 'DATE_RECEIVED']].head(10)"
|
|
107
|
+
]
|
|
108
|
+
},
|
|
109
|
+
{
|
|
110
|
+
"cell_type": "code",
|
|
111
|
+
"execution_count": null,
|
|
112
|
+
"id": "27d566c9",
|
|
113
|
+
"metadata": {},
|
|
114
|
+
"outputs": [],
|
|
115
|
+
"source": [
|
|
116
|
+
"# Narrow to a specific brand + date window\n",
|
|
117
|
+
"bsci_jan_thru_aug_2021 = db.query_device(\n",
|
|
118
|
+
" brand_name='Boston Scientific',\n",
|
|
119
|
+
" start_date='2021-01-01',\n",
|
|
120
|
+
" end_date='2021-08-31'\n",
|
|
121
|
+
")\n",
|
|
122
|
+
"\n",
|
|
123
|
+
"print(f'Boston Scientific events in 2021 Jan-Aug: {len(bsci_jan_thru_aug_2021):,}')\n",
|
|
124
|
+
"bsci_jan_thru_aug_2021[['MDR_REPORT_KEY', 'BRAND_NAME', 'EVENT_TYPE', 'MANUFACTURER_G1_NAME', 'DATE_RECEIVED']]"
|
|
125
|
+
]
|
|
126
|
+
},
|
|
127
|
+
{
|
|
128
|
+
"cell_type": "markdown",
|
|
129
|
+
"id": "dc8abaa4",
|
|
130
|
+
"metadata": {},
|
|
131
|
+
"source": [
|
|
132
|
+
"---\n",
|
|
133
|
+
"## 5. Substring search <a id=\"5-search\"></a>\n",
|
|
134
|
+
"\n",
|
|
135
|
+
"`search_by_device_names()` does **case-insensitive substring matching** across a synthesized `DEVICE_NAME_CONCAT` column (BRAND_NAME | GENERIC_NAME | MANUFACTURER_D_NAME concatenated). This is useful because MAUDE entries are often inconsistent, with names of medical devices often appearing in one of these three columns. \n",
|
|
136
|
+
"\n",
|
|
137
|
+
"| Criteria format | Logic |\n",
|
|
138
|
+
"|---|---|\n",
|
|
139
|
+
"| `'term'` | single substring |\n",
|
|
140
|
+
"| `['a', 'b']` | a **OR** b |\n",
|
|
141
|
+
"| `[['a', 'b'], 'c']` | (a **AND** b) **OR** c |\n",
|
|
142
|
+
"| `{'group1': ..., 'group2': ...}` | grouped (see [section 6](#6-grouped)) |"
|
|
143
|
+
]
|
|
144
|
+
},
|
|
145
|
+
{
|
|
146
|
+
"cell_type": "code",
|
|
147
|
+
"execution_count": null,
|
|
148
|
+
"id": "29942997",
|
|
149
|
+
"metadata": {},
|
|
150
|
+
"outputs": [],
|
|
151
|
+
"source": [
|
|
152
|
+
"# Simple substring — anything with \"venous stent\" in any name field\n",
|
|
153
|
+
"venous_stents = db.search_by_device_names('venous stent')\n",
|
|
154
|
+
"print(f'Events matching \"venous stent\": {len(venous_stents):,}')\n",
|
|
155
|
+
"venous_stents[['MDR_REPORT_KEY', 'BRAND_NAME', 'GENERIC_NAME', 'DATE_RECEIVED']]"
|
|
156
|
+
]
|
|
157
|
+
},
|
|
158
|
+
{
|
|
159
|
+
"cell_type": "code",
|
|
160
|
+
"execution_count": null,
|
|
161
|
+
"id": "abc1713f",
|
|
162
|
+
"metadata": {},
|
|
163
|
+
"outputs": [],
|
|
164
|
+
"source": [
|
|
165
|
+
"# OR logic — venous stent OR iliac stent\n",
|
|
166
|
+
"venous_iliac = db.search_by_device_names(['venous stent', 'iliac stent'])\n",
|
|
167
|
+
"print(f'Venous OR iliac stents: {len(venous_iliac):,}')\n",
|
|
168
|
+
"venous_iliac['GENERIC_NAME'].value_counts().head(10)"
|
|
169
|
+
]
|
|
170
|
+
},
|
|
171
|
+
{
|
|
172
|
+
"cell_type": "code",
|
|
173
|
+
"execution_count": null,
|
|
174
|
+
"id": "bfcfe120",
|
|
175
|
+
"metadata": {},
|
|
176
|
+
"outputs": [],
|
|
177
|
+
"source": [
|
|
178
|
+
"# AND logic — must contain both \"argon\" AND \"cleaner\" (avoids false positives)\n",
|
|
179
|
+
"argon_cleaner = db.search_by_device_names([['argon','cleaner']])\n",
|
|
180
|
+
"print(f'Argon AND Cleaner events: {len(argon_cleaner):,}')\n",
|
|
181
|
+
"argon_cleaner[['MDR_REPORT_KEY', 'BRAND_NAME', 'GENERIC_NAME', 'MANUFACTURER_D_NAME', 'DATE_RECEIVED']]"
|
|
182
|
+
]
|
|
183
|
+
},
|
|
184
|
+
{
|
|
185
|
+
"cell_type": "code",
|
|
186
|
+
"execution_count": null,
|
|
187
|
+
"id": "d894b316",
|
|
188
|
+
"metadata": {},
|
|
189
|
+
"outputs": [],
|
|
190
|
+
"source": [
|
|
191
|
+
"# Combined: (argon AND cleaner) OR (angiojet) OR (thrombectomy)\n",
|
|
192
|
+
"thrombectomy_broad = db.search_by_device_names(\n",
|
|
193
|
+
" [['argon', 'cleaner'], 'angiojet', 'rotational thrombectomy']\n",
|
|
194
|
+
")\n",
|
|
195
|
+
"print(f'Broad rotational thrombectomy search: {len(thrombectomy_broad):,}')\n",
|
|
196
|
+
"thrombectomy_broad[['MDR_REPORT_KEY', 'BRAND_NAME', 'GENERIC_NAME']].head(10)"
|
|
197
|
+
]
|
|
198
|
+
},
|
|
199
|
+
{
|
|
200
|
+
"cell_type": "markdown",
|
|
201
|
+
"id": "11fa003a",
|
|
202
|
+
"metadata": {},
|
|
203
|
+
"source": [
|
|
204
|
+
"Results are pandas DataFrames — export anytime:"
|
|
205
|
+
]
|
|
206
|
+
},
|
|
207
|
+
{
|
|
208
|
+
"cell_type": "code",
|
|
209
|
+
"execution_count": null,
|
|
210
|
+
"id": "ad725c72",
|
|
211
|
+
"metadata": {},
|
|
212
|
+
"outputs": [],
|
|
213
|
+
"source": [
|
|
214
|
+
"thrombectomy_broad.to_csv('thrombectomy_broad.csv', index=False)\n",
|
|
215
|
+
"thrombectomy_broad.to_excel('thrombectomy_broad.xlsx', index=False)"
|
|
216
|
+
]
|
|
217
|
+
},
|
|
218
|
+
{
|
|
219
|
+
"cell_type": "markdown",
|
|
220
|
+
"id": "defb6b57",
|
|
221
|
+
"metadata": {},
|
|
222
|
+
"source": [
|
|
223
|
+
"---\n",
|
|
224
|
+
"## 6. Grouped search across device classes <a id=\"6-grouped\"></a>\n",
|
|
225
|
+
"\n",
|
|
226
|
+
"A **dict** of criteria runs multiple searches at once and labels each result with a `search_group` column. Useful for comparative studies."
|
|
227
|
+
]
|
|
228
|
+
},
|
|
229
|
+
{
|
|
230
|
+
"cell_type": "code",
|
|
231
|
+
"execution_count": null,
|
|
232
|
+
"id": "ba497816",
|
|
233
|
+
"metadata": {},
|
|
234
|
+
"outputs": [],
|
|
235
|
+
"source": [
|
|
236
|
+
"grouped = db.search_by_device_names({\n",
|
|
237
|
+
" 'venous_stents': ['venous stent', 'venous stenting'],\n",
|
|
238
|
+
" 'thrombectomy': [['argon', 'cleaner'], 'angiojet', 'rotational thrombectomy'],\n",
|
|
239
|
+
"})\n",
|
|
240
|
+
"\n",
|
|
241
|
+
"print(f'Total events: {len(grouped):,}')\n",
|
|
242
|
+
"print('\\nEvents by group:')\n",
|
|
243
|
+
"print(grouped['search_group'].value_counts().to_string())"
|
|
244
|
+
]
|
|
245
|
+
},
|
|
246
|
+
{
|
|
247
|
+
"cell_type": "code",
|
|
248
|
+
"execution_count": null,
|
|
249
|
+
"id": "4f737489",
|
|
250
|
+
"metadata": {},
|
|
251
|
+
"outputs": [],
|
|
252
|
+
"source": [
|
|
253
|
+
"grouped[['MDR_REPORT_KEY', 'BRAND_NAME', 'GENERIC_NAME', 'search_group']].head(15)"
|
|
254
|
+
]
|
|
255
|
+
},
|
|
256
|
+
{
|
|
257
|
+
"cell_type": "markdown",
|
|
258
|
+
"id": "8f8b4418",
|
|
259
|
+
"metadata": {},
|
|
260
|
+
"source": [
|
|
261
|
+
"---\n",
|
|
262
|
+
"## 7. Event narratives <a id=\"7-narratives\"></a>\n",
|
|
263
|
+
"\n",
|
|
264
|
+
"`get_narratives()` fetches the free-text FOI_TEXT descriptions for a set of MDR_REPORT_KEYs."
|
|
265
|
+
]
|
|
266
|
+
},
|
|
267
|
+
{
|
|
268
|
+
"cell_type": "code",
|
|
269
|
+
"execution_count": null,
|
|
270
|
+
"id": "505d9360",
|
|
271
|
+
"metadata": {},
|
|
272
|
+
"outputs": [],
|
|
273
|
+
"source": [
|
|
274
|
+
"narratives = db.get_narratives(thrombectomy_broad['MDR_REPORT_KEY'])\n",
|
|
275
|
+
"print(f'Narratives retrieved: {len(narratives):,}')\n",
|
|
276
|
+
"narratives.head(5)"
|
|
277
|
+
]
|
|
278
|
+
},
|
|
279
|
+
{
|
|
280
|
+
"cell_type": "code",
|
|
281
|
+
"execution_count": null,
|
|
282
|
+
"id": "cb10a454",
|
|
283
|
+
"metadata": {},
|
|
284
|
+
"outputs": [],
|
|
285
|
+
"source": [
|
|
286
|
+
"# Read a few narratives in full\n",
|
|
287
|
+
"for _, row in narratives.head(3).iterrows():\n",
|
|
288
|
+
" print(f\"=== MDR {row['MDR_REPORT_KEY']} ===\")\n",
|
|
289
|
+
" print(row['FOI_TEXT'][:500])\n",
|
|
290
|
+
" print()"
|
|
291
|
+
]
|
|
292
|
+
},
|
|
293
|
+
{
|
|
294
|
+
"cell_type": "markdown",
|
|
295
|
+
"id": "2f8d0518",
|
|
296
|
+
"metadata": {},
|
|
297
|
+
"source": [
|
|
298
|
+
"---\n",
|
|
299
|
+
"## 8. Enrich with patient outcomes <a id=\"8-patient\"></a>\n",
|
|
300
|
+
"\n",
|
|
301
|
+
"`enrich_with_patient_data()` left-joins the patient outcomes table. `SEQUENCE_NUMBER_OUTCOME` codes:\n",
|
|
302
|
+
"- `D` Death · `IN` Injury · `H` Hospitalization · `LT` Life Threatening · `OT` Other · `R` Required Intervention"
|
|
303
|
+
]
|
|
304
|
+
},
|
|
305
|
+
{
|
|
306
|
+
"cell_type": "code",
|
|
307
|
+
"execution_count": null,
|
|
308
|
+
"id": "dc713f30",
|
|
309
|
+
"metadata": {},
|
|
310
|
+
"outputs": [],
|
|
311
|
+
"source": [
|
|
312
|
+
"enriched = db.enrich_with_patient_data(thrombectomy_broad)\n",
|
|
313
|
+
"\n",
|
|
314
|
+
"with_outcome = enriched.dropna(subset=['SEQUENCE_NUMBER_OUTCOME'])\n",
|
|
315
|
+
"print(f'Events with patient outcome data: {len(with_outcome):,} / {len(enriched):,}')\n",
|
|
316
|
+
"with_outcome[['MDR_REPORT_KEY', 'BRAND_NAME', 'SEQUENCE_NUMBER_OUTCOME']].head(10)"
|
|
317
|
+
]
|
|
318
|
+
},
|
|
319
|
+
{
|
|
320
|
+
"cell_type": "code",
|
|
321
|
+
"execution_count": null,
|
|
322
|
+
"id": "8e86fb47",
|
|
323
|
+
"metadata": {},
|
|
324
|
+
"outputs": [],
|
|
325
|
+
"source": [
|
|
326
|
+
"OUTCOME_LABELS = {\n",
|
|
327
|
+
" 'D': 'Death', 'IN': 'Injury', 'H': 'Hospitalization',\n",
|
|
328
|
+
" 'LT': 'Life Threatening', 'OT': 'Other', 'R': 'Required Intervention'\n",
|
|
329
|
+
"}\n",
|
|
330
|
+
"\n",
|
|
331
|
+
"outcome_counts = (\n",
|
|
332
|
+
" with_outcome['SEQUENCE_NUMBER_OUTCOME']\n",
|
|
333
|
+
" .str.split(';')\n",
|
|
334
|
+
" .explode()\n",
|
|
335
|
+
" .map(lambda c: OUTCOME_LABELS.get(c, c))\n",
|
|
336
|
+
" .value_counts()\n",
|
|
337
|
+
")\n",
|
|
338
|
+
"outcome_counts"
|
|
339
|
+
]
|
|
340
|
+
},
|
|
341
|
+
{
|
|
342
|
+
"cell_type": "markdown",
|
|
343
|
+
"id": "074fc0f5",
|
|
344
|
+
"metadata": {},
|
|
345
|
+
"source": [
|
|
346
|
+
"---\n",
|
|
347
|
+
"## 9. Enrich with device problem codes <a id=\"9-problems\"></a>\n",
|
|
348
|
+
"\n",
|
|
349
|
+
"`enrich_with_problems()` left-joins the device problem code table (available from 2019 onwards)."
|
|
350
|
+
]
|
|
351
|
+
},
|
|
352
|
+
{
|
|
353
|
+
"cell_type": "code",
|
|
354
|
+
"execution_count": null,
|
|
355
|
+
"id": "55390046",
|
|
356
|
+
"metadata": {},
|
|
357
|
+
"outputs": [],
|
|
358
|
+
"source": [
|
|
359
|
+
"with_problems = db.enrich_with_problems(thrombectomy_broad)\n",
|
|
360
|
+
"\n",
|
|
361
|
+
"problem_counts = (\n",
|
|
362
|
+
" with_problems\n",
|
|
363
|
+
" .dropna(subset=['DEVICE_PROBLEM_CODE'])\n",
|
|
364
|
+
" ['DEVICE_PROBLEM_CODE']\n",
|
|
365
|
+
" .value_counts()\n",
|
|
366
|
+
")\n",
|
|
367
|
+
"print(f'Events with a problem code: {problem_counts.sum():,}')\n",
|
|
368
|
+
"print('\\nTop problem codes:')\n",
|
|
369
|
+
"problem_counts.head(10)"
|
|
370
|
+
]
|
|
371
|
+
},
|
|
372
|
+
{
|
|
373
|
+
"cell_type": "markdown",
|
|
374
|
+
"id": "b176b2ed",
|
|
375
|
+
"metadata": {},
|
|
376
|
+
"source": [
|
|
377
|
+
"---\n",
|
|
378
|
+
"## 10. Trend analysis by year <a id=\"10-trends\"></a>\n",
|
|
379
|
+
"\n",
|
|
380
|
+
"`get_trends_by_year()` counts events per calendar year. When results include a `search_group` column it breaks down by group automatically."
|
|
381
|
+
]
|
|
382
|
+
},
|
|
383
|
+
{
|
|
384
|
+
"cell_type": "code",
|
|
385
|
+
"execution_count": null,
|
|
386
|
+
"id": "268e04d6",
|
|
387
|
+
"metadata": {},
|
|
388
|
+
"outputs": [],
|
|
389
|
+
"source": [
|
|
390
|
+
"# Overall year-over-year trend\n",
|
|
391
|
+
"trends = db.get_trends_by_year(thrombectomy_broad)\n",
|
|
392
|
+
"print('Rotational thrombectomy — events by year:')\n",
|
|
393
|
+
"print(trends.to_string(index=False))"
|
|
394
|
+
]
|
|
395
|
+
},
|
|
396
|
+
{
|
|
397
|
+
"cell_type": "code",
|
|
398
|
+
"execution_count": null,
|
|
399
|
+
"id": "8fcf1083",
|
|
400
|
+
"metadata": {},
|
|
401
|
+
"outputs": [],
|
|
402
|
+
"source": [
|
|
403
|
+
"# Per-group trends\n",
|
|
404
|
+
"group_trends = db.get_trends_by_year(grouped)\n",
|
|
405
|
+
"print('Events by year and device group:')\n",
|
|
406
|
+
"print(group_trends.to_string(index=False))"
|
|
407
|
+
]
|
|
408
|
+
},
|
|
409
|
+
{
|
|
410
|
+
"cell_type": "code",
|
|
411
|
+
"execution_count": null,
|
|
412
|
+
"id": "8a27263f",
|
|
413
|
+
"metadata": {},
|
|
414
|
+
"outputs": [],
|
|
415
|
+
"source": [
|
|
416
|
+
"# Pivot for side-by-side comparison\n",
|
|
417
|
+
"if 'search_group' in group_trends.columns:\n",
|
|
418
|
+
" pivot = (\n",
|
|
419
|
+
" group_trends\n",
|
|
420
|
+
" .pivot(index='year', columns='search_group', values='event_count')\n",
|
|
421
|
+
" .fillna(0).astype(int)\n",
|
|
422
|
+
" )\n",
|
|
423
|
+
" print(pivot.to_string())"
|
|
424
|
+
]
|
|
425
|
+
},
|
|
426
|
+
{
|
|
427
|
+
"cell_type": "markdown",
|
|
428
|
+
"id": "ba90769d",
|
|
429
|
+
"metadata": {},
|
|
430
|
+
"source": "---\n## 11. Raw SQL <a id=\"11-sql\"></a>\n\n`db.query()` exposes DuckDB directly. Tables available: `master`, `device`, `text`, `patient`, `problem`."
|
|
431
|
+
},
|
|
432
|
+
{
|
|
433
|
+
"cell_type": "code",
|
|
434
|
+
"execution_count": null,
|
|
435
|
+
"id": "6d0a3dd3",
|
|
436
|
+
"metadata": {},
|
|
437
|
+
"outputs": [],
|
|
438
|
+
"source": [
|
|
439
|
+
"# Event type breakdown across all loaded data\n",
|
|
440
|
+
"db.query(\"\"\"\n",
|
|
441
|
+
" SELECT\n",
|
|
442
|
+
" EVENT_TYPE,\n",
|
|
443
|
+
" CASE EVENT_TYPE\n",
|
|
444
|
+
" WHEN 'D' THEN 'Death'\n",
|
|
445
|
+
" WHEN 'IN' THEN 'Injury'\n",
|
|
446
|
+
" WHEN 'M' THEN 'Malfunction'\n",
|
|
447
|
+
" WHEN 'O' THEN 'Other'\n",
|
|
448
|
+
" ELSE EVENT_TYPE\n",
|
|
449
|
+
" END AS label,\n",
|
|
450
|
+
" COUNT(*) AS n\n",
|
|
451
|
+
" FROM master\n",
|
|
452
|
+
" GROUP BY EVENT_TYPE\n",
|
|
453
|
+
" ORDER BY n DESC\n",
|
|
454
|
+
"\"\"\")"
|
|
455
|
+
]
|
|
456
|
+
},
|
|
457
|
+
{
|
|
458
|
+
"cell_type": "code",
|
|
459
|
+
"execution_count": null,
|
|
460
|
+
"id": "c380f28d",
|
|
461
|
+
"metadata": {},
|
|
462
|
+
"outputs": [],
|
|
463
|
+
"source": [
|
|
464
|
+
"# Top 10 manufacturers by adverse event volume (master joined to device)\n",
|
|
465
|
+
"db.query(\"\"\"\n",
|
|
466
|
+
" SELECT\n",
|
|
467
|
+
" d.MANUFACTURER_D_NAME,\n",
|
|
468
|
+
" COUNT(DISTINCT m.MDR_REPORT_KEY) AS events\n",
|
|
469
|
+
" FROM master m\n",
|
|
470
|
+
" JOIN device d USING (MDR_REPORT_KEY)\n",
|
|
471
|
+
" GROUP BY d.MANUFACTURER_D_NAME\n",
|
|
472
|
+
" ORDER BY events DESC\n",
|
|
473
|
+
" LIMIT 10\n",
|
|
474
|
+
"\"\"\")"
|
|
475
|
+
]
|
|
476
|
+
},
|
|
477
|
+
{
|
|
478
|
+
"cell_type": "code",
|
|
479
|
+
"execution_count": null,
|
|
480
|
+
"id": "40bc7adc",
|
|
481
|
+
"metadata": {},
|
|
482
|
+
"outputs": [],
|
|
483
|
+
"source": [
|
|
484
|
+
"# Parameterized query — safe for user-supplied input\n",
|
|
485
|
+
"db.query(\n",
|
|
486
|
+
" \"SELECT MDR_REPORT_KEY, BRAND_NAME, DATE_RECEIVED FROM device WHERE DEVICE_REPORT_PRODUCT_CODE = ? LIMIT 10\",\n",
|
|
487
|
+
" params=['NIQ']\n",
|
|
488
|
+
")"
|
|
489
|
+
]
|
|
490
|
+
},
|
|
491
|
+
{
|
|
492
|
+
"cell_type": "markdown",
|
|
493
|
+
"id": "afaade88",
|
|
494
|
+
"source": "---\n## 12. Archive a snapshot for publication <a id=\"12-archive\"></a>\n\nMAUDE is updated continuously and isn't versioned, so a query run today may return different results in a year. `db.archive()` freezes the exact database backing an analysis so it can be cited or uploaded alongside a paper (e.g. to Zenodo):\n\n- Checkpoints and copies the DuckDB file itself.\n- Writes a `manifest.json` recording, per loaded table/year: source file, SHA-256 checksum, row count, and load timestamp — plus the DuckDB and pymaude versions used to build it.\n- With `include_raw=True`, also copies the raw MAUDE source files referenced in the manifest into `output_dir/raw/`, so reviewers can re-derive the database from scratch.",
|
|
495
|
+
"metadata": {}
|
|
496
|
+
},
|
|
497
|
+
{
|
|
498
|
+
"cell_type": "code",
|
|
499
|
+
"id": "3eb5e80e",
|
|
500
|
+
"source": "manifest_path = db.archive('./maude_archive', include_raw=False)\nprint(f'Archive written, manifest at: {manifest_path}')",
|
|
501
|
+
"metadata": {},
|
|
502
|
+
"execution_count": null,
|
|
503
|
+
"outputs": []
|
|
504
|
+
},
|
|
505
|
+
{
|
|
506
|
+
"cell_type": "code",
|
|
507
|
+
"id": "26bdab3c",
|
|
508
|
+
"source": "# Inspect the generated manifest — this is what accompanies the archived .duckdb file\nimport json\n\nwith open(manifest_path) as f:\n manifest = json.load(f)\n\nprint(f\"pymaude {manifest['pymaude_version']} · duckdb {manifest['duckdb_version']} · {manifest['checksum_algorithm']}\")\nprint(f\"Database: {manifest['database']['filename']} ({manifest['database']['size_bytes']:,} bytes)\")\nprint(f\"Tables tracked: {len(manifest['tables'])}\")\nmanifest['tables'][:5]",
|
|
509
|
+
"metadata": {},
|
|
510
|
+
"execution_count": null,
|
|
511
|
+
"outputs": []
|
|
512
|
+
},
|
|
513
|
+
{
|
|
514
|
+
"cell_type": "code",
|
|
515
|
+
"execution_count": null,
|
|
516
|
+
"id": "2a116968",
|
|
517
|
+
"metadata": {},
|
|
518
|
+
"outputs": [],
|
|
519
|
+
"source": [
|
|
520
|
+
"db.close()\n",
|
|
521
|
+
"\n"
|
|
522
|
+
]
|
|
523
|
+
}
|
|
524
|
+
],
|
|
525
|
+
"metadata": {
|
|
526
|
+
"kernelspec": {
|
|
527
|
+
"display_name": "venv (3.12.3)",
|
|
528
|
+
"language": "python",
|
|
529
|
+
"name": "python3"
|
|
530
|
+
},
|
|
531
|
+
"language_info": {
|
|
532
|
+
"codemirror_mode": {
|
|
533
|
+
"name": "ipython",
|
|
534
|
+
"version": 3
|
|
535
|
+
},
|
|
536
|
+
"file_extension": ".py",
|
|
537
|
+
"mimetype": "text/x-python",
|
|
538
|
+
"name": "python",
|
|
539
|
+
"nbconvert_exporter": "python",
|
|
540
|
+
"pygments_lexer": "ipython3",
|
|
541
|
+
"version": "3.12.3"
|
|
542
|
+
}
|
|
543
|
+
},
|
|
544
|
+
"nbformat": 4,
|
|
545
|
+
"nbformat_minor": 5
|
|
546
|
+
}
|