pubmed-markdown 0.1.0__tar.gz

pubmed_markdown-0.1.0/.claude/settings.local.json

@@ -0,0 +1,43 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(grep:*)",
+      "Bash(rg:*)",
+      "Bash(/opt/homebrew/lib/node_modules/@anthropic-ai/claude-code/vendor/ripgrep/arm64-darwin/rg -n 'class=\"\".*article.*\"\"' \"/Users/shloknatarajan/stanford/research/daneshjou/pmid-article-resolver/data/raw_html/PMC10038974.html\")",
+      "Bash(/opt/homebrew/lib/node_modules/@anthropic-ai/claude-code/vendor/ripgrep/arm64-darwin/rg -n 'section.*abstract|section.*introduction|section.*methods|section.*results|section.*discussion|section.*conclusion' \"/Users/shloknatarajan/stanford/research/daneshjou/pmid-article-resolver/data/raw_html/PMC10038974.html\")",
+      "Bash(/opt/homebrew/lib/node_modules/@anthropic-ai/claude-code/vendor/ripgrep/arm64-darwin/rg -n '<h[1-6]' \"/Users/shloknatarajan/stanford/research/daneshjou/pmid-article-resolver/data/raw_html/PMC10038974.html\")",
+      "Bash(/opt/homebrew/lib/node_modules/@anthropic-ai/claude-code/vendor/ripgrep/arm64-darwin/rg -n '<figure|<table' \"/Users/shloknatarajan/stanford/research/daneshjou/pmid-article-resolver/data/raw_html/PMC10038974.html\")",
+      "Bash(/opt/homebrew/lib/node_modules/@anthropic-ai/claude-code/vendor/ripgrep/arm64-darwin/rg -n 'href=\"\".*\\.pdf\"\"|\\.jpg\"\"|\\.png\"\"|\\.svg\"\"' \"/Users/shloknatarajan/stanford/research/daneshjou/pmid-article-resolver/data/raw_html/PMC10038974.html\")",
+      "Bash(/opt/homebrew/lib/node_modules/@anthropic-ai/claude-code/vendor/ripgrep/arm64-darwin/rg -n 'cdn\\.ncbi\\.nlm\\.nih\\.gov.*\\.jpg|\\.png' \"/Users/shloknatarajan/stanford/research/daneshjou/pmid-article-resolver/data/raw_html/PMC10038974.html\")",
+      "Bash(/opt/homebrew/lib/node_modules/@anthropic-ai/claude-code/vendor/ripgrep/arm64-darwin/rg -n 'References|ref-list|class=\"\"ref\"\"' \"/Users/shloknatarajan/stanford/research/daneshjou/pmid-article-resolver/data/raw_html/PMC10038974.html\")",
+      "Bash(/opt/homebrew/lib/node_modules/@anthropic-ai/claude-code/vendor/ripgrep/arm64-darwin/rg -n 'Discussion|discussion' \"/Users/shloknatarajan/stanford/research/daneshjou/pmid-article-resolver/data/raw_html/PMC10038974.html\")",
+      "Bash(/opt/homebrew/lib/node_modules/@anthropic-ai/claude-code/vendor/ripgrep/arm64-darwin/rg -n '<h[2-6].*class=\"pmc_sec_title\"' \"/Users/shloknatarajan/stanford/research/daneshjou/pmid-article-resolver/data/raw_html/PMC10038974.html\")",
+      "Bash(python:*)",
+      "Bash(pixi run:*)",
+      "Bash(find:*)",
+      "Bash(rm:*)",
+      "Bash(ls:*)",
+      "Bash(du -sh:*)",
+      "Bash(pixi install:*)",
+      "Bash(curl:*)",
+      "Bash(pkill:*)",
+      "Bash(node:*)",
+      "Bash(git config:*)",
+      "Bash(python3:*)",
+      "Bash(test -f ~/.gsd/defaults.json)",
+      "Read(//Users/shloknatarajan/.gsd/**)",
+      "Skill(gsd:progress)",
+      "WebSearch",
+      "Bash(pip install:*)",
+      "Bash(python -c \"from pubmed_downloader.pubmed_downloader import main; print\\('Entry point OK'\\)\" && unzip -l dist/pubmed_downloader-0.1.0-py3-none-any.whl | head -20)",
+      "Read(//private/tmp/**)",
+      "Bash(source pubmed_test_env/bin/activate)",
+      "Bash(source /tmp/pubmed_test_env/bin/activate && pubmed-download --help 2>&1)",
+      "Bash(source /tmp/pubmed_test_env/bin/activate && pip install /Users/shloknatarajan/stanford/research/daneshjou/PubMedDownloader 2>&1 | tail -3)",
+      "Bash(source /tmp/pubmed_test_env/bin/activate && cd /tmp && rm -rf test_output && pubmed-download --file_path=test_pmids.txt --save_dir=test_output 2>&1)",
+      "Bash(twine check:*)",
+      "Bash(pip index:*)"
+    ]
+  },
+  "enableAllProjectMcpServers": false
+}

pubmed_markdown-0.1.0/.env.example

@@ -0,0 +1,6 @@
+# Required: Email for NCBI API compliance
+NCBI_EMAIL=your.email@example.com
+
+# Optional: NCBI API key (raises rate limit from 3 to 10 requests/sec)
+# Register at https://www.ncbi.nlm.nih.gov/account/settings/
+# NCBI_API_KEY=your_api_key_here

pubmed_markdown-0.1.0/.gitattributes

@@ -0,0 +1,2 @@
+# SCM syntax highlighting & preventing 3-way merges
+pixi.lock merge=binary linguist-language=YAML linguist-generated=true

pubmed_markdown-0.1.0/.gitignore

@@ -0,0 +1,18 @@
+
+# pixi environments
+.pixi
+*.egg-info
+
+# environment files
+.env
+
+# python cache
+**/*__pycache__
+**/*.pyc
+**/.DS_Store
+
+# data
+data/*
+
+# cursor
+.cursor

pubmed_markdown-0.1.0/.vscode/launch.json

@@ -0,0 +1,49 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "PMCID to Text",
+            "type": "debugpy",
+            "request": "launch",
+            "cwd": "${workspaceFolder}",
+            "module": "src.pmcid_to_text.fetch_article",
+            "args": ["--pmcid", "PMC1884285", "--save_path", "data/articles"],
+            "console": "integratedTerminal"
+        },
+        {
+            "name": "Check OA Status",
+            "type": "debugpy",
+            "request": "launch",
+            "cwd": "${workspaceFolder}",
+            "module": "src.pmcid_to_text.check_oa_status",
+            "console": "integratedTerminal"
+        },
+        {
+            "name": "Aaron HTML",
+            "type": "debugpy",
+            "request": "launch",
+            "cwd": "${workspaceFolder}",
+            "module": "src.pmcid_to_text.html_getter",
+            "console": "integratedTerminal"
+        },
+        {
+            "name": "Single Article",
+            "type": "debugpy",
+            "request": "launch",
+            "cwd": "${workspaceFolder}",
+            "module": "src.markdown_from_pmid",
+            "args": ["--pmid", "12895196", "--save_dir", "data"],
+            "console": "integratedTerminal"
+        },
+        {
+            "name": "Pubmed Converter",
+            "type": "debugpy",
+            "request": "launch",
+            "cwd": "${workspaceFolder}",
+            "module": "src.pubmed_downloader",
+        }
+    ]
+}

pubmed_markdown-0.1.0/Dockerfile

@@ -0,0 +1,13 @@
+FROM python:3.12-slim
+
+WORKDIR /app
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY pubmed_downloader/ pubmed_downloader/
+COPY api.py .
+
+EXPOSE 8000
+
+CMD ["uvicorn", "api:app", "--host", "0.0.0.0", "--port", "8000"]

pubmed_markdown-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Shlok Natarajan
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

pubmed_markdown-0.1.0/PKG-INFO

@@ -0,0 +1,179 @@
+Metadata-Version: 2.4
+Name: pubmed-markdown
+Version: 0.1.0
+Summary: Convert PubMed articles (PMIDs or PMCIDs) to clean, structured markdown with full text, abstracts, and supplementary materials
+Project-URL: Homepage, https://github.com/shloknatarajan/PubMedDownloader
+Project-URL: Repository, https://github.com/shloknatarajan/PubMedDownloader
+Project-URL: Issues, https://github.com/shloknatarajan/PubMedDownloader/issues
+Author-email: Shlok Natarajan <shlok.natarajan@gmail.com>
+License-Expression: MIT
+License-File: LICENSE
+Keywords: bioinformatics,markdown,pharmacogenomics,pmc,pubmed
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
+Requires-Python: >=3.11
+Requires-Dist: beautifulsoup4>=4.13.0
+Requires-Dist: biopython>=1.85
+Requires-Dist: loguru>=0.7.0
+Requires-Dist: pandas>=2.0.0
+Requires-Dist: python-dotenv>=1.0.0
+Requires-Dist: requests>=2.32.0
+Requires-Dist: tqdm>=4.67.0
+Description-Content-Type: text/markdown
+
+# PubMed Downloader
+
+Convert PubMed articles to clean, structured markdown. Handles the full pipeline: PMID resolution, full-text extraction via PubMed Central, HTML-to-markdown conversion, and supplementary material retrieval.
+
+Articles without open-access full text automatically fall back to abstract-only download.
+
+## Installation
+
+```bash
+pip install git+https://github.com/shloknatarajan/PubMedDownloader.git
+```
+
+## Setup
+
+Set your email for NCBI API identification (optional but recommended):
+
+```bash
+export NCBI_EMAIL=your-email@institution.edu
+```
+
+Or create a `.env` file in your working directory:
+
+```env
+NCBI_EMAIL=your-email@institution.edu
+```
+
+## Usage
+
+### Python API
+
+**Single article (returns markdown string, no files created):**
+
+```python
+from pubmed_downloader import PubMedDownloader
+
+downloader = PubMedDownloader()
+
+# From PMID (resolves to PMCID automatically, falls back to abstract if not open access)
+markdown = downloader.single_pmid_to_markdown("12895196")
+
+# From PMCID directly
+markdown = downloader.single_pmcid_to_markdown("PMC1884285")
+```
+
+**Batch processing (saves HTML and markdown files to disk):**
+
+```python
+from pubmed_downloader import PubMedDownloader
+
+downloader = PubMedDownloader()
+pmids = ["12895196", "17872605", "25051018"]
+downloader.pmids_to_markdown(pmids, save_dir="data")
+```
+
+This creates:
+```
+data/
+├── html/          # Raw HTML from PMC
+├── markdown/      # Converted markdown files
+├── cache/         # PMID-to-PMCID mapping cache
+└── pmcids.txt     # Resolved PMCIDs
+```
+
+**Add supplementary materials to existing markdown files:**
+
+```python
+downloader.add_supplements_to_existing(save_dir="data")
+```
+
+**Individual utility functions:**
+
+```python
+from pubmed_downloader import (
+    get_pmcid_from_pmid,
+    get_html_from_pmcid,
+    get_abstract_markdown_from_pmid,
+    fetch_bioc_supplement,
+)
+
+# Resolve PMIDs to PMCIDs
+mapping = get_pmcid_from_pmid(["12895196", "17872605"])
+
+# Fetch raw HTML from PMC
+html = get_html_from_pmcid("PMC1884285")
+
+# Get abstract for non-open-access articles
+abstract_md = get_abstract_markdown_from_pmid("12345678")
+
+# Get supplementary material text
+supplement = fetch_bioc_supplement("PMC6435416")
+```
+
+### Command Line
+
+```bash
+# Convert PMIDs from a file (one PMID per line)
+pubmed-download --file_path=pmids.txt --save_dir=data
+
+# Add supplementary materials to existing markdown
+pubmed-download --add_supplements --save_dir=data
+
+# Clear all caches
+pubmed-download --clear_caches
+```
+
+### API Reference
+
+| Method | Creates Files | Returns | Use Case |
+|--------|--------------|---------|----------|
+| `single_pmid_to_markdown()` | No | Markdown string | Single article, programmatic use |
+| `single_pmcid_to_markdown()` | No | Markdown string | Direct PMCID conversion |
+| `pmids_to_markdown()` | Yes | None | Batch processing, building datasets |
+| `local_html_to_markdown()` | Yes | None | Re-convert existing HTML files |
+| `add_supplements_to_existing()` | Yes | None | Append supplements to existing markdown |
+
+## PharmGKB Integration
+
+Extract PMIDs from PharmGKB variant annotations for pharmacogenomics research:
+
+```python
+from pubmed_downloader.pharmgkb_annotations import get_pmid_list
+from pubmed_downloader import PubMedDownloader
+
+# Download PharmGKB annotations and extract PMIDs
+pmids = get_pmid_list(save_dir="data")
+
+# Convert to markdown
+downloader = PubMedDownloader()
+downloader.pmids_to_markdown([str(p) for p in pmids], save_dir="data")
+```
+
+## How It Works
+
+1. **PMID to PMCID** -- Uses NCBI's ID Converter API with batching, caching (30-day expiry), and rate limiting
+2. **HTML extraction** -- Fetches full article HTML from PubMed Central
+3. **Markdown conversion** -- Converts HTML to structured markdown preserving tables, figures, citations, and references
+4. **Supplementary materials** -- Fetches pre-processed supplement text via NCBI's BioC API
+5. **Abstract fallback** -- Articles not in PMC Open Access get abstract + metadata via NCBI E-Fetch
+
+## Configuration
+
+| Environment Variable | Default | Description |
+|---------------------|---------|-------------|
+| `NCBI_EMAIL` | None | Email for NCBI API identification |
+| `PMID_CACHE_DIR` | `data/cache` | Cache directory path |
+| `PMID_CACHE_FILE` | `pmid_to_pmcid.json` | Cache filename |
+
+## License
+
+MIT

pubmed_markdown-0.1.0/README.md

@@ -0,0 +1,150 @@
+# PubMed Downloader
+
+Convert PubMed articles to clean, structured markdown. Handles the full pipeline: PMID resolution, full-text extraction via PubMed Central, HTML-to-markdown conversion, and supplementary material retrieval.
+
+Articles without open-access full text automatically fall back to abstract-only download.
+
+## Installation
+
+```bash
+pip install git+https://github.com/shloknatarajan/PubMedDownloader.git
+```
+
+## Setup
+
+Set your email for NCBI API identification (optional but recommended):
+
+```bash
+export NCBI_EMAIL=your-email@institution.edu
+```
+
+Or create a `.env` file in your working directory:
+
+```env
+NCBI_EMAIL=your-email@institution.edu
+```
+
+## Usage
+
+### Python API
+
+**Single article (returns markdown string, no files created):**
+
+```python
+from pubmed_downloader import PubMedDownloader
+
+downloader = PubMedDownloader()
+
+# From PMID (resolves to PMCID automatically, falls back to abstract if not open access)
+markdown = downloader.single_pmid_to_markdown("12895196")
+
+# From PMCID directly
+markdown = downloader.single_pmcid_to_markdown("PMC1884285")
+```
+
+**Batch processing (saves HTML and markdown files to disk):**
+
+```python
+from pubmed_downloader import PubMedDownloader
+
+downloader = PubMedDownloader()
+pmids = ["12895196", "17872605", "25051018"]
+downloader.pmids_to_markdown(pmids, save_dir="data")
+```
+
+This creates:
+```
+data/
+├── html/          # Raw HTML from PMC
+├── markdown/      # Converted markdown files
+├── cache/         # PMID-to-PMCID mapping cache
+└── pmcids.txt     # Resolved PMCIDs
+```
+
+**Add supplementary materials to existing markdown files:**
+
+```python
+downloader.add_supplements_to_existing(save_dir="data")
+```
+
+**Individual utility functions:**
+
+```python
+from pubmed_downloader import (
+    get_pmcid_from_pmid,
+    get_html_from_pmcid,
+    get_abstract_markdown_from_pmid,
+    fetch_bioc_supplement,
+)
+
+# Resolve PMIDs to PMCIDs
+mapping = get_pmcid_from_pmid(["12895196", "17872605"])
+
+# Fetch raw HTML from PMC
+html = get_html_from_pmcid("PMC1884285")
+
+# Get abstract for non-open-access articles
+abstract_md = get_abstract_markdown_from_pmid("12345678")
+
+# Get supplementary material text
+supplement = fetch_bioc_supplement("PMC6435416")
+```
+
+### Command Line
+
+```bash
+# Convert PMIDs from a file (one PMID per line)
+pubmed-download --file_path=pmids.txt --save_dir=data
+
+# Add supplementary materials to existing markdown
+pubmed-download --add_supplements --save_dir=data
+
+# Clear all caches
+pubmed-download --clear_caches
+```
+
+### API Reference
+
+| Method | Creates Files | Returns | Use Case |
+|--------|--------------|---------|----------|
+| `single_pmid_to_markdown()` | No | Markdown string | Single article, programmatic use |
+| `single_pmcid_to_markdown()` | No | Markdown string | Direct PMCID conversion |
+| `pmids_to_markdown()` | Yes | None | Batch processing, building datasets |
+| `local_html_to_markdown()` | Yes | None | Re-convert existing HTML files |
+| `add_supplements_to_existing()` | Yes | None | Append supplements to existing markdown |
+
+## PharmGKB Integration
+
+Extract PMIDs from PharmGKB variant annotations for pharmacogenomics research:
+
+```python
+from pubmed_downloader.pharmgkb_annotations import get_pmid_list
+from pubmed_downloader import PubMedDownloader
+
+# Download PharmGKB annotations and extract PMIDs
+pmids = get_pmid_list(save_dir="data")
+
+# Convert to markdown
+downloader = PubMedDownloader()
+downloader.pmids_to_markdown([str(p) for p in pmids], save_dir="data")
+```
+
+## How It Works
+
+1. **PMID to PMCID** -- Uses NCBI's ID Converter API with batching, caching (30-day expiry), and rate limiting
+2. **HTML extraction** -- Fetches full article HTML from PubMed Central
+3. **Markdown conversion** -- Converts HTML to structured markdown preserving tables, figures, citations, and references
+4. **Supplementary materials** -- Fetches pre-processed supplement text via NCBI's BioC API
+5. **Abstract fallback** -- Articles not in PMC Open Access get abstract + metadata via NCBI E-Fetch
+
+## Configuration
+
+| Environment Variable | Default | Description |
+|---------------------|---------|-------------|
+| `NCBI_EMAIL` | None | Email for NCBI API identification |
+| `PMID_CACHE_DIR` | `data/cache` | Cache directory path |
+| `PMID_CACHE_FILE` | `pmid_to_pmcid.json` | Cache filename |
+
+## License
+
+MIT