biomarkit 1.0.0__tar.gz

biomarkit-1.0.0/.gitignore

@@ -0,0 +1,20 @@
+__pycache__*
+_data*
+corpora/
+.idea/
+*.xml
+htmlcov/
+secrets.env
+.coverage
+*.sqlite
+/logs
+tmp/
+.venv/
+.pytest_cache/
+.claude/
+.worktrees/
+llm_response.txt
+text_download/apis/clients/am_phyto_soc.py
+text_download/apis/clients/copernicus.py
+evals/
+download_results.xlsx

biomarkit-1.0.0/.python-version

@@ -0,0 +1 @@
+3.12

biomarkit-1.0.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Jack Bruton
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

biomarkit-1.0.0/PKG-INFO

@@ -0,0 +1,185 @@
+Metadata-Version: 2.5
+Name: biomarkit
+Version: 1.0.0
+Summary: Downloads full-text scientific publications from a Scopus query and converts them into standardised Markdown.
+Author-email: Jack Bruton <jack.bruton@hotmail.com>
+License: MIT License
+        
+        Copyright (c) 2025 Jack Bruton
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+License-File: LICENSE
+Requires-Python: <3.14,>=3.12
+Requires-Dist: mineru[pipeline]>=3.4.0
+Requires-Dist: openai>=1.0.0
+Requires-Dist: pandas>=2.3.3
+Requires-Dist: playwright>=1.56.0
+Requires-Dist: plotly>=6.5.0
+Requires-Dist: pydantic>=2.12.0
+Requires-Dist: python-dotenv>=1.0.0
+Requires-Dist: requests>=2.32.5
+Requires-Dist: torch==2.9.1
+Requires-Dist: torchvision==0.24.1
+Requires-Dist: tqdm>=4.67.0
+Requires-Dist: wiley-tdm>=1.0.0
+Provides-Extra: dev
+Requires-Dist: coverage>=7.11.0; extra == 'dev'
+Requires-Dist: pytest>=8.4.2; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# Biomarkit
+
+A Python package for automated publisher-agnostic corpus download and structured Markdown file conversion 
+for full-text scientific publications.
+
+---
+
+## Overview
+### Full-text manuscript download from major publishers with `download_corpus()`.
+Attempt download of full-text PDFs and XML files for all publications listed in a Scopus query CSV file by DOI.
+Download is attempted first via open-access routes including arXiv, bioRxiv and Unpaywall. On failure, download is 
+attempted using specific publisher APIs including Elsevier, Wiley, Springer, MDPI, Frontiers.
+
+**API keys are required for publishers that do not support fully automated text-mining**
+
+### Derive JSON manuscript structures from PDF and XML files with `transform_text()`.
+Converts downloaded PDFs and XML files into a single, structured JSON format. XMLs are parsed directly, and PDFs are
+converted locally via MinerU (GPU can be used here for performance improvements – See [MinerU](https://github.com/opendatalab/mineru)).
+Captures other outputs including raw texts and figures.
+
+
+### Convert all manuscripts to a customisable Markdown format with `standardise_text()`.
+Converts JSON-structured text into a standardised Markdown format. A single Markdown file is generated for each
+document. Supports parameters for toggling inclusion of manuscript sections: figures, tables, references, and Latex 
+equations.
+
+The "force_imrad_structure" parameter will force each output Markdown file to contain marked sections for "Introduction"
+"Methods", "Results" and "Discussion" (can be omitted).
+
+---
+
+## Example:
+### Complete a full download and generate markdown files for each DOI in a given Scopus query:
+
+```python
+import biomarkit
+
+# Make a new corpus and set it as active (writes CORPUS_NAME to secrets.env)
+biomarkit.build_new_corpus(name="my_corpus", scopus_file="path/to/scopus.csv", set_active=True)
+
+# Attempt download for all DOIs:
+publications = biomarkit.download_corpus(check_opensource=True, generate_report=True)
+
+# Run OCR and XML parser to generate JSON document structures:
+publications = biomarkit.transform_text(publications)
+
+# Build final markdown files:
+biomarkit.standardise_text(publications, keep_figures=False, keep_tables=True, keep_latex=True, force_imrad_structure=True)
+```
+
+### Publications as Python objects:
+`Publication` Python objects are returned at each step and are useful for integrating the package within larger 
+Python workflows. For example, storing each publication as a Python object makes it easy to query metadata and filepaths:
+
+```python
+import biomarkit
+
+publications = biomarkit.download_corpus(check_opensource=True)
+
+# Query metadata for all DOIs in the active corpus: 
+for pub in publications:
+    print(pub.doi,
+          pub.title,
+          pub.abstract,
+          pub.publisher,
+          pub.document_type,
+          pub.publication_filepath,
+          pub.final_md_filepath
+          )
+```
+
+---
+## Configuration
+
+### Workspace directory
+
+By default, biomarkit creates a `corpora/` folder in your **current working directory**. To use a different location, set the `BIOMARKIT_DIR` environment variable before running:
+
+```bash
+# Linux/macOS
+export BIOMARKIT_DIR=/path/to/my/workspace
+
+# Windows (PowerShell)
+$env:BIOMARKIT_DIR = "C:\path\to\my\workspace"
+```
+
+Biomarkit also looks for `secrets.env` inside this directory.
+
+### API keys and settings
+
+All user-facing settings live in `secrets.env` (copy from [`secrets.env.example`](secrets.env.example) and place it in your workspace directory):
+
+| Setting | Description |
+|---|---|
+| `CORPUS_NAME` | Name of the active corpus folder under `corpora/` |
+| `USER_EMAIL` | Your email address (required by some publisher APIs) |
+| `WILEY_TDM_TOKEN` | Wiley TDM API token (optional) |
+| `SPRINGER_API_KEY` | Springer API key (optional) |
+| `ELSEVIER_API_KEY` | Elsevier API key (optional) |
+| `LLM_BASE_URL` | OpenAI-compatible endpoint for the fallback section classifier (defaults to local [Ollama](https://ollama.com)) |
+| `LLM_MODEL_NAME` | Model used by the fallback classifier (default `gemma3:12b`) |
+
+### The corpus folder structure
+
+All inputs and outputs for a Scopus query live under `corpora/<corpus_name>/` inside your workspace directory. The active corpus is selected by setting `CORPUS_NAME` in `secrets.env`. Each corpus folder has the following structure:
+
+```
+corpora/
+└── <corpus_name>/
+    ├── scopus.csv       # Scopus query export (input)
+    ├── manuscripts/     # Downloaded full-text PDFs/XMLs — output of `download_corpus()`
+    ├── intermediates/   # MinerU output files and JSON structures — output of `transform_text()`
+    ├── results/         # Final standardised Markdown files — output of `standardise_text()`
+    ├── reports/         # All HTML output reports
+    ├── logs/            # Run logs
+    └── sqlite.db        # SQLite cache for this corpus
+```
+Everything except `scopus.csv` is created automatically. Create a new corpus with `build_new_corpus(name="my_corpus", scopus_file="path/to/scopus.csv")`.
+
+---
+
+## Dependencies
+- Python 3.12–3.13:
+- Optional: A CUDA-capable GPU (big speedup for PDF conversion via MinerU) + `torch` (CUDA 12.8)
+- Optional: [Ollama](https://ollama.com/download) running `gemma3:12b` (~8 GB), used as an LLM fallback to classify 
+ambiguous section headings. Without it, the pipeline runs, but accuracy in removing paper boilerplate is decreased.
+
+Core dependencies: `pandas`, `pydantic`, `requests`, `playwright`, `plotly`, `openai`, `wiley-tdm`, `mineru[pipeline]`, `torch`
+
+## Installation
+Install with [uv](https://github.com/astral-sh/uv):
+
+```bash
+uv pip install biomarkit
+```
+
+Or with pip:
+```python
+pip install biomarkit
+```

biomarkit-1.0.0/README.md

@@ -0,0 +1,140 @@
+# Biomarkit
+
+A Python package for automated publisher-agnostic corpus download and structured Markdown file conversion 
+for full-text scientific publications.
+
+---
+
+## Overview
+### Full-text manuscript download from major publishers with `download_corpus()`.
+Attempt download of full-text PDFs and XML files for all publications listed in a Scopus query CSV file by DOI.
+Download is attempted first via open-access routes including arXiv, bioRxiv and Unpaywall. On failure, download is 
+attempted using specific publisher APIs including Elsevier, Wiley, Springer, MDPI, Frontiers.
+
+**API keys are required for publishers that do not support fully automated text-mining**
+
+### Derive JSON manuscript structures from PDF and XML files with `transform_text()`.
+Converts downloaded PDFs and XML files into a single, structured JSON format. XMLs are parsed directly, and PDFs are
+converted locally via MinerU (GPU can be used here for performance improvements – See [MinerU](https://github.com/opendatalab/mineru)).
+Captures other outputs including raw texts and figures.
+
+
+### Convert all manuscripts to a customisable Markdown format with `standardise_text()`.
+Converts JSON-structured text into a standardised Markdown format. A single Markdown file is generated for each
+document. Supports parameters for toggling inclusion of manuscript sections: figures, tables, references, and Latex 
+equations.
+
+The "force_imrad_structure" parameter will force each output Markdown file to contain marked sections for "Introduction"
+"Methods", "Results" and "Discussion" (can be omitted).
+
+---
+
+## Example:
+### Complete a full download and generate markdown files for each DOI in a given Scopus query:
+
+```python
+import biomarkit
+
+# Make a new corpus and set it as active (writes CORPUS_NAME to secrets.env)
+biomarkit.build_new_corpus(name="my_corpus", scopus_file="path/to/scopus.csv", set_active=True)
+
+# Attempt download for all DOIs:
+publications = biomarkit.download_corpus(check_opensource=True, generate_report=True)
+
+# Run OCR and XML parser to generate JSON document structures:
+publications = biomarkit.transform_text(publications)
+
+# Build final markdown files:
+biomarkit.standardise_text(publications, keep_figures=False, keep_tables=True, keep_latex=True, force_imrad_structure=True)
+```
+
+### Publications as Python objects:
+`Publication` Python objects are returned at each step and are useful for integrating the package within larger 
+Python workflows. For example, storing each publication as a Python object makes it easy to query metadata and filepaths:
+
+```python
+import biomarkit
+
+publications = biomarkit.download_corpus(check_opensource=True)
+
+# Query metadata for all DOIs in the active corpus: 
+for pub in publications:
+    print(pub.doi,
+          pub.title,
+          pub.abstract,
+          pub.publisher,
+          pub.document_type,
+          pub.publication_filepath,
+          pub.final_md_filepath
+          )
+```
+
+---
+## Configuration
+
+### Workspace directory
+
+By default, biomarkit creates a `corpora/` folder in your **current working directory**. To use a different location, set the `BIOMARKIT_DIR` environment variable before running:
+
+```bash
+# Linux/macOS
+export BIOMARKIT_DIR=/path/to/my/workspace
+
+# Windows (PowerShell)
+$env:BIOMARKIT_DIR = "C:\path\to\my\workspace"
+```
+
+Biomarkit also looks for `secrets.env` inside this directory.
+
+### API keys and settings
+
+All user-facing settings live in `secrets.env` (copy from [`secrets.env.example`](secrets.env.example) and place it in your workspace directory):
+
+| Setting | Description |
+|---|---|
+| `CORPUS_NAME` | Name of the active corpus folder under `corpora/` |
+| `USER_EMAIL` | Your email address (required by some publisher APIs) |
+| `WILEY_TDM_TOKEN` | Wiley TDM API token (optional) |
+| `SPRINGER_API_KEY` | Springer API key (optional) |
+| `ELSEVIER_API_KEY` | Elsevier API key (optional) |
+| `LLM_BASE_URL` | OpenAI-compatible endpoint for the fallback section classifier (defaults to local [Ollama](https://ollama.com)) |
+| `LLM_MODEL_NAME` | Model used by the fallback classifier (default `gemma3:12b`) |
+
+### The corpus folder structure
+
+All inputs and outputs for a Scopus query live under `corpora/<corpus_name>/` inside your workspace directory. The active corpus is selected by setting `CORPUS_NAME` in `secrets.env`. Each corpus folder has the following structure:
+
+```
+corpora/
+└── <corpus_name>/
+    ├── scopus.csv       # Scopus query export (input)
+    ├── manuscripts/     # Downloaded full-text PDFs/XMLs — output of `download_corpus()`
+    ├── intermediates/   # MinerU output files and JSON structures — output of `transform_text()`
+    ├── results/         # Final standardised Markdown files — output of `standardise_text()`
+    ├── reports/         # All HTML output reports
+    ├── logs/            # Run logs
+    └── sqlite.db        # SQLite cache for this corpus
+```
+Everything except `scopus.csv` is created automatically. Create a new corpus with `build_new_corpus(name="my_corpus", scopus_file="path/to/scopus.csv")`.
+
+---
+
+## Dependencies
+- Python 3.12–3.13:
+- Optional: A CUDA-capable GPU (big speedup for PDF conversion via MinerU) + `torch` (CUDA 12.8)
+- Optional: [Ollama](https://ollama.com/download) running `gemma3:12b` (~8 GB), used as an LLM fallback to classify 
+ambiguous section headings. Without it, the pipeline runs, but accuracy in removing paper boilerplate is decreased.
+
+Core dependencies: `pandas`, `pydantic`, `requests`, `playwright`, `plotly`, `openai`, `wiley-tdm`, `mineru[pipeline]`, `torch`
+
+## Installation
+Install with [uv](https://github.com/astral-sh/uv):
+
+```bash
+uv pip install biomarkit
+```
+
+Or with pip:
+```python
+pip install biomarkit
+```

biomarkit-1.0.0/biomarkit/__init__.py

@@ -0,0 +1,3 @@
+from biomarkit.main import build_new_corpus, download_corpus, transform_text, standardise_text
+
+__all__ = ["build_new_corpus", "download_corpus", "transform_text", "standardise_text"]

biomarkit-1.0.0/biomarkit/config.py

@@ -0,0 +1,85 @@
+from pathlib import Path
+from dotenv import load_dotenv
+from os import getenv
+
+# Workspace root: users set BIOMARKIT_DIR to their preferred storage location;
+# falls back to the current working directory if unset.
+BASE_DIR = Path(getenv("BIOMARKIT_DIR", ".")).resolve()
+
+# Filepath for secrets.env — lives alongside the corpora in the workspace root.
+SECRETS_FILE = BASE_DIR / "secrets.env"
+
+# Load secrets before reading any values.
+load_dotenv(dotenv_path=SECRETS_FILE)
+
+# Corpus selection — set CORPUS_NAME in secrets.env to choose the active corpus.
+CORPUS_NAME = getenv("CORPUS_NAME", "default")
+
+# Corpus folder layout.
+CORPORA_DIR = BASE_DIR / "corpora"
+CORPUS_DIR = CORPORA_DIR / CORPUS_NAME
+SCOPUS_INPUT_CSV_NAME = CORPUS_DIR / "scopus.csv"
+DB_CACHE_FILE_NAME = CORPUS_DIR / "sqlite.db"
+DOWNLOAD_DIR = CORPUS_DIR / "manuscripts"
+JSON_STRUCT_DIR = CORPUS_DIR / "intermediates"
+FINAL_MARKDOWN_DIR = CORPUS_DIR / "results"
+REPORT_DIR = CORPUS_DIR / "reports"
+LOG_DIR = CORPUS_DIR / "logs"
+
+# Shared scratch space (not corpus-specific).
+TMP_DIR = BASE_DIR / "tmp"
+
+# Client Parameters.
+USER_EMAIL = getenv("USER_EMAIL")
+WILEY_TDM_TOKEN = getenv("WILEY_TDM_TOKEN")
+SPRINGER_API_KEY = getenv("SPRINGER_API_KEY")
+ELSEVIER_API_KEY = getenv("ELSEVIER_API_KEY")
+
+# LLM Settings - fallback section classifier, served by a local Ollama instance by default.
+# Any OpenAI-compatible endpoint can be substituted via secrets.env.
+LLM_BASE_URL = getenv("LLM_BASE_URL", "http://localhost:11434/v1")
+LLM_MODEL_NAME = getenv("LLM_MODEL_NAME", "gemma3:12b")
+
+# MinerU settings (optional). Integer GB of GPU VRAM MinerU is allowed to budget for;
+# overrides its auto-detection and raises inference batch sizes on larger GPUs.
+# Set via secrets.env only if you know what you are doing; unset means auto-detect.
+MINERU_VIRTUAL_VRAM_SIZE = getenv("MINERU_VIRTUAL_VRAM_SIZE")
+
+# MinerU remote inference (optional). Both must be set in secrets.env when
+# transform_text(mineru_endpoint="vllm") is used: the URL of a vLLM server hosting
+# the MinerU VLM model, and the API key the server was started with.
+MINERU_VLLM_ENDPOINT = getenv("MINERU_VLLM_ENDPOINT")
+MINERU_API_KEY = getenv("MINERU_API_KEY")
+
+# Program settings (don't touch these unless you know what you are doing).
+MAX_THREADS = 10
+PLOTLY_THEME = "plotly_white"
+
+# MAPS.
+API_URL_TO_NAME = {
+    # "opensource" is the open-source download client (preprint routes + Unpaywall fallback); its api_url is
+    # the Unpaywall endpoint. The "unpaywall" key is kept for clients that do their own Unpaywall lookups.
+    "opensource": "https://api.unpaywall.org/v2/",
+    "unpaywall": "https://api.unpaywall.org/v2/",
+    "wiley": "https://api.wiley.com/onlinelibrary/tdm/v1/",
+    "springer": "https://link.springer.com",
+    "elsevier": "https://api.elsevier.com/",
+    "mdpi": "https://www.mdpi.com",
+    "frontiers": "https://www.frontiersin.org/journals",
+    "american_phytopathological_society": "https://apsjournals.apsnet.org",
+    "taylor_and_francis": "https://www.tandfonline.com",
+    "copernicus": "https://www.copernicus.org"
+}
+
+API_KEY_TO_NAME = {
+    "opensource": "",
+    "unpaywall": "",
+    "wiley": WILEY_TDM_TOKEN,
+    "springer": SPRINGER_API_KEY,
+    "elsevier": ELSEVIER_API_KEY,
+    "mdpi": "",
+    "frontiers": "",
+    "american_phytopathological_society": "",
+    "taylor_and_francis": "",
+    "copernicus": ""
+}