sciwrite-lint 0.2.2__tar.gz → 0.4.0__tar.gz

{sciwrite_lint-0.2.2/sciwrite_lint.egg-info → sciwrite_lint-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sciwrite-lint
-Version: 0.2.2
+Version: 0.4.0
 Summary: A linter for scientific manuscripts — reference verification, consistency checking, and structural validation.
 Author: Sergey Samsonau
 Maintainer-email: Authentic Research Partners LLC <info@arpconnect.com>
@@ -41,13 +41,27 @@ Requires-Dist: numpy>=2.0
 Requires-Dist: sqlite-vec>=0.1.6
 Requires-Dist: torch>=2.5
 Requires-Dist: transformers>=4.50
-Requires-Dist: Pillow>=12.0
-Requires-Dist: pypdf>=6.0
+Requires-Dist: Pillow>=12.2
+Requires-Dist: pypdf>=6.10.2
 Requires-Dist: pypdfium2>=5.0
 Requires-Dist: pdf2image>=1.17
+Requires-Dist: lxml>=6.1
+Requires-Dist: pypandoc-binary>=1.15
+Requires-Dist: rfc3986>=2.0
+Requires-Dist: psutil>=5.9
+Requires-Dist: annotated-types>=0.6
 Provides-Extra: dev
-Requires-Dist: pytest>=8.0; extra == "dev"
+Requires-Dist: pytest>=9.0.3; extra == "dev"
 Requires-Dist: pytest-xdist>=3.0; extra == "dev"
+Requires-Dist: ruff>=0.15; extra == "dev"
+Requires-Dist: mypy>=1.18; extra == "dev"
+Requires-Dist: ast-grep-cli>=0.35; extra == "dev"
+Requires-Dist: deptry>=0.25; extra == "dev"
+Requires-Dist: bandit>=1.9; extra == "dev"
+Requires-Dist: pip-audit>=2.10; extra == "dev"
+Requires-Dist: pip-licenses>=5.5; extra == "dev"
+Requires-Dist: twine>=6.0; extra == "dev"
+Requires-Dist: build>=1.0; extra == "dev"
 Dynamic: license-file
 
 # sciwrite-lint
@@ -66,7 +80,7 @@ AI writing tools produce text that *looks* like good science — fluent prose, c
 
 ## Features
 
-**22 automated checks:**
+**23 automated checks:**
 
 ### Reference verification
 - **Do your references exist?** — checked against CrossRef, OpenAlex, Semantic Scholar, Open Library, and Library of Congress
@@ -75,7 +89,7 @@ AI writing tools produce text that *looks* like good science — fluent prose, c
 - **Robust matching** — when references lack DOIs, a multi-signal matching engine scores candidates across title, author, year, and venue (handles the metadata errors that LLMs routinely introduce)
 
 ### Claim verification (local LLM)
-- **Do cited papers support your claims?** — downloads full text from 8 open-access sources (arXiv, Semantic Scholar, OpenAlex, PubMed Central, Europe PMC, Unpaywall, bioRxiv/medRxiv, CORE), parses via GROBID, embeds sections, and verifies each claim against the actual source text
+- **Do cited papers support your claims?** — downloads full text from 14 open-access sources (arXiv, Semantic Scholar, OpenAlex, PubMed Central, Europe PMC, Unpaywall, bioRxiv/medRxiv, NBER, RePEc/IDEAS, HAL, ERIC, NASA ADS, OSF Preprints, CORE), parses via GROBID, embeds sections, and verifies each claim against the actual source text
 - **What role does each citation play?** — classifies citation purpose (evidence, contrast, method, attribution, context…) with graduated weights: an unsupported *evidence* citation is serious; an unsupported *context* citation barely matters
 - **Are your references' own bibliographies real?** — batch-checks cited papers' reference lists for existence, metadata accuracy, and retraction. Papers built on fabricated evidence are flagged
 
@@ -88,6 +102,7 @@ AI writing tools produce text that *looks* like good science — fluent prose, c
 - **Abstract–body alignment** — abstract makes factual claims the body contradicts
 - **Statistical reporting** — p-values vs. their verbal interpretation
 - **Structure promises** — contributions promised in the introduction but never delivered
+- **Prose quality** — syntactic grammar errors and semantic word-choice mistakes (e.g. "object" where "purpose" is intended). Per-sentence review with paragraph context; severity driven by the model's confidence
 
 ### Figure checks (vision model + LLM)
 - **Caption vs. content** — does the caption match what the figure actually shows?
@@ -128,26 +143,26 @@ sciwrite-lint contributions paper.pdf --format json
 ### Two audiences
 
 - **Humans** — colored terminal output with severity levels, locations, and explanations. Decide in seconds whether each finding is real
-- **AI writing agents** — `--format json` output with structured fields (`level`, `rule_id`, `message`, `context`). Run sciwrite-lint in a write → check → fix → recheck loop. Configurable exit codes for CI integration
+- **AI writing agents** — `--format json` output with structured fields (`level`, `rule_id`, `message`, `context`). Run sciwrite-lint in a write → check → fix → recheck loop. Standard `ruff`/`mypy`-style exit codes: `0` clean, `1` findings at error level, `2` tool error. Python-API consumers can `except sciwrite_lint.LLMConnectionError` (typed exception hierarchy under `SciWriteLintError`) instead of matching on error strings
 
 ### Optimizations
 
-Three models — a vision model (Qwen3-VL-2B default, or 8B FP8 via `--vision-backend vllm` for +15% accuracy), an embedding model (Arctic Embed), and an 8B reasoning LLM (Qwen3 via vLLM) — share a single consumer GPU. Models run in separate pipeline stages; on WSL2, CUDA memory virtualization pages idle allocations to system RAM, letting all three share physical VRAM. FP8 weights and KV cache (Ada Lovelace+) and per-paper SQLite caching with hash-based invalidation are baseline. On top of that:
+Three models — a vision model (Qwen3-VL-2B default, or 8B FP8 via `--vision-backend vllm` for +15% accuracy), an embedding model (Arctic Embed), and an 8B reasoning LLM (Qwen3 via vLLM) — share a single consumer GPU. The pipeline runs each in its own stage and explicitly stops one before starting the next, so only one ever owns GPU memory at a time (same code path on WSL2 and native Linux). FP8 weights and KV cache (Ada Lovelace+) and per-paper SQLite caching with hash-based invalidation are baseline. On top of that:
 
-- **Semantic section filtering** — embedding-based KNN retrieval sends only the ~5 most relevant sections per claim to the LLM, reducing LLM calls ~4x
+- **Cost-aware verify-claim ladder** — sentence chunk → paragraph chunk → whole section. Each level fans out top-N candidates in parallel; stops on a conclusive verdict. Most claims resolve at the cheap sentence level (~200-token prompts) instead of paying for whole sections (~5K tokens), with full-section escalation reserved for hard cases
 - **Prefix-first prompt structure** — shared context placed before variable content in all prompts, maximizing vLLM's automatic prefix caching
-- **Per-call-site thinking budgets** — each LLM call site has empirically tuned (max_tokens, thinking_preset) pairs, measured via grid search to maximize detection quality
+- **Per-call-site thinking budgets** — each LLM call site has its own (max_tokens, thinking_preset) pair tuned for that check's prompt and output shape
 - **Adaptive embedding batches** — token-aware batch sizing gives ~50x speedup over CPU while staying within VRAM limits
 - **Batch-staged multi-paper pipeline** — when checking 2+ papers, GPU models load once per batch (vision/embedding/cited-vision) and vLLM/network stages run concurrently, giving a meaningful speedup over sequential per-paper runs. Tune via `--concurrency` (default 2, validated up to 4 on a single consumer GPU)
 - **Phased API resolution** — citations flow through OpenAlex → Semantic Scholar → CrossRef → Open Library/LoC, each phase only processing what previous phases didn't resolve
-- **8-source full-text cascade** — early exit on first successful download across arXiv, Semantic Scholar, OpenAlex, PubMed Central, Europe PMC, Unpaywall, bioRxiv, CORE
+- **14-source full-text cascade** — early exit on first successful download across arXiv, Semantic Scholar, OpenAlex, PubMed Central, Europe PMC, Unpaywall, bioRxiv, NBER, RePEc/IDEAS, HAL, ERIC, NASA ADS, OSF Preprints, CORE
 - **Live monitoring** *(advanced)* — `sciwrite-lint containers monitor` shows service health, VRAM usage, and KV cache utilization in a terminal dashboard
 
-Full pipeline on a 50-reference paper: ~30 minutes initial (dominated by downloads and claim verification), minutes on cached reruns. On native Linux, the pipeline automatically swaps vLLM containers to free GPU for embedding and vision stages (~50x faster than CPU). *(Native Linux GPU swap is preliminary — tested on WSL2 only; expected to work, may need minor fixes.)*
+Full pipeline on a 50-reference paper: ~30 minutes initial (dominated by downloads and claim verification), minutes on cached reruns. The pipeline automatically swaps vLLM containers to free GPU for embedding and vision stages (~50× faster than CPU) — same code path on WSL2 and native Linux.
 
 ## Install
 
-**Assumed setup:** A workstation with an NVIDIA GPU (16+ GB VRAM). Developed and tested on Windows (WSL2). Native Linux is likely to work with GPU memory allocation tuning (see [docs/services.md](https://github.com/authentic-research-partners/sciwrite-lint/blob/main/docs/services.md#embedding-device)). Not tested on macOS.
+**Assumed setup:** A workstation with an NVIDIA GPU (16+ GB VRAM). Tested on WSL2 with NVIDIA driver 546.01+. Native Linux is likely to work with GPU memory allocation tuning (see [docs/services.md](https://github.com/authentic-research-partners/sciwrite-lint/blob/main/docs/services.md#embedding-device)). Not tested on macOS.
 
 Requires [uv](https://docs.astral.sh/uv/getting-started/installation/), a container runtime (podman or docker), CUDA drivers, and [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html).
 
@@ -186,30 +201,63 @@ Activate the venv in each new terminal with `source .venv/bin/activate`. For mos
 ## Setup
 
 ```bash
-sciwrite-lint init                     # scaffold .sciwrite-lint.toml + references/ + local_pdfs/
+sciwrite-lint init                     # scaffold .sciwrite-lint.toml + references/ + local_pdfs/ + local_web/
 sciwrite-lint config set-email you@example.com  # required for Unpaywall + Retraction Watch
 sciwrite-lint containers start         # start GROBID + vLLM (needs GPU for vLLM)
 sciwrite-lint containers monitor       # live dashboard: service health, VRAM, KV cache
 ```
 
+### WSL2: disable CUDA Sysmem Fallback (recommended)
+
+By default Windows silently spills GPU overflow into system RAM via PCIe (30–100× slower than VRAM); you'll see this as a sudden throughput collapse during the heaviest pipeline stages with no error in the logs. Tell the NVIDIA driver to fail loudly instead:
+
+> NVIDIA Control Panel → 3D Settings → Manage 3D Settings → *CUDA - Sysmem Fallback Policy* → **Prefer No Sysmem Fallback** → Apply.
+
+Driver 546.01+. After this, WSL2 behaves like native Linux on overflow (CUDA OOM error). The pipeline already sequences GPU consumers so OOM should not happen in normal use; the setting just prevents silent slowdowns when something does go wrong. No setting needed on native Linux — `cudaMalloc` is already loud there.
+
 ![Monitor dashboard](https://github.com/authentic-research-partners/sciwrite-lint/raw/main/docs/monitor.png)
 
 `init` detects `.tex` files and their `.bib` references and generates a `.sciwrite-lint.toml` config. Review to confirm the right files were detected.
 
-**Providing PDFs manually (`local_pdfs/`):** Drop any reference PDF into the `local_pdfs/` directory to skip the download step entirely. This is useful when you already have the file, when the paper is paywalled, or when the publisher blocks automated downloads (Cell, Springer, some PMC pages). Name files by the reference title:
+**Providing references manually — two drop folders.** sciwrite-lint reads two local directories before going to the open-access waterfall, kept separate because the folder itself signals how much to trust the source:
 
+- `local_pdfs/` — **academic sources**. Accepts `.pdf` (primary) and `.md` summaries you've written yourself. Use this for paywalled papers, for OA papers whose publisher requires a browser (Cloudflare/JS walls, Cell, Springer, some PMC pages), and for anything you'd cite as peer-reviewed evidence.
+- `local_web/` — **web captures**. Accepts `.md` (hand-written or pre-extracted) and `.mhtml` / `.mht` saved from a browser via **File → Save As → Webpage (Single File)**. Use this for JavaScript-heavy pages where a headless HTTP fetch would only see the pre-hydration shell; MHTML captures the rendered DOM post-JS. sciwrite-lint converts MHTML to markdown at ingest using the same trafilatura extractor it uses for live web fetches, so the output format is consistent either way. Anything landing here is classified `local_type=web_page` — softer evidence than a peer-reviewed PDF.
+
+Name files by the reference title, or prefix them with the citekey:
+
+```
+local_pdfs/                                     local_web/
+├── States of Curiosity … Learning.pdf          ├── dewey1910_How_We_Think.mhtml
+├── Mind in Society.pdf                         ├── earthwatch2025_About.md
+├── vanfraassen1980_The_Scientific_Image.pdf    └── bca2026_Research_Teachers.md
+└── dewey2016_summary.md   # reader-written
 ```
-local_pdfs/
-├── States of Curiosity Modulate Hippocampus-Dependent Learning.pdf
-├── Mind in Society.pdf
-└── The System of Professions.pdf
+
+Matching runs in two passes for both folders. **(1) Citekey prefix:** if the filename begins with a known bib citekey (case-insensitive, e.g. `vanfraassen1980_…`), the file is matched directly to that reference — no fuzzy scoring. This is the authoritative path and works reliably for archives that follow a `{citekey}_{Title}.<ext>` convention, including short titles where length differences would otherwise penalize the score. **(2) Fuzzy title:** filenames without a recognized citekey prefix fall through to fuzzy matching against `.bib` titles (threshold 0.80). Academic matches win over web matches when the same reference is present in both folders — PDF trumps web capture. Matched files are copied into the workspace; on the next run the reference upgrades from T2 to T1 and goes through GROBID parsing (for PDFs) or direct read (for `.md`) during claim verification.
+
+**Per-paper curated archives.** Each paper can point both drop folders at its own directories in its `[[papers]]` block:
+
+```toml
+[[papers]]
+name = "my-paper"
+file_path = "papers/my-paper/my-paper.tex"
+bib = "papers/my-paper/my-paper.bib"
+local_pdfs_dir = "papers/my-paper/Sources/full_text"      # academic archive
+local_web_dir  = "papers/my-paper/Sources/full_text_web"  # web captures
 ```
 
-The tool fuzzy-matches filenames against your `.bib` titles (threshold 0.80) and copies matches to the workspace. On the next run, matched references upgrade from T2 to T1 and go through GROBID parsing and claim verification like any other full-text reference.
+If you don't set them explicitly, sciwrite-lint auto-detects sibling `Sources/full_text/` and `Sources/full_text_web/` directories next to `file_path`. Resolution order per paper: explicit override → auto-detected `Sources/full_text{,_web}/` → project-wide default.
+
+**Refreshing a source file in place.** sciwrite-lint records the SHA-256 of each drop-folder file when it first ingests it. On every subsequent run the hash is re-checked: if you overwrite `dewey1910.pdf` with a corrected scan (or re-capture an MHTML, or edit a markdown summary) under the same filename, the hash changes and the tool automatically re-copies the file into the workspace — triggering a fresh GROBID parse and embedding for PDFs, a fresh MHTML-to-markdown conversion for web captures. You don't need `--fresh`, and you don't need to rename files. Unchanged files cost a single hash read per run and are otherwise skipped.
 
-**Finding what to download:** `sciwrite-lint verify` lists T2 references that are confirmed open access with direct URLs — open in your browser and save to `local_pdfs/`. For paywalled papers, use your institution's library access.
+**Footnote URLs (`\footnote{\url{…}}`) get verified too.** Papers often cite informational web pages inline as a footnote URL rather than through a formal `.bib` entry — organisation pages, program descriptions, press releases, FAQs. sciwrite-lint matches every such URL to a `.md` file in `local_web_dir` by reading a `Source: https://…` (or `Source URL: …`) header from the first 20 lines of each file. The matched capture becomes a T1 source for claim verification — same as a cited reference. Browser-saved `.mhtml` gets this header written automatically at ingest; for hand-authored `.md` you add one line at the top. See [docs/local-sources.md](https://github.com/authentic-research-partners/sciwrite-lint/blob/main/docs/local-sources.md) for details and examples.
 
-**Optional API keys** increase rate limits for Semantic Scholar, NCBI, and CORE:
+**Finding what to download:** `sciwrite-lint verify` lists T2 references that are confirmed open access with direct URLs — open each in your browser and save to `local_pdfs/` (or `local_web/` if the "paper" is actually a blog post or documentation page that the publisher serves as HTML). For paywalled papers, use your institution's library access.
+
+**Fetched-PDF match validation.** Every downloaded PDF goes through a multi-signal match gate (title similarity, author surname, DOI, year, plus hard rejects for common bibliographic-record landing pages) before being cached. Wrong-paper downloads from API mismatches and template landing pages are rejected and deleted; the tool moves on to the next open-access source. You can still drop the correct PDF into `local_pdfs_dir` to skip the waterfall entirely.
+
+**API keys** increase rate limits for Semantic Scholar, NCBI, and CORE, and enable NASA ADS (required for that source):
 
 ```bash
 sciwrite-lint config show              # see what's configured
@@ -227,6 +275,7 @@ sciwrite-lint check                             # all papers (batch-staged when
 sciwrite-lint check --concurrency 4             # batch parallelism (default 2, validated up to 4)
 sciwrite-lint check paper.tex                   # text + LLM rules on a .tex file
 sciwrite-lint check paper.pdf                   # check a PDF (GROBID required)
+sciwrite-lint check --checks prose-quality paper.tex   # run only the listed checks (comma-separated)
 sciwrite-lint contributions --paper my_paper    # add contribution axes to SciLint Score
 sciwrite-lint contributions paper.pdf           # standalone file scoring
 ```
@@ -235,6 +284,8 @@ sciwrite-lint contributions paper.pdf           # standalone file scoring
 
 Use `--fresh` to start from scratch (backs up the existing workspace before recreating it).
 
+Use `--checks ID[,ID...]` to run only the listed checks (comma-separated), disabling the rest for this invocation. Useful for fast iteration during editing (`--checks prose-quality` runs just the prose review), for debugging a single check, or for scripting a two-pass workflow. Unknown check IDs fail loudly — run `sciwrite-lint checks` to list available IDs.
+
 ### Contribution axes (`sciwrite-lint contributions`)
 
 `contributions` computes 5 contribution axes from philosophy of science (Popper, Lakatos, Kitcher, Laudan, Mayo) and updates the SciLint Score. Requires vLLM.
@@ -303,6 +354,7 @@ Output formats: terminal (default) or `--format json`.
 - `sciwrite-lint checks` — list all checks
 - `sciwrite-lint <command> --help` — detailed usage for any command
 - [docs/services.md](https://github.com/authentic-research-partners/sciwrite-lint/blob/main/docs/services.md) — GROBID, vLLM, external APIs, configuration
+- [docs/local-sources.md](https://github.com/authentic-research-partners/sciwrite-lint/blob/main/docs/local-sources.md) — drop directories, filename conventions, `Source:` header for footnote URLs
 
 For contributors and advanced users:
 

{sciwrite_lint-0.2.2 → sciwrite_lint-0.4.0}/README.md

@@ -14,7 +14,7 @@ AI writing tools produce text that *looks* like good science — fluent prose, c
 
 ## Features
 
-**22 automated checks:**
+**23 automated checks:**
 
 ### Reference verification
 - **Do your references exist?** — checked against CrossRef, OpenAlex, Semantic Scholar, Open Library, and Library of Congress
@@ -23,7 +23,7 @@ AI writing tools produce text that *looks* like good science — fluent prose, c
 - **Robust matching** — when references lack DOIs, a multi-signal matching engine scores candidates across title, author, year, and venue (handles the metadata errors that LLMs routinely introduce)
 
 ### Claim verification (local LLM)
-- **Do cited papers support your claims?** — downloads full text from 8 open-access sources (arXiv, Semantic Scholar, OpenAlex, PubMed Central, Europe PMC, Unpaywall, bioRxiv/medRxiv, CORE), parses via GROBID, embeds sections, and verifies each claim against the actual source text
+- **Do cited papers support your claims?** — downloads full text from 14 open-access sources (arXiv, Semantic Scholar, OpenAlex, PubMed Central, Europe PMC, Unpaywall, bioRxiv/medRxiv, NBER, RePEc/IDEAS, HAL, ERIC, NASA ADS, OSF Preprints, CORE), parses via GROBID, embeds sections, and verifies each claim against the actual source text
 - **What role does each citation play?** — classifies citation purpose (evidence, contrast, method, attribution, context…) with graduated weights: an unsupported *evidence* citation is serious; an unsupported *context* citation barely matters
 - **Are your references' own bibliographies real?** — batch-checks cited papers' reference lists for existence, metadata accuracy, and retraction. Papers built on fabricated evidence are flagged
 
@@ -36,6 +36,7 @@ AI writing tools produce text that *looks* like good science — fluent prose, c
 - **Abstract–body alignment** — abstract makes factual claims the body contradicts
 - **Statistical reporting** — p-values vs. their verbal interpretation
 - **Structure promises** — contributions promised in the introduction but never delivered
+- **Prose quality** — syntactic grammar errors and semantic word-choice mistakes (e.g. "object" where "purpose" is intended). Per-sentence review with paragraph context; severity driven by the model's confidence
 
 ### Figure checks (vision model + LLM)
 - **Caption vs. content** — does the caption match what the figure actually shows?
@@ -76,26 +77,26 @@ sciwrite-lint contributions paper.pdf --format json
 ### Two audiences
 
 - **Humans** — colored terminal output with severity levels, locations, and explanations. Decide in seconds whether each finding is real
-- **AI writing agents** — `--format json` output with structured fields (`level`, `rule_id`, `message`, `context`). Run sciwrite-lint in a write → check → fix → recheck loop. Configurable exit codes for CI integration
+- **AI writing agents** — `--format json` output with structured fields (`level`, `rule_id`, `message`, `context`). Run sciwrite-lint in a write → check → fix → recheck loop. Standard `ruff`/`mypy`-style exit codes: `0` clean, `1` findings at error level, `2` tool error. Python-API consumers can `except sciwrite_lint.LLMConnectionError` (typed exception hierarchy under `SciWriteLintError`) instead of matching on error strings
 
 ### Optimizations
 
-Three models — a vision model (Qwen3-VL-2B default, or 8B FP8 via `--vision-backend vllm` for +15% accuracy), an embedding model (Arctic Embed), and an 8B reasoning LLM (Qwen3 via vLLM) — share a single consumer GPU. Models run in separate pipeline stages; on WSL2, CUDA memory virtualization pages idle allocations to system RAM, letting all three share physical VRAM. FP8 weights and KV cache (Ada Lovelace+) and per-paper SQLite caching with hash-based invalidation are baseline. On top of that:
+Three models — a vision model (Qwen3-VL-2B default, or 8B FP8 via `--vision-backend vllm` for +15% accuracy), an embedding model (Arctic Embed), and an 8B reasoning LLM (Qwen3 via vLLM) — share a single consumer GPU. The pipeline runs each in its own stage and explicitly stops one before starting the next, so only one ever owns GPU memory at a time (same code path on WSL2 and native Linux). FP8 weights and KV cache (Ada Lovelace+) and per-paper SQLite caching with hash-based invalidation are baseline. On top of that:
 
-- **Semantic section filtering** — embedding-based KNN retrieval sends only the ~5 most relevant sections per claim to the LLM, reducing LLM calls ~4x
+- **Cost-aware verify-claim ladder** — sentence chunk → paragraph chunk → whole section. Each level fans out top-N candidates in parallel; stops on a conclusive verdict. Most claims resolve at the cheap sentence level (~200-token prompts) instead of paying for whole sections (~5K tokens), with full-section escalation reserved for hard cases
 - **Prefix-first prompt structure** — shared context placed before variable content in all prompts, maximizing vLLM's automatic prefix caching
-- **Per-call-site thinking budgets** — each LLM call site has empirically tuned (max_tokens, thinking_preset) pairs, measured via grid search to maximize detection quality
+- **Per-call-site thinking budgets** — each LLM call site has its own (max_tokens, thinking_preset) pair tuned for that check's prompt and output shape
 - **Adaptive embedding batches** — token-aware batch sizing gives ~50x speedup over CPU while staying within VRAM limits
 - **Batch-staged multi-paper pipeline** — when checking 2+ papers, GPU models load once per batch (vision/embedding/cited-vision) and vLLM/network stages run concurrently, giving a meaningful speedup over sequential per-paper runs. Tune via `--concurrency` (default 2, validated up to 4 on a single consumer GPU)
 - **Phased API resolution** — citations flow through OpenAlex → Semantic Scholar → CrossRef → Open Library/LoC, each phase only processing what previous phases didn't resolve
-- **8-source full-text cascade** — early exit on first successful download across arXiv, Semantic Scholar, OpenAlex, PubMed Central, Europe PMC, Unpaywall, bioRxiv, CORE
+- **14-source full-text cascade** — early exit on first successful download across arXiv, Semantic Scholar, OpenAlex, PubMed Central, Europe PMC, Unpaywall, bioRxiv, NBER, RePEc/IDEAS, HAL, ERIC, NASA ADS, OSF Preprints, CORE
 - **Live monitoring** *(advanced)* — `sciwrite-lint containers monitor` shows service health, VRAM usage, and KV cache utilization in a terminal dashboard
 
-Full pipeline on a 50-reference paper: ~30 minutes initial (dominated by downloads and claim verification), minutes on cached reruns. On native Linux, the pipeline automatically swaps vLLM containers to free GPU for embedding and vision stages (~50x faster than CPU). *(Native Linux GPU swap is preliminary — tested on WSL2 only; expected to work, may need minor fixes.)*
+Full pipeline on a 50-reference paper: ~30 minutes initial (dominated by downloads and claim verification), minutes on cached reruns. The pipeline automatically swaps vLLM containers to free GPU for embedding and vision stages (~50× faster than CPU) — same code path on WSL2 and native Linux.
 
 ## Install
 
-**Assumed setup:** A workstation with an NVIDIA GPU (16+ GB VRAM). Developed and tested on Windows (WSL2). Native Linux is likely to work with GPU memory allocation tuning (see [docs/services.md](https://github.com/authentic-research-partners/sciwrite-lint/blob/main/docs/services.md#embedding-device)). Not tested on macOS.
+**Assumed setup:** A workstation with an NVIDIA GPU (16+ GB VRAM). Tested on WSL2 with NVIDIA driver 546.01+. Native Linux is likely to work with GPU memory allocation tuning (see [docs/services.md](https://github.com/authentic-research-partners/sciwrite-lint/blob/main/docs/services.md#embedding-device)). Not tested on macOS.
 
 Requires [uv](https://docs.astral.sh/uv/getting-started/installation/), a container runtime (podman or docker), CUDA drivers, and [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html).
 
@@ -134,30 +135,63 @@ Activate the venv in each new terminal with `source .venv/bin/activate`. For mos
 ## Setup
 
 ```bash
-sciwrite-lint init                     # scaffold .sciwrite-lint.toml + references/ + local_pdfs/
+sciwrite-lint init                     # scaffold .sciwrite-lint.toml + references/ + local_pdfs/ + local_web/
 sciwrite-lint config set-email you@example.com  # required for Unpaywall + Retraction Watch
 sciwrite-lint containers start         # start GROBID + vLLM (needs GPU for vLLM)
 sciwrite-lint containers monitor       # live dashboard: service health, VRAM, KV cache
 ```
 
+### WSL2: disable CUDA Sysmem Fallback (recommended)
+
+By default Windows silently spills GPU overflow into system RAM via PCIe (30–100× slower than VRAM); you'll see this as a sudden throughput collapse during the heaviest pipeline stages with no error in the logs. Tell the NVIDIA driver to fail loudly instead:
+
+> NVIDIA Control Panel → 3D Settings → Manage 3D Settings → *CUDA - Sysmem Fallback Policy* → **Prefer No Sysmem Fallback** → Apply.
+
+Driver 546.01+. After this, WSL2 behaves like native Linux on overflow (CUDA OOM error). The pipeline already sequences GPU consumers so OOM should not happen in normal use; the setting just prevents silent slowdowns when something does go wrong. No setting needed on native Linux — `cudaMalloc` is already loud there.
+
 ![Monitor dashboard](https://github.com/authentic-research-partners/sciwrite-lint/raw/main/docs/monitor.png)
 
 `init` detects `.tex` files and their `.bib` references and generates a `.sciwrite-lint.toml` config. Review to confirm the right files were detected.
 
-**Providing PDFs manually (`local_pdfs/`):** Drop any reference PDF into the `local_pdfs/` directory to skip the download step entirely. This is useful when you already have the file, when the paper is paywalled, or when the publisher blocks automated downloads (Cell, Springer, some PMC pages). Name files by the reference title:
+**Providing references manually — two drop folders.** sciwrite-lint reads two local directories before going to the open-access waterfall, kept separate because the folder itself signals how much to trust the source:
 
+- `local_pdfs/` — **academic sources**. Accepts `.pdf` (primary) and `.md` summaries you've written yourself. Use this for paywalled papers, for OA papers whose publisher requires a browser (Cloudflare/JS walls, Cell, Springer, some PMC pages), and for anything you'd cite as peer-reviewed evidence.
+- `local_web/` — **web captures**. Accepts `.md` (hand-written or pre-extracted) and `.mhtml` / `.mht` saved from a browser via **File → Save As → Webpage (Single File)**. Use this for JavaScript-heavy pages where a headless HTTP fetch would only see the pre-hydration shell; MHTML captures the rendered DOM post-JS. sciwrite-lint converts MHTML to markdown at ingest using the same trafilatura extractor it uses for live web fetches, so the output format is consistent either way. Anything landing here is classified `local_type=web_page` — softer evidence than a peer-reviewed PDF.
+
+Name files by the reference title, or prefix them with the citekey:
+
+```
+local_pdfs/                                     local_web/
+├── States of Curiosity … Learning.pdf          ├── dewey1910_How_We_Think.mhtml
+├── Mind in Society.pdf                         ├── earthwatch2025_About.md
+├── vanfraassen1980_The_Scientific_Image.pdf    └── bca2026_Research_Teachers.md
+└── dewey2016_summary.md   # reader-written
 ```
-local_pdfs/
-├── States of Curiosity Modulate Hippocampus-Dependent Learning.pdf
-├── Mind in Society.pdf
-└── The System of Professions.pdf
+
+Matching runs in two passes for both folders. **(1) Citekey prefix:** if the filename begins with a known bib citekey (case-insensitive, e.g. `vanfraassen1980_…`), the file is matched directly to that reference — no fuzzy scoring. This is the authoritative path and works reliably for archives that follow a `{citekey}_{Title}.<ext>` convention, including short titles where length differences would otherwise penalize the score. **(2) Fuzzy title:** filenames without a recognized citekey prefix fall through to fuzzy matching against `.bib` titles (threshold 0.80). Academic matches win over web matches when the same reference is present in both folders — PDF trumps web capture. Matched files are copied into the workspace; on the next run the reference upgrades from T2 to T1 and goes through GROBID parsing (for PDFs) or direct read (for `.md`) during claim verification.
+
+**Per-paper curated archives.** Each paper can point both drop folders at its own directories in its `[[papers]]` block:
+
+```toml
+[[papers]]
+name = "my-paper"
+file_path = "papers/my-paper/my-paper.tex"
+bib = "papers/my-paper/my-paper.bib"
+local_pdfs_dir = "papers/my-paper/Sources/full_text"      # academic archive
+local_web_dir  = "papers/my-paper/Sources/full_text_web"  # web captures
 ```
 
-The tool fuzzy-matches filenames against your `.bib` titles (threshold 0.80) and copies matches to the workspace. On the next run, matched references upgrade from T2 to T1 and go through GROBID parsing and claim verification like any other full-text reference.
+If you don't set them explicitly, sciwrite-lint auto-detects sibling `Sources/full_text/` and `Sources/full_text_web/` directories next to `file_path`. Resolution order per paper: explicit override → auto-detected `Sources/full_text{,_web}/` → project-wide default.
+
+**Refreshing a source file in place.** sciwrite-lint records the SHA-256 of each drop-folder file when it first ingests it. On every subsequent run the hash is re-checked: if you overwrite `dewey1910.pdf` with a corrected scan (or re-capture an MHTML, or edit a markdown summary) under the same filename, the hash changes and the tool automatically re-copies the file into the workspace — triggering a fresh GROBID parse and embedding for PDFs, a fresh MHTML-to-markdown conversion for web captures. You don't need `--fresh`, and you don't need to rename files. Unchanged files cost a single hash read per run and are otherwise skipped.
 
-**Finding what to download:** `sciwrite-lint verify` lists T2 references that are confirmed open access with direct URLs — open in your browser and save to `local_pdfs/`. For paywalled papers, use your institution's library access.
+**Footnote URLs (`\footnote{\url{…}}`) get verified too.** Papers often cite informational web pages inline as a footnote URL rather than through a formal `.bib` entry — organisation pages, program descriptions, press releases, FAQs. sciwrite-lint matches every such URL to a `.md` file in `local_web_dir` by reading a `Source: https://…` (or `Source URL: …`) header from the first 20 lines of each file. The matched capture becomes a T1 source for claim verification — same as a cited reference. Browser-saved `.mhtml` gets this header written automatically at ingest; for hand-authored `.md` you add one line at the top. See [docs/local-sources.md](https://github.com/authentic-research-partners/sciwrite-lint/blob/main/docs/local-sources.md) for details and examples.
 
-**Optional API keys** increase rate limits for Semantic Scholar, NCBI, and CORE:
+**Finding what to download:** `sciwrite-lint verify` lists T2 references that are confirmed open access with direct URLs — open each in your browser and save to `local_pdfs/` (or `local_web/` if the "paper" is actually a blog post or documentation page that the publisher serves as HTML). For paywalled papers, use your institution's library access.
+
+**Fetched-PDF match validation.** Every downloaded PDF goes through a multi-signal match gate (title similarity, author surname, DOI, year, plus hard rejects for common bibliographic-record landing pages) before being cached. Wrong-paper downloads from API mismatches and template landing pages are rejected and deleted; the tool moves on to the next open-access source. You can still drop the correct PDF into `local_pdfs_dir` to skip the waterfall entirely.
+
+**API keys** increase rate limits for Semantic Scholar, NCBI, and CORE, and enable NASA ADS (required for that source):
 
 ```bash
 sciwrite-lint config show              # see what's configured
@@ -175,6 +209,7 @@ sciwrite-lint check                             # all papers (batch-staged when
 sciwrite-lint check --concurrency 4             # batch parallelism (default 2, validated up to 4)
 sciwrite-lint check paper.tex                   # text + LLM rules on a .tex file
 sciwrite-lint check paper.pdf                   # check a PDF (GROBID required)
+sciwrite-lint check --checks prose-quality paper.tex   # run only the listed checks (comma-separated)
 sciwrite-lint contributions --paper my_paper    # add contribution axes to SciLint Score
 sciwrite-lint contributions paper.pdf           # standalone file scoring
 ```
@@ -183,6 +218,8 @@ sciwrite-lint contributions paper.pdf           # standalone file scoring
 
 Use `--fresh` to start from scratch (backs up the existing workspace before recreating it).
 
+Use `--checks ID[,ID...]` to run only the listed checks (comma-separated), disabling the rest for this invocation. Useful for fast iteration during editing (`--checks prose-quality` runs just the prose review), for debugging a single check, or for scripting a two-pass workflow. Unknown check IDs fail loudly — run `sciwrite-lint checks` to list available IDs.
+
 ### Contribution axes (`sciwrite-lint contributions`)
 
 `contributions` computes 5 contribution axes from philosophy of science (Popper, Lakatos, Kitcher, Laudan, Mayo) and updates the SciLint Score. Requires vLLM.
@@ -251,6 +288,7 @@ Output formats: terminal (default) or `--format json`.
 - `sciwrite-lint checks` — list all checks
 - `sciwrite-lint <command> --help` — detailed usage for any command
 - [docs/services.md](https://github.com/authentic-research-partners/sciwrite-lint/blob/main/docs/services.md) — GROBID, vLLM, external APIs, configuration
+- [docs/local-sources.md](https://github.com/authentic-research-partners/sciwrite-lint/blob/main/docs/local-sources.md) — drop directories, filename conventions, `Source:` header for footnote URLs
 
 For contributors and advanced users:
 

{sciwrite_lint-0.2.2 → sciwrite_lint-0.4.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "sciwrite-lint"
-version = "0.2.2"
+version = "0.4.0"
 description = "A linter for scientific manuscripts — reference verification, consistency checking, and structural validation."
 requires-python = ">=3.13,<3.14"
 license = "MIT"
@@ -42,10 +42,15 @@ dependencies = [
     "sqlite-vec>=0.1.6",
     "torch>=2.5",
     "transformers>=4.50",
-    "Pillow>=12.0",
-    "pypdf>=6.0",
+    "Pillow>=12.2",
+    "pypdf>=6.10.2",
     "pypdfium2>=5.0",
     "pdf2image>=1.17",
+    "lxml>=6.1",
+    "pypandoc-binary>=1.15",
+    "rfc3986>=2.0",
+    "psutil>=5.9",
+    "annotated-types>=0.6",
 ]
 
 [project.urls]
@@ -56,9 +61,23 @@ Issues = "https://github.com/authentic-research-partners/sciwrite-lint/issues"
 Paper = "https://arxiv.org/abs/2604.08501"
 
 [project.optional-dependencies]
+# `pip install -e ".[dev]"` brings in every tool used by the mandatory checks
+# and the release pipeline. Exception: the `claude` CLI is a Node app —
+# install separately per https://docs.anthropic.com/en/docs/claude-code.
 dev = [
-    "pytest>=8.0",
+    # Mandatory checks — run after every code change
+    "pytest>=9.0.3",
     "pytest-xdist>=3.0",
+    "ruff>=0.15",
+    "mypy>=1.18",
+    "ast-grep-cli>=0.35",
+    # Release pipeline (scripts/release.sh phases 3–6)
+    "deptry>=0.25",
+    "bandit>=1.9",
+    "pip-audit>=2.10",
+    "pip-licenses>=5.5",
+    "twine>=6.0",
+    "build>=1.0",
 ]
 
 [project.scripts]
@@ -67,44 +86,3 @@ sciwrite-lint = "sciwrite_lint.__main__:main"
 [tool.setuptools.packages.find]
 include = ["sciwrite_lint*"]
 
-[tool.pytest.ini_options]
-testpaths = ["tests"]
-pythonpath = ["."]
-markers = [
-    "integration: tests that read actual paper files",
-    "network: tests that make real network connections (slow)",
-]
-# PID-unique basetemp set in conftest.py — prevents FileExistsError
-# when multiple pytest processes run concurrently.
-addopts = "-n auto -p no:cacheprovider"
-
-[tool.bandit]
-skips = [
-    "B101",  # assert — used for type narrowing and internal invariants
-    "B104",  # bind 0.0.0.0 — container start commands for GROBID/vLLM
-    "B314",  # xml.etree.fromstring — parsing trusted GROBID TEI output
-    "B404",  # subprocess import — CLI tool, subprocess is core functionality
-    "B405",  # xml.etree import — parsing trusted GROBID TEI output
-    "B603",  # subprocess call — CLI invokes grobid/vllm/claude by design
-    "B607",  # partial path — CLI tools resolved via PATH
-    "B608",  # SQL string — internal usage.db, no user input in queries
-    "B615",  # HF from_pretrained without revision pin — trusted publisher (Qwen), research tool
-]
-
-[tool.deptry]
-optional_dependencies_dev_groups = ["dev"]
-
-[tool.deptry.per_rule_ignores]
-# Self-imports of shipped non-package directories
-DEP001 = ["evals", "eval_real_world", "bench"]
-# Lazy/conditional imports in optional groups — deptry can't detect these
-DEP002 = ["bibtexparser", "pdfplumber", "sentence-transformers", "numpy", "sqlite-vec", "grobid-tei-xml", "torch", "transformers", "Pillow", "pypdf", "pdf2image"]
-# Intra-package imports
-DEP003 = ["sciwrite_lint", "evals", "eval_real_world"]
-
-[tool.mypy]
-plugins = ["pydantic.mypy"]
-[[tool.mypy.overrides]]
-# No py.typed marker or typeshed stubs available
-module = ["bibtexparser", "sqlite_vec", "pypdf", "pypdfium2", "pypdfium2.raw", "transformers", "pdf2image"]
-ignore_missing_imports = true

sciwrite_lint-0.4.0/sciwrite_lint/__init__.py

@@ -0,0 +1,28 @@
+"""sciwrite-lint: a linter for scientific manuscripts."""
+
+from sciwrite_lint.exceptions import LLMConnectionError, SciWriteLintError
+
+__version__ = "0.4.0"
+
+from sciwrite_lint.oa import (
+    DownloadResult,
+    FetchConfig,
+    SearchHit,
+    WebResult,
+    download_pdf,
+    fetch_web,
+    search_by_title,
+)
+
+__all__ = [
+    "DownloadResult",
+    "FetchConfig",
+    "LLMConnectionError",
+    "SciWriteLintError",
+    "SearchHit",
+    "WebResult",
+    "__version__",
+    "download_pdf",
+    "fetch_web",
+    "search_by_title",
+]