groundmark 0.1.0__tar.gz

groundmark-0.1.0/.github/workflows/build.yaml

@@ -0,0 +1,33 @@
+name: CI Test Build
+
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+
+jobs:
+  build:
+    name: Build pure Python package
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Update uv lock
+        run: uv lock
+
+      - name: Build package
+        run: uv build
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/*

groundmark-0.1.0/.github/workflows/lint.yaml

@@ -0,0 +1,21 @@
+name: Lint
+on: [push]
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+
+    - uses: actions/setup-python@v5
+      with:
+        python-version: '3.11'
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v5
+
+    - name: Install dependencies
+      run: uv sync --group dev
+
+    - name: Run pre-commit
+      run: uv run pre-commit run --all-files

groundmark-0.1.0/.github/workflows/release.yaml

@@ -0,0 +1,68 @@
+name: Release
+
+on:
+  release:
+    types: [created]
+
+jobs:
+  build:
+    name: Build distribution
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Update uv lock
+        run: uv lock
+
+      - name: Build package
+        run: uv build
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/*
+
+  upload_release_assets:
+    name: Upload Assets to Release
+    runs-on: ubuntu-latest
+    needs: [build]
+    permissions:
+      contents: write
+    steps:
+      - name: Download all artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: artifacts
+          merge-multiple: true
+      - name: Upload Wheels and sdist
+        uses: softprops/action-gh-release@v2
+        with:
+          tag_name: ${{ github.event.release.tag_name }}
+          files: artifacts/*
+          overwrite_files: true
+
+  publish-to-pypi:
+    name: Publish to PyPI
+    runs-on: ubuntu-latest
+    needs: [build]
+    environment:
+      name: pypi
+      url: https://pypi.org/p/groundmark
+    permissions:
+      id-token: write # Required for trusted publishing
+    steps:
+      - name: Download all wheels and sdist
+        uses: actions/download-artifact@v4
+        with:
+          path: dist
+          merge-multiple: true
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

groundmark-0.1.0/.gitignore

@@ -0,0 +1,23 @@
+# Python-generated files
+__pycache__/
+*.py[oc]
+build/
+dist/
+wheels/
+*.egg-info
+
+# Virtual environments
+.venv
+
+# Tool caches
+.mypy_cache/
+.pytest_cache/
+.ruff_cache/
+
+# Specs (local working docs)
+specs/
+
+# Output and Data
+output.md
+*.pdf
+!tests/data/*.pdf

groundmark-0.1.0/.markdownlint.json

@@ -0,0 +1,5 @@
+{
+  "MD033": false,
+  "MD013": false,
+  "MD041": false
+}

groundmark-0.1.0/.pre-commit-config.yaml

@@ -0,0 +1,48 @@
+default_language_version:
+    python: python3.11
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v6.0.0
+    hooks:
+      - id: check-yaml
+        exclude: '\.*conda/.*'
+      - id: end-of-file-fixer
+        exclude: 'tests/fixtures'
+      - id: trailing-whitespace
+        exclude: '\.txt$|\.tsv$'
+      - id: check-case-conflict
+      - id: check-merge-conflict
+      - id: detect-private-key
+      - id: debug-statements
+      - id: check-added-large-files
+
+  - repo: https://github.com/igorshubovych/markdownlint-cli
+    rev: v0.45.0
+    hooks:
+      - id: markdownlint
+        exclude: 'tests/fixtures'
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    # Ruff version.
+    rev: v0.14.1
+    hooks:
+      - id: ruff
+        args: ["--fix"]
+      - id: ruff-format
+
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.18.2
+    hooks:
+      - id: mypy
+        exclude: "docs/"
+        args:
+          [
+            --pretty,
+            --show-error-codes,
+            --no-strict-optional,
+            --ignore-missing-imports,
+            --install-types,
+            --non-interactive,
+            --config-file=./pyproject.toml
+          ]
+        additional_dependencies: []

groundmark-0.1.0/.python-version

@@ -0,0 +1 @@
+3.11

groundmark-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Centre for Population Genomics
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

groundmark-0.1.0/PKG-INFO

@@ -0,0 +1,109 @@
+Metadata-Version: 2.4
+Name: groundmark
+Version: 0.1.0
+Summary: PDF to grounded Markdown with bounding box annotations
+Project-URL: Homepage, https://github.com/populationgenomics/groundmark
+Project-URL: Bug Tracker, https://github.com/populationgenomics/groundmark/issues
+Author-email: Tobias Sargeant <tobias.sargeant@gmail.com>
+License: MIT
+License-File: LICENSE
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Python: >=3.11
+Requires-Dist: anchorite>=0.1.1
+Requires-Dist: pdfplumber>=0.11.9
+Requires-Dist: pydantic-ai-slim[anthropic,bedrock,google,openai]>=1.67.0
+Requires-Dist: pypdf>=6.8.0
+Description-Content-Type: text/markdown
+
+# groundmark
+
+<img src="groundmark.webp" alt="groundmark" width="200">
+
+## Grounded Markdown for PDFs
+
+**groundmark is a thin, batteries-included wrapper around [anchorite](https://github.com/populationgenomics/anchorite).** It provides concrete implementations of anchorite's provider protocols — [Pydantic AI](https://ai.pydantic.dev/) for LLM-based Markdown generation and [pdfplumber](https://github.com/jsvine/pdfplumber) for bounding box extraction — so you can go from PDF bytes to annotated Markdown in a single call. All the heavy lifting (Smith-Waterman alignment, annotation, stripping, quote resolution) lives in anchorite.
+
+Give it a PDF and a model string, get back Markdown with embedded bounding box coordinates that trace every text span back to its location in the source PDF.
+
+## Architecture
+
+The library processes documents in two streams that are then merged:
+
+1. **Semantic Stream**: The PDF is sent to an LLM (via Pydantic AI) to produce clean Markdown with `<!--page-->` markers between pages.
+2. **Positional Stream**: The PDF is parsed locally by pdfplumber to extract line-level text segments and their bounding boxes.
+3. **Alignment**: Smith-Waterman alignment (via anchorite) maps each parsed line to its position in the Markdown, constrained by page boundaries.
+4. **Annotation**: Bounding box coordinates are injected as HTML span attributes:
+
+   ```html
+   <span data-bbox="120,45,180,890" data-page="3">The patient presented with</span>
+   ```
+
+## Quick Start
+
+```python
+import asyncio
+import groundmark as gm
+
+async def main():
+    pdf_bytes = open("document.pdf", "rb").read()
+
+    config = gm.Config(model="bedrock:au.anthropic.claude-sonnet-4-6")
+
+    # PDF -> annotated Markdown (one call)
+    result = await gm.process(pdf_bytes, config)
+    print(f"Coverage: {result.coverage_percent:.2%}")
+    print(result.annotated_markdown[:500])
+
+    # Strip for LLM consumption
+    stripped = gm.strip(result.annotated_markdown)
+    # stripped.plain_text: clean Markdown with spans removed
+    # stripped.validation_map: list of (start, end, Anchor) ranges
+
+    # Resolve verbatim quotes to PDF coordinates
+    resolved = gm.resolve(result.annotated_markdown, ["the patient presented with"])
+    # -> {"the patient presented with": [(page, BBox), ...]}
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+## Debug Visualizer
+
+The included visualizer overlays extracted bounding boxes onto the source PDF, useful for diagnosing alignment issues. Blue highlights show raw extracted boxes from pdfplumber; red highlights show aligned boxes from the annotated Markdown.
+
+```bash
+python -m groundmark.visualize input.pdf output.pdf --model "bedrock:au.anthropic.claude-sonnet-4-6"
+
+# Or with cached Markdown:
+python -m groundmark.visualize input.pdf output.pdf --markdown cached.md
+```
+
+![Visualizer output showing blue (raw) and red (aligned) bounding box overlays](visualize_example.jpg)
+
+*Screenshot from Santoro et al., "Health outcomes and drug utilisation in children with Noonan syndrome: a European cohort study," Orphanet J Rare Dis 20:76 (2025). [doi:10.1186/s13023-025-03594-7](https://doi.org/10.1186/s13023-025-03594-7). CC-BY 4.0.*
+
+## Configuration
+
+### Timeouts
+
+The LLM call for PDF-to-Markdown conversion can take several minutes for large documents, especially with Opus on Bedrock. Timeout defaults by provider:
+
+| Provider | Default | Environment Variable |
+|----------|---------|---------------------|
+| Bedrock (boto3) | 300s | `AWS_READ_TIMEOUT` |
+| Anthropic (httpx) | 600s | — (use `ModelSettings(timeout=...)`) |
+
+For Bedrock with Opus, 300s may not be enough. Set a higher timeout:
+
+```bash
+export AWS_READ_TIMEOUT=600
+```
+
+## License
+
+This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.

groundmark-0.1.0/README.md

@@ -0,0 +1,87 @@
+# groundmark
+
+<img src="groundmark.webp" alt="groundmark" width="200">
+
+## Grounded Markdown for PDFs
+
+**groundmark is a thin, batteries-included wrapper around [anchorite](https://github.com/populationgenomics/anchorite).** It provides concrete implementations of anchorite's provider protocols — [Pydantic AI](https://ai.pydantic.dev/) for LLM-based Markdown generation and [pdfplumber](https://github.com/jsvine/pdfplumber) for bounding box extraction — so you can go from PDF bytes to annotated Markdown in a single call. All the heavy lifting (Smith-Waterman alignment, annotation, stripping, quote resolution) lives in anchorite.
+
+Give it a PDF and a model string, get back Markdown with embedded bounding box coordinates that trace every text span back to its location in the source PDF.
+
+## Architecture
+
+The library processes documents in two streams that are then merged:
+
+1. **Semantic Stream**: The PDF is sent to an LLM (via Pydantic AI) to produce clean Markdown with `<!--page-->` markers between pages.
+2. **Positional Stream**: The PDF is parsed locally by pdfplumber to extract line-level text segments and their bounding boxes.
+3. **Alignment**: Smith-Waterman alignment (via anchorite) maps each parsed line to its position in the Markdown, constrained by page boundaries.
+4. **Annotation**: Bounding box coordinates are injected as HTML span attributes:
+
+   ```html
+   <span data-bbox="120,45,180,890" data-page="3">The patient presented with</span>
+   ```
+
+## Quick Start
+
+```python
+import asyncio
+import groundmark as gm
+
+async def main():
+    pdf_bytes = open("document.pdf", "rb").read()
+
+    config = gm.Config(model="bedrock:au.anthropic.claude-sonnet-4-6")
+
+    # PDF -> annotated Markdown (one call)
+    result = await gm.process(pdf_bytes, config)
+    print(f"Coverage: {result.coverage_percent:.2%}")
+    print(result.annotated_markdown[:500])
+
+    # Strip for LLM consumption
+    stripped = gm.strip(result.annotated_markdown)
+    # stripped.plain_text: clean Markdown with spans removed
+    # stripped.validation_map: list of (start, end, Anchor) ranges
+
+    # Resolve verbatim quotes to PDF coordinates
+    resolved = gm.resolve(result.annotated_markdown, ["the patient presented with"])
+    # -> {"the patient presented with": [(page, BBox), ...]}
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+## Debug Visualizer
+
+The included visualizer overlays extracted bounding boxes onto the source PDF, useful for diagnosing alignment issues. Blue highlights show raw extracted boxes from pdfplumber; red highlights show aligned boxes from the annotated Markdown.
+
+```bash
+python -m groundmark.visualize input.pdf output.pdf --model "bedrock:au.anthropic.claude-sonnet-4-6"
+
+# Or with cached Markdown:
+python -m groundmark.visualize input.pdf output.pdf --markdown cached.md
+```
+
+![Visualizer output showing blue (raw) and red (aligned) bounding box overlays](visualize_example.jpg)
+
+*Screenshot from Santoro et al., "Health outcomes and drug utilisation in children with Noonan syndrome: a European cohort study," Orphanet J Rare Dis 20:76 (2025). [doi:10.1186/s13023-025-03594-7](https://doi.org/10.1186/s13023-025-03594-7). CC-BY 4.0.*
+
+## Configuration
+
+### Timeouts
+
+The LLM call for PDF-to-Markdown conversion can take several minutes for large documents, especially with Opus on Bedrock. Timeout defaults by provider:
+
+| Provider | Default | Environment Variable |
+|----------|---------|---------------------|
+| Bedrock (boto3) | 300s | `AWS_READ_TIMEOUT` |
+| Anthropic (httpx) | 600s | — (use `ModelSettings(timeout=...)`) |
+
+For Bedrock with Opus, 300s may not be enough. Set a higher timeout:
+
+```bash
+export AWS_READ_TIMEOUT=600
+```
+
+## License
+
+This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.

groundmark-0.1.0/pyproject.toml

@@ -0,0 +1,75 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "groundmark"
+version = "0.1.0"
+authors = [
+    { name = "Tobias Sargeant", email = "tobias.sargeant@gmail.com" },
+]
+description = "PDF to grounded Markdown with bounding box annotations"
+readme = "README.md"
+license = { text = "MIT" }
+requires-python = ">=3.11"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+]
+dependencies = [
+    "anchorite>=0.1.1",
+    "pydantic-ai-slim[anthropic,bedrock,google,openai]>=1.67.0",
+    "pdfplumber>=0.11.9",
+    "pypdf>=6.8.0",
+]
+
+[dependency-groups]
+dev = [
+    "pre-commit>=4.5.1",
+    "pytest",
+    "pytest-asyncio",
+    "ruff>=0.14.6",
+    "typer>=0.24.1",
+]
+
+[project.urls]
+"Homepage" = "https://github.com/populationgenomics/groundmark"
+"Bug Tracker" = "https://github.com/populationgenomics/groundmark/issues"
+
+[tool.ruff]
+line-length = 120
+target-version = "py311"
+indent-width = 4
+
+[tool.ruff.lint]
+select = ["A", "B", "C",  "E", "F", "G", "I", "N", "Q", "S", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "ERA", "EXE", "ICN", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "UP", "YTT"]
+ignore = [
+    "C901",    # function complexity — too aggressive for straightforward extraction loops
+    "COM812",  # trailing comma — conflicts with ruff formatter
+    "PD011",   # pandas-use-of-dot-values (false positive)
+    "PLR0912", # too many branches — same as C901
+    "PLR0913", # too many arguments — sometimes unavoidable
+]
+fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "ERA", "EXE", "FBT", "ICN", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "UP", "YTT"]
+
+[tool.ruff.lint.isort]
+known-first-party = ["groundmark"]
+
+[tool.ruff.lint.per-file-ignores]
+"src/groundmark/markdown.py" = [
+    "RUF001",  # ambiguous unicode — intentional in LLM prompt about preserving unicode symbols
+]
+"tests/*" = [
+    "ARG001",   # unused function arguments (mock.patch positional injection).
+    "S101",     # asserts.
+    "S102",     # exec().
+    "PLR2004",  # magic value comparisons.
+    "SLF001"    # private method access.
+]
+
+[tool.mypy]
+python_version = "3.11"

groundmark-0.1.0/src/groundmark/__init__.py

@@ -0,0 +1,6 @@
+from anchorite import Anchor, BBox, annotate, resolve, strip
+
+from groundmark.markdown import PROMPT
+from groundmark.process import Config, ProcessResult, process
+
+__all__ = ["PROMPT", "Anchor", "BBox", "Config", "ProcessResult", "annotate", "process", "resolve", "strip"]

groundmark-0.1.0/src/groundmark/markdown.py

@@ -0,0 +1,104 @@
+"""PDF to Markdown conversion via Pydantic AI (any supported LLM)."""
+
+import re
+import unicodedata
+from typing import Final
+
+from anchorite.document import DocumentChunk
+from pydantic_ai import Agent
+from pydantic_ai.messages import BinaryContent
+
+# Apparently, faithfully analyzing a PDF's complicated layout and transcribing
+# it into well-structured Markdown isn't creative enough for Claude's content
+# filter. Asking the model to add line numbers gives it something "original"
+# to contribute, which satisfies the anti-regurgitation heuristic. We strip
+# them right after.
+# https://privacy.claude.com/en/articles/10023638-why-am-i-receiving-an-output-blocked-by-content-filtering-policy-error
+_LINE_NUM_PREFIX = """
+IMPORTANT: Prefix every output line with its line number followed by a
+pipe character (no trailing space), e.g.:
+  1|# Heading
+  2|
+  3|Some paragraph text here.
+Start numbering at 1. This is required for all output.
+"""
+
+_LINE_NUM_RE = re.compile(r"^\d+\|", re.MULTILINE)
+
+PROMPT: Final[str] = (
+    """
+Carefully transcribe the text for this pdf into a text file with
+markdown annotations.
+"""
+    + _LINE_NUM_PREFIX
+    + """
+**The final output must be formatted as text that visually
+mimics in markdown the layout and hierarchy of the original PDF
+when rendered (ignoring the line-number prefixes).**
+
+* Do not include headers or footers that are repeated on each page.
+* Do not include page numbers.
+* Preserve the reading order of the text as it appears in the PDF.
+* Remove hyphens that break words at the end of lines.
+  * e.g. "uti- lized" -> "utilized"
+* Use Markdown headings (`#`, `##`, `###`) to reflect the size and
+  hierarchy of titles and subtitles in the PDF.
+* Ensure that there are blank lines before and after headings, lists,
+  tables, and images.
+* End each paragraph with a blank line.
+* Do not break lines within paragraphs or headings.
+* Render bullet points and numbered lettered lists as markdown lists.
+  * It is ok to remove brackets and other consistent punctuation around
+    list identifiers
+    * e.g. "a)" -> "a."
+* Use blockquotes for any sidebars or highlighted text.
+* Bold all words and phrases that appear bolded in the original
+  source material. Similarly, italicise all text in italics.
+* Render tables as markdown, paying particular attention to copying
+  identifiers exactly.
+* Break text into paragraphs and lists exactly as they appear in
+  the PDF.
+* Preserve figure/chart captions verbatim — do not paraphrase, extend,
+  or interleave them with descriptions. If useful context is only visible
+  in the image (e.g. axis labels, legend entries, data values), add it
+  as a separate paragraph after the caption.
+* Convert bar charts into markdown tables where possible.
+* Convert tables contained in images into markdown.
+* Keep mathematical expressions as close to the PDF's own characters as
+  possible — use the same Unicode symbols (×, ≥, α, β, etc.) rather than
+  converting to LaTeX commands.
+  * Only use LaTeX (`$...$` / `$$...$$`) for complex display equations
+    with fractions, integrals, summations, or multi-level notation that
+    cannot be represented legibly in plain text.
+* Insert markers at the start of each page of the form `<!--page-->`
+* Surround tables and figure descriptions with markers:
+  * `<!--table-->` ... `<!--end-->`
+  * `<!--figure-->` ... `<!--end-->`
+"""
+)
+
+_agent: Agent[None, str] = Agent(output_type=str)
+
+
+class PydanticAIMarkdownProvider:
+    """MarkdownProvider that converts PDF chunks to Markdown via a vision-capable LLM."""
+
+    def __init__(self, model: str, *, prompt: str | None = None) -> None:
+        self.model = model
+        self._prompt = prompt or PROMPT
+
+    async def generate_markdown(self, chunk: DocumentChunk) -> str:
+        """Convert a document chunk to Markdown.
+
+        Returns:
+            Markdown string with ``<!--page-->`` markers between pages.
+        """
+        result = await _agent.run(
+            [BinaryContent(data=chunk.data, media_type=chunk.mime_type), self._prompt],
+            model=self.model,
+        )
+        # Strip the line-number prefixes added to bypass Claude's content filter.
+        markdown = _LINE_NUM_RE.sub("", result.output)
+        # NFKC-normalize so superscript digits, ligatures, etc. match the
+        # NFKC-normalized anchor text from pdfplumber extraction.
+        return unicodedata.normalize("NFKC", markdown)