paper2mdviallm 0.1.0__tar.gz

paper2mdviallm-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 s1n4sh4
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

paper2mdviallm-0.1.0/MANIFEST.in

@@ -0,0 +1,2 @@
+include README.md
+include LICENSE

paper2mdviallm-0.1.0/PKG-INFO

@@ -0,0 +1,112 @@
+Metadata-Version: 2.4
+Name: paper2mdviallm
+Version: 0.1.0
+Summary: LLM-native academic PDF to Obsidian Markdown converter.
+Author: s1n4sh4
+License-Expression: MIT
+Keywords: pdf,markdown,obsidian,llm,academic
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Text Processing :: Markup :: Markdown
+Classifier: Topic :: Utilities
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: anthropic>=0.40.0
+Requires-Dist: openai>=1.0.0
+Requires-Dist: PyMuPDF>=1.24.0
+Requires-Dist: pydantic>=2.0
+Requires-Dist: python-dotenv>=1.0.0
+Requires-Dist: tenacity>=8.0.0
+Requires-Dist: rich>=13.0.0
+Requires-Dist: typer>=0.12.0
+Dynamic: license-file
+
+# paper2mdviallm
+
+`paper2mdviallm` is a Python CLI for converting digital academic PDFs into
+Obsidian-friendly Markdown.
+
+It is the external conversion backend used by the Scholia Obsidian plugin, but
+it can also be installed and run on its own.
+
+## Install
+
+When `paper2mdviallm` is published to PyPI, install it into a dedicated environment:
+
+```sh
+conda create -n scholia python=3.11
+conda activate scholia
+pip install paper2mdviallm
+```
+
+For local development from this repository:
+
+```sh
+python -m venv .venv
+.venv/bin/pip install -r requirements-dev.txt
+.venv/bin/pip install -e .
+```
+
+## Requirements
+
+- Python 3.10+
+- An OpenAI or Anthropic API key
+- Digital PDFs with a text layer
+
+The CLI reads API keys from the environment:
+
+- `OPENAI_API_KEY`
+- `ANTHROPIC_API_KEY`
+
+## Usage
+
+Inspect a PDF before conversion:
+
+```sh
+paper2mdviallm inspect paper.pdf
+```
+
+Convert a PDF to Markdown:
+
+```sh
+paper2mdviallm convert paper.pdf -o out/
+```
+
+Override the model or concurrency when needed:
+
+```sh
+paper2mdviallm convert paper.pdf -o out/ --model claude-sonnet-4-6 --concurrency 3
+```
+
+## With Scholia
+
+After installing into a conda environment, point Scholia at either:
+
+- the environment root, such as `/Users/me/miniconda3/envs/scholia`, or
+- the concrete executable, such as `/Users/me/miniconda3/envs/scholia/bin/paper2mdviallm`
+
+Scholia resolves the binary and executes it directly.
+
+## Development
+
+Run tests:
+
+```sh
+python -m pytest tests
+```
+
+Bootstrap from the repo root:
+
+```sh
+tools/paper2mdviallm/bootstrap.sh
+```
+
+## Boundaries
+
+- The current target is digital PDFs, not OCR-heavy scanned documents.
+- Formula-heavy papers still need manual inspection after conversion.
+- The output is optimized for Obsidian reading workflows rather than general
+  document fidelity.

paper2mdviallm-0.1.0/README.md

@@ -0,0 +1,86 @@
+# paper2mdviallm
+
+`paper2mdviallm` is a Python CLI for converting digital academic PDFs into
+Obsidian-friendly Markdown.
+
+It is the external conversion backend used by the Scholia Obsidian plugin, but
+it can also be installed and run on its own.
+
+## Install
+
+When `paper2mdviallm` is published to PyPI, install it into a dedicated environment:
+
+```sh
+conda create -n scholia python=3.11
+conda activate scholia
+pip install paper2mdviallm
+```
+
+For local development from this repository:
+
+```sh
+python -m venv .venv
+.venv/bin/pip install -r requirements-dev.txt
+.venv/bin/pip install -e .
+```
+
+## Requirements
+
+- Python 3.10+
+- An OpenAI or Anthropic API key
+- Digital PDFs with a text layer
+
+The CLI reads API keys from the environment:
+
+- `OPENAI_API_KEY`
+- `ANTHROPIC_API_KEY`
+
+## Usage
+
+Inspect a PDF before conversion:
+
+```sh
+paper2mdviallm inspect paper.pdf
+```
+
+Convert a PDF to Markdown:
+
+```sh
+paper2mdviallm convert paper.pdf -o out/
+```
+
+Override the model or concurrency when needed:
+
+```sh
+paper2mdviallm convert paper.pdf -o out/ --model claude-sonnet-4-6 --concurrency 3
+```
+
+## With Scholia
+
+After installing into a conda environment, point Scholia at either:
+
+- the environment root, such as `/Users/me/miniconda3/envs/scholia`, or
+- the concrete executable, such as `/Users/me/miniconda3/envs/scholia/bin/paper2mdviallm`
+
+Scholia resolves the binary and executes it directly.
+
+## Development
+
+Run tests:
+
+```sh
+python -m pytest tests
+```
+
+Bootstrap from the repo root:
+
+```sh
+tools/paper2mdviallm/bootstrap.sh
+```
+
+## Boundaries
+
+- The current target is digital PDFs, not OCR-heavy scanned documents.
+- Formula-heavy papers still need manual inspection after conversion.
+- The output is optimized for Obsidian reading workflows rather than general
+  document fidelity.

paper2mdviallm-0.1.0/paper2mdviallm/__init__.py

@@ -0,0 +1,3 @@
+"""paper2mdviallm: Academic PDF to Obsidian Markdown converter."""
+
+__version__ = "0.1.0"

paper2mdviallm-0.1.0/paper2mdviallm/cli.py

@@ -0,0 +1,189 @@
+from __future__ import annotations
+
+import logging
+import re
+import shutil
+import tempfile
+from pathlib import Path
+from typing import Optional
+
+import typer
+from rich.console import Console
+from rich.progress import Progress, SpinnerColumn, TextColumn, TimeElapsedColumn
+from rich.table import Table
+
+from .config import load_config
+from .converter import convert_all
+from .metadata import extract_metadata
+from .postprocess import postprocess
+from .splitter import split_pdf
+from .stitcher import stitch
+from .verifier import verify
+
+app = typer.Typer(
+    name="paper2mdviallm",
+    help="Convert academic paper PDFs to Obsidian-compatible Markdown.",
+    add_completion=False
+)
+console = Console()
+
+logging.basicConfig(level=logging.WARNING, format="%(levelname)s: %(message)s")
+
+
+@app.command()
+def convert(
+    pdf: Path = typer.Argument(..., help="Path to the PDF file"),
+    output: Optional[Path] = typer.Option(None, "-o", "--output", help="Output directory"),
+    model: Optional[str] = typer.Option(None, "--model", help="Override model"),
+    concurrency: Optional[int] = typer.Option(None, "--concurrency", help="Max concurrent API calls"),
+    keep_intermediate: bool = typer.Option(False, "--keep-intermediate", help="Keep sub-PDFs and raw responses"),
+    verbose: bool = typer.Option(False, "--verbose", "-v", help="Verbose logging")
+) -> None:
+    if verbose:
+        logging.getLogger().setLevel(logging.DEBUG)
+
+    if not pdf.exists():
+        console.print(f"[red]Error:[/red] File not found: {pdf}")
+        raise typer.Exit(1)
+
+    cfg = load_config()
+    if model:
+        cfg.api.model = model
+    if concurrency:
+        cfg.api.concurrency = concurrency
+
+    from .converter import _is_openai
+
+    if _is_openai(cfg.api.model):
+        if not cfg.openai_api_key:
+            console.print("[red]Error:[/red] OPENAI_API_KEY not set. Add it to .env or environment.")
+            raise typer.Exit(1)
+    else:
+        if not cfg.api_key:
+            console.print("[red]Error:[/red] ANTHROPIC_API_KEY not set. Add it to .env or environment.")
+            raise typer.Exit(1)
+
+    out_dir = output or Path(cfg.output.default_dir).expanduser()
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    with tempfile.TemporaryDirectory() as tmp:
+        tmp_path = Path(tmp)
+        _run_conversion(pdf, out_dir, tmp_path, cfg, keep_intermediate)
+
+
+def _run_conversion(pdf: Path, out_dir: Path, tmp_path: Path, cfg, keep_intermediate: bool) -> None:
+    with Progress(
+        SpinnerColumn(),
+        TextColumn("[progress.description]{task.description}"),
+        TimeElapsedColumn(),
+        console=console
+    ) as progress:
+        metadata_task = progress.add_task("Extracting metadata...", total=None)
+        meta = extract_metadata(pdf)
+        progress.update(metadata_task, description=f"[green]Metadata:[/green] {meta.title or pdf.stem}")
+        progress.stop_task(metadata_task)
+
+        split_task = progress.add_task("Splitting PDF...", total=None)
+        specs = split_pdf(
+            pdf,
+            tmp_path,
+            chunk_size=cfg.chunking.max_pages_per_chunk,
+            overlap=cfg.chunking.overlap_pages,
+            single_pass_threshold=cfg.chunking.single_pass_threshold
+        )
+        progress.update(split_task, description=f"[green]Split:[/green] {len(specs)} chunk(s)")
+        progress.stop_task(split_task)
+
+        convert_task = progress.add_task(f"Converting {len(specs)} chunk(s)...", total=None)
+        title = meta.title or pdf.stem
+        raw_chunks = convert_all(specs, title, meta.authors, cfg)
+        progress.update(convert_task, description=f"[green]Converted[/green] {len(raw_chunks)} chunk(s)")
+        progress.stop_task(convert_task)
+
+        stitch_task = progress.add_task("Stitching...", total=None)
+        stitched = stitch(raw_chunks)
+        progress.stop_task(stitch_task)
+
+        post_task = progress.add_task("Post-processing...", total=None)
+        final_md = postprocess(stitched, meta)
+        progress.stop_task(post_task)
+
+        verify_task = progress.add_task("Verifying...", total=None)
+        result = verify(pdf, final_md)
+        progress.stop_task(verify_task)
+
+    if result.warnings:
+        console.print("\n[yellow]Verification warnings:[/yellow]")
+        for warning in result.warnings:
+            console.print(f"  • {warning}")
+        if result.missing_passages:
+            console.print("\n[dim]Sample missing passages:[/dim]")
+            for passage in result.missing_passages[:5]:
+                console.print(f"  - {passage}")
+
+    safe_title = re.sub(r'[\\/:*?"<>|]', "_", meta.title or pdf.stem)[:120]
+    out_file = out_dir / f"{safe_title}.md"
+    out_file.write_text(final_md, encoding="utf-8")
+
+    status = "[green]OK[/green]" if result.passed else "[yellow]WARN[/yellow]"
+    console.print(f"\n{status} Written -> {out_file}")
+
+    if keep_intermediate:
+        keep_dir = out_dir / f"{safe_title}_intermediate"
+        keep_dir.mkdir(exist_ok=True)
+        for spec in (specs if specs and hasattr(specs[0], "path") else []):
+            try:
+                shutil.copy(spec.path, keep_dir)
+            except Exception:
+                pass
+        console.print(f"   Intermediate artifacts saved to {keep_dir}")
+
+
+@app.command()
+def inspect(
+    pdf: Path = typer.Argument(..., help="Path to the PDF file")
+) -> None:
+    if not pdf.exists():
+        console.print(f"[red]Error:[/red] File not found: {pdf}")
+        raise typer.Exit(1)
+
+    cfg = load_config()
+
+    console.print("\n[bold]Metadata[/bold]")
+    meta = extract_metadata(pdf)
+    table = Table(show_header=False)
+    table.add_row("Title", meta.title or "[dim]not found[/dim]")
+    table.add_row("Authors", ", ".join(meta.authors) or "[dim]not found[/dim]")
+    table.add_row("Year", meta.year or "[dim]not found[/dim]")
+    table.add_row("Venue", meta.venue or "[dim]not found[/dim]")
+    table.add_row("DOI", meta.doi or "[dim]not found[/dim]")
+    table.add_row("arXiv", meta.arxiv or "[dim]not found[/dim]")
+    console.print(table)
+
+    console.print("\n[bold]Split Plan[/bold]")
+    with tempfile.TemporaryDirectory() as tmp:
+        specs = split_pdf(
+            pdf,
+            Path(tmp),
+            chunk_size=cfg.chunking.max_pages_per_chunk,
+            overlap=cfg.chunking.overlap_pages,
+            single_pass_threshold=cfg.chunking.single_pass_threshold
+        )
+
+    chunk_table = Table("Chunk", "Pages", "Context pages")
+    for spec in specs:
+        chunk_table.add_row(
+            str(spec.chunk_idx + 1),
+            f"{spec.start + 1}-{spec.end + 1}",
+            str(spec.context_pages)
+        )
+    console.print(chunk_table)
+    console.print(f"\nTotal chunks: {len(specs)}")
+
+
+def main() -> None:
+    app()
+
+
+if __name__ == "__main__":
+    main()

paper2mdviallm-0.1.0/paper2mdviallm/config.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+import os
+import tomllib
+from pathlib import Path
+from typing import Optional
+
+from dotenv import load_dotenv
+from pydantic import BaseModel, Field
+
+load_dotenv()
+
+_CONFIG_PATH = Path.home() / ".paper2mdviallm" / "config.toml"
+
+
+class ApiConfig(BaseModel):
+    model: str = "claude-sonnet-4-6"
+    max_tokens: int = 16000
+    concurrency: int = 3
+
+
+class OutputConfig(BaseModel):
+    default_dir: str = "~/Documents/ObsidianVault/Papers"
+
+
+class ChunkingConfig(BaseModel):
+    max_pages_per_chunk: int = 10
+    overlap_pages: int = 1
+    single_pass_threshold: int = 12
+
+
+class Config(BaseModel):
+    api: ApiConfig = Field(default_factory=ApiConfig)
+    output: OutputConfig = Field(default_factory=OutputConfig)
+    chunking: ChunkingConfig = Field(default_factory=ChunkingConfig)
+    api_key: Optional[str] = None
+    openai_api_key: Optional[str] = None
+
+    @classmethod
+    def load(cls) -> "Config":
+        raw: dict = {}
+        if _CONFIG_PATH.exists():
+            with open(_CONFIG_PATH, "rb") as f:
+                raw = tomllib.load(f)
+        obj = cls(**raw)
+        obj.api_key = os.environ.get("ANTHROPIC_API_KEY")
+        obj.openai_api_key = os.environ.get("OPENAI_API_KEY")
+        return obj
+
+
+def load_config() -> Config:
+    return Config.load()

paper2mdviallm-0.1.0/paper2mdviallm/converter.py

@@ -0,0 +1,233 @@
+from __future__ import annotations
+
+import asyncio
+import base64
+import logging
+from pathlib import Path
+from typing import Any
+
+import anthropic
+import openai
+from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential
+
+from .config import Config
+from .prompts import SYSTEM_PROMPT, build_user_instruction
+from .splitter import ChunkSpec
+
+log = logging.getLogger(__name__)
+
+_ANTHROPIC_RETRYABLE = (
+    anthropic.RateLimitError,
+    anthropic.APITimeoutError,
+    anthropic.APIConnectionError
+)
+
+_OPENAI_RETRYABLE = (
+    openai.RateLimitError,
+    openai.APITimeoutError,
+    openai.APIConnectionError
+)
+
+_RETRY_KWARGS = dict(
+    wait=wait_exponential(multiplier=1, min=2, max=60),
+    stop=stop_after_attempt(5),
+    reraise=True
+)
+
+
+def _is_openai(model: str) -> bool:
+    return model.startswith(("gpt-", "o1-", "o3-", "o4-"))
+
+
+def _make_client(cfg: Config) -> Any:
+    if _is_openai(cfg.api.model):
+        return openai.OpenAI(api_key=cfg.openai_api_key)
+    return anthropic.Anthropic(api_key=cfg.api_key)
+
+
+def convert_chunk(
+    spec: ChunkSpec,
+    title: str,
+    authors: list[str],
+    cfg: Config,
+    client: Any = None
+) -> str:
+    if client is None:
+        client = _make_client(cfg)
+    return _call_with_retry(spec, title, authors, cfg, client)
+
+
+def _call_with_retry(
+    spec: ChunkSpec,
+    title: str,
+    authors: list[str],
+    cfg: Config,
+    client: Any,
+    attempt: int = 0
+) -> str:
+    bad_request = openai.BadRequestError if _is_openai(cfg.api.model) else anthropic.BadRequestError
+    try:
+        if _is_openai(cfg.api.model):
+            return _do_call_openai(spec, title, authors, cfg, client)
+        return _do_call_anthropic(spec, title, authors, cfg, client)
+    except bad_request as exc:
+        if attempt >= 2:
+            raise
+        log.warning("Chunk %d failed (attempt %d), halving and retrying: %s", spec.chunk_idx, attempt, exc)
+        return _halve_and_retry(spec, title, authors, cfg, client, attempt)
+
+
+@retry(retry=retry_if_exception_type(_ANTHROPIC_RETRYABLE), **_RETRY_KWARGS)
+def _do_call_anthropic(
+    spec: ChunkSpec,
+    title: str,
+    authors: list[str],
+    cfg: Config,
+    client: anthropic.Anthropic
+) -> str:
+    pdf_bytes = spec.path.read_bytes()
+    pdf_b64 = base64.standard_b64encode(pdf_bytes).decode("utf-8")
+    user_text = build_user_instruction(
+        title=title,
+        authors=authors,
+        chunk_idx=spec.chunk_idx,
+        total=spec.total,
+        start=spec.start,
+        end=spec.end,
+        context_pages=spec.context_pages
+    )
+
+    response = client.messages.create(
+        model=cfg.api.model,
+        max_tokens=cfg.api.max_tokens,
+        system=SYSTEM_PROMPT,
+        messages=[
+            {
+                "role": "user",
+                "content": [
+                    {
+                        "type": "document",
+                        "source": {
+                            "type": "base64",
+                            "media_type": "application/pdf",
+                            "data": pdf_b64
+                        },
+                        "cache_control": {"type": "ephemeral"}
+                    },
+                    {"type": "text", "text": user_text}
+                ]
+            }
+        ]
+    )
+    return response.content[0].text
+
+
+@retry(retry=retry_if_exception_type(_OPENAI_RETRYABLE), **_RETRY_KWARGS)
+def _do_call_openai(
+    spec: ChunkSpec,
+    title: str,
+    authors: list[str],
+    cfg: Config,
+    client: openai.OpenAI
+) -> str:
+    import fitz
+
+    doc = fitz.open(spec.path)
+    content: list[dict] = []
+
+    mat = fitz.Matrix(150 / 72, 150 / 72)
+    for page in doc:
+        pix = page.get_pixmap(matrix=mat)
+        img_b64 = base64.standard_b64encode(pix.tobytes("png")).decode("utf-8")
+        content.append({
+            "type": "image_url",
+            "image_url": {"url": f"data:image/png;base64,{img_b64}", "detail": "high"}
+        })
+    doc.close()
+
+    user_text = build_user_instruction(
+        title=title,
+        authors=authors,
+        chunk_idx=spec.chunk_idx,
+        total=spec.total,
+        start=spec.start,
+        end=spec.end,
+        context_pages=spec.context_pages
+    )
+    content.append({"type": "text", "text": user_text})
+
+    response = client.chat.completions.create(
+        model=cfg.api.model,
+        max_completion_tokens=cfg.api.max_tokens,
+        messages=[
+            {"role": "system", "content": SYSTEM_PROMPT},
+            {"role": "user", "content": content}
+        ]
+    )
+    return response.choices[0].message.content
+
+
+def _halve_and_retry(
+    spec: ChunkSpec,
+    title: str,
+    authors: list[str],
+    cfg: Config,
+    client: Any,
+    attempt: int
+) -> str:
+    import fitz
+
+    doc = fitz.open(spec.path)
+    n = len(doc)
+    mid = n // 2
+    tmp_dir = spec.path.parent
+
+    halves = []
+    for hi, (fr, to) in enumerate([(0, mid - 1), (mid, n - 1)]):
+        sub = fitz.open()
+        sub.insert_pdf(doc, from_page=fr, to_page=to)
+        sub_path = tmp_dir / f"{spec.path.stem}_half{hi}.pdf"
+        sub.save(sub_path)
+        sub.close()
+
+        half_spec = ChunkSpec(
+            path=sub_path,
+            start=spec.start + fr,
+            end=spec.start + to,
+            context_pages=spec.context_pages if hi == 0 else 0,
+            chunk_idx=spec.chunk_idx,
+            total=spec.total
+        )
+        halves.append(half_spec)
+
+    doc.close()
+
+    results = [_call_with_retry(half, title, authors, cfg, client, attempt + 1) for half in halves]
+    return "\n\n".join(results)
+
+
+async def convert_chunks_async(
+    specs: list[ChunkSpec],
+    title: str,
+    authors: list[str],
+    cfg: Config
+) -> list[str]:
+    client = _make_client(cfg)
+    sem = asyncio.Semaphore(cfg.api.concurrency)
+
+    async def bounded(spec: ChunkSpec) -> str:
+        async with sem:
+            loop = asyncio.get_event_loop()
+            return await loop.run_in_executor(None, convert_chunk, spec, title, authors, cfg, client)
+
+    tasks = [bounded(spec) for spec in specs]
+    return await asyncio.gather(*tasks)
+
+
+def convert_all(
+    specs: list[ChunkSpec],
+    title: str,
+    authors: list[str],
+    cfg: Config
+) -> list[str]:
+    return asyncio.run(convert_chunks_async(specs, title, authors, cfg))