file2records 0.2.0__tar.gz

file2records-0.2.0/.gitignore

@@ -0,0 +1,29 @@
+.venv/
+__pycache__/
+*.pyc
+.env
+workspace/pdfs/*
+workspace/parsed/*
+workspace/extracted/*
+workspace/judged/*
+!workspace/pdfs/.gitkeep
+!workspace/parsed/.gitkeep
+!workspace/extracted/.gitkeep
+!workspace/judged/.gitkeep
+
+# the Docker workspace volume
+data/*
+!data/.gitkeep
+
+# user config is local: prompts, schema and model choice are per-project, and timings are
+# measurements of one machine
+workspace/config/*
+!workspace/config/.gitkeep
+
+# the verification run's console log; its report (artifacts/overnight-report.md) is worth keeping
+artifacts/overnight.log
+
+# built docs site (zensical build)
+site/
+# downloaded by `vale sync`
+.vale/styles/Google/

file2records-0.2.0/CITATION.cff

@@ -0,0 +1,26 @@
+cff-version: 1.2.0
+title: file2records
+message: >-
+  If you use this software in work that you publish, please cite it.
+type: software
+authors:
+  - family-names: Battulga
+    given-names: Enkhnyam
+    affiliation: RWTH Aachen University
+repository-code: https://github.com/Enkhnyam/chemistry-data-extractor-toolkit
+abstract: >-
+  A local tool for building auditable experimental datasets from scientific papers (PDF, JATS and Elsevier XML, HTML, Word). One language
+  model extracts records against a schema you define; a second re-reads the paper and audits
+  every record against it; the reviewer checks what remains with the source text alongside and
+  the passage behind each value highlighted. Schema, prompts and worked examples are
+  configuration rather than code, so the same pipeline serves any chemistry. Generalised from a
+  PET-depolymerisation database built the same way.
+keywords:
+  - information extraction
+  - large language models
+  - chemistry
+  - literature mining
+  - LLM-as-a-judge
+  - research data
+license: MIT
+version: 0.2.0

file2records-0.2.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Enkhnyam Battulga, RWTH Aachen University
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

file2records-0.2.0/PKG-INFO

@@ -0,0 +1,75 @@
+Metadata-Version: 2.5
+Name: file2records
+Version: 0.2.0
+Summary: Turn scientific papers (PDF, JATS and Elsevier XML, HTML, Word) into a structured dataset: one model extracts, a second audits, you review against the source text.
+Project-URL: Homepage, https://github.com/Enkhnyam/chemistry-data-extractor-toolkit
+Project-URL: Repository, https://github.com/Enkhnyam/chemistry-data-extractor-toolkit
+Project-URL: Issues, https://github.com/Enkhnyam/chemistry-data-extractor-toolkit/issues
+Author: Enkhnyam Battulga
+License-Expression: MIT
+License-File: LICENSE
+Keywords: chemistry,data extraction,elsevier,jats,llm,pdf,scientific papers
+Classifier: Development Status :: 4 - Beta
+Classifier: Framework :: FastAPI
+Classifier: Intended Audience :: Science/Research
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Chemistry
+Requires-Python: >=3.10
+Requires-Dist: fastapi>=0.115.0
+Requires-Dist: httpx>=0.27
+Requires-Dist: litellm>=1.91.0
+Requires-Dist: lxml>=5.0
+Requires-Dist: markdown-it-py>=3.0.0
+Requires-Dist: pydantic>=2.0
+Requires-Dist: python-docx>=1.1
+Requires-Dist: python-dotenv>=1.0.1
+Requires-Dist: python-multipart>=0.0.12
+Requires-Dist: tenacity>=8.0
+Requires-Dist: uvicorn[standard]>=0.32.0
+Provides-Extra: pdf
+Requires-Dist: docling>=2.110.0; extra == 'pdf'
+Description-Content-Type: text/markdown
+
+# file2records
+
+file2records builds a dataset from a folder of scientific papers. You say which fields a
+record has. A language model reads each paper and fills them in, a second model checks every
+record against the paper, and you review the result with the source passage next to each
+value. It reads PDF, JATS XML, Elsevier XML, HTML, and Word files, runs on your own machine,
+and comes set up for RWTH's free KI:connect models.
+
+Documentation: <https://enkhnyam.github.io/chemistry-data-extractor-toolkit/>
+
+```bash
+pip install "file2records[pdf]"        # or: pip install file2records   (no PDFs, ~250 MB)
+file2records serve my-first-project    # opens a finished example in your browser
+```
+
+```python
+import file2records as fr
+
+project = fr.Project("my-review")
+project.add("papers/")                                   # PDF, XML, HTML, Word, Markdown
+project.extract(model=fr.rwth(), only=r"glycoly[sz]is")  # RWTH_API_KEY from the environment
+project.judge(model=fr.rwth())
+project.export("dataset.csv")                            # one row per record, with its DOI
+```
+
+Start with the [tutorial](https://enkhnyam.github.io/chemistry-data-extractor-toolkit/tutorial/):
+a PET glycolysis dataset from three real papers in 15 minutes.
+
+## Development
+
+```bash
+uv sync                                       # includes docling for PDF tests
+uv run python -m unittest discover -s tests   # no network, no key
+uvx zensical serve                            # the docs, at http://localhost:8000
+```
+
+Releasing: [RELEASING.md](RELEASING.md). Licence: the code is MIT; the demo paper is CC BY
+([NOTICE](src/file2records/demo/NOTICE.md)). Please cite: [CITATION.cff](CITATION.cff).

file2records-0.2.0/README.md

@@ -0,0 +1,38 @@
+# file2records
+
+file2records builds a dataset from a folder of scientific papers. You say which fields a
+record has. A language model reads each paper and fills them in, a second model checks every
+record against the paper, and you review the result with the source passage next to each
+value. It reads PDF, JATS XML, Elsevier XML, HTML, and Word files, runs on your own machine,
+and comes set up for RWTH's free KI:connect models.
+
+Documentation: <https://enkhnyam.github.io/chemistry-data-extractor-toolkit/>
+
+```bash
+pip install "file2records[pdf]"        # or: pip install file2records   (no PDFs, ~250 MB)
+file2records serve my-first-project    # opens a finished example in your browser
+```
+
+```python
+import file2records as fr
+
+project = fr.Project("my-review")
+project.add("papers/")                                   # PDF, XML, HTML, Word, Markdown
+project.extract(model=fr.rwth(), only=r"glycoly[sz]is")  # RWTH_API_KEY from the environment
+project.judge(model=fr.rwth())
+project.export("dataset.csv")                            # one row per record, with its DOI
+```
+
+Start with the [tutorial](https://enkhnyam.github.io/chemistry-data-extractor-toolkit/tutorial/):
+a PET glycolysis dataset from three real papers in 15 minutes.
+
+## Development
+
+```bash
+uv sync                                       # includes docling for PDF tests
+uv run python -m unittest discover -s tests   # no network, no key
+uvx zensical serve                            # the docs, at http://localhost:8000
+```
+
+Releasing: [RELEASING.md](RELEASING.md). Licence: the code is MIT; the demo paper is CC BY
+([NOTICE](src/file2records/demo/NOTICE.md)). Please cite: [CITATION.cff](CITATION.cff).

file2records-0.2.0/pyproject.toml

@@ -0,0 +1,70 @@
+[project]
+name = "file2records"
+version = "0.2.0"
+description = "Turn scientific papers (PDF, JATS and Elsevier XML, HTML, Word) into a structured dataset: one model extracts, a second audits, you review against the source text."
+readme = "README.md"
+requires-python = ">=3.10"
+license = "MIT"
+license-files = ["LICENSE"]
+authors = [{ name = "Enkhnyam Battulga" }]
+keywords = ["chemistry", "data extraction", "llm", "scientific papers", "jats", "elsevier", "pdf"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Science/Research",
+    "Topic :: Scientific/Engineering :: Chemistry",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Operating System :: OS Independent",
+    "Framework :: FastAPI",
+]
+
+dependencies = [
+    "fastapi>=0.115.0",
+    "uvicorn[standard]>=0.32.0",
+    "python-multipart>=0.0.12",
+    "litellm>=1.91.0",
+    # litellm.completion(num_retries=...) imports tenacity lazily and litellm does not declare
+    # it, so every model call on a clean install died with "tenacity import failed". We pass
+    # num_retries on every call, which made that every call.
+    "tenacity>=8.0",
+    "pydantic>=2.0",
+    "python-dotenv>=1.0.1",
+    "markdown-it-py>=3.0.0",
+    # Only for asking an endpoint which models it serves. It arrives anyway under litellm, but
+    # a direct import belongs in a direct dependency -- otherwise it vanishes the day litellm
+    # changes its HTTP client.
+    "httpx>=0.27",
+    # The XML and HTML readers (JATS, Elsevier, publisher pages) and the Word reader. Small, and
+    # no machine-learning model is involved in reading a structured format.
+    "lxml>=5.0",
+    "python-docx>=1.1",
+]
+
+[project.optional-dependencies]
+# PDF reading: docling and its layout/table models, via PyTorch -- a download of several GB.
+# Kept out of the base install so the structured formats work everywhere in seconds.
+pdf = ["docling>=2.110.0"]
+
+[project.scripts]
+file2records = "file2records.cli:main"
+
+[project.urls]
+Homepage = "https://github.com/Enkhnyam/chemistry-data-extractor-toolkit"
+Repository = "https://github.com/Enkhnyam/chemistry-data-extractor-toolkit"
+Issues = "https://github.com/Enkhnyam/chemistry-data-extractor-toolkit/issues"
+
+[build-system]
+requires = ["hatchling>=1.27"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/file2records"]
+
+[tool.hatch.build.targets.sdist]
+include = ["src/file2records", "tests", "README.md", "LICENSE", "CITATION.cff"]
+
+[dependency-groups]
+dev = ["docling>=2.110.0"]

file2records-0.2.0/src/file2records/__init__.py

@@ -0,0 +1,15 @@
+"""file2records: turn a folder of papers into a structured dataset you can check.
+
+    import file2records as fr
+    project = fr.Project("my-review")
+    project.add("papers/")
+    project.extract(model=fr.rwth())
+    project.export("dataset.csv")
+
+See project.py for the API, cli.py for the command line, main.py for the web app.
+"""
+__version__ = "0.2.0"
+
+from .project import Project, rwth  # noqa: E402
+
+__all__ = ["Project", "rwth", "__version__"]

file2records-0.2.0/src/file2records/__main__.py

@@ -0,0 +1,5 @@
+import sys
+
+from .cli import main
+
+sys.exit(main())

file2records-0.2.0/src/file2records/bundle.py

@@ -0,0 +1,155 @@
+"""The export bundle: a dataset together with what produced it.
+
+A CSV of records on its own is not reproducible -- it cannot say which model wrote it, under
+which prompt, against which schema, or which rows a human then corrected. The bundle is the data
+beside the config and a manifest that pins both.
+
+What it leaves out by default matters as much. Papers obtained through publishers' text-and-data
+mining agreements may be read and mined, not redistributed -- and a bundle is made to be shared.
+So the paper text and the source files go in only when asked for (`include_text`,
+`include_files`), which is the right call for open-access papers and the caller's to make.
+Without them every record still names its paper's DOI and the ids of the chunks it came from,
+which is enough for anyone with access to the paper to check it.
+
+No key ever enters the bundle; model settings are copied without their key variables.
+"""
+import csv
+import io
+import json
+import subprocess
+import time
+import zipfile
+from pathlib import Path
+
+from . import __version__, config, models, report, storage
+from .storage import read_json
+
+
+def csv_bytes(columns, rows) -> bytes:
+    buffer = io.StringIO()
+    writer = csv.DictWriter(buffer, fieldnames=columns, extrasaction="ignore")
+    writer.writeheader()
+    writer.writerows(rows)
+    return buffer.getvalue().encode("utf-8")
+
+
+def git_commit() -> str | None:
+    """Which checkout produced this, when run from one. None for an installed package, whose
+    version is recorded instead."""
+    try:
+        return subprocess.check_output(["git", "-C", str(Path(__file__).resolve().parent),
+                                        "rev-parse", "HEAD"],
+                                       text=True, stderr=subprocess.DEVNULL, timeout=5).strip()
+    except Exception:
+        return None
+
+
+def build(paper_ids: list[str] | None = None, *, include_text: bool = False,
+          include_files: bool = False, profiles: list[dict] | None = None) -> bytes:
+    """The zip as bytes. `paper_ids` limits it to those papers (None: all of them)."""
+    record_columns, record_rows = report.flat_records(paper_ids)
+    paper_columns, paper_rows = report.papers_table(paper_ids)
+    summary = report.build()
+    settings = config.get_settings()
+    if profiles is None:
+        profiles = [{k: v for k, v in p.items()} for p in models.listing(lambda _: False)]
+    chosen = {r["paper_id"] for r in paper_rows}
+
+    manifest = {
+        "exported_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+        "tool": "file2records",
+        "version": __version__,
+        "git_commit": git_commit(),
+        "papers": len(paper_rows),
+        "records": len(record_rows),
+        "includes_paper_text": include_text,
+        "models": {
+            "extract": (models.get(settings.get("extract_model", "")) or {}).get("model"),
+            "judge": (models.get(settings.get("judge_model", "")) or {}).get("model"),
+            # per paper too: a corpus is often built across more than one model
+            "per_paper": {r["paper_id"]: {"extract": r["extract_model"], "judge": r["judge_model"]}
+                          for r in paper_rows},
+        },
+        "spend": summary.get("totals", {}).get("spend", {}),
+        "source_tracking_default": settings.get("source_tracking_default", True),
+    }
+
+    buffer = io.BytesIO()
+    with zipfile.ZipFile(buffer, "w", zipfile.ZIP_DEFLATED) as bundle:
+        bundle.writestr("manifest.json", json.dumps(manifest, indent=2, ensure_ascii=False))
+        bundle.writestr("README.md", _readme(manifest))
+        bundle.writestr("data/records.csv", csv_bytes(record_columns, record_rows))
+        bundle.writestr("data/records.json", json.dumps(record_rows, indent=1, ensure_ascii=False))
+        bundle.writestr("data/papers.csv", csv_bytes(paper_columns, paper_rows))
+        bundle.writestr("data/report.json", json.dumps(summary, indent=2, ensure_ascii=False))
+
+        # config: everything that decides what a run produces, and nothing that authenticates it
+        bundle.writestr("config/schema.json",
+                        json.dumps({"fields": config.get_schema()}, indent=2, ensure_ascii=False))
+        bundle.writestr("config/extract_prompt.txt", config.get_extract_prompt())
+        bundle.writestr("config/judge_prompt.txt", config.get_judge_prompt())
+        bundle.writestr("config/few_shot.json",
+                        json.dumps(config.get_few_shot(), indent=2, ensure_ascii=False))
+        bundle.writestr("config/models.json", json.dumps(
+            [{k: v for k, v in m.items()
+              if k in ("id", "name", "model", "api_base", "api_version")} for m in profiles],
+            indent=2, ensure_ascii=False))
+
+        # the per-paper working files, so a reviewer can trace any row back to its chunk
+        for stage, directory in (("extracted", storage.EXTRACTED), ("judged", storage.JUDGED)):
+            for path in sorted(directory.glob("*.json")):
+                if path.stem in chosen:
+                    bundle.write(path, f"{stage}/{path.name}")
+        for path in sorted(storage.PARSED.glob("*.json")):
+            if path.stem not in chosen:
+                continue
+            paper = read_json(path, {})
+            if not include_text:
+                paper["chunks"] = [{"id": c["id"]} for c in paper.get("chunks", [])]
+            bundle.writestr(f"parsed/{path.name}", json.dumps(paper, indent=1, ensure_ascii=False))
+        if include_files:
+            for pid in sorted(chosen):
+                if (source := storage.source_file(pid)) is not None:
+                    bundle.write(source, f"papers/{source.name}")
+    return buffer.getvalue()
+
+
+def _readme(manifest: dict) -> str:
+    m = manifest["models"]
+    text_note = (
+        "`chunks[].text` holds the paper text, because this bundle was exported with it. Check "
+        "the papers' licences before sharing it."
+        if manifest["includes_paper_text"] else
+        "Paper text is not included -- papers obtained under text-and-data-mining terms may be "
+        "mined, not redistributed. Each chunk keeps its `id`, and each record its paper's DOI, "
+        "so anyone with access to the paper can check a value against its source.")
+    return f"""# file2records bundle
+
+{manifest['records']} records from {manifest['papers']} paper(s), exported
+{manifest['exported_at']} by file2records {manifest['version']}.
+
+## What is here
+
+- `data/records.csv`, `data/records.json` — one row per record, with the paper's DOI, the
+  judge's verdict and any reviewer flag or note. `extract_model` and `judge_model` say what
+  produced each row.
+- `data/papers.csv` — one row per paper: DOI, chunks in, records out, model, tokens, cost.
+- `data/report.json` — completeness by field, what the judge changed, totals.
+- `config/` — the schema, both prompts, the worked examples and the model settings that
+  produced this. API keys are not included.
+- `parsed/`, `extracted/`, `judged/` — the working files, so any row can be traced back to the
+  chunk it came from. `source_chunk_ids` on a record refers to `chunks[].id` in `parsed/`.
+- `papers/` — the source files, if you exported with them.
+
+{text_note}
+
+## Reading it
+
+Extraction model: `{m['extract'] or 'not set'}` · judge model: `{m['judge'] or 'not set'}`.
+Papers may differ from these if the corpus was built across more than one model; see
+`models.per_paper` in `manifest.json`.
+
+`model_records` in `extracted/*.json` is what the model originally said, kept beside the
+corrected records the moment anything was edited. A corrected dataset that cannot be diffed
+against the model's own output is not evidence of anything.
+"""

file2records-0.2.0/src/file2records/cli.py

@@ -0,0 +1,207 @@
+"""The `file2records` command.
+
+    file2records serve my-review                 the web app on that project folder
+    file2records add my-review papers/           read files and folders into it
+    file2records papers my-review                what is in it
+    file2records search my-review "glycoly[sz]is"
+    file2records check my-review                 what is missing before a run
+    file2records extract my-review --only "glycoly[sz]is" --exclude "positron|tomograph"
+    file2records judge my-review
+    file2records export my-review dataset.csv    (.csv, .json, or .zip for the full bundle)
+
+Every command takes the project folder first. Model commands use the model chosen in the web
+app's Settings unless --model is given: any litellm model string, or rwth/<name> for RWTH's
+KI:connect (key from RWTH_API_KEY).
+"""
+import argparse
+import os
+import sys
+from pathlib import Path
+
+from dotenv import load_dotenv
+
+from . import __version__
+
+
+def _project(folder):
+    from .project import Project
+    project = Project(folder)
+    # Keys saved through the web app live in the project's .env; a .env in the current folder
+    # is read too. Neither overrides a variable already set in the shell.
+    load_dotenv(project.path / ".env")
+    load_dotenv(Path.cwd() / ".env")
+    return project
+
+
+def _filters(p):
+    p.add_argument("--only", metavar="REGEX",
+                   help="only papers whose full text matches this regular expression")
+    p.add_argument("--exclude", metavar="REGEX",
+                   help="skip papers whose full text matches this regular expression")
+
+
+def _print_result(r):
+    name = r.get("filename") or r["id"]
+    if r.get("error"):
+        print(f"  ✗ {name}: {r['error']}")
+    elif "n_chunks" in r:
+        doi = f"  doi:{r['doi']}" if r.get("doi") else ""
+        print(f"  ✓ {name}  [{r['format']}]  {r['n_chunks']} chunks{doi}")
+    elif "n_records" in r:
+        print(f"  ✓ {name}: {r['n_records']} records in {r['seconds']}s")
+    else:
+        print(f"  ✓ {name}: {r['n_verdicts']} verdicts in {r['seconds']}s")
+
+
+def cmd_serve(args):
+    os.environ["WORKSPACE_DIR"] = str(Path(args.folder).resolve())
+    _project(args.folder)                        # creates the folder, opens it
+    import uvicorn
+    from .main import app
+    url = f"http://{args.host}:{args.port}"
+    print(f"file2records {__version__} — project {Path(args.folder).resolve()}\nOpen {url}")
+    if not args.no_browser:
+        import threading
+        import webbrowser
+        threading.Timer(1.0, webbrowser.open, [url]).start()
+    uvicorn.run(app, host=args.host, port=args.port, log_level="warning")
+
+
+def cmd_add(args):
+    project = _project(args.folder)
+    print(f"Reading into {project.path}")
+    results = project.add(*args.paths, source_tracking=not args.no_source_tracking,
+                          on_file=_print_result)
+    failed = sum(1 for r in results if r.get("error"))
+    print(f"{len(results) - failed} added, {failed} failed")
+    return 1 if failed and failed == len(results) else 0
+
+
+def cmd_papers(args):
+    papers = _project(args.folder).papers()
+    if not papers:
+        print("No papers yet. Add some:  file2records add <folder> <files or folders>")
+    for p in papers:
+        state = "judged" if p["judged"] else "extracted" if p["extracted"] else "parsed"
+        records = "" if p["n_records"] is None else f"{p['n_records']} records"
+        print(f"{p['id']:50.50}  {p['format']:8}  {state:9}  {records:11}  {p['doi']}")
+
+
+def cmd_search(args):
+    hits = _project(args.folder).search(args.pattern, ignore_case=not args.case_sensitive)
+    for h in hits:
+        print(f"\n{h['filename']}  ({h['matches']} matches)")
+        for s in h["snippets"]:
+            print(f"    …{s['before'][-60:]}[{s['match']}]{s['after'][:60]}…".replace("\n", " "))
+    total = sum(h["matches"] for h in hits)
+    print(f"\n{len(hits)} papers, {total} matches")
+
+
+def cmd_check(args):
+    project = _project(args.folder)
+    ok = True
+    for stage in ("extract", "judge"):
+        missing = project.check(stage, args.model)
+        ok &= stage == "judge" or not missing
+        print(f"{stage}: {'ready' if not missing else 'not ready'}")
+        for m in missing:
+            hint = " Or pass --model, e.g. rwth/gpt-oss-120b." if m.startswith("Choose a model") else ""
+            print(f"  - {m}{hint}")
+    return 0 if ok else 1
+
+
+def _run_stage(args, stage):
+    project = _project(args.folder)
+    run = project.extract if stage == "extract" else project.judge
+    results = run(args.model, only=args.only, exclude=args.exclude, redo=args.redo,
+                  on_paper=_print_result)
+    if not results:
+        print(f"Nothing to {stage}: every chosen paper is done already (--redo to run again).")
+    failed = sum(1 for r in results if r.get("error"))
+    print(f"{len(results) - failed} done, {failed} failed")
+    return 1 if failed else 0
+
+
+def cmd_export(args):
+    path = _project(args.folder).export(args.output, only=args.only, exclude=args.exclude,
+                                        include_text=args.include_text,
+                                        include_files=args.include_files)
+    print(f"Wrote {path}")
+
+
+FOLDER = "the project folder (created if it does not exist)"
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="file2records",
+        description="Turn papers (PDF, XML, HTML, Word, Markdown) into a structured dataset.",
+        epilog="examples:\n" + "\n".join(__doc__.splitlines()[2:10]) +
+               "\n\ndocs: https://enkhnyam.github.io/chemistry-data-extractor-toolkit/",
+        formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("--version", action="version", version=f"file2records {__version__}")
+    sub = parser.add_subparsers(dest="command", required=True, metavar="COMMAND")
+
+    p = sub.add_parser("serve", help="open the web app on a project folder")
+    p.add_argument("folder", nargs="?", default="workspace", help=FOLDER + " (default: %(default)s)")
+    p.add_argument("--port", type=int, default=8000)
+    p.add_argument("--host", default="127.0.0.1",
+                   help="keep the default unless you know why: the app has no login")
+    p.add_argument("--no-browser", action="store_true")
+    p.set_defaults(func=cmd_serve)
+
+    p = sub.add_parser("add", help="read files or folders of papers into a project")
+    p.add_argument("folder", help=FOLDER)
+    p.add_argument("paths", nargs="+", help="paper files, or folders of them")
+    p.add_argument("--no-source-tracking", action="store_true",
+                   help="don't tag chunks, so records won't cite the passage they came from")
+    p.set_defaults(func=cmd_add)
+
+    p = sub.add_parser("papers", help="list the papers in a project")
+    p.add_argument("folder", help=FOLDER)
+    p.set_defaults(func=cmd_papers)
+
+    p = sub.add_parser("search", help="search the full text of every paper with a regex")
+    p.add_argument("folder", help=FOLDER)
+    p.add_argument("pattern", help="a regular expression, e.g. \"glycoly[sz]is\"")
+    p.add_argument("--case-sensitive", action="store_true")
+    p.set_defaults(func=cmd_search)
+
+    p = sub.add_parser("check", help="say what is missing before a run")
+    p.add_argument("folder", help=FOLDER)
+    p.add_argument("--model")
+    p.set_defaults(func=cmd_check)
+
+    for stage, text in (("extract", "extract records from papers not extracted yet"),
+                        ("judge", "audit extracted records with a second model")):
+        p = sub.add_parser(stage, help=text)
+        p.add_argument("folder", help=FOLDER)
+        p.add_argument("--model", help='litellm model string, or rwth/<name> '
+                                       '(default: the model chosen in Settings)')
+        p.add_argument("--redo", action="store_true", help="also re-run papers already done")
+        _filters(p)
+        p.set_defaults(func=lambda a, s=stage: _run_stage(a, s))
+
+    p = sub.add_parser("export", help="write records to .csv / .json, or a .zip bundle")
+    p.add_argument("folder", help=FOLDER)
+    p.add_argument("output", help="a .csv or .json file of records, or a .zip bundle")
+    p.add_argument("--include-text", action="store_true",
+                   help="put the paper text in a .zip bundle (check the papers' licences)")
+    p.add_argument("--include-files", action="store_true",
+                   help="put the source files in a .zip bundle (check the papers' licences)")
+    _filters(p)
+    p.set_defaults(func=cmd_export)
+    return parser
+
+
+def main(argv=None) -> int:
+    args = build_parser().parse_args(argv)
+    try:
+        return args.func(args) or 0
+    except (ValueError, RuntimeError, FileNotFoundError) as e:
+        print(f"error: {e}", file=sys.stderr)
+        return 2
+
+
+if __name__ == "__main__":
+    sys.exit(main())