paratext-cli 0.2.4__tar.gz → 0.4.0__tar.gz

{paratext_cli-0.2.4 → paratext_cli-0.4.0}/.helical/tasks/PTX-4 genericise-the-defaults-away-from-local-inference-servers.md

@@ -4,11 +4,10 @@ title: Genericise the defaults away from local inference servers
 status: done
 horizon: now
 flow: clear
-priority: high
 labels:
   - docs
 created: '2026-06-28'
-updated: '2026-09-02'
+updated: '2026-09-03'
 ---
 
 Defaults naming Lemonade and LM Studio make the framework look like it only runs against one person's setup.

{paratext_cli-0.2.4 → paratext_cli-0.4.0}/AGENTS.md

@@ -267,35 +267,39 @@ alongside `_api_export_catalogue`.
 - **Bump `schema_version`** when a schema changes.
 - **Commits:** short messages, no Co-Authored-By trailer.
 
-## Backlog tasks are public
+## Tasks in this repo are public
 
-This repository is public, and `backlog/` is committed with it. Task titles,
+This repository is public, and `.helical/` is committed with it. Task titles,
 descriptions and notes are visible to anyone.
 
 Keep repo tasks to engineering work. Anything naming NLS internals, staff,
 suppliers, unannounced plans or licensing negotiations belongs in the
-`paratext-nls` backlog or the roadmap at `/opt/backlog` instead.
+`paratext-nls` board or the `admin` project instead.
 
-<!-- BACKLOG.MD GUIDELINES START -->
-<!-- backlog.md-instructions-version: 1.50.1 -->
-<CRITICAL_INSTRUCTION>
+<!-- HELICAL START -->
+## Tasks
 
-## Backlog.md Workflow
+This project's tasks live in `.helical/tasks/` and are managed with the `helical`
+command. Read them before starting, and record what you find.
 
-This project uses Backlog.md for task and project management.
-
-**For every user request in this project, run `backlog instructions overview` before answering or taking action.**
-
-Use the overview to decide whether to search, read, create, or update Backlog tasks.
+```bash
+helical ls -p <project>              # open work here
+helical show <ID>                    # one task in full
+helical set <ID> --status doing      # todo | doing | done
+helical set <ID> --horizon next      # now | next | future
+helical set <ID> --flow waiting --blocker "why"
+helical new "Title" -p <project> --horizon next
+```
 
-Before task lifecycle actions, read the matching detailed guide:
-- `backlog instructions task-creation` before creating or splitting tasks
-- `backlog instructions task-execution` before planning, changing status or assignee, adding a plan or implementation notes, or implementing task work
-- `backlog instructions task-finalization` before checking acceptance criteria, writing final summaries, or moving tasks to terminal statuses
+`--json` works on every command. A task carries both a **status** (how far along)
+and a **horizon** (when it matters); the routemap across all projects is built from
+horizons, so set one deliberately.
 
-Use `backlog <command> --help` before running unfamiliar commands. Help shows options, fields, and examples.
+**Flow** says who can unstick a task: `waiting` sits with someone else and needs
+chasing, `blocked` sits with us and needs a decision. Neither is accepted without
+`--blocker` explaining it.
 
-Do not edit Backlog task, draft, document, decision, or milestone markdown files directly. Use the `backlog` CLI so metadata, relationships, and history stay consistent.
+Labels come from a fixed vocabulary; `helical new --help` lists it.
 
-</CRITICAL_INSTRUCTION>
-<!-- BACKLOG.MD GUIDELINES END -->
+Boards: <https://helical.ai.nls.uk> · read-only <https://projects.ai.nls.uk>
+<!-- HELICAL END -->

{paratext_cli-0.2.4 → paratext_cli-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: paratext-cli
-Version: 0.2.4
+Version: 0.4.0
 Summary: Modular, project-based metadata extraction from digitised library/archive collections with a multimodal model
 License-Expression: Apache-2.0
 License-File: LICENSE

{paratext_cli-0.2.4 → paratext_cli-0.4.0}/pyproject.toml

@@ -3,7 +3,7 @@
 # the plain name on PyPI belongs to an unrelated project, so the distribution
 # carries the `-cli` suffix.
 name = "paratext-cli"
-version = "0.2.4"
+version = "0.4.0"
 description = "Modular, project-based metadata extraction from digitised library/archive collections with a multimodal model"
 readme = "README.md"
 # Core is pure-Python + numpy/pillow/pydantic. The [detector] extra needs

{paratext_cli-0.2.4 → paratext_cli-0.4.0}/src/paratext/__init__.py

@@ -1,3 +1,3 @@
 """paratext — metadata extraction from digitised collections with a multimodal model."""
 
-__version__ = "0.2.4"
+__version__ = "0.4.0"

{paratext_cli-0.2.4 → paratext_cli-0.4.0}/src/paratext/cli.py

@@ -427,6 +427,22 @@ def _cmd_carbon(args: argparse.Namespace) -> int:
 
 
 # ── Review ────────────────────────────────────────────────────────────────
+def _workshop_endpoint(args: argparse.Namespace) -> dict:
+    """Where workshop runs send their model calls. Comes from the same
+    config/env layer as `run`, so a Space configures it with PARATEXT_* vars."""
+    merged = {**HARDCODED_DEFAULTS, **coerce_paths(load_defaults(None))}
+    base_url, note = normalise_base_url(str(merged.get("base_url") or ""))
+    if note:
+        print(note)
+    source = args.workshop_source or merged.get("source")
+    return {
+        "base_url": base_url,
+        "api_key": merged.get("api_key"),
+        "model": merged.get("model"),
+        "source": str(source) if source else None,
+    }
+
+
 def _cmd_review(args: argparse.Namespace) -> int:
     from .review import serve
 
@@ -439,6 +455,8 @@ def _cmd_review(args: argparse.Namespace) -> int:
         # A missing default root means "nothing run yet", not a mistake — serve
         # anyway so Projects and the homepage guidance are reachable.
         allow_empty=args.data_dir == REVIEW_ROOT,
+        workshop=args.workshop,
+        endpoint=_workshop_endpoint(args) if args.workshop else None,
     )
     return 0
 
@@ -779,6 +797,13 @@ def _build_parser() -> tuple[
                          "nginx; use 0.0.0.0 to expose directly on the network)")
     rv.add_argument("--db", type=Path, default=None,
                     help="Annotations SQLite path (default: <data_dir>/annotations.db)")
+    rv.add_argument("--workshop", metavar="DIR", default=None,
+                    help="Workshop mode: give each browser its own workspace under DIR "
+                         "(prompt, fields, rounds and verdicts), seeded with the rounds "
+                         "served here")
+    rv.add_argument("--workshop-source", metavar="DIR", default=None,
+                    help="With --workshop: the images attendees' runs read from "
+                         "(default: the configured `source`)")
     rv.add_argument("--no-open", action="store_true", help="Don't open a browser")
     rv.set_defaults(func=_cmd_review)
 

{paratext_cli-0.2.4 → paratext_cli-0.4.0}/src/paratext/packaging.py

@@ -25,7 +25,7 @@ from pathlib import Path
 from PIL import Image
 
 from .io import iter_records, read_provenance
-from .projects import KEEP, Curation, build_view, get_project
+from .projects import KEEP, Curation, Project, build_view, get_project
 
 logger = logging.getLogger(__name__)
 
@@ -71,9 +71,14 @@ def package(
     *,
     fresh: bool = False,
     image_max_size: int = 1024,
+    proj: "Project | None" = None,
 ) -> tuple[int, dict[str, int]]:
-    """Write samples.json + images/ for `paratext review`. Returns (kept, skipped_by_reason)."""
-    proj = get_project(project)
+    """Write samples.json + images/ for `paratext review`. Returns (kept, skipped_by_reason).
+
+    `proj` overrides entry-point lookup, for a project built at runtime rather
+    than installed — workshop mode, where the schema is edited in the browser.
+    """
+    proj = proj or get_project(project)
     if fresh and out.exists():
         shutil.rmtree(out)
     (out / "images").mkdir(parents=True, exist_ok=True)

paratext_cli-0.4.0/src/paratext/review/runs.py

@@ -0,0 +1,171 @@
+"""Background extraction runs for workshop mode.
+
+`extract.run` is built for a sweep: resume, tqdm, a whole collection. A workshop
+run is five cards that somebody is watching, so this is a small loop that
+reports progress per card and can be polled.
+
+One job per session at a time. A run costs money, so the caps here are the
+thing standing between a public Space and someone's credit balance.
+"""
+
+from __future__ import annotations
+
+import logging
+import threading
+import time
+import uuid
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from openai import OpenAI
+
+from ..extract import _prompt_hash
+from ..io import append_jsonl, write_provenance_header
+from ..packaging import package
+from ..projects import Project
+from ..runner import call_structured
+from ..sources import image_source
+from ..workshop import build_schema
+
+logger = logging.getLogger(__name__)
+
+MAX_CARDS = 5          # per run
+MAX_RUNS_PER_SESSION = 40
+
+
+@dataclass
+class Job:
+    id: str
+    total: int
+    done: int = 0
+    status: str = "running"      # running | done | error
+    error: str = ""
+    round_name: str = ""
+    failures: list[str] = field(default_factory=list)
+
+    def as_dict(self) -> dict:
+        return {
+            "id": self.id,
+            "total": self.total,
+            "done": self.done,
+            "status": self.status,
+            "error": self.error,
+            "round": self.round_name,
+            "failures": self.failures,
+        }
+
+
+class Runs:
+    """Tracks one job per session, and how many runs a session has spent."""
+
+    def __init__(self, max_runs: int = MAX_RUNS_PER_SESSION):
+        self._jobs: dict[str, Job] = {}
+        self._by_session: dict[str, str] = {}
+        self._spent: dict[str, int] = {}
+        self._lock = threading.Lock()
+        self.max_runs = max_runs
+
+    def active(self, session_id: str) -> Job | None:
+        with self._lock:
+            job = self._jobs.get(self._by_session.get(session_id, ""))
+        return job if job and job.status == "running" else None
+
+    def get(self, job_id: str) -> Job | None:
+        with self._lock:
+            return self._jobs.get(job_id)
+
+    def spent(self, session_id: str) -> int:
+        with self._lock:
+            return self._spent.get(session_id, 0)
+
+    def start(self, session_id: str, total: int, work) -> Job:
+        """Register a job and run `work(job)` on a thread."""
+        with self._lock:
+            if self._spent.get(session_id, 0) >= self.max_runs:
+                raise ValueError(
+                    f"This session has used its {self.max_runs} runs. "
+                    f"Reload with a fresh session to continue."
+                )
+            self._spent[session_id] = self._spent.get(session_id, 0) + 1
+            job = Job(id=uuid.uuid4().hex[:12], total=total)
+            self._jobs[job.id] = job
+            self._by_session[session_id] = job.id
+
+        def target():
+            try:
+                work(job)
+                job.status = "done" if job.status == "running" else job.status
+            except Exception as e:                      # noqa: BLE001 — surfaced to the UI
+                logger.exception("workshop run failed")
+                job.status, job.error = "error", str(e)
+
+        threading.Thread(target=target, daemon=True).start()
+        return job
+
+
+def workshop_project(name: str, prompt: str, fields: list[dict]) -> Project:
+    """A Project assembled from what the attendee typed, not from an installed
+    package."""
+    return Project(
+        name=name,
+        schema_version="v1",
+        prompt=prompt,
+        schema=build_schema(fields),
+        source=image_source(),
+    )
+
+
+def extract_and_package(
+    job: Job,
+    *,
+    proj: Project,
+    source: Path,
+    output: Path,
+    review_out: Path,
+    base_url: str,
+    api_key: str,
+    model: str,
+    cards: int,
+) -> None:
+    """Run the model over `cards` samples, then package a round. Updates `job`
+    as it goes so the browser can draw a progress bar."""
+    client = OpenAI(base_url=base_url, api_key=api_key)
+    output.parent.mkdir(parents=True, exist_ok=True)
+    if output.exists():
+        output.unlink()          # a workshop run is always a fresh round
+    write_provenance_header(output, {
+        "project": proj.name,
+        "schema_version": proj.schema_version,
+        "prompt_hash": _prompt_hash(proj.prompt),
+        "prompt": proj.prompt,
+        "model": model,
+        "base_url": base_url,
+    })
+
+    for sample in proj.source.iter_samples(source, cards):
+        t0 = time.monotonic()
+        try:
+            parsed = call_structured(
+                client, model=model, prompt=proj.prompt, images=sample.images,
+                schema=proj.schema, image_max_size=proj.image_max_size,
+                image_quality=proj.image_quality,
+            )
+            append_jsonl(output, {
+                "id": sample.id,
+                "extraction": parsed.model_dump(),
+                "metadata": sample.metadata,
+                "elapsed_s": round(time.monotonic() - t0, 3),
+            })
+        except Exception as e:                          # noqa: BLE001
+            job.failures.append(f"{sample.id}: {e}")
+            logger.warning("[%s] failed: %s", sample.id, e)
+        finally:
+            job.done += 1
+
+    if job.done and len(job.failures) == job.done:
+        raise RuntimeError(f"every card failed — {job.failures[0]}")
+
+    kept, _ = package(output, review_out, proj.name, fresh=True, proj=proj)
+    job.round_name = review_out.name
+    if not kept:
+        raise RuntimeError("nothing survived packaging")

{paratext_cli-0.2.4 → paratext_cli-0.4.0}/src/paratext/review/server.py

@@ -18,6 +18,7 @@ import io
 import json
 import logging
 import webbrowser
+from http.cookies import SimpleCookie
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 from pathlib import Path
 from urllib.parse import parse_qs, unquote, urlparse
@@ -32,6 +33,9 @@ from ..datasets import (
     schema_history,
 )
 from ..store import Store, default_db_path
+from ..workshop import normalise_fields
+from .runs import MAX_CARDS, Runs, extract_and_package, workshop_project
+from .sessions import COOKIE_NAME, Sessions
 
 logger = logging.getLogger(__name__)
 
@@ -67,18 +71,141 @@ _MIME = {
 
 
 class Handler(BaseHTTPRequestHandler):
-    data_dir: Path
-    store: Store
+    # The single-tenant defaults. In workshop mode `data_dir` and `store` below
+    # resolve per request instead, so one container serves a room of people
+    # without them overwriting each other.
+    base_data_dir: Path
+    base_store: Store
+    sessions: Sessions | None = None
+    runs: Runs = Runs()
+    workshop_defaults: dict = {}
 
     def log_message(self, *a):  # quiet by default
         pass
 
+    # -- workshop mode --
+    def _workshop_or_404(self):
+        session = self._session()
+        if session is None:
+            raise FileNotFoundError("workshop mode is not enabled on this server")
+        return session
+
+    def _api_workshop_state(self):
+        """The attendee's prompt and fields, falling back to the defaults this
+        server was started with."""
+        session = self._workshop_or_404()
+        state = session.read_state()
+        self._json({
+            "session": session.id,
+            "prompt": state.get("prompt", type(self).workshop_defaults.get("prompt", "")),
+            "fields": state.get("fields", type(self).workshop_defaults.get("fields", [])),
+            "runs_used": self.runs.spent(session.id),
+            "max_runs": self.runs.max_runs,
+            "max_cards": MAX_CARDS,
+        })
+
+    def _api_workshop_save(self):
+        session = self._workshop_or_404()
+        body = self._body()
+        fields = normalise_fields(body.get("fields") or [])
+        if not fields:
+            return self._json({"error": "a schema needs at least one field"}, 400)
+        session.write_state({"prompt": body.get("prompt") or "", "fields": fields})
+        self._json({"ok": True, "fields": fields})
+
+    def _api_workshop_run(self):
+        session = self._workshop_or_404()
+        if self.runs.active(session.id):
+            return self._json({"error": "a run is already going"}, 409)
+
+        body = self._body()
+        prompt = (body.get("prompt") or "").strip()
+        fields = normalise_fields(body.get("fields") or [])
+        if not prompt:
+            return self._json({"error": "the prompt is empty"}, 400)
+        if not fields:
+            return self._json({"error": "a schema needs at least one field"}, 400)
+        cards = max(1, min(int(body.get("cards") or MAX_CARDS), MAX_CARDS))
+
+        cfg = type(self).workshop_defaults
+        if not cfg.get("source") or not Path(cfg["source"]).is_dir():
+            return self._json({"error": "this server has no source images configured"}, 400)
+
+        session.write_state({"prompt": prompt, "fields": fields})
+        name = cfg.get("project", "workshop")
+        proj = workshop_project(name, prompt, fields)
+        # A new round per prompt: the round number is just how many the attendee
+        # has made, so r1/r2/r3 read as their own iteration history.
+        existing = sorted(session.data_dir.glob(f"{name}-r*"))
+        n = len(existing) + 1
+        review_out = session.data_dir / f"{name}-r{n}"
+
+        try:
+            job = self.runs.start(
+                session.id, cards,
+                lambda j: extract_and_package(
+                    j, proj=proj, source=Path(cfg["source"]),
+                    output=session.dir / "output" / f"{name}-r{n}.jsonl",
+                    review_out=review_out,
+                    base_url=cfg["base_url"], api_key=cfg["api_key"],
+                    model=cfg["model"], cards=cards,
+                ),
+            )
+        except ValueError as e:
+            return self._json({"error": str(e)}, 429)
+        self._json(job.as_dict(), 202)
+
+    def _api_workshop_job(self, job_id):
+        self._workshop_or_404()
+        job = self.runs.get(job_id)
+        if not job:
+            raise FileNotFoundError(f"no such job: {job_id}")
+        self._json(job.as_dict())
+
+    # -- per-request session --
+    def _session(self):
+        """The caller's workshop session, creating one on first request. None
+        outside workshop mode, where the server is single-tenant."""
+        if self.sessions is None:
+            return None
+        if getattr(self, "_session_cache", None) is None:
+            cookie = SimpleCookie(self.headers.get("cookie") or "")
+            morsel = cookie.get(COOKIE_NAME)
+            session, created = self.sessions.get_or_create(morsel.value if morsel else None)
+            self._session_cache = session
+            self._set_cookie = created
+        return self._session_cache
+
+    @property
+    def data_dir(self) -> Path:
+        session = self._session()
+        return session.data_dir if session else type(self).base_data_dir
+
+    @property
+    def store(self) -> Store:
+        session = self._session()
+        if not session:
+            return type(self).base_store
+        if getattr(self, "_store_cache", None) is None:
+            self._store_cache = session.store()
+        return self._store_cache
+
+    def _session_headers(self) -> None:
+        """Hand the browser its session on the response that created it."""
+        if getattr(self, "_set_cookie", False):
+            self.send_header(
+                "set-cookie",
+                f"{COOKIE_NAME}={self._session_cache.id}; Path=/; SameSite=Lax; Max-Age=86400",
+            )
+            self._set_cookie = False
+
     # -- helpers --
     def _json(self, obj, status=200):
         body = json.dumps(obj).encode()
         self.send_response(status)
         self.send_header("content-type", "application/json")
         self.send_header("content-length", str(len(body)))
+        self._session_headers()
         self.end_headers()
         self.wfile.write(body)
 
@@ -88,6 +215,7 @@ class Handler(BaseHTTPRequestHandler):
         self.send_header("content-length", str(len(body)))
         for k, v in (headers or {}).items():
             self.send_header(k, v)
+        self._session_headers()
         self.end_headers()
         self.wfile.write(body)
 
@@ -129,6 +257,10 @@ class Handler(BaseHTTPRequestHandler):
                 return self._api_prompts(self._dataset(qs))
             if path == "/api/schema":
                 return self._api_schema(self._dataset(qs))
+            if path == "/api/workshop/state":
+                return self._api_workshop_state()
+            if path.startswith("/api/workshop/job/"):
+                return self._api_workshop_job(path.rsplit("/", 1)[1])
             if path == "/api/export/fields":
                 return self._api_export_fields(self._dataset(qs), qs)
             if path in ("/api/export/marc", "/api/export/dc"):
@@ -164,6 +296,10 @@ class Handler(BaseHTTPRequestHandler):
             return self._api_export_catalogue(
                 self._dataset(qs), u.path.rsplit("/", 1)[1], qs, mapping=body.get("mapping")
             )
+        if u.path == "/api/workshop/state":
+            return self._api_workshop_save()
+        if u.path == "/api/workshop/run":
+            return self._api_workshop_run()
         if u.path == "/api/oauth/hf/exchange":
             return self._api_hf_exchange()
         if u.path == "/api/export/hf":
@@ -623,6 +759,40 @@ class Handler(BaseHTTPRequestHandler):
         )
 
 
+def _workshop_defaults(datasets: list[dict], endpoint: dict) -> dict:
+    """Seed an attendee's starting prompt and fields from the newest round being
+    served, so the Space needs no separate copy of either. The endpoint comes
+    from the usual config/env layer."""
+    newest = max(datasets, key=lambda d: d.get("round") or 0, default=None)
+    prompt, fields, project = "", [], "workshop"
+    if newest:
+        project = newest.get("base") or project
+        try:
+            prov = json.loads((newest["dir"] / "provenance.json").read_text())
+            prompt = prov.get("prompt", "")
+        except (OSError, json.JSONDecodeError, KeyError):
+            pass
+        try:
+            view = json.loads((newest["dir"] / "view.json").read_text())
+            for panel in view.get("panels", []):
+                if panel.get("source") == "model_output":
+                    fields = [
+                        {"name": f["key"], "type": "text", "description": ""}
+                        for f in panel.get("fields", [])
+                    ]
+        except (OSError, json.JSONDecodeError):
+            pass
+    return {
+        "prompt": prompt,
+        "fields": fields,
+        "project": project,
+        "source": endpoint.get("source"),
+        "base_url": endpoint.get("base_url"),
+        "api_key": endpoint.get("api_key") or "EMPTY",
+        "model": endpoint.get("model"),
+    }
+
+
 def serve(
     data_dir: Path,
     port: int = DEFAULT_PORT,
@@ -630,6 +800,8 @@ def serve(
     host: str = "127.0.0.1",
     db_path: Path | None = None,
     allow_empty: bool = False,
+    workshop: Path | str | None = None,
+    endpoint: dict | None = None,
 ) -> None:
     """Serve the review UI over ``data_dir``.
 
@@ -648,11 +820,20 @@ def serve(
                 f"or pass a dataset directory: paratext review <dir>"
             )
         data_dir.mkdir(parents=True, exist_ok=True)
-    Handler.data_dir = data_dir
+    Handler.base_data_dir = data_dir
     # Default the annotation store to <data_dir>/annotations.db; --db can point it
     # elsewhere (e.g. an existing DB when swapping in for another review app).
-    Handler.store = Store(default_db_path(data_dir, Path(db_path).resolve() if db_path else None))
+    Handler.base_store = Store(
+        default_db_path(data_dir, Path(db_path).resolve() if db_path else None)
+    )
     datasets = discover_datasets(data_dir)
+    # Workshop mode: every browser gets its own workspace, seeded with the rounds
+    # shipped here so an attendee's own runs sit beside the worked example.
+    Handler.sessions = (
+        Sessions(Path(workshop).resolve(), examples=[d["dir"] for d in datasets])
+        if workshop
+        else None
+    )
     httpd = ThreadingHTTPServer((host, port), Handler)
     # When bound to all interfaces there's no single canonical URL; show
     # localhost for clicking and note the bind address.
@@ -663,6 +844,12 @@ def serve(
     print(
         f"  data dir: {data_dir}  ({len(datasets)} dataset(s): {names})"
     )
+    if workshop:
+        Handler.workshop_defaults = _workshop_defaults(datasets, endpoint or {})
+        d = Handler.workshop_defaults
+        print(f"  workshop mode: per-session workspaces under {Handler.sessions.root}")
+        print(f"    endpoint {d.get('model') or '(no model set)'} at {d.get('base_url')}")
+        print(f"    source   {d.get('source') or '(none — runs disabled)'}")
     if open_browser:
         try:
             webbrowser.open(url)