lorebase-scribe 0.2.3__py3-none-any.whl

lorebase_scribe/extraction/values.py

@@ -0,0 +1,282 @@
+"""Rule-based Attribute values (tier 0), sentence by sentence.
+
+In each sentence, dates (ISO and written forms such as `8 February 2024`), numbers (with
+thousands separators) and enum values are found. A value goes to an Attribute of a Record when
+a cue word of that Attribute (from its api name, display name and longer description words)
+precedes it by at most `value_cue_words` words in the sentence; enum values written as
+upper-case codes (`DDP`, `EUR`) need no cue. The Records a sentence can describe are the ones
+it names; a sentence that names none describes the Record of its section heading. The closer
+the cue (and the Record), the higher the confidence, up to `confidence.value_max`. Each
+Attribute of a Record takes at most one value per sentence and each occurrence goes to at most
+one Attribute. Booleans and free text are left to tier 2, except the title of a Record whose
+key appears in a heading, which is taken from the heading.
+
+These rules propose; reviewers decide. Their confidence stays below the default
+`auto_accept_threshold`, so nothing they find is accepted without review by default.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Sequence
+from dataclasses import dataclass
+from datetime import date
+from typing import Any
+
+from lorebase_core.model import Attribute
+from lorebase_core.types import AttributeType, EncodingError, encode_value
+from lorebase_scribe.extraction.candidates import Evidence, Resolved, ValueCandidate
+from lorebase_scribe.extraction.ontology import Ontology, humanize
+from lorebase_scribe.extraction.settings import ExtractionSettings
+
+_MONTHS = {
+    name: index
+    for index, names in enumerate(
+        (
+            ("jan", "january"),
+            ("feb", "february"),
+            ("mar", "march"),
+            ("apr", "april"),
+            ("may",),
+            ("jun", "june"),
+            ("jul", "july"),
+            ("aug", "august"),
+            ("sep", "sept", "september"),
+            ("oct", "october"),
+            ("nov", "november"),
+            ("dec", "december"),
+        ),
+        start=1,
+    )
+    for name in names
+}
+_MONTH = (
+    r"(Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|June?|July?|Aug(?:ust)?"
+    r"|Sept?(?:ember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)"
+)
+_ISO_DATE = re.compile(r"(?<![\w-])(\d{4})-(\d{2})-(\d{2})(?![\w-])")
+_DMY = re.compile(rf"(?<!\w)(\d{{1,2}})\s+{_MONTH}\.?,?\s+(\d{{4}})(?!\w)")
+_MDY = re.compile(rf"(?<!\w){_MONTH}\.?\s+(\d{{1,2}}),?\s+(\d{{4}})(?!\w)")
+_MONTH_YEAR = re.compile(rf"(?<!\w){_MONTH}\.?,?\s+(\d{{4}})(?!\w)")
+_NUMBER = re.compile(r"(?<![\w.,-])(\d{1,3}(?:,\d{3})+|\d+)(?:\.(\d+))?(?![\w-]|[.,]\d)")
+_TOKEN = re.compile(r"\w+")
+_GENERIC = frozenset(
+    {
+        *("date", "day", "days", "time", "first", "last", "name", "type", "kind", "number"),
+        *("code", "value", "iso", "when", "units", "unit"),
+    }
+)
+_DATE_TYPES = (AttributeType.DATE,)
+_NUMBER_TYPES = (AttributeType.INTEGER, AttributeType.DECIMAL, AttributeType.FLOAT)
+
+
+@dataclass(frozen=True, slots=True)
+class Occurrence:
+    start: int
+    end: int
+    kind: str  # date | number | enum
+    value: Any
+
+
+def find_dates(text: str, start: int, end: int) -> list[Occurrence]:
+    """Full dates in `text[start:end]`, as ISO strings."""
+    found: list[Occurrence] = []
+    for match in _ISO_DATE.finditer(text, start, end):
+        _add_date(found, match, int(match[1]), int(match[2]), int(match[3]))
+    for match in _DMY.finditer(text, start, end):
+        _add_date(found, match, int(match[3]), _MONTHS[match[2].lower()], int(match[1]))
+    for match in _MDY.finditer(text, start, end):
+        _add_date(found, match, int(match[3]), _MONTHS[match[1].lower()], int(match[2]))
+    return found
+
+
+def _add_date(found: list[Occurrence], match: re.Match[str], y: int, m: int, d: int) -> None:
+    try:
+        found.append(Occurrence(match.start(), match.end(), "date", date(y, m, d).isoformat()))
+    except ValueError:
+        return
+
+
+def find_numbers(
+    text: str, start: int, end: int, taken: Sequence[tuple[int, int]]
+) -> list[Occurrence]:
+    """Numbers in `text[start:end]` outside `taken` spans and month-year phrases, as strings
+    without thousands separators."""
+    blocked = [*taken, *((m.start(), m.end()) for m in _MONTH_YEAR.finditer(text, start, end))]
+    found: list[Occurrence] = []
+    for match in _NUMBER.finditer(text, start, end):
+        if any(match.start() < e and match.end() > s for s, e in blocked):
+            continue
+        whole = match[1].replace(",", "")
+        value = f"{whole}.{match[2]}" if match[2] else whole
+        found.append(Occurrence(match.start(), match.end(), "number", value))
+    return found
+
+
+def _enums(text: str, start: int, end: int, attribute: Attribute) -> list[Occurrence]:
+    found: list[Occurrence] = []
+    for option in attribute.type.enum_values or ():
+        if len(option) < 2:
+            continue
+        pattern = re.compile(rf"(?<![\w-]){re.escape(option)}(?![\w-])")
+        found.extend(
+            Occurrence(m.start(), m.end(), "enum", option)
+            for m in pattern.finditer(text, start, end)
+        )
+    return found
+
+
+def _cue_matches(token: str, cue: str) -> bool:
+    return token == cue or (len(cue) >= 3 and token.startswith(cue))
+
+
+def _encode(attribute: Attribute, occurrence: Occurrence) -> Any:
+    name = attribute.type.name
+    raw: Any = occurrence.value
+    if occurrence.kind == "number":
+        if name is AttributeType.INTEGER:
+            if "." in raw:
+                return None
+            raw = int(raw)
+        elif name is AttributeType.FLOAT:
+            raw = float(raw)
+    try:
+        return encode_value(attribute.type, raw)
+    except EncodingError:
+        return None
+
+
+def _compatible(attribute: Attribute, occurrence: Occurrence) -> bool:
+    name = attribute.type.name
+    if occurrence.kind == "date":
+        return name in _DATE_TYPES
+    if occurrence.kind == "number":
+        return name in _NUMBER_TYPES
+    return name is AttributeType.ENUM and occurrence.value in (attribute.type.enum_values or ())
+
+
+@dataclass(frozen=True, slots=True)
+class _Option:
+    score: float
+    owner_distance: int
+    owner: Resolved
+    attribute: Attribute
+    occurrence: Occurrence
+
+
+def sentence_values(
+    ontology: Ontology,
+    owners: Sequence[Resolved],
+    text: str,
+    sentence: tuple[int, int],
+    *,
+    excluded: Sequence[tuple[int, int]],
+    settings: ExtractionSettings,
+) -> list[ValueCandidate]:
+    """Values the sentence `text[sentence[0]:sentence[1]]` gives the Records `owners` name."""
+    start, end = sentence
+    dates = find_dates(text, start, end)
+    numbers = find_numbers(text, start, end, [*excluded, *((d.start, d.end) for d in dates)])
+    tokens = [(m.start(), m.group().lower()) for m in _TOKEN.finditer(text, start, end)]
+    limit = settings.value_cue_words
+    names = {view.id: set(humanize(view.api_name).split()) for view in ontology.kinds.values()}
+    options: list[_Option] = []
+    for owner in owners:
+        view = ontology.kinds[owner.kind_id]
+        own = names[view.id]
+        others = set().union(*(n for k, n in names.items() if k != view.id)) - own
+        skip = {a.id for a in (view.key_attribute, view.title_attribute) if a is not None}
+        for attribute in view.attributes:
+            if attribute.id in skip:
+                continue
+            cues = [c for c in view.cues.get(attribute.id, ()) if c not in own | _GENERIC]
+            pool = (
+                _enums(text, start, end, attribute)
+                if attribute.type.name is AttributeType.ENUM
+                else [*dates, *numbers]
+            )
+            for occurrence in pool:
+                if not _compatible(attribute, occurrence):
+                    continue
+                before = [
+                    i for i, (position, _t) in enumerate(tokens) if position < occurrence.start
+                ]
+                cue_gaps = [
+                    len(before) - 1 - index
+                    for index in before
+                    if any(_cue_matches(tokens[index][1], cue) for cue in cues)
+                    # "supplier status" announces a value of a Supplier, not of this Kind
+                    and not (index > 0 and tokens[index - 1][1] in others)
+                ]
+                gap = min((g for g in cue_gaps if g <= limit), default=None)
+                if gap is None:
+                    if occurrence.kind == "enum" and str(occurrence.value).isupper():
+                        gap = limit
+                    else:
+                        continue
+                score = settings.confidence.value_max * (1 - 0.5 * gap / max(limit, 1))
+                distance = abs(owner.mention.start - occurrence.start)
+                options.append(_Option(score, distance, owner, attribute, occurrence))
+    options.sort(
+        key=lambda o: (-o.score, o.owner_distance, o.occurrence.start, o.attribute.api_name)
+    )
+    used: set[tuple[str, str | None, str]] = set()
+    places: set[int] = set()
+    found: list[ValueCandidate] = []
+    for option in options:
+        slot = (option.owner.kind_id, option.owner.key, option.attribute.id)
+        if slot in used or option.occurrence.start in places:
+            continue
+        value = _encode(option.attribute, option.occurrence)
+        if value is None:
+            continue
+        used.add(slot)
+        places.add(option.occurrence.start)
+        mention = option.owner.mention
+        found.append(
+            ValueCandidate(
+                subject=option.owner,
+                attribute_id=option.attribute.id,
+                value=value,
+                evidence=Evidence(mention.document_id, mention.chunk, start, end, text[start:end]),
+                confidence=option.score,
+                tier=0,
+                model="rules",
+            )
+        )
+    return found
+
+
+def heading_title(
+    ontology: Ontology, subject: Resolved, text: str, *, settings: ExtractionSettings
+) -> ValueCandidate | None:
+    """The title of a Record named by key in a heading: the heading text."""
+    view = ontology.kinds[subject.kind_id]
+    attribute = view.title_attribute
+    if attribute is None or attribute.type.name not in (AttributeType.STRING, AttributeType.TEXT):
+        return None
+    if view.key_attribute is not None and attribute.id == view.key_attribute.id:
+        return None
+    mention = subject.mention
+    line_start = text.rfind("\n", 0, mention.start) + 1
+    line_end = text.find("\n", mention.end)
+    line_end = len(text) if line_end == -1 else line_end
+    heading = text[line_start:line_end].lstrip("#").strip()
+    if not heading:
+        return None
+    try:
+        value = encode_value(attribute.type, heading)
+    except EncodingError:
+        return None
+    offset = line_start + text[line_start:line_end].index(heading)
+    return ValueCandidate(
+        subject=subject,
+        attribute_id=attribute.id,
+        value=value,
+        evidence=Evidence(
+            mention.document_id, mention.chunk, offset, offset + len(heading), heading
+        ),
+        confidence=settings.confidence.value_max,
+        tier=0,
+        model="rules",
+    )

lorebase_scribe/jobs.py

@@ -0,0 +1,418 @@
+"""Job types Scribe runs and their handlers.
+
+| Type | Payload | Handler |
+|------|---------|---------|
+| `source_sync` | `{"source": id or api name, "dry_run": bool}`, Domain set | `SourceSync.run` |
+| `webhook_delivery` | none | `lorebase_store.webhooks.deliver_pending`, one pass |
+| `store.index` | `{"action", "name", "branch", "kind"}`, Domain set | `indexes.run_index_job` |
+| `gc` | `{"dry_run": bool, "grace_seconds": int}`, Domain set | `lorebase_store.gc.run_gc_job` |
+| `store.materialize` | `{"branch"}`, Domain set | `materialize.run_materialize_job` |
+| `store.branch_delete` | `{"branch"}`, Domain set | `materialize.run_branch_delete_job` |
+| `store.sensitivity_refresh` | `{"branch", "resume"}`, Domain set | `run_sensitivity_refresh_job` |
+| `extraction` | `{"documents", "tiers", "requested_by"}`, Domain set | `ExtractionRun.run` |
+| `embeddings` | `{"branches", "reindex", "resume"}` optional, Domain set | `EmbeddingPass.run` |
+| `audit_verify` | none | `lorebase_store.auth.AuditRepository.verify` over every chain |
+| `audit_checkpoint` | none | `checkpoints.publish_checkpoints` |
+| `vault.rotate` | `{"after": [domain id, token]}` optional | `lorebase_store.vault.rotate` |
+
+An `extraction` job reads `documents` (ids) with `tiers` (a list, or null for
+`scribe.extraction.tiers`); `requested_by` is recorded on the extraction job.
+
+An `embeddings` job that reads `scribe.embeddings.max_records_per_job` rows before its pass is
+complete queues its continuation (same payload plus `resume`) and succeeds.
+
+`store.materialize` and `store.branch_delete` copy or delete a large branch's rows in batches
+(`store.materialize.job_batch_rows` per transaction); at most `store.materialize.job_concurrency`
+of them run at once across workers, and one that has to wait queues itself again.
+
+`store.sensitivity_refresh` rewrites, in batches, the titles and search vectors a branch
+materialized before the default branch tagged an Attribute (`materialize.stale`); one whose
+branch stays locked queues its continuation (same branch plus `resume`).
+
+An `audit_verify` job succeeds whether the chains verify or not: its result and the
+`lorebase.audit.verifications` counter (outcome `intact` or `broken`, by stream) carry the
+answer, and a broken chain is logged at error level. It verifies the chains of actions and,
+under `reads`, those of the read stream.
+
+An `audit_checkpoint` job signs the head of every chain of `scribe.audit_checkpoints.streams`
+and stores one checkpoint per stream in the configured sink; `lorebase.audit.checkpoints`
+counts them by outcome. A failure fails the attempt (it is retried like any job).
+
+A `vault.rotate` job makes the vault rows of every Domain sealed or fingerprinted under
+earlier keys again under the current ones (`vault.rekey_batch` rows per transaction); after
+`vault.rekey_rows_per_job` rows it queues its continuation from where it stopped and succeeds.
+Without vault keys it fails at once.
+
+Index DDL runs on `JobContext.ddl_engine` (the maintenance role) when one is configured.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import dataclasses
+import logging
+from collections.abc import Callable, Iterable, Mapping, Sequence
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any
+
+import httpx
+from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker
+
+from lorebase_inference import InferenceSettings
+from lorebase_scribe.checkpoints import CheckpointSink, build_sink, publish_checkpoints
+from lorebase_scribe.embeddings import PERMANENT_ERRORS as EMBEDDING_ERRORS
+from lorebase_scribe.embeddings import EmbeddingModels, EmbeddingPass, Templates
+from lorebase_scribe.extraction.pipeline import PERMANENT_ERRORS as EXTRACTION_ERRORS
+from lorebase_scribe.extraction.pipeline import ExtractionModels, ExtractionRun
+from lorebase_scribe.settings import ScribeSettings
+from lorebase_scribe.sources import PERMANENT_ERRORS, SourceSync
+from lorebase_scribe.sources.connectors import ConnectorFactory, default_connector
+from lorebase_scribe.sources.sync import utc_now
+from lorebase_scribe.telemetry import AUDIT_CHECKPOINTS, AUDIT_VERIFICATIONS
+from lorebase_scribe.webhooks import HttpxSender
+from lorebase_scribe.worker import JobHandler, PermanentJobError
+from lorebase_store.auth import AuditRepository, AuditStream, Chains
+from lorebase_store.embeddings import SchemaCache
+from lorebase_store.gc import GcBusyError, GcError, run_gc_job
+from lorebase_store.indexes import run_index_job
+from lorebase_store.jobs import (
+    BRANCH_DELETE_JOB,
+    EMBEDDINGS_JOB,
+    EXTRACTION_JOB,
+    GC_JOB,
+    INDEX_JOB,
+    LINEAGE_BACKFILL_JOB,
+    MATERIALIZE_JOB,
+    SENSITIVITY_REFRESH_JOB,
+    SOURCE_SYNC_JOB,
+    VAULT_ROTATE_JOB,
+    JobRow,
+    embeddings_payload,
+    enqueue_job,
+)
+from lorebase_store.materialize import (
+    run_branch_delete_job,
+    run_lineage_backfill_job,
+    run_materialize_job,
+    run_sensitivity_refresh_job,
+)
+from lorebase_store.runtime import StoreRuntime
+from lorebase_store.storage import constraints_only
+from lorebase_store.vault import rotate
+from lorebase_store.webhooks import WebhookSettings, deliver_pending
+
+WEBHOOK_DELIVERY_JOB = "webhook_delivery"
+AUDIT_VERIFY_JOB = "audit_verify"
+AUDIT_CHECKPOINT_JOB = "audit_checkpoint"
+
+log = logging.getLogger("lorebase.scribe.jobs")
+
+JOB_TYPES = (
+    SOURCE_SYNC_JOB,
+    WEBHOOK_DELIVERY_JOB,
+    INDEX_JOB,
+    MATERIALIZE_JOB,
+    BRANCH_DELETE_JOB,
+    SENSITIVITY_REFRESH_JOB,
+    LINEAGE_BACKFILL_JOB,
+    GC_JOB,
+    EXTRACTION_JOB,
+    EMBEDDINGS_JOB,
+    AUDIT_VERIFY_JOB,
+    AUDIT_CHECKPOINT_JOB,
+    VAULT_ROTATE_JOB,
+)
+"""Every job type Scribe knows."""
+
+
+@dataclass(frozen=True)
+class JobContext:
+    """What the handlers share: database, store runtime, settings and the HTTP client.
+
+    `ddl_engine` runs per-Attribute index DDL: the maintenance engine
+    (`database.maintenance_dsn`) when configured, else `engine`. `extraction_models` replaces
+    the models built from `inference` (tests). `embedding_models` is the process's embedding
+    model, created on first use from `inference` unless given; `embedding_templates` the
+    process's cache of resolved embedding templates (one per pass when not given).
+    `checkpoint_sink` replaces the sink of `scribe.audit_checkpoints` (tests)."""
+
+    engine: AsyncEngine
+    sessions: async_sessionmaker[AsyncSession]
+    runtime: StoreRuntime
+    settings: ScribeSettings
+    webhooks: WebhookSettings
+    http: httpx.AsyncClient
+    worker_id: str
+    connectors: ConnectorFactory = default_connector
+    clock: Callable[[], datetime] = utc_now
+    ddl_engine: AsyncEngine | None = None
+    inference: InferenceSettings = field(default_factory=InferenceSettings)
+    extraction_models: Callable[[Sequence[int]], ExtractionModels] | None = None
+    embedding_models: EmbeddingModels | None = None
+    embedding_templates: SchemaCache[Templates] | None = None
+    checkpoint_sink: CheckpointSink | None = None
+
+    def embedding_pass(self) -> EmbeddingPass:
+        """An embedding pass sharing the process's model."""
+        models = self.embedding_models
+        if models is None:
+            raise RuntimeError("no embedding model: build the context with `embedding_models`")
+        return EmbeddingPass(
+            self.engine,
+            self.sessions,
+            models,
+            self.settings.embeddings,
+            dims=self.runtime.settings.embeddings.dims,
+            clock=self.clock,
+            templates=self.embedding_templates,
+        )
+
+
+def _domain(job: JobRow) -> str:
+    if job.domain_id is None:
+        raise PermanentJobError(f"a {job.type} job needs a Domain")
+    return job.domain_id
+
+
+def build_handlers(context: JobContext, types: Iterable[str] = ()) -> dict[str, JobHandler]:
+    """Handlers of `types` (default: every type); unknown types raise `ValueError`."""
+
+    async def source_sync(job: JobRow) -> Mapping[str, Any]:
+        reference = job.payload.get("source")
+        if not isinstance(reference, str) or not reference:
+            raise PermanentJobError("a source_sync job needs a `source`")
+        sync = SourceSync(
+            context.sessions,
+            context.runtime,
+            context.settings.sources,
+            principal=context.settings.worker.principal,
+            connectors=context.connectors,
+            clock=context.clock,
+        )
+        report = await sync.run(
+            _domain(job), reference, dry_run=bool(job.payload.get("dry_run", False))
+        )
+        return report.as_dict()
+
+    async def webhook_delivery(job: JobRow) -> Mapping[str, Any]:
+        # The pass bounds its attempts itself; past the lease of its deliveries another pass
+        # may send them again, so a pass still running then is abandoned (a failed attempt of
+        # the job) instead of holding a worker slot that its heartbeat keeps from going stale.
+        sender = HttpxSender(context.http, connect_timeout_s=context.webhooks.connect_timeout_s)
+        lease_s = context.webhooks.lease_s
+        budget = asyncio.timeout(lease_s)
+        try:
+            async with budget:
+                report = await deliver_pending(
+                    context.sessions,
+                    context.webhooks,
+                    sender,
+                    writes=context.runtime.settings.writes,
+                    outbox=context.runtime.settings.outbox,
+                    vault=context.runtime.vault,
+                )
+        except TimeoutError:
+            if not budget.expired():
+                raise
+            raise TimeoutError(f"the pass outlasted webhooks.lease_s ({lease_s:g}s)") from None
+        return dataclasses.asdict(report)
+
+    async def index(job: JobRow) -> Mapping[str, Any]:
+        _domain(job)
+        return await run_index_job(
+            context.ddl_engine or context.engine,
+            job,
+            constraints_only=constraints_only(context.runtime.settings),
+        )
+
+    async def materialize(job: JobRow) -> Mapping[str, Any]:
+        _domain(job)
+        return await run_materialize_job(
+            context.engine, job, runtime=context.runtime, clock=context.clock
+        )
+
+    async def branch_delete(job: JobRow) -> Mapping[str, Any]:
+        _domain(job)
+        return await run_branch_delete_job(
+            context.engine, job, runtime=context.runtime, clock=context.clock
+        )
+
+    async def sensitivity_refresh(job: JobRow) -> Mapping[str, Any]:
+        _domain(job)
+        return await run_sensitivity_refresh_job(
+            context.engine, job, runtime=context.runtime, clock=context.clock
+        )
+
+    async def lineage_backfill(job: JobRow) -> Mapping[str, Any]:
+        return await run_lineage_backfill_job(context.engine, job, runtime=context.runtime)
+
+    async def gc(job: JobRow) -> Mapping[str, Any]:
+        _domain(job)
+        try:
+            return await run_gc_job(context.engine, job, runtime=context.runtime)
+        except GcBusyError:
+            raise
+        except GcError as error:
+            raise PermanentJobError(str(error)) from None
+
+    async def extraction(job: JobRow) -> Mapping[str, Any]:
+        documents = job.payload.get("documents")
+        if not isinstance(documents, list) or not documents:
+            raise PermanentJobError("an extraction job needs `documents`")
+        tiers = job.payload.get("tiers")
+        if tiers is not None and not isinstance(tiers, list):
+            raise PermanentJobError("`tiers` must be a list")
+        run = ExtractionRun(
+            context.sessions,
+            context.runtime,
+            context.settings.extraction,
+            context.inference,
+            principal=context.settings.worker.principal,
+            models=context.extraction_models,
+            clock=context.clock,
+        )
+        requested_by = job.payload.get("requested_by")
+        report = await run.run(
+            _domain(job),
+            [str(d) for d in documents],
+            tiers=None if tiers is None else [int(t) for t in tiers],
+            requested_by=str(requested_by) if requested_by else None,
+        )
+        return report.as_dict()
+
+    async def embeddings(job: JobRow) -> Mapping[str, Any]:
+        domain = _domain(job)
+        branches = job.payload.get("branches")
+        if branches is not None and (
+            not isinstance(branches, list) or not all(isinstance(b, str) for b in branches)
+        ):
+            raise PermanentJobError("`branches` must be a list of branch names")
+        resume = job.payload.get("resume")
+        if resume is not None and (
+            not isinstance(resume, list)
+            or len(resume) != 3
+            or not all(isinstance(part, str) for part in resume)
+        ):
+            raise PermanentJobError("`resume` must be [branch, kind id, key]")
+        reindex = bool(job.payload.get("reindex", False))
+        report = await context.embedding_pass().run(
+            domain,
+            branches=branches,
+            reindex=reindex,
+            resume=None if resume is None else (resume[0], resume[1], resume[2]),
+        )
+        result = report.as_dict()
+        if report.resume is not None:
+            async with context.sessions() as session, session.begin():
+                result["continuation"] = await enqueue_job(
+                    session,
+                    EMBEDDINGS_JOB,
+                    domain_id=domain,
+                    payload=embeddings_payload(
+                        branches=branches, reindex=reindex, resume=report.resume
+                    ),
+                    unique=True,
+                )
+        return result
+
+    async def verify_stream(stream: AuditStream) -> dict[str, Any]:
+        async with context.sessions() as session, session.begin():
+            result = await AuditRepository(session, stream).verify(domain_id=Chains.ALL)
+        outcome = "intact" if result.ok else "broken"
+        AUDIT_VERIFICATIONS.add(1, {"outcome": outcome, "stream": stream.value})
+        report: dict[str, Any] = {
+            "intact": result.ok,
+            "checked": result.checked,
+            "chains": result.chains,
+            "last_seq": result.last_seq,
+        }
+        if not result.ok:
+            report["first_broken"] = {
+                "seq": result.bad_seq,
+                "domain_id": result.bad_domain_id,
+                "reason": result.reason,
+            }
+            log.error(
+                "audit chain broken", extra={**report["first_broken"], "stream": stream.value}
+            )
+        return report
+
+    async def audit_verify(job: JobRow) -> Mapping[str, Any]:
+        report = await verify_stream(AuditStream.ACTIONS)
+        report["reads"] = await verify_stream(AuditStream.READS)
+        return report
+
+    async def audit_checkpoint(job: JobRow) -> Mapping[str, Any]:
+        settings = context.settings.audit_checkpoints
+        if not settings.enabled:
+            raise PermanentJobError("scribe.audit_checkpoints is not enabled")
+        sink = context.checkpoint_sink or build_sink(settings, context.http)
+        try:
+            published = await publish_checkpoints(
+                context.sessions, settings, sink, clock=context.clock
+            )
+        except Exception:
+            AUDIT_CHECKPOINTS.add(1, {"outcome": "failed"})
+            raise
+        AUDIT_CHECKPOINTS.add(len(published), {"outcome": "published"})
+        return {"published": [item.as_dict() for item in published]}
+
+    async def vault_rotate(job: JobRow) -> Mapping[str, Any]:
+        keys = context.runtime.vault
+        if keys is None:
+            raise PermanentJobError("vault.rotate needs vault.token_key and vault.encryption_key")
+        after = job.payload.get("after")
+        if after is not None and (
+            not isinstance(after, list)
+            or len(after) != 2
+            or not all(isinstance(part, str) for part in after)
+        ):
+            raise PermanentJobError("`after` must be [domain id, token]")
+        settings = context.runtime.vault_settings
+        report = await rotate(
+            context.sessions,
+            keys,
+            batch=settings.rekey_batch,
+            max_rows=settings.rekey_rows_per_job,
+            after=None if after is None else (after[0], after[1]),
+        )
+        result = report.as_dict()
+        if report.after is not None:
+            async with context.sessions() as session, session.begin():
+                result["continuation"] = await enqueue_job(
+                    session,
+                    VAULT_ROTATE_JOB,
+                    payload={"after": list(report.after)},
+                    unique=True,
+                )
+        return result
+
+    available: dict[str, JobHandler] = {
+        SOURCE_SYNC_JOB: source_sync,
+        WEBHOOK_DELIVERY_JOB: webhook_delivery,
+        INDEX_JOB: index,
+        MATERIALIZE_JOB: materialize,
+        BRANCH_DELETE_JOB: branch_delete,
+        SENSITIVITY_REFRESH_JOB: sensitivity_refresh,
+        LINEAGE_BACKFILL_JOB: lineage_backfill,
+        GC_JOB: gc,
+        EXTRACTION_JOB: extraction,
+        EMBEDDINGS_JOB: embeddings,
+        AUDIT_VERIFY_JOB: audit_verify,
+        AUDIT_CHECKPOINT_JOB: audit_checkpoint,
+        VAULT_ROTATE_JOB: vault_rotate,
+    }
+    chosen = list(types) or list(available)
+    unknown = sorted(set(chosen) - available.keys())
+    if unknown:
+        raise ValueError(f"unknown job type(s): {', '.join(unknown)}")
+    return {name: available[name] for name in chosen}
+
+
+PERMANENT_JOB_ERRORS: tuple[type[Exception], ...] = (
+    *PERMANENT_ERRORS,
+    *EXTRACTION_ERRORS,
+    *EMBEDDING_ERRORS,
+)
+"""Handler failures the worker fails at once instead of retrying."""