jev-mcp-python 0.1.0__py3-none-any.whl

jev_mcp/extract/worker.py

@@ -0,0 +1,198 @@
+"""The production `RegexExecutor`: a pool of killable worker processes (ADR-0004, ADR-0016).
+
+A slot is one process (`python -m jev_mcp.extract.worker`) running one pattern at a time, fed
+length-prefixed pickles of plain values over its stdin and stdout, never the server's stdout. Each
+pattern arrives with one absolute deadline, taken before queue admission and spent across admission,
+worker start, IPC, matching, and the reply — no stage resets the clock (ADR-0016). On the deadline,
+or on cancellation (ADR-0011), that slot's process is killed and a fresh one is started on the next
+demand; other slots keep running. Admission is bounded: a waiter count at `queue_bound` is
+`Saturated` at once, and a pattern the pool cannot admit before its deadline times out unrun.
+
+Matching uses the standard library's `re`, a backtracking matcher like V8's Irregexp: patterns that
+time out in the reference also time out here. The `regex` module short-circuits some of them
+(`(a+)+$` finishes at once), which would turn the reference's `invalid_pattern` into a result
+(ADR-0018).
+"""
+
+import os
+import pickle
+import signal
+import subprocess
+import sys
+from time import monotonic
+from typing import Final
+
+import anyio
+from anyio.abc import Process
+from anyio.streams.buffered import BufferedByteReceiveStream
+
+from jev_mcp.extract.dialect import Translated
+from jev_mcp.extract.executor import (
+    Invalid,
+    Matches,
+    MatchResult,
+    Saturated,
+    Timeout,
+    Unavailable,
+    match_all,
+)
+from jev_mcp.limits import EXTRACT, ExtractCaps
+
+_START_TIMEOUT_S: Final = 30.0
+_SELF_DEADLINE_GRACE_S: Final = 1.0
+
+
+class _WorkerFailed(Exception):
+    def __init__(self, cause: Unavailable) -> None:
+        self.cause = cause
+
+
+def main() -> None:
+    """The worker process: answer one job at a time until stdin closes.
+
+    Each job also arms SIGALRM, whose default action ends the process: if the server dies without
+    killing a worker stuck in a pattern, the worker still ends shortly after the deadline. `re`
+    never returns to the interpreter mid-match, so only a signal's default action can stop it.
+    """
+    signal.signal(signal.SIGINT, signal.SIG_IGN)  # the server owns shutdown; it kills its workers
+    signal.signal(signal.SIGALRM, signal.SIG_DFL)
+    requests, replies = sys.stdin.buffer, sys.stdout.buffer
+    replies.write(_READY)
+    replies.flush()
+    while len(header := requests.read(_HEADER)) == _HEADER:
+        job = pickle.loads(requests.read(int.from_bytes(header)))  # noqa: S301 - our parent
+        source, flags, units, deadline, max_candidates, max_units = job
+        signal.setitimer(signal.ITIMER_REAL, deadline + _SELF_DEADLINE_GRACE_S)
+        matches = match_all(Translated(source, flags), units, max_candidates, max_units)
+        signal.setitimer(signal.ITIMER_REAL, 0)
+        reply = pickle.dumps((matches.candidates, matches.truncated, matches.too_long))
+        replies.write(len(reply).to_bytes(_HEADER) + reply)
+        replies.flush()
+
+
+_READY: Final = b"R"
+_HEADER: Final = 4
+
+
+class _Slot:
+    def __init__(self, process: Process) -> None:
+        assert process.stdin is not None and process.stdout is not None
+        self.process = process
+        self.requests = process.stdin
+        self.replies = BufferedByteReceiveStream(process.stdout)
+
+    async def kill(self) -> None:
+        with anyio.CancelScope(shield=True):
+            try:
+                self.process.kill()
+            except ProcessLookupError:
+                pass  # already reaped (a worker can die on a bad pattern before kill lands); the goal holds
+            await self.process.aclose()
+
+
+class ProcessRegexExecutor:
+    """At most `size` patterns run at once, each in its own killable process.
+
+    The caller's deadline is a whole-request budget (ADR-0016): admission, worker start, IPC, and
+    matching all spend it. `queue_bound` caps how many patterns may wait for a slot. Each job carries
+    the candidate caps from `caps` (`limits.EXTRACT`), so the worker holds no copy of them.
+    """
+
+    def __init__(self, size: int | None = None, queue_bound: int | None = None, caps: ExtractCaps = EXTRACT) -> None:
+        self._caps = caps
+        self.size = size or min(8, os.cpu_count() or 1)
+        self.queue_bound = queue_bound if queue_bound is not None else 8 * self.size
+        self._limit = anyio.Semaphore(self.size)
+        self._waiting = 0
+        self._idle: list[_Slot] = []
+
+    async def find(self, pattern: Translated, text: str, *, deadline: float) -> MatchResult:
+        """Match `pattern` over unit-space `text`; every stage spends what remains of `deadline`."""
+        if self._waiting >= self.queue_bound:
+            return Saturated()
+        self._waiting += 1
+        try:
+            with anyio.fail_after(max(0.0, deadline - monotonic())):
+                await self._limit.acquire()
+        except TimeoutError:
+            return Timeout()
+        finally:
+            self._waiting -= 1
+        try:
+            return await self._find(pattern, text, deadline)
+        except TimeoutError:
+            return Timeout()
+        except _WorkerFailed as failed:
+            return Invalid(failed.cause)
+        finally:
+            self._limit.release()
+
+    async def _find(self, pattern: Translated, text: str, deadline: float) -> Matches:
+        remaining = deadline - monotonic()
+        if remaining <= 0:
+            raise TimeoutError
+        slot = self._idle.pop() if self._idle else await self._start(remaining)
+        healthy = False
+        try:
+            matches = await self._run(slot, pattern, text, deadline - monotonic())
+            healthy = True
+            return matches
+        finally:
+            if healthy:
+                self._idle.append(slot)
+            else:
+                await slot.kill()
+
+    async def _run(self, slot: _Slot, pattern: Translated, units: str, budget: float) -> Matches:
+        if budget <= 0:
+            raise TimeoutError
+        caps = self._caps
+        job = pickle.dumps(
+            (pattern.source, pattern.flags, units, budget, caps.candidates_per_field, caps.candidate_units)
+        )
+        try:
+            await slot.requests.send(len(job).to_bytes(_HEADER) + job)
+            with anyio.fail_after(budget):
+                size = int.from_bytes(await slot.replies.receive_exactly(_HEADER))
+                reply = await slot.replies.receive_exactly(size)
+        except TimeoutError:
+            raise  # an OSError, but the deadline's, not a crash
+        except (anyio.IncompleteRead, anyio.EndOfStream, anyio.BrokenResourceError, OSError):
+            raise _WorkerFailed(Unavailable.NO_RESULT) from None
+        candidates, truncated, too_long = pickle.loads(reply)  # noqa: S301 - our own child
+        return Matches(candidates, truncated, too_long)
+
+    async def _start(self, budget: float = _START_TIMEOUT_S) -> _Slot:
+        """A ready worker. Raises `TimeoutError` if `budget` runs out first, `_WorkerFailed` on a crash."""
+        process = await anyio.open_process(
+            [sys.executable, "-m", __name__], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=None
+        )
+        slot = _Slot(process)
+        try:
+            with anyio.fail_after(budget):
+                ready = await slot.replies.receive_exactly(len(_READY))
+        except TimeoutError:
+            # The budget ran out before the worker signalled readiness — a deadline, not a crash.
+            await slot.kill()
+            raise
+        except (anyio.IncompleteRead, anyio.EndOfStream):
+            await slot.kill()
+            raise _WorkerFailed(Unavailable.NOT_STARTED) from None
+        if ready != _READY:
+            await slot.kill()
+            raise _WorkerFailed(Unavailable.NOT_STARTED)
+        return slot
+
+    async def warm(self) -> None:
+        """Start one worker ahead of demand."""
+        if not self._idle:
+            self._idle.append(await self._start())
+
+    async def aclose(self) -> None:
+        idle, self._idle = self._idle, []
+        for slot in idle:
+            await slot.kill()
+
+
+if __name__ == "__main__":
+    main()

jev_mcp/ids.py

@@ -0,0 +1,49 @@
+"""`sanitize_id` and `ensure_unique_ids` (`lib.ts:13-45`)."""
+
+import re
+from collections.abc import Mapping, Sequence
+from typing import NamedTuple
+
+_UNSAFE = re.compile(r"[^A-Za-z0-9_.-]+")
+MAX_ID_LENGTH = 64
+
+
+def sanitize_id(raw: str) -> str:
+    """Collapse each run of characters outside `[A-Za-z0-9_.-]` to `_`, strip edge underscores, keep 64.
+
+    The result is ASCII, so code points and UTF-16 units coincide. The 64-unit slice runs after the
+    strip, so a cut id may end in `_`, as in the reference.
+    """
+    return _UNSAFE.sub("_", raw).strip("_")[:MAX_ID_LENGTH]
+
+
+class UniqueIds(NamedTuple):
+    items: list[dict[str, object]]
+    """Copies of the input items with `id` set; an existing `id` keeps its key position."""
+    renamed: dict[str, str]
+    """Caller id → id used, for every non-empty caller id that changed."""
+
+
+def ensure_unique_ids(items: Sequence[Mapping[str, object]], fallback_prefix: str) -> UniqueIds:
+    """Give every item a safe, unique id: sanitized, else `{fallback_prefix}{index}`; collisions get `_1`, `_2`, ….
+
+    A missing or null `id` counts as empty. Fallback ids are not checked against later caller ids
+    beyond the shared collision loop, exactly as in the reference.
+    """
+    used: set[str] = set()
+    renamed: dict[str, str] = {}
+    out: list[dict[str, object]] = []
+    for index, item in enumerate(items):
+        raw = item.get("id")
+        raw_id = raw if isinstance(raw, str) else ""
+        base = sanitize_id(raw_id) or f"{fallback_prefix}{index}"
+        new_id = base
+        suffix = 1
+        while new_id in used:
+            new_id = f"{base}_{suffix}"
+            suffix += 1
+        used.add(new_id)
+        if raw_id and raw_id != new_id:
+            renamed[raw_id] = new_id
+        out.append({**item, "id": new_id})
+    return UniqueIds(out, renamed)

jev_mcp/limits.py

@@ -0,0 +1,218 @@
+"""The frozen input caps, typed per tool (ADR-0014); the manifest is the oracle, not the source.
+
+Every value is transcribed from `docs/reference/parity-manifest.json` `caps`, and
+`tests/contract/test_limits.py` fails when the two disagree, so a re-freeze points here. VALUES
+ONLY: UTF-16 measurement stays in `text.py` (ADR-0005) and applying a cap — reject is the schema
+itself — lives in the tools and `validation/caps.py`. A tool that hardcodes one of these numbers is
+a bug.
+
+Text bounds are UTF-16 code units (JS `.length`); `None` marks a cap the reference deliberately
+leaves open (the manifest records it as null: "do not add a bound"). The three behaviors of the
+manifest prose: schema `min*`/`max*` are reject, `*_units` are truncate-to-cap or skip thresholds,
+and the aggregate budgets are strict greater-than errors (the cap itself is allowed).
+
+`jev_extract`'s candidate caps (`candidates_per_field`, `candidate_units`) reach the matcher from
+here, carried in each executor job; `regex_timeout_ms` is enforced in `extract/candidates.py`, which
+keeps its own copy, and the contract test fails on drift between the two.
+"""
+
+from dataclasses import dataclass
+from typing import Final
+
+
+@dataclass(frozen=True, slots=True)
+class CandidatesCaps:
+    """`candidatesSchema` (`index.ts:105-116`), shared by jev_find and jev_rerank."""
+
+    min_items: int
+    max_items: int
+    text_units: int
+
+
+@dataclass(frozen=True, slots=True)
+class VerifyCaps:
+    """jev_verify: only presence is capped; lengths are deliberately open (manifest nulls)."""
+
+    claims_min: int
+    claims_max: int | None
+    claim_units: int | None
+    evidence_min: int
+    evidence_max: int | None
+
+
+@dataclass(frozen=True, slots=True)
+class ScreenCaps:
+    """jev_screen: `text` only needs content; no length bound (manifest nulls)."""
+
+    text_min: int
+    text_max: int | None
+    purpose_max: int | None
+
+
+@dataclass(frozen=True, slots=True)
+class FindCaps:
+    """jev_find: candidate bounds are the shared `CANDIDATES`."""
+
+    top_k_min: int
+    top_k_max: int
+    top_k_default: int
+
+
+@dataclass(frozen=True, slots=True)
+class ClassifyCaps:
+    """jev_classify: per-entry text truncation plus the item-class budget error."""
+
+    items_min: int
+    items_max: int
+    item_units: int
+    classes_min: int
+    classes_max: int
+    class_description_units: int
+    item_class_pairs: int
+
+
+@dataclass(frozen=True, slots=True)
+class DecideCaps:
+    """jev_decide: every bound is a schema reject; nothing is truncated."""
+
+    decision_min: int
+    decision_max: int
+    evidence_min: int
+    evidence_max: int
+    priorities_min: int
+    priorities_max: int
+    candidates_min: int
+    candidates_max: int
+    candidate_id_max: int
+    candidate_description_min: int
+    candidate_description_max: int
+    requirements_min: int
+    requirements_max: int
+    requirement_min: int
+    requirement_max: int
+
+
+@dataclass(frozen=True, slots=True)
+class RerankCaps:
+    """jev_rerank: candidate bounds are the shared `CANDIDATES`."""
+
+    query_min: int
+    query_max: int
+    aggregate_candidate_units: int
+    top_k_min: int
+    top_k_max: int
+
+
+@dataclass(frozen=True, slots=True)
+class CompareCaps:
+    """jev_compare: passages are schema-rejected above the cap; the truncate call is a no-op guard."""
+
+    passage_min: int
+    passage_max: int
+    aspects_min: int
+    aspects_max: int
+    aspect_min: int
+    aspect_max: int
+
+
+@dataclass(frozen=True, slots=True)
+class ExtractCaps:
+    """jev_extract: schema rejects, one aggregate budget error, and the worker's pipeline caps."""
+
+    document_min: int
+    document_max: int
+    fields_min: int
+    fields_max: int
+    field_id_max: int
+    pattern_min: int
+    pattern_max: int
+    flags_max: int
+    description_min: int
+    description_max: int
+    candidates_per_field: int
+    candidate_units: int
+    aggregate_candidate_units: int
+    regex_timeout_ms: int
+
+
+@dataclass(frozen=True, slots=True)
+class ReviewCaps:
+    """jev_review: request, diff, and tests are each truncated at the doc cap, never schema-rejected."""
+
+    doc_units: int
+
+
+@dataclass(frozen=True, slots=True)
+class GateCaps:
+    """jev_gate: claim/item/text caps and the two aggregate isError budgets."""
+
+    claims_min: int
+    claims_max: int
+    claim_units: int
+    evidence_items: int
+    aggregate_evidence_units: int
+    doc_units: int
+
+
+CANDIDATES: Final = CandidatesCaps(min_items=1, max_items=250, text_units=2000)
+"""jev_find `"candidates": "1..250 reject"` / `"candidate_text": "2000 truncate"`; jev_rerank
+`"candidates": "..250 reject"` — one shared schema and one shared truncation."""
+
+VERIFY: Final = VerifyCaps(claims_min=1, claims_max=None, claim_units=None, evidence_min=1, evidence_max=None)
+SCREEN: Final = ScreenCaps(text_min=1, text_max=None, purpose_max=None)
+FIND: Final = FindCaps(top_k_min=1, top_k_max=50, top_k_default=5)
+CLASSIFY: Final = ClassifyCaps(
+    items_min=1,
+    items_max=64,
+    item_units=2000,
+    classes_min=2,
+    classes_max=250,
+    class_description_units=2000,
+    item_class_pairs=8_000,
+)
+DECIDE: Final = DecideCaps(
+    decision_min=1,
+    decision_max=1500,
+    evidence_min=1,
+    evidence_max=12_000,
+    priorities_min=1,
+    priorities_max=2000,
+    candidates_min=2,
+    candidates_max=6,
+    candidate_id_max=64,
+    candidate_description_min=1,
+    candidate_description_max=2000,
+    requirements_min=0,
+    requirements_max=3,
+    requirement_min=1,
+    requirement_max=500,
+)
+RERANK: Final = RerankCaps(query_min=1, query_max=2000, aggregate_candidate_units=100_000, top_k_min=1, top_k_max=250)
+COMPARE: Final = CompareCaps(
+    passage_min=1, passage_max=20_000, aspects_min=0, aspects_max=10, aspect_min=1, aspect_max=200
+)
+EXTRACT: Final = ExtractCaps(
+    document_min=1,
+    document_max=50_000,
+    fields_min=1,
+    fields_max=32,
+    field_id_max=64,
+    pattern_min=1,
+    pattern_max=500,
+    flags_max=8,
+    description_min=1,
+    description_max=2000,
+    candidates_per_field=20,
+    candidate_units=2000,
+    aggregate_candidate_units=50_000,
+    regex_timeout_ms=1000,
+)
+REVIEW: Final = ReviewCaps(doc_units=50_000)
+GATE: Final = GateCaps(
+    claims_min=1,
+    claims_max=16,
+    claim_units=2000,
+    evidence_items=16,
+    aggregate_evidence_units=200_000,
+    doc_units=50_000,
+)

jev_mcp/policy/__init__.py

@@ -0,0 +1,98 @@
+"""Pure policy mapping validated judgments to actions (ADR-0002). Validation rejects; policy decides."""
+
+from jev_mcp.policy.actions import (
+    Action,
+    Decision,
+    classification_decision,
+    require_complete_context,
+    verify_action,
+    worst_action,
+)
+from jev_mcp.policy.claims import (
+    GATE_REASON_CODES,
+    ClaimJudgment,
+    ClaimVerdict,
+    RequirementCheck,
+    claim_action,
+    contradicts_recommendation,
+    gate_reason_codes,
+)
+from jev_mcp.policy.extract import (
+    EXTRACT_REASON_CODES,
+    ExtractFieldDecision,
+    ExtractFieldEvidence,
+    ExtractJudgment,
+    ExtractReasonCode,
+    ExtractStatus,
+    decide_extract_field,
+)
+from jev_mcp.policy.ranking import ExistsVerdict, exists_verdict, rank_candidates, rerank_by_score
+from jev_mcp.policy.review import REVIEW_WEIGHTS, min_confidence, review_action, review_composite
+from jev_mcp.policy.screen import ScreenAction, ScreenRecommendation, screen_fail_closed, screen_recommendation
+from jev_mcp.policy.thresholds import (
+    DEFAULT_AUTO_ACCEPT,
+    DEFAULT_CLASSIFY_AUTO_ACCEPT,
+    DEFAULT_COMPOSITE_FLOOR,
+    DEFAULT_MINIMUM_MARGIN,
+    DEFAULT_REVIEW_AT_CAP,
+    DEFAULT_SCREEN_BLOCK_AT,
+    DEFAULT_SCREEN_REVIEW_AT,
+    EXISTS_ABSENT_BELOW,
+    EXISTS_FOUND_AT,
+    SCREEN_RELEVANCE_SKIP_BELOW,
+    SCREEN_SUBSTANCE_SKIP_BELOW,
+    THRESHOLD_INVARIANT_MESSAGE,
+    PolicyThresholds,
+    resolve_policy_thresholds,
+    validate_policy_thresholds,
+)
+
+__all__ = [
+    "DEFAULT_AUTO_ACCEPT",
+    "DEFAULT_CLASSIFY_AUTO_ACCEPT",
+    "DEFAULT_COMPOSITE_FLOOR",
+    "DEFAULT_MINIMUM_MARGIN",
+    "DEFAULT_REVIEW_AT_CAP",
+    "DEFAULT_SCREEN_BLOCK_AT",
+    "DEFAULT_SCREEN_REVIEW_AT",
+    "EXISTS_ABSENT_BELOW",
+    "EXISTS_FOUND_AT",
+    "EXTRACT_REASON_CODES",
+    "GATE_REASON_CODES",
+    "REVIEW_WEIGHTS",
+    "SCREEN_RELEVANCE_SKIP_BELOW",
+    "SCREEN_SUBSTANCE_SKIP_BELOW",
+    "THRESHOLD_INVARIANT_MESSAGE",
+    "Action",
+    "ClaimJudgment",
+    "ClaimVerdict",
+    "Decision",
+    "ExistsVerdict",
+    "ExtractFieldDecision",
+    "ExtractFieldEvidence",
+    "ExtractJudgment",
+    "ExtractReasonCode",
+    "ExtractStatus",
+    "PolicyThresholds",
+    "RequirementCheck",
+    "ScreenAction",
+    "ScreenRecommendation",
+    "claim_action",
+    "classification_decision",
+    "contradicts_recommendation",
+    "decide_extract_field",
+    "exists_verdict",
+    "gate_reason_codes",
+    "min_confidence",
+    "rank_candidates",
+    "require_complete_context",
+    "rerank_by_score",
+    "resolve_policy_thresholds",
+    "review_action",
+    "review_composite",
+    "screen_fail_closed",
+    "screen_recommendation",
+    "validate_policy_thresholds",
+    "verify_action",
+    "worst_action",
+]

jev_mcp/policy/actions.py

@@ -0,0 +1,41 @@
+"""Actions and the rules that combine them (`lib.ts:60-62`, `lib.ts:126-133`, `lib.ts:288-360`)."""
+
+from collections.abc import Iterable
+from typing import Literal
+
+type Action = Literal["auto", "review", "escalate"]
+"""`PolicyAction`: auto stands alone, review needs confirmation, escalate must not proceed as is."""
+
+type Decision = Literal["auto", "review"]
+"""The two-tier action of verify, classify, compare, and extract: they have no escalate tier."""
+
+
+def verify_action(confidence: float, auto_accept: float) -> Decision:
+    """`verifyAction`: auto iff confidence >= auto_accept.
+
+    Unknown confidence never reaches this function: jev_verify maps it to review at the call site
+    (`index.ts:216`), and that check belongs to the tool.
+    """
+    return "auto" if confidence >= auto_accept else "review"
+
+
+def classification_decision(
+    top_probability: float, margin: float, auto_accept: float, minimum_margin: float
+) -> Decision:
+    """`classificationDecision`: auto iff top_probability >= auto_accept and margin >= minimum_margin."""
+    return "auto" if top_probability >= auto_accept and margin >= minimum_margin else "review"
+
+
+def require_complete_context(action: Action, truncated: bool) -> Action:
+    """`requireCompleteContext`: truncated input demotes auto to review and leaves stronger actions alone."""
+    return "review" if truncated and action == "auto" else action
+
+
+def worst_action(actions: Iterable[Action]) -> Action:
+    """`worstAction`: escalate > review > auto. No actions at all is auto, as in the reference."""
+    seen = set(actions)
+    if "escalate" in seen:
+        return "escalate"
+    if "review" in seen:
+        return "review"
+    return "auto"