jevfilter 0.1.0__py3-none-any.whl

jevfilter/__init__.py

@@ -0,0 +1,50 @@
+"""jevfilter: judge content against plain-English definitions with Jev.
+
+```python
+import jevfilter as jf
+
+jf.choose("I was charged twice", ["billing", "bug", "feature request"])
+
+f = jf.Filter(jf.Topic.load("topics/"))
+r = f.judge("Your application to Acme was received")
+```
+"""
+
+from .content import Content
+from .defaults import configure
+from .engine import Filter, Plan
+from .errors import JevFilterError, JudgeError, TopicError
+from .helpers import check, choose, rate
+from .policy import Decision, Policy, ThresholdPolicy, TopicAnswers
+from .registry import facet
+from .result import Choice, Field, Result, Score, TopicResult
+from .topic import Topic, Topics
+from .version import __version__
+from .wording import WORDING_VERSION
+
+__all__ = [
+    "WORDING_VERSION",
+    "Choice",
+    "Content",
+    "Decision",
+    "Field",
+    "Filter",
+    "JevFilterError",
+    "JudgeError",
+    "Plan",
+    "Policy",
+    "Result",
+    "Score",
+    "ThresholdPolicy",
+    "Topic",
+    "TopicAnswers",
+    "TopicError",
+    "TopicResult",
+    "Topics",
+    "__version__",
+    "check",
+    "configure",
+    "choose",
+    "facet",
+    "rate",
+]

jevfilter/content.py

@@ -0,0 +1,46 @@
+"""What gets judged."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
+from typing import Any
+
+
+@dataclass(frozen=True)
+class Content:
+    """Content to judge, plus optional field candidates and context.
+
+    `state` is text or a JSON-able dict. `candidates` maps topic name → field
+    name → candidate values (use `"*"` as the topic for every topic).
+    `context` is extra JSON-able state the model can read (e.g. who the
+    reader is); it is sent alongside `content`.
+    """
+
+    state: Any
+    candidates: Mapping[str, Mapping[str, Sequence[str]]] | None = None
+    context: Any = None
+
+    def as_state(self) -> dict[str, Any]:
+        state: dict[str, Any] = {"content": self.state}
+        if self.context is not None:
+            state["context"] = self.context
+        return state
+
+    def candidates_for(self, topic: str, field: str) -> list[str]:
+        if not self.candidates:
+            return []
+        values = [
+            *self.candidates.get(topic, {}).get(field, ()),
+            *self.candidates.get("*", {}).get(field, ()),
+        ]
+        out: list[str] = []
+        for v in values:
+            v = v.strip() if isinstance(v, str) else v
+            if isinstance(v, str) and v and v not in out:
+                out.append(v)
+        return out
+
+
+def as_content(content: Any) -> Content:
+    return content if isinstance(content, Content) else Content(content)

jevfilter/defaults.py

@@ -0,0 +1,52 @@
+"""The default judge used by helpers and by `Filter` when none is given."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from .judges.base import Judge
+
+_default: Judge | None = None
+
+
+def configure(
+    *,
+    api_key: str | None = None,
+    model: str | None = None,
+    judge: Judge | None = None,
+    **client_kwargs: Any,
+) -> None:
+    """Set the default judge for helpers and filters created without `judge=`.
+
+    ```python
+    jf.configure(api_key="...", model="jev-1.13.0")   # a JevJudge
+    jf.configure(judge=FakeJudge({...}))               # any Judge
+    ```
+
+    Without this, a `JevJudge` reading `TYPESAFE_API_KEY` from the
+    environment is created on first use.
+    """
+    global _default
+    if judge is not None:
+        if api_key is not None or model is not None or client_kwargs:
+            raise ValueError("pass either judge= or JevJudge options, not both")
+        _default = judge
+        return
+    from .judges.jev import JevJudge
+
+    _default = JevJudge(model=model, api_key=api_key, **client_kwargs)
+
+
+def default_judge() -> Judge:
+    global _default
+    if _default is None:
+        from .judges.jev import JevJudge
+
+        _default = JevJudge()
+    return _default
+
+
+def reset() -> None:
+    """Forget the configured default (mainly for tests)."""
+    global _default
+    _default = None

jevfilter/engine.py

@@ -0,0 +1,285 @@
+"""The judge pipeline: compile → plan → execute → interpret → decide → report.
+
+This version sends every question for one piece of content in a single
+request. Packing / splitting across Jev's context limits comes later.
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from typing import Any, Literal
+
+from . import registry, wording
+from .content import Content, as_content
+from .defaults import default_judge
+from .errors import JevFilterError, JudgeError
+from .facets import BUILTIN, Facet
+from .judges.base import Answer, Judge, NoulAnswer, Question, Response, answer_to_dict
+from .policy import Policy, ThresholdPolicy, TopicAnswers
+from .result import Result, Score, TopicResult
+from .topic import Topic, Topics, TopicSource, as_topics
+from .version import __version__
+
+DEFAULT_PRICE_PER_MTOK = 0.042
+"""Jev's published input price (USD per million tokens) when this was written."""
+
+OnError = Literal["raise", "review"] | Judge
+
+
+@dataclass(frozen=True)
+class _Route:
+    topic: str
+    facet: str  # "membership", a built-in facet key or a custom facet key
+    local: str  # the facet's own question name ("" for a single question)
+
+
+@dataclass
+class _Compiled:
+    state: dict[str, Any]
+    questions: dict[str, Question]
+    routes: dict[str, _Route]
+    warnings: list[str] = field(default_factory=list)
+
+
+@dataclass(frozen=True)
+class Plan:
+    """What `judge()` would send, without sending it."""
+
+    requests: list[dict[str, Any]]
+    estimated_input_tokens: int
+    cost_usd: float
+    warnings: tuple[str, ...] = ()
+
+
+class Filter:
+    """Topics plus engine configuration. `judge(content)` → `Result`.
+
+    ```python
+    f = Filter(Topic.load("topics/"))
+    r = f.judge({"from": "...", "subject": "...", "body": "..."})
+    ```
+    """
+
+    def __init__(
+        self,
+        topics: TopicSource | Topics,
+        *,
+        judge: Judge | None = None,
+        policy: Policy | None = None,
+        on_error: OnError = "raise",
+        price_per_mtok: float = DEFAULT_PRICE_PER_MTOK,
+        include_requests: bool = False,
+    ):
+        self.topics = as_topics(topics)
+        if not self.topics:
+            raise ValueError("Filter needs at least one topic")
+        if isinstance(on_error, str) and on_error not in ("raise", "review"):
+            raise ValueError("on_error must be 'raise', 'review' or a Judge")
+        self._backend = judge
+        self.policy: Policy = policy or ThresholdPolicy()
+        self.on_error = on_error
+        self.price_per_mtok = price_per_mtok
+        self.include_requests = include_requests
+
+    @property
+    def backend(self) -> Judge:
+        """The judge backend: the one given, else the `configure()`d default."""
+        return self._backend or default_judge()
+
+    # -- public -------------------------------------------------------------
+
+    def judge(self, content: Any) -> Result:
+        """Judge one piece of content (text, a dict, or `Content`) against every topic."""
+        c = as_content(content)
+        compiled = self._compile(c)
+        response, degraded, error = self._execute(compiled)
+        return self._report(compiled, response, degraded, error)
+
+    def explain(self, content: Any) -> Plan:
+        """The exact request payloads and an estimated cost, without calling the backend."""
+        compiled = self._compile(as_content(content))
+        request = _payload(compiled)
+        tokens = estimate_tokens(request)
+        return Plan(
+            requests=[request],
+            estimated_input_tokens=tokens,
+            cost_usd=tokens * self.price_per_mtok / 1e6,
+            warnings=tuple(compiled.warnings),
+        )
+
+    # -- 1. compile ---------------------------------------------------------
+
+    def _compile(self, content: Content) -> _Compiled:
+        compiled = _Compiled(content.as_state(), {}, {})
+        for t in self.topics.values():
+            qid = f"{t.name}/membership"
+            compiled.questions[qid] = wording.membership(t)
+            compiled.routes[qid] = _Route(t.name, "membership", "")
+            for key, facet, config in _facets_of(t):
+                for local, q in facet.questions(t, config, content).items():
+                    qid = f"{t.name}/{key}/{local}" if local else f"{t.name}/{key}"
+                    compiled.questions[qid] = q
+                    compiled.routes[qid] = _Route(t.name, key, local)
+                warn = getattr(facet, "warnings", None)
+                if warn is not None:
+                    compiled.warnings.extend(warn(t, content))
+        return compiled
+
+    # -- 3/4. execute -------------------------------------------------------
+
+    def _execute(self, compiled: _Compiled) -> tuple[Response | None, bool, str | None]:
+        try:
+            return self.backend.ask(compiled.state, compiled.questions), False, None
+        except JudgeError as e:
+            if self.on_error == "raise":
+                raise
+            if self.on_error == "review":
+                return None, True, str(e)
+            fallback = self.on_error
+            return fallback.ask(compiled.state, compiled.questions), True, str(e)
+
+    # -- 5/6/7. interpret, decide, report -----------------------------------
+
+    def _report(
+        self,
+        compiled: _Compiled,
+        response: Response | None,
+        degraded: bool,
+        error: str | None,
+    ) -> Result:
+        warnings = list(compiled.warnings)
+        if error:
+            warnings.append(f"backend_error: {error}")
+        topics: dict[str, TopicResult] = {}
+        if response is None:
+            for t in self.topics.values():
+                topics[t.name] = TopicResult(
+                    t.name, "review", 0.0, ("backend_error",), topic_version=t.version
+                )
+        else:
+            grouped = _group(compiled.routes, response.answers)
+            for t in self.topics.values():
+                topics[t.name] = self._topic_result(t, grouped.get(t.name, {}))
+
+        tokens = response.input_tokens if response else None
+        return Result(
+            topics=topics,
+            model=response.model if response else None,
+            request_ids=tuple(r for r in [response.request_id if response else None] if r),
+            input_tokens=tokens,
+            cost_usd=tokens * self.price_per_mtok / 1e6 if tokens is not None else None,
+            wording_version=wording.WORDING_VERSION,
+            jevfilter_version=__version__,
+            degraded=degraded,
+            warnings=tuple(warnings),
+            raw={qid: answer_to_dict(a) for qid, a in response.answers.items()} if response else {},
+            requests=[_payload(compiled)] if self.include_requests else None,
+        )
+
+    def _topic_result(self, t: Topic, answers: dict[str, dict[str, Answer]]) -> TopicResult:
+        membership = answers.get("membership", {}).get("")
+        if not isinstance(membership, NoulAnswer):
+            raise JudgeError(f"no membership answer for topic {t.name!r}")
+
+        values: dict[str, Any] = {}
+        for key, facet, config in _facets_of(t):
+            if key in answers or key == "fields":
+                values[key] = facet.interpret(t, config, answers.get(key, {}))
+
+        # `when: {category: [...]}` is independent of the outcome, so apply it first.
+        category = values.get("categories")
+        _filter(t, values, lambda w: w.mode != "category" or _in(category, w.categories))
+
+        custom = {k: v for k, v in values.items() if k not in BUILTIN}
+        draft = TopicAnswers(
+            p=membership.p,
+            category=values.get("categories"),
+            fields=values.get("fields", {}),
+            scores=values.get("scores", {}),
+            flags=values.get("flags", {}),
+            facets=custom,
+        )
+        decision = self.policy.decide(t, draft)
+
+        # Everything but `always` facets is only used when content belongs.
+        if decision.outcome == "no":
+            _filter(t, values, lambda w: w.mode == "always")
+
+        scores: dict[str, Score] = values.get("scores", {})
+        return TopicResult(
+            topic=t.name,
+            outcome=decision.outcome,
+            p=membership.p,
+            reasons=decision.reasons,
+            category=values.get("categories"),
+            fields=values.get("fields", {}),
+            scores=scores,
+            flags=values.get("flags", {}),
+            composites=_composites(t, scores),
+            facets={k: v for k, v in values.items() if k not in BUILTIN},
+            topic_version=t.version,
+        )
+
+
+# -- helpers ----------------------------------------------------------------
+
+
+def _facets_of(t: Topic) -> list[tuple[str, Facet, Any]]:
+    out: list[tuple[str, Facet, Any]] = []
+    for key in BUILTIN:
+        config = getattr(t, key)
+        if config:
+            out.append((key, BUILTIN[key], config))
+    for key, config in t.custom.items():
+        facet = registry.get_facet(key)
+        if facet is None:
+            raise JevFilterError(f"Topic {t.name!r}: custom facet {key!r} is no longer registered")
+        out.append((key, facet, config))
+    return out
+
+
+def _group(
+    routes: Mapping[str, _Route], answers: Mapping[str, Answer]
+) -> dict[str, dict[str, dict[str, Answer]]]:
+    grouped: dict[str, dict[str, dict[str, Answer]]] = {}
+    for qid, a in answers.items():
+        route = routes.get(qid)
+        if route is not None:
+            grouped.setdefault(route.topic, {}).setdefault(route.facet, {})[route.local] = a
+    return grouped
+
+
+def _filter(t: Topic, values: dict[str, Any], keep: Any) -> None:
+    """Drop facet answers (or items within them) whose `when` fails `keep`."""
+    for key in list(values):
+        value = values[key]
+        if key in ("fields", "scores", "flags"):
+            values[key] = {n: v for n, v in value.items() if keep(t.when_for(key, n))}
+        elif not keep(t.when_for(key)):
+            del values[key]
+
+
+def _in(category: Any, names: tuple[str, ...]) -> bool:
+    return category is not None and category.value in names
+
+
+def _composites(t: Topic, scores: Mapping[str, Score]) -> dict[str, float]:
+    out = {}
+    for name, weights in t.composites.items():
+        if all(s in scores for s in weights):
+            out[name] = sum(w * scores[s].normalized for s, w in weights.items())
+    return out
+
+
+def _payload(compiled: _Compiled) -> dict[str, Any]:
+    return {
+        "state": compiled.state,
+        "questions": {qid: q.to_dict() for qid, q in compiled.questions.items()},
+    }
+
+
+def estimate_tokens(payload: Any) -> int:
+    """Rough input-token estimate: characters ÷ 4."""
+    return len(json.dumps(payload, ensure_ascii=False)) // 4 + 1

jevfilter/errors.py

@@ -0,0 +1,19 @@
+"""Typed errors raised by jevfilter."""
+
+from __future__ import annotations
+
+
+class JevFilterError(Exception):
+    """Base class for every jevfilter error."""
+
+
+class TopicError(JevFilterError, ValueError):
+    """A topic definition is invalid. `problems` lists every issue found."""
+
+    def __init__(self, problems: list[str]):
+        self.problems = problems
+        super().__init__("\n".join(problems) if len(problems) > 1 else problems[0])
+
+
+class JudgeError(JevFilterError):
+    """The judge backend failed to answer."""

jevfilter/facets/__init__.py

@@ -0,0 +1,214 @@
+"""Facets: the kinds of judgment a topic asks for.
+
+A facet turns a topic's config into questions and reads its answers back.
+Built-ins cover categories, fields, scores and flags; register your own with
+`@jevfilter.facet("name")` and use it as a topic key.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import Any, ClassVar, Protocol
+
+from .. import wording
+from ..content import Content
+from ..errors import JevFilterError
+from ..judges.base import Answer, ChoiceAnswer, NoulAnswer, Question, ScoreAnswer
+from ..result import Choice, Field, Score
+from ..topic import Topic
+
+
+class Facet(Protocol):
+    """Questions are keyed by a local name ("" for a single question); the
+    engine prefixes them into question IDs: `<topic>/<facet>[/<name>]`."""
+
+    def questions(self, topic: Topic, config: Any, content: Content) -> dict[str, Question]: ...
+
+    def interpret(self, topic: Topic, config: Any, answers: Mapping[str, Answer]) -> Any: ...
+
+
+def _speculative(topic: Topic, facet: str, item: str | None = None) -> bool:
+    return topic.when_for(facet, item).mode != "always"
+
+
+def to_choice(a: ChoiceAnswer) -> Choice:
+    return Choice(a.choice, a.confidence, dict(a.probabilities))
+
+
+def to_score(a: ScoreAnswer, levels: tuple[Any, ...]) -> Score:
+    def label(i: int) -> str:
+        return levels[i] if isinstance(levels[i], str) else str(i)
+
+    best = max(a.probabilities, key=a.probabilities.__getitem__)
+    return Score(
+        value=a.score,
+        level=label(best),
+        confidence=a.confidence,
+        probabilities={label(i): p for i, p in sorted(a.probabilities.items())},
+        max_level=len(levels) - 1,
+    )
+
+
+# -- built-ins --------------------------------------------------------------
+
+
+class CategoriesFacet:
+    def questions(self, topic: Topic, config: Any, content: Content) -> dict[str, Question]:
+        if any(not c.is_leaf for c in topic.categories.values()):
+            raise JevFilterError(
+                f"Topic {topic.name!r}: nested categories aren't supported by this "
+                "version of jevfilter yet; use a flat list"
+            )
+        return {"": wording.categories(topic, topic.categories, _speculative(topic, "categories"))}
+
+    def interpret(self, topic: Topic, config: Any, answers: Mapping[str, Answer]) -> Choice:
+        a = answers[""]
+        assert isinstance(a, ChoiceAnswer)
+        return to_choice(a)
+
+
+class FieldsFacet:
+    def questions(self, topic: Topic, config: Any, content: Content) -> dict[str, Question]:
+        out = {}
+        for name, spec in topic.fields.items():
+            candidates = content.candidates_for(topic.name, name)
+            candidates = [c for c in candidates if c != wording.NONE_OF_THESE]
+            if candidates:
+                out[name] = wording.field(
+                    topic, spec, candidates, _speculative(topic, "fields", name)
+                )
+        return out
+
+    def warnings(self, topic: Topic, content: Content) -> list[str]:
+        return [
+            f"{topic.name}/fields/{name}: no candidates, so it wasn't asked"
+            for name in topic.fields
+            if not content.candidates_for(topic.name, name)
+        ]
+
+    def interpret(
+        self, topic: Topic, config: Any, answers: Mapping[str, Answer]
+    ) -> dict[str, Field]:
+        out = {}
+        for name in topic.fields:
+            a = answers.get(name)
+            if isinstance(a, ChoiceAnswer):
+                value = None if a.choice == wording.NONE_OF_THESE else a.choice
+                out[name] = Field(value, a.confidence, dict(a.probabilities))
+            else:
+                out[name] = Field(None, 0.0, {})
+        return out
+
+
+class ScoresFacet:
+    def questions(self, topic: Topic, config: Any, content: Content) -> dict[str, Question]:
+        return {
+            name: wording.score(topic, spec, _speculative(topic, "scores", name))
+            for name, spec in topic.scores.items()
+        }
+
+    def interpret(
+        self, topic: Topic, config: Any, answers: Mapping[str, Answer]
+    ) -> dict[str, Score]:
+        out = {}
+        for name, spec in topic.scores.items():
+            a = answers[name]
+            assert isinstance(a, ScoreAnswer)
+            out[name] = to_score(a, spec.levels)
+        return out
+
+
+class FlagsFacet:
+    def questions(self, topic: Topic, config: Any, content: Content) -> dict[str, Question]:
+        return {
+            name: wording.flag(topic, cond, _speculative(topic, "flags", name))
+            for name, cond in topic.flags.items()
+        }
+
+    def interpret(
+        self, topic: Topic, config: Any, answers: Mapping[str, Answer]
+    ) -> dict[str, float]:
+        out = {}
+        for name in topic.flags:
+            a = answers[name]
+            assert isinstance(a, NoulAnswer)
+            out[name] = a.p
+        return out
+
+
+BUILTIN: dict[str, Facet] = {
+    "categories": CategoriesFacet(),
+    "fields": FieldsFacet(),
+    "scores": ScoresFacet(),
+    "flags": FlagsFacet(),
+}
+
+
+# -- bases for custom facets ------------------------------------------------
+
+
+class _SingleQuestionFacet:
+    """Custom facet asking one question. Set `instructions` (and criteria/levels)."""
+
+    key: str = ""  # set by @jevfilter.facet(...)
+    instructions: ClassVar[Any] = None
+
+    def _instructions(self, topic: Topic) -> Any:
+        if self.instructions is None:
+            raise JevFilterError(f"{type(self).__name__} needs `instructions`")
+        return wording.custom(topic, self.instructions, _speculative(topic, self.key))
+
+
+class NoulFacet(_SingleQuestionFacet):
+    """A yes/no custom facet; its result is the probability of yes."""
+
+    criteria: ClassVar[Mapping[str, Any] | None] = None
+
+    def questions(self, topic: Topic, config: Any, content: Content) -> dict[str, Question]:
+        return {"": Question("noul", self._instructions(topic), self.criteria)}
+
+    def interpret(self, topic: Topic, config: Any, answers: Mapping[str, Answer]) -> float:
+        a = answers[""]
+        assert isinstance(a, NoulAnswer)
+        return a.p
+
+
+class ChoiceFacet(_SingleQuestionFacet):
+    """A custom facet picking one of `criteria` (label → description)."""
+
+    criteria: ClassVar[Mapping[str, Any]] = {}
+
+    def questions(self, topic: Topic, config: Any, content: Content) -> dict[str, Question]:
+        if not self.criteria:
+            raise JevFilterError(f"{type(self).__name__} needs `criteria`")
+        return {"": Question("choice", self._instructions(topic), dict(self.criteria))}
+
+    def interpret(self, topic: Topic, config: Any, answers: Mapping[str, Answer]) -> Choice:
+        a = answers[""]
+        assert isinstance(a, ChoiceAnswer)
+        return to_choice(a)
+
+
+class ScoreFacet(_SingleQuestionFacet):
+    """A custom facet rating content on ordered `levels` (low → high)."""
+
+    levels: ClassVar[tuple[Any, ...]] = ()
+
+    def questions(self, topic: Topic, config: Any, content: Content) -> dict[str, Question]:
+        if len(self.levels) < 2:
+            raise JevFilterError(f"{type(self).__name__} needs at least two `levels`")
+        return {"": Question("score", self._instructions(topic), list(self.levels))}
+
+    def interpret(self, topic: Topic, config: Any, answers: Mapping[str, Answer]) -> Score:
+        a = answers[""]
+        assert isinstance(a, ScoreAnswer)
+        return to_score(a, tuple(self.levels))
+
+
+__all__ = [
+    "BUILTIN",
+    "ChoiceFacet",
+    "Facet",
+    "NoulFacet",
+    "ScoreFacet",
+]