llms2jev 0.3.3__py3-none-any.whl

llm2jev/__init__.py

@@ -0,0 +1,80 @@
+"""Public composition facade for decision definitions, services, and runtime ports. Exports
+form the supported import contract; concrete provider dependencies remain lazily loaded
+by their constructors. Internal compilation, JSON, and projection helpers are
+intentionally not re-exported.
+"""
+from .application import AsyncBoundEvaluator, AsyncLLM2Jev, BoundEvaluator, LLM2Jev
+from .contracts import (
+    AsyncBinaryRuntime,
+    BinaryLabels,
+    BinaryRuntime,
+    BinaryScores,
+    ChatMessage,
+    ChatPrompt,
+    InvalidRequestError,
+    ModelIdentity,
+    ModelNotFoundError,
+)
+from .core import (
+    Answer,
+    Choice,
+    ChoiceAnswer,
+    JSONContent,
+    JSONValue,
+    JevRequest,
+    JevResponse,
+    Noul,
+    NoulAnswer,
+    Question,
+    Score,
+    ScoreAnswer,
+    State,
+    Usage,
+)
+from .inference import (
+    CandidateProbe,
+    StructuredProbeEncoder,
+    DistributionPolicy,
+    ProbeEncoder,
+    RelativeSupport,
+)
+from .runtime import AsyncOllamaRuntime, OllamaRuntime, OllamaScoringError, TransformersRuntime
+
+__all__ = [
+    "Answer",
+    "AsyncBinaryRuntime",
+    "AsyncBoundEvaluator",
+    "AsyncLLM2Jev",
+    "AsyncOllamaRuntime",
+    "BinaryLabels",
+    "BinaryRuntime",
+    "BinaryScores",
+    "BoundEvaluator",
+    "CandidateProbe",
+    "ChatMessage",
+    "ChatPrompt",
+    "Choice",
+    "ChoiceAnswer",
+    "DistributionPolicy",
+    "InvalidRequestError",
+    "JSONContent",
+    "JSONValue",
+    "JevRequest",
+    "JevResponse",
+    "LLM2Jev",
+    "ModelIdentity",
+    "ModelNotFoundError",
+    "Noul",
+    "NoulAnswer",
+    "OllamaRuntime",
+    "OllamaScoringError",
+    "ProbeEncoder",
+    "Question",
+    "RelativeSupport",
+    "Score",
+    "ScoreAnswer",
+    "State",
+    "StructuredProbeEncoder",
+    "TransformersRuntime",
+    "Usage",
+]

llm2jev/application/__init__.py

@@ -0,0 +1,8 @@
+"""Application entry points expose services and bound evaluators. This facade exports
+execution handles without allocating, closing, or owning concrete model resources.
+"""
+
+from .bound import AsyncBoundEvaluator, BoundEvaluator
+from .service import AsyncLLM2Jev, LLM2Jev
+
+__all__ = ["AsyncBoundEvaluator", "AsyncLLM2Jev", "BoundEvaluator", "LLM2Jev"]

llm2jev/application/bound.py

@@ -0,0 +1,103 @@
+"""Execution handles for a captured rule plan and a borrowed scoring runtime. Each call
+owns preparation and response projection, and iterators consume records lazily in source
+order. Cancellation and failures propagate without closing a runtime that another
+evaluator may still need.
+"""
+from __future__ import annotations
+
+from typing import AsyncIterable, AsyncIterator, Iterable, Iterator, Union
+
+from dataclasses import dataclass, field
+
+from ..contracts import AsyncBinaryRuntime, BinaryRuntime
+from ..core.response import JevResponse
+from ..core.types import State, JSONValue
+from ..inference.compilation.plan import DecisionPlan
+from .pipeline import EvaluationPipeline
+
+
+@dataclass(frozen=True, init=False)
+class BoundEvaluator:
+    """A validated rule snapshot borrowing a synchronous runtime."""
+
+    _runtime: BinaryRuntime = field(repr=False)
+    _pipeline: EvaluationPipeline = field(repr=False)
+    _plan: DecisionPlan = field(repr=False)
+    model: str
+
+    # Retain the validated plan and borrowed collaborators without capturing any per-record input.
+    def __init__(
+        self,
+        *,
+        _runtime: BinaryRuntime,
+        _pipeline: EvaluationPipeline,
+        _plan: DecisionPlan,
+        model: str,
+    ) -> None:
+        object.__setattr__(self, "_runtime", _runtime)
+        object.__setattr__(self, "_pipeline", _pipeline)
+        object.__setattr__(self, "_plan", _plan)
+        object.__setattr__(self, "model", model)
+
+    # Prepare an isolated record, score its ordered candidates, then project against that same
+    # plan.
+    def evaluate(self, *, state: State) -> JevResponse:
+        prepared = self._pipeline.prepare(plan=self._plan, model=self.model, state=state)
+        output = self._runtime.score(model=prepared.model, prompts=prepared.prompts)
+        return self._pipeline.finish(prepared, output)
+
+    # Pull one record at a time and stop on failure; reject strings that would otherwise stream
+    # characters.
+    def iter_evaluate(self, states: Iterable[State]) -> Iterator[JevResponse]:
+        """Consume states lazily in order, stopping on the first error."""
+        if isinstance(states, (str, bytes, bytearray)):
+            raise ValueError("states must be an iterable of state values, not a string")
+        for state in states:
+            yield self.evaluate(state=state)
+
+
+@dataclass(frozen=True, init=False)
+class AsyncBoundEvaluator:
+    """A validated rule snapshot borrowing a native asynchronous runtime."""
+
+    _runtime: AsyncBinaryRuntime = field(repr=False)
+    _pipeline: EvaluationPipeline = field(repr=False)
+    _plan: DecisionPlan = field(repr=False)
+    model: str
+
+    # Store binding-time policy separately from the state snapshots created by later awaited
+    # calls.
+    def __init__(
+        self,
+        *,
+        _runtime: AsyncBinaryRuntime,
+        _pipeline: EvaluationPipeline,
+        _plan: DecisionPlan,
+        model: str,
+    ) -> None:
+        object.__setattr__(self, "_runtime", _runtime)
+        object.__setattr__(self, "_pipeline", _pipeline)
+        object.__setattr__(self, "_plan", _plan)
+        object.__setattr__(self, "model", model)
+
+    # Keep preparation immutable across the await boundary so caller edits cannot change response
+    # assembly.
+    async def evaluate(self, *, state: State) -> JevResponse:
+        prepared = self._pipeline.prepare(plan=self._plan, model=self.model, state=state)
+        output = await self._runtime.score(model=prepared.model, prompts=prepared.prompts)
+        return self._pipeline.finish(prepared, output)
+
+    # Consume either source protocol sequentially; no hidden buffering, retries, or parallel model
+    # calls occur.
+    async def iter_evaluate(
+        self, states: Union[Iterable[State], AsyncIterable[State]],
+    ) -> AsyncIterator[JevResponse]:
+        """Consume a sync or async source sequentially without buffering it."""
+        if isinstance(states, (str, bytes, bytearray)):
+            raise ValueError("states must be an iterable of state values, not a string")
+        if isinstance(states, AsyncIterable):
+            async for state in states:
+                yield await self.evaluate(state=state)
+        else:
+            for state in states:
+                yield await self.evaluate(state=state)

llm2jev/application/pipeline.py

@@ -0,0 +1,119 @@
+"""Shared preparation and completion boundary for both execution modes. An evaluation
+freezes one state tree and attaches it to the precompiled candidates before encoding.
+Runtime calls occur outside this module; completion interprets evidence only against the
+captured plan, never mutable source rules.
+"""
+from __future__ import annotations
+
+from typing import Optional, Tuple
+
+from dataclasses import dataclass, field
+
+from ..contracts import BinaryLabels, BinaryScores, ChatPrompt, InvalidRequestError, ModelIdentity
+from ..core.response import JevResponse
+from ..core.types import State, JSONValue
+from ..core.validation import document_snapshot, model_name
+from ..inference.projection import DecisionProjector
+from ..inference.probes import CandidateProbe
+from ..inference.compilation.plan import DecisionPlan
+from ..inference.distribution import DistributionPolicy
+from ..inference.encoding import StructuredProbeEncoder, ProbeEncoder
+
+
+@dataclass(frozen=True, init=False)
+class PreparedEvaluation:
+    plan: DecisionPlan = field(repr=False)
+    model: str
+    state: State = field(repr=False)
+    prompts: Tuple[ChatPrompt, ...] = field(repr=False)
+
+    # Keep plan, frozen state, and rendered prompts together for one evaluation's alignment and
+    # lifetime.
+    def __init__(
+        self,
+        *,
+        plan: DecisionPlan,
+        model: str,
+        state: State,
+        prompts: Tuple[ChatPrompt, ...],
+    ) -> None:
+        object.__setattr__(self, "plan", plan)
+        object.__setattr__(self, "model", model)
+        object.__setattr__(self, "state", state)
+        object.__setattr__(self, "prompts", prompts)
+
+
+@dataclass(frozen=True, init=False)
+class EvaluationPipeline:
+    """The shared, model-independent preparation and assembly boundary."""
+
+    encoder: ProbeEncoder = field(repr=False)
+    projector: DecisionProjector = field(repr=False)
+    model_identity: Optional[ModelIdentity] = None
+
+    # Capture encoding, projection, and optional identity policy without importing a concrete
+    # provider.
+    def __init__(
+        self,
+        *,
+        encoder: ProbeEncoder,
+        projector: DecisionProjector,
+        model_identity: Optional[ModelIdentity] = None,
+    ) -> None:
+        object.__setattr__(self, "encoder", encoder)
+        object.__setattr__(self, "projector", projector)
+        object.__setattr__(self, "model_identity", model_identity)
+
+    # Reject invalid or unserved names before candidate materialization or runtime access.
+    def validate_model(self, model: str) -> None:
+        try:
+            model_name(model)
+        except ValueError as error:
+            raise InvalidRequestError(str(error)) from error
+        if self.model_identity is not None:
+            self.model_identity.validate(model)
+
+    # Freeze one state tree, share it across candidate views, and materialize prompts in plan
+    # order.
+    def prepare(self, *, plan: DecisionPlan, model: str, state: State) -> PreparedEvaluation:
+        self.validate_model(model)
+        try:
+            snapshot = document_snapshot(state, "state", immutable=True)
+        except ValueError as error:
+            raise InvalidRequestError(str(error)) from error
+        prompts = tuple(self.encoder.encode(CandidateProbe._from_snapshot(
+            candidate=candidate, state=snapshot,
+        )) for candidate in plan.candidates)
+        return PreparedEvaluation(plan=plan, model=model, state=snapshot, prompts=prompts)
+
+    # Interpret runtime evidence using the prepared plan, never by rereading caller-owned question
+    # mappings.
+    def finish(self, prepared: PreparedEvaluation, output: BinaryScores) -> JevResponse:
+        return self.projector.project(
+            plan=prepared.plan, model=prepared.model, scores=output,
+        )
+
+
+# Resolve default labels once and check extension compatibility before creating shared execution
+# policy.
+def create_pipeline(
+    *, runtime: object, encoder: Optional[ProbeEncoder],
+    distribution: DistributionPolicy, model_identity: Optional[ModelIdentity],
+) -> EvaluationPipeline:
+    # Label metadata is optional for third-party implementations of the score protocol.
+    labels = getattr(runtime, "labels", None)
+    if labels is not None and not isinstance(labels, BinaryLabels):
+        raise TypeError("runtime labels must be BinaryLabels")
+    selected = encoder if encoder is not None else StructuredProbeEncoder(
+        labels=labels if labels is not None else BinaryLabels(),
+    )
+    if not callable(getattr(selected, "encode", None)):
+        raise TypeError("encoder must implement encode")
+    if isinstance(selected, StructuredProbeEncoder) and labels is not None and selected.labels != labels:
+        raise ValueError("encoder and runtime must use the same binary labels")
+    if model_identity is not None and not isinstance(model_identity, ModelIdentity):
+        raise TypeError("model_identity must be ModelIdentity")
+    return EvaluationPipeline(
+        encoder=selected, projector=DecisionProjector(distribution=distribution),
+        model_identity=model_identity,
+    )

llm2jev/application/service.py

@@ -0,0 +1,120 @@
+"""Configure decision evaluation and bind reusable definitions to scoring ports. Sync and
+async entry points share validation and plan compilation, while their execution modes
+remain explicit. Services borrow runtimes; changing service configuration affects future
+bindings rather than existing evaluator snapshots.
+"""
+from __future__ import annotations
+
+from typing import Mapping, Optional, Tuple, Union
+
+
+from ..contracts import AsyncBinaryRuntime, BinaryRuntime, ModelIdentity
+from ..core.request import JevRequest, Question
+from ..core.response import JevResponse
+from ..inference.distribution import DistributionPolicy, RelativeSupport
+from ..inference.compilation.compiler import RuleCompiler
+from ..inference.compilation.plan import DecisionPlan
+from ..inference.encoding import ProbeEncoder
+from .bound import AsyncBoundEvaluator, BoundEvaluator
+from .pipeline import EvaluationPipeline, create_pipeline
+
+
+class _EvaluationService:
+    """One configuration and binding policy for both execution modes."""
+
+    # Capture extension configuration and validate it before accepting bindings; runtime ownership
+    # remains external.
+    def __init__(
+        self,
+        *,
+        runtime: Union[BinaryRuntime, AsyncBinaryRuntime],
+        encoder: Optional[ProbeEncoder] = None,
+        distribution: DistributionPolicy = RelativeSupport(),
+        model_identity: Optional[ModelIdentity] = None,
+    ) -> None:
+        self.runtime = runtime
+        self.encoder = encoder
+        self.distribution = distribution
+        self.model_identity = model_identity
+        self._compiler = RuleCompiler()
+        self.encoder = self._pipeline().encoder
+
+    # Build a validated policy snapshot so future service edits do not change already-bound
+    # evaluators.
+    def _pipeline(self) -> EvaluationPipeline:
+        return create_pipeline(
+            runtime=self.runtime, encoder=self.encoder,
+            distribution=self.distribution, model_identity=self.model_identity,
+        )
+
+    # Validate the model before compiling rules, preventing unsupported identities from reaching
+    # any scorer.
+    def _binding(
+        self, model: str, questions: Mapping[str, Question],
+    ) -> Tuple[EvaluationPipeline, DecisionPlan]:
+        pipeline = self._pipeline()
+        pipeline.validate_model(model)
+        return pipeline, self._compiler.compile(questions)
+
+
+class LLM2Jev(_EvaluationService):
+    """Evaluate dynamic requests or bind reusable rules to a scoring runtime."""
+
+    runtime: BinaryRuntime
+
+    # Keep a concrete keyword-only signature for Python 3.8 introspection while sharing
+    # configuration policy.
+    def __init__(
+        self, *, runtime: BinaryRuntime, encoder: Optional[ProbeEncoder] = None,
+        distribution: DistributionPolicy = RelativeSupport(),
+        model_identity: Optional[ModelIdentity] = None,
+    ) -> None:
+        super().__init__(
+            runtime=runtime, encoder=encoder, distribution=distribution,
+            model_identity=model_identity,
+        )
+
+    # Capture one rule plan and policy set; returned evaluators borrow rather than close the
+    # runtime.
+    def bind(self, *, model: str, questions: Mapping[str, Question]) -> BoundEvaluator:
+        pipeline, plan = self._binding(model, questions)
+        return BoundEvaluator(
+            _runtime=self.runtime, _pipeline=pipeline,
+            _plan=plan, model=model,
+        )
+
+    # Route one-shot requests through binding so dynamic and reusable evaluation have identical
+    # semantics.
+    def evaluate(self, request: JevRequest) -> JevResponse:
+        return self.bind(model=request.model, questions=request.questions).evaluate(state=request.state)
+
+
+class AsyncLLM2Jev(_EvaluationService):
+    """The native async entry point using the same compiler and pipeline."""
+
+    runtime: AsyncBinaryRuntime
+
+    # Expose the asynchronous port explicitly without creating an event loop or acquiring model
+    # resources.
+    def __init__(
+        self, *, runtime: AsyncBinaryRuntime, encoder: Optional[ProbeEncoder] = None,
+        distribution: DistributionPolicy = RelativeSupport(),
+        model_identity: Optional[ModelIdentity] = None,
+    ) -> None:
+        super().__init__(
+            runtime=runtime, encoder=encoder, distribution=distribution,
+            model_identity=model_identity,
+        )
+
+    # Binding is synchronous rule work; only the returned evaluator's model execution needs
+    # awaiting.
+    def bind(self, *, model: str, questions: Mapping[str, Question]) -> AsyncBoundEvaluator:
+        pipeline, plan = self._binding(model, questions)
+        return AsyncBoundEvaluator(
+            _runtime=self.runtime, _pipeline=pipeline,
+            _plan=plan, model=model,
+        )
+
+    # Reuse the binding path and propagate cancellation through the asynchronous scoring call.
+    async def evaluate(self, request: JevRequest) -> JevResponse:
+        return await self.bind(model=request.model, questions=request.questions).evaluate(state=request.state)

llm2jev/contracts.py

@@ -0,0 +1,135 @@
+"""Model-independent ports connecting application execution to binary scorers. Ordered
+scores, label pairs, identity metadata, and error categories cross this boundary; model
+libraries and inference implementations do not. A runtime supplies evidence and measured
+usage, never assembled application answers.
+"""
+from __future__ import annotations
+
+from typing import Literal, Protocol, Sequence, Tuple, TypedDict
+
+from dataclasses import dataclass, field
+
+from .core.response import Usage
+from .core.validation import model_name
+from .utils.probability import support_values
+
+
+class ChatMessage(TypedDict):
+    role: Literal['system', 'user']
+    content: str
+
+
+ChatPrompt = Tuple[ChatMessage, ...]
+
+
+@dataclass(frozen=True, init=False)
+class BinaryLabels:
+    """The two answer labels shared by a encoder and a scoring runtime."""
+
+    yes: str = "yes"
+    no: str = "no"
+
+    # Construct an explicit pair of answer tokens; provider-specific tokenization is checked
+    # later.
+    def __init__(
+        self,
+        *,
+        yes: str = 'yes',
+        no: str = 'no',
+    ) -> None:
+        object.__setattr__(self, "yes", yes)
+        object.__setattr__(self, "no", no)
+        self.__post_init__()
+
+    # Distinct nonblank labels are required to define two separate outcomes of the same
+    # experiment.
+    def __post_init__(self) -> None:
+        if any(not isinstance(value, str) or not value.strip() for value in (self.yes, self.no)):
+            raise ValueError("binary labels must be non-empty strings")
+        if self.yes == self.no:
+            raise ValueError("binary labels must be different")
+
+
+@dataclass(frozen=True, init=False)
+class ModelIdentity:
+    """A loaded model and the explicit request names accepted for it."""
+
+    name: str
+    aliases: Sequence[str] = ()
+
+    # Capture the loaded model name and accepted request aliases without implicitly routing
+    # between models.
+    def __init__(
+        self,
+        *,
+        name: str,
+        aliases: Sequence[str] = (),
+    ) -> None:
+        object.__setattr__(self, "name", name)
+        object.__setattr__(self, "aliases", aliases)
+        self.__post_init__()
+
+    # Detach alias storage and reject malformed names before any identity comparison is attempted.
+    def __post_init__(self) -> None:
+        model_name(self.name)
+        if not isinstance(self.aliases, Sequence) or isinstance(self.aliases, (str, bytes)):
+            raise ValueError("model aliases must be a sequence of names")
+        aliases = tuple(self.aliases)
+        if any(not isinstance(alias, str) or not alias.strip() for alias in aliases):
+            raise ValueError("model aliases must be non-empty strings")
+        object.__setattr__(self, "aliases", aliases)
+
+    # Match only explicitly accepted names; the HTTP layer sanitizes this typed failure for public
+    # responses.
+    def validate(self, model: str) -> None:
+        if model != self.name and model not in self.aliases:
+            raise ModelNotFoundError(f"The model {model!r} does not exist")
+
+
+class InvalidRequestError(ValueError):
+    """Input cannot be evaluated; no scoring call has been made."""
+
+
+class ModelNotFoundError(ValueError):
+    """A request names a model outside an explicitly configured identity."""
+
+
+@dataclass(frozen=True, init=False)
+class BinaryScores:
+    """One ordered P(yes) per prompt, conditional on the two labels."""
+
+    yes_probabilities: Sequence[float]
+    usage: Usage = field(default_factory=Usage)
+
+    # Pair candidate-ordered support with measured usage, allowing unknown counters to remain
+    # unknown.
+    def __init__(
+        self,
+        *,
+        yes_probabilities: Sequence[float],
+        usage: Usage = Usage(),
+    ) -> None:
+        object.__setattr__(self, "yes_probabilities", yes_probabilities)
+        object.__setattr__(self, "usage", usage)
+        self.__post_init__()
+
+    # Copy and validate independent supports without imposing categorical unit-mass normalization.
+    def __post_init__(self) -> None:
+        probabilities = support_values(self.yes_probabilities)
+        if not isinstance(self.usage, Usage):
+            raise ValueError("usage must be a Usage instance")
+        object.__setattr__(self, "yes_probabilities", probabilities)
+
+
+class BinaryRuntime(Protocol):
+    # Implementations must return one label-conditioned support per prompt in the same order, or
+    # raise.
+    def score(self, *, model: str, prompts: Sequence[ChatPrompt]) -> BinaryScores:
+        """Return next-token binary probabilities in input order."""
+
+
+class AsyncBinaryRuntime(Protocol):
+    # Awaitable implementations preserve ordering and cancellation; they return evidence rather
+    # than answers.
+    async def score(self, *, model: str, prompts: Sequence[ChatPrompt]) -> BinaryScores:
+        """Await ordered binary scores without blocking the event loop."""

llm2jev/core/__init__.py

@@ -0,0 +1,26 @@
+"""Domain facade gathers question, answer, request, response, and JSON vocabulary. Imports
+remain independent of runtimes and web frameworks so protocol values work in minimal
+installations.
+"""
+from .answers import Answer, ChoiceAnswer, NoulAnswer, ScoreAnswer
+from .questions import Choice, Noul, Score
+from .request import JevRequest, Question
+from .response import JevResponse, Usage
+from .types import JSONContent, JSONValue, State
+
+__all__ = [
+    "Answer",
+    "Choice",
+    "ChoiceAnswer",
+    "JSONContent",
+    "JSONValue",
+    "JevRequest",
+    "Question",
+    "JevResponse",
+    "Noul",
+    "NoulAnswer",
+    "Score",
+    "ScoreAnswer",
+    "State",
+    "Usage",
+]