slmeval-cli 0.1.0__py3-none-any.whl

slmeval/__init__.py

@@ -0,0 +1,71 @@
+"""slmeval — benchmark agent workflows across small language models.
+
+The public API is intentionally tiny. Integration is two touch points:
+
+    from slmeval import chat_model, node, sandbox_tool
+
+    async def run_case(case_input: dict) -> dict:
+        with node("intent_router"):               # names the row in the scorecard
+            llm = chat_model()                    # routed per candidate, per node
+            decision = await llm.complete(msgs, schema=Routing)
+
+`node` is a no-op outside a benchmark run, and `chat_model` falls through to
+whichever model you name for production, so this is safe to leave in shipped
+code:
+
+    llm = chat_model(provider="openai", model="gpt-4o-mini")
+
+The second touch point is `@sandbox_tool`, which records a tool call once and
+then replays it for every candidate, so a 90-execution matrix never touches your
+real APIs:
+
+    @sandbox_tool(match_on=["order_id"], ignore_args=["request_id"])
+    async def fetch_order(order_id: str, request_id: str) -> dict:
+        ...
+
+It is a no-op outside a benchmark too, so in production your tools do exactly
+what they always did.
+
+With LangGraph there is a third, which replaces the `node()` blocks entirely:
+
+    builder = slmeval.instrument_langgraph(StateGraph(State))
+    builder.add_node("intent_router", route)
+
+Every node then names itself in the scorecard, and the graph still runs
+unchanged outside a benchmark.
+"""
+
+from __future__ import annotations
+
+__version__ = "0.1.0"
+
+from slmeval.adapters import instrument_langgraph, instrument_node
+from slmeval.core.context import RunContext, node
+from slmeval.core.router import ChatModel, chat_model
+from slmeval.providers.base import (
+    Completion,
+    Msg,
+    ProviderError,
+    StructuredOutputError,
+    ToolCall,
+    ToolSpec,
+)
+from slmeval.sandbox.tool import CassetteMiss, sandbox_tool
+
+__all__ = [
+    "CassetteMiss",
+    "ChatModel",
+    "Completion",
+    "Msg",
+    "ProviderError",
+    "RunContext",
+    "StructuredOutputError",
+    "ToolCall",
+    "ToolSpec",
+    "__version__",
+    "chat_model",
+    "instrument_langgraph",
+    "instrument_node",
+    "node",
+    "sandbox_tool",
+]

slmeval/__main__.py

@@ -0,0 +1,14 @@
+"""``python -m slmeval``, for when the console script is not on PATH.
+
+A fresh virtualenv on Windows often is not activated in the shell the user
+happens to be in, and "slmeval: command not found" five seconds after a
+successful install is a bad first five minutes. This is the same entry point
+the ``slmeval`` command uses.
+"""
+
+from __future__ import annotations
+
+from slmeval.cli.main import main
+
+if __name__ == "__main__":
+    main()

slmeval/adapters/__init__.py

@@ -0,0 +1,24 @@
+"""Framework adapters: the same benchmark, without the user marking anything.
+
+There are two, and the gradient between them is deliberate.
+
+:mod:`slmeval.adapters.instrument` is the generic machinery - a wrapper that
+turns one callable into a timed, attributed, traced node. It knows nothing
+about any framework and is what the others are built from.
+
+:mod:`slmeval.adapters.langgraph` wraps every node of a LangGraph builder, so
+a user with a graph writes one line and marks nothing by hand.
+
+The adapter is never where the measurement happens. Model routing comes from
+the ambient context, tool interception from ``@sandbox_tool``, and both work
+with no adapter at all - so an adapter can only ever *add* attribution. That is
+what keeps a framework integration from becoming a second implementation of the
+harness, which is the way integrations usually rot.
+"""
+
+from __future__ import annotations
+
+from slmeval.adapters.instrument import instrument_node, state_keys
+from slmeval.adapters.langgraph import instrument_langgraph
+
+__all__ = ["instrument_langgraph", "instrument_node", "state_keys"]

slmeval/adapters/instrument.py

@@ -0,0 +1,135 @@
+"""Turning one callable into an attributed node.
+
+This is the piece every framework adapter is built from, and it is where the
+blueprint's node wrapper had three bugs worth fixing by name, because each one
+corrupts a number the scorecard prints.
+
+**Exit through ``reset(token)``.** The blueprint restored the contextvar with
+``set(token.old_value)``, which is not the contextvars API and goes wrong the
+first time nodes nest - which is the first thing any real graph does. Entry and
+exit both go through :func:`slmeval.core.context.entering_node` here, so there
+is exactly one place that could get it wrong.
+
+**Record in ``finally``, not per-``except``.** The blueprint's sync wrapper
+caught the schema error and lost every other exception: a node that raised
+``KeyError`` left no trace at all, so the denominator of every per-node rate
+silently shrank. One ``finally``, one record, whatever happened - including
+cancellation, which is how a benchmark's own ``timeout_s`` arrives.
+
+**A crash is not a schema failure.** The blueprint recorded
+``schema_valid=True`` on the generic-exception path, which quietly mixed
+unrelated crashes into the one metric this tool exists to report. Schema
+validity is tri-state, it lives on the model call
+(:class:`~slmeval.core.trace.SchemaStatus`), and nothing here touches it: a
+node that raised records an *error*, which is a different column.
+
+Outside a benchmark the wrapper is a thin pass-through - it enters a node whose
+trajectory nobody is collecting and calls the original. That is what makes it
+safe to leave an instrumented graph in shipped code.
+"""
+
+from __future__ import annotations
+
+import inspect
+import time
+from collections.abc import Callable, Mapping
+from functools import wraps
+from typing import Any
+
+from slmeval.core.context import entering_node
+from slmeval.core.trace import NodeVisit
+
+ERROR_CHARS = 300
+"""How much of an exception message is kept.
+
+Enough to recognise, not enough to be a liability: one of these is stored per
+node per execution, and a 4 kB traceback times ninety executions is a trace
+store that costs more than the thing it measures. The *type* is what
+aggregates anyway.
+"""
+
+
+def state_keys(value: Any) -> tuple[str, ...]:
+    """The keys of a state object, sorted, or ``()`` if it has none.
+
+    Keys and never values. A trace store must not quietly become a copy of a
+    customer's conversation state, and the shape is what carries the finding
+    anyway: a node that was supposed to write ``refund_eligible`` and returned
+    an empty dict is visible from the keys alone.
+    """
+    if isinstance(value, Mapping):
+        return tuple(sorted(str(k) for k in value))
+    # Off the class, not the instance: Pydantic deprecated the instance
+    # attribute, and a benchmark that printed a DeprecationWarning per node per
+    # execution would teach people to filter our warnings.
+    owner = value if isinstance(value, type) else type(value)
+    fields = getattr(owner, "model_fields", None) or getattr(owner, "__dataclass_fields__", None)
+    if isinstance(fields, Mapping):
+        return tuple(sorted(str(k) for k in fields))
+    return ()
+
+
+def _record(
+    visit: NodeVisit | None, started: float, *, error: BaseException | None
+) -> None:
+    if visit is None:
+        return
+    visit.latency_ms = (time.perf_counter() - started) * 1000.0
+    if error is not None:
+        visit.error_type = type(error).__name__
+        visit.error = str(error)[:ERROR_CHARS] or None
+
+
+def instrument_node(name: str, func: Callable[..., Any]) -> Callable[..., Any]:
+    """Wrap ``func`` so that calling it enters the node ``name``.
+
+    Async-ness is preserved: a framework that awaits its nodes has to get a
+    coroutine function back, and one that calls them synchronously must not be
+    handed a coroutine it never awaits.
+    """
+    if inspect.iscoroutinefunction(func):
+
+        @wraps(func)
+        async def async_node(*args: Any, **kwargs: Any) -> Any:
+            with entering_node(name) as visit:
+                if visit is not None and args:
+                    visit.input_keys = state_keys(args[0])
+                started = time.perf_counter()
+                failure: BaseException | None = None
+                try:
+                    result = await func(*args, **kwargs)
+                except BaseException as exc:
+                    failure = exc
+                    raise
+                else:
+                    if visit is not None:
+                        visit.output_keys = state_keys(result)
+                    return result
+                finally:
+                    _record(visit, started, error=failure)
+
+        return async_node
+
+    @wraps(func)
+    def sync_node(*args: Any, **kwargs: Any) -> Any:
+        with entering_node(name) as visit:
+            if visit is not None and args:
+                visit.input_keys = state_keys(args[0])
+            started = time.perf_counter()
+            failure: BaseException | None = None
+            try:
+                result = func(*args, **kwargs)
+            except BaseException as exc:
+                failure = exc
+                raise
+            else:
+                if visit is not None:
+                    visit.output_keys = state_keys(result)
+                return result
+            finally:
+                _record(visit, started, error=failure)
+
+    return sync_node
+
+
+__all__ = ["ERROR_CHARS", "instrument_node", "state_keys"]

slmeval/adapters/langgraph.py

@@ -0,0 +1,237 @@
+"""The LangGraph adapter: one line, and every node is attributed.
+
+    import slmeval
+    from langgraph.graph import StateGraph
+
+    builder = slmeval.instrument_langgraph(StateGraph(State))
+    builder.add_node("intent_router", route)
+    builder.add_node("order_lookup", lookup)
+    graph = builder.compile()
+
+Nothing else changes. The user writes no ``with node(...)`` blocks and the
+scorecard still has a row per node - which is the whole value of a framework
+adapter, because per-node attribution is the part nobody will hand-annotate
+across a thirty-node graph.
+
+**Instrument the builder, then add the nodes.** That ordering is the supported
+path and the reason this adapter is boring: it replaces the builder's own
+``add_node``, so each node's callable is wrapped *before* LangGraph ever sees
+it, and from that point on the library's internals are none of our business.
+Calling it after the nodes are added also works - the registered callables are
+wrapped in place - but that path has to know how a given LangGraph version
+stores a node, and it says so loudly rather than wrapping nothing when it does
+not recognise one. An adapter that quietly attributed every call to ``None``
+would print a flat scorecard, which reads as a finding rather than as a bug.
+
+LangGraph itself is never imported here. It is an optional extra
+(``pip install "slmeval-cli[langgraph]"``), the core must install in seconds
+without a model framework, and the only thing this file needs to know is that
+a builder has named nodes holding callables.
+
+The adapter adds attribution and nothing else. Model routing still comes from
+the ambient context and tool interception from ``@sandbox_tool``, both of which
+work with no adapter at all - so an instrumented graph and a hand-marked one
+produce the same trajectory. ``examples/support_agent`` is benchmarked both
+ways to keep that true.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from typing import Any
+
+from slmeval.adapters.instrument import instrument_node
+from slmeval.errors import ConfigError
+
+MARK = "_slmeval_instrumented"
+"""Set on anything this module has wrapped, so a second call is a no-op.
+
+``instrument_langgraph(builder)`` twice - in a test fixture and again in the
+graph's own factory, say - would otherwise enter every node twice, and Tier 2
+would read the doubled trajectory as the agent looping.
+"""
+
+#: Where a node's callable lives, in the versions this has been built against.
+#: ``func``/``afunc`` are LangGraph's own ``RunnableCallable``; ``invoke``/
+#: ``ainvoke`` are the generic Runnable surface, wrapped only as a fallback.
+CALLABLE_ATTRS = ("func", "afunc", "invoke", "ainvoke")
+
+#: Where a builder keeps the runnable for a node.
+NODE_ATTRS = ("runnable", "action", "bound")
+
+
+def instrument_langgraph(builder: Any, *, prefix: str = "") -> Any:
+    """Wrap every node of a LangGraph builder so each one names itself.
+
+    Returns the same builder, so it reads as a pipeline step either way:
+
+        builder = slmeval.instrument_langgraph(StateGraph(State))
+        ...
+        graph = builder.compile()
+
+    ``prefix`` namespaces the node names, for an agent that composes two graphs
+    with a ``plan`` node in each. Without it the two share one scorecard row
+    and "which plan node is failing" has no answer.
+    """
+    if not hasattr(builder, "add_node"):
+        raise ConfigError(
+            f"instrument_langgraph() needs a graph builder, not {type(builder).__name__}",
+            hint=(
+                "Pass the StateGraph, not the compiled graph: a compiled "
+                "graph's nodes are already bound, and wrapping them would not "
+                "change what runs."
+            ),
+        )
+
+    _patch_add_node(builder, prefix)
+    existing = _existing_nodes(builder)
+    for name, holder in existing:
+        _wrap_node(_named(name, prefix), holder)
+
+    # The failure worth reporting is "nothing here was even a callable", not
+    # "nothing was wrapped": a second call finds every node already wrapped,
+    # which is success.
+    if existing and not any(_reachable(holder) for _, holder in existing):
+        raise ConfigError(
+            f"instrument_langgraph() could not reach the callable behind "
+            f"{len(existing)} already-registered node(s)",
+            hint=(
+                "Instrument the builder *before* adding nodes - "
+                "`builder = slmeval.instrument_langgraph(StateGraph(State))` - "
+                "which needs to know nothing about how your LangGraph version "
+                'stores a node. Or use adapter = "generic": call '
+                "slmeval.chat_model() in your node functions and wrap each in "
+                '`with node("name")`, for the same attribution by hand.'
+            ),
+        )
+    return builder
+
+
+# --------------------------------------------------------------------------- #
+# Before the nodes are added: replace add_node
+# --------------------------------------------------------------------------- #
+
+
+def _patch_add_node(builder: Any, prefix: str) -> None:
+    """Wrap each node's callable on its way in.
+
+    The supported path, and the one that needs no knowledge of LangGraph's
+    internals: ``add_node(name, action)`` is the public API, and the action is
+    still the user's own function when it arrives here.
+    """
+    original = builder.add_node
+    if getattr(original, MARK, False):
+        return
+
+    def add_node(node: Any = None, action: Any = None, **kwargs: Any) -> Any:
+        name, func, rest = _split_add_node_args(node, action)
+        if func is not None:
+            name = name or getattr(func, "__name__", None)
+            if name:
+                func = _instrument(_named(str(name), prefix), func)
+        if rest is None:
+            return original(name, func, **kwargs) if name else original(func, **kwargs)
+        return original(*rest, **kwargs)
+
+    add_node._slmeval_instrumented = True  # type: ignore[attr-defined]
+    builder.add_node = add_node
+
+
+def _split_add_node_args(
+    node: Any, action: Any
+) -> tuple[str | None, Callable[..., Any] | None, tuple[Any, ...] | None]:
+    """Make sense of ``add_node``'s two accepted shapes.
+
+    ``add_node("name", func)`` and ``add_node(func)`` - where the node takes
+    the function's own name - are both public, and a wrapper that handled only
+    the first would break every graph written in the second style. Anything
+    else is passed through untouched rather than guessed at.
+    """
+    if action is None and callable(node):
+        return None, node, None
+    if isinstance(node, str) and callable(action):
+        return node, action, None
+    return None, None, (node, action)
+
+
+# --------------------------------------------------------------------------- #
+# After the nodes are added: wrap what is registered
+# --------------------------------------------------------------------------- #
+
+
+def _existing_nodes(builder: Any) -> list[tuple[str, Any]]:
+    nodes = getattr(builder, "nodes", None)
+    if not isinstance(nodes, dict):
+        return []
+    return [(str(name), holder) for name, holder in nodes.items()]
+
+
+def _wrap_node(name: str, holder: Any) -> bool:
+    """Instrument one registered node in place. True if anything was wrapped.
+
+    The holder is a node spec whose ``runnable`` holds the callable, or - in
+    the simplest builders, and in the stub this is tested against - the
+    callable itself. The runnable is wrapped *in place* rather than replaced,
+    because LangGraph's executor calls methods on it that a bare function does
+    not have.
+    """
+    for attribute in NODE_ATTRS:
+        runnable = getattr(holder, attribute, None)
+        if runnable is not None and _wrap_in_place(name, runnable):
+            return True
+    if callable(holder):
+        return _wrap_in_place(name, holder)
+    return False
+
+
+def _wrap_in_place(name: str, runnable: Any) -> bool:
+    if getattr(runnable, MARK, False):
+        return False
+    wrapped = False
+    for attribute in CALLABLE_ATTRS:
+        current = getattr(runnable, attribute, None)
+        if current is None or not callable(current):
+            continue
+        if _set(runnable, attribute, _instrument(name, current)):
+            wrapped = True
+    if wrapped:
+        _set(runnable, MARK, True)
+    return wrapped
+
+
+def _instrument(name: str, func: Callable[..., Any]) -> Callable[..., Any]:
+    if getattr(func, MARK, False):
+        return func
+    wrapped = instrument_node(name, func)
+    _set(wrapped, MARK, True)
+    return wrapped
+
+
+def _set(obj: Any, attribute: str, value: Any) -> bool:
+    """Assign, through whatever the object allows. False if it allows nothing."""
+    try:
+        setattr(obj, attribute, value)
+    except (AttributeError, TypeError, ValueError):
+        try:
+            object.__setattr__(obj, attribute, value)
+        except (AttributeError, TypeError, ValueError):
+            return False
+    return True
+
+
+def _reachable(holder: Any) -> bool:
+    """Whether this node holds a callable the adapter could reach at all."""
+    for attribute in NODE_ATTRS:
+        runnable = getattr(holder, attribute, None)
+        if runnable is None:
+            continue
+        if any(callable(getattr(runnable, a, None)) for a in CALLABLE_ATTRS):
+            return True
+    return callable(holder)
+
+
+def _named(name: str, prefix: str) -> str:
+    return f"{prefix}{name}" if prefix else name
+
+
+__all__ = ["CALLABLE_ATTRS", "MARK", "NODE_ATTRS", "instrument_langgraph"]

slmeval/cli/__init__.py

@@ -0,0 +1,15 @@
+"""Terminal interface."""
+
+from __future__ import annotations
+
+__all__ = ["app", "main"]
+
+
+def __getattr__(name: str) -> object:
+    # Lazy so `import slmeval.cli` stays cheap and import cycles cannot form
+    # between main.py and the command modules.
+    if name in __all__:
+        from slmeval.cli import main as _main
+
+        return getattr(_main, name)
+    raise AttributeError(name)

slmeval/cli/commands/__init__.py

@@ -0,0 +1,35 @@
+"""One module per command."""
+
+from __future__ import annotations
+
+from slmeval.cli.commands import (
+    baseline_cmd,
+    cache_cmd,
+    dataset_cmd,
+    doctor_cmd,
+    gate_cmd,
+    init_cmd,
+    judge_cmd,
+    record_cmd,
+    report_cmd,
+    run_cmd,
+    show_cmd,
+    stubs,
+    validate_cmd,
+)
+
+__all__ = [
+    "baseline_cmd",
+    "cache_cmd",
+    "dataset_cmd",
+    "doctor_cmd",
+    "gate_cmd",
+    "init_cmd",
+    "judge_cmd",
+    "record_cmd",
+    "report_cmd",
+    "run_cmd",
+    "show_cmd",
+    "stubs",
+    "validate_cmd",
+]