premanmcp 1.1.13 → 1.1.15

package/bin/preman/__init__.py

@@ -0,0 +1,267 @@
+"""Record an agent's tool calls during a PreMan eval, and do nothing otherwise.
+
+PreMan's `eval target` writes this into each file it decorates:
+
+    try:
+        from preman import tool as preman_tool
+    except ImportError:  # PreMan's eval recorder; without it, decorate with nothing
+        def preman_tool(name):
+            return lambda fn: fn
+
+    @preman_tool("query_prices")      # the name the model sees for this tool
+    def query_prices(gpu: str) -> list[dict]:
+        ...
+
+This package ships inside PreMan's CLI and is importable only inside an eval,
+so everywhere else the fallback leaves the function untouched. Inside an eval
+with ``PREMAN_EVAL_RUN`` set, each call runs through OpenInference's tool
+helper, which creates the OpenTelemetry TOOL span the eval collects.
+
+Recording must never change what the agent sees. If recording fails, the tool
+still runs exactly once and the failure is reported to the eval instead.
+"""
+
+from __future__ import annotations
+
+import functools
+import inspect
+import os
+import sys
+import threading
+import time
+from pathlib import Path
+from typing import Any, Callable, Optional
+
+__all__ = ["tool", "recording_errors", "decorated"]
+__version__ = "0.1.0"
+
+_EVAL_VAR = "PREMAN_EVAL_RUN"
+
+#: Set by PreMan's eval wrapper (.preman/preman_agent_entry.py) and called as
+#: observer(event, tool_name, detail), event one of "start", "end", "broken".
+#: None everywhere else.
+_observer: Optional[Callable[[str, str, str], None]] = None
+
+_lock = threading.Lock()
+_errors: list[tuple[float, str, str]] = []
+_tracer: list[Any] = []
+#: (file, tool name, placed) for each @preman_tool applied in this process.
+#: placed is False when it was handed something other than a function: it sits
+#: above a decorator that returned an object, and calls can go around it.
+_decorated: list[tuple[str, str, bool]] = []
+
+
+def recording_errors(since: float = 0.0) -> list[tuple[float, str, str]]:
+    """Recording failures as (time.monotonic(), tool name, message), oldest first."""
+    with _lock:
+        return [e for e in _errors if e[0] >= since]
+
+
+def decorated() -> list[tuple[str, str, bool]]:
+    """Every @preman_tool applied so far, as (resolved file, tool name, placed)."""
+    with _lock:
+        return list(_decorated)
+
+
+def _register(name: str, fn: Any) -> None:
+    # Never raises: a caller it cannot read only leaves the tool off the list.
+    try:
+        file = str(Path(sys._getframe(2).f_code.co_filename).resolve())
+        placed = inspect.isroutine(inspect.unwrap(fn))
+    except Exception:
+        return
+    with _lock:
+        _decorated.append((file, name, placed))
+
+
+def _notify(event: str, name: str, detail: str = "") -> None:
+    observer = _observer
+    if observer is None:
+        return
+    try:
+        observer(event, name, detail)
+    except Exception:
+        pass
+
+
+def _describe(exc: BaseException) -> str:
+    # str(exc) runs the exception's own __str__, which may itself raise.
+    try:
+        return "%s: %s" % (type(exc).__name__, exc)
+    except Exception:
+        return type(exc).__name__
+
+
+def _broken(name: str, exc: BaseException | str) -> None:
+    message = exc if isinstance(exc, str) else _describe(exc)
+    with _lock:
+        _errors.append((time.monotonic(), name, message))
+    _notify("broken", name, message)
+
+
+def _unrecorded() -> str:
+    """Why the current tool span records nothing, or "" when it records.
+
+    Without an SDK TracerProvider (or with OTEL_SDK_DISABLED) the tool span is
+    a no-op, so the call would go unrecorded with no error to show for it.
+    Reported once the call is over, so one call gives at most one error."""
+    try:
+        from opentelemetry import trace
+
+        if trace.get_current_span().is_recording():
+            return ""
+        return "no span collector is listening in this run, so the call was not recorded"
+    except Exception as exc:
+        return _describe(exc)
+
+
+def _oi_tracer() -> Any:
+    # Imported on first recorded call, so nothing outside a recording eval
+    # loads OpenTelemetry's SDK or OpenInference because of this package.
+    if not _tracer:
+        from openinference.instrumentation import OITracer, TraceConfig
+        from opentelemetry import trace
+
+        _tracer.append(OITracer(trace.get_tracer("preman-trace"), config=TraceConfig()))
+    return _tracer[0]
+
+
+def _target(fn: Callable[..., Any], args: tuple) -> tuple[Callable[..., Any], tuple]:
+    """The callable whose arguments are recorded, and its positional arguments.
+    A method's self/cls is bound first so it is not recorded as an argument."""
+    params = list(inspect.signature(fn).parameters)
+    if params and params[0] in ("self", "cls") and args:
+        return functools.partial(fn, args[0]), args[1:]
+    return fn, args
+
+
+def _stand_in(fn: Callable[..., Any], target: Callable[..., Any], state: dict, is_async: bool):
+    # A fresh stand-in per call, so 'entered', 'done' and 'error' belong to
+    # this call only. Its signature is the target's, so OpenInference records
+    # arguments by name; its name and docstring are the tool's.
+    name = state["name"]
+    if is_async:
+        async def inner(*a, **k):
+            state["entered"] = True
+            _notify("start", name)
+            state["unrecorded"] = _unrecorded()
+            try:
+                result = await target(*a, **k)
+            except BaseException as exc:
+                state["error"] = exc
+                raise
+            finally:
+                _notify("end", name)
+            state["done"], state["result"] = True, result
+            return result
+    else:
+        def inner(*a, **k):
+            state["entered"] = True
+            _notify("start", name)
+            state["unrecorded"] = _unrecorded()
+            try:
+                result = target(*a, **k)
+            except BaseException as exc:
+                state["error"] = exc
+                raise
+            finally:
+                _notify("end", name)
+            state["done"], state["result"] = True, result
+            return result
+    inner.__signature__ = inspect.signature(target)
+    inner.__name__ = getattr(fn, "__name__", "tool")
+    inner.__qualname__ = getattr(fn, "__qualname__", inner.__name__)
+    inner.__doc__ = fn.__doc__
+    return inner
+
+
+def tool(name: str):
+    """Record each call of the decorated function as the tool ``name``."""
+    if not isinstance(name, str) or not name.strip():
+        raise TypeError('preman.tool needs the tool name the model sees, e.g. @preman_tool("query_prices")')
+
+    def decorate(fn):
+        if inspect.isgeneratorfunction(fn) or inspect.isasyncgenfunction(fn):
+            raise TypeError("preman.tool cannot record a generator function (%s)" % getattr(fn, "__qualname__", fn))
+        _register(name, fn)
+        is_async = inspect.iscoroutinefunction(fn)
+
+        def prepare(args, kwargs):
+            # None means "call the tool directly": either the call does not fit
+            # its signature (so it fails exactly as it would), or the call
+            # cannot be prepared for recording, which is reported.
+            try:
+                signature = inspect.signature(fn)
+            except Exception as exc:  # e.g. ValueError: a builtin with no signature
+                _broken(name, exc)
+                return None
+            try:
+                signature.bind(*args, **kwargs)
+            except TypeError:
+                return None
+            try:
+                target, rest = _target(fn, args)
+                state = {"name": name, "entered": False, "done": False, "result": None, "error": None,
+                         "unrecorded": ""}
+                return state, rest, _stand_in(fn, target, state, is_async)
+            except Exception as exc:
+                _broken(name, exc)
+                return None
+
+        if is_async:
+            @functools.wraps(fn)
+            async def recorded(*args, **kwargs):
+                if not os.environ.get(_EVAL_VAR):
+                    return await fn(*args, **kwargs)
+                prepared = prepare(args, kwargs)
+                if prepared is None:
+                    return await fn(*args, **kwargs)
+                state, rest, inner = prepared
+                try:
+                    result = await _oi_tracer().tool(name=name)(inner)(*rest, **kwargs)
+                except Exception as exc:
+                    if state["done"]:
+                        _broken(name, exc)
+                        return state["result"]
+                    if state["error"] is not None:
+                        if exc is not state["error"]:
+                            _broken(name, exc)
+                        elif state["unrecorded"]:
+                            _broken(name, state["unrecorded"])
+                        raise state["error"]
+                    _broken(name, exc)
+                    return await fn(*args, **kwargs)
+                if state["unrecorded"]:
+                    _broken(name, state["unrecorded"])
+                return result
+        else:
+            @functools.wraps(fn)
+            def recorded(*args, **kwargs):
+                if not os.environ.get(_EVAL_VAR):
+                    return fn(*args, **kwargs)
+                prepared = prepare(args, kwargs)
+                if prepared is None:
+                    return fn(*args, **kwargs)
+                state, rest, inner = prepared
+                try:
+                    result = _oi_tracer().tool(name=name)(inner)(*rest, **kwargs)
+                except Exception as exc:
+                    if state["done"]:
+                        _broken(name, exc)
+                        return state["result"]
+                    if state["error"] is not None:
+                        if exc is not state["error"]:
+                            _broken(name, exc)
+                        elif state["unrecorded"]:
+                            _broken(name, state["unrecorded"])
+                        raise state["error"]
+                    _broken(name, exc)
+                    return fn(*args, **kwargs)
+                if state["unrecorded"]:
+                    _broken(name, state["unrecorded"])
+                return result
+
+        recorded.__preman_tool__ = name
+        return recorded
+
+    return decorate

package/bin/shared.js

@@ -23,6 +23,11 @@ import path from "node:path";
 import { createInterface } from "node:readline/promises";
 import { fileURLToPath } from "node:url";
 
+// Generated wrappers import tool_proxy.py from this folder (design spec §6.0.2).
+// Set here because every entry point imports this module, and the wrapper check
+// (`python -c`) and PYTHONSAFEPATH both drop the script folder from sys.path.
+process.env.PREMAN_CLI_BIN = path.dirname(fileURLToPath(import.meta.url));
+
 export const DEFAULT_BACKEND = "https://api.preman.live";
 export const DEFAULT_FRONTEND = "https://app.preman.live";
 export const CREDENTIALS_DIR = path.join(os.homedir(), ".preman");