variably-sdk 2.13.0__tar.gz → 2.14.0__tar.gz

{variably_sdk-2.13.0 → variably_sdk-2.14.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: variably-sdk
-Version: 2.13.0
+Version: 2.14.0
 Summary: Official Python SDK for Variably feature flags, LLM experimentation, and prompt optimization platform
 Author-email: Variably <support@variably.com>
 License: MIT

{variably_sdk-2.13.0 → variably_sdk-2.14.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "variably-sdk"
-version = "2.13.0"
+version = "2.14.0"
 description = "Official Python SDK for Variably feature flags, LLM experimentation, and prompt optimization platform"
 readme = "README.md"
 requires-python = ">=3.7"

{variably_sdk-2.13.0 → variably_sdk-2.14.0}/src/variably/client.py

@@ -819,9 +819,10 @@ class VariablyClient:
         # Multi-agent context — see docs/improvements/42.1-implementation-plan.md
         agent_name: Optional[str] = None,          # This step's agent, within a session_id trace
         is_aggregator: bool = False,               # True on the final step that aggregates the trace
-        tools_called: Optional[List[Dict[str, Any]]] = None,   # [{"name","arguments","output"}] actually called
+        tools_called: Optional[List[Dict[str, Any]]] = None,   # [{"name","arguments","output","error"}] actually called; "error" on a failed call
         expected_tools: Optional[List[Dict[str, Any]]] = None, # golden tools for deterministic trajectory scoring
         allowed_tools: Optional[List[str]] = None,             # permitted tool names; calls outside this list flag as unauthorized
+        max_steps: Optional[int] = None,                       # the golden's budget of tool calls for this run
         # User & organization context
         user_id: Optional[str] = None,             # End-user ID for per-user analytics
         tags: Optional[List[str]] = None,          # Grouping labels, e.g. ["production", "rag"]
@@ -939,6 +940,10 @@ class VariablyClient:
             data["expected_tools"] = expected_tools
         if allowed_tools:
             data["allowed_tools"] = allowed_tools
+        if max_steps is not None:
+            if not isinstance(max_steps, int) or isinstance(max_steps, bool) or max_steps < 1:
+                raise ValueError(f"max_steps must be a positive integer, got {max_steps!r}")
+            data["max_steps"] = max_steps
         if provider:
             data["provider"] = provider
         if model:

{variably_sdk-2.13.0 → variably_sdk-2.14.0}/src/variably/integrations/adk.py

@@ -490,6 +490,8 @@ class VariablyPlugin(BasePlugin):
         entry: Dict[str, Any] = {"name": name, "arguments": _args(tool_args)}
         if error is not None:
             entry["output"] = f"ERROR: {type(error).__name__}: {error}"[:500]
+            # Structured: a failed call is a wasted step (never read from output text).
+            entry["error"] = f"{type(error).__name__}: {error}"[:500]
         else:
             entry["output"] = _text_of(result)
             extractor = self._extractors.get(name)

{variably_sdk-2.13.0 → variably_sdk-2.14.0}/src/variably/integrations/langgraph.py

@@ -149,6 +149,40 @@ def observe_node(
     return decorate
 
 
+def tool_calls_from_messages(messages: Iterable[Any]) -> list:
+    """``tools_called`` from LangChain messages, for a ``tools_extractor``.
+
+    Each tool call an ``AIMessage`` requested (``.tool_calls``) is matched to the
+    ``ToolMessage`` that answered it (by ``tool_call_id``): its content becomes the
+    call's ``output``, and a ``ToolMessage`` with ``status="error"`` — how
+    LangGraph's ``ToolNode`` reports a tool that raised — sets ``error``, so the
+    failed call counts as a wasted step. Calls never answered are kept without an
+    output. Duck-typed: works with message objects or their dict form::
+
+        instrument(graph, tools_extractor=lambda state, update: {
+            "tools_called": tool_calls_from_messages(update.get("messages", [])),
+        })
+    """
+    def get(m, k, default=None):
+        return m.get(k, default) if isinstance(m, dict) else getattr(m, k, default)
+
+    calls, by_id = [], {}
+    for m in messages or []:
+        for tc in get(m, "tool_calls", None) or []:
+            entry = {"name": get(tc, "name", "") or "", "arguments": dict(get(tc, "args", None) or {})}
+            calls.append(entry)
+            if get(tc, "id"):
+                by_id[get(tc, "id")] = entry
+        tid = get(m, "tool_call_id")
+        if tid and tid in by_id:
+            content = get(m, "content", "")
+            text = content if isinstance(content, str) else str(content)
+            by_id[tid]["output"] = text[:2000]
+            if get(m, "status") == "error":
+                by_id[tid]["error"] = text[:500] or "error"
+    return calls
+
+
 def _detect_terminal_nodes(graph) -> set:
     """Terminal nodes = those with an edge to END, plus any with no outgoing
     edge. Best-effort across langgraph versions."""
@@ -236,8 +270,10 @@ def instrument(
             grounding cannot be scored (only the non-grounding dimensions).
         tools_extractor: custom fn ``(state, update) -> {"tools_called": [...],
             "expected_tools": [...]}`` (each tool a ``{"name","arguments","output"}``
-            dict) enabling the trajectory / tool-correctness / order-adherence
-            metrics. Return None when the node makes no tool calls.
+            dict, plus ``"error"`` on a failed call) enabling the trajectory /
+            tool-correctness / order-adherence and step-efficiency metrics.
+            ``tool_calls_from_messages`` builds ``tools_called`` from the
+            standard LangChain messages. Return None when the node makes no tool calls.
 
     Returns the same graph (for chaining). Best-effort; never raises.
     """

{variably_sdk-2.13.0 → variably_sdk-2.14.0}/src/variably/observe.py

@@ -185,13 +185,20 @@ def observe(
     agent_name: Optional[str] = None,
     is_aggregator: bool = False,
     # Tools this step actually called / was expected to call. Each dict:
-    # {"name": "...", "arguments": {...}, "output": "..."}. Enables the
+    # {"name": "...", "arguments": {...}, "output": "...", "error": "..."}.
+    # Set "error" (a short message) on a call that failed — the tool raised,
+    # timed out or exited non-zero: a failed call is a wasted step in Step
+    # efficiency (failures are never read from the output text). Enables the
     # deterministic trajectory precision/recall + tool-correctness metrics.
     tools_called: Optional[List[Dict[str, Any]]] = None,
     expected_tools: Optional[List[Dict[str, Any]]] = None,
     # Permitted-tool allowlist (names). Any call outside it flags as an
     # unauthorized action — the deterministic security/adversarial signal.
     allowed_tools: Optional[List[str]] = None,
+    # The golden's budget of tool calls for this run. A run over it fails Step
+    # efficiency; without it, only wasted calls (failed, repeated, after the
+    # golden's final tool) count against the run.
+    max_steps: Optional[int] = None,
 
     # --- User & organization context ---
     user_id: Optional[str] = None,            # Your end-user's ID — enables per-user analytics
@@ -297,6 +304,7 @@ def observe(
             tools_called=tools_called,
             expected_tools=expected_tools,
             allowed_tools=allowed_tools,
+            max_steps=max_steps,
             metadata=metadata,
             kind=kind,
         )

{variably_sdk-2.13.0 → variably_sdk-2.14.0}/src/variably/version.py

@@ -1,3 +1,3 @@
 """Version information for Variably SDK."""
 
-__version__ = "2.13.0"
+__version__ = "2.14.0"

{variably_sdk-2.13.0 → variably_sdk-2.14.0}/src/variably_sdk.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: variably-sdk
-Version: 2.13.0
+Version: 2.14.0
 Summary: Official Python SDK for Variably feature flags, LLM experimentation, and prompt optimization platform
 Author-email: Variably <support@variably.com>
 License: MIT

{variably_sdk-2.13.0 → variably_sdk-2.14.0}/tests/test_integrations_adk.py

@@ -164,6 +164,8 @@ def test_tool_error_recorded(captured):
     o = _by_agent(captured)["solo"]
     assert o["tools_called"][0]["name"] == "broken_tool"
     assert o["tools_called"][0]["output"].startswith("ERROR: RuntimeError: upstream 503")
+    # Structured, so Step efficiency counts the failed call as wasted (v4.1.1).
+    assert o["tools_called"][0]["error"] == "RuntimeError: upstream 503"
     assert "upstream 503" in o["metadata"]["error"]
 
 

{variably_sdk-2.13.0 → variably_sdk-2.14.0}/tests/test_integrations_langgraph.py

@@ -129,3 +129,26 @@ def test_explicit_aggregator_override(monkeypatch):
     by_agent = {c["agent_name"]: c for c in calls}
     assert by_agent["a"]["is_aggregator"] is True
     assert by_agent["b"]["is_aggregator"] is False
+
+
+class _Msg:
+    def __init__(self, **kw):
+        self.__dict__.update(kw)
+
+
+def test_tool_calls_from_messages_marks_failed_calls():
+    from variably.integrations.langgraph import tool_calls_from_messages
+    msgs = [
+        _Msg(content="", tool_calls=[{"name": "search", "args": {"q": "refund"}, "id": "t1"},
+                                     {"name": "get_order", "args": {"id": "A1"}, "id": "t2"}]),
+        _Msg(content="refunds take 5 days", tool_call_id="t1", status="success"),
+        _Msg(content="Error: TimeoutError('upstream')", tool_call_id="t2", status="error"),
+        {"tool_calls": [{"name": "get_order", "args": {"id": "A1"}, "id": "t3"}]},
+        {"tool_call_id": "t3", "content": "order A1: shipped"},
+    ]
+    calls = tool_calls_from_messages(msgs)
+    assert [c["name"] for c in calls] == ["search", "get_order", "get_order"]
+    assert calls[0] == {"name": "search", "arguments": {"q": "refund"}, "output": "refunds take 5 days"}
+    assert calls[1]["error"] == "Error: TimeoutError('upstream')"
+    assert "error" not in calls[2] and calls[2]["output"] == "order A1: shipped"
+    assert tool_calls_from_messages(None) == []