variably-sdk 2.9.1__tar.gz → 2.12.0__tar.gz

{variably_sdk-2.9.1 → variably_sdk-2.12.0}/PKG-INFO

@@ -1,8 +1,7 @@
 Metadata-Version: 2.4
 Name: variably-sdk
-Version: 2.9.1
+Version: 2.12.0
 Summary: Official Python SDK for Variably feature flags, LLM experimentation, and prompt optimization platform
-Author: Variably
 Author-email: Variably <support@variably.com>
 License: MIT
 Project-URL: Homepage, https://github.com/variably/variably-python-sdk
@@ -23,10 +22,12 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Topic :: Internet :: WWW/HTTP
-Requires-Python: >=3.8
+Requires-Python: >=3.7
 Description-Content-Type: text/markdown
 Requires-Dist: requests>=2.25.0
 Requires-Dist: typing-extensions>=3.7.4; python_version < "3.8"
+Provides-Extra: langgraph
+Requires-Dist: langgraph>=0.1; extra == "langgraph"
 Provides-Extra: dev
 Requires-Dist: pytest>=6.0; extra == "dev"
 Requires-Dist: pytest-asyncio>=0.18.0; extra == "dev"
@@ -40,8 +41,6 @@ Provides-Extra: test
 Requires-Dist: pytest>=6.0; extra == "test"
 Requires-Dist: pytest-cov>=2.10; extra == "test"
 Requires-Dist: responses>=0.18.0; extra == "test"
-Dynamic: author
-Dynamic: requires-python
 
 # Variably Python SDK
 

{variably_sdk-2.9.1 → variably_sdk-2.12.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "variably-sdk"
-version = "2.9.1"
+version = "2.12.0"
 description = "Official Python SDK for Variably feature flags, LLM experimentation, and prompt optimization platform"
 readme = "README.md"
 requires-python = ">=3.7"
@@ -35,6 +35,10 @@ dependencies = [
 ]
 
 [project.optional-dependencies]
+# Framework integrations (Task 42.4). Install e.g. `pip install variably-sdk[langgraph]`.
+langgraph = [
+    "langgraph>=0.1",
+]
 dev = [
     "pytest>=6.0",
     "pytest-asyncio>=0.18.0",

variably_sdk-2.12.0/src/variably/assignment.py

@@ -0,0 +1,65 @@
+"""Deterministic, sticky variant assignment (#54).
+
+Mirrors the Go ``internal/assignment`` package **byte-for-byte** — same FNV-1a
+hash, same bucketing, same weighted walk — so the SDK assigns the *identical*
+variant the server would, with no per-event call. Same (variants, experiment_id,
+key) -> same variant, always.
+"""
+from typing import Any, Dict, List, Optional
+
+_FNV_OFFSET = 2166136261
+_FNV_PRIME = 16777619
+_MASK32 = 0xFFFFFFFF
+
+
+def bucket(salt: str, key: str) -> float:
+    """Stable value in [0, 100) for (salt, key). FNV-1a 32-bit over
+    ``f"{salt}:{key}"`` — identical output to the Go implementation."""
+    h = _FNV_OFFSET
+    for b in (salt + ":" + key).encode("utf-8"):
+        h ^= b
+        h = (h * _FNV_PRIME) & _MASK32
+    return (h % 10000) / 100.0
+
+
+def key_from_context(
+    user_id: Optional[str] = None, session_id: Optional[str] = None
+) -> str:
+    """Bucketing key — prefer user_id, then session_id. Empty when neither."""
+    if user_id:
+        return user_id
+    if session_id:
+        return session_id
+    return ""
+
+
+def select_variant(
+    variants: List[Dict[str, Any]], experiment_id: str, key: str
+) -> Optional[Dict[str, Any]]:
+    """Deterministic, sticky, weighted assignment. Mirrors Go SelectVariant.
+
+    ``variants``: dicts with at least ``weight`` and ``is_control`` (as returned
+    by the /prompts/config endpoint). Empty key (no user/session id) -> control
+    (or first) deterministically, never random.
+    """
+    if not variants:
+        return None
+    if not key:
+        for v in variants:
+            if v.get("is_control"):
+                return v
+        return variants[0]
+
+    total = sum(
+        float(v.get("weight", 0)) for v in variants if float(v.get("weight", 0)) > 0
+    )
+    b = bucket(experiment_id, key)  # [0, 100)
+    cumulative = 0.0
+    n = len(variants)
+    for v in variants:
+        w = float(v.get("weight", 0))
+        w = (w / total * 100.0) if total > 0 else (100.0 / n)
+        cumulative += w
+        if b < cumulative:
+            return v
+    return variants[-1]

{variably_sdk-2.9.1 → variably_sdk-2.12.0}/src/variably/client.py

@@ -10,6 +10,7 @@ from .config import VariablyConfig
 from .errors import ConfigurationError, VariablyError
 from .http_client import HTTPClient
 from .logger import VariablyLogger, create_logger
+from .prompt_config import PromptConfigStore, http_config_fetcher
 from .types import (
     EvaluateResponse,
     Event,
@@ -49,6 +50,10 @@ class VariablyClient:
         
         # Initialize HTTP client
         self.http_client = HTTPClient(config, self.logger, self.metrics)
+
+        # Lazily-started prompt config store for local (in-process) variant
+        # assignment — bootstrapped + polled on first get_variant_local() call.
+        self._prompt_config_store: Optional[PromptConfigStore] = None
         
         self.logger.info(
             "Variably client initialized",
@@ -486,6 +491,66 @@ class VariablyClient:
             is_control=variant.get("is_control", variant.get("isControl", False)),
         )
 
+    def get_variant_local(
+        self,
+        experiment_key: str,
+        user_context: UserContext,
+        poll_interval_seconds: float = 45.0,
+    ) -> PromptVariant:
+        """Assign a prompt variant **in-process** from a cached, background-polled
+        config snapshot — no per-call DB query on the server (#54.3).
+
+        Deterministic and sticky: the same (experiment_key, user_id/session_id)
+        always maps to the same variant, using the exact hash the server would.
+        The config is bootstrapped on first call and refreshed every
+        ``poll_interval_seconds`` with ETag (a 304 costs one cheap round trip).
+
+        Falls back to the server ``get_variant`` if the experiment isn't in the
+        snapshot yet (e.g. created between polls), so a brand-new experiment is
+        never silently missed.
+        """
+        if not experiment_key or not isinstance(experiment_key, str):
+            raise ConfigurationError(
+                "Experiment key must be a non-empty string", "experiment_key"
+            )
+        key = user_context.get("user_id") or user_context.get("session_id")
+        if not key:
+            raise ConfigurationError(
+                "User context must include user_id or session_id",
+                "user_context.user_id",
+            )
+
+        if self._prompt_config_store is None:
+            store = PromptConfigStore(
+                http_config_fetcher(self.config.base_url, self.config.api_key)
+            )
+            store.start_polling(poll_interval_seconds)
+            self._prompt_config_store = store
+
+        variant = self._prompt_config_store.assign(
+            experiment_key,
+            user_id=user_context.get("user_id"),
+            session_id=user_context.get("session_id"),
+        )
+        if variant is None:
+            # Not in snapshot (new/just-changed experiment) — authoritative fallback.
+            return self.get_variant(experiment_key, user_context)
+
+        # Extract model/provider from parameters.llmConfig, mirroring get_variant.
+        params = variant.get("parameters", {}) or {}
+        llm_config = params.get("llmConfig", params)
+        return PromptVariant(
+            experiment_id=variant.get("experiment_id", ""),
+            experiment_key=experiment_key,
+            variant_id=variant.get("variant_id", ""),
+            variant_key=variant.get("name", variant.get("variant_key", "")),
+            prompt_template=variant.get("prompt_template", ""),
+            model=llm_config.get("model", ""),
+            provider=llm_config.get("provider", ""),
+            parameters=params,
+            is_control=variant.get("is_control", False),
+        )
+
     def evaluate_prompt(
         self,
         experiment_key: str,
@@ -678,6 +743,12 @@ class VariablyClient:
         # Multi-turn / Session context
         session_id: Optional[str] = None,          # Groups conversation turns into sessions
         conversation_history: Optional[List[Dict[str, Any]]] = None,  # Prior turns for coherence scoring
+        # Multi-agent context — see docs/improvements/42.1-implementation-plan.md
+        agent_name: Optional[str] = None,          # This step's agent, within a session_id trace
+        is_aggregator: bool = False,               # True on the final step that aggregates the trace
+        tools_called: Optional[List[Dict[str, Any]]] = None,   # [{"name","arguments","output"}] actually called
+        expected_tools: Optional[List[Dict[str, Any]]] = None, # golden tools for deterministic trajectory scoring
+        allowed_tools: Optional[List[str]] = None,             # permitted tool names; calls outside this list flag as unauthorized
         # User & organization context
         user_id: Optional[str] = None,             # End-user ID for per-user analytics
         tags: Optional[List[str]] = None,          # Grouping labels, e.g. ["production", "rag"]
@@ -741,6 +812,28 @@ class VariablyClient:
             metadata = dict(metadata) if metadata else {}
             metadata.setdefault("kind", kind)
 
+        # Validate multi-agent inputs up front (mirrors server-side checks) so
+        # a malformed trace fails fast at the call site rather than silently
+        # producing an unscoreable session.
+        if is_aggregator and not agent_name:
+            raise ConfigurationError(
+                "is_aggregator=True requires a non-empty agent_name",
+                "agent_name",
+            )
+        if is_aggregator and not session_id:
+            raise ConfigurationError(
+                "is_aggregator=True requires a session_id to assemble the multi-agent trace",
+                "session_id",
+            )
+        for _label, _tools in (("tools_called", tools_called), ("expected_tools", expected_tools)):
+            if _tools:
+                for _t in _tools:
+                    if not isinstance(_t, dict) or not _t.get("name"):
+                        raise ConfigurationError(
+                            f"{_label} entries must be dicts with a non-empty 'name'",
+                            _label,
+                        )
+
         data: Dict[str, Any] = {
             "prompt": prompt_str,
             "response": response,
@@ -763,6 +856,16 @@ class VariablyClient:
             data["user_id"] = user_id
         if session_id:
             data["session_id"] = session_id
+        if agent_name:
+            data["agent_name"] = agent_name
+        if is_aggregator:
+            data["is_aggregator"] = True
+        if tools_called:
+            data["tools_called"] = tools_called
+        if expected_tools:
+            data["expected_tools"] = expected_tools
+        if allowed_tools:
+            data["allowed_tools"] = allowed_tools
         if provider:
             data["provider"] = provider
         if model:

variably_sdk-2.12.0/src/variably/integrations/__init__.py

@@ -0,0 +1,15 @@
+"""Framework auto-instrumentation for variА/Bly (improvement #42, Task 42.4).
+
+Per-framework integrations that wire multi-agent workflows into variА/Bly's
+deterministic cross-step evaluation with little or no per-node code:
+
+    from variably.integrations.langgraph import instrument
+    instrument(graph, session_id_key="session_id")
+
+Each integration maps the framework's node/agent identity to ``agent_name``,
+detects the terminal node(s) as ``is_aggregator``, pulls session id + context
+from the framework's state, and logs via ``variably.observe`` — always
+best-effort, never raising into the customer's workflow.
+
+Install extras per framework, e.g. ``pip install variably-sdk[langgraph]``.
+"""

variably_sdk-2.12.0/src/variably/integrations/_common.py

@@ -0,0 +1,103 @@
+"""Shared helpers for variА/Bly framework integrations (Task 42.4).
+
+Best-effort extraction of the fields the evaluator needs (response text, an
+optional query/prompt, and the session id) from a framework's state / node
+return value. Everything here is defensive: extraction never raises, so a
+framework whose state shape we don't recognise degrades to an empty string
+rather than breaking the customer's workflow.
+"""
+
+from __future__ import annotations
+
+import sys
+from typing import Any, Optional
+
+# Keys we look for when extracting a human-readable response from a node's
+# return value, in priority order.
+_RESPONSE_KEYS = ("response", "output", "answer", "result", "text", "content")
+# Keys we look for when extracting the user query/prompt from graph state.
+_QUERY_KEYS = ("query", "question", "input", "prompt", "user_input")
+# Frameworks commonly nest config/inputs under one of these container keys
+# (e.g. gpt-researcher's state["task"]["query"]). Checked one level deep.
+_QUERY_CONTAINERS = ("task", "input", "config", "configurable", "state", "metadata")
+
+
+def warn(msg: str) -> None:
+    print(f"[variably.integrations] {msg}", file=sys.stderr)
+
+
+def _message_text(msgs: Any) -> str:
+    """Extract text from a LangChain-style messages list (or single message)."""
+    try:
+        last = msgs[-1] if isinstance(msgs, (list, tuple)) and msgs else msgs
+        content = getattr(last, "content", None)
+        if isinstance(content, str):
+            return content
+        if isinstance(last, dict) and isinstance(last.get("content"), str):
+            return last["content"]
+    except Exception:
+        pass
+    return ""
+
+
+def extract_response_text(update: Any) -> str:
+    """Best-effort extraction of response text from a node's return value."""
+    try:
+        if update is None:
+            return ""
+        if isinstance(update, str):
+            return update
+        if isinstance(update, dict):
+            for k in _RESPONSE_KEYS:
+                v = update.get(k)
+                if isinstance(v, str) and v.strip():
+                    return v
+            if "messages" in update:
+                text = _message_text(update["messages"])
+                if text:
+                    return text
+            for v in update.values():  # fallback: first non-empty string
+                if isinstance(v, str) and v.strip():
+                    return v
+            return ""
+        # objects: try common attributes
+        for k in _RESPONSE_KEYS:
+            v = getattr(update, k, None)
+            if isinstance(v, str) and v.strip():
+                return v
+        return str(update)
+    except Exception:
+        return ""
+
+
+def extract_query(state: Any) -> str:
+    """Best-effort extraction of the user query/prompt from graph state.
+
+    Checks top-level keys first, then one level into common container dicts
+    (``task``/``input``/``config``/…) so frameworks that nest the query — e.g.
+    gpt-researcher's ``state["task"]["query"]`` — are handled without a custom
+    extractor. Falls back to "" (caller then uses a ``[node:name]`` placeholder)."""
+    try:
+        get = state.get if isinstance(state, dict) else (lambda k, d=None: getattr(state, k, d))
+        for k in _QUERY_KEYS:
+            v = get(k)
+            if isinstance(v, str) and v.strip():
+                return v
+        for container in _QUERY_CONTAINERS:
+            c = get(container)
+            if isinstance(c, dict):
+                for k in _QUERY_KEYS:
+                    v = c.get(k)
+                    if isinstance(v, str) and v.strip():
+                        return v
+    except Exception:
+        pass
+    return ""
+
+
+def extract_session_id(state: Any, key: str) -> Optional[str]:
+    try:
+        v = state.get(key) if isinstance(state, dict) else getattr(state, key, None)
+        return str(v) if v else None
+    except Exception:
+        return None

variably_sdk-2.12.0/src/variably/integrations/langgraph.py

@@ -0,0 +1,324 @@
+"""LangGraph auto-instrumentation for variА/Bly (Task 42.4).
+
+Two entry points:
+
+1. ``instrument(graph, session_id_key=...)`` — zero-touch. Call it once after
+   you've added nodes/edges (before ``.compile()``). Wraps every node so each
+   execution logs to variА/Bly: ``agent_name`` = node name, ``is_aggregator``
+   on the graph's terminal node(s), ``session_id`` pulled from graph state.
+
+2. ``@observe_node(...)`` — an explicit per-node decorator. Version-independent
+   and the most robust option if ``instrument`` can't wrap your graph's node
+   representation.
+
+Both are best-effort: instrumentation failures degrade to a stderr warning and
+NEVER raise into your graph. Targets langgraph >= 0.1; call ``instrument`` after
+``add_node`` / ``add_edge`` and before ``compile()``.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import functools
+from typing import Any, Callable, Iterable, Optional
+
+from variably import observe as _observe
+from variably.integrations._common import (
+    extract_query,
+    extract_response_text,
+    extract_session_id,
+    warn,
+)
+
+
+# Node names we've already warned about (no response text) — warn once per node
+# per process so the message is visible without spamming every run.
+_warned_no_response: set = set()
+
+
+def _shape(value: Any) -> str:
+    """Describe a value's STRUCTURE (type + dict keys) without exposing its
+    contents — safe to print/share for diagnostics."""
+    try:
+        if isinstance(value, dict):
+            keys = [str(k) for k in list(value.keys())[:25]]
+            more = "…" if len(value) > 25 else ""
+            return f"dict(keys=[{', '.join(keys)}{more}])"
+        if isinstance(value, (list, tuple)):
+            return f"{type(value).__name__}(len={len(value)})"
+        return type(value).__name__
+    except Exception:
+        return "<unknown>"
+
+
+def _log_node(
+    name: str,
+    state: Any,
+    update: Any,
+    *,
+    is_aggregator: bool,
+    session_id_key: str,
+    response_extractor: Optional[Callable[[Any], str]],
+    context_extractor: Optional[Callable[[Any, Any], Any]] = None,
+    tools_extractor: Optional[Callable[[Any, Any], Any]] = None,
+) -> None:
+    """Log one node execution to variА/Bly. Never raises."""
+    try:
+        resp = (response_extractor or extract_response_text)(update)
+        # No response text → do NOT fabricate one (a serialized dict would be scored
+        # as meaningless prose) and do NOT silently drop it. Warn the developer once
+        # per node (visible on stderr, non-blocking) so they notice and either add a
+        # response_extractor= or accept that this node has no scorable output.
+        if not (resp and resp.strip()):
+            if name not in _warned_no_response:
+                _warned_no_response.add(name)
+                warn(
+                    f"node '{name}' produced no response text — this step was NOT scored.\n"
+                    f"    node       = {name!r}\n"
+                    f"    returned   = {_shape(update)}\n"
+                    f"    extractor  = {'custom response_extractor' if response_extractor else 'default (extract_response_text)'}\n"
+                    f"    aggregator = {is_aggregator}\n"
+                    f"  Fix: pass response_extractor= to instrument() mapping this node's output to a "
+                    f"string, e.g. response_extractor=lambda upd: upd.get('report', '').\n"
+                    f"  Only the structure above (type + keys) is shown — never your data — so this "
+                    f"line is safe to share with variА/Bly support if a step you expected isn't scored."
+                )
+            return
+        sid = extract_session_id(state, session_id_key)
+        # is_aggregator requires a session_id (SDK write-time rule); only claim
+        # it when we actually have one, else it degrades to a normal step.
+        agg = bool(is_aggregator and sid)
+        # Retrieved context enables grounding/faithfulness scoring. Without it,
+        # only the non-grounding dimensions are scored. Best-effort; never fatal.
+        refs = None
+        if context_extractor is not None:
+            try:
+                refs = context_extractor(state, update)
+            except Exception:
+                refs = None
+        # Tool calls enable the trajectory / tool-correctness / order-adherence
+        # metrics. Extractor returns {"tools_called": [...], "expected_tools": [...]}.
+        tools_called = expected_tools = None
+        if tools_extractor is not None:
+            try:
+                t = tools_extractor(state, update) or {}
+                tools_called = t.get("tools_called")
+                expected_tools = t.get("expected_tools")
+            except Exception:
+                tools_called = expected_tools = None
+        _observe(
+            prompt=extract_query(state) or f"[node:{name}]",
+            response=resp or "",
+            session_id=sid,
+            agent_name=name,
+            is_aggregator=agg,
+            reference_materials=refs,
+            tools_called=tools_called,
+            expected_tools=expected_tools,
+        )
+    except Exception as e:  # pragma: no cover - defensive
+        warn(f"node '{name}' instrumentation failed: {e}")
+
+
+def observe_node(
+    *,
+    agent_name: Optional[str] = None,
+    is_aggregator: bool = False,
+    session_id_key: str = "session_id",
+    response_extractor: Optional[Callable[[Any], str]] = None,
+) -> Callable[[Callable], Callable]:
+    """Decorator for a LangGraph node function. Logs the node's output to
+    variА/Bly after it runs. ``agent_name`` defaults to the function name."""
+
+    def decorate(fn: Callable) -> Callable:
+        name = agent_name or getattr(fn, "__name__", "node")
+
+        @functools.wraps(fn)
+        def wrapper(state, *args, **kwargs):
+            update = fn(state, *args, **kwargs)
+            _log_node(
+                name, state, update,
+                is_aggregator=is_aggregator,
+                session_id_key=session_id_key,
+                response_extractor=response_extractor,
+            )
+            return update
+
+        return wrapper
+
+    return decorate
+
+
+def _detect_terminal_nodes(graph) -> set:
+    """Terminal nodes = those with an edge to END, plus any with no outgoing
+    edge. Best-effort across langgraph versions."""
+    terminals: set = set()
+    try:
+        try:
+            from langgraph.graph import END
+        except Exception:
+            END = "__end__"
+
+        node_names = set(getattr(graph, "nodes", {}) or {})
+        edges = getattr(graph, "edges", None) or set()
+        has_outgoing: set = set()
+        for edge in edges:
+            try:
+                u, v = edge
+            except Exception:
+                continue
+            has_outgoing.add(u)
+            if v == END or v == "__end__":
+                terminals.add(u)
+        # nodes with no outgoing plain edge (may route via conditional branches,
+        # but this is a reasonable default when no END edge is declared)
+        terminals |= (node_names - has_outgoing)
+    except Exception as e:  # pragma: no cover - defensive
+        warn(f"terminal-node detection failed: {e}")
+    return terminals
+
+
+def _wrap_callable(name, fn, is_agg, session_id_key, response_extractor, context_extractor=None, tools_extractor=None):
+    @functools.wraps(fn)
+    def wrapped(state, *args, **kwargs):
+        update = fn(state, *args, **kwargs)
+        _log_node(
+            name, state, update,
+            is_aggregator=is_agg,
+            session_id_key=session_id_key,
+            response_extractor=response_extractor,
+            context_extractor=context_extractor,
+            tools_extractor=tools_extractor,
+        )
+        return update
+    return wrapped
+
+
+def _wrap_callable_async(name, fn, is_agg, session_id_key, response_extractor, context_extractor=None, tools_extractor=None):
+    """Async counterpart of _wrap_callable. Most real LangGraph nodes are
+    ``async def`` (invoked via ``ainvoke``), so their fn lives on the runnable's
+    ``.afunc`` — this wraps that path. ``_observe`` is non-blocking, so logging
+    from inside the event loop is safe."""
+    @functools.wraps(fn)
+    async def wrapped(state, *args, **kwargs):
+        update = await fn(state, *args, **kwargs)
+        _log_node(
+            name, state, update,
+            is_aggregator=is_agg,
+            session_id_key=session_id_key,
+            response_extractor=response_extractor,
+            context_extractor=context_extractor,
+            tools_extractor=tools_extractor,
+        )
+        return update
+    return wrapped
+
+
+def instrument(
+    graph,
+    *,
+    session_id_key: str = "session_id",
+    aggregator_nodes: Optional[Iterable[str]] = None,
+    response_extractor: Optional[Callable[[Any], str]] = None,
+    context_extractor: Optional[Callable[[Any, Any], Any]] = None,
+    tools_extractor: Optional[Callable[[Any, Any], Any]] = None,
+):
+    """Instrument every node in a LangGraph ``StateGraph`` in place.
+
+    Args:
+        graph: an uncompiled ``StateGraph``.
+        session_id_key: state key holding the session/thread id.
+        aggregator_nodes: override terminal-node auto-detection with an explicit set.
+        response_extractor: custom fn to pull response text from a node's return.
+        context_extractor: custom fn ``(state, update) -> list[{"id","content"}]``
+            returning the retrieved reference materials for grounding/faithfulness
+            scoring. Return None/[] when the node has no context. Without it,
+            grounding cannot be scored (only the non-grounding dimensions).
+        tools_extractor: custom fn ``(state, update) -> {"tools_called": [...],
+            "expected_tools": [...]}`` (each tool a ``{"name","arguments","output"}``
+            dict) enabling the trajectory / tool-correctness / order-adherence
+            metrics. Return None when the node makes no tool calls.
+
+    Returns the same graph (for chaining). Best-effort; never raises.
+    """
+    nodes = getattr(graph, "nodes", None)
+    if not nodes:
+        warn("graph has no .nodes — nothing to instrument (pass an uncompiled StateGraph)")
+        return graph
+
+    terminals = set(aggregator_nodes) if aggregator_nodes is not None else _detect_terminal_nodes(graph)
+
+    for name in list(nodes.keys()):
+        is_agg = name in terminals
+        try:
+            spec = nodes[name]
+            # langgraph StateNodeSpec exposes the node fn on .runnable; older /
+            # simpler representations store the callable directly.
+            runnable = getattr(spec, "runnable", None)
+            if runnable is not None:
+                wrapped_any = False
+                # Sync path — RunnableCallable stores a sync fn on .func.
+                sync_fn = getattr(runnable, "func", None)
+                if callable(sync_fn):
+                    try:
+                        runnable.func = _wrap_callable(
+                            name, sync_fn, is_agg, session_id_key, response_extractor,
+                            context_extractor, tools_extractor)
+                        wrapped_any = True
+                    except Exception:
+                        pass
+                # Async path — async nodes store the coroutine fn on .afunc, which
+                # is what langgraph calls under ainvoke. Missing this = no events
+                # for any async graph (the common case).
+                async_fn = getattr(runnable, "afunc", None)
+                if callable(async_fn):
+                    try:
+                        runnable.afunc = _wrap_callable_async(
+                            name, async_fn, is_agg, session_id_key, response_extractor,
+                            context_extractor, tools_extractor)
+                        wrapped_any = True
+                    except Exception:
+                        pass
+                if wrapped_any:
+                    continue
+                # No .func/.afunc — the runnable itself may be the callable.
+                if callable(runnable):
+                    wrapped = (
+                        _wrap_callable_async(name, runnable, is_agg, session_id_key,
+                                             response_extractor, context_extractor, tools_extractor)
+                        if asyncio.iscoroutinefunction(runnable)
+                        else _wrap_callable(name, runnable, is_agg, session_id_key,
+                                            response_extractor, context_extractor, tools_extractor)
+                    )
+                    if _replace_spec_runnable(nodes, name, spec, wrapped):
+                        continue
+                warn(f"node '{name}': could not rewrap runnable (unsupported spec)")
+            elif callable(spec):
+                nodes[name] = (
+                    _wrap_callable_async(name, spec, is_agg, session_id_key, response_extractor)
+                    if asyncio.iscoroutinefunction(spec)
+                    else _wrap_callable(name, spec, is_agg, session_id_key, response_extractor)
+                )
+            else:
+                warn(f"node '{name}': unrecognised spec type {type(spec).__name__}; skipped")
+        except Exception as e:  # pragma: no cover - defensive
+            warn(f"failed to instrument node '{name}': {e}")
+
+    return graph
+
+
+def _replace_spec_runnable(nodes, name, spec, wrapped) -> bool:
+    """Try to put `wrapped` back onto the node spec across spec representations."""
+    # settable attribute
+    try:
+        spec.runnable = wrapped  # type: ignore[attr-defined]
+        return True
+    except Exception:
+        pass
+    # namedtuple-style: rebuild via _replace
+    try:
+        nodes[name] = spec._replace(runnable=wrapped)  # type: ignore[attr-defined]
+        return True
+    except Exception:
+        pass
+    return False

{variably_sdk-2.9.1 → variably_sdk-2.12.0}/src/variably/observe.py

@@ -172,6 +172,22 @@ def observe(
     # Advanced: not needed if you pass session_id (variA/Bly auto-builds history from previous turns).
     prior_turns: Optional[List[Dict[str, str]]] = None,
 
+    # --- Multi-agent context ---
+    # Turn the session_id grouping into an evaluatable multi-agent trace.
+    # agent_name labels which agent produced this step; set is_aggregator=True
+    # on the final step that produces the aggregated answer to trigger
+    # deterministic cross-step scoring (coherence + trajectory).
+    agent_name: Optional[str] = None,
+    is_aggregator: bool = False,
+    # Tools this step actually called / was expected to call. Each dict:
+    # {"name": "...", "arguments": {...}, "output": "..."}. Enables the
+    # deterministic trajectory precision/recall + tool-correctness metrics.
+    tools_called: Optional[List[Dict[str, Any]]] = None,
+    expected_tools: Optional[List[Dict[str, Any]]] = None,
+    # Permitted-tool allowlist (names). Any call outside it flags as an
+    # unauthorized action — the deterministic security/adversarial signal.
+    allowed_tools: Optional[List[str]] = None,
+
     # --- User & organization context ---
     user_id: Optional[str] = None,            # Your end-user's ID — enables per-user analytics
     tags: Optional[List[str]] = None,         # Grouping labels, e.g. ["production", "rag", "v2"]
@@ -271,6 +287,11 @@ def observe(
             tags=tags,
             user_id=user_id,
             session_id=session_id,
+            agent_name=agent_name,
+            is_aggregator=is_aggregator,
+            tools_called=tools_called,
+            expected_tools=expected_tools,
+            allowed_tools=allowed_tools,
             metadata=metadata,
             kind=kind,
         )

variably_sdk-2.12.0/src/variably/prompt_config.py

@@ -0,0 +1,115 @@
+"""SDK-side prompt config store + background poll (#54.2/54.3).
+
+Bootstraps the project's prompt-experiment config from
+``GET /internal/sdk/prompts/config`` once, refreshes it in the background with
+ETag/If-None-Match (a 304 is a no-op), and assigns variants **locally** from the
+cached snapshot — zero backend call per event. Deterministic + sticky via
+``assignment`` (which is byte-for-byte identical to the server).
+"""
+import threading
+from typing import Any, Callable, Dict, List, Optional, Tuple
+
+from .assignment import key_from_context, select_variant
+
+# A fetcher takes the current ETag/version and returns (version, config_dict) on
+# change, or None when unchanged (304), errored, or nothing to update. Injectable
+# so the store is testable without a live server.
+Fetcher = Callable[[Optional[str]], Optional[Tuple[str, Dict[str, Any]]]]
+
+
+class PromptConfigStore:
+    """Thread-safe snapshot of the SDK prompt config + local assignment."""
+
+    def __init__(self, fetcher: Fetcher):
+        self._fetcher = fetcher
+        self._lock = threading.Lock()
+        self._version: Optional[str] = None
+        self._experiments: Dict[str, Dict[str, Any]] = {}  # experiment_key -> experiment
+        self._stop = threading.Event()
+        self._thread: Optional[threading.Thread] = None
+
+    def refresh(self) -> bool:
+        """Fetch config and atomically swap the snapshot on change. Returns True
+        if the snapshot was updated, False on 304/unchanged/error. Never raises."""
+        try:
+            result = self._fetcher(self._version)
+        except Exception:
+            return False
+        if result is None:
+            return False
+        version, cfg = result
+        index = {
+            e["experiment_key"]: e for e in (cfg.get("experiments") or []) if e.get("experiment_key")
+        }
+        with self._lock:
+            self._version = version
+            self._experiments = index
+        return True
+
+    def assign(
+        self,
+        experiment_key: str,
+        user_id: Optional[str] = None,
+        session_id: Optional[str] = None,
+    ) -> Optional[Dict[str, Any]]:
+        """Assign a variant locally from the cached snapshot. Returns the variant
+        dict, or None if the experiment isn't in the snapshot (not running /
+        unknown / config not yet loaded)."""
+        with self._lock:
+            exp = self._experiments.get(experiment_key)
+        if not exp:
+            return None
+        experiment_id = exp["experiment_id"]
+        key = key_from_context(user_id, session_id)
+        variant = select_variant(exp.get("variants") or [], experiment_id, key)
+        if variant is None:
+            return None
+        # Stamp experiment_id (lives on the experiment, not the variant) so callers
+        # get a self-contained variant. Copy to avoid mutating the snapshot.
+        return {**variant, "experiment_id": experiment_id}
+
+    def start_polling(self, interval_seconds: float = 45.0) -> None:
+        """Bootstrap once, then refresh every ``interval_seconds`` in a daemon
+        thread. Idempotent."""
+        if self._thread is not None:
+            return
+        self.refresh()  # initial bootstrap
+
+        def _loop() -> None:
+            while not self._stop.wait(interval_seconds):
+                self.refresh()
+
+        self._thread = threading.Thread(
+            target=_loop, name="variably-config-poll", daemon=True
+        )
+        self._thread.start()
+
+    def stop(self) -> None:
+        self._stop.set()
+
+    @property
+    def version(self) -> Optional[str]:
+        return self._version
+
+
+def http_config_fetcher(base_url: str, api_key: str, timeout: float = 5.0) -> Fetcher:
+    """Build a fetcher that GETs /internal/sdk/prompts/config with ETag. Uses
+    ``requests`` (already an SDK dep). Returns (version, config) on 200, None on
+    304 or any error — the store keeps its last-good snapshot."""
+    import requests
+
+    url = base_url.rstrip("/") + "/api/v1/internal/sdk/prompts/config"
+
+    def _fetch(etag: Optional[str]) -> Optional[Tuple[str, Dict[str, Any]]]:
+        headers = {"Authorization": f"Bearer {api_key}"}
+        if etag:
+            headers["If-None-Match"] = '"' + etag + '"'
+        resp = requests.get(url, headers=headers, timeout=timeout)
+        if resp.status_code == 304:
+            return None
+        if resp.status_code != 200:
+            return None
+        cfg = resp.json()
+        return cfg.get("version", ""), cfg
+
+    return _fetch

{variably_sdk-2.9.1 → variably_sdk-2.12.0}/src/variably/version.py

@@ -1,3 +1,3 @@
 """Version information for Variably SDK."""
 
-__version__ = "2.9.1"
+__version__ = "2.11.0"

{variably_sdk-2.9.1 → variably_sdk-2.12.0}/src/variably_sdk.egg-info/PKG-INFO

@@ -1,8 +1,7 @@
 Metadata-Version: 2.4
 Name: variably-sdk
-Version: 2.9.1
+Version: 2.12.0
 Summary: Official Python SDK for Variably feature flags, LLM experimentation, and prompt optimization platform
-Author: Variably
 Author-email: Variably <support@variably.com>
 License: MIT
 Project-URL: Homepage, https://github.com/variably/variably-python-sdk
@@ -23,10 +22,12 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Topic :: Internet :: WWW/HTTP
-Requires-Python: >=3.8
+Requires-Python: >=3.7
 Description-Content-Type: text/markdown
 Requires-Dist: requests>=2.25.0
 Requires-Dist: typing-extensions>=3.7.4; python_version < "3.8"
+Provides-Extra: langgraph
+Requires-Dist: langgraph>=0.1; extra == "langgraph"
 Provides-Extra: dev
 Requires-Dist: pytest>=6.0; extra == "dev"
 Requires-Dist: pytest-asyncio>=0.18.0; extra == "dev"
@@ -40,8 +41,6 @@ Provides-Extra: test
 Requires-Dist: pytest>=6.0; extra == "test"
 Requires-Dist: pytest-cov>=2.10; extra == "test"
 Requires-Dist: responses>=0.18.0; extra == "test"
-Dynamic: author
-Dynamic: requires-python
 
 # Variably Python SDK
 

{variably_sdk-2.9.1 → variably_sdk-2.12.0}/src/variably_sdk.egg-info/SOURCES.txt

@@ -1,7 +1,7 @@
 README.md
 pyproject.toml
-setup.py
 src/variably/__init__.py
+src/variably/assignment.py
 src/variably/cache.py
 src/variably/client.py
 src/variably/config.py
@@ -9,11 +9,17 @@ src/variably/errors.py
 src/variably/http_client.py
 src/variably/logger.py
 src/variably/observe.py
+src/variably/prompt_config.py
 src/variably/types.py
 src/variably/version.py
+src/variably/integrations/__init__.py
+src/variably/integrations/_common.py
+src/variably/integrations/langgraph.py
 src/variably_sdk.egg-info/PKG-INFO
 src/variably_sdk.egg-info/SOURCES.txt
 src/variably_sdk.egg-info/dependency_links.txt
 src/variably_sdk.egg-info/requires.txt
 src/variably_sdk.egg-info/top_level.txt
+tests/test_integrations_langgraph.py
+tests/test_prompt_assignment.py
 tests/test_streaming.py

{variably_sdk-2.9.1 → variably_sdk-2.12.0}/src/variably_sdk.egg-info/requires.txt

@@ -13,6 +13,9 @@ mypy>=0.800
 isort>=5.0
 responses>=0.18.0
 
+[langgraph]
+langgraph>=0.1
+
 [test]
 pytest>=6.0
 pytest-cov>=2.10

variably_sdk-2.12.0/tests/test_integrations_langgraph.py

@@ -0,0 +1,131 @@
+"""Tests for the LangGraph integration (Task 42.4).
+
+Uses a fake StateGraph and a captured `observe` so no real langgraph is needed.
+"""
+
+import sys
+import os
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "src"))
+
+import variably.integrations.langgraph as lg
+from variably.integrations.langgraph import instrument, observe_node
+
+
+class FakeGraph:
+    """Minimal StateGraph stand-in: nodes as name->callable, edges as a set."""
+    def __init__(self):
+        self.nodes = {}
+        self.edges = set()
+
+    def add_node(self, name, fn):
+        self.nodes[name] = fn
+
+    def add_edge(self, u, v):
+        self.edges.add((u, v))
+
+
+def _capture(monkeypatch):
+    calls = []
+    monkeypatch.setattr(lg, "_observe", lambda **kw: calls.append(kw))
+    return calls
+
+
+def test_observe_node_decorator(monkeypatch):
+    calls = _capture(monkeypatch)
+
+    @observe_node(agent_name="reco", is_aggregator=True)
+    def node(state):
+        return {"response": "final recommendation"}
+
+    out = node({"session_id": "s1", "query": "what to buy?"})
+    assert out == {"response": "final recommendation"}
+    assert len(calls) == 1
+    c = calls[0]
+    assert c["agent_name"] == "reco"
+    assert c["is_aggregator"] is True
+    assert c["session_id"] == "s1"
+    assert c["response"] == "final recommendation"
+    assert c["prompt"] == "what to buy?"
+
+
+def test_is_aggregator_requires_session_id(monkeypatch):
+    calls = _capture(monkeypatch)
+
+    @observe_node(agent_name="reco", is_aggregator=True)
+    def node(state):
+        return {"response": "x"}
+
+    node({})  # no session_id
+    assert calls[0]["is_aggregator"] is False
+    assert calls[0]["session_id"] is None
+
+
+def test_instrument_wraps_all_nodes_and_marks_terminal(monkeypatch):
+    calls = _capture(monkeypatch)
+    g = FakeGraph()
+    g.add_node("profiler", lambda s: {"profile": "eco-friendly"})
+    g.add_node("recommender", lambda s: {"response": "buy X"})
+    g.add_edge("profiler", "recommender")
+    g.add_edge("recommender", "__end__")  # terminal
+
+    instrument(g, session_id_key="session_id")
+
+    state = {"session_id": "sess", "query": "q"}
+    g.nodes["profiler"](state)
+    g.nodes["recommender"](state)
+
+    by_agent = {c["agent_name"]: c for c in calls}
+    assert set(by_agent) == {"profiler", "recommender"}
+    assert by_agent["profiler"]["is_aggregator"] is False
+    assert by_agent["recommender"]["is_aggregator"] is True
+    assert by_agent["profiler"]["response"] == "eco-friendly"
+    assert by_agent["recommender"]["response"] == "buy X"
+
+
+def test_instrument_never_breaks_node_when_logging_fails(monkeypatch):
+    def boom(**kw):
+        raise RuntimeError("logging down")
+    monkeypatch.setattr(lg, "_observe", boom)
+
+    g = FakeGraph()
+    g.add_node("n", lambda s: {"response": "still works"})
+    instrument(g)
+    # node must still return its update even though logging raises
+    assert g.nodes["n"]({"session_id": "s"}) == {"response": "still works"}
+
+
+def test_instrument_rewraps_runnable_spec(monkeypatch):
+    calls = _capture(monkeypatch)
+
+    class FakeRunnable:
+        def __init__(self, func):
+            self.func = func
+
+    class Spec:
+        def __init__(self, runnable):
+            self.runnable = runnable
+
+    g = FakeGraph()
+    g.nodes["x"] = Spec(FakeRunnable(lambda s: {"response": "r"}))
+    g.add_edge("x", "__end__")
+
+    instrument(g)
+    # the wrapped fn now lives on .runnable.func
+    g.nodes["x"].runnable.func({"session_id": "s"})
+    assert len(calls) == 1
+    assert calls[0]["agent_name"] == "x"
+    assert calls[0]["is_aggregator"] is True
+
+
+def test_explicit_aggregator_override(monkeypatch):
+    calls = _capture(monkeypatch)
+    g = FakeGraph()
+    g.add_node("a", lambda s: {"response": "1"})
+    g.add_node("b", lambda s: {"response": "2"})
+    instrument(g, aggregator_nodes={"a"})
+    g.nodes["a"]({"session_id": "s"})
+    g.nodes["b"]({"session_id": "s"})
+    by_agent = {c["agent_name"]: c for c in calls}
+    assert by_agent["a"]["is_aggregator"] is True
+    assert by_agent["b"]["is_aggregator"] is False

variably_sdk-2.12.0/tests/test_prompt_assignment.py

@@ -0,0 +1,103 @@
+"""Tests for SDK-side local assignment (#54.3) + config store (#54.2).
+
+Run via pytest, or directly: `python tests/test_prompt_assignment.py` (the direct
+runner avoids the legacy top-level `variably` package shadowing `src/variably`).
+"""
+import os
+import sys
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "src"))
+
+from variably.assignment import bucket, key_from_context, select_variant  # noqa: E402
+from variably.prompt_config import PromptConfigStore  # noqa: E402
+
+# Reference values computed by the Go internal/assignment package — the SDK MUST
+# match these byte-for-byte, or server and client assign different variants.
+GO_BUCKETS = {
+    ("exp-1", "user-42"): 41.20,
+    ("exp-1", "user-99"): 64.00,
+    ("exp-2", "user-42"): 81.89,
+    ("clinical", "abc-123"): 53.23,
+    ("e", "s"): 99.59,
+}
+
+
+def _vars():
+    return [
+        {"variant_id": "a", "name": "control", "is_control": True, "weight": 70},
+        {"variant_id": "b", "name": "treatment", "is_control": False, "weight": 30},
+    ]
+
+
+def test_bucket_parity_with_go():
+    for (salt, key), expected in GO_BUCKETS.items():
+        got = bucket(salt, key)
+        assert abs(got - expected) < 0.005, f"{salt}:{key} py={got} go={expected}"
+
+
+def test_sticky():
+    first = select_variant(_vars(), "exp-1", "user-42")
+    for _ in range(1000):
+        assert select_variant(_vars(), "exp-1", "user-42")["variant_id"] == first["variant_id"]
+
+
+def test_distribution():
+    counts = {"a": 0, "b": 0}
+    n = 10000
+    for i in range(n):
+        counts[select_variant(_vars(), "exp-1", f"user-{i}")["variant_id"]] += 1
+    assert 65 <= counts["a"] / n * 100 <= 75
+    assert 25 <= counts["b"] / n * 100 <= 35
+
+
+def test_keyless_returns_control():
+    for _ in range(50):
+        v = select_variant(_vars(), "exp-1", "")
+        assert v is not None and v["is_control"]
+
+
+def test_key_from_context():
+    assert key_from_context("u1", "s1") == "u1"
+    assert key_from_context(None, "s1") == "s1"
+    assert key_from_context(None, None) == ""
+
+
+def test_store_refresh_and_assign():
+    cfg = {
+        "version": "v1",
+        "experiments": [
+            {
+                "experiment_id": "exp-1",
+                "experiment_key": "greeting",
+                "status": "running",
+                "traffic_allocation": 100,
+                "variants": _vars(),
+            }
+        ],
+    }
+    calls = {"n": 0}
+
+    def fetcher(etag):
+        calls["n"] += 1
+        # first call returns config; subsequent (same version) -> None (304)
+        return ("v1", cfg) if etag != "v1" else None
+
+    store = PromptConfigStore(fetcher)
+    assert store.refresh() is True                 # initial load
+    assert store.version == "v1"
+    assert store.refresh() is False                # 304 no-op
+    v = store.assign("greeting", user_id="user-42")
+    assert v is not None and v["variant_id"] in ("a", "b")
+    assert store.assign("unknown-exp", user_id="x") is None  # not in snapshot
+
+
+def _run_all():
+    fns = [g for n, g in sorted(globals().items()) if n.startswith("test_") and callable(g)]
+    for fn in fns:
+        fn()
+        print(f"  PASS {fn.__name__}")
+    print(f"\nALL {len(fns)} TESTS PASSED")
+
+
+if __name__ == "__main__":
+    _run_all()

variably_sdk-2.9.1/setup.py

@@ -1,23 +0,0 @@
-from setuptools import setup, find_packages
-
-setup(
-    name="variably",
-    version="2.4.0",
-    description="Variably Python SDK for LLM experimentation platform",
-    author="Variably",
-    packages=find_packages(),
-    install_requires=[
-        "requests>=2.25.0",
-    ],
-    python_requires=">=3.8",
-    classifiers=[
-        "Development Status :: 4 - Beta",
-        "Intended Audience :: Developers",
-        "License :: OSI Approved :: MIT License",
-        "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.8",
-        "Programming Language :: Python :: 3.9",
-        "Programming Language :: Python :: 3.10",
-        "Programming Language :: Python :: 3.11",
-    ],
-)