variably-sdk 2.11.0__tar.gz → 2.12.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {variably_sdk-2.11.0 → variably_sdk-2.12.0}/PKG-INFO +1 -1
- {variably_sdk-2.11.0 → variably_sdk-2.12.0}/pyproject.toml +1 -1
- variably_sdk-2.12.0/src/variably/assignment.py +65 -0
- {variably_sdk-2.11.0 → variably_sdk-2.12.0}/src/variably/client.py +68 -0
- {variably_sdk-2.11.0 → variably_sdk-2.12.0}/src/variably/integrations/_common.py +16 -1
- variably_sdk-2.12.0/src/variably/integrations/langgraph.py +324 -0
- {variably_sdk-2.11.0 → variably_sdk-2.12.0}/src/variably/observe.py +4 -0
- variably_sdk-2.12.0/src/variably/prompt_config.py +115 -0
- {variably_sdk-2.11.0 → variably_sdk-2.12.0}/src/variably_sdk.egg-info/PKG-INFO +1 -1
- {variably_sdk-2.11.0 → variably_sdk-2.12.0}/src/variably_sdk.egg-info/SOURCES.txt +3 -0
- variably_sdk-2.12.0/tests/test_prompt_assignment.py +103 -0
- variably_sdk-2.11.0/src/variably/integrations/langgraph.py +0 -202
- {variably_sdk-2.11.0 → variably_sdk-2.12.0}/README.md +0 -0
- {variably_sdk-2.11.0 → variably_sdk-2.12.0}/setup.cfg +0 -0
- {variably_sdk-2.11.0 → variably_sdk-2.12.0}/src/variably/__init__.py +0 -0
- {variably_sdk-2.11.0 → variably_sdk-2.12.0}/src/variably/cache.py +0 -0
- {variably_sdk-2.11.0 → variably_sdk-2.12.0}/src/variably/config.py +0 -0
- {variably_sdk-2.11.0 → variably_sdk-2.12.0}/src/variably/errors.py +0 -0
- {variably_sdk-2.11.0 → variably_sdk-2.12.0}/src/variably/http_client.py +0 -0
- {variably_sdk-2.11.0 → variably_sdk-2.12.0}/src/variably/integrations/__init__.py +0 -0
- {variably_sdk-2.11.0 → variably_sdk-2.12.0}/src/variably/logger.py +0 -0
- {variably_sdk-2.11.0 → variably_sdk-2.12.0}/src/variably/types.py +0 -0
- {variably_sdk-2.11.0 → variably_sdk-2.12.0}/src/variably/version.py +0 -0
- {variably_sdk-2.11.0 → variably_sdk-2.12.0}/src/variably_sdk.egg-info/dependency_links.txt +0 -0
- {variably_sdk-2.11.0 → variably_sdk-2.12.0}/src/variably_sdk.egg-info/requires.txt +0 -0
- {variably_sdk-2.11.0 → variably_sdk-2.12.0}/src/variably_sdk.egg-info/top_level.txt +0 -0
- {variably_sdk-2.11.0 → variably_sdk-2.12.0}/tests/test_integrations_langgraph.py +0 -0
- {variably_sdk-2.11.0 → variably_sdk-2.12.0}/tests/test_streaming.py +0 -0
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "variably-sdk"
|
|
7
|
-
version = "2.
|
|
7
|
+
version = "2.12.0"
|
|
8
8
|
description = "Official Python SDK for Variably feature flags, LLM experimentation, and prompt optimization platform"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.7"
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
"""Deterministic, sticky variant assignment (#54).
|
|
2
|
+
|
|
3
|
+
Mirrors the Go ``internal/assignment`` package **byte-for-byte** — same FNV-1a
|
|
4
|
+
hash, same bucketing, same weighted walk — so the SDK assigns the *identical*
|
|
5
|
+
variant the server would, with no per-event call. Same (variants, experiment_id,
|
|
6
|
+
key) -> same variant, always.
|
|
7
|
+
"""
|
|
8
|
+
from typing import Any, Dict, List, Optional
|
|
9
|
+
|
|
10
|
+
_FNV_OFFSET = 2166136261
|
|
11
|
+
_FNV_PRIME = 16777619
|
|
12
|
+
_MASK32 = 0xFFFFFFFF
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def bucket(salt: str, key: str) -> float:
|
|
16
|
+
"""Stable value in [0, 100) for (salt, key). FNV-1a 32-bit over
|
|
17
|
+
``f"{salt}:{key}"`` — identical output to the Go implementation."""
|
|
18
|
+
h = _FNV_OFFSET
|
|
19
|
+
for b in (salt + ":" + key).encode("utf-8"):
|
|
20
|
+
h ^= b
|
|
21
|
+
h = (h * _FNV_PRIME) & _MASK32
|
|
22
|
+
return (h % 10000) / 100.0
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def key_from_context(
|
|
26
|
+
user_id: Optional[str] = None, session_id: Optional[str] = None
|
|
27
|
+
) -> str:
|
|
28
|
+
"""Bucketing key — prefer user_id, then session_id. Empty when neither."""
|
|
29
|
+
if user_id:
|
|
30
|
+
return user_id
|
|
31
|
+
if session_id:
|
|
32
|
+
return session_id
|
|
33
|
+
return ""
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def select_variant(
|
|
37
|
+
variants: List[Dict[str, Any]], experiment_id: str, key: str
|
|
38
|
+
) -> Optional[Dict[str, Any]]:
|
|
39
|
+
"""Deterministic, sticky, weighted assignment. Mirrors Go SelectVariant.
|
|
40
|
+
|
|
41
|
+
``variants``: dicts with at least ``weight`` and ``is_control`` (as returned
|
|
42
|
+
by the /prompts/config endpoint). Empty key (no user/session id) -> control
|
|
43
|
+
(or first) deterministically, never random.
|
|
44
|
+
"""
|
|
45
|
+
if not variants:
|
|
46
|
+
return None
|
|
47
|
+
if not key:
|
|
48
|
+
for v in variants:
|
|
49
|
+
if v.get("is_control"):
|
|
50
|
+
return v
|
|
51
|
+
return variants[0]
|
|
52
|
+
|
|
53
|
+
total = sum(
|
|
54
|
+
float(v.get("weight", 0)) for v in variants if float(v.get("weight", 0)) > 0
|
|
55
|
+
)
|
|
56
|
+
b = bucket(experiment_id, key) # [0, 100)
|
|
57
|
+
cumulative = 0.0
|
|
58
|
+
n = len(variants)
|
|
59
|
+
for v in variants:
|
|
60
|
+
w = float(v.get("weight", 0))
|
|
61
|
+
w = (w / total * 100.0) if total > 0 else (100.0 / n)
|
|
62
|
+
cumulative += w
|
|
63
|
+
if b < cumulative:
|
|
64
|
+
return v
|
|
65
|
+
return variants[-1]
|
|
@@ -10,6 +10,7 @@ from .config import VariablyConfig
|
|
|
10
10
|
from .errors import ConfigurationError, VariablyError
|
|
11
11
|
from .http_client import HTTPClient
|
|
12
12
|
from .logger import VariablyLogger, create_logger
|
|
13
|
+
from .prompt_config import PromptConfigStore, http_config_fetcher
|
|
13
14
|
from .types import (
|
|
14
15
|
EvaluateResponse,
|
|
15
16
|
Event,
|
|
@@ -49,6 +50,10 @@ class VariablyClient:
|
|
|
49
50
|
|
|
50
51
|
# Initialize HTTP client
|
|
51
52
|
self.http_client = HTTPClient(config, self.logger, self.metrics)
|
|
53
|
+
|
|
54
|
+
# Lazily-started prompt config store for local (in-process) variant
|
|
55
|
+
# assignment — bootstrapped + polled on first get_variant_local() call.
|
|
56
|
+
self._prompt_config_store: Optional[PromptConfigStore] = None
|
|
52
57
|
|
|
53
58
|
self.logger.info(
|
|
54
59
|
"Variably client initialized",
|
|
@@ -486,6 +491,66 @@ class VariablyClient:
|
|
|
486
491
|
is_control=variant.get("is_control", variant.get("isControl", False)),
|
|
487
492
|
)
|
|
488
493
|
|
|
494
|
+
def get_variant_local(
|
|
495
|
+
self,
|
|
496
|
+
experiment_key: str,
|
|
497
|
+
user_context: UserContext,
|
|
498
|
+
poll_interval_seconds: float = 45.0,
|
|
499
|
+
) -> PromptVariant:
|
|
500
|
+
"""Assign a prompt variant **in-process** from a cached, background-polled
|
|
501
|
+
config snapshot — no per-call DB query on the server (#54.3).
|
|
502
|
+
|
|
503
|
+
Deterministic and sticky: the same (experiment_key, user_id/session_id)
|
|
504
|
+
always maps to the same variant, using the exact hash the server would.
|
|
505
|
+
The config is bootstrapped on first call and refreshed every
|
|
506
|
+
``poll_interval_seconds`` with ETag (a 304 costs one cheap round trip).
|
|
507
|
+
|
|
508
|
+
Falls back to the server ``get_variant`` if the experiment isn't in the
|
|
509
|
+
snapshot yet (e.g. created between polls), so a brand-new experiment is
|
|
510
|
+
never silently missed.
|
|
511
|
+
"""
|
|
512
|
+
if not experiment_key or not isinstance(experiment_key, str):
|
|
513
|
+
raise ConfigurationError(
|
|
514
|
+
"Experiment key must be a non-empty string", "experiment_key"
|
|
515
|
+
)
|
|
516
|
+
key = user_context.get("user_id") or user_context.get("session_id")
|
|
517
|
+
if not key:
|
|
518
|
+
raise ConfigurationError(
|
|
519
|
+
"User context must include user_id or session_id",
|
|
520
|
+
"user_context.user_id",
|
|
521
|
+
)
|
|
522
|
+
|
|
523
|
+
if self._prompt_config_store is None:
|
|
524
|
+
store = PromptConfigStore(
|
|
525
|
+
http_config_fetcher(self.config.base_url, self.config.api_key)
|
|
526
|
+
)
|
|
527
|
+
store.start_polling(poll_interval_seconds)
|
|
528
|
+
self._prompt_config_store = store
|
|
529
|
+
|
|
530
|
+
variant = self._prompt_config_store.assign(
|
|
531
|
+
experiment_key,
|
|
532
|
+
user_id=user_context.get("user_id"),
|
|
533
|
+
session_id=user_context.get("session_id"),
|
|
534
|
+
)
|
|
535
|
+
if variant is None:
|
|
536
|
+
# Not in snapshot (new/just-changed experiment) — authoritative fallback.
|
|
537
|
+
return self.get_variant(experiment_key, user_context)
|
|
538
|
+
|
|
539
|
+
# Extract model/provider from parameters.llmConfig, mirroring get_variant.
|
|
540
|
+
params = variant.get("parameters", {}) or {}
|
|
541
|
+
llm_config = params.get("llmConfig", params)
|
|
542
|
+
return PromptVariant(
|
|
543
|
+
experiment_id=variant.get("experiment_id", ""),
|
|
544
|
+
experiment_key=experiment_key,
|
|
545
|
+
variant_id=variant.get("variant_id", ""),
|
|
546
|
+
variant_key=variant.get("name", variant.get("variant_key", "")),
|
|
547
|
+
prompt_template=variant.get("prompt_template", ""),
|
|
548
|
+
model=llm_config.get("model", ""),
|
|
549
|
+
provider=llm_config.get("provider", ""),
|
|
550
|
+
parameters=params,
|
|
551
|
+
is_control=variant.get("is_control", False),
|
|
552
|
+
)
|
|
553
|
+
|
|
489
554
|
def evaluate_prompt(
|
|
490
555
|
self,
|
|
491
556
|
experiment_key: str,
|
|
@@ -683,6 +748,7 @@ class VariablyClient:
|
|
|
683
748
|
is_aggregator: bool = False, # True on the final step that aggregates the trace
|
|
684
749
|
tools_called: Optional[List[Dict[str, Any]]] = None, # [{"name","arguments","output"}] actually called
|
|
685
750
|
expected_tools: Optional[List[Dict[str, Any]]] = None, # golden tools for deterministic trajectory scoring
|
|
751
|
+
allowed_tools: Optional[List[str]] = None, # permitted tool names; calls outside this list flag as unauthorized
|
|
686
752
|
# User & organization context
|
|
687
753
|
user_id: Optional[str] = None, # End-user ID for per-user analytics
|
|
688
754
|
tags: Optional[List[str]] = None, # Grouping labels, e.g. ["production", "rag"]
|
|
@@ -798,6 +864,8 @@ class VariablyClient:
|
|
|
798
864
|
data["tools_called"] = tools_called
|
|
799
865
|
if expected_tools:
|
|
800
866
|
data["expected_tools"] = expected_tools
|
|
867
|
+
if allowed_tools:
|
|
868
|
+
data["allowed_tools"] = allowed_tools
|
|
801
869
|
if provider:
|
|
802
870
|
data["provider"] = provider
|
|
803
871
|
if model:
|
|
@@ -17,6 +17,9 @@ from typing import Any, Optional
|
|
|
17
17
|
_RESPONSE_KEYS = ("response", "output", "answer", "result", "text", "content")
|
|
18
18
|
# Keys we look for when extracting the user query/prompt from graph state.
|
|
19
19
|
_QUERY_KEYS = ("query", "question", "input", "prompt", "user_input")
|
|
20
|
+
# Frameworks commonly nest config/inputs under one of these container keys
|
|
21
|
+
# (e.g. gpt-researcher's state["task"]["query"]). Checked one level deep.
|
|
22
|
+
_QUERY_CONTAINERS = ("task", "input", "config", "configurable", "state", "metadata")
|
|
20
23
|
|
|
21
24
|
|
|
22
25
|
def warn(msg: str) -> None:
|
|
@@ -68,13 +71,25 @@ def extract_response_text(update: Any) -> str:
|
|
|
68
71
|
|
|
69
72
|
|
|
70
73
|
def extract_query(state: Any) -> str:
|
|
71
|
-
"""Best-effort extraction of the user query/prompt from graph state.
|
|
74
|
+
"""Best-effort extraction of the user query/prompt from graph state.
|
|
75
|
+
|
|
76
|
+
Checks top-level keys first, then one level into common container dicts
|
|
77
|
+
(``task``/``input``/``config``/…) so frameworks that nest the query — e.g.
|
|
78
|
+
gpt-researcher's ``state["task"]["query"]`` — are handled without a custom
|
|
79
|
+
extractor. Falls back to "" (caller then uses a ``[node:name]`` placeholder)."""
|
|
72
80
|
try:
|
|
73
81
|
get = state.get if isinstance(state, dict) else (lambda k, d=None: getattr(state, k, d))
|
|
74
82
|
for k in _QUERY_KEYS:
|
|
75
83
|
v = get(k)
|
|
76
84
|
if isinstance(v, str) and v.strip():
|
|
77
85
|
return v
|
|
86
|
+
for container in _QUERY_CONTAINERS:
|
|
87
|
+
c = get(container)
|
|
88
|
+
if isinstance(c, dict):
|
|
89
|
+
for k in _QUERY_KEYS:
|
|
90
|
+
v = c.get(k)
|
|
91
|
+
if isinstance(v, str) and v.strip():
|
|
92
|
+
return v
|
|
78
93
|
except Exception:
|
|
79
94
|
pass
|
|
80
95
|
return ""
|
|
@@ -0,0 +1,324 @@
|
|
|
1
|
+
"""LangGraph auto-instrumentation for variА/Bly (Task 42.4).
|
|
2
|
+
|
|
3
|
+
Two entry points:
|
|
4
|
+
|
|
5
|
+
1. ``instrument(graph, session_id_key=...)`` — zero-touch. Call it once after
|
|
6
|
+
you've added nodes/edges (before ``.compile()``). Wraps every node so each
|
|
7
|
+
execution logs to variА/Bly: ``agent_name`` = node name, ``is_aggregator``
|
|
8
|
+
on the graph's terminal node(s), ``session_id`` pulled from graph state.
|
|
9
|
+
|
|
10
|
+
2. ``@observe_node(...)`` — an explicit per-node decorator. Version-independent
|
|
11
|
+
and the most robust option if ``instrument`` can't wrap your graph's node
|
|
12
|
+
representation.
|
|
13
|
+
|
|
14
|
+
Both are best-effort: instrumentation failures degrade to a stderr warning and
|
|
15
|
+
NEVER raise into your graph. Targets langgraph >= 0.1; call ``instrument`` after
|
|
16
|
+
``add_node`` / ``add_edge`` and before ``compile()``.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import asyncio
|
|
22
|
+
import functools
|
|
23
|
+
from typing import Any, Callable, Iterable, Optional
|
|
24
|
+
|
|
25
|
+
from variably import observe as _observe
|
|
26
|
+
from variably.integrations._common import (
|
|
27
|
+
extract_query,
|
|
28
|
+
extract_response_text,
|
|
29
|
+
extract_session_id,
|
|
30
|
+
warn,
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
# Node names we've already warned about (no response text) — warn once per node
|
|
35
|
+
# per process so the message is visible without spamming every run.
|
|
36
|
+
_warned_no_response: set = set()
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _shape(value: Any) -> str:
|
|
40
|
+
"""Describe a value's STRUCTURE (type + dict keys) without exposing its
|
|
41
|
+
contents — safe to print/share for diagnostics."""
|
|
42
|
+
try:
|
|
43
|
+
if isinstance(value, dict):
|
|
44
|
+
keys = [str(k) for k in list(value.keys())[:25]]
|
|
45
|
+
more = "…" if len(value) > 25 else ""
|
|
46
|
+
return f"dict(keys=[{', '.join(keys)}{more}])"
|
|
47
|
+
if isinstance(value, (list, tuple)):
|
|
48
|
+
return f"{type(value).__name__}(len={len(value)})"
|
|
49
|
+
return type(value).__name__
|
|
50
|
+
except Exception:
|
|
51
|
+
return "<unknown>"
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _log_node(
|
|
55
|
+
name: str,
|
|
56
|
+
state: Any,
|
|
57
|
+
update: Any,
|
|
58
|
+
*,
|
|
59
|
+
is_aggregator: bool,
|
|
60
|
+
session_id_key: str,
|
|
61
|
+
response_extractor: Optional[Callable[[Any], str]],
|
|
62
|
+
context_extractor: Optional[Callable[[Any, Any], Any]] = None,
|
|
63
|
+
tools_extractor: Optional[Callable[[Any, Any], Any]] = None,
|
|
64
|
+
) -> None:
|
|
65
|
+
"""Log one node execution to variА/Bly. Never raises."""
|
|
66
|
+
try:
|
|
67
|
+
resp = (response_extractor or extract_response_text)(update)
|
|
68
|
+
# No response text → do NOT fabricate one (a serialized dict would be scored
|
|
69
|
+
# as meaningless prose) and do NOT silently drop it. Warn the developer once
|
|
70
|
+
# per node (visible on stderr, non-blocking) so they notice and either add a
|
|
71
|
+
# response_extractor= or accept that this node has no scorable output.
|
|
72
|
+
if not (resp and resp.strip()):
|
|
73
|
+
if name not in _warned_no_response:
|
|
74
|
+
_warned_no_response.add(name)
|
|
75
|
+
warn(
|
|
76
|
+
f"node '{name}' produced no response text — this step was NOT scored.\n"
|
|
77
|
+
f" node = {name!r}\n"
|
|
78
|
+
f" returned = {_shape(update)}\n"
|
|
79
|
+
f" extractor = {'custom response_extractor' if response_extractor else 'default (extract_response_text)'}\n"
|
|
80
|
+
f" aggregator = {is_aggregator}\n"
|
|
81
|
+
f" Fix: pass response_extractor= to instrument() mapping this node's output to a "
|
|
82
|
+
f"string, e.g. response_extractor=lambda upd: upd.get('report', '').\n"
|
|
83
|
+
f" Only the structure above (type + keys) is shown — never your data — so this "
|
|
84
|
+
f"line is safe to share with variА/Bly support if a step you expected isn't scored."
|
|
85
|
+
)
|
|
86
|
+
return
|
|
87
|
+
sid = extract_session_id(state, session_id_key)
|
|
88
|
+
# is_aggregator requires a session_id (SDK write-time rule); only claim
|
|
89
|
+
# it when we actually have one, else it degrades to a normal step.
|
|
90
|
+
agg = bool(is_aggregator and sid)
|
|
91
|
+
# Retrieved context enables grounding/faithfulness scoring. Without it,
|
|
92
|
+
# only the non-grounding dimensions are scored. Best-effort; never fatal.
|
|
93
|
+
refs = None
|
|
94
|
+
if context_extractor is not None:
|
|
95
|
+
try:
|
|
96
|
+
refs = context_extractor(state, update)
|
|
97
|
+
except Exception:
|
|
98
|
+
refs = None
|
|
99
|
+
# Tool calls enable the trajectory / tool-correctness / order-adherence
|
|
100
|
+
# metrics. Extractor returns {"tools_called": [...], "expected_tools": [...]}.
|
|
101
|
+
tools_called = expected_tools = None
|
|
102
|
+
if tools_extractor is not None:
|
|
103
|
+
try:
|
|
104
|
+
t = tools_extractor(state, update) or {}
|
|
105
|
+
tools_called = t.get("tools_called")
|
|
106
|
+
expected_tools = t.get("expected_tools")
|
|
107
|
+
except Exception:
|
|
108
|
+
tools_called = expected_tools = None
|
|
109
|
+
_observe(
|
|
110
|
+
prompt=extract_query(state) or f"[node:{name}]",
|
|
111
|
+
response=resp or "",
|
|
112
|
+
session_id=sid,
|
|
113
|
+
agent_name=name,
|
|
114
|
+
is_aggregator=agg,
|
|
115
|
+
reference_materials=refs,
|
|
116
|
+
tools_called=tools_called,
|
|
117
|
+
expected_tools=expected_tools,
|
|
118
|
+
)
|
|
119
|
+
except Exception as e: # pragma: no cover - defensive
|
|
120
|
+
warn(f"node '{name}' instrumentation failed: {e}")
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def observe_node(
|
|
124
|
+
*,
|
|
125
|
+
agent_name: Optional[str] = None,
|
|
126
|
+
is_aggregator: bool = False,
|
|
127
|
+
session_id_key: str = "session_id",
|
|
128
|
+
response_extractor: Optional[Callable[[Any], str]] = None,
|
|
129
|
+
) -> Callable[[Callable], Callable]:
|
|
130
|
+
"""Decorator for a LangGraph node function. Logs the node's output to
|
|
131
|
+
variА/Bly after it runs. ``agent_name`` defaults to the function name."""
|
|
132
|
+
|
|
133
|
+
def decorate(fn: Callable) -> Callable:
|
|
134
|
+
name = agent_name or getattr(fn, "__name__", "node")
|
|
135
|
+
|
|
136
|
+
@functools.wraps(fn)
|
|
137
|
+
def wrapper(state, *args, **kwargs):
|
|
138
|
+
update = fn(state, *args, **kwargs)
|
|
139
|
+
_log_node(
|
|
140
|
+
name, state, update,
|
|
141
|
+
is_aggregator=is_aggregator,
|
|
142
|
+
session_id_key=session_id_key,
|
|
143
|
+
response_extractor=response_extractor,
|
|
144
|
+
)
|
|
145
|
+
return update
|
|
146
|
+
|
|
147
|
+
return wrapper
|
|
148
|
+
|
|
149
|
+
return decorate
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def _detect_terminal_nodes(graph) -> set:
|
|
153
|
+
"""Terminal nodes = those with an edge to END, plus any with no outgoing
|
|
154
|
+
edge. Best-effort across langgraph versions."""
|
|
155
|
+
terminals: set = set()
|
|
156
|
+
try:
|
|
157
|
+
try:
|
|
158
|
+
from langgraph.graph import END
|
|
159
|
+
except Exception:
|
|
160
|
+
END = "__end__"
|
|
161
|
+
|
|
162
|
+
node_names = set(getattr(graph, "nodes", {}) or {})
|
|
163
|
+
edges = getattr(graph, "edges", None) or set()
|
|
164
|
+
has_outgoing: set = set()
|
|
165
|
+
for edge in edges:
|
|
166
|
+
try:
|
|
167
|
+
u, v = edge
|
|
168
|
+
except Exception:
|
|
169
|
+
continue
|
|
170
|
+
has_outgoing.add(u)
|
|
171
|
+
if v == END or v == "__end__":
|
|
172
|
+
terminals.add(u)
|
|
173
|
+
# nodes with no outgoing plain edge (may route via conditional branches,
|
|
174
|
+
# but this is a reasonable default when no END edge is declared)
|
|
175
|
+
terminals |= (node_names - has_outgoing)
|
|
176
|
+
except Exception as e: # pragma: no cover - defensive
|
|
177
|
+
warn(f"terminal-node detection failed: {e}")
|
|
178
|
+
return terminals
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def _wrap_callable(name, fn, is_agg, session_id_key, response_extractor, context_extractor=None, tools_extractor=None):
|
|
182
|
+
@functools.wraps(fn)
|
|
183
|
+
def wrapped(state, *args, **kwargs):
|
|
184
|
+
update = fn(state, *args, **kwargs)
|
|
185
|
+
_log_node(
|
|
186
|
+
name, state, update,
|
|
187
|
+
is_aggregator=is_agg,
|
|
188
|
+
session_id_key=session_id_key,
|
|
189
|
+
response_extractor=response_extractor,
|
|
190
|
+
context_extractor=context_extractor,
|
|
191
|
+
tools_extractor=tools_extractor,
|
|
192
|
+
)
|
|
193
|
+
return update
|
|
194
|
+
return wrapped
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def _wrap_callable_async(name, fn, is_agg, session_id_key, response_extractor, context_extractor=None, tools_extractor=None):
|
|
198
|
+
"""Async counterpart of _wrap_callable. Most real LangGraph nodes are
|
|
199
|
+
``async def`` (invoked via ``ainvoke``), so their fn lives on the runnable's
|
|
200
|
+
``.afunc`` — this wraps that path. ``_observe`` is non-blocking, so logging
|
|
201
|
+
from inside the event loop is safe."""
|
|
202
|
+
@functools.wraps(fn)
|
|
203
|
+
async def wrapped(state, *args, **kwargs):
|
|
204
|
+
update = await fn(state, *args, **kwargs)
|
|
205
|
+
_log_node(
|
|
206
|
+
name, state, update,
|
|
207
|
+
is_aggregator=is_agg,
|
|
208
|
+
session_id_key=session_id_key,
|
|
209
|
+
response_extractor=response_extractor,
|
|
210
|
+
context_extractor=context_extractor,
|
|
211
|
+
tools_extractor=tools_extractor,
|
|
212
|
+
)
|
|
213
|
+
return update
|
|
214
|
+
return wrapped
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def instrument(
|
|
218
|
+
graph,
|
|
219
|
+
*,
|
|
220
|
+
session_id_key: str = "session_id",
|
|
221
|
+
aggregator_nodes: Optional[Iterable[str]] = None,
|
|
222
|
+
response_extractor: Optional[Callable[[Any], str]] = None,
|
|
223
|
+
context_extractor: Optional[Callable[[Any, Any], Any]] = None,
|
|
224
|
+
tools_extractor: Optional[Callable[[Any, Any], Any]] = None,
|
|
225
|
+
):
|
|
226
|
+
"""Instrument every node in a LangGraph ``StateGraph`` in place.
|
|
227
|
+
|
|
228
|
+
Args:
|
|
229
|
+
graph: an uncompiled ``StateGraph``.
|
|
230
|
+
session_id_key: state key holding the session/thread id.
|
|
231
|
+
aggregator_nodes: override terminal-node auto-detection with an explicit set.
|
|
232
|
+
response_extractor: custom fn to pull response text from a node's return.
|
|
233
|
+
context_extractor: custom fn ``(state, update) -> list[{"id","content"}]``
|
|
234
|
+
returning the retrieved reference materials for grounding/faithfulness
|
|
235
|
+
scoring. Return None/[] when the node has no context. Without it,
|
|
236
|
+
grounding cannot be scored (only the non-grounding dimensions).
|
|
237
|
+
tools_extractor: custom fn ``(state, update) -> {"tools_called": [...],
|
|
238
|
+
"expected_tools": [...]}`` (each tool a ``{"name","arguments","output"}``
|
|
239
|
+
dict) enabling the trajectory / tool-correctness / order-adherence
|
|
240
|
+
metrics. Return None when the node makes no tool calls.
|
|
241
|
+
|
|
242
|
+
Returns the same graph (for chaining). Best-effort; never raises.
|
|
243
|
+
"""
|
|
244
|
+
nodes = getattr(graph, "nodes", None)
|
|
245
|
+
if not nodes:
|
|
246
|
+
warn("graph has no .nodes — nothing to instrument (pass an uncompiled StateGraph)")
|
|
247
|
+
return graph
|
|
248
|
+
|
|
249
|
+
terminals = set(aggregator_nodes) if aggregator_nodes is not None else _detect_terminal_nodes(graph)
|
|
250
|
+
|
|
251
|
+
for name in list(nodes.keys()):
|
|
252
|
+
is_agg = name in terminals
|
|
253
|
+
try:
|
|
254
|
+
spec = nodes[name]
|
|
255
|
+
# langgraph StateNodeSpec exposes the node fn on .runnable; older /
|
|
256
|
+
# simpler representations store the callable directly.
|
|
257
|
+
runnable = getattr(spec, "runnable", None)
|
|
258
|
+
if runnable is not None:
|
|
259
|
+
wrapped_any = False
|
|
260
|
+
# Sync path — RunnableCallable stores a sync fn on .func.
|
|
261
|
+
sync_fn = getattr(runnable, "func", None)
|
|
262
|
+
if callable(sync_fn):
|
|
263
|
+
try:
|
|
264
|
+
runnable.func = _wrap_callable(
|
|
265
|
+
name, sync_fn, is_agg, session_id_key, response_extractor,
|
|
266
|
+
context_extractor, tools_extractor)
|
|
267
|
+
wrapped_any = True
|
|
268
|
+
except Exception:
|
|
269
|
+
pass
|
|
270
|
+
# Async path — async nodes store the coroutine fn on .afunc, which
|
|
271
|
+
# is what langgraph calls under ainvoke. Missing this = no events
|
|
272
|
+
# for any async graph (the common case).
|
|
273
|
+
async_fn = getattr(runnable, "afunc", None)
|
|
274
|
+
if callable(async_fn):
|
|
275
|
+
try:
|
|
276
|
+
runnable.afunc = _wrap_callable_async(
|
|
277
|
+
name, async_fn, is_agg, session_id_key, response_extractor,
|
|
278
|
+
context_extractor, tools_extractor)
|
|
279
|
+
wrapped_any = True
|
|
280
|
+
except Exception:
|
|
281
|
+
pass
|
|
282
|
+
if wrapped_any:
|
|
283
|
+
continue
|
|
284
|
+
# No .func/.afunc — the runnable itself may be the callable.
|
|
285
|
+
if callable(runnable):
|
|
286
|
+
wrapped = (
|
|
287
|
+
_wrap_callable_async(name, runnable, is_agg, session_id_key,
|
|
288
|
+
response_extractor, context_extractor, tools_extractor)
|
|
289
|
+
if asyncio.iscoroutinefunction(runnable)
|
|
290
|
+
else _wrap_callable(name, runnable, is_agg, session_id_key,
|
|
291
|
+
response_extractor, context_extractor, tools_extractor)
|
|
292
|
+
)
|
|
293
|
+
if _replace_spec_runnable(nodes, name, spec, wrapped):
|
|
294
|
+
continue
|
|
295
|
+
warn(f"node '{name}': could not rewrap runnable (unsupported spec)")
|
|
296
|
+
elif callable(spec):
|
|
297
|
+
nodes[name] = (
|
|
298
|
+
_wrap_callable_async(name, spec, is_agg, session_id_key, response_extractor)
|
|
299
|
+
if asyncio.iscoroutinefunction(spec)
|
|
300
|
+
else _wrap_callable(name, spec, is_agg, session_id_key, response_extractor)
|
|
301
|
+
)
|
|
302
|
+
else:
|
|
303
|
+
warn(f"node '{name}': unrecognised spec type {type(spec).__name__}; skipped")
|
|
304
|
+
except Exception as e: # pragma: no cover - defensive
|
|
305
|
+
warn(f"failed to instrument node '{name}': {e}")
|
|
306
|
+
|
|
307
|
+
return graph
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
def _replace_spec_runnable(nodes, name, spec, wrapped) -> bool:
|
|
311
|
+
"""Try to put `wrapped` back onto the node spec across spec representations."""
|
|
312
|
+
# settable attribute
|
|
313
|
+
try:
|
|
314
|
+
spec.runnable = wrapped # type: ignore[attr-defined]
|
|
315
|
+
return True
|
|
316
|
+
except Exception:
|
|
317
|
+
pass
|
|
318
|
+
# namedtuple-style: rebuild via _replace
|
|
319
|
+
try:
|
|
320
|
+
nodes[name] = spec._replace(runnable=wrapped) # type: ignore[attr-defined]
|
|
321
|
+
return True
|
|
322
|
+
except Exception:
|
|
323
|
+
pass
|
|
324
|
+
return False
|
|
@@ -184,6 +184,9 @@ def observe(
|
|
|
184
184
|
# deterministic trajectory precision/recall + tool-correctness metrics.
|
|
185
185
|
tools_called: Optional[List[Dict[str, Any]]] = None,
|
|
186
186
|
expected_tools: Optional[List[Dict[str, Any]]] = None,
|
|
187
|
+
# Permitted-tool allowlist (names). Any call outside it flags as an
|
|
188
|
+
# unauthorized action — the deterministic security/adversarial signal.
|
|
189
|
+
allowed_tools: Optional[List[str]] = None,
|
|
187
190
|
|
|
188
191
|
# --- User & organization context ---
|
|
189
192
|
user_id: Optional[str] = None, # Your end-user's ID — enables per-user analytics
|
|
@@ -288,6 +291,7 @@ def observe(
|
|
|
288
291
|
is_aggregator=is_aggregator,
|
|
289
292
|
tools_called=tools_called,
|
|
290
293
|
expected_tools=expected_tools,
|
|
294
|
+
allowed_tools=allowed_tools,
|
|
291
295
|
metadata=metadata,
|
|
292
296
|
kind=kind,
|
|
293
297
|
)
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
"""SDK-side prompt config store + background poll (#54.2/54.3).
|
|
2
|
+
|
|
3
|
+
Bootstraps the project's prompt-experiment config from
|
|
4
|
+
``GET /internal/sdk/prompts/config`` once, refreshes it in the background with
|
|
5
|
+
ETag/If-None-Match (a 304 is a no-op), and assigns variants **locally** from the
|
|
6
|
+
cached snapshot — zero backend call per event. Deterministic + sticky via
|
|
7
|
+
``assignment`` (which is byte-for-byte identical to the server).
|
|
8
|
+
"""
|
|
9
|
+
import threading
|
|
10
|
+
from typing import Any, Callable, Dict, List, Optional, Tuple
|
|
11
|
+
|
|
12
|
+
from .assignment import key_from_context, select_variant
|
|
13
|
+
|
|
14
|
+
# A fetcher takes the current ETag/version and returns (version, config_dict) on
|
|
15
|
+
# change, or None when unchanged (304), errored, or nothing to update. Injectable
|
|
16
|
+
# so the store is testable without a live server.
|
|
17
|
+
Fetcher = Callable[[Optional[str]], Optional[Tuple[str, Dict[str, Any]]]]
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class PromptConfigStore:
|
|
21
|
+
"""Thread-safe snapshot of the SDK prompt config + local assignment."""
|
|
22
|
+
|
|
23
|
+
def __init__(self, fetcher: Fetcher):
|
|
24
|
+
self._fetcher = fetcher
|
|
25
|
+
self._lock = threading.Lock()
|
|
26
|
+
self._version: Optional[str] = None
|
|
27
|
+
self._experiments: Dict[str, Dict[str, Any]] = {} # experiment_key -> experiment
|
|
28
|
+
self._stop = threading.Event()
|
|
29
|
+
self._thread: Optional[threading.Thread] = None
|
|
30
|
+
|
|
31
|
+
def refresh(self) -> bool:
|
|
32
|
+
"""Fetch config and atomically swap the snapshot on change. Returns True
|
|
33
|
+
if the snapshot was updated, False on 304/unchanged/error. Never raises."""
|
|
34
|
+
try:
|
|
35
|
+
result = self._fetcher(self._version)
|
|
36
|
+
except Exception:
|
|
37
|
+
return False
|
|
38
|
+
if result is None:
|
|
39
|
+
return False
|
|
40
|
+
version, cfg = result
|
|
41
|
+
index = {
|
|
42
|
+
e["experiment_key"]: e for e in (cfg.get("experiments") or []) if e.get("experiment_key")
|
|
43
|
+
}
|
|
44
|
+
with self._lock:
|
|
45
|
+
self._version = version
|
|
46
|
+
self._experiments = index
|
|
47
|
+
return True
|
|
48
|
+
|
|
49
|
+
def assign(
|
|
50
|
+
self,
|
|
51
|
+
experiment_key: str,
|
|
52
|
+
user_id: Optional[str] = None,
|
|
53
|
+
session_id: Optional[str] = None,
|
|
54
|
+
) -> Optional[Dict[str, Any]]:
|
|
55
|
+
"""Assign a variant locally from the cached snapshot. Returns the variant
|
|
56
|
+
dict, or None if the experiment isn't in the snapshot (not running /
|
|
57
|
+
unknown / config not yet loaded)."""
|
|
58
|
+
with self._lock:
|
|
59
|
+
exp = self._experiments.get(experiment_key)
|
|
60
|
+
if not exp:
|
|
61
|
+
return None
|
|
62
|
+
experiment_id = exp["experiment_id"]
|
|
63
|
+
key = key_from_context(user_id, session_id)
|
|
64
|
+
variant = select_variant(exp.get("variants") or [], experiment_id, key)
|
|
65
|
+
if variant is None:
|
|
66
|
+
return None
|
|
67
|
+
# Stamp experiment_id (lives on the experiment, not the variant) so callers
|
|
68
|
+
# get a self-contained variant. Copy to avoid mutating the snapshot.
|
|
69
|
+
return {**variant, "experiment_id": experiment_id}
|
|
70
|
+
|
|
71
|
+
def start_polling(self, interval_seconds: float = 45.0) -> None:
|
|
72
|
+
"""Bootstrap once, then refresh every ``interval_seconds`` in a daemon
|
|
73
|
+
thread. Idempotent."""
|
|
74
|
+
if self._thread is not None:
|
|
75
|
+
return
|
|
76
|
+
self.refresh() # initial bootstrap
|
|
77
|
+
|
|
78
|
+
def _loop() -> None:
|
|
79
|
+
while not self._stop.wait(interval_seconds):
|
|
80
|
+
self.refresh()
|
|
81
|
+
|
|
82
|
+
self._thread = threading.Thread(
|
|
83
|
+
target=_loop, name="variably-config-poll", daemon=True
|
|
84
|
+
)
|
|
85
|
+
self._thread.start()
|
|
86
|
+
|
|
87
|
+
def stop(self) -> None:
|
|
88
|
+
self._stop.set()
|
|
89
|
+
|
|
90
|
+
@property
|
|
91
|
+
def version(self) -> Optional[str]:
|
|
92
|
+
return self._version
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def http_config_fetcher(base_url: str, api_key: str, timeout: float = 5.0) -> Fetcher:
|
|
96
|
+
"""Build a fetcher that GETs /internal/sdk/prompts/config with ETag. Uses
|
|
97
|
+
``requests`` (already an SDK dep). Returns (version, config) on 200, None on
|
|
98
|
+
304 or any error — the store keeps its last-good snapshot."""
|
|
99
|
+
import requests
|
|
100
|
+
|
|
101
|
+
url = base_url.rstrip("/") + "/api/v1/internal/sdk/prompts/config"
|
|
102
|
+
|
|
103
|
+
def _fetch(etag: Optional[str]) -> Optional[Tuple[str, Dict[str, Any]]]:
|
|
104
|
+
headers = {"Authorization": f"Bearer {api_key}"}
|
|
105
|
+
if etag:
|
|
106
|
+
headers["If-None-Match"] = '"' + etag + '"'
|
|
107
|
+
resp = requests.get(url, headers=headers, timeout=timeout)
|
|
108
|
+
if resp.status_code == 304:
|
|
109
|
+
return None
|
|
110
|
+
if resp.status_code != 200:
|
|
111
|
+
return None
|
|
112
|
+
cfg = resp.json()
|
|
113
|
+
return cfg.get("version", ""), cfg
|
|
114
|
+
|
|
115
|
+
return _fetch
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
README.md
|
|
2
2
|
pyproject.toml
|
|
3
3
|
src/variably/__init__.py
|
|
4
|
+
src/variably/assignment.py
|
|
4
5
|
src/variably/cache.py
|
|
5
6
|
src/variably/client.py
|
|
6
7
|
src/variably/config.py
|
|
@@ -8,6 +9,7 @@ src/variably/errors.py
|
|
|
8
9
|
src/variably/http_client.py
|
|
9
10
|
src/variably/logger.py
|
|
10
11
|
src/variably/observe.py
|
|
12
|
+
src/variably/prompt_config.py
|
|
11
13
|
src/variably/types.py
|
|
12
14
|
src/variably/version.py
|
|
13
15
|
src/variably/integrations/__init__.py
|
|
@@ -19,4 +21,5 @@ src/variably_sdk.egg-info/dependency_links.txt
|
|
|
19
21
|
src/variably_sdk.egg-info/requires.txt
|
|
20
22
|
src/variably_sdk.egg-info/top_level.txt
|
|
21
23
|
tests/test_integrations_langgraph.py
|
|
24
|
+
tests/test_prompt_assignment.py
|
|
22
25
|
tests/test_streaming.py
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
"""Tests for SDK-side local assignment (#54.3) + config store (#54.2).
|
|
2
|
+
|
|
3
|
+
Run via pytest, or directly: `python tests/test_prompt_assignment.py` (the direct
|
|
4
|
+
runner avoids the legacy top-level `variably` package shadowing `src/variably`).
|
|
5
|
+
"""
|
|
6
|
+
import os
|
|
7
|
+
import sys
|
|
8
|
+
|
|
9
|
+
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "src"))
|
|
10
|
+
|
|
11
|
+
from variably.assignment import bucket, key_from_context, select_variant # noqa: E402
|
|
12
|
+
from variably.prompt_config import PromptConfigStore # noqa: E402
|
|
13
|
+
|
|
14
|
+
# Reference values computed by the Go internal/assignment package — the SDK MUST
|
|
15
|
+
# match these byte-for-byte, or server and client assign different variants.
|
|
16
|
+
GO_BUCKETS = {
|
|
17
|
+
("exp-1", "user-42"): 41.20,
|
|
18
|
+
("exp-1", "user-99"): 64.00,
|
|
19
|
+
("exp-2", "user-42"): 81.89,
|
|
20
|
+
("clinical", "abc-123"): 53.23,
|
|
21
|
+
("e", "s"): 99.59,
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _vars():
|
|
26
|
+
return [
|
|
27
|
+
{"variant_id": "a", "name": "control", "is_control": True, "weight": 70},
|
|
28
|
+
{"variant_id": "b", "name": "treatment", "is_control": False, "weight": 30},
|
|
29
|
+
]
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def test_bucket_parity_with_go():
|
|
33
|
+
for (salt, key), expected in GO_BUCKETS.items():
|
|
34
|
+
got = bucket(salt, key)
|
|
35
|
+
assert abs(got - expected) < 0.005, f"{salt}:{key} py={got} go={expected}"
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def test_sticky():
|
|
39
|
+
first = select_variant(_vars(), "exp-1", "user-42")
|
|
40
|
+
for _ in range(1000):
|
|
41
|
+
assert select_variant(_vars(), "exp-1", "user-42")["variant_id"] == first["variant_id"]
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def test_distribution():
|
|
45
|
+
counts = {"a": 0, "b": 0}
|
|
46
|
+
n = 10000
|
|
47
|
+
for i in range(n):
|
|
48
|
+
counts[select_variant(_vars(), "exp-1", f"user-{i}")["variant_id"]] += 1
|
|
49
|
+
assert 65 <= counts["a"] / n * 100 <= 75
|
|
50
|
+
assert 25 <= counts["b"] / n * 100 <= 35
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def test_keyless_returns_control():
|
|
54
|
+
for _ in range(50):
|
|
55
|
+
v = select_variant(_vars(), "exp-1", "")
|
|
56
|
+
assert v is not None and v["is_control"]
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def test_key_from_context():
|
|
60
|
+
assert key_from_context("u1", "s1") == "u1"
|
|
61
|
+
assert key_from_context(None, "s1") == "s1"
|
|
62
|
+
assert key_from_context(None, None) == ""
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def test_store_refresh_and_assign():
|
|
66
|
+
cfg = {
|
|
67
|
+
"version": "v1",
|
|
68
|
+
"experiments": [
|
|
69
|
+
{
|
|
70
|
+
"experiment_id": "exp-1",
|
|
71
|
+
"experiment_key": "greeting",
|
|
72
|
+
"status": "running",
|
|
73
|
+
"traffic_allocation": 100,
|
|
74
|
+
"variants": _vars(),
|
|
75
|
+
}
|
|
76
|
+
],
|
|
77
|
+
}
|
|
78
|
+
calls = {"n": 0}
|
|
79
|
+
|
|
80
|
+
def fetcher(etag):
|
|
81
|
+
calls["n"] += 1
|
|
82
|
+
# first call returns config; subsequent (same version) -> None (304)
|
|
83
|
+
return ("v1", cfg) if etag != "v1" else None
|
|
84
|
+
|
|
85
|
+
store = PromptConfigStore(fetcher)
|
|
86
|
+
assert store.refresh() is True # initial load
|
|
87
|
+
assert store.version == "v1"
|
|
88
|
+
assert store.refresh() is False # 304 no-op
|
|
89
|
+
v = store.assign("greeting", user_id="user-42")
|
|
90
|
+
assert v is not None and v["variant_id"] in ("a", "b")
|
|
91
|
+
assert store.assign("unknown-exp", user_id="x") is None # not in snapshot
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _run_all():
|
|
95
|
+
fns = [g for n, g in sorted(globals().items()) if n.startswith("test_") and callable(g)]
|
|
96
|
+
for fn in fns:
|
|
97
|
+
fn()
|
|
98
|
+
print(f" PASS {fn.__name__}")
|
|
99
|
+
print(f"\nALL {len(fns)} TESTS PASSED")
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
if __name__ == "__main__":
|
|
103
|
+
_run_all()
|
|
@@ -1,202 +0,0 @@
|
|
|
1
|
-
"""LangGraph auto-instrumentation for variА/Bly (Task 42.4).
|
|
2
|
-
|
|
3
|
-
Two entry points:
|
|
4
|
-
|
|
5
|
-
1. ``instrument(graph, session_id_key=...)`` — zero-touch. Call it once after
|
|
6
|
-
you've added nodes/edges (before ``.compile()``). Wraps every node so each
|
|
7
|
-
execution logs to variА/Bly: ``agent_name`` = node name, ``is_aggregator``
|
|
8
|
-
on the graph's terminal node(s), ``session_id`` pulled from graph state.
|
|
9
|
-
|
|
10
|
-
2. ``@observe_node(...)`` — an explicit per-node decorator. Version-independent
|
|
11
|
-
and the most robust option if ``instrument`` can't wrap your graph's node
|
|
12
|
-
representation.
|
|
13
|
-
|
|
14
|
-
Both are best-effort: instrumentation failures degrade to a stderr warning and
|
|
15
|
-
NEVER raise into your graph. Targets langgraph >= 0.1; call ``instrument`` after
|
|
16
|
-
``add_node`` / ``add_edge`` and before ``compile()``.
|
|
17
|
-
"""
|
|
18
|
-
|
|
19
|
-
from __future__ import annotations
|
|
20
|
-
|
|
21
|
-
import functools
|
|
22
|
-
from typing import Any, Callable, Iterable, Optional
|
|
23
|
-
|
|
24
|
-
from variably import observe as _observe
|
|
25
|
-
from variably.integrations._common import (
|
|
26
|
-
extract_query,
|
|
27
|
-
extract_response_text,
|
|
28
|
-
extract_session_id,
|
|
29
|
-
warn,
|
|
30
|
-
)
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
def _log_node(
|
|
34
|
-
name: str,
|
|
35
|
-
state: Any,
|
|
36
|
-
update: Any,
|
|
37
|
-
*,
|
|
38
|
-
is_aggregator: bool,
|
|
39
|
-
session_id_key: str,
|
|
40
|
-
response_extractor: Optional[Callable[[Any], str]],
|
|
41
|
-
) -> None:
|
|
42
|
-
"""Log one node execution to variА/Bly. Never raises."""
|
|
43
|
-
try:
|
|
44
|
-
resp = (response_extractor or extract_response_text)(update)
|
|
45
|
-
sid = extract_session_id(state, session_id_key)
|
|
46
|
-
# is_aggregator requires a session_id (SDK write-time rule); only claim
|
|
47
|
-
# it when we actually have one, else it degrades to a normal step.
|
|
48
|
-
agg = bool(is_aggregator and sid)
|
|
49
|
-
_observe(
|
|
50
|
-
prompt=extract_query(state) or f"[node:{name}]",
|
|
51
|
-
response=resp or "",
|
|
52
|
-
session_id=sid,
|
|
53
|
-
agent_name=name,
|
|
54
|
-
is_aggregator=agg,
|
|
55
|
-
)
|
|
56
|
-
except Exception as e: # pragma: no cover - defensive
|
|
57
|
-
warn(f"node '{name}' instrumentation failed: {e}")
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
def observe_node(
|
|
61
|
-
*,
|
|
62
|
-
agent_name: Optional[str] = None,
|
|
63
|
-
is_aggregator: bool = False,
|
|
64
|
-
session_id_key: str = "session_id",
|
|
65
|
-
response_extractor: Optional[Callable[[Any], str]] = None,
|
|
66
|
-
) -> Callable[[Callable], Callable]:
|
|
67
|
-
"""Decorator for a LangGraph node function. Logs the node's output to
|
|
68
|
-
variА/Bly after it runs. ``agent_name`` defaults to the function name."""
|
|
69
|
-
|
|
70
|
-
def decorate(fn: Callable) -> Callable:
|
|
71
|
-
name = agent_name or getattr(fn, "__name__", "node")
|
|
72
|
-
|
|
73
|
-
@functools.wraps(fn)
|
|
74
|
-
def wrapper(state, *args, **kwargs):
|
|
75
|
-
update = fn(state, *args, **kwargs)
|
|
76
|
-
_log_node(
|
|
77
|
-
name, state, update,
|
|
78
|
-
is_aggregator=is_aggregator,
|
|
79
|
-
session_id_key=session_id_key,
|
|
80
|
-
response_extractor=response_extractor,
|
|
81
|
-
)
|
|
82
|
-
return update
|
|
83
|
-
|
|
84
|
-
return wrapper
|
|
85
|
-
|
|
86
|
-
return decorate
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
def _detect_terminal_nodes(graph) -> set:
|
|
90
|
-
"""Terminal nodes = those with an edge to END, plus any with no outgoing
|
|
91
|
-
edge. Best-effort across langgraph versions."""
|
|
92
|
-
terminals: set = set()
|
|
93
|
-
try:
|
|
94
|
-
try:
|
|
95
|
-
from langgraph.graph import END
|
|
96
|
-
except Exception:
|
|
97
|
-
END = "__end__"
|
|
98
|
-
|
|
99
|
-
node_names = set(getattr(graph, "nodes", {}) or {})
|
|
100
|
-
edges = getattr(graph, "edges", None) or set()
|
|
101
|
-
has_outgoing: set = set()
|
|
102
|
-
for edge in edges:
|
|
103
|
-
try:
|
|
104
|
-
u, v = edge
|
|
105
|
-
except Exception:
|
|
106
|
-
continue
|
|
107
|
-
has_outgoing.add(u)
|
|
108
|
-
if v == END or v == "__end__":
|
|
109
|
-
terminals.add(u)
|
|
110
|
-
# nodes with no outgoing plain edge (may route via conditional branches,
|
|
111
|
-
# but this is a reasonable default when no END edge is declared)
|
|
112
|
-
terminals |= (node_names - has_outgoing)
|
|
113
|
-
except Exception as e: # pragma: no cover - defensive
|
|
114
|
-
warn(f"terminal-node detection failed: {e}")
|
|
115
|
-
return terminals
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
def _wrap_callable(name, fn, is_agg, session_id_key, response_extractor):
|
|
119
|
-
@functools.wraps(fn)
|
|
120
|
-
def wrapped(state, *args, **kwargs):
|
|
121
|
-
update = fn(state, *args, **kwargs)
|
|
122
|
-
_log_node(
|
|
123
|
-
name, state, update,
|
|
124
|
-
is_aggregator=is_agg,
|
|
125
|
-
session_id_key=session_id_key,
|
|
126
|
-
response_extractor=response_extractor,
|
|
127
|
-
)
|
|
128
|
-
return update
|
|
129
|
-
return wrapped
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
def instrument(
|
|
133
|
-
graph,
|
|
134
|
-
*,
|
|
135
|
-
session_id_key: str = "session_id",
|
|
136
|
-
aggregator_nodes: Optional[Iterable[str]] = None,
|
|
137
|
-
response_extractor: Optional[Callable[[Any], str]] = None,
|
|
138
|
-
):
|
|
139
|
-
"""Instrument every node in a LangGraph ``StateGraph`` in place.
|
|
140
|
-
|
|
141
|
-
Args:
|
|
142
|
-
graph: an uncompiled ``StateGraph``.
|
|
143
|
-
session_id_key: state key holding the session/thread id.
|
|
144
|
-
aggregator_nodes: override terminal-node auto-detection with an explicit set.
|
|
145
|
-
response_extractor: custom fn to pull response text from a node's return.
|
|
146
|
-
|
|
147
|
-
Returns the same graph (for chaining). Best-effort; never raises.
|
|
148
|
-
"""
|
|
149
|
-
nodes = getattr(graph, "nodes", None)
|
|
150
|
-
if not nodes:
|
|
151
|
-
warn("graph has no .nodes — nothing to instrument (pass an uncompiled StateGraph)")
|
|
152
|
-
return graph
|
|
153
|
-
|
|
154
|
-
terminals = set(aggregator_nodes) if aggregator_nodes is not None else _detect_terminal_nodes(graph)
|
|
155
|
-
|
|
156
|
-
for name in list(nodes.keys()):
|
|
157
|
-
is_agg = name in terminals
|
|
158
|
-
try:
|
|
159
|
-
spec = nodes[name]
|
|
160
|
-
# langgraph StateNodeSpec exposes the node fn on .runnable; older /
|
|
161
|
-
# simpler representations store the callable directly.
|
|
162
|
-
runnable = getattr(spec, "runnable", None)
|
|
163
|
-
if runnable is not None:
|
|
164
|
-
inner = getattr(runnable, "func", None) or runnable
|
|
165
|
-
if not callable(inner):
|
|
166
|
-
warn(f"node '{name}': runnable is not callable; skipped")
|
|
167
|
-
continue
|
|
168
|
-
wrapped = _wrap_callable(name, inner, is_agg, session_id_key, response_extractor)
|
|
169
|
-
if hasattr(runnable, "func"):
|
|
170
|
-
try:
|
|
171
|
-
runnable.func = wrapped # RunnableCallable/Lambda
|
|
172
|
-
continue
|
|
173
|
-
except Exception:
|
|
174
|
-
pass
|
|
175
|
-
if _replace_spec_runnable(nodes, name, spec, wrapped):
|
|
176
|
-
continue
|
|
177
|
-
warn(f"node '{name}': could not rewrap runnable (unsupported spec)")
|
|
178
|
-
elif callable(spec):
|
|
179
|
-
nodes[name] = _wrap_callable(name, spec, is_agg, session_id_key, response_extractor)
|
|
180
|
-
else:
|
|
181
|
-
warn(f"node '{name}': unrecognised spec type {type(spec).__name__}; skipped")
|
|
182
|
-
except Exception as e: # pragma: no cover - defensive
|
|
183
|
-
warn(f"failed to instrument node '{name}': {e}")
|
|
184
|
-
|
|
185
|
-
return graph
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
def _replace_spec_runnable(nodes, name, spec, wrapped) -> bool:
|
|
189
|
-
"""Try to put `wrapped` back onto the node spec across spec representations."""
|
|
190
|
-
# settable attribute
|
|
191
|
-
try:
|
|
192
|
-
spec.runnable = wrapped # type: ignore[attr-defined]
|
|
193
|
-
return True
|
|
194
|
-
except Exception:
|
|
195
|
-
pass
|
|
196
|
-
# namedtuple-style: rebuild via _replace
|
|
197
|
-
try:
|
|
198
|
-
nodes[name] = spec._replace(runnable=wrapped) # type: ignore[attr-defined]
|
|
199
|
-
return True
|
|
200
|
-
except Exception:
|
|
201
|
-
pass
|
|
202
|
-
return False
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|