variably-sdk 2.11.0__tar.gz → 2.12.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (28) hide show
  1. {variably_sdk-2.11.0 → variably_sdk-2.12.0}/PKG-INFO +1 -1
  2. {variably_sdk-2.11.0 → variably_sdk-2.12.0}/pyproject.toml +1 -1
  3. variably_sdk-2.12.0/src/variably/assignment.py +65 -0
  4. {variably_sdk-2.11.0 → variably_sdk-2.12.0}/src/variably/client.py +68 -0
  5. {variably_sdk-2.11.0 → variably_sdk-2.12.0}/src/variably/integrations/_common.py +16 -1
  6. variably_sdk-2.12.0/src/variably/integrations/langgraph.py +324 -0
  7. {variably_sdk-2.11.0 → variably_sdk-2.12.0}/src/variably/observe.py +4 -0
  8. variably_sdk-2.12.0/src/variably/prompt_config.py +115 -0
  9. {variably_sdk-2.11.0 → variably_sdk-2.12.0}/src/variably_sdk.egg-info/PKG-INFO +1 -1
  10. {variably_sdk-2.11.0 → variably_sdk-2.12.0}/src/variably_sdk.egg-info/SOURCES.txt +3 -0
  11. variably_sdk-2.12.0/tests/test_prompt_assignment.py +103 -0
  12. variably_sdk-2.11.0/src/variably/integrations/langgraph.py +0 -202
  13. {variably_sdk-2.11.0 → variably_sdk-2.12.0}/README.md +0 -0
  14. {variably_sdk-2.11.0 → variably_sdk-2.12.0}/setup.cfg +0 -0
  15. {variably_sdk-2.11.0 → variably_sdk-2.12.0}/src/variably/__init__.py +0 -0
  16. {variably_sdk-2.11.0 → variably_sdk-2.12.0}/src/variably/cache.py +0 -0
  17. {variably_sdk-2.11.0 → variably_sdk-2.12.0}/src/variably/config.py +0 -0
  18. {variably_sdk-2.11.0 → variably_sdk-2.12.0}/src/variably/errors.py +0 -0
  19. {variably_sdk-2.11.0 → variably_sdk-2.12.0}/src/variably/http_client.py +0 -0
  20. {variably_sdk-2.11.0 → variably_sdk-2.12.0}/src/variably/integrations/__init__.py +0 -0
  21. {variably_sdk-2.11.0 → variably_sdk-2.12.0}/src/variably/logger.py +0 -0
  22. {variably_sdk-2.11.0 → variably_sdk-2.12.0}/src/variably/types.py +0 -0
  23. {variably_sdk-2.11.0 → variably_sdk-2.12.0}/src/variably/version.py +0 -0
  24. {variably_sdk-2.11.0 → variably_sdk-2.12.0}/src/variably_sdk.egg-info/dependency_links.txt +0 -0
  25. {variably_sdk-2.11.0 → variably_sdk-2.12.0}/src/variably_sdk.egg-info/requires.txt +0 -0
  26. {variably_sdk-2.11.0 → variably_sdk-2.12.0}/src/variably_sdk.egg-info/top_level.txt +0 -0
  27. {variably_sdk-2.11.0 → variably_sdk-2.12.0}/tests/test_integrations_langgraph.py +0 -0
  28. {variably_sdk-2.11.0 → variably_sdk-2.12.0}/tests/test_streaming.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: variably-sdk
3
- Version: 2.11.0
3
+ Version: 2.12.0
4
4
  Summary: Official Python SDK for Variably feature flags, LLM experimentation, and prompt optimization platform
5
5
  Author-email: Variably <support@variably.com>
6
6
  License: MIT
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "variably-sdk"
7
- version = "2.11.0"
7
+ version = "2.12.0"
8
8
  description = "Official Python SDK for Variably feature flags, LLM experimentation, and prompt optimization platform"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.7"
@@ -0,0 +1,65 @@
1
+ """Deterministic, sticky variant assignment (#54).
2
+
3
+ Mirrors the Go ``internal/assignment`` package **byte-for-byte** — same FNV-1a
4
+ hash, same bucketing, same weighted walk — so the SDK assigns the *identical*
5
+ variant the server would, with no per-event call. Same (variants, experiment_id,
6
+ key) -> same variant, always.
7
+ """
8
+ from typing import Any, Dict, List, Optional
9
+
10
+ _FNV_OFFSET = 2166136261
11
+ _FNV_PRIME = 16777619
12
+ _MASK32 = 0xFFFFFFFF
13
+
14
+
15
+ def bucket(salt: str, key: str) -> float:
16
+ """Stable value in [0, 100) for (salt, key). FNV-1a 32-bit over
17
+ ``f"{salt}:{key}"`` — identical output to the Go implementation."""
18
+ h = _FNV_OFFSET
19
+ for b in (salt + ":" + key).encode("utf-8"):
20
+ h ^= b
21
+ h = (h * _FNV_PRIME) & _MASK32
22
+ return (h % 10000) / 100.0
23
+
24
+
25
+ def key_from_context(
26
+ user_id: Optional[str] = None, session_id: Optional[str] = None
27
+ ) -> str:
28
+ """Bucketing key — prefer user_id, then session_id. Empty when neither."""
29
+ if user_id:
30
+ return user_id
31
+ if session_id:
32
+ return session_id
33
+ return ""
34
+
35
+
36
+ def select_variant(
37
+ variants: List[Dict[str, Any]], experiment_id: str, key: str
38
+ ) -> Optional[Dict[str, Any]]:
39
+ """Deterministic, sticky, weighted assignment. Mirrors Go SelectVariant.
40
+
41
+ ``variants``: dicts with at least ``weight`` and ``is_control`` (as returned
42
+ by the /prompts/config endpoint). Empty key (no user/session id) -> control
43
+ (or first) deterministically, never random.
44
+ """
45
+ if not variants:
46
+ return None
47
+ if not key:
48
+ for v in variants:
49
+ if v.get("is_control"):
50
+ return v
51
+ return variants[0]
52
+
53
+ total = sum(
54
+ float(v.get("weight", 0)) for v in variants if float(v.get("weight", 0)) > 0
55
+ )
56
+ b = bucket(experiment_id, key) # [0, 100)
57
+ cumulative = 0.0
58
+ n = len(variants)
59
+ for v in variants:
60
+ w = float(v.get("weight", 0))
61
+ w = (w / total * 100.0) if total > 0 else (100.0 / n)
62
+ cumulative += w
63
+ if b < cumulative:
64
+ return v
65
+ return variants[-1]
@@ -10,6 +10,7 @@ from .config import VariablyConfig
10
10
  from .errors import ConfigurationError, VariablyError
11
11
  from .http_client import HTTPClient
12
12
  from .logger import VariablyLogger, create_logger
13
+ from .prompt_config import PromptConfigStore, http_config_fetcher
13
14
  from .types import (
14
15
  EvaluateResponse,
15
16
  Event,
@@ -49,6 +50,10 @@ class VariablyClient:
49
50
 
50
51
  # Initialize HTTP client
51
52
  self.http_client = HTTPClient(config, self.logger, self.metrics)
53
+
54
+ # Lazily-started prompt config store for local (in-process) variant
55
+ # assignment — bootstrapped + polled on first get_variant_local() call.
56
+ self._prompt_config_store: Optional[PromptConfigStore] = None
52
57
 
53
58
  self.logger.info(
54
59
  "Variably client initialized",
@@ -486,6 +491,66 @@ class VariablyClient:
486
491
  is_control=variant.get("is_control", variant.get("isControl", False)),
487
492
  )
488
493
 
494
+ def get_variant_local(
495
+ self,
496
+ experiment_key: str,
497
+ user_context: UserContext,
498
+ poll_interval_seconds: float = 45.0,
499
+ ) -> PromptVariant:
500
+ """Assign a prompt variant **in-process** from a cached, background-polled
501
+ config snapshot — no per-call DB query on the server (#54.3).
502
+
503
+ Deterministic and sticky: the same (experiment_key, user_id/session_id)
504
+ always maps to the same variant, using the exact hash the server would.
505
+ The config is bootstrapped on first call and refreshed every
506
+ ``poll_interval_seconds`` with ETag (a 304 costs one cheap round trip).
507
+
508
+ Falls back to the server ``get_variant`` if the experiment isn't in the
509
+ snapshot yet (e.g. created between polls), so a brand-new experiment is
510
+ never silently missed.
511
+ """
512
+ if not experiment_key or not isinstance(experiment_key, str):
513
+ raise ConfigurationError(
514
+ "Experiment key must be a non-empty string", "experiment_key"
515
+ )
516
+ key = user_context.get("user_id") or user_context.get("session_id")
517
+ if not key:
518
+ raise ConfigurationError(
519
+ "User context must include user_id or session_id",
520
+ "user_context.user_id",
521
+ )
522
+
523
+ if self._prompt_config_store is None:
524
+ store = PromptConfigStore(
525
+ http_config_fetcher(self.config.base_url, self.config.api_key)
526
+ )
527
+ store.start_polling(poll_interval_seconds)
528
+ self._prompt_config_store = store
529
+
530
+ variant = self._prompt_config_store.assign(
531
+ experiment_key,
532
+ user_id=user_context.get("user_id"),
533
+ session_id=user_context.get("session_id"),
534
+ )
535
+ if variant is None:
536
+ # Not in snapshot (new/just-changed experiment) — authoritative fallback.
537
+ return self.get_variant(experiment_key, user_context)
538
+
539
+ # Extract model/provider from parameters.llmConfig, mirroring get_variant.
540
+ params = variant.get("parameters", {}) or {}
541
+ llm_config = params.get("llmConfig", params)
542
+ return PromptVariant(
543
+ experiment_id=variant.get("experiment_id", ""),
544
+ experiment_key=experiment_key,
545
+ variant_id=variant.get("variant_id", ""),
546
+ variant_key=variant.get("name", variant.get("variant_key", "")),
547
+ prompt_template=variant.get("prompt_template", ""),
548
+ model=llm_config.get("model", ""),
549
+ provider=llm_config.get("provider", ""),
550
+ parameters=params,
551
+ is_control=variant.get("is_control", False),
552
+ )
553
+
489
554
  def evaluate_prompt(
490
555
  self,
491
556
  experiment_key: str,
@@ -683,6 +748,7 @@ class VariablyClient:
683
748
  is_aggregator: bool = False, # True on the final step that aggregates the trace
684
749
  tools_called: Optional[List[Dict[str, Any]]] = None, # [{"name","arguments","output"}] actually called
685
750
  expected_tools: Optional[List[Dict[str, Any]]] = None, # golden tools for deterministic trajectory scoring
751
+ allowed_tools: Optional[List[str]] = None, # permitted tool names; calls outside this list flag as unauthorized
686
752
  # User & organization context
687
753
  user_id: Optional[str] = None, # End-user ID for per-user analytics
688
754
  tags: Optional[List[str]] = None, # Grouping labels, e.g. ["production", "rag"]
@@ -798,6 +864,8 @@ class VariablyClient:
798
864
  data["tools_called"] = tools_called
799
865
  if expected_tools:
800
866
  data["expected_tools"] = expected_tools
867
+ if allowed_tools:
868
+ data["allowed_tools"] = allowed_tools
801
869
  if provider:
802
870
  data["provider"] = provider
803
871
  if model:
@@ -17,6 +17,9 @@ from typing import Any, Optional
17
17
  _RESPONSE_KEYS = ("response", "output", "answer", "result", "text", "content")
18
18
  # Keys we look for when extracting the user query/prompt from graph state.
19
19
  _QUERY_KEYS = ("query", "question", "input", "prompt", "user_input")
20
+ # Frameworks commonly nest config/inputs under one of these container keys
21
+ # (e.g. gpt-researcher's state["task"]["query"]). Checked one level deep.
22
+ _QUERY_CONTAINERS = ("task", "input", "config", "configurable", "state", "metadata")
20
23
 
21
24
 
22
25
  def warn(msg: str) -> None:
@@ -68,13 +71,25 @@ def extract_response_text(update: Any) -> str:
68
71
 
69
72
 
70
73
  def extract_query(state: Any) -> str:
71
- """Best-effort extraction of the user query/prompt from graph state."""
74
+ """Best-effort extraction of the user query/prompt from graph state.
75
+
76
+ Checks top-level keys first, then one level into common container dicts
77
+ (``task``/``input``/``config``/…) so frameworks that nest the query — e.g.
78
+ gpt-researcher's ``state["task"]["query"]`` — are handled without a custom
79
+ extractor. Falls back to "" (caller then uses a ``[node:name]`` placeholder)."""
72
80
  try:
73
81
  get = state.get if isinstance(state, dict) else (lambda k, d=None: getattr(state, k, d))
74
82
  for k in _QUERY_KEYS:
75
83
  v = get(k)
76
84
  if isinstance(v, str) and v.strip():
77
85
  return v
86
+ for container in _QUERY_CONTAINERS:
87
+ c = get(container)
88
+ if isinstance(c, dict):
89
+ for k in _QUERY_KEYS:
90
+ v = c.get(k)
91
+ if isinstance(v, str) and v.strip():
92
+ return v
78
93
  except Exception:
79
94
  pass
80
95
  return ""
@@ -0,0 +1,324 @@
1
+ """LangGraph auto-instrumentation for variА/Bly (Task 42.4).
2
+
3
+ Two entry points:
4
+
5
+ 1. ``instrument(graph, session_id_key=...)`` — zero-touch. Call it once after
6
+ you've added nodes/edges (before ``.compile()``). Wraps every node so each
7
+ execution logs to variА/Bly: ``agent_name`` = node name, ``is_aggregator``
8
+ on the graph's terminal node(s), ``session_id`` pulled from graph state.
9
+
10
+ 2. ``@observe_node(...)`` — an explicit per-node decorator. Version-independent
11
+ and the most robust option if ``instrument`` can't wrap your graph's node
12
+ representation.
13
+
14
+ Both are best-effort: instrumentation failures degrade to a stderr warning and
15
+ NEVER raise into your graph. Targets langgraph >= 0.1; call ``instrument`` after
16
+ ``add_node`` / ``add_edge`` and before ``compile()``.
17
+ """
18
+
19
+ from __future__ import annotations
20
+
21
+ import asyncio
22
+ import functools
23
+ from typing import Any, Callable, Iterable, Optional
24
+
25
+ from variably import observe as _observe
26
+ from variably.integrations._common import (
27
+ extract_query,
28
+ extract_response_text,
29
+ extract_session_id,
30
+ warn,
31
+ )
32
+
33
+
34
+ # Node names we've already warned about (no response text) — warn once per node
35
+ # per process so the message is visible without spamming every run.
36
+ _warned_no_response: set = set()
37
+
38
+
39
+ def _shape(value: Any) -> str:
40
+ """Describe a value's STRUCTURE (type + dict keys) without exposing its
41
+ contents — safe to print/share for diagnostics."""
42
+ try:
43
+ if isinstance(value, dict):
44
+ keys = [str(k) for k in list(value.keys())[:25]]
45
+ more = "…" if len(value) > 25 else ""
46
+ return f"dict(keys=[{', '.join(keys)}{more}])"
47
+ if isinstance(value, (list, tuple)):
48
+ return f"{type(value).__name__}(len={len(value)})"
49
+ return type(value).__name__
50
+ except Exception:
51
+ return "<unknown>"
52
+
53
+
54
+ def _log_node(
55
+ name: str,
56
+ state: Any,
57
+ update: Any,
58
+ *,
59
+ is_aggregator: bool,
60
+ session_id_key: str,
61
+ response_extractor: Optional[Callable[[Any], str]],
62
+ context_extractor: Optional[Callable[[Any, Any], Any]] = None,
63
+ tools_extractor: Optional[Callable[[Any, Any], Any]] = None,
64
+ ) -> None:
65
+ """Log one node execution to variА/Bly. Never raises."""
66
+ try:
67
+ resp = (response_extractor or extract_response_text)(update)
68
+ # No response text → do NOT fabricate one (a serialized dict would be scored
69
+ # as meaningless prose) and do NOT silently drop it. Warn the developer once
70
+ # per node (visible on stderr, non-blocking) so they notice and either add a
71
+ # response_extractor= or accept that this node has no scorable output.
72
+ if not (resp and resp.strip()):
73
+ if name not in _warned_no_response:
74
+ _warned_no_response.add(name)
75
+ warn(
76
+ f"node '{name}' produced no response text — this step was NOT scored.\n"
77
+ f" node = {name!r}\n"
78
+ f" returned = {_shape(update)}\n"
79
+ f" extractor = {'custom response_extractor' if response_extractor else 'default (extract_response_text)'}\n"
80
+ f" aggregator = {is_aggregator}\n"
81
+ f" Fix: pass response_extractor= to instrument() mapping this node's output to a "
82
+ f"string, e.g. response_extractor=lambda upd: upd.get('report', '').\n"
83
+ f" Only the structure above (type + keys) is shown — never your data — so this "
84
+ f"line is safe to share with variА/Bly support if a step you expected isn't scored."
85
+ )
86
+ return
87
+ sid = extract_session_id(state, session_id_key)
88
+ # is_aggregator requires a session_id (SDK write-time rule); only claim
89
+ # it when we actually have one, else it degrades to a normal step.
90
+ agg = bool(is_aggregator and sid)
91
+ # Retrieved context enables grounding/faithfulness scoring. Without it,
92
+ # only the non-grounding dimensions are scored. Best-effort; never fatal.
93
+ refs = None
94
+ if context_extractor is not None:
95
+ try:
96
+ refs = context_extractor(state, update)
97
+ except Exception:
98
+ refs = None
99
+ # Tool calls enable the trajectory / tool-correctness / order-adherence
100
+ # metrics. Extractor returns {"tools_called": [...], "expected_tools": [...]}.
101
+ tools_called = expected_tools = None
102
+ if tools_extractor is not None:
103
+ try:
104
+ t = tools_extractor(state, update) or {}
105
+ tools_called = t.get("tools_called")
106
+ expected_tools = t.get("expected_tools")
107
+ except Exception:
108
+ tools_called = expected_tools = None
109
+ _observe(
110
+ prompt=extract_query(state) or f"[node:{name}]",
111
+ response=resp or "",
112
+ session_id=sid,
113
+ agent_name=name,
114
+ is_aggregator=agg,
115
+ reference_materials=refs,
116
+ tools_called=tools_called,
117
+ expected_tools=expected_tools,
118
+ )
119
+ except Exception as e: # pragma: no cover - defensive
120
+ warn(f"node '{name}' instrumentation failed: {e}")
121
+
122
+
123
+ def observe_node(
124
+ *,
125
+ agent_name: Optional[str] = None,
126
+ is_aggregator: bool = False,
127
+ session_id_key: str = "session_id",
128
+ response_extractor: Optional[Callable[[Any], str]] = None,
129
+ ) -> Callable[[Callable], Callable]:
130
+ """Decorator for a LangGraph node function. Logs the node's output to
131
+ variА/Bly after it runs. ``agent_name`` defaults to the function name."""
132
+
133
+ def decorate(fn: Callable) -> Callable:
134
+ name = agent_name or getattr(fn, "__name__", "node")
135
+
136
+ @functools.wraps(fn)
137
+ def wrapper(state, *args, **kwargs):
138
+ update = fn(state, *args, **kwargs)
139
+ _log_node(
140
+ name, state, update,
141
+ is_aggregator=is_aggregator,
142
+ session_id_key=session_id_key,
143
+ response_extractor=response_extractor,
144
+ )
145
+ return update
146
+
147
+ return wrapper
148
+
149
+ return decorate
150
+
151
+
152
+ def _detect_terminal_nodes(graph) -> set:
153
+ """Terminal nodes = those with an edge to END, plus any with no outgoing
154
+ edge. Best-effort across langgraph versions."""
155
+ terminals: set = set()
156
+ try:
157
+ try:
158
+ from langgraph.graph import END
159
+ except Exception:
160
+ END = "__end__"
161
+
162
+ node_names = set(getattr(graph, "nodes", {}) or {})
163
+ edges = getattr(graph, "edges", None) or set()
164
+ has_outgoing: set = set()
165
+ for edge in edges:
166
+ try:
167
+ u, v = edge
168
+ except Exception:
169
+ continue
170
+ has_outgoing.add(u)
171
+ if v == END or v == "__end__":
172
+ terminals.add(u)
173
+ # nodes with no outgoing plain edge (may route via conditional branches,
174
+ # but this is a reasonable default when no END edge is declared)
175
+ terminals |= (node_names - has_outgoing)
176
+ except Exception as e: # pragma: no cover - defensive
177
+ warn(f"terminal-node detection failed: {e}")
178
+ return terminals
179
+
180
+
181
+ def _wrap_callable(name, fn, is_agg, session_id_key, response_extractor, context_extractor=None, tools_extractor=None):
182
+ @functools.wraps(fn)
183
+ def wrapped(state, *args, **kwargs):
184
+ update = fn(state, *args, **kwargs)
185
+ _log_node(
186
+ name, state, update,
187
+ is_aggregator=is_agg,
188
+ session_id_key=session_id_key,
189
+ response_extractor=response_extractor,
190
+ context_extractor=context_extractor,
191
+ tools_extractor=tools_extractor,
192
+ )
193
+ return update
194
+ return wrapped
195
+
196
+
197
+ def _wrap_callable_async(name, fn, is_agg, session_id_key, response_extractor, context_extractor=None, tools_extractor=None):
198
+ """Async counterpart of _wrap_callable. Most real LangGraph nodes are
199
+ ``async def`` (invoked via ``ainvoke``), so their fn lives on the runnable's
200
+ ``.afunc`` — this wraps that path. ``_observe`` is non-blocking, so logging
201
+ from inside the event loop is safe."""
202
+ @functools.wraps(fn)
203
+ async def wrapped(state, *args, **kwargs):
204
+ update = await fn(state, *args, **kwargs)
205
+ _log_node(
206
+ name, state, update,
207
+ is_aggregator=is_agg,
208
+ session_id_key=session_id_key,
209
+ response_extractor=response_extractor,
210
+ context_extractor=context_extractor,
211
+ tools_extractor=tools_extractor,
212
+ )
213
+ return update
214
+ return wrapped
215
+
216
+
217
+ def instrument(
218
+ graph,
219
+ *,
220
+ session_id_key: str = "session_id",
221
+ aggregator_nodes: Optional[Iterable[str]] = None,
222
+ response_extractor: Optional[Callable[[Any], str]] = None,
223
+ context_extractor: Optional[Callable[[Any, Any], Any]] = None,
224
+ tools_extractor: Optional[Callable[[Any, Any], Any]] = None,
225
+ ):
226
+ """Instrument every node in a LangGraph ``StateGraph`` in place.
227
+
228
+ Args:
229
+ graph: an uncompiled ``StateGraph``.
230
+ session_id_key: state key holding the session/thread id.
231
+ aggregator_nodes: override terminal-node auto-detection with an explicit set.
232
+ response_extractor: custom fn to pull response text from a node's return.
233
+ context_extractor: custom fn ``(state, update) -> list[{"id","content"}]``
234
+ returning the retrieved reference materials for grounding/faithfulness
235
+ scoring. Return None/[] when the node has no context. Without it,
236
+ grounding cannot be scored (only the non-grounding dimensions).
237
+ tools_extractor: custom fn ``(state, update) -> {"tools_called": [...],
238
+ "expected_tools": [...]}`` (each tool a ``{"name","arguments","output"}``
239
+ dict) enabling the trajectory / tool-correctness / order-adherence
240
+ metrics. Return None when the node makes no tool calls.
241
+
242
+ Returns the same graph (for chaining). Best-effort; never raises.
243
+ """
244
+ nodes = getattr(graph, "nodes", None)
245
+ if not nodes:
246
+ warn("graph has no .nodes — nothing to instrument (pass an uncompiled StateGraph)")
247
+ return graph
248
+
249
+ terminals = set(aggregator_nodes) if aggregator_nodes is not None else _detect_terminal_nodes(graph)
250
+
251
+ for name in list(nodes.keys()):
252
+ is_agg = name in terminals
253
+ try:
254
+ spec = nodes[name]
255
+ # langgraph StateNodeSpec exposes the node fn on .runnable; older /
256
+ # simpler representations store the callable directly.
257
+ runnable = getattr(spec, "runnable", None)
258
+ if runnable is not None:
259
+ wrapped_any = False
260
+ # Sync path — RunnableCallable stores a sync fn on .func.
261
+ sync_fn = getattr(runnable, "func", None)
262
+ if callable(sync_fn):
263
+ try:
264
+ runnable.func = _wrap_callable(
265
+ name, sync_fn, is_agg, session_id_key, response_extractor,
266
+ context_extractor, tools_extractor)
267
+ wrapped_any = True
268
+ except Exception:
269
+ pass
270
+ # Async path — async nodes store the coroutine fn on .afunc, which
271
+ # is what langgraph calls under ainvoke. Missing this = no events
272
+ # for any async graph (the common case).
273
+ async_fn = getattr(runnable, "afunc", None)
274
+ if callable(async_fn):
275
+ try:
276
+ runnable.afunc = _wrap_callable_async(
277
+ name, async_fn, is_agg, session_id_key, response_extractor,
278
+ context_extractor, tools_extractor)
279
+ wrapped_any = True
280
+ except Exception:
281
+ pass
282
+ if wrapped_any:
283
+ continue
284
+ # No .func/.afunc — the runnable itself may be the callable.
285
+ if callable(runnable):
286
+ wrapped = (
287
+ _wrap_callable_async(name, runnable, is_agg, session_id_key,
288
+ response_extractor, context_extractor, tools_extractor)
289
+ if asyncio.iscoroutinefunction(runnable)
290
+ else _wrap_callable(name, runnable, is_agg, session_id_key,
291
+ response_extractor, context_extractor, tools_extractor)
292
+ )
293
+ if _replace_spec_runnable(nodes, name, spec, wrapped):
294
+ continue
295
+ warn(f"node '{name}': could not rewrap runnable (unsupported spec)")
296
+ elif callable(spec):
297
+ nodes[name] = (
298
+ _wrap_callable_async(name, spec, is_agg, session_id_key, response_extractor)
299
+ if asyncio.iscoroutinefunction(spec)
300
+ else _wrap_callable(name, spec, is_agg, session_id_key, response_extractor)
301
+ )
302
+ else:
303
+ warn(f"node '{name}': unrecognised spec type {type(spec).__name__}; skipped")
304
+ except Exception as e: # pragma: no cover - defensive
305
+ warn(f"failed to instrument node '{name}': {e}")
306
+
307
+ return graph
308
+
309
+
310
+ def _replace_spec_runnable(nodes, name, spec, wrapped) -> bool:
311
+ """Try to put `wrapped` back onto the node spec across spec representations."""
312
+ # settable attribute
313
+ try:
314
+ spec.runnable = wrapped # type: ignore[attr-defined]
315
+ return True
316
+ except Exception:
317
+ pass
318
+ # namedtuple-style: rebuild via _replace
319
+ try:
320
+ nodes[name] = spec._replace(runnable=wrapped) # type: ignore[attr-defined]
321
+ return True
322
+ except Exception:
323
+ pass
324
+ return False
@@ -184,6 +184,9 @@ def observe(
184
184
  # deterministic trajectory precision/recall + tool-correctness metrics.
185
185
  tools_called: Optional[List[Dict[str, Any]]] = None,
186
186
  expected_tools: Optional[List[Dict[str, Any]]] = None,
187
+ # Permitted-tool allowlist (names). Any call outside it flags as an
188
+ # unauthorized action — the deterministic security/adversarial signal.
189
+ allowed_tools: Optional[List[str]] = None,
187
190
 
188
191
  # --- User & organization context ---
189
192
  user_id: Optional[str] = None, # Your end-user's ID — enables per-user analytics
@@ -288,6 +291,7 @@ def observe(
288
291
  is_aggregator=is_aggregator,
289
292
  tools_called=tools_called,
290
293
  expected_tools=expected_tools,
294
+ allowed_tools=allowed_tools,
291
295
  metadata=metadata,
292
296
  kind=kind,
293
297
  )
@@ -0,0 +1,115 @@
1
+ """SDK-side prompt config store + background poll (#54.2/54.3).
2
+
3
+ Bootstraps the project's prompt-experiment config from
4
+ ``GET /internal/sdk/prompts/config`` once, refreshes it in the background with
5
+ ETag/If-None-Match (a 304 is a no-op), and assigns variants **locally** from the
6
+ cached snapshot — zero backend call per event. Deterministic + sticky via
7
+ ``assignment`` (which is byte-for-byte identical to the server).
8
+ """
9
+ import threading
10
+ from typing import Any, Callable, Dict, List, Optional, Tuple
11
+
12
+ from .assignment import key_from_context, select_variant
13
+
14
+ # A fetcher takes the current ETag/version and returns (version, config_dict) on
15
+ # change, or None when unchanged (304), errored, or nothing to update. Injectable
16
+ # so the store is testable without a live server.
17
+ Fetcher = Callable[[Optional[str]], Optional[Tuple[str, Dict[str, Any]]]]
18
+
19
+
20
+ class PromptConfigStore:
21
+ """Thread-safe snapshot of the SDK prompt config + local assignment."""
22
+
23
+ def __init__(self, fetcher: Fetcher):
24
+ self._fetcher = fetcher
25
+ self._lock = threading.Lock()
26
+ self._version: Optional[str] = None
27
+ self._experiments: Dict[str, Dict[str, Any]] = {} # experiment_key -> experiment
28
+ self._stop = threading.Event()
29
+ self._thread: Optional[threading.Thread] = None
30
+
31
+ def refresh(self) -> bool:
32
+ """Fetch config and atomically swap the snapshot on change. Returns True
33
+ if the snapshot was updated, False on 304/unchanged/error. Never raises."""
34
+ try:
35
+ result = self._fetcher(self._version)
36
+ except Exception:
37
+ return False
38
+ if result is None:
39
+ return False
40
+ version, cfg = result
41
+ index = {
42
+ e["experiment_key"]: e for e in (cfg.get("experiments") or []) if e.get("experiment_key")
43
+ }
44
+ with self._lock:
45
+ self._version = version
46
+ self._experiments = index
47
+ return True
48
+
49
+ def assign(
50
+ self,
51
+ experiment_key: str,
52
+ user_id: Optional[str] = None,
53
+ session_id: Optional[str] = None,
54
+ ) -> Optional[Dict[str, Any]]:
55
+ """Assign a variant locally from the cached snapshot. Returns the variant
56
+ dict, or None if the experiment isn't in the snapshot (not running /
57
+ unknown / config not yet loaded)."""
58
+ with self._lock:
59
+ exp = self._experiments.get(experiment_key)
60
+ if not exp:
61
+ return None
62
+ experiment_id = exp["experiment_id"]
63
+ key = key_from_context(user_id, session_id)
64
+ variant = select_variant(exp.get("variants") or [], experiment_id, key)
65
+ if variant is None:
66
+ return None
67
+ # Stamp experiment_id (lives on the experiment, not the variant) so callers
68
+ # get a self-contained variant. Copy to avoid mutating the snapshot.
69
+ return {**variant, "experiment_id": experiment_id}
70
+
71
+ def start_polling(self, interval_seconds: float = 45.0) -> None:
72
+ """Bootstrap once, then refresh every ``interval_seconds`` in a daemon
73
+ thread. Idempotent."""
74
+ if self._thread is not None:
75
+ return
76
+ self.refresh() # initial bootstrap
77
+
78
+ def _loop() -> None:
79
+ while not self._stop.wait(interval_seconds):
80
+ self.refresh()
81
+
82
+ self._thread = threading.Thread(
83
+ target=_loop, name="variably-config-poll", daemon=True
84
+ )
85
+ self._thread.start()
86
+
87
+ def stop(self) -> None:
88
+ self._stop.set()
89
+
90
+ @property
91
+ def version(self) -> Optional[str]:
92
+ return self._version
93
+
94
+
95
+ def http_config_fetcher(base_url: str, api_key: str, timeout: float = 5.0) -> Fetcher:
96
+ """Build a fetcher that GETs /internal/sdk/prompts/config with ETag. Uses
97
+ ``requests`` (already an SDK dep). Returns (version, config) on 200, None on
98
+ 304 or any error — the store keeps its last-good snapshot."""
99
+ import requests
100
+
101
+ url = base_url.rstrip("/") + "/api/v1/internal/sdk/prompts/config"
102
+
103
+ def _fetch(etag: Optional[str]) -> Optional[Tuple[str, Dict[str, Any]]]:
104
+ headers = {"Authorization": f"Bearer {api_key}"}
105
+ if etag:
106
+ headers["If-None-Match"] = '"' + etag + '"'
107
+ resp = requests.get(url, headers=headers, timeout=timeout)
108
+ if resp.status_code == 304:
109
+ return None
110
+ if resp.status_code != 200:
111
+ return None
112
+ cfg = resp.json()
113
+ return cfg.get("version", ""), cfg
114
+
115
+ return _fetch
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: variably-sdk
3
- Version: 2.11.0
3
+ Version: 2.12.0
4
4
  Summary: Official Python SDK for Variably feature flags, LLM experimentation, and prompt optimization platform
5
5
  Author-email: Variably <support@variably.com>
6
6
  License: MIT
@@ -1,6 +1,7 @@
1
1
  README.md
2
2
  pyproject.toml
3
3
  src/variably/__init__.py
4
+ src/variably/assignment.py
4
5
  src/variably/cache.py
5
6
  src/variably/client.py
6
7
  src/variably/config.py
@@ -8,6 +9,7 @@ src/variably/errors.py
8
9
  src/variably/http_client.py
9
10
  src/variably/logger.py
10
11
  src/variably/observe.py
12
+ src/variably/prompt_config.py
11
13
  src/variably/types.py
12
14
  src/variably/version.py
13
15
  src/variably/integrations/__init__.py
@@ -19,4 +21,5 @@ src/variably_sdk.egg-info/dependency_links.txt
19
21
  src/variably_sdk.egg-info/requires.txt
20
22
  src/variably_sdk.egg-info/top_level.txt
21
23
  tests/test_integrations_langgraph.py
24
+ tests/test_prompt_assignment.py
22
25
  tests/test_streaming.py
@@ -0,0 +1,103 @@
1
+ """Tests for SDK-side local assignment (#54.3) + config store (#54.2).
2
+
3
+ Run via pytest, or directly: `python tests/test_prompt_assignment.py` (the direct
4
+ runner avoids the legacy top-level `variably` package shadowing `src/variably`).
5
+ """
6
+ import os
7
+ import sys
8
+
9
+ sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "src"))
10
+
11
+ from variably.assignment import bucket, key_from_context, select_variant # noqa: E402
12
+ from variably.prompt_config import PromptConfigStore # noqa: E402
13
+
14
+ # Reference values computed by the Go internal/assignment package — the SDK MUST
15
+ # match these byte-for-byte, or server and client assign different variants.
16
+ GO_BUCKETS = {
17
+ ("exp-1", "user-42"): 41.20,
18
+ ("exp-1", "user-99"): 64.00,
19
+ ("exp-2", "user-42"): 81.89,
20
+ ("clinical", "abc-123"): 53.23,
21
+ ("e", "s"): 99.59,
22
+ }
23
+
24
+
25
+ def _vars():
26
+ return [
27
+ {"variant_id": "a", "name": "control", "is_control": True, "weight": 70},
28
+ {"variant_id": "b", "name": "treatment", "is_control": False, "weight": 30},
29
+ ]
30
+
31
+
32
+ def test_bucket_parity_with_go():
33
+ for (salt, key), expected in GO_BUCKETS.items():
34
+ got = bucket(salt, key)
35
+ assert abs(got - expected) < 0.005, f"{salt}:{key} py={got} go={expected}"
36
+
37
+
38
+ def test_sticky():
39
+ first = select_variant(_vars(), "exp-1", "user-42")
40
+ for _ in range(1000):
41
+ assert select_variant(_vars(), "exp-1", "user-42")["variant_id"] == first["variant_id"]
42
+
43
+
44
+ def test_distribution():
45
+ counts = {"a": 0, "b": 0}
46
+ n = 10000
47
+ for i in range(n):
48
+ counts[select_variant(_vars(), "exp-1", f"user-{i}")["variant_id"]] += 1
49
+ assert 65 <= counts["a"] / n * 100 <= 75
50
+ assert 25 <= counts["b"] / n * 100 <= 35
51
+
52
+
53
+ def test_keyless_returns_control():
54
+ for _ in range(50):
55
+ v = select_variant(_vars(), "exp-1", "")
56
+ assert v is not None and v["is_control"]
57
+
58
+
59
+ def test_key_from_context():
60
+ assert key_from_context("u1", "s1") == "u1"
61
+ assert key_from_context(None, "s1") == "s1"
62
+ assert key_from_context(None, None) == ""
63
+
64
+
65
+ def test_store_refresh_and_assign():
66
+ cfg = {
67
+ "version": "v1",
68
+ "experiments": [
69
+ {
70
+ "experiment_id": "exp-1",
71
+ "experiment_key": "greeting",
72
+ "status": "running",
73
+ "traffic_allocation": 100,
74
+ "variants": _vars(),
75
+ }
76
+ ],
77
+ }
78
+ calls = {"n": 0}
79
+
80
+ def fetcher(etag):
81
+ calls["n"] += 1
82
+ # first call returns config; subsequent (same version) -> None (304)
83
+ return ("v1", cfg) if etag != "v1" else None
84
+
85
+ store = PromptConfigStore(fetcher)
86
+ assert store.refresh() is True # initial load
87
+ assert store.version == "v1"
88
+ assert store.refresh() is False # 304 no-op
89
+ v = store.assign("greeting", user_id="user-42")
90
+ assert v is not None and v["variant_id"] in ("a", "b")
91
+ assert store.assign("unknown-exp", user_id="x") is None # not in snapshot
92
+
93
+
94
+ def _run_all():
95
+ fns = [g for n, g in sorted(globals().items()) if n.startswith("test_") and callable(g)]
96
+ for fn in fns:
97
+ fn()
98
+ print(f" PASS {fn.__name__}")
99
+ print(f"\nALL {len(fns)} TESTS PASSED")
100
+
101
+
102
+ if __name__ == "__main__":
103
+ _run_all()
@@ -1,202 +0,0 @@
1
- """LangGraph auto-instrumentation for variА/Bly (Task 42.4).
2
-
3
- Two entry points:
4
-
5
- 1. ``instrument(graph, session_id_key=...)`` — zero-touch. Call it once after
6
- you've added nodes/edges (before ``.compile()``). Wraps every node so each
7
- execution logs to variА/Bly: ``agent_name`` = node name, ``is_aggregator``
8
- on the graph's terminal node(s), ``session_id`` pulled from graph state.
9
-
10
- 2. ``@observe_node(...)`` — an explicit per-node decorator. Version-independent
11
- and the most robust option if ``instrument`` can't wrap your graph's node
12
- representation.
13
-
14
- Both are best-effort: instrumentation failures degrade to a stderr warning and
15
- NEVER raise into your graph. Targets langgraph >= 0.1; call ``instrument`` after
16
- ``add_node`` / ``add_edge`` and before ``compile()``.
17
- """
18
-
19
- from __future__ import annotations
20
-
21
- import functools
22
- from typing import Any, Callable, Iterable, Optional
23
-
24
- from variably import observe as _observe
25
- from variably.integrations._common import (
26
- extract_query,
27
- extract_response_text,
28
- extract_session_id,
29
- warn,
30
- )
31
-
32
-
33
- def _log_node(
34
- name: str,
35
- state: Any,
36
- update: Any,
37
- *,
38
- is_aggregator: bool,
39
- session_id_key: str,
40
- response_extractor: Optional[Callable[[Any], str]],
41
- ) -> None:
42
- """Log one node execution to variА/Bly. Never raises."""
43
- try:
44
- resp = (response_extractor or extract_response_text)(update)
45
- sid = extract_session_id(state, session_id_key)
46
- # is_aggregator requires a session_id (SDK write-time rule); only claim
47
- # it when we actually have one, else it degrades to a normal step.
48
- agg = bool(is_aggregator and sid)
49
- _observe(
50
- prompt=extract_query(state) or f"[node:{name}]",
51
- response=resp or "",
52
- session_id=sid,
53
- agent_name=name,
54
- is_aggregator=agg,
55
- )
56
- except Exception as e: # pragma: no cover - defensive
57
- warn(f"node '{name}' instrumentation failed: {e}")
58
-
59
-
60
- def observe_node(
61
- *,
62
- agent_name: Optional[str] = None,
63
- is_aggregator: bool = False,
64
- session_id_key: str = "session_id",
65
- response_extractor: Optional[Callable[[Any], str]] = None,
66
- ) -> Callable[[Callable], Callable]:
67
- """Decorator for a LangGraph node function. Logs the node's output to
68
- variА/Bly after it runs. ``agent_name`` defaults to the function name."""
69
-
70
- def decorate(fn: Callable) -> Callable:
71
- name = agent_name or getattr(fn, "__name__", "node")
72
-
73
- @functools.wraps(fn)
74
- def wrapper(state, *args, **kwargs):
75
- update = fn(state, *args, **kwargs)
76
- _log_node(
77
- name, state, update,
78
- is_aggregator=is_aggregator,
79
- session_id_key=session_id_key,
80
- response_extractor=response_extractor,
81
- )
82
- return update
83
-
84
- return wrapper
85
-
86
- return decorate
87
-
88
-
89
- def _detect_terminal_nodes(graph) -> set:
90
- """Terminal nodes = those with an edge to END, plus any with no outgoing
91
- edge. Best-effort across langgraph versions."""
92
- terminals: set = set()
93
- try:
94
- try:
95
- from langgraph.graph import END
96
- except Exception:
97
- END = "__end__"
98
-
99
- node_names = set(getattr(graph, "nodes", {}) or {})
100
- edges = getattr(graph, "edges", None) or set()
101
- has_outgoing: set = set()
102
- for edge in edges:
103
- try:
104
- u, v = edge
105
- except Exception:
106
- continue
107
- has_outgoing.add(u)
108
- if v == END or v == "__end__":
109
- terminals.add(u)
110
- # nodes with no outgoing plain edge (may route via conditional branches,
111
- # but this is a reasonable default when no END edge is declared)
112
- terminals |= (node_names - has_outgoing)
113
- except Exception as e: # pragma: no cover - defensive
114
- warn(f"terminal-node detection failed: {e}")
115
- return terminals
116
-
117
-
118
- def _wrap_callable(name, fn, is_agg, session_id_key, response_extractor):
119
- @functools.wraps(fn)
120
- def wrapped(state, *args, **kwargs):
121
- update = fn(state, *args, **kwargs)
122
- _log_node(
123
- name, state, update,
124
- is_aggregator=is_agg,
125
- session_id_key=session_id_key,
126
- response_extractor=response_extractor,
127
- )
128
- return update
129
- return wrapped
130
-
131
-
132
- def instrument(
133
- graph,
134
- *,
135
- session_id_key: str = "session_id",
136
- aggregator_nodes: Optional[Iterable[str]] = None,
137
- response_extractor: Optional[Callable[[Any], str]] = None,
138
- ):
139
- """Instrument every node in a LangGraph ``StateGraph`` in place.
140
-
141
- Args:
142
- graph: an uncompiled ``StateGraph``.
143
- session_id_key: state key holding the session/thread id.
144
- aggregator_nodes: override terminal-node auto-detection with an explicit set.
145
- response_extractor: custom fn to pull response text from a node's return.
146
-
147
- Returns the same graph (for chaining). Best-effort; never raises.
148
- """
149
- nodes = getattr(graph, "nodes", None)
150
- if not nodes:
151
- warn("graph has no .nodes — nothing to instrument (pass an uncompiled StateGraph)")
152
- return graph
153
-
154
- terminals = set(aggregator_nodes) if aggregator_nodes is not None else _detect_terminal_nodes(graph)
155
-
156
- for name in list(nodes.keys()):
157
- is_agg = name in terminals
158
- try:
159
- spec = nodes[name]
160
- # langgraph StateNodeSpec exposes the node fn on .runnable; older /
161
- # simpler representations store the callable directly.
162
- runnable = getattr(spec, "runnable", None)
163
- if runnable is not None:
164
- inner = getattr(runnable, "func", None) or runnable
165
- if not callable(inner):
166
- warn(f"node '{name}': runnable is not callable; skipped")
167
- continue
168
- wrapped = _wrap_callable(name, inner, is_agg, session_id_key, response_extractor)
169
- if hasattr(runnable, "func"):
170
- try:
171
- runnable.func = wrapped # RunnableCallable/Lambda
172
- continue
173
- except Exception:
174
- pass
175
- if _replace_spec_runnable(nodes, name, spec, wrapped):
176
- continue
177
- warn(f"node '{name}': could not rewrap runnable (unsupported spec)")
178
- elif callable(spec):
179
- nodes[name] = _wrap_callable(name, spec, is_agg, session_id_key, response_extractor)
180
- else:
181
- warn(f"node '{name}': unrecognised spec type {type(spec).__name__}; skipped")
182
- except Exception as e: # pragma: no cover - defensive
183
- warn(f"failed to instrument node '{name}': {e}")
184
-
185
- return graph
186
-
187
-
188
- def _replace_spec_runnable(nodes, name, spec, wrapped) -> bool:
189
- """Try to put `wrapped` back onto the node spec across spec representations."""
190
- # settable attribute
191
- try:
192
- spec.runnable = wrapped # type: ignore[attr-defined]
193
- return True
194
- except Exception:
195
- pass
196
- # namedtuple-style: rebuild via _replace
197
- try:
198
- nodes[name] = spec._replace(runnable=wrapped) # type: ignore[attr-defined]
199
- return True
200
- except Exception:
201
- pass
202
- return False
File without changes
File without changes