variably-sdk 2.13.0__tar.gz → 2.15.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. {variably_sdk-2.13.0 → variably_sdk-2.15.0}/PKG-INFO +5 -2
  2. {variably_sdk-2.13.0 → variably_sdk-2.15.0}/README.md +2 -1
  3. {variably_sdk-2.13.0 → variably_sdk-2.15.0}/pyproject.toml +5 -1
  4. variably_sdk-2.15.0/src/variably/_json_safe.py +92 -0
  5. {variably_sdk-2.13.0 → variably_sdk-2.15.0}/src/variably/client.py +147 -1
  6. {variably_sdk-2.13.0 → variably_sdk-2.15.0}/src/variably/http_client.py +10 -2
  7. {variably_sdk-2.13.0 → variably_sdk-2.15.0}/src/variably/integrations/adk.py +21 -0
  8. {variably_sdk-2.13.0 → variably_sdk-2.15.0}/src/variably/integrations/langgraph.py +190 -11
  9. {variably_sdk-2.13.0 → variably_sdk-2.15.0}/src/variably/integrations/otel.py +4 -0
  10. variably_sdk-2.15.0/src/variably/integrations/otel_attributes.py +54 -0
  11. {variably_sdk-2.13.0 → variably_sdk-2.15.0}/src/variably/observe.py +24 -2
  12. variably_sdk-2.15.0/src/variably/settings.py +198 -0
  13. variably_sdk-2.15.0/src/variably/sql/__init__.py +42 -0
  14. variably_sdk-2.15.0/src/variably/sql/checks.py +843 -0
  15. variably_sdk-2.15.0/src/variably/sql/execute.py +415 -0
  16. {variably_sdk-2.13.0 → variably_sdk-2.15.0}/src/variably/version.py +1 -1
  17. {variably_sdk-2.13.0 → variably_sdk-2.15.0}/src/variably_sdk.egg-info/PKG-INFO +5 -2
  18. {variably_sdk-2.13.0 → variably_sdk-2.15.0}/src/variably_sdk.egg-info/SOURCES.txt +13 -0
  19. {variably_sdk-2.13.0 → variably_sdk-2.15.0}/src/variably_sdk.egg-info/requires.txt +5 -0
  20. {variably_sdk-2.13.0 → variably_sdk-2.15.0}/tests/test_integrations_adk.py +23 -0
  21. variably_sdk-2.15.0/tests/test_integrations_langgraph.py +237 -0
  22. variably_sdk-2.15.0/tests/test_json_safe_rows.py +134 -0
  23. variably_sdk-2.15.0/tests/test_otel_handoff_attributes.py +23 -0
  24. variably_sdk-2.15.0/tests/test_scope_rows_passthrough.py +178 -0
  25. variably_sdk-2.15.0/tests/test_settings.py +150 -0
  26. variably_sdk-2.15.0/tests/test_sql_checks.py +525 -0
  27. variably_sdk-2.15.0/tests/test_sql_execute.py +272 -0
  28. variably_sdk-2.15.0/tests/test_sql_execute_safety.py +193 -0
  29. variably_sdk-2.13.0/tests/test_integrations_langgraph.py +0 -131
  30. {variably_sdk-2.13.0 → variably_sdk-2.15.0}/setup.cfg +0 -0
  31. {variably_sdk-2.13.0 → variably_sdk-2.15.0}/src/variably/__init__.py +0 -0
  32. {variably_sdk-2.13.0 → variably_sdk-2.15.0}/src/variably/assignment.py +0 -0
  33. {variably_sdk-2.13.0 → variably_sdk-2.15.0}/src/variably/cache.py +0 -0
  34. {variably_sdk-2.13.0 → variably_sdk-2.15.0}/src/variably/config.py +0 -0
  35. {variably_sdk-2.13.0 → variably_sdk-2.15.0}/src/variably/errors.py +0 -0
  36. {variably_sdk-2.13.0 → variably_sdk-2.15.0}/src/variably/integrations/__init__.py +0 -0
  37. {variably_sdk-2.13.0 → variably_sdk-2.15.0}/src/variably/integrations/_common.py +0 -0
  38. {variably_sdk-2.13.0 → variably_sdk-2.15.0}/src/variably/logger.py +0 -0
  39. {variably_sdk-2.13.0 → variably_sdk-2.15.0}/src/variably/prompt_config.py +0 -0
  40. {variably_sdk-2.13.0 → variably_sdk-2.15.0}/src/variably/types.py +0 -0
  41. {variably_sdk-2.13.0 → variably_sdk-2.15.0}/src/variably_sdk.egg-info/dependency_links.txt +0 -0
  42. {variably_sdk-2.13.0 → variably_sdk-2.15.0}/src/variably_sdk.egg-info/top_level.txt +0 -0
  43. {variably_sdk-2.13.0 → variably_sdk-2.15.0}/tests/test_integrations_otel.py +0 -0
  44. {variably_sdk-2.13.0 → variably_sdk-2.15.0}/tests/test_prompt_assignment.py +0 -0
  45. {variably_sdk-2.13.0 → variably_sdk-2.15.0}/tests/test_streaming.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: variably-sdk
3
- Version: 2.13.0
3
+ Version: 2.15.0
4
4
  Summary: Official Python SDK for Variably feature flags, LLM experimentation, and prompt optimization platform
5
5
  Author-email: Variably <support@variably.com>
6
6
  License: MIT
@@ -30,6 +30,8 @@ Provides-Extra: langgraph
30
30
  Requires-Dist: langgraph>=0.1; extra == "langgraph"
31
31
  Provides-Extra: adk
32
32
  Requires-Dist: google-adk<3,>=2.0; python_version >= "3.10" and extra == "adk"
33
+ Provides-Extra: sql
34
+ Requires-Dist: sqlglot<31,>=25.0; python_version >= "3.8" and extra == "sql"
33
35
  Provides-Extra: otel
34
36
  Requires-Dist: opentelemetry-sdk<2,>=1.20; extra == "otel"
35
37
  Requires-Dist: opentelemetry-exporter-otlp-proto-http<2,>=1.20; extra == "otel"
@@ -856,11 +858,12 @@ python3 -c "from variably import VariablyClient, PromptVariant; print('OK')"
856
858
 
857
859
  ### Version bumping checklist
858
860
 
859
- When releasing a new version, update these three files then clean-build-publish:
861
+ When releasing a new version, update these files then clean-build-publish:
860
862
 
861
863
  1. `src/variably/version.py` — `__version__`
862
864
  2. `pyproject.toml` — `version`
863
865
  3. `src/variably/http_client.py` — `User-Agent` header string
866
+ 4. `CHANGELOG.md` — the release notes (new features and behaviour changes)
864
867
 
865
868
  ```bash
866
869
  # Example: bumping from 2.0.0 to 2.0.1
@@ -807,11 +807,12 @@ python3 -c "from variably import VariablyClient, PromptVariant; print('OK')"
807
807
 
808
808
  ### Version bumping checklist
809
809
 
810
- When releasing a new version, update these three files then clean-build-publish:
810
+ When releasing a new version, update these files then clean-build-publish:
811
811
 
812
812
  1. `src/variably/version.py` — `__version__`
813
813
  2. `pyproject.toml` — `version`
814
814
  3. `src/variably/http_client.py` — `User-Agent` header string
815
+ 4. `CHANGELOG.md` — the release notes (new features and behaviour changes)
815
816
 
816
817
  ```bash
817
818
  # Example: bumping from 2.0.0 to 2.0.1
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "variably-sdk"
7
- version = "2.13.0"
7
+ version = "2.15.0"
8
8
  description = "Official Python SDK for Variably feature flags, LLM experimentation, and prompt optimization platform"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.7"
@@ -43,6 +43,10 @@ langgraph = [
43
43
  adk = [
44
44
  "google-adk>=2.0,<3; python_version>='3.10'",
45
45
  ]
46
+ # SQL checks without a database (#96 §7): schema-linking and declared query rules.
47
+ sql = [
48
+ "sqlglot>=25.0,<31; python_version>='3.8'",
49
+ ]
46
50
  # OpenTelemetry exporter: send existing OTel traces to be evaluated.
47
51
  otel = [
48
52
  "opentelemetry-sdk>=1.20,<2",
@@ -0,0 +1,92 @@
1
+ """Values from databases and drivers, made JSON-safe before they are sent (116 §15 P1 item 26).
2
+
3
+ Databricks and most SQL drivers return ``Decimal``, ``date`` and ``datetime``, and rows as tuples or
4
+ ``Row`` objects; ``json`` (and so ``requests``) refuses them. Nothing is rounded:
5
+
6
+ - ``Decimal``: an integer when it is whole and exact in JSON (|n| <= 2**53); a float when that float
7
+ is exactly the same number; otherwise its exact text ("123456789012345678901234567890"), which the
8
+ scorer reads as a number. NaN and infinities are sent as text.
9
+ - ``date`` / ``datetime`` / ``time``: ISO 8601. ``UUID``: its text. ``bytes``: hex. Sets: lists.
10
+ - Rows (any dict's ``"rows"`` list): a mapping is kept; a Databricks / Spark ``Row`` (``asDict()``),
11
+ a namedtuple (``_asdict()``) and a ``sqlite3.Row`` (``keys()``) become dicts; a plain DB-API tuple
12
+ becomes a dict only with the column names given beside it (``"columns": [d[0] for d in
13
+ cursor.description]``), which are then dropped. A tuple row with no names is refused — never
14
+ guessed.
15
+ """
16
+ from __future__ import annotations
17
+
18
+ import datetime as dt
19
+ import enum
20
+ import uuid
21
+ from decimal import Decimal
22
+ from typing import Any
23
+
24
+ from .errors import ConfigurationError
25
+
26
+ _SAFE_INT = 2 ** 53
27
+
28
+
29
+ def _decimal(d: Decimal) -> Any:
30
+ if not d.is_finite():
31
+ return str(d)
32
+ if d == d.to_integral_value() and abs(d) <= _SAFE_INT:
33
+ return int(d)
34
+ f = float(d)
35
+ if Decimal(repr(f)) == d.normalize():
36
+ return f
37
+ return format(d, "f")
38
+
39
+
40
+ def _row(row: Any, columns: Any, where: str) -> dict:
41
+ if isinstance(row, dict):
42
+ return row
43
+ for name in ("asDict", "_asdict"):
44
+ m = getattr(row, name, None)
45
+ if callable(m):
46
+ return dict(m())
47
+ keys = getattr(row, "keys", None)
48
+ if callable(keys) and not isinstance(row, (tuple, list)): # sqlite3.Row and other mapping-like rows
49
+ return {k: row[k] for k in keys()}
50
+ if isinstance(row, (tuple, list)):
51
+ if not columns:
52
+ raise ConfigurationError(f"{where}: a row is a tuple with no column names; send rows as dicts, or give "
53
+ f"the names beside them: \"columns\": [d[0] for d in cursor.description]", where)
54
+ if len(columns) != len(row):
55
+ raise ConfigurationError(f"{where}: {len(columns)} columns but a row of {len(row)} values", where)
56
+ return dict(zip(columns, row))
57
+ raise ConfigurationError(f"{where}: a row must be a dict, a database Row or a tuple with columns", where)
58
+
59
+
60
+ def json_safe(value: Any, where: str = "payload") -> Any:
61
+ """``value`` with every driver value made JSON-safe and every ``"rows"`` list made dicts."""
62
+ if value is None or isinstance(value, (str, bool, int, float)):
63
+ return value
64
+ if isinstance(value, Decimal):
65
+ return _decimal(value)
66
+ if isinstance(value, (dt.datetime, dt.date, dt.time)):
67
+ return value.isoformat()
68
+ if isinstance(value, uuid.UUID):
69
+ return str(value)
70
+ if isinstance(value, enum.Enum):
71
+ return json_safe(value.value, where)
72
+ if isinstance(value, (bytes, bytearray, memoryview)):
73
+ return bytes(value).hex()
74
+ if isinstance(value, dict):
75
+ out = {}
76
+ rows = value.get("rows")
77
+ if isinstance(rows, (list, tuple)) and rows:
78
+ columns = value.get("columns")
79
+ out["rows"] = [json_safe(_row(r, columns, f"{where}.rows[{i}]"), f"{where}.rows[{i}]")
80
+ for i, r in enumerate(rows)]
81
+ for k, v in value.items():
82
+ if k not in ("rows", "columns"):
83
+ out[str(k)] = json_safe(v, f"{where}.{k}")
84
+ return out
85
+ return {str(k): json_safe(v, f"{where}.{k}") for k, v in value.items()}
86
+ if isinstance(value, (list, tuple, set, frozenset)):
87
+ return [json_safe(v, f"{where}[{i}]") for i, v in enumerate(value)]
88
+ for name in ("asDict", "_asdict"): # a Row outside "rows"
89
+ m = getattr(value, name, None)
90
+ if callable(m):
91
+ return json_safe(dict(m()), where)
92
+ return value # anything else: json reports it, as before
@@ -5,6 +5,7 @@ import json
5
5
  from datetime import datetime
6
6
  from typing import Any, Dict, List, Optional, Union
7
7
 
8
+ from ._json_safe import json_safe
8
9
  from .cache import CacheManager, NoOpCache
9
10
  from .config import VariablyConfig
10
11
  from .errors import ConfigurationError, FlagNotFoundError, VariablyError
@@ -706,6 +707,10 @@ class VariablyClient:
706
707
  "transformer", # query rewrite / expansion / summarization step
707
708
  )
708
709
 
710
+ # "Answers what was asked" (117): the kinds of answer a question can want (the API's list).
711
+ _ANSWER_KINDS = ("amount", "quantity", "percent", "date", "duration", "count", "yes_no", "list", "procedure",
712
+ "explanation", "other")
713
+
709
714
  def submit_response(
710
715
  self,
711
716
  experiment_key: str,
@@ -813,15 +818,23 @@ class VariablyClient:
813
818
  # RAG / Grounding context
814
819
  reference_materials: Optional[List[Dict[str, Any]]] = None, # Retrieved docs for grounding scoring
815
820
  retrieval_query: Optional[str] = None, # The actual search query (if rewritten/expanded)
821
+ question_scope: Optional[Dict[str, Any]] = None, # What the question is about: as_of and the project's scope dimensions
822
+ handoff: Optional[Dict[str, Any]] = None, # {"to": agent, "fields": {...}, "message": "..."} handed to another agent
823
+ expected_answer_kind: Optional[str] = None, # 117: "amount", "date", "yes_no", … — checked in code
824
+ required_checks: Optional[List[Dict[str, Any]]] = None, # 103: [{"id", "text", "critical", "kind"}] the answer must address
825
+ requirements_ref: Optional[str] = None, # 103: a stored requirement template, "name" or "name@v3"
826
+ sql: Optional[Union[str, List[str]]] = None, # #96: the SQL this answer ran, checked by the server
827
+ expected_answer: Optional[str] = None, # your expected answer: its values compared in code, its own verdict
816
828
  # Multi-turn / Session context
817
829
  session_id: Optional[str] = None, # Groups conversation turns into sessions
818
830
  conversation_history: Optional[List[Dict[str, Any]]] = None, # Prior turns for coherence scoring
819
831
  # Multi-agent context — see docs/improvements/42.1-implementation-plan.md
820
832
  agent_name: Optional[str] = None, # This step's agent, within a session_id trace
821
833
  is_aggregator: bool = False, # True on the final step that aggregates the trace
822
- tools_called: Optional[List[Dict[str, Any]]] = None, # [{"name","arguments","output"}] actually called
834
+ tools_called: Optional[List[Dict[str, Any]]] = None, # [{"name","arguments","output","error"}] actually called; "error" on a failed call
823
835
  expected_tools: Optional[List[Dict[str, Any]]] = None, # golden tools for deterministic trajectory scoring
824
836
  allowed_tools: Optional[List[str]] = None, # permitted tool names; calls outside this list flag as unauthorized
837
+ max_steps: Optional[int] = None, # the golden's budget of tool calls for this run
825
838
  # User & organization context
826
839
  user_id: Optional[str] = None, # End-user ID for per-user analytics
827
840
  tags: Optional[List[str]] = None, # Grouping labels, e.g. ["production", "rag"]
@@ -846,6 +859,21 @@ class VariablyClient:
846
859
  response: The AI's generated response text.
847
860
  reference_materials: Retrieved docs (RAG chunks) for grounding scoring.
848
861
  Enables: faithfulness, hallucination rate, attribution accuracy.
862
+ question_scope: What the question is about, for "Right version of the source":
863
+ "as_of" (a date) and any period or text dimension the project declares
864
+ (scope defaults "dimensions"; without any, the presets plan_year,
865
+ fiscal_year, jurisdiction, product), e.g. {"as_of": "2026-03-01",
866
+ "region": "EMEA"}. Checked against each reference's "metadata" — the same
867
+ dimensions' keys (a date range "coverage" as coverage_from / coverage_to).
868
+ A key no dimension declares is refused (400). Wins over what the scorer
869
+ reads from the question's text.
870
+ handoff: What this step handed to another agent, checked against the
871
+ project's hand-off contracts: {"to": "adjudication", "fields":
872
+ {"onset_date": "2026-03-01", "leave_type": "STD"}, "message": "..."}.
873
+ "to" is required; fields are JSON values (money as {"amount",
874
+ "currency"}, records, lists) per the contract's types. "message" is the
875
+ hand-off as text, grounded in what this step had. An agent called as a
876
+ tool (tools_called) is checked without this.
849
877
  retrieval_query: The query sent to your retriever/search.
850
878
  If your pipeline rewrites queries, pass the rewritten version.
851
879
  session_id: Conversation/thread ID — groups turns into sessions.
@@ -920,6 +948,61 @@ class VariablyClient:
920
948
  evaluation_context["reference_materials"] = reference_materials
921
949
  if retrieval_query:
922
950
  evaluation_context["retrieval_query"] = retrieval_query
951
+ if expected_answer_kind is not None:
952
+ if expected_answer_kind not in self._ANSWER_KINDS:
953
+ raise ConfigurationError(f"expected_answer_kind must be one of {self._ANSWER_KINDS}", "expected_answer_kind")
954
+ evaluation_context["expected_answer_kind"] = expected_answer_kind
955
+ if required_checks is not None and requirements_ref is not None:
956
+ raise ConfigurationError("give required_checks or requirements_ref, not both", "required_checks")
957
+ if required_checks is not None:
958
+ if not isinstance(required_checks, list) or not all(
959
+ isinstance(r, dict) and isinstance(r.get("id"), str) and isinstance(r.get("text"), str)
960
+ for r in required_checks):
961
+ raise ConfigurationError("required_checks must be a list of {'id', 'text', 'critical', 'kind'} dicts",
962
+ "required_checks")
963
+ evaluation_context["required_checks"] = required_checks
964
+ if requirements_ref is not None:
965
+ if not isinstance(requirements_ref, str) or not requirements_ref.strip():
966
+ raise ConfigurationError("requirements_ref must be a template name, e.g. 'onboarding@v2'", "requirements_ref")
967
+ evaluation_context["requirements_ref"] = requirements_ref
968
+ if sql is not None:
969
+ evaluation_context["sql_queries"] = self._sql_queries(sql)
970
+ if expected_answer is not None:
971
+ if not isinstance(expected_answer, str) or not expected_answer.strip() or len(expected_answer) > 20_000:
972
+ raise ConfigurationError("expected_answer must be non-empty text of at most 20,000 characters",
973
+ "expected_answer")
974
+ evaluation_context["expected_answer"] = expected_answer
975
+ if handoff is not None:
976
+ handoff = json_safe(handoff, "handoff") if isinstance(handoff, dict) else handoff
977
+ if not isinstance(handoff, dict) or not isinstance(handoff.get("to"), str) or not handoff["to"].strip():
978
+ raise ConfigurationError("handoff must be a dict with a non-empty 'to' (the receiving agent)", "handoff")
979
+ unknown = set(handoff) - {"to", "fields", "message"}
980
+ if unknown:
981
+ raise ConfigurationError(f"handoff: unknown keys {sorted(unknown)}", "handoff")
982
+ fields = handoff.get("fields") or {}
983
+
984
+ def _json_value(v, depth=1):
985
+ if v is None or isinstance(v, (str, int, float, bool)):
986
+ return True
987
+ if depth > 4:
988
+ return False
989
+ if isinstance(v, list):
990
+ return all(_json_value(x, depth + 1) for x in v)
991
+ if isinstance(v, dict):
992
+ return all(isinstance(k, str) and k.strip() and _json_value(x, depth + 1) for k, x in v.items())
993
+ return False
994
+
995
+ if not isinstance(fields, dict) or not all(isinstance(k, str) and k.strip() and _json_value(v)
996
+ for k, v in fields.items()):
997
+ raise ConfigurationError("handoff.fields must map names to JSON values (records and lists "
998
+ "nest at most 4 deep)", "handoff")
999
+ if handoff.get("message") is not None and not isinstance(handoff["message"], str):
1000
+ raise ConfigurationError("handoff.message must be text", "handoff")
1001
+ evaluation_context["handoff"] = handoff
1002
+ if question_scope:
1003
+ if not isinstance(question_scope, dict):
1004
+ raise ConfigurationError("question_scope must be a dict", "question_scope")
1005
+ evaluation_context["question_scope"] = question_scope
923
1006
  if evaluation_context:
924
1007
  data["evaluation_context"] = evaluation_context
925
1008
 
@@ -939,6 +1022,10 @@ class VariablyClient:
939
1022
  data["expected_tools"] = expected_tools
940
1023
  if allowed_tools:
941
1024
  data["allowed_tools"] = allowed_tools
1025
+ if max_steps is not None:
1026
+ if not isinstance(max_steps, int) or isinstance(max_steps, bool) or max_steps < 1:
1027
+ raise ValueError(f"max_steps must be a positive integer, got {max_steps!r}")
1028
+ data["max_steps"] = max_steps
942
1029
  if provider:
943
1030
  data["provider"] = provider
944
1031
  if model:
@@ -956,6 +1043,9 @@ class VariablyClient:
956
1043
  if kind:
957
1044
  data["kind"] = kind
958
1045
 
1046
+ # Database values and rows (Decimal, dates, Databricks / DB-API rows) made JSON-safe here,
1047
+ # so a row that cannot be named is refused at the call (116 §15 item 26).
1048
+ data = json_safe(data)
959
1049
  try:
960
1050
  result = self.http_client.log_observation(data)
961
1051
  return LogResult(
@@ -970,6 +1060,62 @@ class VariablyClient:
970
1060
  )
971
1061
  raise
972
1062
 
1063
+ # --- Check settings as code (versioned; see variably.settings) ---------------------------
1064
+
1065
+ def set_code_lists(self, experiment_id: str, lists: Any, note: Optional[str] = None) -> Dict[str, Any]:
1066
+ """Save the experiment's code lists (a dict, or a JSON/YAML file path). {} removes them.
1067
+ Refused when a saved hand-off contract uses a list this removes."""
1068
+ from . import settings
1069
+ return settings.set_code_lists(self.http_client, experiment_id, lists, note)
1070
+
1071
+ def set_handoff_contracts(self, experiment_id: str, contracts: Any, note: Optional[str] = None) -> Dict[str, Any]:
1072
+ """Save the experiment's hand-off contracts (a list, or a JSON/YAML file path), checked
1073
+ against its saved code lists. [] removes them."""
1074
+ from . import settings
1075
+ return settings.set_handoff_contracts(self.http_client, experiment_id, contracts, note)
1076
+
1077
+ # The server's limits on the SQL one answer sends (sql_settings.py).
1078
+ _MAX_SQL_QUERIES = 20
1079
+ _MAX_SQL_CHARS = 100_000
1080
+
1081
+ @classmethod
1082
+ def _sql_queries(cls, sql: Any) -> List[str]:
1083
+ """The SQL text an answer ran, as a list. Only the text is sent: the server checks it
1084
+ against the project's sql_checks; the local check (variably.sql) is a developer-time
1085
+ warning and its result is never sent."""
1086
+ queries = [sql] if isinstance(sql, str) else sql
1087
+ if (not isinstance(queries, list) or not queries or len(queries) > cls._MAX_SQL_QUERIES
1088
+ or not all(isinstance(q, str) and q.strip() and len(q) <= cls._MAX_SQL_CHARS for q in queries)):
1089
+ raise ConfigurationError(
1090
+ f"sql must be the SQL text the answer ran, or a list of at most {cls._MAX_SQL_QUERIES} such texts "
1091
+ f"(each non-empty, at most {cls._MAX_SQL_CHARS} characters)", "sql")
1092
+ return list(queries)
1093
+
1094
+ def set_sql_checks(self, experiment_id: str, sql_checks: Any, note: Optional[str] = None) -> Dict[str, Any]:
1095
+ """Save the experiment's SQL checks (a dict, or a JSON/YAML file path): {"dialect",
1096
+ "schema", "rules"}. The server checks each answer's SQL text against them — it never
1097
+ connects to your database; the schema is table and column names (and types), no data.
1098
+ A rule that names columns needs the schema. {} removes them."""
1099
+ from . import settings
1100
+ return settings.set_sql_checks(self.http_client, experiment_id, sql_checks, note)
1101
+
1102
+ def set_scope_defaults(self, experiment_id: str, defaults: Any, note: Optional[str] = None) -> Dict[str, Any]:
1103
+ """Save the experiment's scope defaults (a dict, or a JSON/YAML file path): when plan
1104
+ and fiscal years start, which request metadata keys state the question's scope, and
1105
+ whether document headers are read. {} removes them."""
1106
+ from . import settings
1107
+ return settings.set_scope_defaults(self.http_client, experiment_id, defaults, note)
1108
+
1109
+ def settings_history(self, experiment_id: str, kind: Optional[str] = None, limit: int = 100) -> List[Dict[str, Any]]:
1110
+ """Recorded versions of the experiment's check settings, newest first, without content."""
1111
+ from . import settings
1112
+ return settings.settings_history(self.http_client, experiment_id, kind, limit)
1113
+
1114
+ def settings_version(self, experiment_id: str, kind: str, version: int) -> Dict[str, Any]:
1115
+ """One recorded version of a setting, with its content."""
1116
+ from . import settings
1117
+ return settings.settings_version(self.http_client, experiment_id, kind, version)
1118
+
973
1119
  def clear_cache(self) -> None:
974
1120
  """Clear the cache."""
975
1121
  self.cache.clear()
@@ -2,6 +2,7 @@
2
2
 
3
3
  import json
4
4
  import time
5
+ from ._json_safe import json_safe
5
6
  from typing import Any, Dict, List, Optional
6
7
 
7
8
  import requests
@@ -94,7 +95,7 @@ class HTTPClient:
94
95
  session.headers.update({
95
96
  "Content-Type": "application/json",
96
97
  "X-API-Key": self.config.api_key,
97
- "User-Agent": "Variably-Python-SDK/2.3.0",
98
+ "User-Agent": "Variably-Python-SDK/2.15.0",
98
99
  })
99
100
 
100
101
  return session
@@ -271,7 +272,7 @@ class HTTPClient:
271
272
 
272
273
  response = self.session.post(
273
274
  url,
274
- json=request_data,
275
+ json=json_safe(request_data),
275
276
  timeout=self.config.timeout,
276
277
  stream=True,
277
278
  )
@@ -299,6 +300,10 @@ class HTTPClient:
299
300
  retryable=False,
300
301
  )
301
302
 
303
+ def request_json(self, method: str, path: str, **kwargs: Any) -> Any:
304
+ """A management call (settings, history): never retried, errors raised as-is."""
305
+ return self._make_request(method, path, retryable=False, **kwargs)
306
+
302
307
  def _make_request(
303
308
  self,
304
309
  method: str,
@@ -309,6 +314,9 @@ class HTTPClient:
309
314
  """Make HTTP request with error handling and metrics."""
310
315
  url = f"{self.config.base_url}{path}"
311
316
  start_time = time.time()
317
+ if "json" in kwargs:
318
+ # Driver values (Decimal, dates, Row objects) never reach json.dumps raw (116 §15 item 26).
319
+ kwargs["json"] = json_safe(kwargs["json"])
312
320
 
313
321
  try:
314
322
  self.logger.debug(f"Making {method} request to {path}")
@@ -242,6 +242,11 @@ class VariablyPlugin(BasePlugin):
242
242
  ``{"researcher": [{"name": "search_policy"}]}`` — enables deterministic
243
243
  tool-correctness / trajectory scoring against ground truth.
244
244
  allowed_tools: optional allow-list per agent (tool-permission checks).
245
+ required_checks: optional requirements per agent — what its answer must address,
246
+ ``{"advisor": [{"id": "kyc", "text": "Identity is verified", "critical": True}]}``
247
+ (requirement coverage). Or ``requirements_ref``: a stored template per agent,
248
+ ``{"advisor": "onboarding@v2"}``. Not both for one agent. Not sent with
249
+ ``capture_text=False`` (there is no answer text to check).
245
250
  evidence_extractors: ``{tool_name: fn(result) -> [{"id","content","source"}]}``
246
251
  for function-tool retrievers whose result shape isn't auto-detected.
247
252
  capture_text: ``False`` sends structure only (no prompt/response/tool payload
@@ -255,6 +260,8 @@ class VariablyPlugin(BasePlugin):
255
260
  *,
256
261
  expected_tools: Optional[Dict[str, List[Dict[str, Any]]]] = None,
257
262
  allowed_tools: Optional[Dict[str, List[str]]] = None,
263
+ required_checks: Optional[Dict[str, List[Dict[str, Any]]]] = None,
264
+ requirements_ref: Optional[Dict[str, str]] = None,
258
265
  evidence_extractors: Optional[Dict[str, Callable[[Any], List[Dict[str, Any]]]]] = None,
259
266
  capture_text: bool = True,
260
267
  tags: Optional[List[str]] = None,
@@ -265,6 +272,11 @@ class VariablyPlugin(BasePlugin):
265
272
  super().__init__(name=name)
266
273
  self._expected_tools = expected_tools or {}
267
274
  self._allowed_tools = allowed_tools or {}
275
+ self._required_checks = dict(required_checks or {})
276
+ self._requirements_ref = dict(requirements_ref or {})
277
+ both = sorted(set(self._required_checks) & set(self._requirements_ref))
278
+ if both:
279
+ raise ValueError(f"adk: give required_checks or requirements_ref per agent, not both: {both}")
268
280
  self._extractors = evidence_extractors or {}
269
281
  self._capture_text = capture_text
270
282
  self._tags = list(tags or [])
@@ -397,6 +409,13 @@ class VariablyPlugin(BasePlugin):
397
409
  )
398
410
  if kind:
399
411
  payload["kind"] = kind
412
+ reqs, ref = self._required_checks.get(step.agent_name), self._requirements_ref.get(step.agent_name)
413
+ if reqs is not None or ref is not None:
414
+ if self._capture_text:
415
+ payload["required_checks"], payload["requirements_ref"] = reqs, ref
416
+ else:
417
+ self._warn_once("requirements_no_text",
418
+ "adk: requirements are not checked with capture_text=False (no answer text is sent)")
400
419
  self._sender.put(payload)
401
420
 
402
421
  def _finish(self, invocation_id: str, error: Optional[str] = None) -> None:
@@ -490,6 +509,8 @@ class VariablyPlugin(BasePlugin):
490
509
  entry: Dict[str, Any] = {"name": name, "arguments": _args(tool_args)}
491
510
  if error is not None:
492
511
  entry["output"] = f"ERROR: {type(error).__name__}: {error}"[:500]
512
+ # Structured: a failed call is a wasted step (never read from output text).
513
+ entry["error"] = f"{type(error).__name__}: {error}"[:500]
493
514
  else:
494
515
  entry["output"] = _text_of(result)
495
516
  extractor = self._extractors.get(name)