variably-sdk 2.13.0__tar.gz → 2.15.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {variably_sdk-2.13.0 → variably_sdk-2.15.0}/PKG-INFO +5 -2
- {variably_sdk-2.13.0 → variably_sdk-2.15.0}/README.md +2 -1
- {variably_sdk-2.13.0 → variably_sdk-2.15.0}/pyproject.toml +5 -1
- variably_sdk-2.15.0/src/variably/_json_safe.py +92 -0
- {variably_sdk-2.13.0 → variably_sdk-2.15.0}/src/variably/client.py +147 -1
- {variably_sdk-2.13.0 → variably_sdk-2.15.0}/src/variably/http_client.py +10 -2
- {variably_sdk-2.13.0 → variably_sdk-2.15.0}/src/variably/integrations/adk.py +21 -0
- {variably_sdk-2.13.0 → variably_sdk-2.15.0}/src/variably/integrations/langgraph.py +190 -11
- {variably_sdk-2.13.0 → variably_sdk-2.15.0}/src/variably/integrations/otel.py +4 -0
- variably_sdk-2.15.0/src/variably/integrations/otel_attributes.py +54 -0
- {variably_sdk-2.13.0 → variably_sdk-2.15.0}/src/variably/observe.py +24 -2
- variably_sdk-2.15.0/src/variably/settings.py +198 -0
- variably_sdk-2.15.0/src/variably/sql/__init__.py +42 -0
- variably_sdk-2.15.0/src/variably/sql/checks.py +843 -0
- variably_sdk-2.15.0/src/variably/sql/execute.py +415 -0
- {variably_sdk-2.13.0 → variably_sdk-2.15.0}/src/variably/version.py +1 -1
- {variably_sdk-2.13.0 → variably_sdk-2.15.0}/src/variably_sdk.egg-info/PKG-INFO +5 -2
- {variably_sdk-2.13.0 → variably_sdk-2.15.0}/src/variably_sdk.egg-info/SOURCES.txt +13 -0
- {variably_sdk-2.13.0 → variably_sdk-2.15.0}/src/variably_sdk.egg-info/requires.txt +5 -0
- {variably_sdk-2.13.0 → variably_sdk-2.15.0}/tests/test_integrations_adk.py +23 -0
- variably_sdk-2.15.0/tests/test_integrations_langgraph.py +237 -0
- variably_sdk-2.15.0/tests/test_json_safe_rows.py +134 -0
- variably_sdk-2.15.0/tests/test_otel_handoff_attributes.py +23 -0
- variably_sdk-2.15.0/tests/test_scope_rows_passthrough.py +178 -0
- variably_sdk-2.15.0/tests/test_settings.py +150 -0
- variably_sdk-2.15.0/tests/test_sql_checks.py +525 -0
- variably_sdk-2.15.0/tests/test_sql_execute.py +272 -0
- variably_sdk-2.15.0/tests/test_sql_execute_safety.py +193 -0
- variably_sdk-2.13.0/tests/test_integrations_langgraph.py +0 -131
- {variably_sdk-2.13.0 → variably_sdk-2.15.0}/setup.cfg +0 -0
- {variably_sdk-2.13.0 → variably_sdk-2.15.0}/src/variably/__init__.py +0 -0
- {variably_sdk-2.13.0 → variably_sdk-2.15.0}/src/variably/assignment.py +0 -0
- {variably_sdk-2.13.0 → variably_sdk-2.15.0}/src/variably/cache.py +0 -0
- {variably_sdk-2.13.0 → variably_sdk-2.15.0}/src/variably/config.py +0 -0
- {variably_sdk-2.13.0 → variably_sdk-2.15.0}/src/variably/errors.py +0 -0
- {variably_sdk-2.13.0 → variably_sdk-2.15.0}/src/variably/integrations/__init__.py +0 -0
- {variably_sdk-2.13.0 → variably_sdk-2.15.0}/src/variably/integrations/_common.py +0 -0
- {variably_sdk-2.13.0 → variably_sdk-2.15.0}/src/variably/logger.py +0 -0
- {variably_sdk-2.13.0 → variably_sdk-2.15.0}/src/variably/prompt_config.py +0 -0
- {variably_sdk-2.13.0 → variably_sdk-2.15.0}/src/variably/types.py +0 -0
- {variably_sdk-2.13.0 → variably_sdk-2.15.0}/src/variably_sdk.egg-info/dependency_links.txt +0 -0
- {variably_sdk-2.13.0 → variably_sdk-2.15.0}/src/variably_sdk.egg-info/top_level.txt +0 -0
- {variably_sdk-2.13.0 → variably_sdk-2.15.0}/tests/test_integrations_otel.py +0 -0
- {variably_sdk-2.13.0 → variably_sdk-2.15.0}/tests/test_prompt_assignment.py +0 -0
- {variably_sdk-2.13.0 → variably_sdk-2.15.0}/tests/test_streaming.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: variably-sdk
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.15.0
|
|
4
4
|
Summary: Official Python SDK for Variably feature flags, LLM experimentation, and prompt optimization platform
|
|
5
5
|
Author-email: Variably <support@variably.com>
|
|
6
6
|
License: MIT
|
|
@@ -30,6 +30,8 @@ Provides-Extra: langgraph
|
|
|
30
30
|
Requires-Dist: langgraph>=0.1; extra == "langgraph"
|
|
31
31
|
Provides-Extra: adk
|
|
32
32
|
Requires-Dist: google-adk<3,>=2.0; python_version >= "3.10" and extra == "adk"
|
|
33
|
+
Provides-Extra: sql
|
|
34
|
+
Requires-Dist: sqlglot<31,>=25.0; python_version >= "3.8" and extra == "sql"
|
|
33
35
|
Provides-Extra: otel
|
|
34
36
|
Requires-Dist: opentelemetry-sdk<2,>=1.20; extra == "otel"
|
|
35
37
|
Requires-Dist: opentelemetry-exporter-otlp-proto-http<2,>=1.20; extra == "otel"
|
|
@@ -856,11 +858,12 @@ python3 -c "from variably import VariablyClient, PromptVariant; print('OK')"
|
|
|
856
858
|
|
|
857
859
|
### Version bumping checklist
|
|
858
860
|
|
|
859
|
-
When releasing a new version, update these
|
|
861
|
+
When releasing a new version, update these files then clean-build-publish:
|
|
860
862
|
|
|
861
863
|
1. `src/variably/version.py` — `__version__`
|
|
862
864
|
2. `pyproject.toml` — `version`
|
|
863
865
|
3. `src/variably/http_client.py` — `User-Agent` header string
|
|
866
|
+
4. `CHANGELOG.md` — the release notes (new features and behaviour changes)
|
|
864
867
|
|
|
865
868
|
```bash
|
|
866
869
|
# Example: bumping from 2.0.0 to 2.0.1
|
|
@@ -807,11 +807,12 @@ python3 -c "from variably import VariablyClient, PromptVariant; print('OK')"
|
|
|
807
807
|
|
|
808
808
|
### Version bumping checklist
|
|
809
809
|
|
|
810
|
-
When releasing a new version, update these
|
|
810
|
+
When releasing a new version, update these files then clean-build-publish:
|
|
811
811
|
|
|
812
812
|
1. `src/variably/version.py` — `__version__`
|
|
813
813
|
2. `pyproject.toml` — `version`
|
|
814
814
|
3. `src/variably/http_client.py` — `User-Agent` header string
|
|
815
|
+
4. `CHANGELOG.md` — the release notes (new features and behaviour changes)
|
|
815
816
|
|
|
816
817
|
```bash
|
|
817
818
|
# Example: bumping from 2.0.0 to 2.0.1
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "variably-sdk"
|
|
7
|
-
version = "2.
|
|
7
|
+
version = "2.15.0"
|
|
8
8
|
description = "Official Python SDK for Variably feature flags, LLM experimentation, and prompt optimization platform"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.7"
|
|
@@ -43,6 +43,10 @@ langgraph = [
|
|
|
43
43
|
adk = [
|
|
44
44
|
"google-adk>=2.0,<3; python_version>='3.10'",
|
|
45
45
|
]
|
|
46
|
+
# SQL checks without a database (#96 §7): schema-linking and declared query rules.
|
|
47
|
+
sql = [
|
|
48
|
+
"sqlglot>=25.0,<31; python_version>='3.8'",
|
|
49
|
+
]
|
|
46
50
|
# OpenTelemetry exporter: send existing OTel traces to be evaluated.
|
|
47
51
|
otel = [
|
|
48
52
|
"opentelemetry-sdk>=1.20,<2",
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
"""Values from databases and drivers, made JSON-safe before they are sent (116 §15 P1 item 26).
|
|
2
|
+
|
|
3
|
+
Databricks and most SQL drivers return ``Decimal``, ``date`` and ``datetime``, and rows as tuples or
|
|
4
|
+
``Row`` objects; ``json`` (and so ``requests``) refuses them. Nothing is rounded:
|
|
5
|
+
|
|
6
|
+
- ``Decimal``: an integer when it is whole and exact in JSON (|n| <= 2**53); a float when that float
|
|
7
|
+
is exactly the same number; otherwise its exact text ("123456789012345678901234567890"), which the
|
|
8
|
+
scorer reads as a number. NaN and infinities are sent as text.
|
|
9
|
+
- ``date`` / ``datetime`` / ``time``: ISO 8601. ``UUID``: its text. ``bytes``: hex. Sets: lists.
|
|
10
|
+
- Rows (any dict's ``"rows"`` list): a mapping is kept; a Databricks / Spark ``Row`` (``asDict()``),
|
|
11
|
+
a namedtuple (``_asdict()``) and a ``sqlite3.Row`` (``keys()``) become dicts; a plain DB-API tuple
|
|
12
|
+
becomes a dict only with the column names given beside it (``"columns": [d[0] for d in
|
|
13
|
+
cursor.description]``), which are then dropped. A tuple row with no names is refused — never
|
|
14
|
+
guessed.
|
|
15
|
+
"""
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import datetime as dt
|
|
19
|
+
import enum
|
|
20
|
+
import uuid
|
|
21
|
+
from decimal import Decimal
|
|
22
|
+
from typing import Any
|
|
23
|
+
|
|
24
|
+
from .errors import ConfigurationError
|
|
25
|
+
|
|
26
|
+
_SAFE_INT = 2 ** 53
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _decimal(d: Decimal) -> Any:
|
|
30
|
+
if not d.is_finite():
|
|
31
|
+
return str(d)
|
|
32
|
+
if d == d.to_integral_value() and abs(d) <= _SAFE_INT:
|
|
33
|
+
return int(d)
|
|
34
|
+
f = float(d)
|
|
35
|
+
if Decimal(repr(f)) == d.normalize():
|
|
36
|
+
return f
|
|
37
|
+
return format(d, "f")
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _row(row: Any, columns: Any, where: str) -> dict:
|
|
41
|
+
if isinstance(row, dict):
|
|
42
|
+
return row
|
|
43
|
+
for name in ("asDict", "_asdict"):
|
|
44
|
+
m = getattr(row, name, None)
|
|
45
|
+
if callable(m):
|
|
46
|
+
return dict(m())
|
|
47
|
+
keys = getattr(row, "keys", None)
|
|
48
|
+
if callable(keys) and not isinstance(row, (tuple, list)): # sqlite3.Row and other mapping-like rows
|
|
49
|
+
return {k: row[k] for k in keys()}
|
|
50
|
+
if isinstance(row, (tuple, list)):
|
|
51
|
+
if not columns:
|
|
52
|
+
raise ConfigurationError(f"{where}: a row is a tuple with no column names; send rows as dicts, or give "
|
|
53
|
+
f"the names beside them: \"columns\": [d[0] for d in cursor.description]", where)
|
|
54
|
+
if len(columns) != len(row):
|
|
55
|
+
raise ConfigurationError(f"{where}: {len(columns)} columns but a row of {len(row)} values", where)
|
|
56
|
+
return dict(zip(columns, row))
|
|
57
|
+
raise ConfigurationError(f"{where}: a row must be a dict, a database Row or a tuple with columns", where)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def json_safe(value: Any, where: str = "payload") -> Any:
|
|
61
|
+
"""``value`` with every driver value made JSON-safe and every ``"rows"`` list made dicts."""
|
|
62
|
+
if value is None or isinstance(value, (str, bool, int, float)):
|
|
63
|
+
return value
|
|
64
|
+
if isinstance(value, Decimal):
|
|
65
|
+
return _decimal(value)
|
|
66
|
+
if isinstance(value, (dt.datetime, dt.date, dt.time)):
|
|
67
|
+
return value.isoformat()
|
|
68
|
+
if isinstance(value, uuid.UUID):
|
|
69
|
+
return str(value)
|
|
70
|
+
if isinstance(value, enum.Enum):
|
|
71
|
+
return json_safe(value.value, where)
|
|
72
|
+
if isinstance(value, (bytes, bytearray, memoryview)):
|
|
73
|
+
return bytes(value).hex()
|
|
74
|
+
if isinstance(value, dict):
|
|
75
|
+
out = {}
|
|
76
|
+
rows = value.get("rows")
|
|
77
|
+
if isinstance(rows, (list, tuple)) and rows:
|
|
78
|
+
columns = value.get("columns")
|
|
79
|
+
out["rows"] = [json_safe(_row(r, columns, f"{where}.rows[{i}]"), f"{where}.rows[{i}]")
|
|
80
|
+
for i, r in enumerate(rows)]
|
|
81
|
+
for k, v in value.items():
|
|
82
|
+
if k not in ("rows", "columns"):
|
|
83
|
+
out[str(k)] = json_safe(v, f"{where}.{k}")
|
|
84
|
+
return out
|
|
85
|
+
return {str(k): json_safe(v, f"{where}.{k}") for k, v in value.items()}
|
|
86
|
+
if isinstance(value, (list, tuple, set, frozenset)):
|
|
87
|
+
return [json_safe(v, f"{where}[{i}]") for i, v in enumerate(value)]
|
|
88
|
+
for name in ("asDict", "_asdict"): # a Row outside "rows"
|
|
89
|
+
m = getattr(value, name, None)
|
|
90
|
+
if callable(m):
|
|
91
|
+
return json_safe(dict(m()), where)
|
|
92
|
+
return value # anything else: json reports it, as before
|
|
@@ -5,6 +5,7 @@ import json
|
|
|
5
5
|
from datetime import datetime
|
|
6
6
|
from typing import Any, Dict, List, Optional, Union
|
|
7
7
|
|
|
8
|
+
from ._json_safe import json_safe
|
|
8
9
|
from .cache import CacheManager, NoOpCache
|
|
9
10
|
from .config import VariablyConfig
|
|
10
11
|
from .errors import ConfigurationError, FlagNotFoundError, VariablyError
|
|
@@ -706,6 +707,10 @@ class VariablyClient:
|
|
|
706
707
|
"transformer", # query rewrite / expansion / summarization step
|
|
707
708
|
)
|
|
708
709
|
|
|
710
|
+
# "Answers what was asked" (117): the kinds of answer a question can want (the API's list).
|
|
711
|
+
_ANSWER_KINDS = ("amount", "quantity", "percent", "date", "duration", "count", "yes_no", "list", "procedure",
|
|
712
|
+
"explanation", "other")
|
|
713
|
+
|
|
709
714
|
def submit_response(
|
|
710
715
|
self,
|
|
711
716
|
experiment_key: str,
|
|
@@ -813,15 +818,23 @@ class VariablyClient:
|
|
|
813
818
|
# RAG / Grounding context
|
|
814
819
|
reference_materials: Optional[List[Dict[str, Any]]] = None, # Retrieved docs for grounding scoring
|
|
815
820
|
retrieval_query: Optional[str] = None, # The actual search query (if rewritten/expanded)
|
|
821
|
+
question_scope: Optional[Dict[str, Any]] = None, # What the question is about: as_of and the project's scope dimensions
|
|
822
|
+
handoff: Optional[Dict[str, Any]] = None, # {"to": agent, "fields": {...}, "message": "..."} handed to another agent
|
|
823
|
+
expected_answer_kind: Optional[str] = None, # 117: "amount", "date", "yes_no", … — checked in code
|
|
824
|
+
required_checks: Optional[List[Dict[str, Any]]] = None, # 103: [{"id", "text", "critical", "kind"}] the answer must address
|
|
825
|
+
requirements_ref: Optional[str] = None, # 103: a stored requirement template, "name" or "name@v3"
|
|
826
|
+
sql: Optional[Union[str, List[str]]] = None, # #96: the SQL this answer ran, checked by the server
|
|
827
|
+
expected_answer: Optional[str] = None, # your expected answer: its values compared in code, its own verdict
|
|
816
828
|
# Multi-turn / Session context
|
|
817
829
|
session_id: Optional[str] = None, # Groups conversation turns into sessions
|
|
818
830
|
conversation_history: Optional[List[Dict[str, Any]]] = None, # Prior turns for coherence scoring
|
|
819
831
|
# Multi-agent context — see docs/improvements/42.1-implementation-plan.md
|
|
820
832
|
agent_name: Optional[str] = None, # This step's agent, within a session_id trace
|
|
821
833
|
is_aggregator: bool = False, # True on the final step that aggregates the trace
|
|
822
|
-
tools_called: Optional[List[Dict[str, Any]]] = None, # [{"name","arguments","output"}] actually called
|
|
834
|
+
tools_called: Optional[List[Dict[str, Any]]] = None, # [{"name","arguments","output","error"}] actually called; "error" on a failed call
|
|
823
835
|
expected_tools: Optional[List[Dict[str, Any]]] = None, # golden tools for deterministic trajectory scoring
|
|
824
836
|
allowed_tools: Optional[List[str]] = None, # permitted tool names; calls outside this list flag as unauthorized
|
|
837
|
+
max_steps: Optional[int] = None, # the golden's budget of tool calls for this run
|
|
825
838
|
# User & organization context
|
|
826
839
|
user_id: Optional[str] = None, # End-user ID for per-user analytics
|
|
827
840
|
tags: Optional[List[str]] = None, # Grouping labels, e.g. ["production", "rag"]
|
|
@@ -846,6 +859,21 @@ class VariablyClient:
|
|
|
846
859
|
response: The AI's generated response text.
|
|
847
860
|
reference_materials: Retrieved docs (RAG chunks) for grounding scoring.
|
|
848
861
|
Enables: faithfulness, hallucination rate, attribution accuracy.
|
|
862
|
+
question_scope: What the question is about, for "Right version of the source":
|
|
863
|
+
"as_of" (a date) and any period or text dimension the project declares
|
|
864
|
+
(scope defaults "dimensions"; without any, the presets plan_year,
|
|
865
|
+
fiscal_year, jurisdiction, product), e.g. {"as_of": "2026-03-01",
|
|
866
|
+
"region": "EMEA"}. Checked against each reference's "metadata" — the same
|
|
867
|
+
dimensions' keys (a date range "coverage" as coverage_from / coverage_to).
|
|
868
|
+
A key no dimension declares is refused (400). Wins over what the scorer
|
|
869
|
+
reads from the question's text.
|
|
870
|
+
handoff: What this step handed to another agent, checked against the
|
|
871
|
+
project's hand-off contracts: {"to": "adjudication", "fields":
|
|
872
|
+
{"onset_date": "2026-03-01", "leave_type": "STD"}, "message": "..."}.
|
|
873
|
+
"to" is required; fields are JSON values (money as {"amount",
|
|
874
|
+
"currency"}, records, lists) per the contract's types. "message" is the
|
|
875
|
+
hand-off as text, grounded in what this step had. An agent called as a
|
|
876
|
+
tool (tools_called) is checked without this.
|
|
849
877
|
retrieval_query: The query sent to your retriever/search.
|
|
850
878
|
If your pipeline rewrites queries, pass the rewritten version.
|
|
851
879
|
session_id: Conversation/thread ID — groups turns into sessions.
|
|
@@ -920,6 +948,61 @@ class VariablyClient:
|
|
|
920
948
|
evaluation_context["reference_materials"] = reference_materials
|
|
921
949
|
if retrieval_query:
|
|
922
950
|
evaluation_context["retrieval_query"] = retrieval_query
|
|
951
|
+
if expected_answer_kind is not None:
|
|
952
|
+
if expected_answer_kind not in self._ANSWER_KINDS:
|
|
953
|
+
raise ConfigurationError(f"expected_answer_kind must be one of {self._ANSWER_KINDS}", "expected_answer_kind")
|
|
954
|
+
evaluation_context["expected_answer_kind"] = expected_answer_kind
|
|
955
|
+
if required_checks is not None and requirements_ref is not None:
|
|
956
|
+
raise ConfigurationError("give required_checks or requirements_ref, not both", "required_checks")
|
|
957
|
+
if required_checks is not None:
|
|
958
|
+
if not isinstance(required_checks, list) or not all(
|
|
959
|
+
isinstance(r, dict) and isinstance(r.get("id"), str) and isinstance(r.get("text"), str)
|
|
960
|
+
for r in required_checks):
|
|
961
|
+
raise ConfigurationError("required_checks must be a list of {'id', 'text', 'critical', 'kind'} dicts",
|
|
962
|
+
"required_checks")
|
|
963
|
+
evaluation_context["required_checks"] = required_checks
|
|
964
|
+
if requirements_ref is not None:
|
|
965
|
+
if not isinstance(requirements_ref, str) or not requirements_ref.strip():
|
|
966
|
+
raise ConfigurationError("requirements_ref must be a template name, e.g. 'onboarding@v2'", "requirements_ref")
|
|
967
|
+
evaluation_context["requirements_ref"] = requirements_ref
|
|
968
|
+
if sql is not None:
|
|
969
|
+
evaluation_context["sql_queries"] = self._sql_queries(sql)
|
|
970
|
+
if expected_answer is not None:
|
|
971
|
+
if not isinstance(expected_answer, str) or not expected_answer.strip() or len(expected_answer) > 20_000:
|
|
972
|
+
raise ConfigurationError("expected_answer must be non-empty text of at most 20,000 characters",
|
|
973
|
+
"expected_answer")
|
|
974
|
+
evaluation_context["expected_answer"] = expected_answer
|
|
975
|
+
if handoff is not None:
|
|
976
|
+
handoff = json_safe(handoff, "handoff") if isinstance(handoff, dict) else handoff
|
|
977
|
+
if not isinstance(handoff, dict) or not isinstance(handoff.get("to"), str) or not handoff["to"].strip():
|
|
978
|
+
raise ConfigurationError("handoff must be a dict with a non-empty 'to' (the receiving agent)", "handoff")
|
|
979
|
+
unknown = set(handoff) - {"to", "fields", "message"}
|
|
980
|
+
if unknown:
|
|
981
|
+
raise ConfigurationError(f"handoff: unknown keys {sorted(unknown)}", "handoff")
|
|
982
|
+
fields = handoff.get("fields") or {}
|
|
983
|
+
|
|
984
|
+
def _json_value(v, depth=1):
|
|
985
|
+
if v is None or isinstance(v, (str, int, float, bool)):
|
|
986
|
+
return True
|
|
987
|
+
if depth > 4:
|
|
988
|
+
return False
|
|
989
|
+
if isinstance(v, list):
|
|
990
|
+
return all(_json_value(x, depth + 1) for x in v)
|
|
991
|
+
if isinstance(v, dict):
|
|
992
|
+
return all(isinstance(k, str) and k.strip() and _json_value(x, depth + 1) for k, x in v.items())
|
|
993
|
+
return False
|
|
994
|
+
|
|
995
|
+
if not isinstance(fields, dict) or not all(isinstance(k, str) and k.strip() and _json_value(v)
|
|
996
|
+
for k, v in fields.items()):
|
|
997
|
+
raise ConfigurationError("handoff.fields must map names to JSON values (records and lists "
|
|
998
|
+
"nest at most 4 deep)", "handoff")
|
|
999
|
+
if handoff.get("message") is not None and not isinstance(handoff["message"], str):
|
|
1000
|
+
raise ConfigurationError("handoff.message must be text", "handoff")
|
|
1001
|
+
evaluation_context["handoff"] = handoff
|
|
1002
|
+
if question_scope:
|
|
1003
|
+
if not isinstance(question_scope, dict):
|
|
1004
|
+
raise ConfigurationError("question_scope must be a dict", "question_scope")
|
|
1005
|
+
evaluation_context["question_scope"] = question_scope
|
|
923
1006
|
if evaluation_context:
|
|
924
1007
|
data["evaluation_context"] = evaluation_context
|
|
925
1008
|
|
|
@@ -939,6 +1022,10 @@ class VariablyClient:
|
|
|
939
1022
|
data["expected_tools"] = expected_tools
|
|
940
1023
|
if allowed_tools:
|
|
941
1024
|
data["allowed_tools"] = allowed_tools
|
|
1025
|
+
if max_steps is not None:
|
|
1026
|
+
if not isinstance(max_steps, int) or isinstance(max_steps, bool) or max_steps < 1:
|
|
1027
|
+
raise ValueError(f"max_steps must be a positive integer, got {max_steps!r}")
|
|
1028
|
+
data["max_steps"] = max_steps
|
|
942
1029
|
if provider:
|
|
943
1030
|
data["provider"] = provider
|
|
944
1031
|
if model:
|
|
@@ -956,6 +1043,9 @@ class VariablyClient:
|
|
|
956
1043
|
if kind:
|
|
957
1044
|
data["kind"] = kind
|
|
958
1045
|
|
|
1046
|
+
# Database values and rows (Decimal, dates, Databricks / DB-API rows) made JSON-safe here,
|
|
1047
|
+
# so a row that cannot be named is refused at the call (116 §15 item 26).
|
|
1048
|
+
data = json_safe(data)
|
|
959
1049
|
try:
|
|
960
1050
|
result = self.http_client.log_observation(data)
|
|
961
1051
|
return LogResult(
|
|
@@ -970,6 +1060,62 @@ class VariablyClient:
|
|
|
970
1060
|
)
|
|
971
1061
|
raise
|
|
972
1062
|
|
|
1063
|
+
# --- Check settings as code (versioned; see variably.settings) ---------------------------
|
|
1064
|
+
|
|
1065
|
+
def set_code_lists(self, experiment_id: str, lists: Any, note: Optional[str] = None) -> Dict[str, Any]:
|
|
1066
|
+
"""Save the experiment's code lists (a dict, or a JSON/YAML file path). {} removes them.
|
|
1067
|
+
Refused when a saved hand-off contract uses a list this removes."""
|
|
1068
|
+
from . import settings
|
|
1069
|
+
return settings.set_code_lists(self.http_client, experiment_id, lists, note)
|
|
1070
|
+
|
|
1071
|
+
def set_handoff_contracts(self, experiment_id: str, contracts: Any, note: Optional[str] = None) -> Dict[str, Any]:
|
|
1072
|
+
"""Save the experiment's hand-off contracts (a list, or a JSON/YAML file path), checked
|
|
1073
|
+
against its saved code lists. [] removes them."""
|
|
1074
|
+
from . import settings
|
|
1075
|
+
return settings.set_handoff_contracts(self.http_client, experiment_id, contracts, note)
|
|
1076
|
+
|
|
1077
|
+
# The server's limits on the SQL one answer sends (sql_settings.py).
|
|
1078
|
+
_MAX_SQL_QUERIES = 20
|
|
1079
|
+
_MAX_SQL_CHARS = 100_000
|
|
1080
|
+
|
|
1081
|
+
@classmethod
|
|
1082
|
+
def _sql_queries(cls, sql: Any) -> List[str]:
|
|
1083
|
+
"""The SQL text an answer ran, as a list. Only the text is sent: the server checks it
|
|
1084
|
+
against the project's sql_checks; the local check (variably.sql) is a developer-time
|
|
1085
|
+
warning and its result is never sent."""
|
|
1086
|
+
queries = [sql] if isinstance(sql, str) else sql
|
|
1087
|
+
if (not isinstance(queries, list) or not queries or len(queries) > cls._MAX_SQL_QUERIES
|
|
1088
|
+
or not all(isinstance(q, str) and q.strip() and len(q) <= cls._MAX_SQL_CHARS for q in queries)):
|
|
1089
|
+
raise ConfigurationError(
|
|
1090
|
+
f"sql must be the SQL text the answer ran, or a list of at most {cls._MAX_SQL_QUERIES} such texts "
|
|
1091
|
+
f"(each non-empty, at most {cls._MAX_SQL_CHARS} characters)", "sql")
|
|
1092
|
+
return list(queries)
|
|
1093
|
+
|
|
1094
|
+
def set_sql_checks(self, experiment_id: str, sql_checks: Any, note: Optional[str] = None) -> Dict[str, Any]:
|
|
1095
|
+
"""Save the experiment's SQL checks (a dict, or a JSON/YAML file path): {"dialect",
|
|
1096
|
+
"schema", "rules"}. The server checks each answer's SQL text against them — it never
|
|
1097
|
+
connects to your database; the schema is table and column names (and types), no data.
|
|
1098
|
+
A rule that names columns needs the schema. {} removes them."""
|
|
1099
|
+
from . import settings
|
|
1100
|
+
return settings.set_sql_checks(self.http_client, experiment_id, sql_checks, note)
|
|
1101
|
+
|
|
1102
|
+
def set_scope_defaults(self, experiment_id: str, defaults: Any, note: Optional[str] = None) -> Dict[str, Any]:
|
|
1103
|
+
"""Save the experiment's scope defaults (a dict, or a JSON/YAML file path): when plan
|
|
1104
|
+
and fiscal years start, which request metadata keys state the question's scope, and
|
|
1105
|
+
whether document headers are read. {} removes them."""
|
|
1106
|
+
from . import settings
|
|
1107
|
+
return settings.set_scope_defaults(self.http_client, experiment_id, defaults, note)
|
|
1108
|
+
|
|
1109
|
+
def settings_history(self, experiment_id: str, kind: Optional[str] = None, limit: int = 100) -> List[Dict[str, Any]]:
|
|
1110
|
+
"""Recorded versions of the experiment's check settings, newest first, without content."""
|
|
1111
|
+
from . import settings
|
|
1112
|
+
return settings.settings_history(self.http_client, experiment_id, kind, limit)
|
|
1113
|
+
|
|
1114
|
+
def settings_version(self, experiment_id: str, kind: str, version: int) -> Dict[str, Any]:
|
|
1115
|
+
"""One recorded version of a setting, with its content."""
|
|
1116
|
+
from . import settings
|
|
1117
|
+
return settings.settings_version(self.http_client, experiment_id, kind, version)
|
|
1118
|
+
|
|
973
1119
|
def clear_cache(self) -> None:
|
|
974
1120
|
"""Clear the cache."""
|
|
975
1121
|
self.cache.clear()
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
import json
|
|
4
4
|
import time
|
|
5
|
+
from ._json_safe import json_safe
|
|
5
6
|
from typing import Any, Dict, List, Optional
|
|
6
7
|
|
|
7
8
|
import requests
|
|
@@ -94,7 +95,7 @@ class HTTPClient:
|
|
|
94
95
|
session.headers.update({
|
|
95
96
|
"Content-Type": "application/json",
|
|
96
97
|
"X-API-Key": self.config.api_key,
|
|
97
|
-
"User-Agent": "Variably-Python-SDK/2.
|
|
98
|
+
"User-Agent": "Variably-Python-SDK/2.15.0",
|
|
98
99
|
})
|
|
99
100
|
|
|
100
101
|
return session
|
|
@@ -271,7 +272,7 @@ class HTTPClient:
|
|
|
271
272
|
|
|
272
273
|
response = self.session.post(
|
|
273
274
|
url,
|
|
274
|
-
json=request_data,
|
|
275
|
+
json=json_safe(request_data),
|
|
275
276
|
timeout=self.config.timeout,
|
|
276
277
|
stream=True,
|
|
277
278
|
)
|
|
@@ -299,6 +300,10 @@ class HTTPClient:
|
|
|
299
300
|
retryable=False,
|
|
300
301
|
)
|
|
301
302
|
|
|
303
|
+
def request_json(self, method: str, path: str, **kwargs: Any) -> Any:
|
|
304
|
+
"""A management call (settings, history): never retried, errors raised as-is."""
|
|
305
|
+
return self._make_request(method, path, retryable=False, **kwargs)
|
|
306
|
+
|
|
302
307
|
def _make_request(
|
|
303
308
|
self,
|
|
304
309
|
method: str,
|
|
@@ -309,6 +314,9 @@ class HTTPClient:
|
|
|
309
314
|
"""Make HTTP request with error handling and metrics."""
|
|
310
315
|
url = f"{self.config.base_url}{path}"
|
|
311
316
|
start_time = time.time()
|
|
317
|
+
if "json" in kwargs:
|
|
318
|
+
# Driver values (Decimal, dates, Row objects) never reach json.dumps raw (116 §15 item 26).
|
|
319
|
+
kwargs["json"] = json_safe(kwargs["json"])
|
|
312
320
|
|
|
313
321
|
try:
|
|
314
322
|
self.logger.debug(f"Making {method} request to {path}")
|
|
@@ -242,6 +242,11 @@ class VariablyPlugin(BasePlugin):
|
|
|
242
242
|
``{"researcher": [{"name": "search_policy"}]}`` — enables deterministic
|
|
243
243
|
tool-correctness / trajectory scoring against ground truth.
|
|
244
244
|
allowed_tools: optional allow-list per agent (tool-permission checks).
|
|
245
|
+
required_checks: optional requirements per agent — what its answer must address,
|
|
246
|
+
``{"advisor": [{"id": "kyc", "text": "Identity is verified", "critical": True}]}``
|
|
247
|
+
(requirement coverage). Or ``requirements_ref``: a stored template per agent,
|
|
248
|
+
``{"advisor": "onboarding@v2"}``. Not both for one agent. Not sent with
|
|
249
|
+
``capture_text=False`` (there is no answer text to check).
|
|
245
250
|
evidence_extractors: ``{tool_name: fn(result) -> [{"id","content","source"}]}``
|
|
246
251
|
for function-tool retrievers whose result shape isn't auto-detected.
|
|
247
252
|
capture_text: ``False`` sends structure only (no prompt/response/tool payload
|
|
@@ -255,6 +260,8 @@ class VariablyPlugin(BasePlugin):
|
|
|
255
260
|
*,
|
|
256
261
|
expected_tools: Optional[Dict[str, List[Dict[str, Any]]]] = None,
|
|
257
262
|
allowed_tools: Optional[Dict[str, List[str]]] = None,
|
|
263
|
+
required_checks: Optional[Dict[str, List[Dict[str, Any]]]] = None,
|
|
264
|
+
requirements_ref: Optional[Dict[str, str]] = None,
|
|
258
265
|
evidence_extractors: Optional[Dict[str, Callable[[Any], List[Dict[str, Any]]]]] = None,
|
|
259
266
|
capture_text: bool = True,
|
|
260
267
|
tags: Optional[List[str]] = None,
|
|
@@ -265,6 +272,11 @@ class VariablyPlugin(BasePlugin):
|
|
|
265
272
|
super().__init__(name=name)
|
|
266
273
|
self._expected_tools = expected_tools or {}
|
|
267
274
|
self._allowed_tools = allowed_tools or {}
|
|
275
|
+
self._required_checks = dict(required_checks or {})
|
|
276
|
+
self._requirements_ref = dict(requirements_ref or {})
|
|
277
|
+
both = sorted(set(self._required_checks) & set(self._requirements_ref))
|
|
278
|
+
if both:
|
|
279
|
+
raise ValueError(f"adk: give required_checks or requirements_ref per agent, not both: {both}")
|
|
268
280
|
self._extractors = evidence_extractors or {}
|
|
269
281
|
self._capture_text = capture_text
|
|
270
282
|
self._tags = list(tags or [])
|
|
@@ -397,6 +409,13 @@ class VariablyPlugin(BasePlugin):
|
|
|
397
409
|
)
|
|
398
410
|
if kind:
|
|
399
411
|
payload["kind"] = kind
|
|
412
|
+
reqs, ref = self._required_checks.get(step.agent_name), self._requirements_ref.get(step.agent_name)
|
|
413
|
+
if reqs is not None or ref is not None:
|
|
414
|
+
if self._capture_text:
|
|
415
|
+
payload["required_checks"], payload["requirements_ref"] = reqs, ref
|
|
416
|
+
else:
|
|
417
|
+
self._warn_once("requirements_no_text",
|
|
418
|
+
"adk: requirements are not checked with capture_text=False (no answer text is sent)")
|
|
400
419
|
self._sender.put(payload)
|
|
401
420
|
|
|
402
421
|
def _finish(self, invocation_id: str, error: Optional[str] = None) -> None:
|
|
@@ -490,6 +509,8 @@ class VariablyPlugin(BasePlugin):
|
|
|
490
509
|
entry: Dict[str, Any] = {"name": name, "arguments": _args(tool_args)}
|
|
491
510
|
if error is not None:
|
|
492
511
|
entry["output"] = f"ERROR: {type(error).__name__}: {error}"[:500]
|
|
512
|
+
# Structured: a failed call is a wasted step (never read from output text).
|
|
513
|
+
entry["error"] = f"{type(error).__name__}: {error}"[:500]
|
|
493
514
|
else:
|
|
494
515
|
entry["output"] = _text_of(result)
|
|
495
516
|
extractor = self._extractors.get(name)
|