variably-sdk 2.8.0__tar.gz → 2.9.1__tar.gz

{variably_sdk-2.8.0 → variably_sdk-2.9.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: variably-sdk
-Version: 2.8.0
+Version: 2.9.1
 Summary: Official Python SDK for Variably feature flags, LLM experimentation, and prompt optimization platform
 Author: Variably
 Author-email: Variably <support@variably.com>

{variably_sdk-2.8.0 → variably_sdk-2.9.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "variably-sdk"
-version = "2.8.0"
+version = "2.9.1"
 description = "Official Python SDK for Variably feature flags, LLM experimentation, and prompt optimization platform"
 readme = "README.md"
 requires-python = ">=3.7"

{variably_sdk-2.8.0 → variably_sdk-2.9.1}/src/variably/client.py

@@ -554,6 +554,20 @@ class VariablyClient:
 
         return StreamingEvaluateResponse(raw_response)
 
+    # Allowed values for the `kind` parameter on submit_response. Marks
+    # what type of LLM turn this is so the dashboard can suppress
+    # grounding/hallucination/attribution scoring on turns where those
+    # dimensions don't apply (router, classifier, tool-use). Default
+    # "generative" preserves existing behavior for callers not yet on
+    # this version.
+    _SUBMIT_RESPONSE_KINDS = (
+        "generative",   # default — narrative answer; full 41D scoring
+        "classifier",   # intent / safety / routing decision; no claims
+        "router",       # workflow control turn; no claims
+        "tool_use",     # function/tool call; no narrative response
+        "transformer",  # query rewrite / expansion / summarization step
+    )
+
     def submit_response(
         self,
         experiment_key: str,
@@ -570,6 +584,7 @@ class VariablyClient:
         evaluation_context: Optional[Dict[str, Any]] = None,
         tags: Optional[List[str]] = None,
         metadata: Optional[Dict[str, Any]] = None,
+        kind: Optional[str] = None,
     ) -> SubmitResponseResult:
         """Submit your own LLM response for 41-dimensional evaluation (BYOR step 3).
 
@@ -577,6 +592,22 @@ class VariablyClient:
             tags: Grouping labels for filtering (e.g. ["production", "rag-v2"]).
             metadata: Arbitrary key-value data attached to this evaluation
                 (e.g. {"customer_id": "...", "release": "..."}).
+            kind: What type of LLM turn this is. One of:
+                - "generative" (default) — a narrative answer with claims
+                  to ground. Full 41-dimensional scoring applies.
+                - "classifier" — an intent / safety / routing decision
+                  (e.g. {"is_relevant": true, "should_generate": true}).
+                  No factual claims, so grounding/hallucination/attribution
+                  scores will be suppressed in dashboards.
+                - "router" — a workflow control turn (chooses next node).
+                - "tool_use" — a function/tool call with no narrative.
+                - "transformer" — a query rewrite / expansion / summary
+                  step that's part of the pipeline, not the answer.
+
+                When non-generative kinds are passed, the SDK injects the
+                kind into tags (as `kind:<value>`) and metadata so the
+                dashboard can filter and skip grounding scores. Latency,
+                token, and variant-level analytics still work.
         """
         if not experiment_key or not isinstance(experiment_key, str):
             raise ConfigurationError(
@@ -586,6 +617,22 @@ class VariablyClient:
             raise ConfigurationError(
                 "Variant key must be a non-empty string", "variant_key"
             )
+        if kind is not None and kind not in self._SUBMIT_RESPONSE_KINDS:
+            raise ConfigurationError(
+                f"kind must be one of {self._SUBMIT_RESPONSE_KINDS}, got {kind!r}",
+                "kind",
+            )
+
+        # Inject kind into tags + metadata so the dashboard can filter
+        # without needing a backend deploy. Preserves any caller-supplied
+        # values; we only add, never overwrite.
+        if kind and kind != "generative":
+            kind_tag = f"kind:{kind}"
+            tags = list(tags) if tags else []
+            if kind_tag not in tags:
+                tags.append(kind_tag)
+            metadata = dict(metadata) if metadata else {}
+            metadata.setdefault("kind", kind)
 
         data: Dict[str, Any] = {
             "experiment_key": experiment_key,
@@ -613,6 +660,8 @@ class VariablyClient:
             data["tags"] = tags
         if metadata:
             data["metadata"] = metadata
+        if kind:
+            data["kind"] = kind
 
         result = self.http_client.submit_response(data)
 
@@ -640,6 +689,8 @@ class VariablyClient:
         prompt_tokens: Optional[int] = None,       # Input token count
         completion_tokens: Optional[int] = None,   # Output token count
         cost: Optional[float] = None,              # Cost in USD
+        # Turn-type marker — see _SUBMIT_RESPONSE_KINDS for allowed values.
+        kind: Optional[str] = None,
     ) -> LogResult:
         """Log a prompt/response pair for multi-dimensional evaluation (Observe Mode).
 
@@ -673,6 +724,23 @@ class VariablyClient:
         # Normalize prompt to string
         prompt_str = prompt if isinstance(prompt, str) else str(prompt)
 
+        # Validate kind and inject the marker into tags + metadata so the
+        # dashboard can suppress grounding/hallucination/attribution
+        # scores on non-generative turns. See submit_response for the
+        # full rationale; the same logic applies in Observe Mode.
+        if kind is not None and kind not in self._SUBMIT_RESPONSE_KINDS:
+            raise ConfigurationError(
+                f"kind must be one of {self._SUBMIT_RESPONSE_KINDS}, got {kind!r}",
+                "kind",
+            )
+        if kind and kind != "generative":
+            kind_tag = f"kind:{kind}"
+            tags = list(tags) if tags else []
+            if kind_tag not in tags:
+                tags.append(kind_tag)
+            metadata = dict(metadata) if metadata else {}
+            metadata.setdefault("kind", kind)
+
         data: Dict[str, Any] = {
             "prompt": prompt_str,
             "response": response,
@@ -709,6 +777,8 @@ class VariablyClient:
             data["cost"] = cost
         if metadata:
             data["metadata"] = metadata
+        if kind:
+            data["kind"] = kind
 
         try:
             result = self.http_client.log_observation(data)

{variably_sdk-2.8.0 → variably_sdk-2.9.1}/src/variably/observe.py

@@ -177,6 +177,15 @@ def observe(
     tags: Optional[List[str]] = None,         # Grouping labels, e.g. ["production", "rag", "v2"]
     metadata: Optional[Dict[str, Any]] = None,  # Any arbitrary key-value data for filtering
 
+    # --- Turn type ---
+    # Marks what kind of LLM turn this is. Use "classifier" / "router" /
+    # "tool_use" / "transformer" for non-generative turns so the dashboard
+    # suppresses grounding/hallucination/attribution scores (which don't
+    # apply to JSON control structures or function calls). Default
+    # "generative" preserves existing behavior.
+    # Allowed: "generative" (default), "classifier", "router", "tool_use", "transformer".
+    kind: Optional[str] = None,
+
     # --- Backward compatibility ---
     conversation_history: Optional[List[Dict[str, str]]] = None,  # Deprecated: use prior_turns instead
 ) -> Optional[Dict[str, Any]]:
@@ -263,6 +272,7 @@ def observe(
             user_id=user_id,
             session_id=session_id,
             metadata=metadata,
+            kind=kind,
         )
 
         return {

{variably_sdk-2.8.0 → variably_sdk-2.9.1}/src/variably/version.py

@@ -1,3 +1,3 @@
 """Version information for Variably SDK."""
 
-__version__ = "2.8.0"
+__version__ = "2.9.1"

{variably_sdk-2.8.0 → variably_sdk-2.9.1}/src/variably_sdk.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: variably-sdk
-Version: 2.8.0
+Version: 2.9.1
 Summary: Official Python SDK for Variably feature flags, LLM experimentation, and prompt optimization platform
 Author: Variably
 Author-email: Variably <support@variably.com>