tobeverified 0.5.0__tar.gz

tobeverified-0.5.0/PKG-INFO

@@ -0,0 +1,103 @@
+Metadata-Version: 2.4
+Name: tobeverified
+Version: 0.5.0
+Summary: Runtime verification for non-deterministic LLM outputs — Python SDK
+Project-URL: Homepage, https://tobeverified.com
+Project-URL: Documentation, https://docs.tobeverified.com
+Project-URL: Repository, https://github.com/pvinjamuri/gitproduct
+Project-URL: Issues, https://github.com/pvinjamuri/gitproduct/issues
+Author-email: Prasanna Vinjamuri <pvinjamuri@gmail.com>
+License-Expression: MIT
+Keywords: ai-safety,evals,llm,model-validation,tbv,tobeverified,verification
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.10
+Requires-Dist: httpx>=0.24
+Description-Content-Type: text/markdown
+
+# tobeverified — Python SDK
+
+Runtime verification for non-deterministic LLM outputs.
+
+`tobeverified` submits each AI-generated decision to an independent verifier (LLM judge with policy + low-confidence routing to a human reviewer). Use it where unit tests don't reach: refund classifications, fraud screens, PII redaction, intent routing, tool-call validation.
+
+## Install
+
+```bash
+pip install tobeverified
+```
+
+## Use
+
+```python
+from tobeverified import TBV
+
+tbv = TBV()  # reads TBV_API_KEY and TBV_BASE_URL from env
+
+result = tbv.verify(
+    prompt="Is this customer message a refund request?",
+    context={"message": "I'd like my money back."},
+    allowed_verdicts=["yes", "no"],
+    confidence_threshold=0.85,
+)
+
+print(result["final_verdict"], result["final_tier"])
+# → 'yes' 'agent'   (or 'human' if confidence was below threshold)
+```
+
+`verify()` is fire-and-wait: submits, polls until terminal state. For non-blocking flows use `submit()` + `wait()` separately.
+
+## Decorator pattern
+
+```python
+from tobeverified import verified
+
+@verified(
+    allowed_verdicts=["safe", "unsafe", "needs_review"],
+    confidence_threshold=0.9,
+    source="my-app/comment-mod",
+)
+def moderate(text: str) -> dict:
+    return {
+        "prompt": "Is this comment safe to publish?",
+        "context": {"text": text},
+    }
+
+result = moderate("user-supplied content")
+# result.verdict, result.tier, result.output
+```
+
+## Get an API key
+
+Sign in at [app.tobeverified.com](https://app.tobeverified.com) → **API keys** → create. Plaintext is shown exactly once.
+
+```bash
+export TBV_API_KEY=tbv_live_...
+```
+
+## Reviewer queue
+
+Tasks routed to `human_needed` (confidence below threshold) appear at [review.tobeverified.com](https://review.tobeverified.com) for the account that submitted them. Anyone signed in to the account can resolve. The agent's first-pass verdict, confidence, and rationale are shown alongside the verdict form.
+
+## Reference
+
+| Method | Purpose |
+|---|---|
+| `TBV(base_url=None, api_key=None, timeout=30.0)` | Construct a client. Defaults read `TBV_BASE_URL` and `TBV_API_KEY` from env. |
+| `submit(prompt, allowed_verdicts, ...)` | Enqueue a verification task; returns `{id, status, ...}`. |
+| `get(task_id)` | Fetch current state. |
+| `wait(task_id, timeout=300.0, poll_interval=1.5)` | Block until terminal state. |
+| `verify(prompt, allowed_verdicts, ...)` | `submit()` + `wait()` in one call. |
+
+Errors raise `TBVError` (or `TBVTimeoutError` for `wait()` timeouts). Both inherit from `Exception`.
+
+## Status
+
+Developer preview. Versioned `0.x` until the API contract is frozen; expect minor surface changes. Production deployments tracked via [docs.tobeverified.com](https://docs.tobeverified.com).

tobeverified-0.5.0/pyproject.toml

@@ -0,0 +1,52 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "tobeverified"
+version = "0.5.0"
+description = "Runtime verification for non-deterministic LLM outputs — Python SDK"
+readme = "tobeverified/README.md"
+requires-python = ">=3.10"
+license = "MIT"
+authors = [
+  { name = "Prasanna Vinjamuri", email = "pvinjamuri@gmail.com" }
+]
+keywords = [
+  "llm",
+  "verification",
+  "evals",
+  "ai-safety",
+  "model-validation",
+  "tobeverified",
+  "tbv",
+]
+classifiers = [
+  "Development Status :: 4 - Beta",
+  "Intended Audience :: Developers",
+  "License :: OSI Approved :: MIT License",
+  "Operating System :: OS Independent",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Topic :: Software Development :: Libraries :: Python Modules",
+]
+dependencies = [
+  "httpx>=0.24",
+]
+
+[project.urls]
+Homepage = "https://tobeverified.com"
+Documentation = "https://docs.tobeverified.com"
+Repository = "https://github.com/pvinjamuri/gitproduct"
+Issues = "https://github.com/pvinjamuri/gitproduct/issues"
+
+[tool.hatch.build.targets.wheel]
+packages = ["tobeverified"]
+
+[tool.hatch.build.targets.sdist]
+include = [
+  "tobeverified/**",
+  "pyproject.toml",
+]

tobeverified-0.5.0/tobeverified/README.md

@@ -0,0 +1,79 @@
+# tobeverified — Python SDK
+
+Runtime verification for non-deterministic LLM outputs.
+
+`tobeverified` submits each AI-generated decision to an independent verifier (LLM judge with policy + low-confidence routing to a human reviewer). Use it where unit tests don't reach: refund classifications, fraud screens, PII redaction, intent routing, tool-call validation.
+
+## Install
+
+```bash
+pip install tobeverified
+```
+
+## Use
+
+```python
+from tobeverified import TBV
+
+tbv = TBV()  # reads TBV_API_KEY and TBV_BASE_URL from env
+
+result = tbv.verify(
+    prompt="Is this customer message a refund request?",
+    context={"message": "I'd like my money back."},
+    allowed_verdicts=["yes", "no"],
+    confidence_threshold=0.85,
+)
+
+print(result["final_verdict"], result["final_tier"])
+# → 'yes' 'agent'   (or 'human' if confidence was below threshold)
+```
+
+`verify()` is fire-and-wait: submits, polls until terminal state. For non-blocking flows use `submit()` + `wait()` separately.
+
+## Decorator pattern
+
+```python
+from tobeverified import verified
+
+@verified(
+    allowed_verdicts=["safe", "unsafe", "needs_review"],
+    confidence_threshold=0.9,
+    source="my-app/comment-mod",
+)
+def moderate(text: str) -> dict:
+    return {
+        "prompt": "Is this comment safe to publish?",
+        "context": {"text": text},
+    }
+
+result = moderate("user-supplied content")
+# result.verdict, result.tier, result.output
+```
+
+## Get an API key
+
+Sign in at [app.tobeverified.com](https://app.tobeverified.com) → **API keys** → create. Plaintext is shown exactly once.
+
+```bash
+export TBV_API_KEY=tbv_live_...
+```
+
+## Reviewer queue
+
+Tasks routed to `human_needed` (confidence below threshold) appear at [review.tobeverified.com](https://review.tobeverified.com) for the account that submitted them. Anyone signed in to the account can resolve. The agent's first-pass verdict, confidence, and rationale are shown alongside the verdict form.
+
+## Reference
+
+| Method | Purpose |
+|---|---|
+| `TBV(base_url=None, api_key=None, timeout=30.0)` | Construct a client. Defaults read `TBV_BASE_URL` and `TBV_API_KEY` from env. |
+| `submit(prompt, allowed_verdicts, ...)` | Enqueue a verification task; returns `{id, status, ...}`. |
+| `get(task_id)` | Fetch current state. |
+| `wait(task_id, timeout=300.0, poll_interval=1.5)` | Block until terminal state. |
+| `verify(prompt, allowed_verdicts, ...)` | `submit()` + `wait()` in one call. |
+
+Errors raise `TBVError` (or `TBVTimeoutError` for `wait()` timeouts). Both inherit from `Exception`.
+
+## Status
+
+Developer preview. Versioned `0.x` until the API contract is frozen; expect minor surface changes. Production deployments tracked via [docs.tobeverified.com](https://docs.tobeverified.com).

tobeverified-0.5.0/tobeverified/__init__.py

@@ -0,0 +1,6 @@
+from .client import TBV, TBVError, TBVTimeoutError
+from .decorator import verified
+
+__version__ = "0.5.0"
+
+__all__ = ["TBV", "TBVError", "TBVTimeoutError", "verified", "__version__"]

tobeverified-0.5.0/tobeverified/client.py

@@ -0,0 +1,103 @@
+import os
+import time
+from typing import Any
+
+import httpx
+
+
+class TBVError(Exception):
+    pass
+
+
+class TBVTimeoutError(TBVError):
+    pass
+
+
+class TBV:
+    def __init__(
+        self,
+        base_url: str | None = None,
+        api_key: str | None = None,
+        timeout: float = 30.0,
+    ) -> None:
+        self.base_url = (
+            base_url or os.getenv("TBV_BASE_URL", "https://api.tobeverified.com")
+        ).rstrip("/")
+        self.api_key = api_key or os.getenv("TBV_API_KEY")
+        self._http = httpx.Client(timeout=timeout)
+
+    def _headers(self) -> dict[str, str]:
+        h: dict[str, str] = {}
+        if self.api_key:
+            h["authorization"] = f"Bearer {self.api_key}"
+        return h
+
+    def submit(
+        self,
+        prompt: str,
+        allowed_verdicts: list[str],
+        context: dict[str, Any] | None = None,
+        task_type: str = "decide",
+        confidence_threshold: float = 0.8,
+        touchpoint: str = "chat",
+        source: str = "sdk",
+    ) -> dict[str, Any]:
+        resp = self._http.post(
+            f"{self.base_url}/v1/tasks",
+            headers=self._headers(),
+            json={
+                "task_type": task_type,
+                "prompt": prompt,
+                "context": context or {},
+                "allowed_verdicts": allowed_verdicts,
+                "confidence_threshold": confidence_threshold,
+                "touchpoint": touchpoint,
+                "source": source,
+            },
+        )
+        if resp.status_code >= 400:
+            raise TBVError(f"submit failed {resp.status_code}: {resp.text}")
+        return resp.json()
+
+    def get(self, task_id: str) -> dict[str, Any]:
+        resp = self._http.get(
+            f"{self.base_url}/v1/tasks/{task_id}",
+            headers=self._headers(),
+        )
+        if resp.status_code >= 400:
+            raise TBVError(f"get failed {resp.status_code}: {resp.text}")
+        return resp.json()
+
+    def wait(
+        self,
+        task_id: str,
+        timeout: float = 300.0,
+        poll_interval: float = 1.5,
+    ) -> dict[str, Any]:
+        deadline = time.time() + timeout
+        while time.time() < deadline:
+            task = self.get(task_id)
+            if task["status"] == "completed":
+                return task
+            time.sleep(poll_interval)
+        raise TBVTimeoutError(f"task {task_id} did not complete within {timeout}s")
+
+    def verify(
+        self,
+        prompt: str,
+        allowed_verdicts: list[str],
+        context: dict[str, Any] | None = None,
+        confidence_threshold: float = 0.8,
+        touchpoint: str = "chat",
+        source: str = "sdk",
+        timeout: float = 300.0,
+    ) -> dict[str, Any]:
+        submitted = self.submit(
+            prompt=prompt,
+            allowed_verdicts=allowed_verdicts,
+            context=context,
+            confidence_threshold=confidence_threshold,
+            touchpoint=touchpoint,
+            source=source,
+        )
+        return self.wait(submitted["id"], timeout=timeout)

tobeverified-0.5.0/tobeverified/decorator.py

@@ -0,0 +1,56 @@
+import functools
+from typing import Any, Callable
+
+from .client import TBV
+
+
+def verified(
+    allowed_verdicts: list[str],
+    confidence_threshold: float = 0.8,
+    task_type: str = "decide",
+    touchpoint: str = "chat",
+    source: str | None = None,
+    base_url: str | None = None,
+    timeout: float = 300.0,
+) -> Callable[..., Any]:
+    """Wrap a function so its output is verified by TBV before being returned.
+
+    The wrapped function must return a dict containing at least a "prompt"
+    key. Any other keys are sent as context. The decorator submits a
+    verification task, waits for completion, and returns the final verdict
+    string plus the tier that produced it.
+
+    `touchpoint` and `source` route the event in the Control Tower stream.
+    `source` defaults to the wrapped function's qualified name.
+    """
+
+    def decorator(fn: Callable[..., dict[str, Any]]) -> Callable[..., dict[str, Any]]:
+        client = TBV(base_url=base_url)
+        resolved_source = source or fn.__qualname__
+
+        @functools.wraps(fn)
+        def wrapper(*args: Any, **kwargs: Any) -> dict[str, Any]:
+            payload = fn(*args, **kwargs)
+            if not isinstance(payload, dict) or "prompt" not in payload:
+                raise TypeError(
+                    "@verified function must return a dict containing a 'prompt' key"
+                )
+            prompt = payload.pop("prompt")
+            task = client.verify(
+                prompt=prompt,
+                allowed_verdicts=allowed_verdicts,
+                context=payload,
+                confidence_threshold=confidence_threshold,
+                touchpoint=touchpoint,
+                source=resolved_source,
+                timeout=timeout,
+            )
+            return {
+                "verdict": task["final_verdict"],
+                "tier": task["final_tier"],
+                "task_id": task["id"],
+            }
+
+        return wrapper
+
+    return decorator