provekit 0.7.0__tar.gz → 0.7.1__tar.gz

{provekit-0.7.0 → provekit-0.7.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: provekit
-Version: 0.7.0
+Version: 0.7.1
 Summary: Drop-in agent tracing — add one decorator, get a project key, review every run your agent makes.
 Author: ProveKit contributors
 License: MIT

{provekit-0.7.0 → provekit-0.7.1}/provekit/eval.py

@@ -18,6 +18,8 @@ import json
 
 import httpx
 
+from .trace import _no_self_trace
+
 from . import scorers as _scorers
 from . import trace as _trace
 
@@ -25,24 +27,40 @@ _TIMEOUT = 60
 
 
 def _get(endpoint, headers, path):
-    r = httpx.get(f"{endpoint}{path}", headers=headers, timeout=_TIMEOUT, follow_redirects=False)
+    with _no_self_trace():
+        r = httpx.get(f"{endpoint}{path}", headers=headers, timeout=_TIMEOUT, follow_redirects=False)
     r.raise_for_status()
     return r.json()
 
 
 def _post(endpoint, headers, path, body):
-    r = httpx.post(f"{endpoint}{path}", headers=headers, json=body, timeout=_TIMEOUT, follow_redirects=False)
+    with _no_self_trace():
+        r = httpx.post(f"{endpoint}{path}", headers=headers, json=body, timeout=_TIMEOUT,
+                       follow_redirects=False)
     r.raise_for_status()
     return r.json()
 
 
 def _resolve_dataset_id(endpoint, headers, dataset) -> int:
+    """Accept an id or a name — and an id that arrived as a string.
+
+    The id is a number everywhere it is shown (the portal, `provekit datasets list`), and the
+    two places it is most often read from — an environment variable and a CLI argument — hand
+    it back as text. `evaluate("2", …)` then fell through to the name lookup and reported
+    `dataset '2' not found`, which is true and completely misleading: the dataset exists, and
+    nothing in the message suggests the type is what is wrong.
+    """
     if isinstance(dataset, int):
         return dataset
+    if isinstance(dataset, str) and dataset.strip().isdigit():
+        return int(dataset.strip())
+    names = []
     for d in _get(endpoint, headers, "/v1/datasets"):
         if d.get("name") == dataset:
             return d["id"]
-    raise ValueError(f"dataset {dataset!r} not found")
+        names.append(d.get("name"))
+    raise ValueError(
+        f"dataset {dataset!r} not found — this project has: {', '.join(map(str, names)) or 'none'}")
 
 
 def evaluate(dataset, target, scorers, *, name: str = "experiment") -> dict:

{provekit-0.7.0 → provekit-0.7.1}/provekit/services/calibration.py

@@ -225,10 +225,11 @@ def calibrate(rows, *, pass_at: float = PASS_AT, judge_name: str = "", human_nam
         # label more. The rates are the part that would get quoted as a credibility claim, so
         # they are withheld rather than published with a disclaimer nobody carries forward.
         out["caution"] = (
-            f"{n} trace(s) carry both a human label and a judge score — fewer than "
-            f"{MIN_LABELLED_N}, so no agreement rate, kappa or verdict is reported. A "
-            f"calibration number from a handful of labels is worse than none: it gets quoted. "
-            f"Label more traces in the portal.")
+            f"Only {n} run{'' if n == 1 else 's'} {'has' if n == 1 else 'have'} been judged both "
+            f"ways — by you and by the scorer. We need at least {MIN_LABELLED_N} before the "
+            f"numbers mean anything. A score from a handful of examples looks solid and is not, "
+            f"and once someone quotes it the damage is done — so we show nothing instead. "
+            f"Judge a few more runs in the Review queue.")
         return out
 
     out["agreement"] = (a + d) / n

{provekit-0.7.0 → provekit-0.7.1}/provekit/trace.py

@@ -160,6 +160,33 @@ def _span_to_otlp(span) -> dict:
     return out
 
 
+@contextlib.contextmanager
+def _no_self_trace():
+    """Don't instrument ProveKit's own HTTP calls.
+
+    `provekit[http]` instruments httpx, and the SDK talks to the portal over httpx — so
+    exporting a batch, fetching a prompt and posting a score were each captured as spans, which
+    were then exported, which produced more spans. They arrive parentless, so they land in the
+    trace list as top-level runs named "GET" and "POST" carrying no model, no tokens and no
+    input: in a first real run of the demo agent, 19 of 24 traces were this noise.
+
+    It is worse than clutter. Those rows are counted: they drag the error rate and p95 toward
+    the latency of ProveKit's own API, and they push real runs off the first page of a list
+    people scan by eye.
+
+    Best-effort by design — the helper only exists once an instrumentation package is
+    installed, and tracing must never be the thing that breaks a customer's process, so a
+    missing import degrades to "no suppression" rather than raising.
+    """
+    try:
+        from opentelemetry.instrumentation.utils import suppress_instrumentation
+    except Exception:                                     # noqa: BLE001 — optional dependency
+        yield
+        return
+    with suppress_instrumentation():
+        yield
+
+
 class _ProveKitExporter:
     """A minimal OTel SpanExporter that ships spans to ProveKit's /v1/traces as OTLP-JSON.
 
@@ -179,8 +206,9 @@ class _ProveKitExporter:
         by never looking — silently drops exactly the spans the retry path exists for.
         """
         import httpx
-        r = httpx.post(self._url, json=body, headers=self._headers, timeout=5,
-                       follow_redirects=False)
+        with _no_self_trace():
+            r = httpx.post(self._url, json=body, headers=self._headers, timeout=5,
+                           follow_redirects=False)
         if r.status_code >= 400:
             log.debug("provekit trace export rejected: HTTP %s", r.status_code)
             return False
@@ -453,9 +481,10 @@ def get_prompt(name: str, *, label: str = "", key: str = "", default=None) -> di
         return default
     try:
         import httpx
-        r = httpx.get(f"{endpoint}/v1/prompts/{name}",
-                      params={"label": label, "key": key},
-                      headers={"Authorization": f"Bearer {api_key}"}, timeout=5)
+        with _no_self_trace():
+            r = httpx.get(f"{endpoint}/v1/prompts/{name}",
+                          params={"label": label, "key": key},
+                          headers={"Authorization": f"Bearer {api_key}"}, timeout=5)
         if r.status_code >= 400:
             log.debug("provekit: prompt %r fetch returned %s", name, r.status_code)
             return default
@@ -499,8 +528,10 @@ def score(name: str, *, score: float | None = None, value: str | None = None,
     body = {"name": name, "score": score, "value": value, "comment": comment, "source": "sdk"}
     try:
         import httpx
-        httpx.post(f"{_endpoint}/v1/traces/{trace_id}/feedback", json=body,
-                   headers={"Authorization": f"Bearer {_api_key}"}, timeout=5, follow_redirects=False)
+        with _no_self_trace():
+            httpx.post(f"{_endpoint}/v1/traces/{trace_id}/feedback", json=body,
+                       headers={"Authorization": f"Bearer {_api_key}"}, timeout=5,
+                       follow_redirects=False)
         return True
     except Exception as exc:  # never let a feedback post surface into the user's app
         log.debug("provekit score post failed: %s", exc)

{provekit-0.7.0 → provekit-0.7.1}/provekit.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: provekit
-Version: 0.7.0
+Version: 0.7.1
 Summary: Drop-in agent tracing — add one decorator, get a project key, review every run your agent makes.
 Author: ProveKit contributors
 License: MIT

{provekit-0.7.0 → provekit-0.7.1}/provekit.egg-info/SOURCES.txt

@@ -195,6 +195,7 @@ tests/test_scim.py
 tests/test_scorers.py
 tests/test_scorers_extra.py
 tests/test_search.py
+tests/test_self_trace.py
 tests/test_share.py
 tests/test_share_redaction.py
 tests/test_spool.py

{provekit-0.7.0 → provekit-0.7.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "provekit"
-version = "0.7.0"
+version = "0.7.1"
 description = "Drop-in agent tracing — add one decorator, get a project key, review every run your agent makes."
 requires-python = ">=3.11"
 readme = "README.md"

provekit-0.7.1/tests/test_self_trace.py

@@ -0,0 +1,86 @@
+"""The SDK must not trace its own calls to the portal (#noise).
+
+`provekit[http]` instruments httpx, and the SDK talks to the portal over httpx. Without
+suppression, exporting a batch, fetching a prompt and posting a score each produced their own
+parentless span, which arrived in the customer's trace list as a top-level run named "GET" or
+"POST" with no model, no tokens and no input. In a first real run of the bundled demo agent,
+19 of 24 traces were this. They are also counted, so they drag error rate and p95 toward the
+latency of ProveKit's own API.
+"""
+from provekit import trace as pk_trace
+
+
+def test_the_suppression_helper_is_a_context_manager():
+    with pk_trace._no_self_trace():
+        pass
+
+
+def test_it_degrades_to_a_no_op_without_the_instrumentation_package(monkeypatch):
+    """Tracing must never be the thing that breaks a customer's process, so a missing optional
+    import has to mean "no suppression", not an exception."""
+    import builtins
+    real = builtins.__import__
+
+    def _no_otel_utils(name, *a, **kw):
+        if name == "opentelemetry.instrumentation.utils":
+            raise ImportError("not installed")
+        return real(name, *a, **kw)
+
+    monkeypatch.setattr(builtins, "__import__", _no_otel_utils)
+    with pk_trace._no_self_trace():
+        pass                                  # did not raise
+
+
+def test_every_sdk_call_to_the_portal_is_wrapped():
+    """A drift guard. A new httpx call added to the SDK without suppression re-opens the bug,
+    and the symptom (junk traces in someone else's portal) never shows up in our own tests."""
+    import inspect
+    import re
+
+    from provekit import eval as pk_eval
+
+    for mod in (pk_trace, pk_eval):
+        src = inspect.getsource(mod)
+        for m in re.finditer(r'^(\s*)(?:r = )?httpx\.(get|post)\(', src, re.M):
+            line_no = src[:m.start()].count("\n") + 1
+            before = src[:m.start()].rsplit("\n", 3)[-3:]
+            assert any("_no_self_trace()" in b for b in before), (
+                f"{mod.__name__}:{line_no} calls httpx.{m.group(2)} outside _no_self_trace() — "
+                "it will appear as a junk trace in the customer's own portal")
+
+
+# ---------------------------------------------------------------- dataset id as a string
+
+def test_a_dataset_id_that_arrived_as_text_still_resolves():
+    """An id read from os.environ or argv is a str. It used to fall through to the name lookup
+    and report `dataset '2' not found` — true, and entirely misleading, because the dataset is
+    right there and the type is what was wrong."""
+    from provekit.eval import _resolve_dataset_id
+
+    def _never_called(*a, **kw):
+        raise AssertionError("a numeric id must not need a network lookup")
+
+    import provekit.eval as ev
+    real = ev._get
+    ev._get = _never_called
+    try:
+        assert _resolve_dataset_id("http://x", {}, "2") == 2
+        assert _resolve_dataset_id("http://x", {}, " 7 ") == 7
+        assert _resolve_dataset_id("http://x", {}, 3) == 3
+    finally:
+        ev._get = real
+
+
+def test_an_unknown_name_says_what_the_project_does_have():
+    """"not found" alone leaves you guessing whether the name is wrong, the project is wrong,
+    or the key is. Listing what is actually there settles it in one line."""
+    import pytest
+
+    import provekit.eval as ev
+    real = ev._get
+    ev._get = lambda *a, **kw: [{"id": 1, "name": "support-qa"}, {"id": 2, "name": "billing"}]
+    try:
+        with pytest.raises(ValueError, match="support-qa, billing"):
+            ev._resolve_dataset_id("http://x", {}, "typo-name")
+    finally:
+        ev._get = real