tracecast 0.2.1__tar.gz → 0.2.3__tar.gz

{tracecast-0.2.1/tracecast.egg-info → tracecast-0.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tracecast
-Version: 0.2.1
+Version: 0.2.3
 Summary: LLM observability SDK — framework-agnostic tracing for AI applications
 Author: Pedro Castanheira Costa
 License: MIT

{tracecast-0.2.1 → tracecast-0.2.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "tracecast"
-version = "0.2.1"
+version = "0.2.3"
 description = "LLM observability SDK — framework-agnostic tracing for AI applications"
 readme = "README.md"
 license = { text = "MIT" }

tracecast-0.2.3/tests/test_prompt_dashboard.py

@@ -0,0 +1,56 @@
+import pytest
+
+from tracecast import Tracer
+from tracecast.prompts import create_prompt
+from tracecast.prompts.client import clear_cache
+from tracecast.exporters.dict_exporter import DictExporter
+
+fastapi = pytest.importorskip("fastapi")
+from fastapi import FastAPI
+from fastapi.testclient import TestClient
+
+
+def _client(exporter):
+    tracer = Tracer(exporters=[exporter])
+    app = FastAPI()
+    tracer.mount(app, prefix="/tc")
+    return TestClient(app)
+
+
+def test_list_and_detail_prompts():
+    clear_cache()
+    exp = DictExporter()
+    create_prompt("greeting", "v1", labels=["production"], exporters=[exp])
+    create_prompt("greeting", "v2", exporters=[exp])
+    create_prompt("farewell", "bye", exporters=[exp])
+    client = _client(exp)
+
+    listing = client.get("/tc/api/prompts")
+    assert listing.status_code == 200
+    body = listing.json()
+    assert body["total"] == 2  # 2 distinct names
+    names = {p["name"] for p in body["prompts"]}
+    assert names == {"greeting", "farewell"}
+
+    detail = client.get("/tc/api/prompts/greeting")
+    assert detail.status_code == 200
+    versions = detail.json()["versions"]
+    assert [v["version"] for v in versions] == [1, 2]
+
+
+def test_detail_unknown_prompt_404():
+    exp = DictExporter()
+    client = _client(exp)
+    assert client.get("/tc/api/prompts/ghost").status_code == 404
+
+
+def test_prompt_reader_groups_versions():
+    from tracecast.dashboard.prompt_reader import PromptReader
+    clear_cache()
+    exp = DictExporter()
+    create_prompt("p", "v1", exporters=[exp])
+    create_prompt("p", "v2", exporters=[exp])
+    reader = PromptReader([exp])
+    summaries = reader.list_prompts()
+    assert summaries[0]["name"] == "p"
+    assert summaries[0]["latest_version"] == 2

tracecast-0.2.3/tests/test_prompts.py

@@ -0,0 +1,84 @@
+import pytest
+
+from tracecast import Tracer
+from tracecast.prompts import create_prompt, get_prompt, set_label, PromptVersion
+from tracecast.exporters.dict_exporter import DictExporter
+
+
+def test_create_increments_version():
+    exp = DictExporter()
+    v1 = create_prompt("greeting", "Hello {name}", exporters=[exp])
+    v2 = create_prompt("greeting", "Hi {name}", exporters=[exp])
+    assert v1.version == 1
+    assert v2.version == 2
+
+
+def test_get_prompt_defaults_to_highest_version():
+    exp = DictExporter()
+    create_prompt("p", "a", exporters=[exp])
+    create_prompt("p", "b", exporters=[exp])
+    assert get_prompt("p", label=None, exporters=[exp]).version == 2
+
+
+def test_get_prompt_by_explicit_version():
+    exp = DictExporter()
+    create_prompt("p", "a", exporters=[exp])
+    create_prompt("p", "b", exporters=[exp])
+    assert get_prompt("p", version=1, exporters=[exp]).template == "a"
+
+
+def test_label_resolution_and_move():
+    exp = DictExporter()
+    create_prompt("p", "v1", labels=["production"], exporters=[exp])
+    create_prompt("p", "v2", exporters=[exp])
+    assert get_prompt("p", label="production", exporters=[exp]).version == 1
+    set_label("p", 2, "production", exporters=[exp])
+    assert get_prompt("p", label="production", exporters=[exp]).version == 2
+
+
+def test_unknown_label_raises_with_available():
+    exp = DictExporter()
+    create_prompt("p", "v1", labels=["staging"], exporters=[exp])
+    with pytest.raises(ValueError) as ei:
+        get_prompt("p", label="production", exporters=[exp])
+    assert "staging" in str(ei.value)
+
+
+def test_no_exporter_raises():
+    with pytest.raises(RuntimeError):
+        get_prompt("p", exporters=[])
+
+
+def test_cache_avoids_repeat_query():
+    class CountingExp(DictExporter):
+        def __init__(self):
+            super().__init__()
+            self.query_count = 0
+
+        def query_prompts(self, *, name=None):
+            self.query_count += 1
+            return super().query_prompts(name=name)
+
+    exp = CountingExp()
+    create_prompt("p", "v1", labels=["production"], exporters=[exp])
+    before = exp.query_count
+    get_prompt("p", label="production", exporters=[exp], cache_ttl=60)
+    get_prompt("p", label="production", exporters=[exp], cache_ttl=60)
+    assert exp.query_count == before + 1
+
+
+def test_auto_link_to_active_trace():
+    exp = DictExporter()
+    create_prompt("p", "v1", labels=["production"], exporters=[exp])
+    tracer = Tracer(exporters=[exp])
+    with tracer.trace(name="t") as tr:
+        get_prompt("p", label="production", exporters=[exp])
+        assert tr.metadata.get("prompt_name") == "p"
+        assert tr.metadata.get("prompt_version") == 1
+
+
+def test_promptversion_roundtrip():
+    pv = PromptVersion(name="p", version=1, template="t", labels=["production"])
+    back = PromptVersion.from_dict(pv.to_dict())
+    assert back.name == "p"
+    assert back.labels == ["production"]

tracecast-0.2.3/tests/test_score.py

@@ -0,0 +1,56 @@
+import pytest
+from datetime import datetime, timezone
+
+from tracecast.models.score import Score
+
+
+def test_numeric_score_roundtrip():
+    s = Score(trace_id="t1", name="quality", value=0.8)
+    s.validate()
+    d = s.to_dict()
+    assert d["trace_id"] == "t1"
+    assert d["name"] == "quality"
+    assert d["value"] == 0.8
+    assert d["data_type"] == "numeric"
+    assert d["kind"] == "human"
+    back = Score.from_dict(d)
+    assert back.trace_id == "t1"
+    assert back.value == 0.8
+
+
+def test_boolean_score_rejects_non_binary():
+    Score(trace_id="t1", name="thumb", value=1, data_type="boolean").validate()
+    Score(trace_id="t1", name="thumb", value=0, data_type="boolean").validate()
+    with pytest.raises(ValueError):
+        Score(trace_id="t1", name="thumb", value=0.5, data_type="boolean").validate()
+
+
+def test_categorical_requires_string_value():
+    with pytest.raises(ValueError):
+        Score(trace_id="t1", name="sentiment", value=0, data_type="categorical").validate()
+    ok = Score(trace_id="t1", name="sentiment", value=2,
+               data_type="categorical", string_value="positive")
+    ok.validate()
+    assert ok.to_dict()["string_value"] == "positive"
+
+
+def test_invalid_data_type_rejected():
+    with pytest.raises(ValueError):
+        Score(trace_id="t1", name="x", value=1, data_type="bogus").validate()
+
+
+def test_invalid_kind_rejected():
+    with pytest.raises(ValueError):
+        Score(trace_id="t1", name="x", value=1, kind="robot").validate()
+
+
+def test_score_has_id_and_created_at():
+    s = Score(trace_id="t1", name="x", value=1)
+    assert s.score_id
+    assert isinstance(s.created_at, datetime)
+
+
+def test_span_id_optional():
+    s = Score(trace_id="t1", span_id="sp1", name="x", value=1)
+    assert s.to_dict()["span_id"] == "sp1"
+    assert Score(trace_id="t1", name="x", value=1).to_dict()["span_id"] is None

tracecast-0.2.3/tests/test_score_dashboard.py

@@ -0,0 +1,49 @@
+import pytest
+
+import tracecast
+from tracecast import Tracer
+from tracecast.exporters.dict_exporter import DictExporter
+
+fastapi = pytest.importorskip("fastapi")
+from fastapi import FastAPI
+from fastapi.testclient import TestClient
+
+
+def _client(exporter):
+    tracer = Tracer(exporters=[exporter])
+    app = FastAPI()
+    tracer.mount(app, prefix="/tc")
+    return TestClient(app), tracer
+
+
+def test_trace_scores_endpoint():
+    exp = DictExporter()
+    client, _ = _client(exp)
+    tracecast.score("t1", name="user_feedback", value=1, comment="good", exporters=[exp])
+    tracecast.score("t1", name="quality", value=0.9, exporters=[exp])
+    tracecast.score("other", name="quality", value=0.1, exporters=[exp])
+
+    resp = client.get("/tc/api/traces/t1/scores")
+    assert resp.status_code == 200
+    body = resp.json()
+    assert body["total"] == 2
+    names = {s["name"] for s in body["scores"]}
+    assert names == {"user_feedback", "quality"}
+
+
+def test_trace_scores_empty():
+    exp = DictExporter()
+    client, _ = _client(exp)
+    resp = client.get("/tc/api/traces/nope/scores")
+    assert resp.status_code == 200
+    assert resp.json() == {"scores": [], "total": 0}
+
+
+def test_score_reader_list_for_trace():
+    from tracecast.dashboard.score_reader import ScoreReader
+    exp = DictExporter()
+    tracecast.score("t1", name="a", value=1, exporters=[exp])
+    reader = ScoreReader([exp])
+    rows = reader.list_for_trace("t1")
+    assert len(rows) == 1
+    assert rows[0]["name"] == "a"

tracecast-0.2.3/tests/test_scoring_api.py

@@ -0,0 +1,49 @@
+import pytest
+
+import tracecast
+from tracecast import Tracer, set_default_tracer
+from tracecast.exporters.dict_exporter import DictExporter
+
+
+def test_score_persists_via_explicit_exporter():
+    exp = DictExporter()
+    s = tracecast.score("t1", name="user_feedback", value=1, comment="useful", exporters=[exp])
+    assert s.trace_id == "t1"
+    rows = exp.query_scores(trace_id="t1")
+    assert len(rows) == 1
+    assert rows[0]["name"] == "user_feedback"
+    assert rows[0]["comment"] == "useful"
+    assert rows[0]["source"] == "sdk"
+
+
+def test_score_uses_default_tracer_exporters():
+    exp = DictExporter()
+    set_default_tracer(Tracer(exporters=[exp]))
+    try:
+        tracecast.score("t2", name="q", value=0.5)
+        assert len(exp.query_scores(trace_id="t2")) == 1
+    finally:
+        set_default_tracer(Tracer(exporters=[]))
+
+
+def test_invalid_value_rejected_before_persist():
+    exp = DictExporter()
+    with pytest.raises(ValueError):
+        tracecast.score("t1", name="thumb", value=0.5, data_type="boolean", exporters=[exp])
+    assert exp.query_scores(trace_id="t1") == []
+
+
+def test_exporter_failure_does_not_propagate():
+    class Boom(DictExporter):
+        def export_score(self, score):
+            raise RuntimeError("db down")
+    s = tracecast.score("t3", name="q", value=1, exporters=[Boom()])
+    assert s.trace_id == "t3"  # returned despite export failure
+
+
+def test_query_scores_filter_by_trace():
+    exp = DictExporter()
+    tracecast.score("a", name="q", value=1, exporters=[exp])
+    tracecast.score("b", name="q", value=1, exporters=[exp])
+    assert {r["trace_id"] for r in exp.query_scores(trace_id="a")} == {"a"}
+    assert len(exp.query_scores()) == 2

{tracecast-0.2.1 → tracecast-0.2.3}/tracecast/__init__.py

@@ -3,7 +3,10 @@ from .core.cost_calculator import calculate_cost
 from .core.token_counter import extract_tokens, extract_content, extract_input_text
 from .models.trace import Trace
 from .models.span import Span, SpanType, SpanStatus
+from .models.score import Score
 from .decorators import trace_cast, trace_span, set_default_tracer
+from .core.scoring import score
+from .prompts import create_prompt, get_prompt, set_label, PromptVersion
 from .integrations.llm import trace_llm_call, wrap_openai, wrap_anthropic
 from .instrument import auto_instrument
 from .exporters.dict_exporter import DictExporter
@@ -11,8 +14,9 @@ from .exporters.dict_exporter import DictExporter
 __all__ = [
     "Tracer", "bind_context", "calculate_cost",
     "extract_tokens", "extract_content", "extract_input_text",
-    "Trace", "Span", "SpanType", "SpanStatus",
-    "trace_cast", "trace_span", "set_default_tracer",
+    "Trace", "Span", "SpanType", "SpanStatus", "Score",
+    "trace_cast", "trace_span", "set_default_tracer", "score",
+    "create_prompt", "get_prompt", "set_label", "PromptVersion",
     "trace_llm_call", "wrap_openai", "wrap_anthropic",
     "auto_instrument", "DictExporter",
 ]

tracecast-0.2.3/tracecast/core/scoring.py

@@ -0,0 +1,53 @@
+from typing import Any, Dict, List, Optional
+
+from ..models.score import Score
+from .logger import _logger
+
+
+def _resolve_exporters(exporters, tracer) -> List:
+    if exporters is not None:
+        return exporters
+    if tracer is not None:
+        return getattr(tracer, "exporters", []) or []
+    from ..decorators import _default_tracer
+    if _default_tracer is not None:
+        return getattr(_default_tracer, "exporters", []) or []
+    return []
+
+
+def score(
+    trace_id: str,
+    name: str,
+    value: float,
+    *,
+    span_id: Optional[str] = None,
+    kind: str = "human",
+    data_type: str = "numeric",
+    string_value: Optional[str] = None,
+    comment: Optional[str] = None,
+    source: str = "sdk",
+    project_id: Optional[str] = None,
+    metadata: Optional[Dict[str, Any]] = None,
+    exporters: Optional[List] = None,
+    tracer=None,
+) -> Score:
+    """Attach a Score to a production trace (or span) and persist it.
+
+    Validates before persisting; exporter failures are logged, never raised.
+    """
+    s = Score(
+        trace_id=trace_id, name=name, value=value, span_id=span_id, kind=kind,
+        data_type=data_type, string_value=string_value, comment=comment,
+        source=source, project_id=project_id, metadata=metadata or {},
+    )
+    s.validate()
+
+    for exporter in _resolve_exporters(exporters, tracer):
+        if callable(getattr(exporter, "export_score", None)):
+            try:
+                exporter.export_score(s)
+            except Exception as exc:
+                _logger.error(
+                    "TraceCast: export_score failed for %s: %s", s.score_id, exc, exc_info=True
+                )
+    return s

{tracecast-0.2.1 → tracecast-0.2.3}/tracecast/core/tracer.py

@@ -40,22 +40,34 @@ class Tracer:
         logging: bool = False,
         log_prefix: Optional[str] = None,
         on_export_error: Optional[Callable[[Exception, Trace, BaseExporter], None]] = None,
+        online_eval=None,
     ):
         self.exporters = exporters or []
         self.on_export_error = on_export_error
+        self.online_eval = online_eval
         self._tc_logger = None
         if logging:
             from .logger import TraceCastLogger
             self._tc_logger = TraceCastLogger(prefix=log_prefix)
 
+    def _run_online_eval(self, trace: Trace) -> None:
+        if self.online_eval is None:
+            return
+        try:
+            self.online_eval.maybe_evaluate(trace, default_exporters=self.exporters)
+        except Exception:
+            from .logger import _logger
+            _logger.error("TraceCast: online_eval failed for trace %s", trace.trace_id, exc_info=True)
+
     @contextmanager
-    def trace(self, name: str, session_id=None, user_id=None, project_id=None, metadata=None):
+    def trace(self, name: str, session_id=None, user_id=None, project_id=None, project_name=None, metadata=None):
         t = Trace(
             trace_id=str(uuid.uuid4()),
             name=name,
             session_id=session_id,
             user_id=user_id,
             project_id=project_id,
+            project_name=project_name,
             started_at=datetime.now(timezone.utc),
             metadata=metadata or {},
         )
@@ -79,13 +91,14 @@ class Tracer:
             self._export(t)
 
     @asynccontextmanager
-    async def atrace(self, name: str, session_id=None, user_id=None, project_id=None, metadata=None):
+    async def atrace(self, name: str, session_id=None, user_id=None, project_id=None, project_name=None, metadata=None):
         t = Trace(
             trace_id=str(uuid.uuid4()),
             name=name,
             session_id=session_id,
             user_id=user_id,
             project_id=project_id,
+            project_name=project_name,
             started_at=datetime.now(timezone.utc),
             metadata=metadata or {},
         )
@@ -127,6 +140,7 @@ class Tracer:
                 exporter.export(trace)
             except Exception as exc:
                 self._handle_export_error(exc, trace, exporter)
+        self._run_online_eval(trace)
 
     async def _aexport(self, trace: Trace) -> None:
         for exporter in self.exporters:
@@ -134,6 +148,7 @@ class Tracer:
                 await exporter.aexport(trace)
             except Exception as exc:
                 self._handle_export_error(exc, trace, exporter)
+        self._run_online_eval(trace)
 
     @staticmethod
     def current() -> Optional[Trace]: