tracecast 0.2.1__tar.gz → 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tracecast-0.2.1/tracecast.egg-info → tracecast-0.2.2}/PKG-INFO +1 -1
- {tracecast-0.2.1 → tracecast-0.2.2}/pyproject.toml +1 -1
- tracecast-0.2.2/tests/test_prompt_dashboard.py +56 -0
- tracecast-0.2.2/tests/test_prompts.py +84 -0
- tracecast-0.2.2/tests/test_score.py +56 -0
- tracecast-0.2.2/tests/test_score_dashboard.py +49 -0
- tracecast-0.2.2/tests/test_scoring_api.py +49 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/__init__.py +6 -2
- tracecast-0.2.2/tracecast/core/scoring.py +53 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/core/tracer.py +17 -2
- {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/dashboard/aggregator.py +94 -4
- tracecast-0.2.2/tracecast/dashboard/eval_reader.py +28 -0
- tracecast-0.2.2/tracecast/dashboard/prompt_reader.py +40 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/dashboard/reader.py +22 -2
- {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/dashboard/router.py +94 -3
- tracecast-0.2.2/tracecast/dashboard/score_reader.py +28 -0
- tracecast-0.2.2/tracecast/dashboard/static/assets/index-CBra-omk.js +151 -0
- tracecast-0.2.2/tracecast/dashboard/static/assets/index-DllJJzmE.css +1 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/dashboard/static/index.html +2 -2
- tracecast-0.2.2/tracecast/eval/__init__.py +25 -0
- tracecast-0.2.2/tracecast/eval/cli.py +59 -0
- tracecast-0.2.2/tracecast/eval/compare.py +42 -0
- tracecast-0.2.2/tracecast/eval/dataset.py +136 -0
- tracecast-0.2.2/tracecast/eval/decorator.py +72 -0
- tracecast-0.2.2/tracecast/eval/judge.py +124 -0
- tracecast-0.2.2/tracecast/eval/metrics.py +128 -0
- tracecast-0.2.2/tracecast/eval/models.py +221 -0
- tracecast-0.2.2/tracecast/eval/online.py +71 -0
- tracecast-0.2.2/tracecast/eval/runner.py +160 -0
- tracecast-0.2.2/tracecast/eval/scorers.py +42 -0
- tracecast-0.2.2/tracecast/exporters/_eval_store.py +59 -0
- tracecast-0.2.2/tracecast/exporters/base.py +44 -0
- tracecast-0.2.2/tracecast/exporters/dict_exporter.py +79 -0
- tracecast-0.2.2/tracecast/exporters/json_file.py +117 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/exporters/mongo.py +79 -2
- {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/exporters/postgres.py +195 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/instrumentors/openai_inst.py +20 -2
- {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/integrations/langchain.py +55 -1
- tracecast-0.2.2/tracecast/models/score.py +84 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/models/trace.py +2 -0
- tracecast-0.2.2/tracecast/prompts/__init__.py +4 -0
- tracecast-0.2.2/tracecast/prompts/client.py +134 -0
- tracecast-0.2.2/tracecast/prompts/models.py +49 -0
- {tracecast-0.2.1 → tracecast-0.2.2/tracecast.egg-info}/PKG-INFO +1 -1
- {tracecast-0.2.1 → tracecast-0.2.2}/tracecast.egg-info/SOURCES.txt +28 -3
- tracecast-0.2.1/tracecast/dashboard/static/assets/index-BAxBR6Dz.css +0 -1
- tracecast-0.2.1/tracecast/dashboard/static/assets/index-DaMldkDH.js +0 -155
- tracecast-0.2.1/tracecast/exporters/base.py +0 -16
- tracecast-0.2.1/tracecast/exporters/dict_exporter.py +0 -35
- tracecast-0.2.1/tracecast/exporters/json_file.py +0 -37
- {tracecast-0.2.1 → tracecast-0.2.2}/LICENSE +0 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/README.md +0 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/setup.cfg +0 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tests/test_context_propagation.py +0 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tests/test_cost_calculator.py +0 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tests/test_dashboard_mount.py +0 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tests/test_dashboard_pushdown.py +0 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tests/test_dashboard_sessions.py +0 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tests/test_decorators.py +0 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tests/test_e2e_langgraph.py +0 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tests/test_exporters_db.py +0 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tests/test_integration_frameworks.py +0 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tests/test_integration_langchain.py +0 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tests/test_json_exporter.py +0 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tests/test_langchain_callback.py +0 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tests/test_llm_wrappers.py +0 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tests/test_logging.py +0 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tests/test_mongo_exporter.py +0 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tests/test_roundtrip.py +0 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tests/test_schema_v2.py +0 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tests/test_serve.py +0 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tests/test_span.py +0 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tests/test_streaming.py +0 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tests/test_token_counter.py +0 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tests/test_trace.py +0 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tests/test_trace_span.py +0 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tests/test_tracer.py +0 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/core/__init__.py +0 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/core/cost_calculator.py +0 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/core/logger.py +0 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/core/token_counter.py +0 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/dashboard/__init__.py +0 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/dashboard/asgi_middleware.py +0 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/dashboard/blueprint.py +0 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/dashboard/standalone.py +0 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/decorators.py +0 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/exporters/__init__.py +0 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/exporters/query.py +0 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/instrument.py +0 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/instrumentors/__init__.py +0 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/instrumentors/_llamaindex_handler.py +0 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/instrumentors/_streaming.py +0 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/instrumentors/anthropic_inst.py +0 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/instrumentors/base.py +0 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/instrumentors/crewai_inst.py +0 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/instrumentors/gemini_inst.py +0 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/instrumentors/langchain_inst.py +0 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/instrumentors/llamaindex_inst.py +0 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/integrations/__init__.py +0 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/integrations/llm.py +0 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/middleware.py +0 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/models/__init__.py +0 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/models/span.py +0 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/serve.py +0 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tracecast.egg-info/dependency_links.txt +0 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tracecast.egg-info/entry_points.txt +0 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tracecast.egg-info/requires.txt +0 -0
- {tracecast-0.2.1 → tracecast-0.2.2}/tracecast.egg-info/top_level.txt +0 -0
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import pytest
|
|
2
|
+
|
|
3
|
+
from tracecast import Tracer
|
|
4
|
+
from tracecast.prompts import create_prompt
|
|
5
|
+
from tracecast.prompts.client import clear_cache
|
|
6
|
+
from tracecast.exporters.dict_exporter import DictExporter
|
|
7
|
+
|
|
8
|
+
fastapi = pytest.importorskip("fastapi")
|
|
9
|
+
from fastapi import FastAPI
|
|
10
|
+
from fastapi.testclient import TestClient
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _client(exporter):
|
|
14
|
+
tracer = Tracer(exporters=[exporter])
|
|
15
|
+
app = FastAPI()
|
|
16
|
+
tracer.mount(app, prefix="/tc")
|
|
17
|
+
return TestClient(app)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def test_list_and_detail_prompts():
|
|
21
|
+
clear_cache()
|
|
22
|
+
exp = DictExporter()
|
|
23
|
+
create_prompt("greeting", "v1", labels=["production"], exporters=[exp])
|
|
24
|
+
create_prompt("greeting", "v2", exporters=[exp])
|
|
25
|
+
create_prompt("farewell", "bye", exporters=[exp])
|
|
26
|
+
client = _client(exp)
|
|
27
|
+
|
|
28
|
+
listing = client.get("/tc/api/prompts")
|
|
29
|
+
assert listing.status_code == 200
|
|
30
|
+
body = listing.json()
|
|
31
|
+
assert body["total"] == 2 # 2 distinct names
|
|
32
|
+
names = {p["name"] for p in body["prompts"]}
|
|
33
|
+
assert names == {"greeting", "farewell"}
|
|
34
|
+
|
|
35
|
+
detail = client.get("/tc/api/prompts/greeting")
|
|
36
|
+
assert detail.status_code == 200
|
|
37
|
+
versions = detail.json()["versions"]
|
|
38
|
+
assert [v["version"] for v in versions] == [1, 2]
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def test_detail_unknown_prompt_404():
|
|
42
|
+
exp = DictExporter()
|
|
43
|
+
client = _client(exp)
|
|
44
|
+
assert client.get("/tc/api/prompts/ghost").status_code == 404
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def test_prompt_reader_groups_versions():
|
|
48
|
+
from tracecast.dashboard.prompt_reader import PromptReader
|
|
49
|
+
clear_cache()
|
|
50
|
+
exp = DictExporter()
|
|
51
|
+
create_prompt("p", "v1", exporters=[exp])
|
|
52
|
+
create_prompt("p", "v2", exporters=[exp])
|
|
53
|
+
reader = PromptReader([exp])
|
|
54
|
+
summaries = reader.list_prompts()
|
|
55
|
+
assert summaries[0]["name"] == "p"
|
|
56
|
+
assert summaries[0]["latest_version"] == 2
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
import pytest
|
|
2
|
+
|
|
3
|
+
from tracecast import Tracer
|
|
4
|
+
from tracecast.prompts import create_prompt, get_prompt, set_label, PromptVersion
|
|
5
|
+
from tracecast.exporters.dict_exporter import DictExporter
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def test_create_increments_version():
|
|
9
|
+
exp = DictExporter()
|
|
10
|
+
v1 = create_prompt("greeting", "Hello {name}", exporters=[exp])
|
|
11
|
+
v2 = create_prompt("greeting", "Hi {name}", exporters=[exp])
|
|
12
|
+
assert v1.version == 1
|
|
13
|
+
assert v2.version == 2
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def test_get_prompt_defaults_to_highest_version():
|
|
17
|
+
exp = DictExporter()
|
|
18
|
+
create_prompt("p", "a", exporters=[exp])
|
|
19
|
+
create_prompt("p", "b", exporters=[exp])
|
|
20
|
+
assert get_prompt("p", label=None, exporters=[exp]).version == 2
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def test_get_prompt_by_explicit_version():
|
|
24
|
+
exp = DictExporter()
|
|
25
|
+
create_prompt("p", "a", exporters=[exp])
|
|
26
|
+
create_prompt("p", "b", exporters=[exp])
|
|
27
|
+
assert get_prompt("p", version=1, exporters=[exp]).template == "a"
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def test_label_resolution_and_move():
|
|
31
|
+
exp = DictExporter()
|
|
32
|
+
create_prompt("p", "v1", labels=["production"], exporters=[exp])
|
|
33
|
+
create_prompt("p", "v2", exporters=[exp])
|
|
34
|
+
assert get_prompt("p", label="production", exporters=[exp]).version == 1
|
|
35
|
+
set_label("p", 2, "production", exporters=[exp])
|
|
36
|
+
assert get_prompt("p", label="production", exporters=[exp]).version == 2
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def test_unknown_label_raises_with_available():
|
|
40
|
+
exp = DictExporter()
|
|
41
|
+
create_prompt("p", "v1", labels=["staging"], exporters=[exp])
|
|
42
|
+
with pytest.raises(ValueError) as ei:
|
|
43
|
+
get_prompt("p", label="production", exporters=[exp])
|
|
44
|
+
assert "staging" in str(ei.value)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def test_no_exporter_raises():
|
|
48
|
+
with pytest.raises(RuntimeError):
|
|
49
|
+
get_prompt("p", exporters=[])
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def test_cache_avoids_repeat_query():
|
|
53
|
+
class CountingExp(DictExporter):
|
|
54
|
+
def __init__(self):
|
|
55
|
+
super().__init__()
|
|
56
|
+
self.query_count = 0
|
|
57
|
+
|
|
58
|
+
def query_prompts(self, *, name=None):
|
|
59
|
+
self.query_count += 1
|
|
60
|
+
return super().query_prompts(name=name)
|
|
61
|
+
|
|
62
|
+
exp = CountingExp()
|
|
63
|
+
create_prompt("p", "v1", labels=["production"], exporters=[exp])
|
|
64
|
+
before = exp.query_count
|
|
65
|
+
get_prompt("p", label="production", exporters=[exp], cache_ttl=60)
|
|
66
|
+
get_prompt("p", label="production", exporters=[exp], cache_ttl=60)
|
|
67
|
+
assert exp.query_count == before + 1
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def test_auto_link_to_active_trace():
|
|
71
|
+
exp = DictExporter()
|
|
72
|
+
create_prompt("p", "v1", labels=["production"], exporters=[exp])
|
|
73
|
+
tracer = Tracer(exporters=[exp])
|
|
74
|
+
with tracer.trace(name="t") as tr:
|
|
75
|
+
get_prompt("p", label="production", exporters=[exp])
|
|
76
|
+
assert tr.metadata.get("prompt_name") == "p"
|
|
77
|
+
assert tr.metadata.get("prompt_version") == 1
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def test_promptversion_roundtrip():
|
|
81
|
+
pv = PromptVersion(name="p", version=1, template="t", labels=["production"])
|
|
82
|
+
back = PromptVersion.from_dict(pv.to_dict())
|
|
83
|
+
assert back.name == "p"
|
|
84
|
+
assert back.labels == ["production"]
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import pytest
|
|
2
|
+
from datetime import datetime, timezone
|
|
3
|
+
|
|
4
|
+
from tracecast.models.score import Score
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def test_numeric_score_roundtrip():
|
|
8
|
+
s = Score(trace_id="t1", name="quality", value=0.8)
|
|
9
|
+
s.validate()
|
|
10
|
+
d = s.to_dict()
|
|
11
|
+
assert d["trace_id"] == "t1"
|
|
12
|
+
assert d["name"] == "quality"
|
|
13
|
+
assert d["value"] == 0.8
|
|
14
|
+
assert d["data_type"] == "numeric"
|
|
15
|
+
assert d["kind"] == "human"
|
|
16
|
+
back = Score.from_dict(d)
|
|
17
|
+
assert back.trace_id == "t1"
|
|
18
|
+
assert back.value == 0.8
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def test_boolean_score_rejects_non_binary():
|
|
22
|
+
Score(trace_id="t1", name="thumb", value=1, data_type="boolean").validate()
|
|
23
|
+
Score(trace_id="t1", name="thumb", value=0, data_type="boolean").validate()
|
|
24
|
+
with pytest.raises(ValueError):
|
|
25
|
+
Score(trace_id="t1", name="thumb", value=0.5, data_type="boolean").validate()
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def test_categorical_requires_string_value():
|
|
29
|
+
with pytest.raises(ValueError):
|
|
30
|
+
Score(trace_id="t1", name="sentiment", value=0, data_type="categorical").validate()
|
|
31
|
+
ok = Score(trace_id="t1", name="sentiment", value=2,
|
|
32
|
+
data_type="categorical", string_value="positive")
|
|
33
|
+
ok.validate()
|
|
34
|
+
assert ok.to_dict()["string_value"] == "positive"
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def test_invalid_data_type_rejected():
|
|
38
|
+
with pytest.raises(ValueError):
|
|
39
|
+
Score(trace_id="t1", name="x", value=1, data_type="bogus").validate()
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def test_invalid_kind_rejected():
|
|
43
|
+
with pytest.raises(ValueError):
|
|
44
|
+
Score(trace_id="t1", name="x", value=1, kind="robot").validate()
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def test_score_has_id_and_created_at():
|
|
48
|
+
s = Score(trace_id="t1", name="x", value=1)
|
|
49
|
+
assert s.score_id
|
|
50
|
+
assert isinstance(s.created_at, datetime)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def test_span_id_optional():
|
|
54
|
+
s = Score(trace_id="t1", span_id="sp1", name="x", value=1)
|
|
55
|
+
assert s.to_dict()["span_id"] == "sp1"
|
|
56
|
+
assert Score(trace_id="t1", name="x", value=1).to_dict()["span_id"] is None
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import pytest
|
|
2
|
+
|
|
3
|
+
import tracecast
|
|
4
|
+
from tracecast import Tracer
|
|
5
|
+
from tracecast.exporters.dict_exporter import DictExporter
|
|
6
|
+
|
|
7
|
+
fastapi = pytest.importorskip("fastapi")
|
|
8
|
+
from fastapi import FastAPI
|
|
9
|
+
from fastapi.testclient import TestClient
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def _client(exporter):
|
|
13
|
+
tracer = Tracer(exporters=[exporter])
|
|
14
|
+
app = FastAPI()
|
|
15
|
+
tracer.mount(app, prefix="/tc")
|
|
16
|
+
return TestClient(app), tracer
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def test_trace_scores_endpoint():
|
|
20
|
+
exp = DictExporter()
|
|
21
|
+
client, _ = _client(exp)
|
|
22
|
+
tracecast.score("t1", name="user_feedback", value=1, comment="good", exporters=[exp])
|
|
23
|
+
tracecast.score("t1", name="quality", value=0.9, exporters=[exp])
|
|
24
|
+
tracecast.score("other", name="quality", value=0.1, exporters=[exp])
|
|
25
|
+
|
|
26
|
+
resp = client.get("/tc/api/traces/t1/scores")
|
|
27
|
+
assert resp.status_code == 200
|
|
28
|
+
body = resp.json()
|
|
29
|
+
assert body["total"] == 2
|
|
30
|
+
names = {s["name"] for s in body["scores"]}
|
|
31
|
+
assert names == {"user_feedback", "quality"}
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def test_trace_scores_empty():
|
|
35
|
+
exp = DictExporter()
|
|
36
|
+
client, _ = _client(exp)
|
|
37
|
+
resp = client.get("/tc/api/traces/nope/scores")
|
|
38
|
+
assert resp.status_code == 200
|
|
39
|
+
assert resp.json() == {"scores": [], "total": 0}
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def test_score_reader_list_for_trace():
|
|
43
|
+
from tracecast.dashboard.score_reader import ScoreReader
|
|
44
|
+
exp = DictExporter()
|
|
45
|
+
tracecast.score("t1", name="a", value=1, exporters=[exp])
|
|
46
|
+
reader = ScoreReader([exp])
|
|
47
|
+
rows = reader.list_for_trace("t1")
|
|
48
|
+
assert len(rows) == 1
|
|
49
|
+
assert rows[0]["name"] == "a"
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import pytest
|
|
2
|
+
|
|
3
|
+
import tracecast
|
|
4
|
+
from tracecast import Tracer, set_default_tracer
|
|
5
|
+
from tracecast.exporters.dict_exporter import DictExporter
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def test_score_persists_via_explicit_exporter():
|
|
9
|
+
exp = DictExporter()
|
|
10
|
+
s = tracecast.score("t1", name="user_feedback", value=1, comment="useful", exporters=[exp])
|
|
11
|
+
assert s.trace_id == "t1"
|
|
12
|
+
rows = exp.query_scores(trace_id="t1")
|
|
13
|
+
assert len(rows) == 1
|
|
14
|
+
assert rows[0]["name"] == "user_feedback"
|
|
15
|
+
assert rows[0]["comment"] == "useful"
|
|
16
|
+
assert rows[0]["source"] == "sdk"
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def test_score_uses_default_tracer_exporters():
|
|
20
|
+
exp = DictExporter()
|
|
21
|
+
set_default_tracer(Tracer(exporters=[exp]))
|
|
22
|
+
try:
|
|
23
|
+
tracecast.score("t2", name="q", value=0.5)
|
|
24
|
+
assert len(exp.query_scores(trace_id="t2")) == 1
|
|
25
|
+
finally:
|
|
26
|
+
set_default_tracer(Tracer(exporters=[]))
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def test_invalid_value_rejected_before_persist():
|
|
30
|
+
exp = DictExporter()
|
|
31
|
+
with pytest.raises(ValueError):
|
|
32
|
+
tracecast.score("t1", name="thumb", value=0.5, data_type="boolean", exporters=[exp])
|
|
33
|
+
assert exp.query_scores(trace_id="t1") == []
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def test_exporter_failure_does_not_propagate():
|
|
37
|
+
class Boom(DictExporter):
|
|
38
|
+
def export_score(self, score):
|
|
39
|
+
raise RuntimeError("db down")
|
|
40
|
+
s = tracecast.score("t3", name="q", value=1, exporters=[Boom()])
|
|
41
|
+
assert s.trace_id == "t3" # returned despite export failure
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def test_query_scores_filter_by_trace():
|
|
45
|
+
exp = DictExporter()
|
|
46
|
+
tracecast.score("a", name="q", value=1, exporters=[exp])
|
|
47
|
+
tracecast.score("b", name="q", value=1, exporters=[exp])
|
|
48
|
+
assert {r["trace_id"] for r in exp.query_scores(trace_id="a")} == {"a"}
|
|
49
|
+
assert len(exp.query_scores()) == 2
|
|
@@ -3,7 +3,10 @@ from .core.cost_calculator import calculate_cost
|
|
|
3
3
|
from .core.token_counter import extract_tokens, extract_content, extract_input_text
|
|
4
4
|
from .models.trace import Trace
|
|
5
5
|
from .models.span import Span, SpanType, SpanStatus
|
|
6
|
+
from .models.score import Score
|
|
6
7
|
from .decorators import trace_cast, trace_span, set_default_tracer
|
|
8
|
+
from .core.scoring import score
|
|
9
|
+
from .prompts import create_prompt, get_prompt, set_label, PromptVersion
|
|
7
10
|
from .integrations.llm import trace_llm_call, wrap_openai, wrap_anthropic
|
|
8
11
|
from .instrument import auto_instrument
|
|
9
12
|
from .exporters.dict_exporter import DictExporter
|
|
@@ -11,8 +14,9 @@ from .exporters.dict_exporter import DictExporter
|
|
|
11
14
|
__all__ = [
|
|
12
15
|
"Tracer", "bind_context", "calculate_cost",
|
|
13
16
|
"extract_tokens", "extract_content", "extract_input_text",
|
|
14
|
-
"Trace", "Span", "SpanType", "SpanStatus",
|
|
15
|
-
"trace_cast", "trace_span", "set_default_tracer",
|
|
17
|
+
"Trace", "Span", "SpanType", "SpanStatus", "Score",
|
|
18
|
+
"trace_cast", "trace_span", "set_default_tracer", "score",
|
|
19
|
+
"create_prompt", "get_prompt", "set_label", "PromptVersion",
|
|
16
20
|
"trace_llm_call", "wrap_openai", "wrap_anthropic",
|
|
17
21
|
"auto_instrument", "DictExporter",
|
|
18
22
|
]
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
from typing import Any, Dict, List, Optional
|
|
2
|
+
|
|
3
|
+
from ..models.score import Score
|
|
4
|
+
from .logger import _logger
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def _resolve_exporters(exporters, tracer) -> List:
|
|
8
|
+
if exporters is not None:
|
|
9
|
+
return exporters
|
|
10
|
+
if tracer is not None:
|
|
11
|
+
return getattr(tracer, "exporters", []) or []
|
|
12
|
+
from ..decorators import _default_tracer
|
|
13
|
+
if _default_tracer is not None:
|
|
14
|
+
return getattr(_default_tracer, "exporters", []) or []
|
|
15
|
+
return []
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def score(
|
|
19
|
+
trace_id: str,
|
|
20
|
+
name: str,
|
|
21
|
+
value: float,
|
|
22
|
+
*,
|
|
23
|
+
span_id: Optional[str] = None,
|
|
24
|
+
kind: str = "human",
|
|
25
|
+
data_type: str = "numeric",
|
|
26
|
+
string_value: Optional[str] = None,
|
|
27
|
+
comment: Optional[str] = None,
|
|
28
|
+
source: str = "sdk",
|
|
29
|
+
project_id: Optional[str] = None,
|
|
30
|
+
metadata: Optional[Dict[str, Any]] = None,
|
|
31
|
+
exporters: Optional[List] = None,
|
|
32
|
+
tracer=None,
|
|
33
|
+
) -> Score:
|
|
34
|
+
"""Attach a Score to a production trace (or span) and persist it.
|
|
35
|
+
|
|
36
|
+
Validates before persisting; exporter failures are logged, never raised.
|
|
37
|
+
"""
|
|
38
|
+
s = Score(
|
|
39
|
+
trace_id=trace_id, name=name, value=value, span_id=span_id, kind=kind,
|
|
40
|
+
data_type=data_type, string_value=string_value, comment=comment,
|
|
41
|
+
source=source, project_id=project_id, metadata=metadata or {},
|
|
42
|
+
)
|
|
43
|
+
s.validate()
|
|
44
|
+
|
|
45
|
+
for exporter in _resolve_exporters(exporters, tracer):
|
|
46
|
+
if callable(getattr(exporter, "export_score", None)):
|
|
47
|
+
try:
|
|
48
|
+
exporter.export_score(s)
|
|
49
|
+
except Exception as exc:
|
|
50
|
+
_logger.error(
|
|
51
|
+
"TraceCast: export_score failed for %s: %s", s.score_id, exc, exc_info=True
|
|
52
|
+
)
|
|
53
|
+
return s
|
|
@@ -40,22 +40,34 @@ class Tracer:
|
|
|
40
40
|
logging: bool = False,
|
|
41
41
|
log_prefix: Optional[str] = None,
|
|
42
42
|
on_export_error: Optional[Callable[[Exception, Trace, BaseExporter], None]] = None,
|
|
43
|
+
online_eval=None,
|
|
43
44
|
):
|
|
44
45
|
self.exporters = exporters or []
|
|
45
46
|
self.on_export_error = on_export_error
|
|
47
|
+
self.online_eval = online_eval
|
|
46
48
|
self._tc_logger = None
|
|
47
49
|
if logging:
|
|
48
50
|
from .logger import TraceCastLogger
|
|
49
51
|
self._tc_logger = TraceCastLogger(prefix=log_prefix)
|
|
50
52
|
|
|
53
|
+
def _run_online_eval(self, trace: Trace) -> None:
|
|
54
|
+
if self.online_eval is None:
|
|
55
|
+
return
|
|
56
|
+
try:
|
|
57
|
+
self.online_eval.maybe_evaluate(trace, default_exporters=self.exporters)
|
|
58
|
+
except Exception:
|
|
59
|
+
from .logger import _logger
|
|
60
|
+
_logger.error("TraceCast: online_eval failed for trace %s", trace.trace_id, exc_info=True)
|
|
61
|
+
|
|
51
62
|
@contextmanager
|
|
52
|
-
def trace(self, name: str, session_id=None, user_id=None, project_id=None, metadata=None):
|
|
63
|
+
def trace(self, name: str, session_id=None, user_id=None, project_id=None, project_name=None, metadata=None):
|
|
53
64
|
t = Trace(
|
|
54
65
|
trace_id=str(uuid.uuid4()),
|
|
55
66
|
name=name,
|
|
56
67
|
session_id=session_id,
|
|
57
68
|
user_id=user_id,
|
|
58
69
|
project_id=project_id,
|
|
70
|
+
project_name=project_name,
|
|
59
71
|
started_at=datetime.now(timezone.utc),
|
|
60
72
|
metadata=metadata or {},
|
|
61
73
|
)
|
|
@@ -79,13 +91,14 @@ class Tracer:
|
|
|
79
91
|
self._export(t)
|
|
80
92
|
|
|
81
93
|
@asynccontextmanager
|
|
82
|
-
async def atrace(self, name: str, session_id=None, user_id=None, project_id=None, metadata=None):
|
|
94
|
+
async def atrace(self, name: str, session_id=None, user_id=None, project_id=None, project_name=None, metadata=None):
|
|
83
95
|
t = Trace(
|
|
84
96
|
trace_id=str(uuid.uuid4()),
|
|
85
97
|
name=name,
|
|
86
98
|
session_id=session_id,
|
|
87
99
|
user_id=user_id,
|
|
88
100
|
project_id=project_id,
|
|
101
|
+
project_name=project_name,
|
|
89
102
|
started_at=datetime.now(timezone.utc),
|
|
90
103
|
metadata=metadata or {},
|
|
91
104
|
)
|
|
@@ -127,6 +140,7 @@ class Tracer:
|
|
|
127
140
|
exporter.export(trace)
|
|
128
141
|
except Exception as exc:
|
|
129
142
|
self._handle_export_error(exc, trace, exporter)
|
|
143
|
+
self._run_online_eval(trace)
|
|
130
144
|
|
|
131
145
|
async def _aexport(self, trace: Trace) -> None:
|
|
132
146
|
for exporter in self.exporters:
|
|
@@ -134,6 +148,7 @@ class Tracer:
|
|
|
134
148
|
await exporter.aexport(trace)
|
|
135
149
|
except Exception as exc:
|
|
136
150
|
self._handle_export_error(exc, trace, exporter)
|
|
151
|
+
self._run_online_eval(trace)
|
|
137
152
|
|
|
138
153
|
@staticmethod
|
|
139
154
|
def current() -> Optional[Trace]:
|
|
@@ -50,6 +50,7 @@ def compute_metrics(
|
|
|
50
50
|
cost_by_project[t.project_id] += t.cost_usd
|
|
51
51
|
|
|
52
52
|
traces_over_time = _group_by_day(filtered)
|
|
53
|
+
tokens_by_model_over_time = _group_by_day_and_model(filtered)
|
|
53
54
|
|
|
54
55
|
return {
|
|
55
56
|
"period": period,
|
|
@@ -63,6 +64,7 @@ def compute_metrics(
|
|
|
63
64
|
"cost_by_model": dict(cost_by_model),
|
|
64
65
|
"cost_by_project": dict(cost_by_project),
|
|
65
66
|
"traces_over_time": traces_over_time,
|
|
67
|
+
"tokens_by_model_over_time": tokens_by_model_over_time,
|
|
66
68
|
}
|
|
67
69
|
|
|
68
70
|
|
|
@@ -116,6 +118,7 @@ def _trace_summary(trace: Trace) -> dict:
|
|
|
116
118
|
"trace_id": trace.trace_id,
|
|
117
119
|
"name": trace.name,
|
|
118
120
|
"project_id": trace.project_id,
|
|
121
|
+
"project_name": trace.project_name,
|
|
119
122
|
"user_id": trace.user_id,
|
|
120
123
|
"session_id": trace.session_id,
|
|
121
124
|
"model": trace.model,
|
|
@@ -157,16 +160,33 @@ def build_graph(trace: Trace) -> dict:
|
|
|
157
160
|
}
|
|
158
161
|
|
|
159
162
|
|
|
160
|
-
def compute_sessions(
|
|
161
|
-
|
|
163
|
+
def compute_sessions(
|
|
164
|
+
traces: List[Trace],
|
|
165
|
+
*,
|
|
166
|
+
project_name: Optional[str] = None,
|
|
167
|
+
project_id: Optional[str] = None,
|
|
168
|
+
user_id: Optional[str] = None,
|
|
169
|
+
) -> list:
|
|
170
|
+
"""Aggregate traces by session_id with optional hierarchical filtering."""
|
|
171
|
+
filtered = traces
|
|
172
|
+
if project_name:
|
|
173
|
+
filtered = [t for t in filtered if t.project_name == project_name]
|
|
174
|
+
if project_id:
|
|
175
|
+
filtered = [t for t in filtered if t.project_id == project_id]
|
|
176
|
+
if user_id:
|
|
177
|
+
filtered = [t for t in filtered if t.user_id == user_id]
|
|
178
|
+
|
|
162
179
|
groups: dict[str, dict] = {}
|
|
163
|
-
for t in
|
|
180
|
+
for t in filtered:
|
|
164
181
|
sid = t.session_id
|
|
165
182
|
if not sid:
|
|
166
183
|
continue
|
|
167
184
|
if sid not in groups:
|
|
168
185
|
groups[sid] = {
|
|
169
186
|
"session_id": sid,
|
|
187
|
+
"project_name": t.project_name,
|
|
188
|
+
"project_id": t.project_id,
|
|
189
|
+
"user_id": t.user_id,
|
|
170
190
|
"trace_count": 0,
|
|
171
191
|
"total_cost_usd": 0.0,
|
|
172
192
|
"total_tokens": 0,
|
|
@@ -195,7 +215,45 @@ def compute_sessions(traces: List[Trace]) -> list:
|
|
|
195
215
|
|
|
196
216
|
|
|
197
217
|
def compute_projects(traces: List[Trace]) -> list:
|
|
198
|
-
"""
|
|
218
|
+
"""Group by project_name if available; fall back to project_id."""
|
|
219
|
+
use_name = any(t.project_name for t in traces)
|
|
220
|
+
groups: dict[str, dict] = {}
|
|
221
|
+
for t in traces:
|
|
222
|
+
key = t.project_name if use_name else t.project_id
|
|
223
|
+
if not key:
|
|
224
|
+
continue
|
|
225
|
+
if key not in groups:
|
|
226
|
+
groups[key] = {
|
|
227
|
+
"project_name" if use_name else "project_id": key,
|
|
228
|
+
"trace_count": 0,
|
|
229
|
+
"total_cost_usd": 0.0,
|
|
230
|
+
"total_tokens": 0,
|
|
231
|
+
"total_tokens_in": 0,
|
|
232
|
+
"total_tokens_out": 0,
|
|
233
|
+
"_first_dt": t.started_at,
|
|
234
|
+
"_last_dt": t.started_at,
|
|
235
|
+
}
|
|
236
|
+
g = groups[key]
|
|
237
|
+
g["trace_count"] += 1
|
|
238
|
+
g["total_cost_usd"] += t.cost_usd
|
|
239
|
+
g["total_tokens"] += t.total_tokens
|
|
240
|
+
g["total_tokens_in"] += t.total_tokens_in
|
|
241
|
+
g["total_tokens_out"] += t.total_tokens_out
|
|
242
|
+
dt = t.started_at
|
|
243
|
+
if dt < g["_first_dt"]:
|
|
244
|
+
g["_first_dt"] = dt
|
|
245
|
+
if dt > g["_last_dt"]:
|
|
246
|
+
g["_last_dt"] = dt
|
|
247
|
+
result = sorted(groups.values(), key=lambda x: x["_last_dt"], reverse=True)
|
|
248
|
+
for r in result:
|
|
249
|
+
r["total_cost_usd"] = round(r["total_cost_usd"], 6)
|
|
250
|
+
r["first_trace_at"] = r.pop("_first_dt").isoformat()
|
|
251
|
+
r["last_trace_at"] = r.pop("_last_dt").isoformat()
|
|
252
|
+
return result
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def compute_projects_by_id(traces: List[Trace]) -> list:
|
|
256
|
+
"""Aggregate traces by project_id (filial level, used for drill-down under a project_name)."""
|
|
199
257
|
groups: dict[str, dict] = {}
|
|
200
258
|
for t in traces:
|
|
201
259
|
pid = t.project_id
|
|
@@ -231,6 +289,23 @@ def compute_projects(traces: List[Trace]) -> list:
|
|
|
231
289
|
return result
|
|
232
290
|
|
|
233
291
|
|
|
292
|
+
def compute_filter_options(traces: List[Trace]) -> dict:
|
|
293
|
+
"""Return available values for cascading filter dropdowns: project_name → project_id → user_id."""
|
|
294
|
+
project_names = sorted({t.project_name for t in traces if t.project_name})
|
|
295
|
+
by_name: dict[str, set] = defaultdict(set)
|
|
296
|
+
by_pid: dict[str, set] = defaultdict(set)
|
|
297
|
+
for t in traces:
|
|
298
|
+
if t.project_name and t.project_id:
|
|
299
|
+
by_name[t.project_name].add(t.project_id)
|
|
300
|
+
if t.project_id and t.user_id:
|
|
301
|
+
by_pid[t.project_id].add(t.user_id)
|
|
302
|
+
return {
|
|
303
|
+
"project_names": project_names,
|
|
304
|
+
"project_ids": {k: sorted(v) for k, v in by_name.items()},
|
|
305
|
+
"user_ids": {k: sorted(v) for k, v in by_pid.items()},
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
|
|
234
309
|
def _period_delta(period: str) -> timedelta:
|
|
235
310
|
mapping = {
|
|
236
311
|
"1h": timedelta(hours=1),
|
|
@@ -249,3 +324,18 @@ def _group_by_day(traces: List[Trace]) -> list:
|
|
|
249
324
|
groups[day]["traces"] += 1
|
|
250
325
|
groups[day]["cost_usd"] += t.cost_usd
|
|
251
326
|
return sorted(groups.values(), key=lambda x: x["date"])
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
def _group_by_day_and_model(traces: List[Trace]) -> list:
|
|
330
|
+
"""Group by (date, model) for multi-series token/cost chart."""
|
|
331
|
+
groups: dict[tuple, dict] = {}
|
|
332
|
+
for t in traces:
|
|
333
|
+
day = t.started_at.strftime("%Y-%m-%d")
|
|
334
|
+
model = t.model or "unknown"
|
|
335
|
+
key = (day, model)
|
|
336
|
+
if key not in groups:
|
|
337
|
+
groups[key] = {"date": day, "model": model, "tokens_in": 0, "tokens_out": 0, "cost_usd": 0.0}
|
|
338
|
+
groups[key]["tokens_in"] += t.total_tokens_in
|
|
339
|
+
groups[key]["tokens_out"] += t.total_tokens_out
|
|
340
|
+
groups[key]["cost_usd"] = round(groups[key]["cost_usd"] + t.cost_usd, 6)
|
|
341
|
+
return sorted(groups.values(), key=lambda x: (x["date"], x["model"]))
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
from typing import List, Optional
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class EvalReader:
|
|
5
|
+
def __init__(self, exporters: list):
|
|
6
|
+
self._exporters = exporters
|
|
7
|
+
|
|
8
|
+
def _readable(self):
|
|
9
|
+
for exporter in self._exporters:
|
|
10
|
+
if callable(getattr(exporter, "query_evals", None)):
|
|
11
|
+
return exporter
|
|
12
|
+
return None
|
|
13
|
+
|
|
14
|
+
def list_runs(self, *, project_id=None, dataset_name=None, from_dt=None, to_dt=None,
|
|
15
|
+
limit: int = 50, offset: int = 0) -> List[dict]:
|
|
16
|
+
exporter = self._readable()
|
|
17
|
+
if exporter is None:
|
|
18
|
+
return []
|
|
19
|
+
return exporter.query_evals(
|
|
20
|
+
project_id=project_id, dataset_name=dataset_name,
|
|
21
|
+
from_dt=from_dt, to_dt=to_dt, limit=limit, offset=offset,
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
def get_run(self, run_id: str) -> Optional[dict]:
|
|
25
|
+
exporter = self._readable()
|
|
26
|
+
if exporter is None:
|
|
27
|
+
return None
|
|
28
|
+
return exporter.get_eval(run_id)
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
from typing import List, Optional
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class PromptReader:
|
|
5
|
+
def __init__(self, exporters: list):
|
|
6
|
+
self._exporters = exporters
|
|
7
|
+
|
|
8
|
+
def _readable(self):
|
|
9
|
+
for exporter in self._exporters:
|
|
10
|
+
if callable(getattr(exporter, "query_prompts", None)):
|
|
11
|
+
return exporter
|
|
12
|
+
return None
|
|
13
|
+
|
|
14
|
+
def list_prompts(self) -> List[dict]:
|
|
15
|
+
"""One summary per prompt name (latest version + labels)."""
|
|
16
|
+
exporter = self._readable()
|
|
17
|
+
if exporter is None:
|
|
18
|
+
return []
|
|
19
|
+
rows = exporter.query_prompts()
|
|
20
|
+
by_name: dict = {}
|
|
21
|
+
for row in rows:
|
|
22
|
+
name = row.get("name")
|
|
23
|
+
entry = by_name.setdefault(name, {"name": name, "latest_version": 0,
|
|
24
|
+
"versions": 0, "labels": []})
|
|
25
|
+
entry["versions"] += 1
|
|
26
|
+
if row.get("version", 0) >= entry["latest_version"]:
|
|
27
|
+
entry["latest_version"] = row.get("version", 0)
|
|
28
|
+
for label in row.get("labels", []):
|
|
29
|
+
if label not in entry["labels"]:
|
|
30
|
+
entry["labels"].append(label)
|
|
31
|
+
return sorted(by_name.values(), key=lambda e: e["name"])
|
|
32
|
+
|
|
33
|
+
def get_versions(self, name: str) -> Optional[List[dict]]:
|
|
34
|
+
exporter = self._readable()
|
|
35
|
+
if exporter is None:
|
|
36
|
+
return None
|
|
37
|
+
rows = [r for r in exporter.query_prompts(name=name) if r.get("name") == name]
|
|
38
|
+
if not rows:
|
|
39
|
+
return None
|
|
40
|
+
return sorted(rows, key=lambda r: r.get("version", 0))
|