tracecast 0.2.1__tar.gz → 0.2.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (108) hide show
  1. {tracecast-0.2.1/tracecast.egg-info → tracecast-0.2.2}/PKG-INFO +1 -1
  2. {tracecast-0.2.1 → tracecast-0.2.2}/pyproject.toml +1 -1
  3. tracecast-0.2.2/tests/test_prompt_dashboard.py +56 -0
  4. tracecast-0.2.2/tests/test_prompts.py +84 -0
  5. tracecast-0.2.2/tests/test_score.py +56 -0
  6. tracecast-0.2.2/tests/test_score_dashboard.py +49 -0
  7. tracecast-0.2.2/tests/test_scoring_api.py +49 -0
  8. {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/__init__.py +6 -2
  9. tracecast-0.2.2/tracecast/core/scoring.py +53 -0
  10. {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/core/tracer.py +17 -2
  11. {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/dashboard/aggregator.py +94 -4
  12. tracecast-0.2.2/tracecast/dashboard/eval_reader.py +28 -0
  13. tracecast-0.2.2/tracecast/dashboard/prompt_reader.py +40 -0
  14. {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/dashboard/reader.py +22 -2
  15. {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/dashboard/router.py +94 -3
  16. tracecast-0.2.2/tracecast/dashboard/score_reader.py +28 -0
  17. tracecast-0.2.2/tracecast/dashboard/static/assets/index-CBra-omk.js +151 -0
  18. tracecast-0.2.2/tracecast/dashboard/static/assets/index-DllJJzmE.css +1 -0
  19. {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/dashboard/static/index.html +2 -2
  20. tracecast-0.2.2/tracecast/eval/__init__.py +25 -0
  21. tracecast-0.2.2/tracecast/eval/cli.py +59 -0
  22. tracecast-0.2.2/tracecast/eval/compare.py +42 -0
  23. tracecast-0.2.2/tracecast/eval/dataset.py +136 -0
  24. tracecast-0.2.2/tracecast/eval/decorator.py +72 -0
  25. tracecast-0.2.2/tracecast/eval/judge.py +124 -0
  26. tracecast-0.2.2/tracecast/eval/metrics.py +128 -0
  27. tracecast-0.2.2/tracecast/eval/models.py +221 -0
  28. tracecast-0.2.2/tracecast/eval/online.py +71 -0
  29. tracecast-0.2.2/tracecast/eval/runner.py +160 -0
  30. tracecast-0.2.2/tracecast/eval/scorers.py +42 -0
  31. tracecast-0.2.2/tracecast/exporters/_eval_store.py +59 -0
  32. tracecast-0.2.2/tracecast/exporters/base.py +44 -0
  33. tracecast-0.2.2/tracecast/exporters/dict_exporter.py +79 -0
  34. tracecast-0.2.2/tracecast/exporters/json_file.py +117 -0
  35. {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/exporters/mongo.py +79 -2
  36. {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/exporters/postgres.py +195 -0
  37. {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/instrumentors/openai_inst.py +20 -2
  38. {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/integrations/langchain.py +55 -1
  39. tracecast-0.2.2/tracecast/models/score.py +84 -0
  40. {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/models/trace.py +2 -0
  41. tracecast-0.2.2/tracecast/prompts/__init__.py +4 -0
  42. tracecast-0.2.2/tracecast/prompts/client.py +134 -0
  43. tracecast-0.2.2/tracecast/prompts/models.py +49 -0
  44. {tracecast-0.2.1 → tracecast-0.2.2/tracecast.egg-info}/PKG-INFO +1 -1
  45. {tracecast-0.2.1 → tracecast-0.2.2}/tracecast.egg-info/SOURCES.txt +28 -3
  46. tracecast-0.2.1/tracecast/dashboard/static/assets/index-BAxBR6Dz.css +0 -1
  47. tracecast-0.2.1/tracecast/dashboard/static/assets/index-DaMldkDH.js +0 -155
  48. tracecast-0.2.1/tracecast/exporters/base.py +0 -16
  49. tracecast-0.2.1/tracecast/exporters/dict_exporter.py +0 -35
  50. tracecast-0.2.1/tracecast/exporters/json_file.py +0 -37
  51. {tracecast-0.2.1 → tracecast-0.2.2}/LICENSE +0 -0
  52. {tracecast-0.2.1 → tracecast-0.2.2}/README.md +0 -0
  53. {tracecast-0.2.1 → tracecast-0.2.2}/setup.cfg +0 -0
  54. {tracecast-0.2.1 → tracecast-0.2.2}/tests/test_context_propagation.py +0 -0
  55. {tracecast-0.2.1 → tracecast-0.2.2}/tests/test_cost_calculator.py +0 -0
  56. {tracecast-0.2.1 → tracecast-0.2.2}/tests/test_dashboard_mount.py +0 -0
  57. {tracecast-0.2.1 → tracecast-0.2.2}/tests/test_dashboard_pushdown.py +0 -0
  58. {tracecast-0.2.1 → tracecast-0.2.2}/tests/test_dashboard_sessions.py +0 -0
  59. {tracecast-0.2.1 → tracecast-0.2.2}/tests/test_decorators.py +0 -0
  60. {tracecast-0.2.1 → tracecast-0.2.2}/tests/test_e2e_langgraph.py +0 -0
  61. {tracecast-0.2.1 → tracecast-0.2.2}/tests/test_exporters_db.py +0 -0
  62. {tracecast-0.2.1 → tracecast-0.2.2}/tests/test_integration_frameworks.py +0 -0
  63. {tracecast-0.2.1 → tracecast-0.2.2}/tests/test_integration_langchain.py +0 -0
  64. {tracecast-0.2.1 → tracecast-0.2.2}/tests/test_json_exporter.py +0 -0
  65. {tracecast-0.2.1 → tracecast-0.2.2}/tests/test_langchain_callback.py +0 -0
  66. {tracecast-0.2.1 → tracecast-0.2.2}/tests/test_llm_wrappers.py +0 -0
  67. {tracecast-0.2.1 → tracecast-0.2.2}/tests/test_logging.py +0 -0
  68. {tracecast-0.2.1 → tracecast-0.2.2}/tests/test_mongo_exporter.py +0 -0
  69. {tracecast-0.2.1 → tracecast-0.2.2}/tests/test_roundtrip.py +0 -0
  70. {tracecast-0.2.1 → tracecast-0.2.2}/tests/test_schema_v2.py +0 -0
  71. {tracecast-0.2.1 → tracecast-0.2.2}/tests/test_serve.py +0 -0
  72. {tracecast-0.2.1 → tracecast-0.2.2}/tests/test_span.py +0 -0
  73. {tracecast-0.2.1 → tracecast-0.2.2}/tests/test_streaming.py +0 -0
  74. {tracecast-0.2.1 → tracecast-0.2.2}/tests/test_token_counter.py +0 -0
  75. {tracecast-0.2.1 → tracecast-0.2.2}/tests/test_trace.py +0 -0
  76. {tracecast-0.2.1 → tracecast-0.2.2}/tests/test_trace_span.py +0 -0
  77. {tracecast-0.2.1 → tracecast-0.2.2}/tests/test_tracer.py +0 -0
  78. {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/core/__init__.py +0 -0
  79. {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/core/cost_calculator.py +0 -0
  80. {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/core/logger.py +0 -0
  81. {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/core/token_counter.py +0 -0
  82. {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/dashboard/__init__.py +0 -0
  83. {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/dashboard/asgi_middleware.py +0 -0
  84. {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/dashboard/blueprint.py +0 -0
  85. {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/dashboard/standalone.py +0 -0
  86. {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/decorators.py +0 -0
  87. {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/exporters/__init__.py +0 -0
  88. {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/exporters/query.py +0 -0
  89. {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/instrument.py +0 -0
  90. {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/instrumentors/__init__.py +0 -0
  91. {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/instrumentors/_llamaindex_handler.py +0 -0
  92. {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/instrumentors/_streaming.py +0 -0
  93. {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/instrumentors/anthropic_inst.py +0 -0
  94. {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/instrumentors/base.py +0 -0
  95. {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/instrumentors/crewai_inst.py +0 -0
  96. {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/instrumentors/gemini_inst.py +0 -0
  97. {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/instrumentors/langchain_inst.py +0 -0
  98. {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/instrumentors/llamaindex_inst.py +0 -0
  99. {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/integrations/__init__.py +0 -0
  100. {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/integrations/llm.py +0 -0
  101. {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/middleware.py +0 -0
  102. {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/models/__init__.py +0 -0
  103. {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/models/span.py +0 -0
  104. {tracecast-0.2.1 → tracecast-0.2.2}/tracecast/serve.py +0 -0
  105. {tracecast-0.2.1 → tracecast-0.2.2}/tracecast.egg-info/dependency_links.txt +0 -0
  106. {tracecast-0.2.1 → tracecast-0.2.2}/tracecast.egg-info/entry_points.txt +0 -0
  107. {tracecast-0.2.1 → tracecast-0.2.2}/tracecast.egg-info/requires.txt +0 -0
  108. {tracecast-0.2.1 → tracecast-0.2.2}/tracecast.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: tracecast
3
- Version: 0.2.1
3
+ Version: 0.2.2
4
4
  Summary: LLM observability SDK — framework-agnostic tracing for AI applications
5
5
  Author: Pedro Castanheira Costa
6
6
  License: MIT
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "tracecast"
7
- version = "0.2.1"
7
+ version = "0.2.2"
8
8
  description = "LLM observability SDK — framework-agnostic tracing for AI applications"
9
9
  readme = "README.md"
10
10
  license = { text = "MIT" }
@@ -0,0 +1,56 @@
1
+ import pytest
2
+
3
+ from tracecast import Tracer
4
+ from tracecast.prompts import create_prompt
5
+ from tracecast.prompts.client import clear_cache
6
+ from tracecast.exporters.dict_exporter import DictExporter
7
+
8
+ fastapi = pytest.importorskip("fastapi")
9
+ from fastapi import FastAPI
10
+ from fastapi.testclient import TestClient
11
+
12
+
13
+ def _client(exporter):
14
+ tracer = Tracer(exporters=[exporter])
15
+ app = FastAPI()
16
+ tracer.mount(app, prefix="/tc")
17
+ return TestClient(app)
18
+
19
+
20
+ def test_list_and_detail_prompts():
21
+ clear_cache()
22
+ exp = DictExporter()
23
+ create_prompt("greeting", "v1", labels=["production"], exporters=[exp])
24
+ create_prompt("greeting", "v2", exporters=[exp])
25
+ create_prompt("farewell", "bye", exporters=[exp])
26
+ client = _client(exp)
27
+
28
+ listing = client.get("/tc/api/prompts")
29
+ assert listing.status_code == 200
30
+ body = listing.json()
31
+ assert body["total"] == 2 # 2 distinct names
32
+ names = {p["name"] for p in body["prompts"]}
33
+ assert names == {"greeting", "farewell"}
34
+
35
+ detail = client.get("/tc/api/prompts/greeting")
36
+ assert detail.status_code == 200
37
+ versions = detail.json()["versions"]
38
+ assert [v["version"] for v in versions] == [1, 2]
39
+
40
+
41
+ def test_detail_unknown_prompt_404():
42
+ exp = DictExporter()
43
+ client = _client(exp)
44
+ assert client.get("/tc/api/prompts/ghost").status_code == 404
45
+
46
+
47
+ def test_prompt_reader_groups_versions():
48
+ from tracecast.dashboard.prompt_reader import PromptReader
49
+ clear_cache()
50
+ exp = DictExporter()
51
+ create_prompt("p", "v1", exporters=[exp])
52
+ create_prompt("p", "v2", exporters=[exp])
53
+ reader = PromptReader([exp])
54
+ summaries = reader.list_prompts()
55
+ assert summaries[0]["name"] == "p"
56
+ assert summaries[0]["latest_version"] == 2
@@ -0,0 +1,84 @@
1
+ import pytest
2
+
3
+ from tracecast import Tracer
4
+ from tracecast.prompts import create_prompt, get_prompt, set_label, PromptVersion
5
+ from tracecast.exporters.dict_exporter import DictExporter
6
+
7
+
8
+ def test_create_increments_version():
9
+ exp = DictExporter()
10
+ v1 = create_prompt("greeting", "Hello {name}", exporters=[exp])
11
+ v2 = create_prompt("greeting", "Hi {name}", exporters=[exp])
12
+ assert v1.version == 1
13
+ assert v2.version == 2
14
+
15
+
16
+ def test_get_prompt_defaults_to_highest_version():
17
+ exp = DictExporter()
18
+ create_prompt("p", "a", exporters=[exp])
19
+ create_prompt("p", "b", exporters=[exp])
20
+ assert get_prompt("p", label=None, exporters=[exp]).version == 2
21
+
22
+
23
+ def test_get_prompt_by_explicit_version():
24
+ exp = DictExporter()
25
+ create_prompt("p", "a", exporters=[exp])
26
+ create_prompt("p", "b", exporters=[exp])
27
+ assert get_prompt("p", version=1, exporters=[exp]).template == "a"
28
+
29
+
30
+ def test_label_resolution_and_move():
31
+ exp = DictExporter()
32
+ create_prompt("p", "v1", labels=["production"], exporters=[exp])
33
+ create_prompt("p", "v2", exporters=[exp])
34
+ assert get_prompt("p", label="production", exporters=[exp]).version == 1
35
+ set_label("p", 2, "production", exporters=[exp])
36
+ assert get_prompt("p", label="production", exporters=[exp]).version == 2
37
+
38
+
39
+ def test_unknown_label_raises_with_available():
40
+ exp = DictExporter()
41
+ create_prompt("p", "v1", labels=["staging"], exporters=[exp])
42
+ with pytest.raises(ValueError) as ei:
43
+ get_prompt("p", label="production", exporters=[exp])
44
+ assert "staging" in str(ei.value)
45
+
46
+
47
+ def test_no_exporter_raises():
48
+ with pytest.raises(RuntimeError):
49
+ get_prompt("p", exporters=[])
50
+
51
+
52
+ def test_cache_avoids_repeat_query():
53
+ class CountingExp(DictExporter):
54
+ def __init__(self):
55
+ super().__init__()
56
+ self.query_count = 0
57
+
58
+ def query_prompts(self, *, name=None):
59
+ self.query_count += 1
60
+ return super().query_prompts(name=name)
61
+
62
+ exp = CountingExp()
63
+ create_prompt("p", "v1", labels=["production"], exporters=[exp])
64
+ before = exp.query_count
65
+ get_prompt("p", label="production", exporters=[exp], cache_ttl=60)
66
+ get_prompt("p", label="production", exporters=[exp], cache_ttl=60)
67
+ assert exp.query_count == before + 1
68
+
69
+
70
+ def test_auto_link_to_active_trace():
71
+ exp = DictExporter()
72
+ create_prompt("p", "v1", labels=["production"], exporters=[exp])
73
+ tracer = Tracer(exporters=[exp])
74
+ with tracer.trace(name="t") as tr:
75
+ get_prompt("p", label="production", exporters=[exp])
76
+ assert tr.metadata.get("prompt_name") == "p"
77
+ assert tr.metadata.get("prompt_version") == 1
78
+
79
+
80
+ def test_promptversion_roundtrip():
81
+ pv = PromptVersion(name="p", version=1, template="t", labels=["production"])
82
+ back = PromptVersion.from_dict(pv.to_dict())
83
+ assert back.name == "p"
84
+ assert back.labels == ["production"]
@@ -0,0 +1,56 @@
1
+ import pytest
2
+ from datetime import datetime, timezone
3
+
4
+ from tracecast.models.score import Score
5
+
6
+
7
+ def test_numeric_score_roundtrip():
8
+ s = Score(trace_id="t1", name="quality", value=0.8)
9
+ s.validate()
10
+ d = s.to_dict()
11
+ assert d["trace_id"] == "t1"
12
+ assert d["name"] == "quality"
13
+ assert d["value"] == 0.8
14
+ assert d["data_type"] == "numeric"
15
+ assert d["kind"] == "human"
16
+ back = Score.from_dict(d)
17
+ assert back.trace_id == "t1"
18
+ assert back.value == 0.8
19
+
20
+
21
+ def test_boolean_score_rejects_non_binary():
22
+ Score(trace_id="t1", name="thumb", value=1, data_type="boolean").validate()
23
+ Score(trace_id="t1", name="thumb", value=0, data_type="boolean").validate()
24
+ with pytest.raises(ValueError):
25
+ Score(trace_id="t1", name="thumb", value=0.5, data_type="boolean").validate()
26
+
27
+
28
+ def test_categorical_requires_string_value():
29
+ with pytest.raises(ValueError):
30
+ Score(trace_id="t1", name="sentiment", value=0, data_type="categorical").validate()
31
+ ok = Score(trace_id="t1", name="sentiment", value=2,
32
+ data_type="categorical", string_value="positive")
33
+ ok.validate()
34
+ assert ok.to_dict()["string_value"] == "positive"
35
+
36
+
37
+ def test_invalid_data_type_rejected():
38
+ with pytest.raises(ValueError):
39
+ Score(trace_id="t1", name="x", value=1, data_type="bogus").validate()
40
+
41
+
42
+ def test_invalid_kind_rejected():
43
+ with pytest.raises(ValueError):
44
+ Score(trace_id="t1", name="x", value=1, kind="robot").validate()
45
+
46
+
47
+ def test_score_has_id_and_created_at():
48
+ s = Score(trace_id="t1", name="x", value=1)
49
+ assert s.score_id
50
+ assert isinstance(s.created_at, datetime)
51
+
52
+
53
+ def test_span_id_optional():
54
+ s = Score(trace_id="t1", span_id="sp1", name="x", value=1)
55
+ assert s.to_dict()["span_id"] == "sp1"
56
+ assert Score(trace_id="t1", name="x", value=1).to_dict()["span_id"] is None
@@ -0,0 +1,49 @@
1
+ import pytest
2
+
3
+ import tracecast
4
+ from tracecast import Tracer
5
+ from tracecast.exporters.dict_exporter import DictExporter
6
+
7
+ fastapi = pytest.importorskip("fastapi")
8
+ from fastapi import FastAPI
9
+ from fastapi.testclient import TestClient
10
+
11
+
12
+ def _client(exporter):
13
+ tracer = Tracer(exporters=[exporter])
14
+ app = FastAPI()
15
+ tracer.mount(app, prefix="/tc")
16
+ return TestClient(app), tracer
17
+
18
+
19
+ def test_trace_scores_endpoint():
20
+ exp = DictExporter()
21
+ client, _ = _client(exp)
22
+ tracecast.score("t1", name="user_feedback", value=1, comment="good", exporters=[exp])
23
+ tracecast.score("t1", name="quality", value=0.9, exporters=[exp])
24
+ tracecast.score("other", name="quality", value=0.1, exporters=[exp])
25
+
26
+ resp = client.get("/tc/api/traces/t1/scores")
27
+ assert resp.status_code == 200
28
+ body = resp.json()
29
+ assert body["total"] == 2
30
+ names = {s["name"] for s in body["scores"]}
31
+ assert names == {"user_feedback", "quality"}
32
+
33
+
34
+ def test_trace_scores_empty():
35
+ exp = DictExporter()
36
+ client, _ = _client(exp)
37
+ resp = client.get("/tc/api/traces/nope/scores")
38
+ assert resp.status_code == 200
39
+ assert resp.json() == {"scores": [], "total": 0}
40
+
41
+
42
+ def test_score_reader_list_for_trace():
43
+ from tracecast.dashboard.score_reader import ScoreReader
44
+ exp = DictExporter()
45
+ tracecast.score("t1", name="a", value=1, exporters=[exp])
46
+ reader = ScoreReader([exp])
47
+ rows = reader.list_for_trace("t1")
48
+ assert len(rows) == 1
49
+ assert rows[0]["name"] == "a"
@@ -0,0 +1,49 @@
1
+ import pytest
2
+
3
+ import tracecast
4
+ from tracecast import Tracer, set_default_tracer
5
+ from tracecast.exporters.dict_exporter import DictExporter
6
+
7
+
8
+ def test_score_persists_via_explicit_exporter():
9
+ exp = DictExporter()
10
+ s = tracecast.score("t1", name="user_feedback", value=1, comment="useful", exporters=[exp])
11
+ assert s.trace_id == "t1"
12
+ rows = exp.query_scores(trace_id="t1")
13
+ assert len(rows) == 1
14
+ assert rows[0]["name"] == "user_feedback"
15
+ assert rows[0]["comment"] == "useful"
16
+ assert rows[0]["source"] == "sdk"
17
+
18
+
19
+ def test_score_uses_default_tracer_exporters():
20
+ exp = DictExporter()
21
+ set_default_tracer(Tracer(exporters=[exp]))
22
+ try:
23
+ tracecast.score("t2", name="q", value=0.5)
24
+ assert len(exp.query_scores(trace_id="t2")) == 1
25
+ finally:
26
+ set_default_tracer(Tracer(exporters=[]))
27
+
28
+
29
+ def test_invalid_value_rejected_before_persist():
30
+ exp = DictExporter()
31
+ with pytest.raises(ValueError):
32
+ tracecast.score("t1", name="thumb", value=0.5, data_type="boolean", exporters=[exp])
33
+ assert exp.query_scores(trace_id="t1") == []
34
+
35
+
36
+ def test_exporter_failure_does_not_propagate():
37
+ class Boom(DictExporter):
38
+ def export_score(self, score):
39
+ raise RuntimeError("db down")
40
+ s = tracecast.score("t3", name="q", value=1, exporters=[Boom()])
41
+ assert s.trace_id == "t3" # returned despite export failure
42
+
43
+
44
+ def test_query_scores_filter_by_trace():
45
+ exp = DictExporter()
46
+ tracecast.score("a", name="q", value=1, exporters=[exp])
47
+ tracecast.score("b", name="q", value=1, exporters=[exp])
48
+ assert {r["trace_id"] for r in exp.query_scores(trace_id="a")} == {"a"}
49
+ assert len(exp.query_scores()) == 2
@@ -3,7 +3,10 @@ from .core.cost_calculator import calculate_cost
3
3
  from .core.token_counter import extract_tokens, extract_content, extract_input_text
4
4
  from .models.trace import Trace
5
5
  from .models.span import Span, SpanType, SpanStatus
6
+ from .models.score import Score
6
7
  from .decorators import trace_cast, trace_span, set_default_tracer
8
+ from .core.scoring import score
9
+ from .prompts import create_prompt, get_prompt, set_label, PromptVersion
7
10
  from .integrations.llm import trace_llm_call, wrap_openai, wrap_anthropic
8
11
  from .instrument import auto_instrument
9
12
  from .exporters.dict_exporter import DictExporter
@@ -11,8 +14,9 @@ from .exporters.dict_exporter import DictExporter
11
14
  __all__ = [
12
15
  "Tracer", "bind_context", "calculate_cost",
13
16
  "extract_tokens", "extract_content", "extract_input_text",
14
- "Trace", "Span", "SpanType", "SpanStatus",
15
- "trace_cast", "trace_span", "set_default_tracer",
17
+ "Trace", "Span", "SpanType", "SpanStatus", "Score",
18
+ "trace_cast", "trace_span", "set_default_tracer", "score",
19
+ "create_prompt", "get_prompt", "set_label", "PromptVersion",
16
20
  "trace_llm_call", "wrap_openai", "wrap_anthropic",
17
21
  "auto_instrument", "DictExporter",
18
22
  ]
@@ -0,0 +1,53 @@
1
+ from typing import Any, Dict, List, Optional
2
+
3
+ from ..models.score import Score
4
+ from .logger import _logger
5
+
6
+
7
+ def _resolve_exporters(exporters, tracer) -> List:
8
+ if exporters is not None:
9
+ return exporters
10
+ if tracer is not None:
11
+ return getattr(tracer, "exporters", []) or []
12
+ from ..decorators import _default_tracer
13
+ if _default_tracer is not None:
14
+ return getattr(_default_tracer, "exporters", []) or []
15
+ return []
16
+
17
+
18
+ def score(
19
+ trace_id: str,
20
+ name: str,
21
+ value: float,
22
+ *,
23
+ span_id: Optional[str] = None,
24
+ kind: str = "human",
25
+ data_type: str = "numeric",
26
+ string_value: Optional[str] = None,
27
+ comment: Optional[str] = None,
28
+ source: str = "sdk",
29
+ project_id: Optional[str] = None,
30
+ metadata: Optional[Dict[str, Any]] = None,
31
+ exporters: Optional[List] = None,
32
+ tracer=None,
33
+ ) -> Score:
34
+ """Attach a Score to a production trace (or span) and persist it.
35
+
36
+ Validates before persisting; exporter failures are logged, never raised.
37
+ """
38
+ s = Score(
39
+ trace_id=trace_id, name=name, value=value, span_id=span_id, kind=kind,
40
+ data_type=data_type, string_value=string_value, comment=comment,
41
+ source=source, project_id=project_id, metadata=metadata or {},
42
+ )
43
+ s.validate()
44
+
45
+ for exporter in _resolve_exporters(exporters, tracer):
46
+ if callable(getattr(exporter, "export_score", None)):
47
+ try:
48
+ exporter.export_score(s)
49
+ except Exception as exc:
50
+ _logger.error(
51
+ "TraceCast: export_score failed for %s: %s", s.score_id, exc, exc_info=True
52
+ )
53
+ return s
@@ -40,22 +40,34 @@ class Tracer:
40
40
  logging: bool = False,
41
41
  log_prefix: Optional[str] = None,
42
42
  on_export_error: Optional[Callable[[Exception, Trace, BaseExporter], None]] = None,
43
+ online_eval=None,
43
44
  ):
44
45
  self.exporters = exporters or []
45
46
  self.on_export_error = on_export_error
47
+ self.online_eval = online_eval
46
48
  self._tc_logger = None
47
49
  if logging:
48
50
  from .logger import TraceCastLogger
49
51
  self._tc_logger = TraceCastLogger(prefix=log_prefix)
50
52
 
53
+ def _run_online_eval(self, trace: Trace) -> None:
54
+ if self.online_eval is None:
55
+ return
56
+ try:
57
+ self.online_eval.maybe_evaluate(trace, default_exporters=self.exporters)
58
+ except Exception:
59
+ from .logger import _logger
60
+ _logger.error("TraceCast: online_eval failed for trace %s", trace.trace_id, exc_info=True)
61
+
51
62
  @contextmanager
52
- def trace(self, name: str, session_id=None, user_id=None, project_id=None, metadata=None):
63
+ def trace(self, name: str, session_id=None, user_id=None, project_id=None, project_name=None, metadata=None):
53
64
  t = Trace(
54
65
  trace_id=str(uuid.uuid4()),
55
66
  name=name,
56
67
  session_id=session_id,
57
68
  user_id=user_id,
58
69
  project_id=project_id,
70
+ project_name=project_name,
59
71
  started_at=datetime.now(timezone.utc),
60
72
  metadata=metadata or {},
61
73
  )
@@ -79,13 +91,14 @@ class Tracer:
79
91
  self._export(t)
80
92
 
81
93
  @asynccontextmanager
82
- async def atrace(self, name: str, session_id=None, user_id=None, project_id=None, metadata=None):
94
+ async def atrace(self, name: str, session_id=None, user_id=None, project_id=None, project_name=None, metadata=None):
83
95
  t = Trace(
84
96
  trace_id=str(uuid.uuid4()),
85
97
  name=name,
86
98
  session_id=session_id,
87
99
  user_id=user_id,
88
100
  project_id=project_id,
101
+ project_name=project_name,
89
102
  started_at=datetime.now(timezone.utc),
90
103
  metadata=metadata or {},
91
104
  )
@@ -127,6 +140,7 @@ class Tracer:
127
140
  exporter.export(trace)
128
141
  except Exception as exc:
129
142
  self._handle_export_error(exc, trace, exporter)
143
+ self._run_online_eval(trace)
130
144
 
131
145
  async def _aexport(self, trace: Trace) -> None:
132
146
  for exporter in self.exporters:
@@ -134,6 +148,7 @@ class Tracer:
134
148
  await exporter.aexport(trace)
135
149
  except Exception as exc:
136
150
  self._handle_export_error(exc, trace, exporter)
151
+ self._run_online_eval(trace)
137
152
 
138
153
  @staticmethod
139
154
  def current() -> Optional[Trace]:
@@ -50,6 +50,7 @@ def compute_metrics(
50
50
  cost_by_project[t.project_id] += t.cost_usd
51
51
 
52
52
  traces_over_time = _group_by_day(filtered)
53
+ tokens_by_model_over_time = _group_by_day_and_model(filtered)
53
54
 
54
55
  return {
55
56
  "period": period,
@@ -63,6 +64,7 @@ def compute_metrics(
63
64
  "cost_by_model": dict(cost_by_model),
64
65
  "cost_by_project": dict(cost_by_project),
65
66
  "traces_over_time": traces_over_time,
67
+ "tokens_by_model_over_time": tokens_by_model_over_time,
66
68
  }
67
69
 
68
70
 
@@ -116,6 +118,7 @@ def _trace_summary(trace: Trace) -> dict:
116
118
  "trace_id": trace.trace_id,
117
119
  "name": trace.name,
118
120
  "project_id": trace.project_id,
121
+ "project_name": trace.project_name,
119
122
  "user_id": trace.user_id,
120
123
  "session_id": trace.session_id,
121
124
  "model": trace.model,
@@ -157,16 +160,33 @@ def build_graph(trace: Trace) -> dict:
157
160
  }
158
161
 
159
162
 
160
- def compute_sessions(traces: List[Trace]) -> list:
161
- """Aggregate traces by session_id. Traces without session_id are skipped."""
163
+ def compute_sessions(
164
+ traces: List[Trace],
165
+ *,
166
+ project_name: Optional[str] = None,
167
+ project_id: Optional[str] = None,
168
+ user_id: Optional[str] = None,
169
+ ) -> list:
170
+ """Aggregate traces by session_id with optional hierarchical filtering."""
171
+ filtered = traces
172
+ if project_name:
173
+ filtered = [t for t in filtered if t.project_name == project_name]
174
+ if project_id:
175
+ filtered = [t for t in filtered if t.project_id == project_id]
176
+ if user_id:
177
+ filtered = [t for t in filtered if t.user_id == user_id]
178
+
162
179
  groups: dict[str, dict] = {}
163
- for t in traces:
180
+ for t in filtered:
164
181
  sid = t.session_id
165
182
  if not sid:
166
183
  continue
167
184
  if sid not in groups:
168
185
  groups[sid] = {
169
186
  "session_id": sid,
187
+ "project_name": t.project_name,
188
+ "project_id": t.project_id,
189
+ "user_id": t.user_id,
170
190
  "trace_count": 0,
171
191
  "total_cost_usd": 0.0,
172
192
  "total_tokens": 0,
@@ -195,7 +215,45 @@ def compute_sessions(traces: List[Trace]) -> list:
195
215
 
196
216
 
197
217
  def compute_projects(traces: List[Trace]) -> list:
198
- """Aggregate traces by project_id. Traces without project_id are skipped."""
218
+ """Group by project_name if available; fall back to project_id."""
219
+ use_name = any(t.project_name for t in traces)
220
+ groups: dict[str, dict] = {}
221
+ for t in traces:
222
+ key = t.project_name if use_name else t.project_id
223
+ if not key:
224
+ continue
225
+ if key not in groups:
226
+ groups[key] = {
227
+ "project_name" if use_name else "project_id": key,
228
+ "trace_count": 0,
229
+ "total_cost_usd": 0.0,
230
+ "total_tokens": 0,
231
+ "total_tokens_in": 0,
232
+ "total_tokens_out": 0,
233
+ "_first_dt": t.started_at,
234
+ "_last_dt": t.started_at,
235
+ }
236
+ g = groups[key]
237
+ g["trace_count"] += 1
238
+ g["total_cost_usd"] += t.cost_usd
239
+ g["total_tokens"] += t.total_tokens
240
+ g["total_tokens_in"] += t.total_tokens_in
241
+ g["total_tokens_out"] += t.total_tokens_out
242
+ dt = t.started_at
243
+ if dt < g["_first_dt"]:
244
+ g["_first_dt"] = dt
245
+ if dt > g["_last_dt"]:
246
+ g["_last_dt"] = dt
247
+ result = sorted(groups.values(), key=lambda x: x["_last_dt"], reverse=True)
248
+ for r in result:
249
+ r["total_cost_usd"] = round(r["total_cost_usd"], 6)
250
+ r["first_trace_at"] = r.pop("_first_dt").isoformat()
251
+ r["last_trace_at"] = r.pop("_last_dt").isoformat()
252
+ return result
253
+
254
+
255
+ def compute_projects_by_id(traces: List[Trace]) -> list:
256
+ """Aggregate traces by project_id (filial level, used for drill-down under a project_name)."""
199
257
  groups: dict[str, dict] = {}
200
258
  for t in traces:
201
259
  pid = t.project_id
@@ -231,6 +289,23 @@ def compute_projects(traces: List[Trace]) -> list:
231
289
  return result
232
290
 
233
291
 
292
+ def compute_filter_options(traces: List[Trace]) -> dict:
293
+ """Return available values for cascading filter dropdowns: project_name → project_id → user_id."""
294
+ project_names = sorted({t.project_name for t in traces if t.project_name})
295
+ by_name: dict[str, set] = defaultdict(set)
296
+ by_pid: dict[str, set] = defaultdict(set)
297
+ for t in traces:
298
+ if t.project_name and t.project_id:
299
+ by_name[t.project_name].add(t.project_id)
300
+ if t.project_id and t.user_id:
301
+ by_pid[t.project_id].add(t.user_id)
302
+ return {
303
+ "project_names": project_names,
304
+ "project_ids": {k: sorted(v) for k, v in by_name.items()},
305
+ "user_ids": {k: sorted(v) for k, v in by_pid.items()},
306
+ }
307
+
308
+
234
309
  def _period_delta(period: str) -> timedelta:
235
310
  mapping = {
236
311
  "1h": timedelta(hours=1),
@@ -249,3 +324,18 @@ def _group_by_day(traces: List[Trace]) -> list:
249
324
  groups[day]["traces"] += 1
250
325
  groups[day]["cost_usd"] += t.cost_usd
251
326
  return sorted(groups.values(), key=lambda x: x["date"])
327
+
328
+
329
+ def _group_by_day_and_model(traces: List[Trace]) -> list:
330
+ """Group by (date, model) for multi-series token/cost chart."""
331
+ groups: dict[tuple, dict] = {}
332
+ for t in traces:
333
+ day = t.started_at.strftime("%Y-%m-%d")
334
+ model = t.model or "unknown"
335
+ key = (day, model)
336
+ if key not in groups:
337
+ groups[key] = {"date": day, "model": model, "tokens_in": 0, "tokens_out": 0, "cost_usd": 0.0}
338
+ groups[key]["tokens_in"] += t.total_tokens_in
339
+ groups[key]["tokens_out"] += t.total_tokens_out
340
+ groups[key]["cost_usd"] = round(groups[key]["cost_usd"] + t.cost_usd, 6)
341
+ return sorted(groups.values(), key=lambda x: (x["date"], x["model"]))
@@ -0,0 +1,28 @@
1
+ from typing import List, Optional
2
+
3
+
4
+ class EvalReader:
5
+ def __init__(self, exporters: list):
6
+ self._exporters = exporters
7
+
8
+ def _readable(self):
9
+ for exporter in self._exporters:
10
+ if callable(getattr(exporter, "query_evals", None)):
11
+ return exporter
12
+ return None
13
+
14
+ def list_runs(self, *, project_id=None, dataset_name=None, from_dt=None, to_dt=None,
15
+ limit: int = 50, offset: int = 0) -> List[dict]:
16
+ exporter = self._readable()
17
+ if exporter is None:
18
+ return []
19
+ return exporter.query_evals(
20
+ project_id=project_id, dataset_name=dataset_name,
21
+ from_dt=from_dt, to_dt=to_dt, limit=limit, offset=offset,
22
+ )
23
+
24
+ def get_run(self, run_id: str) -> Optional[dict]:
25
+ exporter = self._readable()
26
+ if exporter is None:
27
+ return None
28
+ return exporter.get_eval(run_id)
@@ -0,0 +1,40 @@
1
+ from typing import List, Optional
2
+
3
+
4
+ class PromptReader:
5
+ def __init__(self, exporters: list):
6
+ self._exporters = exporters
7
+
8
+ def _readable(self):
9
+ for exporter in self._exporters:
10
+ if callable(getattr(exporter, "query_prompts", None)):
11
+ return exporter
12
+ return None
13
+
14
+ def list_prompts(self) -> List[dict]:
15
+ """One summary per prompt name (latest version + labels)."""
16
+ exporter = self._readable()
17
+ if exporter is None:
18
+ return []
19
+ rows = exporter.query_prompts()
20
+ by_name: dict = {}
21
+ for row in rows:
22
+ name = row.get("name")
23
+ entry = by_name.setdefault(name, {"name": name, "latest_version": 0,
24
+ "versions": 0, "labels": []})
25
+ entry["versions"] += 1
26
+ if row.get("version", 0) >= entry["latest_version"]:
27
+ entry["latest_version"] = row.get("version", 0)
28
+ for label in row.get("labels", []):
29
+ if label not in entry["labels"]:
30
+ entry["labels"].append(label)
31
+ return sorted(by_name.values(), key=lambda e: e["name"])
32
+
33
+ def get_versions(self, name: str) -> Optional[List[dict]]:
34
+ exporter = self._readable()
35
+ if exporter is None:
36
+ return None
37
+ rows = [r for r in exporter.query_prompts(name=name) if r.get("name") == name]
38
+ if not rows:
39
+ return None
40
+ return sorted(rows, key=lambda r: r.get("version", 0))