trace-agent-server 0.5.0__tar.gz → 0.7.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/PKG-INFO +1 -1
- {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/pyproject.toml +1 -1
- {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/api/runs.py +4 -2
- {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/db.py +2 -1
- {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/schemas.py +4 -0
- {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/services/events.py +6 -2
- {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/services/graph.py +42 -13
- {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/services/otlp.py +19 -5
- {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/services/run_lifecycle.py +12 -3
- {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server.egg-info/PKG-INFO +1 -1
- {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server.egg-info/SOURCES.txt +2 -0
- trace_agent_server-0.7.0/tests/test_concurrent_writers.py +85 -0
- {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/tests/test_framework_observations.py +39 -0
- trace_agent_server-0.7.0/tests/test_graph.py +60 -0
- {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/tests/test_otlp_ingest.py +19 -0
- {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/tests/test_run_lifecycle.py +30 -0
- {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/tests/test_runs_api.py +4 -1
- {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/README.md +0 -0
- {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/setup.cfg +0 -0
- {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/__init__.py +0 -0
- {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/api/__init__.py +0 -0
- {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/api/otlp.py +0 -0
- {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/cli.py +0 -0
- {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/config.py +0 -0
- {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/main.py +0 -0
- {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/models.py +0 -0
- {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/services/__init__.py +0 -0
- {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/services/analytics.py +0 -0
- {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/services/audit_guardrails.py +0 -0
- {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/services/compare.py +0 -0
- {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/services/dashboard.py +0 -0
- {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/services/explanation.py +0 -0
- {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/services/observability.py +0 -0
- {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/services/otlp_decode.py +0 -0
- {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/services/provider.py +0 -0
- {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/services/run_summary.py +0 -0
- {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/services/stored_redaction.py +0 -0
- {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server.egg-info/dependency_links.txt +0 -0
- {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server.egg-info/entry_points.txt +0 -0
- {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server.egg-info/requires.txt +0 -0
- {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server.egg-info/top_level.txt +0 -0
- {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/tests/test_audit_guardrails_api.py +0 -0
- {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/tests/test_dashboard.py +0 -0
- {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/tests/test_events.py +0 -0
- {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/tests/test_inactivity_sweep.py +0 -0
- {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/tests/test_otlp_decode.py +0 -0
- {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/tests/test_providers.py +0 -0
- {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/tests/test_sdk_wrappers.py +0 -0
- {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/tests/test_sse.py +0 -0
- {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/tests/test_stored_redaction.py +0 -0
- {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/tests/test_turns.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: trace-agent-server
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.7.0
|
|
4
4
|
Summary: TraceAgent FastAPI backend server.
|
|
5
5
|
Author-email: Enrique Javier Villar Cea <villarcea.enriquejavier@gmail.com>, Javier Rodriguez Castellano <javirodriguezcastellano@gmail.com>
|
|
6
6
|
License: MIT
|
|
@@ -720,7 +720,7 @@ def store_observations(
|
|
|
720
720
|
model=observation.model,
|
|
721
721
|
error_code=observation.error_code,
|
|
722
722
|
payload_hash=stable_hash(observation.metadata),
|
|
723
|
-
payload_full=observation.metadata if capture_full else None,
|
|
723
|
+
payload_full={**observation.metadata, **(observation.content or {})} if capture_full else None,
|
|
724
724
|
metadata={
|
|
725
725
|
"provider": observation.provider,
|
|
726
726
|
"model": observation.model,
|
|
@@ -809,10 +809,12 @@ def store_observations(
|
|
|
809
809
|
def finish_run(
|
|
810
810
|
run_id: str,
|
|
811
811
|
payload: FinishRunRequest,
|
|
812
|
+
request: Request,
|
|
812
813
|
session: Session = Depends(get_session),
|
|
813
814
|
) -> dict[str, Any]:
|
|
814
815
|
run = _get_run_or_404(session, run_id)
|
|
815
|
-
|
|
816
|
+
capture_full = getattr(request.app.state.settings, "capture_full_payloads", True)
|
|
817
|
+
derivation = complete_run(session, run, payload.final_output, payload.artifacts, capture_full=capture_full)
|
|
816
818
|
session.commit()
|
|
817
819
|
return {"status": "completed", "run_id": run.id, "decisions": len(derivation["decisions"])}
|
|
818
820
|
|
|
@@ -12,7 +12,8 @@ class Base(DeclarativeBase):
|
|
|
12
12
|
|
|
13
13
|
|
|
14
14
|
def create_session_factory(database_url: str) -> tuple[object, sessionmaker[Session]]:
|
|
15
|
-
|
|
15
|
+
# SQLite lets one writer at a time: a request that finds the database busy waits for it instead of failing.
|
|
16
|
+
connect_args = {"check_same_thread": False, "timeout": 30} if database_url.startswith("sqlite") else {}
|
|
16
17
|
engine = create_engine(database_url, connect_args=connect_args, future=True)
|
|
17
18
|
session_factory = sessionmaker(bind=engine, autoflush=False, autocommit=False, expire_on_commit=False)
|
|
18
19
|
return engine, session_factory
|
|
@@ -61,6 +61,8 @@ class ToolResultInput(BaseModel):
|
|
|
61
61
|
status: Literal["succeeded", "failed"]
|
|
62
62
|
args: dict[str, Any] = Field(default_factory=dict)
|
|
63
63
|
output_summary: str | None = None
|
|
64
|
+
# What the tool returned, whole (masked by the sender). Stored only with full payloads on.
|
|
65
|
+
output: Any | None = None
|
|
64
66
|
output_hash: str | None = None
|
|
65
67
|
error_code: str | None = None
|
|
66
68
|
error_summary: str | None = None
|
|
@@ -113,6 +115,8 @@ class ObservationInput(BaseModel):
|
|
|
113
115
|
provider: str | None = None
|
|
114
116
|
input_summary: str | None = None
|
|
115
117
|
artifact: ArtifactInput | None = None
|
|
118
|
+
# What a model call was asked and answered (input_messages / output_messages). Stored only with full payloads on.
|
|
119
|
+
content: dict[str, Any] | None = None
|
|
116
120
|
metadata: dict[str, Any] = Field(default_factory=dict)
|
|
117
121
|
|
|
118
122
|
|
{trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/services/events.py
RENAMED
|
@@ -4,6 +4,7 @@ import asyncio
|
|
|
4
4
|
from datetime import datetime, timezone
|
|
5
5
|
from typing import Any
|
|
6
6
|
|
|
7
|
+
from sqlalchemy import update
|
|
7
8
|
from sqlalchemy.orm import Session
|
|
8
9
|
|
|
9
10
|
from trace_agent_server.models import Artifact, Event, Run
|
|
@@ -66,8 +67,11 @@ def record_event(
|
|
|
66
67
|
timestamp: datetime | None = None,
|
|
67
68
|
) -> Event:
|
|
68
69
|
"""Record an event; ``timestamp`` overrides "now" for sources that report when it happened."""
|
|
69
|
-
run
|
|
70
|
-
|
|
70
|
+
# The database increments the counter, it is not computed here: two requests writing to the same run at once
|
|
71
|
+
# (the agent recording by hand while the capability posts from its thread) would read the same value and
|
|
72
|
+
# collide on (run_id, seq). The second request waits for the first one's commit and gets the next number.
|
|
73
|
+
session.execute(update(Run).where(Run.id == run.id).values(last_event_seq=Run.last_event_seq + 1, updated_at=utcnow()))
|
|
74
|
+
session.refresh(run, ["last_event_seq"])
|
|
71
75
|
event = Event(
|
|
72
76
|
run_id=run.id,
|
|
73
77
|
turn_id=turn_id,
|
{trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/services/graph.py
RENAMED
|
@@ -5,36 +5,65 @@ from typing import Any
|
|
|
5
5
|
from trace_agent_server.models import Artifact, Decision, Event
|
|
6
6
|
|
|
7
7
|
|
|
8
|
+
def _steps(events: list[Event]) -> list[list[Event]]:
|
|
9
|
+
"""Events grouped by the step they belong to (a tool call's start and end, a model call's request and
|
|
10
|
+
response), in the order the steps began. An event without a step is a step of its own."""
|
|
11
|
+
steps: list[list[Event]] = []
|
|
12
|
+
position: dict[str, int] = {}
|
|
13
|
+
for event in events:
|
|
14
|
+
if event.step_id and event.step_id in position:
|
|
15
|
+
steps[position[event.step_id]].append(event)
|
|
16
|
+
continue
|
|
17
|
+
if event.step_id:
|
|
18
|
+
position[event.step_id] = len(steps)
|
|
19
|
+
steps.append([event])
|
|
20
|
+
return steps
|
|
21
|
+
|
|
22
|
+
|
|
8
23
|
def build_execution_graph(events: list[Event], artifacts: list[Artifact]) -> dict[str, Any]:
|
|
24
|
+
"""One node per step, not per event: a tool is one node with its result, not a 'started' and a 'finished'."""
|
|
9
25
|
nodes: list[dict[str, Any]] = []
|
|
10
26
|
edges: list[dict[str, Any]] = []
|
|
27
|
+
node_of_event: dict[int, str] = {}
|
|
11
28
|
x = 0.0
|
|
12
29
|
y = 0.0
|
|
13
30
|
|
|
14
|
-
|
|
31
|
+
steps = _steps(events)
|
|
32
|
+
for index, step in enumerate(steps):
|
|
33
|
+
last = step[-1]
|
|
34
|
+
node_id = f"event-{last.id}"
|
|
35
|
+
for event in step:
|
|
36
|
+
node_of_event[event.id] = node_id
|
|
37
|
+
failed = any(event.status == "failed" for event in step)
|
|
38
|
+
command = next((e.event_metadata.get("command") for e in step if e.type.startswith("command.") and e.event_metadata), None)
|
|
15
39
|
nodes.append(
|
|
16
40
|
{
|
|
17
|
-
"id":
|
|
41
|
+
"id": node_id,
|
|
18
42
|
"type": "event",
|
|
19
|
-
"label":
|
|
43
|
+
"label": last.type,
|
|
20
44
|
"position": {"x": x, "y": y},
|
|
21
45
|
"data": {
|
|
22
|
-
"summary":
|
|
23
|
-
|
|
24
|
-
"
|
|
25
|
-
"
|
|
26
|
-
"
|
|
46
|
+
"summary": last.summary,
|
|
47
|
+
# The last event of the step (its result) is the one the inspector opens.
|
|
48
|
+
"eventId": last.id,
|
|
49
|
+
"toolName": last.tool_name,
|
|
50
|
+
"status": "failed" if failed else last.status,
|
|
51
|
+
"durationMs": last.duration_ms,
|
|
52
|
+
"stepKind": step[0].type.split(".")[0],
|
|
53
|
+
"command": command,
|
|
54
|
+
"evidenceIds": [event.id for event in step],
|
|
27
55
|
},
|
|
28
56
|
}
|
|
29
57
|
)
|
|
30
58
|
if index > 0:
|
|
59
|
+
previous = nodes[index - 1]
|
|
31
60
|
edges.append(
|
|
32
61
|
{
|
|
33
|
-
"id": f"edge-
|
|
34
|
-
"source":
|
|
35
|
-
"target":
|
|
62
|
+
"id": f"edge-{previous['id']}-{node_id}",
|
|
63
|
+
"source": previous["id"],
|
|
64
|
+
"target": node_id,
|
|
36
65
|
"label": None,
|
|
37
|
-
"animated":
|
|
66
|
+
"animated": failed or last.type == "run.failed",
|
|
38
67
|
}
|
|
39
68
|
)
|
|
40
69
|
x += 220
|
|
@@ -62,7 +91,7 @@ def build_execution_graph(events: list[Event], artifacts: list[Artifact]) -> dic
|
|
|
62
91
|
edges.append(
|
|
63
92
|
{
|
|
64
93
|
"id": f"edge-artifact-{artifact.id}",
|
|
65
|
-
"source": f"event-{artifact.source_event_id}",
|
|
94
|
+
"source": node_of_event.get(artifact.source_event_id, f"event-{artifact.source_event_id}"),
|
|
66
95
|
"target": f"artifact-{artifact.id}",
|
|
67
96
|
"label": "artifact",
|
|
68
97
|
"animated": False,
|
{trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/services/otlp.py
RENAMED
|
@@ -120,6 +120,17 @@ def _first_user_text(input_messages: Any) -> str | None:
|
|
|
120
120
|
return None
|
|
121
121
|
|
|
122
122
|
|
|
123
|
+
def _with_instructions(input_messages: Any, instructions: Any) -> Any:
|
|
124
|
+
"""The instructions of the agent come beside the messages in the GenAI conventions: show them first, as a system message."""
|
|
125
|
+
parts = _parse_json(instructions)
|
|
126
|
+
if isinstance(parts, str):
|
|
127
|
+
parts = [{"type": "text", "content": parts}]
|
|
128
|
+
if not isinstance(parts, list) or not parts:
|
|
129
|
+
return input_messages
|
|
130
|
+
messages = input_messages if isinstance(input_messages, list) else []
|
|
131
|
+
return [{"role": "system", "parts": parts}, *messages]
|
|
132
|
+
|
|
133
|
+
|
|
123
134
|
def _requested_tools(output_messages: Any) -> list[str]:
|
|
124
135
|
names: list[str] = []
|
|
125
136
|
for message in output_messages if isinstance(output_messages, list) else []:
|
|
@@ -196,7 +207,7 @@ def _ingest_run(session: Session, run_id: str, spans: list[SpanRecord], capture_
|
|
|
196
207
|
_track_agent_calls(run, fresh)
|
|
197
208
|
closing = _closing_span(run, fresh) if was_open else None
|
|
198
209
|
if closing is not None:
|
|
199
|
-
_close_run(session, run, closing)
|
|
210
|
+
_close_run(session, run, closing, capture_full)
|
|
200
211
|
elif not was_open:
|
|
201
212
|
# Late spans of a finished run: refresh its decisions and flags.
|
|
202
213
|
derive_explanation(session, run)
|
|
@@ -328,7 +339,9 @@ def _model_steps(session: Session, run: Run, span: SpanRecord, capture_full: boo
|
|
|
328
339
|
payload = None
|
|
329
340
|
if capture_full:
|
|
330
341
|
payload = {
|
|
331
|
-
"input_messages":
|
|
342
|
+
"input_messages": _with_instructions(
|
|
343
|
+
_parse_json(attributes.get("gen_ai.input.messages")), attributes.get("gen_ai.system_instructions")
|
|
344
|
+
),
|
|
332
345
|
"output_messages": output_messages,
|
|
333
346
|
}
|
|
334
347
|
record_event(
|
|
@@ -359,7 +372,8 @@ def _tool_steps(session: Session, run: Run, span: SpanRecord, capture_full: bool
|
|
|
359
372
|
parent_step_id=span.parent_span_id,
|
|
360
373
|
status="failed" if failed else "succeeded",
|
|
361
374
|
args=redact_payload(arguments if isinstance(arguments, dict) else {"input": arguments} if arguments is not None else {}),
|
|
362
|
-
output_summary=summarize_value(_readable(result_text)) if result_text is not None else None,
|
|
375
|
+
output_summary=summarize_value(_readable(result_text), sort_keys=False) if result_text is not None else None,
|
|
376
|
+
output=_parse_json(result_text) if result_text is not None and not failed else None,
|
|
363
377
|
output_hash=stable_hash(result_text if not failed else {"error": message}),
|
|
364
378
|
error_code=code,
|
|
365
379
|
error_summary=message,
|
|
@@ -448,9 +462,9 @@ def _closing_span(run: Run, spans: list[SpanRecord]) -> SpanRecord | None:
|
|
|
448
462
|
return max(agents, key=lambda s: s.end)
|
|
449
463
|
|
|
450
464
|
|
|
451
|
-
def _close_run(session: Session, run: Run, span: SpanRecord) -> None:
|
|
465
|
+
def _close_run(session: Session, run: Run, span: SpanRecord, capture_full: bool) -> None:
|
|
452
466
|
if span.status_code == "ERROR":
|
|
453
467
|
code, message = _error_of(span)
|
|
454
468
|
fail_run_with(session, run, code, message, ended_at=span.end)
|
|
455
469
|
else:
|
|
456
|
-
complete_run(session, run, span.attributes.get("final_result"), ended_at=span.end)
|
|
470
|
+
complete_run(session, run, span.attributes.get("final_result"), ended_at=span.end, capture_full=capture_full)
|
|
@@ -86,7 +86,11 @@ def record_tool_result(
|
|
|
86
86
|
tool_name=result.name,
|
|
87
87
|
duration_ms=result.duration_ms,
|
|
88
88
|
payload_hash=result.output_hash,
|
|
89
|
-
payload_full=
|
|
89
|
+
payload_full=(
|
|
90
|
+
{"output_summary": result.output_summary, **({"output": result.output} if result.output is not None else {})}
|
|
91
|
+
if capture_full
|
|
92
|
+
else None
|
|
93
|
+
),
|
|
90
94
|
metadata={"call_id": result.call_id, "artifact_count": len(result.artifacts)},
|
|
91
95
|
timestamp=ended_at,
|
|
92
96
|
)
|
|
@@ -147,9 +151,13 @@ def complete_run(
|
|
|
147
151
|
artifacts: Sequence[ArtifactInput] = (),
|
|
148
152
|
*,
|
|
149
153
|
ended_at: datetime | None = None,
|
|
154
|
+
capture_full: bool = False,
|
|
150
155
|
) -> dict[str, Any]:
|
|
151
|
-
"""Mark a run completed, record its final output and derive its explanation.
|
|
152
|
-
|
|
156
|
+
"""Mark a run completed, record its final output and derive its explanation.
|
|
157
|
+
|
|
158
|
+
The summary of the output is always kept; the whole output only with ``capture_full``.
|
|
159
|
+
"""
|
|
160
|
+
final_summary = summarize_value(final_output, sort_keys=False) if final_output is not None else "Run finished without final output."
|
|
153
161
|
run.final_output_summary = final_summary
|
|
154
162
|
run.status = "completed"
|
|
155
163
|
run.ended_at = ended_at or utcnow()
|
|
@@ -161,6 +169,7 @@ def complete_run(
|
|
|
161
169
|
summary=final_summary,
|
|
162
170
|
status="completed",
|
|
163
171
|
payload_hash=stable_hash(final_output),
|
|
172
|
+
payload_full={"final_output": final_output} if capture_full and final_output is not None else None,
|
|
164
173
|
timestamp=ended_at,
|
|
165
174
|
)
|
|
166
175
|
for artifact_input in artifacts:
|
{trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server.egg-info/PKG-INFO
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: trace-agent-server
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.7.0
|
|
4
4
|
Summary: TraceAgent FastAPI backend server.
|
|
5
5
|
Author-email: Enrique Javier Villar Cea <villarcea.enriquejavier@gmail.com>, Javier Rodriguez Castellano <javirodriguezcastellano@gmail.com>
|
|
6
6
|
License: MIT
|
{trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server.egg-info/SOURCES.txt
RENAMED
|
@@ -32,9 +32,11 @@ src/trace_agent_server/services/run_lifecycle.py
|
|
|
32
32
|
src/trace_agent_server/services/run_summary.py
|
|
33
33
|
src/trace_agent_server/services/stored_redaction.py
|
|
34
34
|
tests/test_audit_guardrails_api.py
|
|
35
|
+
tests/test_concurrent_writers.py
|
|
35
36
|
tests/test_dashboard.py
|
|
36
37
|
tests/test_events.py
|
|
37
38
|
tests/test_framework_observations.py
|
|
39
|
+
tests/test_graph.py
|
|
38
40
|
tests/test_inactivity_sweep.py
|
|
39
41
|
tests/test_otlp_decode.py
|
|
40
42
|
tests/test_otlp_ingest.py
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import threading
|
|
4
|
+
|
|
5
|
+
import pytest
|
|
6
|
+
from fastapi.testclient import TestClient
|
|
7
|
+
|
|
8
|
+
from trace_agent_server.config import Settings
|
|
9
|
+
from trace_agent_server.main import create_app
|
|
10
|
+
from trace_agent_server.models import Event, Run
|
|
11
|
+
from trace_agent_server.services.events import record_event
|
|
12
|
+
|
|
13
|
+
WRITERS = 6
|
|
14
|
+
EVENTS_EACH = 25
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@pytest.fixture
|
|
18
|
+
def app(tmp_path):
|
|
19
|
+
return create_app(Settings(database_url=f"sqlite:///{tmp_path / 'writers.db'}", cors_origins=["http://localhost:5173"]))
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class TestSeveralWritersOnOneRun:
|
|
23
|
+
"""The capability posts from a background thread while the agent records by hand: both write to the same run."""
|
|
24
|
+
|
|
25
|
+
def test_events_recorded_at_the_same_time_never_collide(self, app) -> None:
|
|
26
|
+
with app.state.session_factory() as session:
|
|
27
|
+
run = Run(agent_name="agent", goal="goal")
|
|
28
|
+
session.add(run)
|
|
29
|
+
session.commit()
|
|
30
|
+
run_id = run.id
|
|
31
|
+
|
|
32
|
+
errors: list[BaseException] = []
|
|
33
|
+
start = threading.Barrier(WRITERS)
|
|
34
|
+
|
|
35
|
+
def write(writer: int) -> None:
|
|
36
|
+
try:
|
|
37
|
+
start.wait()
|
|
38
|
+
for index in range(EVENTS_EACH):
|
|
39
|
+
# Each request loads the run, records an event and commits, as the endpoints do.
|
|
40
|
+
with app.state.session_factory() as session:
|
|
41
|
+
run = session.get(Run, run_id)
|
|
42
|
+
record_event(session, run, actor="sdk", event_type="tool.started", summary=f"writer {writer} event {index}")
|
|
43
|
+
session.commit()
|
|
44
|
+
except BaseException as exc: # noqa: BLE001 - reported by the test below
|
|
45
|
+
errors.append(exc)
|
|
46
|
+
|
|
47
|
+
threads = [threading.Thread(target=write, args=(writer,)) for writer in range(WRITERS)]
|
|
48
|
+
for thread in threads:
|
|
49
|
+
thread.start()
|
|
50
|
+
for thread in threads:
|
|
51
|
+
thread.join(timeout=120)
|
|
52
|
+
|
|
53
|
+
assert errors == []
|
|
54
|
+
with app.state.session_factory() as session:
|
|
55
|
+
sequence = [seq for (seq,) in session.query(Event.seq).filter_by(run_id=run_id).order_by(Event.seq)]
|
|
56
|
+
assert session.get(Run, run_id).last_event_seq == WRITERS * EVENTS_EACH
|
|
57
|
+
assert sequence == list(range(1, WRITERS * EVENTS_EACH + 1))
|
|
58
|
+
|
|
59
|
+
def test_requests_to_the_same_run_at_the_same_time_all_succeed(self, app) -> None:
|
|
60
|
+
client = TestClient(app)
|
|
61
|
+
run_id = client.post("/api/runs", json={"agent_name": "agent", "goal": "goal"}).json()["run_id"]
|
|
62
|
+
failures: list[int] = []
|
|
63
|
+
start = threading.Barrier(4)
|
|
64
|
+
|
|
65
|
+
def post(writer: int) -> None:
|
|
66
|
+
start.wait()
|
|
67
|
+
for index in range(10):
|
|
68
|
+
response = client.post(
|
|
69
|
+
f"/api/runs/{run_id}/observations",
|
|
70
|
+
json={"observations": [{"kind": "command", "command": f"echo {writer}-{index}", "step_id": f"c-{writer}-{index}", "status": "succeeded", "exit_code": 0}]},
|
|
71
|
+
)
|
|
72
|
+
if response.status_code != 200:
|
|
73
|
+
failures.append(response.status_code)
|
|
74
|
+
|
|
75
|
+
threads = [threading.Thread(target=post, args=(writer,)) for writer in range(4)]
|
|
76
|
+
for thread in threads:
|
|
77
|
+
thread.start()
|
|
78
|
+
for thread in threads:
|
|
79
|
+
thread.join(timeout=120)
|
|
80
|
+
|
|
81
|
+
assert failures == []
|
|
82
|
+
with app.state.session_factory() as session:
|
|
83
|
+
seqs = [seq for (seq,) in session.query(Event.seq).filter_by(run_id=run_id).order_by(Event.seq)]
|
|
84
|
+
assert seqs == list(range(1, len(seqs) + 1))
|
|
85
|
+
assert len(seqs) == 1 + 4 * 10 * 2 # run.started, then command.started and command.succeeded for each command
|
|
@@ -120,6 +120,45 @@ class TestLlmCallUsage:
|
|
|
120
120
|
assert execution["assistant_turn_count"] == 2
|
|
121
121
|
|
|
122
122
|
|
|
123
|
+
class TestLlmCallContent:
|
|
124
|
+
content = {
|
|
125
|
+
"input_messages": [{"role": "user", "parts": [{"type": "text", "content": "¿Por qué bajó el OEE?"}]}],
|
|
126
|
+
"output_messages": [{"role": "assistant", "parts": [{"type": "text", "content": "Por las paradas."}]}],
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
def completed_event(self, client, run_id: str) -> Event:
|
|
130
|
+
with client.app.state.session_factory() as session:
|
|
131
|
+
event = session.query(Event).filter_by(run_id=run_id, type="llm.completed").one()
|
|
132
|
+
session.expunge(event)
|
|
133
|
+
return event
|
|
134
|
+
|
|
135
|
+
def test_what_the_model_was_asked_and_answered_is_stored_with_the_event(self, client) -> None:
|
|
136
|
+
run_id = start_run(client)
|
|
137
|
+
observe(client, run_id, kind="llm_call", step_id="l1", status="succeeded", provider="openai", model="m",
|
|
138
|
+
content=self.content, metadata={"prompt_tokens": 3})
|
|
139
|
+
|
|
140
|
+
event = self.completed_event(client, run_id)
|
|
141
|
+
assert event.payload_full["input_messages"] == self.content["input_messages"]
|
|
142
|
+
assert event.payload_full["output_messages"] == self.content["output_messages"]
|
|
143
|
+
assert event.payload_full["prompt_tokens"] == 3
|
|
144
|
+
# The content is not copied into the metadata that is always kept and listed.
|
|
145
|
+
assert "input_messages" not in event.event_metadata
|
|
146
|
+
|
|
147
|
+
def test_it_is_not_stored_when_full_payloads_are_off(self, tmp_path) -> None:
|
|
148
|
+
app = create_app(Settings(database_url=f"sqlite:///{tmp_path / 'off.db'}", cors_origins=["http://localhost:5173"], capture_full_payloads=False))
|
|
149
|
+
client = TestClient(app)
|
|
150
|
+
run_id = start_run(client)
|
|
151
|
+
observe(client, run_id, kind="llm_call", step_id="l1", status="succeeded", provider="openai", model="m", content=self.content)
|
|
152
|
+
|
|
153
|
+
assert self.completed_event(client, run_id).payload_full is None
|
|
154
|
+
|
|
155
|
+
def test_a_call_without_content_is_stored_as_before(self, client) -> None:
|
|
156
|
+
run_id = start_run(client)
|
|
157
|
+
observe(client, run_id, kind="llm_call", step_id="l1", status="succeeded", provider="openai", model="m", metadata={"prompt_tokens": 3})
|
|
158
|
+
|
|
159
|
+
assert self.completed_event(client, run_id).payload_full == {"prompt_tokens": 3}
|
|
160
|
+
|
|
161
|
+
|
|
123
162
|
class TestTokenUsageOfFrameworkRuns:
|
|
124
163
|
def model_call(self, client, run_id, step, **tokens) -> None:
|
|
125
164
|
observe(client, run_id, kind="llm_call", step_id=step, status="succeeded", provider="openai", model="m", metadata=tokens)
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from trace_agent_server.models import Artifact, Event
|
|
4
|
+
from trace_agent_server.services.graph import build_execution_graph
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def event(event_id: int, event_type: str, step_id: str | None, *, status: str | None = None, tool_name: str | None = None,
|
|
8
|
+
summary: str = "", metadata: dict | None = None, duration_ms: int | None = None) -> Event:
|
|
9
|
+
return Event(id=event_id, run_id="r", seq=event_id, actor="sdk", type=event_type, summary=summary, status=status,
|
|
10
|
+
step_id=step_id, tool_name=tool_name, event_metadata=metadata or {}, duration_ms=duration_ms)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class TestExecutionGraphSteps:
|
|
14
|
+
def test_a_tool_is_one_node_with_its_result_not_a_started_and_a_finished_node(self) -> None:
|
|
15
|
+
events = [
|
|
16
|
+
event(1, "run.started", None),
|
|
17
|
+
event(2, "tool.started", "t1", tool_name="search", status="completed"),
|
|
18
|
+
event(3, "tool.succeeded", "t1", tool_name="search", status="succeeded", summary="3 results", duration_ms=40),
|
|
19
|
+
event(4, "run.completed", None),
|
|
20
|
+
]
|
|
21
|
+
|
|
22
|
+
graph = build_execution_graph(events, [])
|
|
23
|
+
|
|
24
|
+
assert [n["data"]["eventId"] for n in graph["nodes"]] == [1, 3, 4]
|
|
25
|
+
tool = graph["nodes"][1]
|
|
26
|
+
assert (tool["label"], tool["data"]["summary"], tool["data"]["durationMs"], tool["data"]["toolName"]) == ("tool.succeeded", "3 results", 40, "search")
|
|
27
|
+
assert tool["data"]["evidenceIds"] == [2, 3]
|
|
28
|
+
assert tool["data"]["stepKind"] == "tool"
|
|
29
|
+
|
|
30
|
+
def test_the_nodes_are_chained_in_the_order_the_steps_began(self) -> None:
|
|
31
|
+
events = [event(1, "llm.started", "l1"), event(2, "tool.started", "t1"), event(3, "llm.completed", "l1"), event(4, "tool.succeeded", "t1")]
|
|
32
|
+
|
|
33
|
+
graph = build_execution_graph(events, [])
|
|
34
|
+
|
|
35
|
+
assert [n["data"]["eventId"] for n in graph["nodes"]] == [3, 4] # each node is named after the last event of its step
|
|
36
|
+
assert [(e["source"], e["target"]) for e in graph["edges"]] == [("event-3", "event-4")]
|
|
37
|
+
|
|
38
|
+
def test_a_step_that_failed_at_any_point_is_failed_and_its_edge_animated(self) -> None:
|
|
39
|
+
events = [event(1, "tool.started", "t1", status="completed"), event(2, "tool.failed", "t1", status="failed"), event(3, "tool.started", "t2")]
|
|
40
|
+
|
|
41
|
+
graph = build_execution_graph(events, [])
|
|
42
|
+
|
|
43
|
+
assert graph["nodes"][0]["data"]["status"] == "failed"
|
|
44
|
+
assert graph["edges"][0]["animated"] is False # the edge into the next step follows that step, which did not fail
|
|
45
|
+
|
|
46
|
+
def test_the_command_of_a_command_step_is_kept_for_its_title(self) -> None:
|
|
47
|
+
events = [event(1, "command.started", "c1", metadata={"command": "python -c # Leer\nprint(1)"}), event(2, "command.succeeded", "c1", metadata={"command": "python -c # Leer\nprint(1)"})]
|
|
48
|
+
|
|
49
|
+
graph = build_execution_graph(events, [])
|
|
50
|
+
|
|
51
|
+
assert graph["nodes"][0]["data"]["command"] == "python -c # Leer\nprint(1)"
|
|
52
|
+
assert graph["nodes"][0]["data"]["stepKind"] == "command"
|
|
53
|
+
|
|
54
|
+
def test_an_artifact_hangs_from_the_node_of_the_step_that_made_it(self) -> None:
|
|
55
|
+
events = [event(1, "tool.started", "t1"), event(2, "tool.succeeded", "t1")]
|
|
56
|
+
artifact = Artifact(id=9, run_id="r", kind="file", label="report", summary="the report", source_event_id=1)
|
|
57
|
+
|
|
58
|
+
graph = build_execution_graph(events, [artifact])
|
|
59
|
+
|
|
60
|
+
assert ("event-2", "artifact-9") in [(e["source"], e["target"]) for e in graph["edges"]]
|
|
@@ -190,6 +190,25 @@ class TestRunCreationAndMapping:
|
|
|
190
190
|
assert completed.event_metadata["prompt_tokens"] == 60
|
|
191
191
|
assert completed.event_metadata["requested_tools"] == ["get_weather"]
|
|
192
192
|
|
|
193
|
+
def test_what_the_model_was_asked_and_answered_is_kept_with_the_instructions_first(self, client) -> None:
|
|
194
|
+
spans = agent_spans()
|
|
195
|
+
spans[1].attributes["gen_ai__system_instructions"] = json.dumps([{"type": "text", "content": "Be brief."}])
|
|
196
|
+
|
|
197
|
+
post(client, export(*spans))
|
|
198
|
+
|
|
199
|
+
_, events = load(client)
|
|
200
|
+
payload = next(e for e in events if e.type == "llm.completed").payload_full
|
|
201
|
+
assert payload["input_messages"][0] == {"role": "system", "parts": [{"type": "text", "content": "Be brief."}]}
|
|
202
|
+
assert payload["input_messages"][1]["role"] == "user"
|
|
203
|
+
assert payload["output_messages"][0]["parts"][0]["name"] == "get_weather"
|
|
204
|
+
|
|
205
|
+
def test_without_instructions_the_messages_are_kept_as_they_came(self, client) -> None:
|
|
206
|
+
post(client, export(*agent_spans()))
|
|
207
|
+
|
|
208
|
+
_, events = load(client)
|
|
209
|
+
payload = next(e for e in events if e.type == "llm.completed").payload_full
|
|
210
|
+
assert [m["role"] for m in payload["input_messages"]] == ["user"]
|
|
211
|
+
|
|
193
212
|
def test_tool_arguments_and_result_are_recorded(self, client) -> None:
|
|
194
213
|
post(client, export(*agent_spans()))
|
|
195
214
|
|
|
@@ -100,6 +100,36 @@ class TestRecordToolResult:
|
|
|
100
100
|
assert all(e.payload_full is None for e in events_of(session, run))
|
|
101
101
|
|
|
102
102
|
|
|
103
|
+
class TestFinalOutputAndToolOutput:
|
|
104
|
+
def test_the_whole_final_output_is_kept_with_full_payloads_on_and_only_the_summary_without(self, session, run) -> None:
|
|
105
|
+
long_output = {"recomendacion": "x" * 600, "causa": "avería"}
|
|
106
|
+
|
|
107
|
+
complete_run(session, run, long_output, capture_full=True)
|
|
108
|
+
completed = [e for e in events_of(session, run) if e.type == "run.completed"][0]
|
|
109
|
+
|
|
110
|
+
assert completed.payload_full == {"final_output": long_output}
|
|
111
|
+
assert len(completed.summary) <= 220
|
|
112
|
+
|
|
113
|
+
def test_without_full_payloads_only_the_summary_is_stored(self, session, run) -> None:
|
|
114
|
+
complete_run(session, run, {"a": 1})
|
|
115
|
+
|
|
116
|
+
assert [e for e in events_of(session, run) if e.type == "run.completed"][0].payload_full is None
|
|
117
|
+
|
|
118
|
+
def test_the_whole_tool_output_is_kept_next_to_its_summary(self, session, run) -> None:
|
|
119
|
+
result = ToolResultInput(call_id="c1", name="check", step_id="s1", status="succeeded", args={}, output_summary="{...}", output={"veredicto": "descartado", "ventana": [60, 120]})
|
|
120
|
+
|
|
121
|
+
record_tool_result(session, run, result, capture_full=True)
|
|
122
|
+
|
|
123
|
+
done = [e for e in events_of(session, run) if e.type == "tool.succeeded"][0]
|
|
124
|
+
assert done.payload_full == {"output_summary": "{...}", "output": {"veredicto": "descartado", "ventana": [60, 120]}}
|
|
125
|
+
|
|
126
|
+
def test_a_tool_without_output_is_stored_as_before(self, session, run) -> None:
|
|
127
|
+
record_tool_result(session, run, ToolResultInput(call_id="c1", name="check", step_id="s1", status="succeeded", args={}, output_summary="ok"), capture_full=True)
|
|
128
|
+
|
|
129
|
+
done = [e for e in events_of(session, run) if e.type == "tool.succeeded"][0]
|
|
130
|
+
assert done.payload_full == {"output_summary": "ok"}
|
|
131
|
+
|
|
132
|
+
|
|
103
133
|
class TestRunEnd:
|
|
104
134
|
def test_complete_run_sets_status_summary_and_end_time(self, session, run) -> None:
|
|
105
135
|
derivation = complete_run(session, run, {"answer": 42}, ended_at=START)
|
|
@@ -282,7 +282,10 @@ def test_successful_run_generates_timeline_graph_and_explanation(tmp_path) -> No
|
|
|
282
282
|
assert all(decision["alternatives"] == [] for decision in explanation["decisions"])
|
|
283
283
|
|
|
284
284
|
graph = test_client.get(f"/api/runs/{run.run_id}/graph?view=execution").json()
|
|
285
|
-
|
|
285
|
+
# One node per step: a tool's start and end are one node, and every event is evidence of some node.
|
|
286
|
+
assert 0 < len(graph["nodes"]) < len(event_types)
|
|
287
|
+
evidence = {event_id for node in graph["nodes"] for event_id in node["data"]["evidenceIds"]}
|
|
288
|
+
assert len(evidence) == len(event_types)
|
|
286
289
|
|
|
287
290
|
analytics = test_client.get(f"/api/runs/{run.run_id}/analytics").json()
|
|
288
291
|
assert analytics["tool_graph"]["view"] == "tool_chain"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/api/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/services/__init__.py
RENAMED
|
File without changes
|
{trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/services/analytics.py
RENAMED
|
File without changes
|
|
File without changes
|
{trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/services/compare.py
RENAMED
|
File without changes
|
{trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/services/dashboard.py
RENAMED
|
File without changes
|
{trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/services/explanation.py
RENAMED
|
File without changes
|
|
File without changes
|
{trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/services/otlp_decode.py
RENAMED
|
File without changes
|
{trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/services/provider.py
RENAMED
|
File without changes
|
{trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/services/run_summary.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server.egg-info/requires.txt
RENAMED
|
File without changes
|
{trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server.egg-info/top_level.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|