trace-agent-server 0.5.0__tar.gz → 0.7.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/PKG-INFO +1 -1
  2. {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/pyproject.toml +1 -1
  3. {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/api/runs.py +4 -2
  4. {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/db.py +2 -1
  5. {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/schemas.py +4 -0
  6. {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/services/events.py +6 -2
  7. {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/services/graph.py +42 -13
  8. {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/services/otlp.py +19 -5
  9. {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/services/run_lifecycle.py +12 -3
  10. {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server.egg-info/PKG-INFO +1 -1
  11. {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server.egg-info/SOURCES.txt +2 -0
  12. trace_agent_server-0.7.0/tests/test_concurrent_writers.py +85 -0
  13. {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/tests/test_framework_observations.py +39 -0
  14. trace_agent_server-0.7.0/tests/test_graph.py +60 -0
  15. {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/tests/test_otlp_ingest.py +19 -0
  16. {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/tests/test_run_lifecycle.py +30 -0
  17. {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/tests/test_runs_api.py +4 -1
  18. {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/README.md +0 -0
  19. {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/setup.cfg +0 -0
  20. {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/__init__.py +0 -0
  21. {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/api/__init__.py +0 -0
  22. {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/api/otlp.py +0 -0
  23. {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/cli.py +0 -0
  24. {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/config.py +0 -0
  25. {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/main.py +0 -0
  26. {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/models.py +0 -0
  27. {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/services/__init__.py +0 -0
  28. {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/services/analytics.py +0 -0
  29. {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/services/audit_guardrails.py +0 -0
  30. {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/services/compare.py +0 -0
  31. {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/services/dashboard.py +0 -0
  32. {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/services/explanation.py +0 -0
  33. {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/services/observability.py +0 -0
  34. {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/services/otlp_decode.py +0 -0
  35. {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/services/provider.py +0 -0
  36. {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/services/run_summary.py +0 -0
  37. {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server/services/stored_redaction.py +0 -0
  38. {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server.egg-info/dependency_links.txt +0 -0
  39. {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server.egg-info/entry_points.txt +0 -0
  40. {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server.egg-info/requires.txt +0 -0
  41. {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/src/trace_agent_server.egg-info/top_level.txt +0 -0
  42. {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/tests/test_audit_guardrails_api.py +0 -0
  43. {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/tests/test_dashboard.py +0 -0
  44. {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/tests/test_events.py +0 -0
  45. {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/tests/test_inactivity_sweep.py +0 -0
  46. {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/tests/test_otlp_decode.py +0 -0
  47. {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/tests/test_providers.py +0 -0
  48. {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/tests/test_sdk_wrappers.py +0 -0
  49. {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/tests/test_sse.py +0 -0
  50. {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/tests/test_stored_redaction.py +0 -0
  51. {trace_agent_server-0.5.0 → trace_agent_server-0.7.0}/tests/test_turns.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: trace-agent-server
3
- Version: 0.5.0
3
+ Version: 0.7.0
4
4
  Summary: TraceAgent FastAPI backend server.
5
5
  Author-email: Enrique Javier Villar Cea <villarcea.enriquejavier@gmail.com>, Javier Rodriguez Castellano <javirodriguezcastellano@gmail.com>
6
6
  License: MIT
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "trace-agent-server"
7
- version = "0.5.0"
7
+ version = "0.7.0"
8
8
  description = "TraceAgent FastAPI backend server."
9
9
  readme = "README.md"
10
10
  authors = [
@@ -720,7 +720,7 @@ def store_observations(
720
720
  model=observation.model,
721
721
  error_code=observation.error_code,
722
722
  payload_hash=stable_hash(observation.metadata),
723
- payload_full=observation.metadata if capture_full else None,
723
+ payload_full={**observation.metadata, **(observation.content or {})} if capture_full else None,
724
724
  metadata={
725
725
  "provider": observation.provider,
726
726
  "model": observation.model,
@@ -809,10 +809,12 @@ def store_observations(
809
809
  def finish_run(
810
810
  run_id: str,
811
811
  payload: FinishRunRequest,
812
+ request: Request,
812
813
  session: Session = Depends(get_session),
813
814
  ) -> dict[str, Any]:
814
815
  run = _get_run_or_404(session, run_id)
815
- derivation = complete_run(session, run, payload.final_output, payload.artifacts)
816
+ capture_full = getattr(request.app.state.settings, "capture_full_payloads", True)
817
+ derivation = complete_run(session, run, payload.final_output, payload.artifacts, capture_full=capture_full)
816
818
  session.commit()
817
819
  return {"status": "completed", "run_id": run.id, "decisions": len(derivation["decisions"])}
818
820
 
@@ -12,7 +12,8 @@ class Base(DeclarativeBase):
12
12
 
13
13
 
14
14
  def create_session_factory(database_url: str) -> tuple[object, sessionmaker[Session]]:
15
- connect_args = {"check_same_thread": False} if database_url.startswith("sqlite") else {}
15
+ # SQLite lets one writer at a time: a request that finds the database busy waits for it instead of failing.
16
+ connect_args = {"check_same_thread": False, "timeout": 30} if database_url.startswith("sqlite") else {}
16
17
  engine = create_engine(database_url, connect_args=connect_args, future=True)
17
18
  session_factory = sessionmaker(bind=engine, autoflush=False, autocommit=False, expire_on_commit=False)
18
19
  return engine, session_factory
@@ -61,6 +61,8 @@ class ToolResultInput(BaseModel):
61
61
  status: Literal["succeeded", "failed"]
62
62
  args: dict[str, Any] = Field(default_factory=dict)
63
63
  output_summary: str | None = None
64
+ # What the tool returned, whole (masked by the sender). Stored only with full payloads on.
65
+ output: Any | None = None
64
66
  output_hash: str | None = None
65
67
  error_code: str | None = None
66
68
  error_summary: str | None = None
@@ -113,6 +115,8 @@ class ObservationInput(BaseModel):
113
115
  provider: str | None = None
114
116
  input_summary: str | None = None
115
117
  artifact: ArtifactInput | None = None
118
+ # What a model call was asked and answered (input_messages / output_messages). Stored only with full payloads on.
119
+ content: dict[str, Any] | None = None
116
120
  metadata: dict[str, Any] = Field(default_factory=dict)
117
121
 
118
122
 
@@ -4,6 +4,7 @@ import asyncio
4
4
  from datetime import datetime, timezone
5
5
  from typing import Any
6
6
 
7
+ from sqlalchemy import update
7
8
  from sqlalchemy.orm import Session
8
9
 
9
10
  from trace_agent_server.models import Artifact, Event, Run
@@ -66,8 +67,11 @@ def record_event(
66
67
  timestamp: datetime | None = None,
67
68
  ) -> Event:
68
69
  """Record an event; ``timestamp`` overrides "now" for sources that report when it happened."""
69
- run.last_event_seq += 1
70
- run.updated_at = utcnow()
70
+ # The database increments the counter, it is not computed here: two requests writing to the same run at once
71
+ # (the agent recording by hand while the capability posts from its thread) would read the same value and
72
+ # collide on (run_id, seq). The second request waits for the first one's commit and gets the next number.
73
+ session.execute(update(Run).where(Run.id == run.id).values(last_event_seq=Run.last_event_seq + 1, updated_at=utcnow()))
74
+ session.refresh(run, ["last_event_seq"])
71
75
  event = Event(
72
76
  run_id=run.id,
73
77
  turn_id=turn_id,
@@ -5,36 +5,65 @@ from typing import Any
5
5
  from trace_agent_server.models import Artifact, Decision, Event
6
6
 
7
7
 
8
+ def _steps(events: list[Event]) -> list[list[Event]]:
9
+ """Events grouped by the step they belong to (a tool call's start and end, a model call's request and
10
+ response), in the order the steps began. An event without a step is a step of its own."""
11
+ steps: list[list[Event]] = []
12
+ position: dict[str, int] = {}
13
+ for event in events:
14
+ if event.step_id and event.step_id in position:
15
+ steps[position[event.step_id]].append(event)
16
+ continue
17
+ if event.step_id:
18
+ position[event.step_id] = len(steps)
19
+ steps.append([event])
20
+ return steps
21
+
22
+
8
23
  def build_execution_graph(events: list[Event], artifacts: list[Artifact]) -> dict[str, Any]:
24
+ """One node per step, not per event: a tool is one node with its result, not a 'started' and a 'finished'."""
9
25
  nodes: list[dict[str, Any]] = []
10
26
  edges: list[dict[str, Any]] = []
27
+ node_of_event: dict[int, str] = {}
11
28
  x = 0.0
12
29
  y = 0.0
13
30
 
14
- for index, event in enumerate(events):
31
+ steps = _steps(events)
32
+ for index, step in enumerate(steps):
33
+ last = step[-1]
34
+ node_id = f"event-{last.id}"
35
+ for event in step:
36
+ node_of_event[event.id] = node_id
37
+ failed = any(event.status == "failed" for event in step)
38
+ command = next((e.event_metadata.get("command") for e in step if e.type.startswith("command.") and e.event_metadata), None)
15
39
  nodes.append(
16
40
  {
17
- "id": f"event-{event.id}",
41
+ "id": node_id,
18
42
  "type": "event",
19
- "label": event.type,
43
+ "label": last.type,
20
44
  "position": {"x": x, "y": y},
21
45
  "data": {
22
- "summary": event.summary,
23
- "eventId": event.id,
24
- "toolName": event.tool_name,
25
- "status": event.status,
26
- "evidenceIds": [event.id],
46
+ "summary": last.summary,
47
+ # The last event of the step (its result) is the one the inspector opens.
48
+ "eventId": last.id,
49
+ "toolName": last.tool_name,
50
+ "status": "failed" if failed else last.status,
51
+ "durationMs": last.duration_ms,
52
+ "stepKind": step[0].type.split(".")[0],
53
+ "command": command,
54
+ "evidenceIds": [event.id for event in step],
27
55
  },
28
56
  }
29
57
  )
30
58
  if index > 0:
59
+ previous = nodes[index - 1]
31
60
  edges.append(
32
61
  {
33
- "id": f"edge-event-{events[index - 1].id}-{event.id}",
34
- "source": f"event-{events[index - 1].id}",
35
- "target": f"event-{event.id}",
62
+ "id": f"edge-{previous['id']}-{node_id}",
63
+ "source": previous["id"],
64
+ "target": node_id,
36
65
  "label": None,
37
- "animated": event.type in {"tool.failed", "run.failed"},
66
+ "animated": failed or last.type == "run.failed",
38
67
  }
39
68
  )
40
69
  x += 220
@@ -62,7 +91,7 @@ def build_execution_graph(events: list[Event], artifacts: list[Artifact]) -> dic
62
91
  edges.append(
63
92
  {
64
93
  "id": f"edge-artifact-{artifact.id}",
65
- "source": f"event-{artifact.source_event_id}",
94
+ "source": node_of_event.get(artifact.source_event_id, f"event-{artifact.source_event_id}"),
66
95
  "target": f"artifact-{artifact.id}",
67
96
  "label": "artifact",
68
97
  "animated": False,
@@ -120,6 +120,17 @@ def _first_user_text(input_messages: Any) -> str | None:
120
120
  return None
121
121
 
122
122
 
123
+ def _with_instructions(input_messages: Any, instructions: Any) -> Any:
124
+ """The instructions of the agent come beside the messages in the GenAI conventions: show them first, as a system message."""
125
+ parts = _parse_json(instructions)
126
+ if isinstance(parts, str):
127
+ parts = [{"type": "text", "content": parts}]
128
+ if not isinstance(parts, list) or not parts:
129
+ return input_messages
130
+ messages = input_messages if isinstance(input_messages, list) else []
131
+ return [{"role": "system", "parts": parts}, *messages]
132
+
133
+
123
134
  def _requested_tools(output_messages: Any) -> list[str]:
124
135
  names: list[str] = []
125
136
  for message in output_messages if isinstance(output_messages, list) else []:
@@ -196,7 +207,7 @@ def _ingest_run(session: Session, run_id: str, spans: list[SpanRecord], capture_
196
207
  _track_agent_calls(run, fresh)
197
208
  closing = _closing_span(run, fresh) if was_open else None
198
209
  if closing is not None:
199
- _close_run(session, run, closing)
210
+ _close_run(session, run, closing, capture_full)
200
211
  elif not was_open:
201
212
  # Late spans of a finished run: refresh its decisions and flags.
202
213
  derive_explanation(session, run)
@@ -328,7 +339,9 @@ def _model_steps(session: Session, run: Run, span: SpanRecord, capture_full: boo
328
339
  payload = None
329
340
  if capture_full:
330
341
  payload = {
331
- "input_messages": _parse_json(attributes.get("gen_ai.input.messages")),
342
+ "input_messages": _with_instructions(
343
+ _parse_json(attributes.get("gen_ai.input.messages")), attributes.get("gen_ai.system_instructions")
344
+ ),
332
345
  "output_messages": output_messages,
333
346
  }
334
347
  record_event(
@@ -359,7 +372,8 @@ def _tool_steps(session: Session, run: Run, span: SpanRecord, capture_full: bool
359
372
  parent_step_id=span.parent_span_id,
360
373
  status="failed" if failed else "succeeded",
361
374
  args=redact_payload(arguments if isinstance(arguments, dict) else {"input": arguments} if arguments is not None else {}),
362
- output_summary=summarize_value(_readable(result_text)) if result_text is not None else None,
375
+ output_summary=summarize_value(_readable(result_text), sort_keys=False) if result_text is not None else None,
376
+ output=_parse_json(result_text) if result_text is not None and not failed else None,
363
377
  output_hash=stable_hash(result_text if not failed else {"error": message}),
364
378
  error_code=code,
365
379
  error_summary=message,
@@ -448,9 +462,9 @@ def _closing_span(run: Run, spans: list[SpanRecord]) -> SpanRecord | None:
448
462
  return max(agents, key=lambda s: s.end)
449
463
 
450
464
 
451
- def _close_run(session: Session, run: Run, span: SpanRecord) -> None:
465
+ def _close_run(session: Session, run: Run, span: SpanRecord, capture_full: bool) -> None:
452
466
  if span.status_code == "ERROR":
453
467
  code, message = _error_of(span)
454
468
  fail_run_with(session, run, code, message, ended_at=span.end)
455
469
  else:
456
- complete_run(session, run, span.attributes.get("final_result"), ended_at=span.end)
470
+ complete_run(session, run, span.attributes.get("final_result"), ended_at=span.end, capture_full=capture_full)
@@ -86,7 +86,11 @@ def record_tool_result(
86
86
  tool_name=result.name,
87
87
  duration_ms=result.duration_ms,
88
88
  payload_hash=result.output_hash,
89
- payload_full={"output_summary": result.output_summary} if capture_full else None,
89
+ payload_full=(
90
+ {"output_summary": result.output_summary, **({"output": result.output} if result.output is not None else {})}
91
+ if capture_full
92
+ else None
93
+ ),
90
94
  metadata={"call_id": result.call_id, "artifact_count": len(result.artifacts)},
91
95
  timestamp=ended_at,
92
96
  )
@@ -147,9 +151,13 @@ def complete_run(
147
151
  artifacts: Sequence[ArtifactInput] = (),
148
152
  *,
149
153
  ended_at: datetime | None = None,
154
+ capture_full: bool = False,
150
155
  ) -> dict[str, Any]:
151
- """Mark a run completed, record its final output and derive its explanation."""
152
- final_summary = summarize_value(final_output) if final_output is not None else "Run finished without final output."
156
+ """Mark a run completed, record its final output and derive its explanation.
157
+
158
+ The summary of the output is always kept; the whole output only with ``capture_full``.
159
+ """
160
+ final_summary = summarize_value(final_output, sort_keys=False) if final_output is not None else "Run finished without final output."
153
161
  run.final_output_summary = final_summary
154
162
  run.status = "completed"
155
163
  run.ended_at = ended_at or utcnow()
@@ -161,6 +169,7 @@ def complete_run(
161
169
  summary=final_summary,
162
170
  status="completed",
163
171
  payload_hash=stable_hash(final_output),
172
+ payload_full={"final_output": final_output} if capture_full and final_output is not None else None,
164
173
  timestamp=ended_at,
165
174
  )
166
175
  for artifact_input in artifacts:
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: trace-agent-server
3
- Version: 0.5.0
3
+ Version: 0.7.0
4
4
  Summary: TraceAgent FastAPI backend server.
5
5
  Author-email: Enrique Javier Villar Cea <villarcea.enriquejavier@gmail.com>, Javier Rodriguez Castellano <javirodriguezcastellano@gmail.com>
6
6
  License: MIT
@@ -32,9 +32,11 @@ src/trace_agent_server/services/run_lifecycle.py
32
32
  src/trace_agent_server/services/run_summary.py
33
33
  src/trace_agent_server/services/stored_redaction.py
34
34
  tests/test_audit_guardrails_api.py
35
+ tests/test_concurrent_writers.py
35
36
  tests/test_dashboard.py
36
37
  tests/test_events.py
37
38
  tests/test_framework_observations.py
39
+ tests/test_graph.py
38
40
  tests/test_inactivity_sweep.py
39
41
  tests/test_otlp_decode.py
40
42
  tests/test_otlp_ingest.py
@@ -0,0 +1,85 @@
1
+ from __future__ import annotations
2
+
3
+ import threading
4
+
5
+ import pytest
6
+ from fastapi.testclient import TestClient
7
+
8
+ from trace_agent_server.config import Settings
9
+ from trace_agent_server.main import create_app
10
+ from trace_agent_server.models import Event, Run
11
+ from trace_agent_server.services.events import record_event
12
+
13
+ WRITERS = 6
14
+ EVENTS_EACH = 25
15
+
16
+
17
+ @pytest.fixture
18
+ def app(tmp_path):
19
+ return create_app(Settings(database_url=f"sqlite:///{tmp_path / 'writers.db'}", cors_origins=["http://localhost:5173"]))
20
+
21
+
22
+ class TestSeveralWritersOnOneRun:
23
+ """The capability posts from a background thread while the agent records by hand: both write to the same run."""
24
+
25
+ def test_events_recorded_at_the_same_time_never_collide(self, app) -> None:
26
+ with app.state.session_factory() as session:
27
+ run = Run(agent_name="agent", goal="goal")
28
+ session.add(run)
29
+ session.commit()
30
+ run_id = run.id
31
+
32
+ errors: list[BaseException] = []
33
+ start = threading.Barrier(WRITERS)
34
+
35
+ def write(writer: int) -> None:
36
+ try:
37
+ start.wait()
38
+ for index in range(EVENTS_EACH):
39
+ # Each request loads the run, records an event and commits, as the endpoints do.
40
+ with app.state.session_factory() as session:
41
+ run = session.get(Run, run_id)
42
+ record_event(session, run, actor="sdk", event_type="tool.started", summary=f"writer {writer} event {index}")
43
+ session.commit()
44
+ except BaseException as exc: # noqa: BLE001 - reported by the test below
45
+ errors.append(exc)
46
+
47
+ threads = [threading.Thread(target=write, args=(writer,)) for writer in range(WRITERS)]
48
+ for thread in threads:
49
+ thread.start()
50
+ for thread in threads:
51
+ thread.join(timeout=120)
52
+
53
+ assert errors == []
54
+ with app.state.session_factory() as session:
55
+ sequence = [seq for (seq,) in session.query(Event.seq).filter_by(run_id=run_id).order_by(Event.seq)]
56
+ assert session.get(Run, run_id).last_event_seq == WRITERS * EVENTS_EACH
57
+ assert sequence == list(range(1, WRITERS * EVENTS_EACH + 1))
58
+
59
+ def test_requests_to_the_same_run_at_the_same_time_all_succeed(self, app) -> None:
60
+ client = TestClient(app)
61
+ run_id = client.post("/api/runs", json={"agent_name": "agent", "goal": "goal"}).json()["run_id"]
62
+ failures: list[int] = []
63
+ start = threading.Barrier(4)
64
+
65
+ def post(writer: int) -> None:
66
+ start.wait()
67
+ for index in range(10):
68
+ response = client.post(
69
+ f"/api/runs/{run_id}/observations",
70
+ json={"observations": [{"kind": "command", "command": f"echo {writer}-{index}", "step_id": f"c-{writer}-{index}", "status": "succeeded", "exit_code": 0}]},
71
+ )
72
+ if response.status_code != 200:
73
+ failures.append(response.status_code)
74
+
75
+ threads = [threading.Thread(target=post, args=(writer,)) for writer in range(4)]
76
+ for thread in threads:
77
+ thread.start()
78
+ for thread in threads:
79
+ thread.join(timeout=120)
80
+
81
+ assert failures == []
82
+ with app.state.session_factory() as session:
83
+ seqs = [seq for (seq,) in session.query(Event.seq).filter_by(run_id=run_id).order_by(Event.seq)]
84
+ assert seqs == list(range(1, len(seqs) + 1))
85
+ assert len(seqs) == 1 + 4 * 10 * 2 # run.started, then command.started and command.succeeded for each command
@@ -120,6 +120,45 @@ class TestLlmCallUsage:
120
120
  assert execution["assistant_turn_count"] == 2
121
121
 
122
122
 
123
+ class TestLlmCallContent:
124
+ content = {
125
+ "input_messages": [{"role": "user", "parts": [{"type": "text", "content": "¿Por qué bajó el OEE?"}]}],
126
+ "output_messages": [{"role": "assistant", "parts": [{"type": "text", "content": "Por las paradas."}]}],
127
+ }
128
+
129
+ def completed_event(self, client, run_id: str) -> Event:
130
+ with client.app.state.session_factory() as session:
131
+ event = session.query(Event).filter_by(run_id=run_id, type="llm.completed").one()
132
+ session.expunge(event)
133
+ return event
134
+
135
+ def test_what_the_model_was_asked_and_answered_is_stored_with_the_event(self, client) -> None:
136
+ run_id = start_run(client)
137
+ observe(client, run_id, kind="llm_call", step_id="l1", status="succeeded", provider="openai", model="m",
138
+ content=self.content, metadata={"prompt_tokens": 3})
139
+
140
+ event = self.completed_event(client, run_id)
141
+ assert event.payload_full["input_messages"] == self.content["input_messages"]
142
+ assert event.payload_full["output_messages"] == self.content["output_messages"]
143
+ assert event.payload_full["prompt_tokens"] == 3
144
+ # The content is not copied into the metadata that is always kept and listed.
145
+ assert "input_messages" not in event.event_metadata
146
+
147
+ def test_it_is_not_stored_when_full_payloads_are_off(self, tmp_path) -> None:
148
+ app = create_app(Settings(database_url=f"sqlite:///{tmp_path / 'off.db'}", cors_origins=["http://localhost:5173"], capture_full_payloads=False))
149
+ client = TestClient(app)
150
+ run_id = start_run(client)
151
+ observe(client, run_id, kind="llm_call", step_id="l1", status="succeeded", provider="openai", model="m", content=self.content)
152
+
153
+ assert self.completed_event(client, run_id).payload_full is None
154
+
155
+ def test_a_call_without_content_is_stored_as_before(self, client) -> None:
156
+ run_id = start_run(client)
157
+ observe(client, run_id, kind="llm_call", step_id="l1", status="succeeded", provider="openai", model="m", metadata={"prompt_tokens": 3})
158
+
159
+ assert self.completed_event(client, run_id).payload_full == {"prompt_tokens": 3}
160
+
161
+
123
162
  class TestTokenUsageOfFrameworkRuns:
124
163
  def model_call(self, client, run_id, step, **tokens) -> None:
125
164
  observe(client, run_id, kind="llm_call", step_id=step, status="succeeded", provider="openai", model="m", metadata=tokens)
@@ -0,0 +1,60 @@
1
+ from __future__ import annotations
2
+
3
+ from trace_agent_server.models import Artifact, Event
4
+ from trace_agent_server.services.graph import build_execution_graph
5
+
6
+
7
+ def event(event_id: int, event_type: str, step_id: str | None, *, status: str | None = None, tool_name: str | None = None,
8
+ summary: str = "", metadata: dict | None = None, duration_ms: int | None = None) -> Event:
9
+ return Event(id=event_id, run_id="r", seq=event_id, actor="sdk", type=event_type, summary=summary, status=status,
10
+ step_id=step_id, tool_name=tool_name, event_metadata=metadata or {}, duration_ms=duration_ms)
11
+
12
+
13
+ class TestExecutionGraphSteps:
14
+ def test_a_tool_is_one_node_with_its_result_not_a_started_and_a_finished_node(self) -> None:
15
+ events = [
16
+ event(1, "run.started", None),
17
+ event(2, "tool.started", "t1", tool_name="search", status="completed"),
18
+ event(3, "tool.succeeded", "t1", tool_name="search", status="succeeded", summary="3 results", duration_ms=40),
19
+ event(4, "run.completed", None),
20
+ ]
21
+
22
+ graph = build_execution_graph(events, [])
23
+
24
+ assert [n["data"]["eventId"] for n in graph["nodes"]] == [1, 3, 4]
25
+ tool = graph["nodes"][1]
26
+ assert (tool["label"], tool["data"]["summary"], tool["data"]["durationMs"], tool["data"]["toolName"]) == ("tool.succeeded", "3 results", 40, "search")
27
+ assert tool["data"]["evidenceIds"] == [2, 3]
28
+ assert tool["data"]["stepKind"] == "tool"
29
+
30
+ def test_the_nodes_are_chained_in_the_order_the_steps_began(self) -> None:
31
+ events = [event(1, "llm.started", "l1"), event(2, "tool.started", "t1"), event(3, "llm.completed", "l1"), event(4, "tool.succeeded", "t1")]
32
+
33
+ graph = build_execution_graph(events, [])
34
+
35
+ assert [n["data"]["eventId"] for n in graph["nodes"]] == [3, 4] # each node is named after the last event of its step
36
+ assert [(e["source"], e["target"]) for e in graph["edges"]] == [("event-3", "event-4")]
37
+
38
+ def test_a_step_that_failed_at_any_point_is_failed_and_its_edge_animated(self) -> None:
39
+ events = [event(1, "tool.started", "t1", status="completed"), event(2, "tool.failed", "t1", status="failed"), event(3, "tool.started", "t2")]
40
+
41
+ graph = build_execution_graph(events, [])
42
+
43
+ assert graph["nodes"][0]["data"]["status"] == "failed"
44
+ assert graph["edges"][0]["animated"] is False # the edge into the next step follows that step, which did not fail
45
+
46
+ def test_the_command_of_a_command_step_is_kept_for_its_title(self) -> None:
47
+ events = [event(1, "command.started", "c1", metadata={"command": "python -c # Leer\nprint(1)"}), event(2, "command.succeeded", "c1", metadata={"command": "python -c # Leer\nprint(1)"})]
48
+
49
+ graph = build_execution_graph(events, [])
50
+
51
+ assert graph["nodes"][0]["data"]["command"] == "python -c # Leer\nprint(1)"
52
+ assert graph["nodes"][0]["data"]["stepKind"] == "command"
53
+
54
+ def test_an_artifact_hangs_from_the_node_of_the_step_that_made_it(self) -> None:
55
+ events = [event(1, "tool.started", "t1"), event(2, "tool.succeeded", "t1")]
56
+ artifact = Artifact(id=9, run_id="r", kind="file", label="report", summary="the report", source_event_id=1)
57
+
58
+ graph = build_execution_graph(events, [artifact])
59
+
60
+ assert ("event-2", "artifact-9") in [(e["source"], e["target"]) for e in graph["edges"]]
@@ -190,6 +190,25 @@ class TestRunCreationAndMapping:
190
190
  assert completed.event_metadata["prompt_tokens"] == 60
191
191
  assert completed.event_metadata["requested_tools"] == ["get_weather"]
192
192
 
193
+ def test_what_the_model_was_asked_and_answered_is_kept_with_the_instructions_first(self, client) -> None:
194
+ spans = agent_spans()
195
+ spans[1].attributes["gen_ai__system_instructions"] = json.dumps([{"type": "text", "content": "Be brief."}])
196
+
197
+ post(client, export(*spans))
198
+
199
+ _, events = load(client)
200
+ payload = next(e for e in events if e.type == "llm.completed").payload_full
201
+ assert payload["input_messages"][0] == {"role": "system", "parts": [{"type": "text", "content": "Be brief."}]}
202
+ assert payload["input_messages"][1]["role"] == "user"
203
+ assert payload["output_messages"][0]["parts"][0]["name"] == "get_weather"
204
+
205
+ def test_without_instructions_the_messages_are_kept_as_they_came(self, client) -> None:
206
+ post(client, export(*agent_spans()))
207
+
208
+ _, events = load(client)
209
+ payload = next(e for e in events if e.type == "llm.completed").payload_full
210
+ assert [m["role"] for m in payload["input_messages"]] == ["user"]
211
+
193
212
  def test_tool_arguments_and_result_are_recorded(self, client) -> None:
194
213
  post(client, export(*agent_spans()))
195
214
 
@@ -100,6 +100,36 @@ class TestRecordToolResult:
100
100
  assert all(e.payload_full is None for e in events_of(session, run))
101
101
 
102
102
 
103
+ class TestFinalOutputAndToolOutput:
104
+ def test_the_whole_final_output_is_kept_with_full_payloads_on_and_only_the_summary_without(self, session, run) -> None:
105
+ long_output = {"recomendacion": "x" * 600, "causa": "avería"}
106
+
107
+ complete_run(session, run, long_output, capture_full=True)
108
+ completed = [e for e in events_of(session, run) if e.type == "run.completed"][0]
109
+
110
+ assert completed.payload_full == {"final_output": long_output}
111
+ assert len(completed.summary) <= 220
112
+
113
+ def test_without_full_payloads_only_the_summary_is_stored(self, session, run) -> None:
114
+ complete_run(session, run, {"a": 1})
115
+
116
+ assert [e for e in events_of(session, run) if e.type == "run.completed"][0].payload_full is None
117
+
118
+ def test_the_whole_tool_output_is_kept_next_to_its_summary(self, session, run) -> None:
119
+ result = ToolResultInput(call_id="c1", name="check", step_id="s1", status="succeeded", args={}, output_summary="{...}", output={"veredicto": "descartado", "ventana": [60, 120]})
120
+
121
+ record_tool_result(session, run, result, capture_full=True)
122
+
123
+ done = [e for e in events_of(session, run) if e.type == "tool.succeeded"][0]
124
+ assert done.payload_full == {"output_summary": "{...}", "output": {"veredicto": "descartado", "ventana": [60, 120]}}
125
+
126
+ def test_a_tool_without_output_is_stored_as_before(self, session, run) -> None:
127
+ record_tool_result(session, run, ToolResultInput(call_id="c1", name="check", step_id="s1", status="succeeded", args={}, output_summary="ok"), capture_full=True)
128
+
129
+ done = [e for e in events_of(session, run) if e.type == "tool.succeeded"][0]
130
+ assert done.payload_full == {"output_summary": "ok"}
131
+
132
+
103
133
  class TestRunEnd:
104
134
  def test_complete_run_sets_status_summary_and_end_time(self, session, run) -> None:
105
135
  derivation = complete_run(session, run, {"answer": 42}, ended_at=START)
@@ -282,7 +282,10 @@ def test_successful_run_generates_timeline_graph_and_explanation(tmp_path) -> No
282
282
  assert all(decision["alternatives"] == [] for decision in explanation["decisions"])
283
283
 
284
284
  graph = test_client.get(f"/api/runs/{run.run_id}/graph?view=execution").json()
285
- assert len(graph["nodes"]) >= len(event_types)
285
+ # One node per step: a tool's start and end are one node, and every event is evidence of some node.
286
+ assert 0 < len(graph["nodes"]) < len(event_types)
287
+ evidence = {event_id for node in graph["nodes"] for event_id in node["data"]["evidenceIds"]}
288
+ assert len(evidence) == len(event_types)
286
289
 
287
290
  analytics = test_client.get(f"/api/runs/{run.run_id}/analytics").json()
288
291
  assert analytics["tool_graph"]["view"] == "tool_chain"