10xgraph-api 0.7.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (203) hide show
  1. 10xgraph_api-0.7.0.dist-info/METADATA +618 -0
  2. 10xgraph_api-0.7.0.dist-info/RECORD +203 -0
  3. 10xgraph_api-0.7.0.dist-info/WHEEL +5 -0
  4. 10xgraph_api-0.7.0.dist-info/entry_points.txt +3 -0
  5. 10xgraph_api-0.7.0.dist-info/licenses/LICENSE +21 -0
  6. 10xgraph_api-0.7.0.dist-info/top_level.txt +2 -0
  7. agentflow_cli/__init__.py +69 -0
  8. tenxgraph_api/__init__.py +23 -0
  9. tenxgraph_api/cli/__init__.py +6 -0
  10. tenxgraph_api/cli/capabilities.py +127 -0
  11. tenxgraph_api/cli/commands/__init__.py +49 -0
  12. tenxgraph_api/cli/commands/api.py +262 -0
  13. tenxgraph_api/cli/commands/audit.py +213 -0
  14. tenxgraph_api/cli/commands/build.py +305 -0
  15. tenxgraph_api/cli/commands/config.py +71 -0
  16. tenxgraph_api/cli/commands/demo.py +93 -0
  17. tenxgraph_api/cli/commands/eval.py +909 -0
  18. tenxgraph_api/cli/commands/init.py +531 -0
  19. tenxgraph_api/cli/commands/skills.py +532 -0
  20. tenxgraph_api/cli/commands/test.py +113 -0
  21. tenxgraph_api/cli/commands/version.py +47 -0
  22. tenxgraph_api/cli/config_editor/__init__.py +15 -0
  23. tenxgraph_api/cli/config_editor/schema.py +566 -0
  24. tenxgraph_api/cli/config_editor/server.py +240 -0
  25. tenxgraph_api/cli/config_editor/static/app.css +2 -0
  26. tenxgraph_api/cli/config_editor/static/app.js +3 -0
  27. tenxgraph_api/cli/config_editor/static/favicon.svg +11 -0
  28. tenxgraph_api/cli/config_editor/static/index.html +20 -0
  29. tenxgraph_api/cli/config_editor/store.py +120 -0
  30. tenxgraph_api/cli/config_editor/validation.py +377 -0
  31. tenxgraph_api/cli/constants.py +112 -0
  32. tenxgraph_api/cli/context.py +46 -0
  33. tenxgraph_api/cli/core/__init__.py +0 -0
  34. tenxgraph_api/cli/core/animations.py +550 -0
  35. tenxgraph_api/cli/core/config.py +279 -0
  36. tenxgraph_api/cli/core/output.py +625 -0
  37. tenxgraph_api/cli/core/prompts.py +220 -0
  38. tenxgraph_api/cli/core/screen.py +287 -0
  39. tenxgraph_api/cli/core/steps.py +621 -0
  40. tenxgraph_api/cli/core/theme.py +242 -0
  41. tenxgraph_api/cli/core/validation.py +251 -0
  42. tenxgraph_api/cli/exceptions.py +129 -0
  43. tenxgraph_api/cli/logger.py +84 -0
  44. tenxgraph_api/cli/main.py +936 -0
  45. tenxgraph_api/cli/templates/__init__.py +0 -0
  46. tenxgraph_api/cli/templates/defaults.py +844 -0
  47. tenxgraph_api/cli/templates/dev/.env.example +1 -0
  48. tenxgraph_api/cli/templates/dev/10xgraph.json +16 -0
  49. tenxgraph_api/cli/templates/dev/graph/__init__.py +0 -0
  50. tenxgraph_api/cli/templates/dev/graph/agent.py +63 -0
  51. tenxgraph_api/cli/templates/prod/.env.example +75 -0
  52. tenxgraph_api/cli/templates/prod/.pre-commit-config.yaml +50 -0
  53. tenxgraph_api/cli/templates/prod/.python-version +1 -0
  54. tenxgraph_api/cli/templates/prod/10xgraph.json +42 -0
  55. tenxgraph_api/cli/templates/prod/auth/__init__.py +0 -0
  56. tenxgraph_api/cli/templates/prod/auth/agent_auth.py +46 -0
  57. tenxgraph_api/cli/templates/prod/evals/__init__.py +0 -0
  58. tenxgraph_api/cli/templates/prod/evals/user_simulator_eval.py +76 -0
  59. tenxgraph_api/cli/templates/prod/evals/weather_agents_eval.py +53 -0
  60. tenxgraph_api/cli/templates/prod/graph/__init__.py +0 -0
  61. tenxgraph_api/cli/templates/prod/graph/agent.py +50 -0
  62. tenxgraph_api/cli/templates/prod/graph/state.py +5 -0
  63. tenxgraph_api/cli/templates/prod/graph/thread_name_generator.py +7 -0
  64. tenxgraph_api/cli/templates/prod/graph/tools/__init__.py +0 -0
  65. tenxgraph_api/cli/templates/prod/graph/tools/weather_tool.py +43 -0
  66. tenxgraph_api/cli/templates/prod/graph/validators/__init__.py +0 -0
  67. tenxgraph_api/cli/templates/prod/graph/validators/lifecyle.py +107 -0
  68. tenxgraph_api/cli/templates/prod/graph/validators/manager.py +9 -0
  69. tenxgraph_api/cli/templates/prod/graph/validators/validators.py +21 -0
  70. tenxgraph_api/cli/templates/prod/pyproject.toml +79 -0
  71. tenxgraph_api/cli/templates/prod/tests/__init__.py +0 -0
  72. tenxgraph_api/cli/templates/prod/tests/conftest.py +6 -0
  73. tenxgraph_api/cli/templates/prod/tests/test_agent_eval.py +78 -0
  74. tenxgraph_api/cli/templates/prod/tests/test_catalog_tools.py +47 -0
  75. tenxgraph_api/cli/templates/prod/tests/test_graph_nodes.py +57 -0
  76. tenxgraph_api/cli/templates/skills/10xgraph/SKILL.md +92 -0
  77. tenxgraph_api/cli/templates/skills/10xgraph/references/agents-and-tools.md +389 -0
  78. tenxgraph_api/cli/templates/skills/10xgraph/references/api-client.md +61 -0
  79. tenxgraph_api/cli/templates/skills/10xgraph/references/api-configuration.md +75 -0
  80. tenxgraph_api/cli/templates/skills/10xgraph/references/api-settings-and-middleware.md +89 -0
  81. tenxgraph_api/cli/templates/skills/10xgraph/references/architecture.md +91 -0
  82. tenxgraph_api/cli/templates/skills/10xgraph/references/auth-and-authorization.md +149 -0
  83. tenxgraph_api/cli/templates/skills/10xgraph/references/callbacks-and-command.md +283 -0
  84. tenxgraph_api/cli/templates/skills/10xgraph/references/checkpointing-and-threads.md +63 -0
  85. tenxgraph_api/cli/templates/skills/10xgraph/references/cli-commands.md +73 -0
  86. tenxgraph_api/cli/templates/skills/10xgraph/references/client-auth-and-errors.md +94 -0
  87. tenxgraph_api/cli/templates/skills/10xgraph/references/client-messages-invoke-stream.md +115 -0
  88. tenxgraph_api/cli/templates/skills/10xgraph/references/client-threads-memory-files.md +80 -0
  89. tenxgraph_api/cli/templates/skills/10xgraph/references/context-id-background.md +80 -0
  90. tenxgraph_api/cli/templates/skills/10xgraph/references/dependency-injection.md +58 -0
  91. tenxgraph_api/cli/templates/skills/10xgraph/references/evaluation.md +500 -0
  92. tenxgraph_api/cli/templates/skills/10xgraph/references/id-and-thread-name-generators.md +69 -0
  93. tenxgraph_api/cli/templates/skills/10xgraph/references/media-and-files.md +66 -0
  94. tenxgraph_api/cli/templates/skills/10xgraph/references/memory-and-store.md +70 -0
  95. tenxgraph_api/cli/templates/skills/10xgraph/references/prebuilt-agents-and-tools.md +394 -0
  96. tenxgraph_api/cli/templates/skills/10xgraph/references/production-runtime.md +56 -0
  97. tenxgraph_api/cli/templates/skills/10xgraph/references/providers-and-adapters.md +89 -0
  98. tenxgraph_api/cli/templates/skills/10xgraph/references/publishers-and-runtime-protocols.md +84 -0
  99. tenxgraph_api/cli/templates/skills/10xgraph/references/rate-limiting.md +194 -0
  100. tenxgraph_api/cli/templates/skills/10xgraph/references/realtime.md +172 -0
  101. tenxgraph_api/cli/templates/skills/10xgraph/references/remote-tools.md +55 -0
  102. tenxgraph_api/cli/templates/skills/10xgraph/references/rest-api-and-errors.md +97 -0
  103. tenxgraph_api/cli/templates/skills/10xgraph/references/security-and-validators.md +86 -0
  104. tenxgraph_api/cli/templates/skills/10xgraph/references/state-and-messages.md +68 -0
  105. tenxgraph_api/cli/templates/skills/10xgraph/references/state-graph.md +232 -0
  106. tenxgraph_api/cli/templates/skills/10xgraph/references/stream-emitter.md +144 -0
  107. tenxgraph_api/cli/templates/skills/10xgraph/references/streaming.md +53 -0
  108. tenxgraph_api/cli/templates/skills/10xgraph/references/testing-and-evaluation.md +64 -0
  109. tenxgraph_api/cli/templates/skills/10xgraph/references/unit-testing.md +202 -0
  110. tenxgraph_api/cli/templates/skills/copilot/10xgraph.instructions.md +108 -0
  111. tenxgraph_api/py.typed +0 -0
  112. tenxgraph_api/src/__init__.py +0 -0
  113. tenxgraph_api/src/app/__init__.py +13 -0
  114. tenxgraph_api/src/app/core/__init__.py +15 -0
  115. tenxgraph_api/src/app/core/auth/__init__.py +0 -0
  116. tenxgraph_api/src/app/core/auth/auth_backend.py +38 -0
  117. tenxgraph_api/src/app/core/auth/authorization.py +379 -0
  118. tenxgraph_api/src/app/core/auth/base_auth.py +27 -0
  119. tenxgraph_api/src/app/core/auth/jwt_auth.py +122 -0
  120. tenxgraph_api/src/app/core/auth/ownership_resolver.py +170 -0
  121. tenxgraph_api/src/app/core/auth/permissions.py +354 -0
  122. tenxgraph_api/src/app/core/auth/request_config.py +130 -0
  123. tenxgraph_api/src/app/core/auth/route_guard.py +105 -0
  124. tenxgraph_api/src/app/core/config/__init__.py +0 -0
  125. tenxgraph_api/src/app/core/config/graph_config.py +532 -0
  126. tenxgraph_api/src/app/core/config/media_settings.py +72 -0
  127. tenxgraph_api/src/app/core/config/sentry_config.py +60 -0
  128. tenxgraph_api/src/app/core/config/settings.py +202 -0
  129. tenxgraph_api/src/app/core/config/setup_logs.py +65 -0
  130. tenxgraph_api/src/app/core/config/setup_middleware.py +483 -0
  131. tenxgraph_api/src/app/core/config/worker_middleware.py +99 -0
  132. tenxgraph_api/src/app/core/exceptions/__init__.py +17 -0
  133. tenxgraph_api/src/app/core/exceptions/general_exception.py +58 -0
  134. tenxgraph_api/src/app/core/exceptions/handle_errors.py +363 -0
  135. tenxgraph_api/src/app/core/exceptions/resources_exceptions.py +72 -0
  136. tenxgraph_api/src/app/core/exceptions/user_exception.py +58 -0
  137. tenxgraph_api/src/app/core/middleware/__init__.py +25 -0
  138. tenxgraph_api/src/app/core/middleware/rate_limit/__init__.py +15 -0
  139. tenxgraph_api/src/app/core/middleware/rate_limit/base.py +27 -0
  140. tenxgraph_api/src/app/core/middleware/rate_limit/factory.py +98 -0
  141. tenxgraph_api/src/app/core/middleware/rate_limit/keying.py +133 -0
  142. tenxgraph_api/src/app/core/middleware/rate_limit/memory.py +98 -0
  143. tenxgraph_api/src/app/core/middleware/rate_limit/middleware.py +90 -0
  144. tenxgraph_api/src/app/core/middleware/rate_limit/redis.py +115 -0
  145. tenxgraph_api/src/app/core/middleware/request_limits.py +154 -0
  146. tenxgraph_api/src/app/core/middleware/security_headers.py +239 -0
  147. tenxgraph_api/src/app/core/utils/__init__.py +14 -0
  148. tenxgraph_api/src/app/core/utils/log_sanitizer.py +216 -0
  149. tenxgraph_api/src/app/loader.py +516 -0
  150. tenxgraph_api/src/app/main.py +180 -0
  151. tenxgraph_api/src/app/routers/__init__.py +4 -0
  152. tenxgraph_api/src/app/routers/ag_ui/__init__.py +10 -0
  153. tenxgraph_api/src/app/routers/ag_ui/converter.py +195 -0
  154. tenxgraph_api/src/app/routers/ag_ui/event_mapper.py +353 -0
  155. tenxgraph_api/src/app/routers/ag_ui/router.py +90 -0
  156. tenxgraph_api/src/app/routers/ag_ui/service.py +193 -0
  157. tenxgraph_api/src/app/routers/checkpointer/__init__.py +1 -0
  158. tenxgraph_api/src/app/routers/checkpointer/router.py +482 -0
  159. tenxgraph_api/src/app/routers/checkpointer/schemas/__init__.py +1 -0
  160. tenxgraph_api/src/app/routers/checkpointer/schemas/checkpointer_schemas.py +121 -0
  161. tenxgraph_api/src/app/routers/checkpointer/services/__init__.py +1 -0
  162. tenxgraph_api/src/app/routers/checkpointer/services/checkpointer_service.py +300 -0
  163. tenxgraph_api/src/app/routers/evals/__init__.py +6 -0
  164. tenxgraph_api/src/app/routers/evals/router.py +38 -0
  165. tenxgraph_api/src/app/routers/evals/services/__init__.py +1 -0
  166. tenxgraph_api/src/app/routers/evals/services/eval_report_service.py +354 -0
  167. tenxgraph_api/src/app/routers/graph/__init__.py +6 -0
  168. tenxgraph_api/src/app/routers/graph/realtime_guard.py +161 -0
  169. tenxgraph_api/src/app/routers/graph/router.py +821 -0
  170. tenxgraph_api/src/app/routers/graph/schemas/__init__.py +1 -0
  171. tenxgraph_api/src/app/routers/graph/schemas/graph_schemas.py +386 -0
  172. tenxgraph_api/src/app/routers/graph/services/__init__.py +1 -0
  173. tenxgraph_api/src/app/routers/graph/services/graph_service.py +1219 -0
  174. tenxgraph_api/src/app/routers/graph/services/multimodal_preprocessor.py +164 -0
  175. tenxgraph_api/src/app/routers/media/__init__.py +506 -0
  176. tenxgraph_api/src/app/routers/media/router.py +242 -0
  177. tenxgraph_api/src/app/routers/media/schemas.py +39 -0
  178. tenxgraph_api/src/app/routers/ping/__init__.py +4 -0
  179. tenxgraph_api/src/app/routers/ping/router.py +28 -0
  180. tenxgraph_api/src/app/routers/setup_router.py +60 -0
  181. tenxgraph_api/src/app/routers/store/__init__.py +4 -0
  182. tenxgraph_api/src/app/routers/store/router.py +200 -0
  183. tenxgraph_api/src/app/routers/store/schemas/__init__.py +28 -0
  184. tenxgraph_api/src/app/routers/store/schemas/store_schemas.py +200 -0
  185. tenxgraph_api/src/app/routers/store/services/__init__.py +4 -0
  186. tenxgraph_api/src/app/routers/store/services/store_service.py +203 -0
  187. tenxgraph_api/src/app/tasks/__init__.py +0 -0
  188. tenxgraph_api/src/app/tasks/user_tasks.py +38 -0
  189. tenxgraph_api/src/app/utils/__init__.py +12 -0
  190. tenxgraph_api/src/app/utils/callable_helper.py +26 -0
  191. tenxgraph_api/src/app/utils/media/__init__.py +15 -0
  192. tenxgraph_api/src/app/utils/media/extractor.py +172 -0
  193. tenxgraph_api/src/app/utils/media/pipeline.py +83 -0
  194. tenxgraph_api/src/app/utils/parse_output.py +17 -0
  195. tenxgraph_api/src/app/utils/response_helper.py +129 -0
  196. tenxgraph_api/src/app/utils/schemas/__init__.py +11 -0
  197. tenxgraph_api/src/app/utils/schemas/output_schemas.py +28 -0
  198. tenxgraph_api/src/app/utils/schemas/user_schemas.py +13 -0
  199. tenxgraph_api/src/app/utils/snowflake_id_generator.py +90 -0
  200. tenxgraph_api/src/app/utils/swagger_helper.py +160 -0
  201. tenxgraph_api/src/app/utils/telemetry_store.py +123 -0
  202. tenxgraph_api/src/app/utils/thread_name_generator.py +304 -0
  203. tenxgraph_api/src/app/worker.py +91 -0
@@ -0,0 +1,354 @@
1
+ """Reads real `10xgraph eval` reports for the evals API.
2
+
3
+ Reports are written by `10xgraph eval` (tenxgraph_api/cli/commands/eval.py) via the core
4
+ `ReporterManager` / `JSONReporter` (tenxgraph/qa/evaluation/reporters/json.py) as
5
+ `eval_reports/*.json`, one file per run. That schema (`EvalReport` / `EvalCaseResult` /
6
+ `EvalSummary` in tenxgraph/qa/evaluation/eval_result.py) has no run-number, no dollar cost
7
+ (only token counts), no per-case "expected" value, and no built-in regression comparison
8
+ between runs — those are derived here so the response keeps the shape the playground's
9
+ Evals inspector already renders (see agentflow-playground/src/pages/evals/data.js).
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import json
15
+ import time
16
+ from pathlib import Path
17
+ from typing import Any
18
+
19
+ from fastapi import HTTPException
20
+
21
+
22
+ DEFAULT_REPORTS_DIR = "eval_reports"
23
+ SECONDS_PER_MINUTE = 60
24
+ MINUTES_PER_HOUR = 60
25
+ HOURS_PER_DAY = 24
26
+ TOKENS_PER_KILO = 1000
27
+ REGRESSION_FLAT_EPSILON = 0.005
28
+
29
+
30
+ def _format_ago(timestamp: float, now: float) -> str:
31
+ delta = max(0.0, now - timestamp)
32
+ if delta < SECONDS_PER_MINUTE:
33
+ return "just now"
34
+ minutes = delta / SECONDS_PER_MINUTE
35
+ if minutes < MINUTES_PER_HOUR:
36
+ return f"{int(minutes)}m ago"
37
+ hours = minutes / MINUTES_PER_HOUR
38
+ if hours < HOURS_PER_DAY:
39
+ return f"{int(hours)}h ago"
40
+ days = hours / HOURS_PER_DAY
41
+ return f"{int(days)}d ago"
42
+
43
+
44
+ def _format_duration(seconds: float) -> str:
45
+ return f"{seconds:.1f}s"
46
+
47
+
48
+ def _format_tokens(token_usage: dict[str, Any] | None) -> str:
49
+ total = (token_usage or {}).get("total_tokens", 0)
50
+ if total >= TOKENS_PER_KILO:
51
+ return f"{total / TOKENS_PER_KILO:.1f}k tok"
52
+ return f"{total} tok"
53
+
54
+
55
+ def _case_type(case: dict[str, Any]) -> str:
56
+ return "sim" if "turns" in (case.get("metadata") or {}) else "eval"
57
+
58
+
59
+ def _case_score(case: dict[str, Any]) -> float:
60
+ criteria = case.get("criterion_results") or []
61
+ if not criteria:
62
+ return 1.0 if case.get("passed") else 0.0
63
+ return sum(cr.get("score", 0.0) for cr in criteria) / len(criteria)
64
+
65
+
66
+ def _case_input(case: dict[str, Any]) -> str:
67
+ for message in case.get("messages") or []:
68
+ if message.get("role") != "user":
69
+ continue
70
+ content = message.get("content")
71
+ if isinstance(content, str) and content:
72
+ return content
73
+ if isinstance(content, list):
74
+ texts = [
75
+ block.get("text", "")
76
+ for block in content
77
+ if isinstance(block, dict) and block.get("type") == "text"
78
+ ]
79
+ if texts:
80
+ return " ".join(texts)
81
+ return "—"
82
+
83
+
84
+ def _case_conversation(case: dict[str, Any]) -> list[dict[str, str]] | None:
85
+ if _case_type(case) != "sim":
86
+ return None
87
+ turns: list[dict[str, str]] = []
88
+ for raw_line in (case.get("actual_response") or "").splitlines():
89
+ stripped = raw_line.strip()
90
+ if not stripped:
91
+ continue
92
+ if stripped.startswith("USER:"):
93
+ turns.append({"role": "sim", "text": stripped[len("USER:") :].strip()})
94
+ elif stripped.startswith("AGENT:"):
95
+ turns.append({"role": "agent", "text": stripped[len("AGENT:") :].strip()})
96
+ else:
97
+ turns.append({"role": "agent", "text": stripped})
98
+ return turns or None
99
+
100
+
101
+ def _case_rubric(case: dict[str, Any]) -> list[dict[str, Any]] | None:
102
+ criteria = case.get("criterion_results") or []
103
+ if not criteria:
104
+ return None
105
+ rubric = [
106
+ {
107
+ "key": cr.get("criterion", ""),
108
+ "value": round(cr.get("score", 0.0), 2),
109
+ "tone": "accent" if cr.get("passed") else "danger",
110
+ }
111
+ for cr in criteria
112
+ ]
113
+ rubric.append(
114
+ {
115
+ "key": "weighted score",
116
+ "value": round(_case_score(case), 2),
117
+ "tone": "accent" if case.get("passed") else "danger",
118
+ }
119
+ )
120
+ return rubric
121
+
122
+
123
+ def _build_case_view(case: dict[str, Any]) -> dict[str, Any]:
124
+ return {
125
+ "id": case.get("eval_id", ""),
126
+ "name": case.get("name") or case.get("eval_id", ""),
127
+ "type": _case_type(case),
128
+ "score": round(_case_score(case), 2),
129
+ "status": "pass" if case.get("passed") else "fail",
130
+ "lat": _format_duration(case.get("duration_seconds", 0.0)),
131
+ "cost": _format_tokens(case.get("token_usage")),
132
+ "input": _case_input(case),
133
+ "expected": "—",
134
+ "actual": case.get("actual_response") or "—",
135
+ "rubric": _case_rubric(case),
136
+ "conversation": _case_conversation(case),
137
+ }
138
+
139
+
140
+ def _run_status(report: dict[str, Any]) -> str:
141
+ return "pass" if report.get("summary", {}).get("pass_rate", 0.0) >= 1.0 else "fail"
142
+
143
+
144
+ def _derive_threshold(report: dict[str, Any]) -> float:
145
+ thresholds = {
146
+ cr.get("threshold", 0.0)
147
+ for case in report.get("results", [])
148
+ for cr in case.get("criterion_results", [])
149
+ }
150
+ if not thresholds:
151
+ return 80.0
152
+ return round(100 * sum(thresholds) / len(thresholds))
153
+
154
+
155
+ def _build_stats(report: dict[str, Any]) -> list[dict[str, Any]]:
156
+ summary = report.get("summary", {})
157
+ cases = report.get("results", [])
158
+ scores = [_case_score(c) for c in cases]
159
+ avg_score = sum(scores) / len(scores) if scores else 0.0
160
+ return [
161
+ {"label": "cases", "value": str(summary.get("total_cases", 0))},
162
+ {"label": "passed", "value": str(summary.get("passed_cases", 0)), "tone": "ok"},
163
+ {
164
+ "label": "failed",
165
+ "value": str(summary.get("failed_cases", 0) + summary.get("error_cases", 0)),
166
+ "tone": "bad",
167
+ },
168
+ {"label": "avg score", "value": f"{avg_score:.2f}"},
169
+ {
170
+ "label": "avg latency",
171
+ "value": _format_duration(summary.get("avg_duration_seconds", 0.0)),
172
+ },
173
+ {"label": "total tokens", "value": _format_tokens(summary.get("total_token_usage"))},
174
+ ]
175
+
176
+
177
+ def _build_run_row(
178
+ report: dict[str, Any], run_id: str, run_label: str, now: float
179
+ ) -> dict[str, Any]:
180
+ summary = report.get("summary", {})
181
+ return {
182
+ "id": run_id,
183
+ "name": report.get("eval_set_name") or report.get("eval_set_id", ""),
184
+ "run": run_label,
185
+ "rate": round(summary.get("pass_rate", 0.0) * 100, 1),
186
+ "status": _run_status(report),
187
+ "cases": summary.get("total_cases", 0),
188
+ "ago": _format_ago(report.get("timestamp", now), now),
189
+ }
190
+
191
+
192
+ def _build_regression(
193
+ current: dict[str, Any],
194
+ current_label: str,
195
+ previous: dict[str, Any],
196
+ previous_label: str,
197
+ ) -> dict[str, Any]:
198
+ cur_cases = {c["eval_id"]: c for c in current.get("results", [])}
199
+ prev_cases = {c["eval_id"]: c for c in previous.get("results", [])}
200
+
201
+ rows: list[dict[str, Any]] = []
202
+ newly_failing = 0
203
+ newly_passing = 0
204
+ for eval_id, cur_case in cur_cases.items():
205
+ prev_case = prev_cases.get(eval_id)
206
+ if prev_case is None:
207
+ continue
208
+ cur_score = _case_score(cur_case)
209
+ prev_score = _case_score(prev_case)
210
+ delta = cur_score - prev_score
211
+
212
+ if prev_case["passed"] and not cur_case["passed"]:
213
+ newly_failing += 1
214
+ flip = "pass → fail"
215
+ elif not prev_case["passed"] and cur_case["passed"]:
216
+ newly_passing += 1
217
+ flip = "fail → pass"
218
+ else:
219
+ flip = f"{'pass' if cur_case['passed'] else 'fail'} (same)"
220
+
221
+ if abs(delta) < REGRESSION_FLAT_EPSILON:
222
+ dir_, arrow = "flat", "—"
223
+ elif delta > 0:
224
+ dir_, arrow = "up", f"▲{delta:.2f}"
225
+ else:
226
+ dir_, arrow = "down", f"▼{abs(delta):.2f}"
227
+
228
+ rows.append(
229
+ {
230
+ "name": cur_case.get("name") or eval_id,
231
+ "delta": f"{prev_score:.2f} → {cur_score:.2f} {arrow}",
232
+ "dir": dir_,
233
+ "flip": flip,
234
+ "stay": cur_case["passed"] == prev_case["passed"],
235
+ }
236
+ )
237
+
238
+ cur_rate = current.get("summary", {}).get("pass_rate", 0.0) * 100
239
+ prev_rate = previous.get("summary", {}).get("pass_rate", 0.0) * 100
240
+ cur_scores = [_case_score(c) for c in current.get("results", [])]
241
+ prev_scores = [_case_score(c) for c in previous.get("results", [])]
242
+ avg_cur = sum(cur_scores) / len(cur_scores) if cur_scores else 0.0
243
+ avg_prev = sum(prev_scores) / len(prev_scores) if prev_scores else 0.0
244
+ score_drift = avg_cur - avg_prev
245
+ rate_drift = cur_rate - prev_rate
246
+
247
+ return {
248
+ "note": {
249
+ "current": f"run {current_label} ({cur_rate:.1f}%)",
250
+ "prev": f"run {previous_label} ({prev_rate:.1f}%)",
251
+ "suite": "same suite",
252
+ },
253
+ "summary": [
254
+ {
255
+ "label": "pass-rate drift",
256
+ "value": f"{rate_drift:+.1f}pp",
257
+ "tone": "bad" if rate_drift < 0 else "ok",
258
+ },
259
+ {
260
+ "label": "newly failing",
261
+ "value": str(newly_failing),
262
+ "tone": "bad" if newly_failing else "ok",
263
+ },
264
+ {"label": "newly passing", "value": str(newly_passing), "tone": "ok"},
265
+ {
266
+ "label": "avg score drift",
267
+ "value": f"{score_drift:+.2f}",
268
+ "tone": "warn" if score_drift < 0 else "ok",
269
+ },
270
+ ],
271
+ "rows": rows,
272
+ }
273
+
274
+
275
+ class EvalReportService:
276
+ """Reads `eval_reports/*.json` and adapts them into the run/detail shape
277
+ the playground's Evals inspector expects."""
278
+
279
+ def __init__(self, reports_dir: str | Path | None = None) -> None:
280
+ base = Path(reports_dir) if reports_dir is not None else Path(DEFAULT_REPORTS_DIR)
281
+ self.reports_dir = base if base.is_absolute() else Path.cwd() / base
282
+
283
+ def _load_reports(self) -> list[tuple[str, dict[str, Any]]]:
284
+ if not self.reports_dir.is_dir():
285
+ return []
286
+ reports = []
287
+ for path in sorted(self.reports_dir.glob("*.json")):
288
+ try:
289
+ data = json.loads(path.read_text(encoding="utf-8"))
290
+ except (json.JSONDecodeError, OSError):
291
+ continue
292
+ reports.append((path.stem, data))
293
+ reports.sort(key=lambda item: item[1].get("timestamp", 0.0))
294
+ return reports
295
+
296
+ def _run_labels(self) -> dict[str, tuple[int, dict[str, Any], str | None]]:
297
+ """run_id -> (run_number, report, previous_run_id), run_number and
298
+ previous_run_id computed per eval_set_id in chronological order."""
299
+ counters: dict[str, int] = {}
300
+ previous_by_name: dict[str, str] = {}
301
+ labels: dict[str, tuple[int, dict[str, Any], str | None]] = {}
302
+ for run_id, report in self._load_reports():
303
+ name = report.get("eval_set_id", "")
304
+ counters[name] = counters.get(name, 0) + 1
305
+ labels[run_id] = (counters[name], report, previous_by_name.get(name))
306
+ previous_by_name[name] = run_id
307
+ return labels
308
+
309
+ def list_runs(self, now: float | None = None) -> list[dict[str, Any]]:
310
+ now = now if now is not None else time.time()
311
+ labels = self._run_labels()
312
+ rows = [
313
+ (report.get("timestamp", 0.0), _build_run_row(report, run_id, f"#{run_number}", now))
314
+ for run_id, (run_number, report, _prev) in labels.items()
315
+ ]
316
+ rows.sort(key=lambda item: item[0], reverse=True)
317
+ return [row for _, row in rows]
318
+
319
+ def get_run_detail(self, run_id: str) -> dict[str, Any]:
320
+ labels = self._run_labels()
321
+ entry = labels.get(run_id)
322
+ if entry is None:
323
+ raise HTTPException(status_code=404, detail=f"Eval run '{run_id}' not found")
324
+ run_number, report, previous_run_id = entry
325
+ summary = report.get("summary", {})
326
+
327
+ regression = None
328
+ if previous_run_id is not None:
329
+ prev_entry = labels.get(previous_run_id)
330
+ if prev_entry is not None:
331
+ prev_number, prev_report, _ = prev_entry
332
+ regression = _build_regression(
333
+ report, f"#{run_number}", prev_report, f"#{prev_number}"
334
+ )
335
+
336
+ model = (report.get("config_used") or {}).get("model")
337
+ run_timestamp = time.gmtime(report.get("timestamp", 0.0))
338
+ sub_parts = [report.get("eval_set_id", "")]
339
+ if model:
340
+ sub_parts.append(str(model))
341
+ sub_parts.append(time.strftime("%Y-%m-%d %H:%M", run_timestamp))
342
+ sub_parts.append(_format_duration(summary.get("total_duration_seconds", 0.0)))
343
+
344
+ set_label = report.get("eval_set_name") or report.get("eval_set_id", "")
345
+ return {
346
+ "title": f"{set_label} · run #{run_number}",
347
+ "sub": " · ".join(sub_parts),
348
+ "rate": round(summary.get("pass_rate", 0.0) * 100, 1),
349
+ "status": _run_status(report),
350
+ "threshold": _derive_threshold(report),
351
+ "stats": _build_stats(report),
352
+ "cases": [_build_case_view(c) for c in report.get("results", [])],
353
+ "regression": regression,
354
+ }
@@ -0,0 +1,6 @@
1
+ # Graph router module
2
+
3
+ from .router import router
4
+
5
+
6
+ __all__ = ["router"]
@@ -0,0 +1,161 @@
1
+ """Connection guard for the WebSocket endpoints.
2
+
3
+ Rate-limit and request-size middleware are ``BaseHTTPMiddleware`` and Starlette runs them
4
+ only for HTTP scopes, so WebSocket handshakes bypass them entirely. This module re-applies
5
+ two protections at the handshake, as a FastAPI dependency:
6
+
7
+ 1. The same global rate limit as REST (shared backend + bucket), so opening a socket counts
8
+ like any other request.
9
+ 2. A per-process cap on concurrent WebSocket connections (``websocket.max_connections`` in
10
+ 10xgraph.json), and a per-user cap (``websocket.max_connections_per_user``) so one
11
+ account cannot hold every slot.
12
+
13
+ A socket lives much longer than its handshake, so two helpers apply for its whole lifetime:
14
+ :func:`ws_run_allowed` counts every graph run against the same rate-limit bucket, and
15
+ :func:`ws_identity_still_valid` re-verifies the credential so an expired or revoked token
16
+ cannot keep a socket working.
17
+
18
+ Rejections raise ``WebSocketException`` before ``accept()``, so the handshake fails with a
19
+ close code instead of leaving a half-open socket. The concurrency slot is released on
20
+ teardown of the (yield) dependency, i.e. when the handler returns or the client disconnects.
21
+ """
22
+
23
+ from collections.abc import AsyncIterator
24
+ from typing import Any
25
+
26
+ from fastapi import WebSocket, WebSocketException
27
+ from injectq.integrations import InjectAPI
28
+
29
+ from tenxgraph_api.src.app.core import logger
30
+ from tenxgraph_api.src.app.core.config.graph_config import GraphConfig
31
+ from tenxgraph_api.src.app.core.middleware.rate_limit import keying
32
+ from tenxgraph_api.src.app.core.middleware.rate_limit.keying import client_key_for
33
+
34
+
35
+ # RFC 6455 close code 1013 "Try Again Later" -- the right signal for shed-load rejections.
36
+ WS_TRY_AGAIN_LATER = 1013
37
+
38
+
39
+ class _ConnectionRegistry:
40
+ """Per-process counter of active WebSocket connections.
41
+
42
+ The event loop is single-threaded, so the check-and-increment in :meth:`try_acquire` is
43
+ atomic (no ``await`` between the test and the mutation) and needs no lock. This counter is
44
+ per process, matching the in-memory rate-limit backend's scope; for a multi-worker
45
+ deployment, set ``max_connections`` per worker accordingly.
46
+ """
47
+
48
+ def __init__(self) -> None:
49
+ self._active = 0
50
+ self._per_user: dict[str, int] = {}
51
+
52
+ @property
53
+ def active(self) -> int:
54
+ return self._active
55
+
56
+ def try_acquire(self, max_connections: int | None) -> bool:
57
+ if max_connections is not None and self._active >= max_connections:
58
+ return False
59
+ self._active += 1
60
+ return True
61
+
62
+ def release(self) -> None:
63
+ if self._active > 0:
64
+ self._active -= 1
65
+
66
+ def try_acquire_user(self, user_key: str, max_per_user: int | None) -> bool:
67
+ held = self._per_user.get(user_key, 0)
68
+ if max_per_user is not None and held >= max_per_user:
69
+ return False
70
+ self._per_user[user_key] = held + 1
71
+ return True
72
+
73
+ def release_user(self, user_key: str) -> None:
74
+ held = self._per_user.get(user_key, 0)
75
+ if held <= 1:
76
+ self._per_user.pop(user_key, None)
77
+ else:
78
+ self._per_user[user_key] = held - 1
79
+
80
+
81
+ # Module-level singleton (per process).
82
+ _registry = _ConnectionRegistry()
83
+
84
+
85
+ async def realtime_connection_guard(
86
+ websocket: WebSocket,
87
+ config: GraphConfig = InjectAPI(GraphConfig),
88
+ ) -> AsyncIterator[None]:
89
+ """Gate a WebSocket handshake on the global rate limit and the concurrent-connection cap."""
90
+ # 1) Shared global rate limit (same backend/bucket as the REST middleware).
91
+ rl_config = config.rate_limit
92
+ backend = getattr(getattr(websocket, "app", None), "state", None)
93
+ backend = getattr(backend, "rate_limit_backend", None)
94
+ if rl_config is not None and backend is not None:
95
+ # Kept on the connection so every run on this socket counts too (ws_run_allowed).
96
+ websocket.state.ws_rate_limit = (rl_config, backend)
97
+ key = client_key_for(websocket, rl_config)
98
+ decision = await backend.check(key, limit=rl_config.requests, window=rl_config.window)
99
+ if not decision.allowed:
100
+ logger.warning("WebSocket rate limit exceeded for %s", key)
101
+ raise WebSocketException(code=WS_TRY_AGAIN_LATER, reason="Rate limit exceeded")
102
+
103
+ # 2) Per-process concurrent-connection cap, then the per-user share of it.
104
+ ws_config = config.websocket
105
+ max_conn = ws_config.max_connections
106
+ if not _registry.try_acquire(max_conn):
107
+ logger.warning(
108
+ "WebSocket connection limit reached (active=%d, max=%s)",
109
+ _registry.active,
110
+ max_conn,
111
+ )
112
+ raise WebSocketException(code=WS_TRY_AGAIN_LATER, reason="Too many connections")
113
+
114
+ user_key = keying.verified_user_id(websocket)
115
+ if user_key and not _registry.try_acquire_user(user_key, ws_config.max_connections_per_user):
116
+ _registry.release()
117
+ logger.warning(
118
+ "WebSocket per-user connection limit reached for %s (max=%s)",
119
+ user_key,
120
+ ws_config.max_connections_per_user,
121
+ )
122
+ raise WebSocketException(
123
+ code=WS_TRY_AGAIN_LATER, reason="Too many connections for this user"
124
+ )
125
+
126
+ try:
127
+ yield
128
+ finally:
129
+ _registry.release()
130
+ if user_key:
131
+ _registry.release_user(user_key)
132
+
133
+
134
+ async def ws_run_allowed(websocket: WebSocket) -> int | None:
135
+ """Count one graph run against the rate limit.
136
+
137
+ Returns ``None`` when the run may start, or the seconds to wait when the caller is over
138
+ the limit. Runs share the handshake's bucket, which is also the REST bucket.
139
+ """
140
+ limiter = getattr(websocket.state, "ws_rate_limit", None)
141
+ if not isinstance(limiter, tuple): # the guard stored none: rate limiting is off
142
+ return None
143
+ rl_config, backend = limiter
144
+ key = client_key_for(websocket, rl_config)
145
+ decision = await backend.check(key, limit=rl_config.requests, window=rl_config.window)
146
+ if decision.allowed:
147
+ return None
148
+ logger.warning("WebSocket run rate limit exceeded for %s", key)
149
+ return decision.reset_after
150
+
151
+
152
+ def ws_identity_still_valid(websocket: Any, user: dict[str, Any]) -> bool:
153
+ """Whether the socket's credential still verifies as the user it was opened by.
154
+
155
+ Checked before each run: without it, a socket keeps working after its token expires or
156
+ is revoked. Always true when auth is not configured (``user`` has no ``user_id``).
157
+ """
158
+ user_id = user.get("user_id") if isinstance(user, dict) else None
159
+ if not user_id:
160
+ return True
161
+ return keying.verified_user_id(websocket) == str(user_id)