10xgraph-api 0.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- 10xgraph_api-0.7.0.dist-info/METADATA +618 -0
- 10xgraph_api-0.7.0.dist-info/RECORD +203 -0
- 10xgraph_api-0.7.0.dist-info/WHEEL +5 -0
- 10xgraph_api-0.7.0.dist-info/entry_points.txt +3 -0
- 10xgraph_api-0.7.0.dist-info/licenses/LICENSE +21 -0
- 10xgraph_api-0.7.0.dist-info/top_level.txt +2 -0
- agentflow_cli/__init__.py +69 -0
- tenxgraph_api/__init__.py +23 -0
- tenxgraph_api/cli/__init__.py +6 -0
- tenxgraph_api/cli/capabilities.py +127 -0
- tenxgraph_api/cli/commands/__init__.py +49 -0
- tenxgraph_api/cli/commands/api.py +262 -0
- tenxgraph_api/cli/commands/audit.py +213 -0
- tenxgraph_api/cli/commands/build.py +305 -0
- tenxgraph_api/cli/commands/config.py +71 -0
- tenxgraph_api/cli/commands/demo.py +93 -0
- tenxgraph_api/cli/commands/eval.py +909 -0
- tenxgraph_api/cli/commands/init.py +531 -0
- tenxgraph_api/cli/commands/skills.py +532 -0
- tenxgraph_api/cli/commands/test.py +113 -0
- tenxgraph_api/cli/commands/version.py +47 -0
- tenxgraph_api/cli/config_editor/__init__.py +15 -0
- tenxgraph_api/cli/config_editor/schema.py +566 -0
- tenxgraph_api/cli/config_editor/server.py +240 -0
- tenxgraph_api/cli/config_editor/static/app.css +2 -0
- tenxgraph_api/cli/config_editor/static/app.js +3 -0
- tenxgraph_api/cli/config_editor/static/favicon.svg +11 -0
- tenxgraph_api/cli/config_editor/static/index.html +20 -0
- tenxgraph_api/cli/config_editor/store.py +120 -0
- tenxgraph_api/cli/config_editor/validation.py +377 -0
- tenxgraph_api/cli/constants.py +112 -0
- tenxgraph_api/cli/context.py +46 -0
- tenxgraph_api/cli/core/__init__.py +0 -0
- tenxgraph_api/cli/core/animations.py +550 -0
- tenxgraph_api/cli/core/config.py +279 -0
- tenxgraph_api/cli/core/output.py +625 -0
- tenxgraph_api/cli/core/prompts.py +220 -0
- tenxgraph_api/cli/core/screen.py +287 -0
- tenxgraph_api/cli/core/steps.py +621 -0
- tenxgraph_api/cli/core/theme.py +242 -0
- tenxgraph_api/cli/core/validation.py +251 -0
- tenxgraph_api/cli/exceptions.py +129 -0
- tenxgraph_api/cli/logger.py +84 -0
- tenxgraph_api/cli/main.py +936 -0
- tenxgraph_api/cli/templates/__init__.py +0 -0
- tenxgraph_api/cli/templates/defaults.py +844 -0
- tenxgraph_api/cli/templates/dev/.env.example +1 -0
- tenxgraph_api/cli/templates/dev/10xgraph.json +16 -0
- tenxgraph_api/cli/templates/dev/graph/__init__.py +0 -0
- tenxgraph_api/cli/templates/dev/graph/agent.py +63 -0
- tenxgraph_api/cli/templates/prod/.env.example +75 -0
- tenxgraph_api/cli/templates/prod/.pre-commit-config.yaml +50 -0
- tenxgraph_api/cli/templates/prod/.python-version +1 -0
- tenxgraph_api/cli/templates/prod/10xgraph.json +42 -0
- tenxgraph_api/cli/templates/prod/auth/__init__.py +0 -0
- tenxgraph_api/cli/templates/prod/auth/agent_auth.py +46 -0
- tenxgraph_api/cli/templates/prod/evals/__init__.py +0 -0
- tenxgraph_api/cli/templates/prod/evals/user_simulator_eval.py +76 -0
- tenxgraph_api/cli/templates/prod/evals/weather_agents_eval.py +53 -0
- tenxgraph_api/cli/templates/prod/graph/__init__.py +0 -0
- tenxgraph_api/cli/templates/prod/graph/agent.py +50 -0
- tenxgraph_api/cli/templates/prod/graph/state.py +5 -0
- tenxgraph_api/cli/templates/prod/graph/thread_name_generator.py +7 -0
- tenxgraph_api/cli/templates/prod/graph/tools/__init__.py +0 -0
- tenxgraph_api/cli/templates/prod/graph/tools/weather_tool.py +43 -0
- tenxgraph_api/cli/templates/prod/graph/validators/__init__.py +0 -0
- tenxgraph_api/cli/templates/prod/graph/validators/lifecyle.py +107 -0
- tenxgraph_api/cli/templates/prod/graph/validators/manager.py +9 -0
- tenxgraph_api/cli/templates/prod/graph/validators/validators.py +21 -0
- tenxgraph_api/cli/templates/prod/pyproject.toml +79 -0
- tenxgraph_api/cli/templates/prod/tests/__init__.py +0 -0
- tenxgraph_api/cli/templates/prod/tests/conftest.py +6 -0
- tenxgraph_api/cli/templates/prod/tests/test_agent_eval.py +78 -0
- tenxgraph_api/cli/templates/prod/tests/test_catalog_tools.py +47 -0
- tenxgraph_api/cli/templates/prod/tests/test_graph_nodes.py +57 -0
- tenxgraph_api/cli/templates/skills/10xgraph/SKILL.md +92 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/agents-and-tools.md +389 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/api-client.md +61 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/api-configuration.md +75 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/api-settings-and-middleware.md +89 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/architecture.md +91 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/auth-and-authorization.md +149 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/callbacks-and-command.md +283 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/checkpointing-and-threads.md +63 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/cli-commands.md +73 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/client-auth-and-errors.md +94 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/client-messages-invoke-stream.md +115 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/client-threads-memory-files.md +80 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/context-id-background.md +80 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/dependency-injection.md +58 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/evaluation.md +500 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/id-and-thread-name-generators.md +69 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/media-and-files.md +66 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/memory-and-store.md +70 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/prebuilt-agents-and-tools.md +394 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/production-runtime.md +56 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/providers-and-adapters.md +89 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/publishers-and-runtime-protocols.md +84 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/rate-limiting.md +194 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/realtime.md +172 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/remote-tools.md +55 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/rest-api-and-errors.md +97 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/security-and-validators.md +86 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/state-and-messages.md +68 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/state-graph.md +232 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/stream-emitter.md +144 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/streaming.md +53 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/testing-and-evaluation.md +64 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/unit-testing.md +202 -0
- tenxgraph_api/cli/templates/skills/copilot/10xgraph.instructions.md +108 -0
- tenxgraph_api/py.typed +0 -0
- tenxgraph_api/src/__init__.py +0 -0
- tenxgraph_api/src/app/__init__.py +13 -0
- tenxgraph_api/src/app/core/__init__.py +15 -0
- tenxgraph_api/src/app/core/auth/__init__.py +0 -0
- tenxgraph_api/src/app/core/auth/auth_backend.py +38 -0
- tenxgraph_api/src/app/core/auth/authorization.py +379 -0
- tenxgraph_api/src/app/core/auth/base_auth.py +27 -0
- tenxgraph_api/src/app/core/auth/jwt_auth.py +122 -0
- tenxgraph_api/src/app/core/auth/ownership_resolver.py +170 -0
- tenxgraph_api/src/app/core/auth/permissions.py +354 -0
- tenxgraph_api/src/app/core/auth/request_config.py +130 -0
- tenxgraph_api/src/app/core/auth/route_guard.py +105 -0
- tenxgraph_api/src/app/core/config/__init__.py +0 -0
- tenxgraph_api/src/app/core/config/graph_config.py +532 -0
- tenxgraph_api/src/app/core/config/media_settings.py +72 -0
- tenxgraph_api/src/app/core/config/sentry_config.py +60 -0
- tenxgraph_api/src/app/core/config/settings.py +202 -0
- tenxgraph_api/src/app/core/config/setup_logs.py +65 -0
- tenxgraph_api/src/app/core/config/setup_middleware.py +483 -0
- tenxgraph_api/src/app/core/config/worker_middleware.py +99 -0
- tenxgraph_api/src/app/core/exceptions/__init__.py +17 -0
- tenxgraph_api/src/app/core/exceptions/general_exception.py +58 -0
- tenxgraph_api/src/app/core/exceptions/handle_errors.py +363 -0
- tenxgraph_api/src/app/core/exceptions/resources_exceptions.py +72 -0
- tenxgraph_api/src/app/core/exceptions/user_exception.py +58 -0
- tenxgraph_api/src/app/core/middleware/__init__.py +25 -0
- tenxgraph_api/src/app/core/middleware/rate_limit/__init__.py +15 -0
- tenxgraph_api/src/app/core/middleware/rate_limit/base.py +27 -0
- tenxgraph_api/src/app/core/middleware/rate_limit/factory.py +98 -0
- tenxgraph_api/src/app/core/middleware/rate_limit/keying.py +133 -0
- tenxgraph_api/src/app/core/middleware/rate_limit/memory.py +98 -0
- tenxgraph_api/src/app/core/middleware/rate_limit/middleware.py +90 -0
- tenxgraph_api/src/app/core/middleware/rate_limit/redis.py +115 -0
- tenxgraph_api/src/app/core/middleware/request_limits.py +154 -0
- tenxgraph_api/src/app/core/middleware/security_headers.py +239 -0
- tenxgraph_api/src/app/core/utils/__init__.py +14 -0
- tenxgraph_api/src/app/core/utils/log_sanitizer.py +216 -0
- tenxgraph_api/src/app/loader.py +516 -0
- tenxgraph_api/src/app/main.py +180 -0
- tenxgraph_api/src/app/routers/__init__.py +4 -0
- tenxgraph_api/src/app/routers/ag_ui/__init__.py +10 -0
- tenxgraph_api/src/app/routers/ag_ui/converter.py +195 -0
- tenxgraph_api/src/app/routers/ag_ui/event_mapper.py +353 -0
- tenxgraph_api/src/app/routers/ag_ui/router.py +90 -0
- tenxgraph_api/src/app/routers/ag_ui/service.py +193 -0
- tenxgraph_api/src/app/routers/checkpointer/__init__.py +1 -0
- tenxgraph_api/src/app/routers/checkpointer/router.py +482 -0
- tenxgraph_api/src/app/routers/checkpointer/schemas/__init__.py +1 -0
- tenxgraph_api/src/app/routers/checkpointer/schemas/checkpointer_schemas.py +121 -0
- tenxgraph_api/src/app/routers/checkpointer/services/__init__.py +1 -0
- tenxgraph_api/src/app/routers/checkpointer/services/checkpointer_service.py +300 -0
- tenxgraph_api/src/app/routers/evals/__init__.py +6 -0
- tenxgraph_api/src/app/routers/evals/router.py +38 -0
- tenxgraph_api/src/app/routers/evals/services/__init__.py +1 -0
- tenxgraph_api/src/app/routers/evals/services/eval_report_service.py +354 -0
- tenxgraph_api/src/app/routers/graph/__init__.py +6 -0
- tenxgraph_api/src/app/routers/graph/realtime_guard.py +161 -0
- tenxgraph_api/src/app/routers/graph/router.py +821 -0
- tenxgraph_api/src/app/routers/graph/schemas/__init__.py +1 -0
- tenxgraph_api/src/app/routers/graph/schemas/graph_schemas.py +386 -0
- tenxgraph_api/src/app/routers/graph/services/__init__.py +1 -0
- tenxgraph_api/src/app/routers/graph/services/graph_service.py +1219 -0
- tenxgraph_api/src/app/routers/graph/services/multimodal_preprocessor.py +164 -0
- tenxgraph_api/src/app/routers/media/__init__.py +506 -0
- tenxgraph_api/src/app/routers/media/router.py +242 -0
- tenxgraph_api/src/app/routers/media/schemas.py +39 -0
- tenxgraph_api/src/app/routers/ping/__init__.py +4 -0
- tenxgraph_api/src/app/routers/ping/router.py +28 -0
- tenxgraph_api/src/app/routers/setup_router.py +60 -0
- tenxgraph_api/src/app/routers/store/__init__.py +4 -0
- tenxgraph_api/src/app/routers/store/router.py +200 -0
- tenxgraph_api/src/app/routers/store/schemas/__init__.py +28 -0
- tenxgraph_api/src/app/routers/store/schemas/store_schemas.py +200 -0
- tenxgraph_api/src/app/routers/store/services/__init__.py +4 -0
- tenxgraph_api/src/app/routers/store/services/store_service.py +203 -0
- tenxgraph_api/src/app/tasks/__init__.py +0 -0
- tenxgraph_api/src/app/tasks/user_tasks.py +38 -0
- tenxgraph_api/src/app/utils/__init__.py +12 -0
- tenxgraph_api/src/app/utils/callable_helper.py +26 -0
- tenxgraph_api/src/app/utils/media/__init__.py +15 -0
- tenxgraph_api/src/app/utils/media/extractor.py +172 -0
- tenxgraph_api/src/app/utils/media/pipeline.py +83 -0
- tenxgraph_api/src/app/utils/parse_output.py +17 -0
- tenxgraph_api/src/app/utils/response_helper.py +129 -0
- tenxgraph_api/src/app/utils/schemas/__init__.py +11 -0
- tenxgraph_api/src/app/utils/schemas/output_schemas.py +28 -0
- tenxgraph_api/src/app/utils/schemas/user_schemas.py +13 -0
- tenxgraph_api/src/app/utils/snowflake_id_generator.py +90 -0
- tenxgraph_api/src/app/utils/swagger_helper.py +160 -0
- tenxgraph_api/src/app/utils/telemetry_store.py +123 -0
- tenxgraph_api/src/app/utils/thread_name_generator.py +304 -0
- tenxgraph_api/src/app/worker.py +91 -0
|
@@ -0,0 +1,354 @@
|
|
|
1
|
+
"""Reads real `10xgraph eval` reports for the evals API.
|
|
2
|
+
|
|
3
|
+
Reports are written by `10xgraph eval` (tenxgraph_api/cli/commands/eval.py) via the core
|
|
4
|
+
`ReporterManager` / `JSONReporter` (tenxgraph/qa/evaluation/reporters/json.py) as
|
|
5
|
+
`eval_reports/*.json`, one file per run. That schema (`EvalReport` / `EvalCaseResult` /
|
|
6
|
+
`EvalSummary` in tenxgraph/qa/evaluation/eval_result.py) has no run-number, no dollar cost
|
|
7
|
+
(only token counts), no per-case "expected" value, and no built-in regression comparison
|
|
8
|
+
between runs — those are derived here so the response keeps the shape the playground's
|
|
9
|
+
Evals inspector already renders (see agentflow-playground/src/pages/evals/data.js).
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import json
|
|
15
|
+
import time
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
from typing import Any
|
|
18
|
+
|
|
19
|
+
from fastapi import HTTPException
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
DEFAULT_REPORTS_DIR = "eval_reports"
|
|
23
|
+
SECONDS_PER_MINUTE = 60
|
|
24
|
+
MINUTES_PER_HOUR = 60
|
|
25
|
+
HOURS_PER_DAY = 24
|
|
26
|
+
TOKENS_PER_KILO = 1000
|
|
27
|
+
REGRESSION_FLAT_EPSILON = 0.005
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _format_ago(timestamp: float, now: float) -> str:
|
|
31
|
+
delta = max(0.0, now - timestamp)
|
|
32
|
+
if delta < SECONDS_PER_MINUTE:
|
|
33
|
+
return "just now"
|
|
34
|
+
minutes = delta / SECONDS_PER_MINUTE
|
|
35
|
+
if minutes < MINUTES_PER_HOUR:
|
|
36
|
+
return f"{int(minutes)}m ago"
|
|
37
|
+
hours = minutes / MINUTES_PER_HOUR
|
|
38
|
+
if hours < HOURS_PER_DAY:
|
|
39
|
+
return f"{int(hours)}h ago"
|
|
40
|
+
days = hours / HOURS_PER_DAY
|
|
41
|
+
return f"{int(days)}d ago"
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _format_duration(seconds: float) -> str:
|
|
45
|
+
return f"{seconds:.1f}s"
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _format_tokens(token_usage: dict[str, Any] | None) -> str:
|
|
49
|
+
total = (token_usage or {}).get("total_tokens", 0)
|
|
50
|
+
if total >= TOKENS_PER_KILO:
|
|
51
|
+
return f"{total / TOKENS_PER_KILO:.1f}k tok"
|
|
52
|
+
return f"{total} tok"
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _case_type(case: dict[str, Any]) -> str:
|
|
56
|
+
return "sim" if "turns" in (case.get("metadata") or {}) else "eval"
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _case_score(case: dict[str, Any]) -> float:
|
|
60
|
+
criteria = case.get("criterion_results") or []
|
|
61
|
+
if not criteria:
|
|
62
|
+
return 1.0 if case.get("passed") else 0.0
|
|
63
|
+
return sum(cr.get("score", 0.0) for cr in criteria) / len(criteria)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _case_input(case: dict[str, Any]) -> str:
|
|
67
|
+
for message in case.get("messages") or []:
|
|
68
|
+
if message.get("role") != "user":
|
|
69
|
+
continue
|
|
70
|
+
content = message.get("content")
|
|
71
|
+
if isinstance(content, str) and content:
|
|
72
|
+
return content
|
|
73
|
+
if isinstance(content, list):
|
|
74
|
+
texts = [
|
|
75
|
+
block.get("text", "")
|
|
76
|
+
for block in content
|
|
77
|
+
if isinstance(block, dict) and block.get("type") == "text"
|
|
78
|
+
]
|
|
79
|
+
if texts:
|
|
80
|
+
return " ".join(texts)
|
|
81
|
+
return "—"
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _case_conversation(case: dict[str, Any]) -> list[dict[str, str]] | None:
|
|
85
|
+
if _case_type(case) != "sim":
|
|
86
|
+
return None
|
|
87
|
+
turns: list[dict[str, str]] = []
|
|
88
|
+
for raw_line in (case.get("actual_response") or "").splitlines():
|
|
89
|
+
stripped = raw_line.strip()
|
|
90
|
+
if not stripped:
|
|
91
|
+
continue
|
|
92
|
+
if stripped.startswith("USER:"):
|
|
93
|
+
turns.append({"role": "sim", "text": stripped[len("USER:") :].strip()})
|
|
94
|
+
elif stripped.startswith("AGENT:"):
|
|
95
|
+
turns.append({"role": "agent", "text": stripped[len("AGENT:") :].strip()})
|
|
96
|
+
else:
|
|
97
|
+
turns.append({"role": "agent", "text": stripped})
|
|
98
|
+
return turns or None
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _case_rubric(case: dict[str, Any]) -> list[dict[str, Any]] | None:
|
|
102
|
+
criteria = case.get("criterion_results") or []
|
|
103
|
+
if not criteria:
|
|
104
|
+
return None
|
|
105
|
+
rubric = [
|
|
106
|
+
{
|
|
107
|
+
"key": cr.get("criterion", ""),
|
|
108
|
+
"value": round(cr.get("score", 0.0), 2),
|
|
109
|
+
"tone": "accent" if cr.get("passed") else "danger",
|
|
110
|
+
}
|
|
111
|
+
for cr in criteria
|
|
112
|
+
]
|
|
113
|
+
rubric.append(
|
|
114
|
+
{
|
|
115
|
+
"key": "weighted score",
|
|
116
|
+
"value": round(_case_score(case), 2),
|
|
117
|
+
"tone": "accent" if case.get("passed") else "danger",
|
|
118
|
+
}
|
|
119
|
+
)
|
|
120
|
+
return rubric
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _build_case_view(case: dict[str, Any]) -> dict[str, Any]:
|
|
124
|
+
return {
|
|
125
|
+
"id": case.get("eval_id", ""),
|
|
126
|
+
"name": case.get("name") or case.get("eval_id", ""),
|
|
127
|
+
"type": _case_type(case),
|
|
128
|
+
"score": round(_case_score(case), 2),
|
|
129
|
+
"status": "pass" if case.get("passed") else "fail",
|
|
130
|
+
"lat": _format_duration(case.get("duration_seconds", 0.0)),
|
|
131
|
+
"cost": _format_tokens(case.get("token_usage")),
|
|
132
|
+
"input": _case_input(case),
|
|
133
|
+
"expected": "—",
|
|
134
|
+
"actual": case.get("actual_response") or "—",
|
|
135
|
+
"rubric": _case_rubric(case),
|
|
136
|
+
"conversation": _case_conversation(case),
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _run_status(report: dict[str, Any]) -> str:
|
|
141
|
+
return "pass" if report.get("summary", {}).get("pass_rate", 0.0) >= 1.0 else "fail"
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def _derive_threshold(report: dict[str, Any]) -> float:
|
|
145
|
+
thresholds = {
|
|
146
|
+
cr.get("threshold", 0.0)
|
|
147
|
+
for case in report.get("results", [])
|
|
148
|
+
for cr in case.get("criterion_results", [])
|
|
149
|
+
}
|
|
150
|
+
if not thresholds:
|
|
151
|
+
return 80.0
|
|
152
|
+
return round(100 * sum(thresholds) / len(thresholds))
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _build_stats(report: dict[str, Any]) -> list[dict[str, Any]]:
|
|
156
|
+
summary = report.get("summary", {})
|
|
157
|
+
cases = report.get("results", [])
|
|
158
|
+
scores = [_case_score(c) for c in cases]
|
|
159
|
+
avg_score = sum(scores) / len(scores) if scores else 0.0
|
|
160
|
+
return [
|
|
161
|
+
{"label": "cases", "value": str(summary.get("total_cases", 0))},
|
|
162
|
+
{"label": "passed", "value": str(summary.get("passed_cases", 0)), "tone": "ok"},
|
|
163
|
+
{
|
|
164
|
+
"label": "failed",
|
|
165
|
+
"value": str(summary.get("failed_cases", 0) + summary.get("error_cases", 0)),
|
|
166
|
+
"tone": "bad",
|
|
167
|
+
},
|
|
168
|
+
{"label": "avg score", "value": f"{avg_score:.2f}"},
|
|
169
|
+
{
|
|
170
|
+
"label": "avg latency",
|
|
171
|
+
"value": _format_duration(summary.get("avg_duration_seconds", 0.0)),
|
|
172
|
+
},
|
|
173
|
+
{"label": "total tokens", "value": _format_tokens(summary.get("total_token_usage"))},
|
|
174
|
+
]
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def _build_run_row(
|
|
178
|
+
report: dict[str, Any], run_id: str, run_label: str, now: float
|
|
179
|
+
) -> dict[str, Any]:
|
|
180
|
+
summary = report.get("summary", {})
|
|
181
|
+
return {
|
|
182
|
+
"id": run_id,
|
|
183
|
+
"name": report.get("eval_set_name") or report.get("eval_set_id", ""),
|
|
184
|
+
"run": run_label,
|
|
185
|
+
"rate": round(summary.get("pass_rate", 0.0) * 100, 1),
|
|
186
|
+
"status": _run_status(report),
|
|
187
|
+
"cases": summary.get("total_cases", 0),
|
|
188
|
+
"ago": _format_ago(report.get("timestamp", now), now),
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def _build_regression(
|
|
193
|
+
current: dict[str, Any],
|
|
194
|
+
current_label: str,
|
|
195
|
+
previous: dict[str, Any],
|
|
196
|
+
previous_label: str,
|
|
197
|
+
) -> dict[str, Any]:
|
|
198
|
+
cur_cases = {c["eval_id"]: c for c in current.get("results", [])}
|
|
199
|
+
prev_cases = {c["eval_id"]: c for c in previous.get("results", [])}
|
|
200
|
+
|
|
201
|
+
rows: list[dict[str, Any]] = []
|
|
202
|
+
newly_failing = 0
|
|
203
|
+
newly_passing = 0
|
|
204
|
+
for eval_id, cur_case in cur_cases.items():
|
|
205
|
+
prev_case = prev_cases.get(eval_id)
|
|
206
|
+
if prev_case is None:
|
|
207
|
+
continue
|
|
208
|
+
cur_score = _case_score(cur_case)
|
|
209
|
+
prev_score = _case_score(prev_case)
|
|
210
|
+
delta = cur_score - prev_score
|
|
211
|
+
|
|
212
|
+
if prev_case["passed"] and not cur_case["passed"]:
|
|
213
|
+
newly_failing += 1
|
|
214
|
+
flip = "pass → fail"
|
|
215
|
+
elif not prev_case["passed"] and cur_case["passed"]:
|
|
216
|
+
newly_passing += 1
|
|
217
|
+
flip = "fail → pass"
|
|
218
|
+
else:
|
|
219
|
+
flip = f"{'pass' if cur_case['passed'] else 'fail'} (same)"
|
|
220
|
+
|
|
221
|
+
if abs(delta) < REGRESSION_FLAT_EPSILON:
|
|
222
|
+
dir_, arrow = "flat", "—"
|
|
223
|
+
elif delta > 0:
|
|
224
|
+
dir_, arrow = "up", f"▲{delta:.2f}"
|
|
225
|
+
else:
|
|
226
|
+
dir_, arrow = "down", f"▼{abs(delta):.2f}"
|
|
227
|
+
|
|
228
|
+
rows.append(
|
|
229
|
+
{
|
|
230
|
+
"name": cur_case.get("name") or eval_id,
|
|
231
|
+
"delta": f"{prev_score:.2f} → {cur_score:.2f} {arrow}",
|
|
232
|
+
"dir": dir_,
|
|
233
|
+
"flip": flip,
|
|
234
|
+
"stay": cur_case["passed"] == prev_case["passed"],
|
|
235
|
+
}
|
|
236
|
+
)
|
|
237
|
+
|
|
238
|
+
cur_rate = current.get("summary", {}).get("pass_rate", 0.0) * 100
|
|
239
|
+
prev_rate = previous.get("summary", {}).get("pass_rate", 0.0) * 100
|
|
240
|
+
cur_scores = [_case_score(c) for c in current.get("results", [])]
|
|
241
|
+
prev_scores = [_case_score(c) for c in previous.get("results", [])]
|
|
242
|
+
avg_cur = sum(cur_scores) / len(cur_scores) if cur_scores else 0.0
|
|
243
|
+
avg_prev = sum(prev_scores) / len(prev_scores) if prev_scores else 0.0
|
|
244
|
+
score_drift = avg_cur - avg_prev
|
|
245
|
+
rate_drift = cur_rate - prev_rate
|
|
246
|
+
|
|
247
|
+
return {
|
|
248
|
+
"note": {
|
|
249
|
+
"current": f"run {current_label} ({cur_rate:.1f}%)",
|
|
250
|
+
"prev": f"run {previous_label} ({prev_rate:.1f}%)",
|
|
251
|
+
"suite": "same suite",
|
|
252
|
+
},
|
|
253
|
+
"summary": [
|
|
254
|
+
{
|
|
255
|
+
"label": "pass-rate drift",
|
|
256
|
+
"value": f"{rate_drift:+.1f}pp",
|
|
257
|
+
"tone": "bad" if rate_drift < 0 else "ok",
|
|
258
|
+
},
|
|
259
|
+
{
|
|
260
|
+
"label": "newly failing",
|
|
261
|
+
"value": str(newly_failing),
|
|
262
|
+
"tone": "bad" if newly_failing else "ok",
|
|
263
|
+
},
|
|
264
|
+
{"label": "newly passing", "value": str(newly_passing), "tone": "ok"},
|
|
265
|
+
{
|
|
266
|
+
"label": "avg score drift",
|
|
267
|
+
"value": f"{score_drift:+.2f}",
|
|
268
|
+
"tone": "warn" if score_drift < 0 else "ok",
|
|
269
|
+
},
|
|
270
|
+
],
|
|
271
|
+
"rows": rows,
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
class EvalReportService:
|
|
276
|
+
"""Reads `eval_reports/*.json` and adapts them into the run/detail shape
|
|
277
|
+
the playground's Evals inspector expects."""
|
|
278
|
+
|
|
279
|
+
def __init__(self, reports_dir: str | Path | None = None) -> None:
|
|
280
|
+
base = Path(reports_dir) if reports_dir is not None else Path(DEFAULT_REPORTS_DIR)
|
|
281
|
+
self.reports_dir = base if base.is_absolute() else Path.cwd() / base
|
|
282
|
+
|
|
283
|
+
def _load_reports(self) -> list[tuple[str, dict[str, Any]]]:
|
|
284
|
+
if not self.reports_dir.is_dir():
|
|
285
|
+
return []
|
|
286
|
+
reports = []
|
|
287
|
+
for path in sorted(self.reports_dir.glob("*.json")):
|
|
288
|
+
try:
|
|
289
|
+
data = json.loads(path.read_text(encoding="utf-8"))
|
|
290
|
+
except (json.JSONDecodeError, OSError):
|
|
291
|
+
continue
|
|
292
|
+
reports.append((path.stem, data))
|
|
293
|
+
reports.sort(key=lambda item: item[1].get("timestamp", 0.0))
|
|
294
|
+
return reports
|
|
295
|
+
|
|
296
|
+
def _run_labels(self) -> dict[str, tuple[int, dict[str, Any], str | None]]:
|
|
297
|
+
"""run_id -> (run_number, report, previous_run_id), run_number and
|
|
298
|
+
previous_run_id computed per eval_set_id in chronological order."""
|
|
299
|
+
counters: dict[str, int] = {}
|
|
300
|
+
previous_by_name: dict[str, str] = {}
|
|
301
|
+
labels: dict[str, tuple[int, dict[str, Any], str | None]] = {}
|
|
302
|
+
for run_id, report in self._load_reports():
|
|
303
|
+
name = report.get("eval_set_id", "")
|
|
304
|
+
counters[name] = counters.get(name, 0) + 1
|
|
305
|
+
labels[run_id] = (counters[name], report, previous_by_name.get(name))
|
|
306
|
+
previous_by_name[name] = run_id
|
|
307
|
+
return labels
|
|
308
|
+
|
|
309
|
+
def list_runs(self, now: float | None = None) -> list[dict[str, Any]]:
|
|
310
|
+
now = now if now is not None else time.time()
|
|
311
|
+
labels = self._run_labels()
|
|
312
|
+
rows = [
|
|
313
|
+
(report.get("timestamp", 0.0), _build_run_row(report, run_id, f"#{run_number}", now))
|
|
314
|
+
for run_id, (run_number, report, _prev) in labels.items()
|
|
315
|
+
]
|
|
316
|
+
rows.sort(key=lambda item: item[0], reverse=True)
|
|
317
|
+
return [row for _, row in rows]
|
|
318
|
+
|
|
319
|
+
def get_run_detail(self, run_id: str) -> dict[str, Any]:
|
|
320
|
+
labels = self._run_labels()
|
|
321
|
+
entry = labels.get(run_id)
|
|
322
|
+
if entry is None:
|
|
323
|
+
raise HTTPException(status_code=404, detail=f"Eval run '{run_id}' not found")
|
|
324
|
+
run_number, report, previous_run_id = entry
|
|
325
|
+
summary = report.get("summary", {})
|
|
326
|
+
|
|
327
|
+
regression = None
|
|
328
|
+
if previous_run_id is not None:
|
|
329
|
+
prev_entry = labels.get(previous_run_id)
|
|
330
|
+
if prev_entry is not None:
|
|
331
|
+
prev_number, prev_report, _ = prev_entry
|
|
332
|
+
regression = _build_regression(
|
|
333
|
+
report, f"#{run_number}", prev_report, f"#{prev_number}"
|
|
334
|
+
)
|
|
335
|
+
|
|
336
|
+
model = (report.get("config_used") or {}).get("model")
|
|
337
|
+
run_timestamp = time.gmtime(report.get("timestamp", 0.0))
|
|
338
|
+
sub_parts = [report.get("eval_set_id", "")]
|
|
339
|
+
if model:
|
|
340
|
+
sub_parts.append(str(model))
|
|
341
|
+
sub_parts.append(time.strftime("%Y-%m-%d %H:%M", run_timestamp))
|
|
342
|
+
sub_parts.append(_format_duration(summary.get("total_duration_seconds", 0.0)))
|
|
343
|
+
|
|
344
|
+
set_label = report.get("eval_set_name") or report.get("eval_set_id", "")
|
|
345
|
+
return {
|
|
346
|
+
"title": f"{set_label} · run #{run_number}",
|
|
347
|
+
"sub": " · ".join(sub_parts),
|
|
348
|
+
"rate": round(summary.get("pass_rate", 0.0) * 100, 1),
|
|
349
|
+
"status": _run_status(report),
|
|
350
|
+
"threshold": _derive_threshold(report),
|
|
351
|
+
"stats": _build_stats(report),
|
|
352
|
+
"cases": [_build_case_view(c) for c in report.get("results", [])],
|
|
353
|
+
"regression": regression,
|
|
354
|
+
}
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
"""Connection guard for the WebSocket endpoints.
|
|
2
|
+
|
|
3
|
+
Rate-limit and request-size middleware are ``BaseHTTPMiddleware`` and Starlette runs them
|
|
4
|
+
only for HTTP scopes, so WebSocket handshakes bypass them entirely. This module re-applies
|
|
5
|
+
two protections at the handshake, as a FastAPI dependency:
|
|
6
|
+
|
|
7
|
+
1. The same global rate limit as REST (shared backend + bucket), so opening a socket counts
|
|
8
|
+
like any other request.
|
|
9
|
+
2. A per-process cap on concurrent WebSocket connections (``websocket.max_connections`` in
|
|
10
|
+
10xgraph.json), and a per-user cap (``websocket.max_connections_per_user``) so one
|
|
11
|
+
account cannot hold every slot.
|
|
12
|
+
|
|
13
|
+
A socket lives much longer than its handshake, so two helpers apply for its whole lifetime:
|
|
14
|
+
:func:`ws_run_allowed` counts every graph run against the same rate-limit bucket, and
|
|
15
|
+
:func:`ws_identity_still_valid` re-verifies the credential so an expired or revoked token
|
|
16
|
+
cannot keep a socket working.
|
|
17
|
+
|
|
18
|
+
Rejections raise ``WebSocketException`` before ``accept()``, so the handshake fails with a
|
|
19
|
+
close code instead of leaving a half-open socket. The concurrency slot is released on
|
|
20
|
+
teardown of the (yield) dependency, i.e. when the handler returns or the client disconnects.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from collections.abc import AsyncIterator
|
|
24
|
+
from typing import Any
|
|
25
|
+
|
|
26
|
+
from fastapi import WebSocket, WebSocketException
|
|
27
|
+
from injectq.integrations import InjectAPI
|
|
28
|
+
|
|
29
|
+
from tenxgraph_api.src.app.core import logger
|
|
30
|
+
from tenxgraph_api.src.app.core.config.graph_config import GraphConfig
|
|
31
|
+
from tenxgraph_api.src.app.core.middleware.rate_limit import keying
|
|
32
|
+
from tenxgraph_api.src.app.core.middleware.rate_limit.keying import client_key_for
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
# RFC 6455 close code 1013 "Try Again Later" -- the right signal for shed-load rejections.
|
|
36
|
+
WS_TRY_AGAIN_LATER = 1013
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class _ConnectionRegistry:
|
|
40
|
+
"""Per-process counter of active WebSocket connections.
|
|
41
|
+
|
|
42
|
+
The event loop is single-threaded, so the check-and-increment in :meth:`try_acquire` is
|
|
43
|
+
atomic (no ``await`` between the test and the mutation) and needs no lock. This counter is
|
|
44
|
+
per process, matching the in-memory rate-limit backend's scope; for a multi-worker
|
|
45
|
+
deployment, set ``max_connections`` per worker accordingly.
|
|
46
|
+
"""
|
|
47
|
+
|
|
48
|
+
def __init__(self) -> None:
|
|
49
|
+
self._active = 0
|
|
50
|
+
self._per_user: dict[str, int] = {}
|
|
51
|
+
|
|
52
|
+
@property
|
|
53
|
+
def active(self) -> int:
|
|
54
|
+
return self._active
|
|
55
|
+
|
|
56
|
+
def try_acquire(self, max_connections: int | None) -> bool:
|
|
57
|
+
if max_connections is not None and self._active >= max_connections:
|
|
58
|
+
return False
|
|
59
|
+
self._active += 1
|
|
60
|
+
return True
|
|
61
|
+
|
|
62
|
+
def release(self) -> None:
|
|
63
|
+
if self._active > 0:
|
|
64
|
+
self._active -= 1
|
|
65
|
+
|
|
66
|
+
def try_acquire_user(self, user_key: str, max_per_user: int | None) -> bool:
|
|
67
|
+
held = self._per_user.get(user_key, 0)
|
|
68
|
+
if max_per_user is not None and held >= max_per_user:
|
|
69
|
+
return False
|
|
70
|
+
self._per_user[user_key] = held + 1
|
|
71
|
+
return True
|
|
72
|
+
|
|
73
|
+
def release_user(self, user_key: str) -> None:
|
|
74
|
+
held = self._per_user.get(user_key, 0)
|
|
75
|
+
if held <= 1:
|
|
76
|
+
self._per_user.pop(user_key, None)
|
|
77
|
+
else:
|
|
78
|
+
self._per_user[user_key] = held - 1
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
# Module-level singleton (per process).
|
|
82
|
+
_registry = _ConnectionRegistry()
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
async def realtime_connection_guard(
|
|
86
|
+
websocket: WebSocket,
|
|
87
|
+
config: GraphConfig = InjectAPI(GraphConfig),
|
|
88
|
+
) -> AsyncIterator[None]:
|
|
89
|
+
"""Gate a WebSocket handshake on the global rate limit and the concurrent-connection cap."""
|
|
90
|
+
# 1) Shared global rate limit (same backend/bucket as the REST middleware).
|
|
91
|
+
rl_config = config.rate_limit
|
|
92
|
+
backend = getattr(getattr(websocket, "app", None), "state", None)
|
|
93
|
+
backend = getattr(backend, "rate_limit_backend", None)
|
|
94
|
+
if rl_config is not None and backend is not None:
|
|
95
|
+
# Kept on the connection so every run on this socket counts too (ws_run_allowed).
|
|
96
|
+
websocket.state.ws_rate_limit = (rl_config, backend)
|
|
97
|
+
key = client_key_for(websocket, rl_config)
|
|
98
|
+
decision = await backend.check(key, limit=rl_config.requests, window=rl_config.window)
|
|
99
|
+
if not decision.allowed:
|
|
100
|
+
logger.warning("WebSocket rate limit exceeded for %s", key)
|
|
101
|
+
raise WebSocketException(code=WS_TRY_AGAIN_LATER, reason="Rate limit exceeded")
|
|
102
|
+
|
|
103
|
+
# 2) Per-process concurrent-connection cap, then the per-user share of it.
|
|
104
|
+
ws_config = config.websocket
|
|
105
|
+
max_conn = ws_config.max_connections
|
|
106
|
+
if not _registry.try_acquire(max_conn):
|
|
107
|
+
logger.warning(
|
|
108
|
+
"WebSocket connection limit reached (active=%d, max=%s)",
|
|
109
|
+
_registry.active,
|
|
110
|
+
max_conn,
|
|
111
|
+
)
|
|
112
|
+
raise WebSocketException(code=WS_TRY_AGAIN_LATER, reason="Too many connections")
|
|
113
|
+
|
|
114
|
+
user_key = keying.verified_user_id(websocket)
|
|
115
|
+
if user_key and not _registry.try_acquire_user(user_key, ws_config.max_connections_per_user):
|
|
116
|
+
_registry.release()
|
|
117
|
+
logger.warning(
|
|
118
|
+
"WebSocket per-user connection limit reached for %s (max=%s)",
|
|
119
|
+
user_key,
|
|
120
|
+
ws_config.max_connections_per_user,
|
|
121
|
+
)
|
|
122
|
+
raise WebSocketException(
|
|
123
|
+
code=WS_TRY_AGAIN_LATER, reason="Too many connections for this user"
|
|
124
|
+
)
|
|
125
|
+
|
|
126
|
+
try:
|
|
127
|
+
yield
|
|
128
|
+
finally:
|
|
129
|
+
_registry.release()
|
|
130
|
+
if user_key:
|
|
131
|
+
_registry.release_user(user_key)
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
async def ws_run_allowed(websocket: WebSocket) -> int | None:
|
|
135
|
+
"""Count one graph run against the rate limit.
|
|
136
|
+
|
|
137
|
+
Returns ``None`` when the run may start, or the seconds to wait when the caller is over
|
|
138
|
+
the limit. Runs share the handshake's bucket, which is also the REST bucket.
|
|
139
|
+
"""
|
|
140
|
+
limiter = getattr(websocket.state, "ws_rate_limit", None)
|
|
141
|
+
if not isinstance(limiter, tuple): # the guard stored none: rate limiting is off
|
|
142
|
+
return None
|
|
143
|
+
rl_config, backend = limiter
|
|
144
|
+
key = client_key_for(websocket, rl_config)
|
|
145
|
+
decision = await backend.check(key, limit=rl_config.requests, window=rl_config.window)
|
|
146
|
+
if decision.allowed:
|
|
147
|
+
return None
|
|
148
|
+
logger.warning("WebSocket run rate limit exceeded for %s", key)
|
|
149
|
+
return decision.reset_after
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def ws_identity_still_valid(websocket: Any, user: dict[str, Any]) -> bool:
|
|
153
|
+
"""Whether the socket's credential still verifies as the user it was opened by.
|
|
154
|
+
|
|
155
|
+
Checked before each run: without it, a socket keeps working after its token expires or
|
|
156
|
+
is revoked. Always true when auth is not configured (``user`` has no ``user_id``).
|
|
157
|
+
"""
|
|
158
|
+
user_id = user.get("user_id") if isinstance(user, dict) else None
|
|
159
|
+
if not user_id:
|
|
160
|
+
return True
|
|
161
|
+
return keying.verified_user_id(websocket) == str(user_id)
|