10xgraph-api 0.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- 10xgraph_api-0.7.0.dist-info/METADATA +618 -0
- 10xgraph_api-0.7.0.dist-info/RECORD +203 -0
- 10xgraph_api-0.7.0.dist-info/WHEEL +5 -0
- 10xgraph_api-0.7.0.dist-info/entry_points.txt +3 -0
- 10xgraph_api-0.7.0.dist-info/licenses/LICENSE +21 -0
- 10xgraph_api-0.7.0.dist-info/top_level.txt +2 -0
- agentflow_cli/__init__.py +69 -0
- tenxgraph_api/__init__.py +23 -0
- tenxgraph_api/cli/__init__.py +6 -0
- tenxgraph_api/cli/capabilities.py +127 -0
- tenxgraph_api/cli/commands/__init__.py +49 -0
- tenxgraph_api/cli/commands/api.py +262 -0
- tenxgraph_api/cli/commands/audit.py +213 -0
- tenxgraph_api/cli/commands/build.py +305 -0
- tenxgraph_api/cli/commands/config.py +71 -0
- tenxgraph_api/cli/commands/demo.py +93 -0
- tenxgraph_api/cli/commands/eval.py +909 -0
- tenxgraph_api/cli/commands/init.py +531 -0
- tenxgraph_api/cli/commands/skills.py +532 -0
- tenxgraph_api/cli/commands/test.py +113 -0
- tenxgraph_api/cli/commands/version.py +47 -0
- tenxgraph_api/cli/config_editor/__init__.py +15 -0
- tenxgraph_api/cli/config_editor/schema.py +566 -0
- tenxgraph_api/cli/config_editor/server.py +240 -0
- tenxgraph_api/cli/config_editor/static/app.css +2 -0
- tenxgraph_api/cli/config_editor/static/app.js +3 -0
- tenxgraph_api/cli/config_editor/static/favicon.svg +11 -0
- tenxgraph_api/cli/config_editor/static/index.html +20 -0
- tenxgraph_api/cli/config_editor/store.py +120 -0
- tenxgraph_api/cli/config_editor/validation.py +377 -0
- tenxgraph_api/cli/constants.py +112 -0
- tenxgraph_api/cli/context.py +46 -0
- tenxgraph_api/cli/core/__init__.py +0 -0
- tenxgraph_api/cli/core/animations.py +550 -0
- tenxgraph_api/cli/core/config.py +279 -0
- tenxgraph_api/cli/core/output.py +625 -0
- tenxgraph_api/cli/core/prompts.py +220 -0
- tenxgraph_api/cli/core/screen.py +287 -0
- tenxgraph_api/cli/core/steps.py +621 -0
- tenxgraph_api/cli/core/theme.py +242 -0
- tenxgraph_api/cli/core/validation.py +251 -0
- tenxgraph_api/cli/exceptions.py +129 -0
- tenxgraph_api/cli/logger.py +84 -0
- tenxgraph_api/cli/main.py +936 -0
- tenxgraph_api/cli/templates/__init__.py +0 -0
- tenxgraph_api/cli/templates/defaults.py +844 -0
- tenxgraph_api/cli/templates/dev/.env.example +1 -0
- tenxgraph_api/cli/templates/dev/10xgraph.json +16 -0
- tenxgraph_api/cli/templates/dev/graph/__init__.py +0 -0
- tenxgraph_api/cli/templates/dev/graph/agent.py +63 -0
- tenxgraph_api/cli/templates/prod/.env.example +75 -0
- tenxgraph_api/cli/templates/prod/.pre-commit-config.yaml +50 -0
- tenxgraph_api/cli/templates/prod/.python-version +1 -0
- tenxgraph_api/cli/templates/prod/10xgraph.json +42 -0
- tenxgraph_api/cli/templates/prod/auth/__init__.py +0 -0
- tenxgraph_api/cli/templates/prod/auth/agent_auth.py +46 -0
- tenxgraph_api/cli/templates/prod/evals/__init__.py +0 -0
- tenxgraph_api/cli/templates/prod/evals/user_simulator_eval.py +76 -0
- tenxgraph_api/cli/templates/prod/evals/weather_agents_eval.py +53 -0
- tenxgraph_api/cli/templates/prod/graph/__init__.py +0 -0
- tenxgraph_api/cli/templates/prod/graph/agent.py +50 -0
- tenxgraph_api/cli/templates/prod/graph/state.py +5 -0
- tenxgraph_api/cli/templates/prod/graph/thread_name_generator.py +7 -0
- tenxgraph_api/cli/templates/prod/graph/tools/__init__.py +0 -0
- tenxgraph_api/cli/templates/prod/graph/tools/weather_tool.py +43 -0
- tenxgraph_api/cli/templates/prod/graph/validators/__init__.py +0 -0
- tenxgraph_api/cli/templates/prod/graph/validators/lifecyle.py +107 -0
- tenxgraph_api/cli/templates/prod/graph/validators/manager.py +9 -0
- tenxgraph_api/cli/templates/prod/graph/validators/validators.py +21 -0
- tenxgraph_api/cli/templates/prod/pyproject.toml +79 -0
- tenxgraph_api/cli/templates/prod/tests/__init__.py +0 -0
- tenxgraph_api/cli/templates/prod/tests/conftest.py +6 -0
- tenxgraph_api/cli/templates/prod/tests/test_agent_eval.py +78 -0
- tenxgraph_api/cli/templates/prod/tests/test_catalog_tools.py +47 -0
- tenxgraph_api/cli/templates/prod/tests/test_graph_nodes.py +57 -0
- tenxgraph_api/cli/templates/skills/10xgraph/SKILL.md +92 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/agents-and-tools.md +389 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/api-client.md +61 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/api-configuration.md +75 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/api-settings-and-middleware.md +89 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/architecture.md +91 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/auth-and-authorization.md +149 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/callbacks-and-command.md +283 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/checkpointing-and-threads.md +63 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/cli-commands.md +73 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/client-auth-and-errors.md +94 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/client-messages-invoke-stream.md +115 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/client-threads-memory-files.md +80 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/context-id-background.md +80 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/dependency-injection.md +58 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/evaluation.md +500 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/id-and-thread-name-generators.md +69 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/media-and-files.md +66 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/memory-and-store.md +70 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/prebuilt-agents-and-tools.md +394 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/production-runtime.md +56 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/providers-and-adapters.md +89 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/publishers-and-runtime-protocols.md +84 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/rate-limiting.md +194 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/realtime.md +172 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/remote-tools.md +55 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/rest-api-and-errors.md +97 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/security-and-validators.md +86 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/state-and-messages.md +68 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/state-graph.md +232 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/stream-emitter.md +144 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/streaming.md +53 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/testing-and-evaluation.md +64 -0
- tenxgraph_api/cli/templates/skills/10xgraph/references/unit-testing.md +202 -0
- tenxgraph_api/cli/templates/skills/copilot/10xgraph.instructions.md +108 -0
- tenxgraph_api/py.typed +0 -0
- tenxgraph_api/src/__init__.py +0 -0
- tenxgraph_api/src/app/__init__.py +13 -0
- tenxgraph_api/src/app/core/__init__.py +15 -0
- tenxgraph_api/src/app/core/auth/__init__.py +0 -0
- tenxgraph_api/src/app/core/auth/auth_backend.py +38 -0
- tenxgraph_api/src/app/core/auth/authorization.py +379 -0
- tenxgraph_api/src/app/core/auth/base_auth.py +27 -0
- tenxgraph_api/src/app/core/auth/jwt_auth.py +122 -0
- tenxgraph_api/src/app/core/auth/ownership_resolver.py +170 -0
- tenxgraph_api/src/app/core/auth/permissions.py +354 -0
- tenxgraph_api/src/app/core/auth/request_config.py +130 -0
- tenxgraph_api/src/app/core/auth/route_guard.py +105 -0
- tenxgraph_api/src/app/core/config/__init__.py +0 -0
- tenxgraph_api/src/app/core/config/graph_config.py +532 -0
- tenxgraph_api/src/app/core/config/media_settings.py +72 -0
- tenxgraph_api/src/app/core/config/sentry_config.py +60 -0
- tenxgraph_api/src/app/core/config/settings.py +202 -0
- tenxgraph_api/src/app/core/config/setup_logs.py +65 -0
- tenxgraph_api/src/app/core/config/setup_middleware.py +483 -0
- tenxgraph_api/src/app/core/config/worker_middleware.py +99 -0
- tenxgraph_api/src/app/core/exceptions/__init__.py +17 -0
- tenxgraph_api/src/app/core/exceptions/general_exception.py +58 -0
- tenxgraph_api/src/app/core/exceptions/handle_errors.py +363 -0
- tenxgraph_api/src/app/core/exceptions/resources_exceptions.py +72 -0
- tenxgraph_api/src/app/core/exceptions/user_exception.py +58 -0
- tenxgraph_api/src/app/core/middleware/__init__.py +25 -0
- tenxgraph_api/src/app/core/middleware/rate_limit/__init__.py +15 -0
- tenxgraph_api/src/app/core/middleware/rate_limit/base.py +27 -0
- tenxgraph_api/src/app/core/middleware/rate_limit/factory.py +98 -0
- tenxgraph_api/src/app/core/middleware/rate_limit/keying.py +133 -0
- tenxgraph_api/src/app/core/middleware/rate_limit/memory.py +98 -0
- tenxgraph_api/src/app/core/middleware/rate_limit/middleware.py +90 -0
- tenxgraph_api/src/app/core/middleware/rate_limit/redis.py +115 -0
- tenxgraph_api/src/app/core/middleware/request_limits.py +154 -0
- tenxgraph_api/src/app/core/middleware/security_headers.py +239 -0
- tenxgraph_api/src/app/core/utils/__init__.py +14 -0
- tenxgraph_api/src/app/core/utils/log_sanitizer.py +216 -0
- tenxgraph_api/src/app/loader.py +516 -0
- tenxgraph_api/src/app/main.py +180 -0
- tenxgraph_api/src/app/routers/__init__.py +4 -0
- tenxgraph_api/src/app/routers/ag_ui/__init__.py +10 -0
- tenxgraph_api/src/app/routers/ag_ui/converter.py +195 -0
- tenxgraph_api/src/app/routers/ag_ui/event_mapper.py +353 -0
- tenxgraph_api/src/app/routers/ag_ui/router.py +90 -0
- tenxgraph_api/src/app/routers/ag_ui/service.py +193 -0
- tenxgraph_api/src/app/routers/checkpointer/__init__.py +1 -0
- tenxgraph_api/src/app/routers/checkpointer/router.py +482 -0
- tenxgraph_api/src/app/routers/checkpointer/schemas/__init__.py +1 -0
- tenxgraph_api/src/app/routers/checkpointer/schemas/checkpointer_schemas.py +121 -0
- tenxgraph_api/src/app/routers/checkpointer/services/__init__.py +1 -0
- tenxgraph_api/src/app/routers/checkpointer/services/checkpointer_service.py +300 -0
- tenxgraph_api/src/app/routers/evals/__init__.py +6 -0
- tenxgraph_api/src/app/routers/evals/router.py +38 -0
- tenxgraph_api/src/app/routers/evals/services/__init__.py +1 -0
- tenxgraph_api/src/app/routers/evals/services/eval_report_service.py +354 -0
- tenxgraph_api/src/app/routers/graph/__init__.py +6 -0
- tenxgraph_api/src/app/routers/graph/realtime_guard.py +161 -0
- tenxgraph_api/src/app/routers/graph/router.py +821 -0
- tenxgraph_api/src/app/routers/graph/schemas/__init__.py +1 -0
- tenxgraph_api/src/app/routers/graph/schemas/graph_schemas.py +386 -0
- tenxgraph_api/src/app/routers/graph/services/__init__.py +1 -0
- tenxgraph_api/src/app/routers/graph/services/graph_service.py +1219 -0
- tenxgraph_api/src/app/routers/graph/services/multimodal_preprocessor.py +164 -0
- tenxgraph_api/src/app/routers/media/__init__.py +506 -0
- tenxgraph_api/src/app/routers/media/router.py +242 -0
- tenxgraph_api/src/app/routers/media/schemas.py +39 -0
- tenxgraph_api/src/app/routers/ping/__init__.py +4 -0
- tenxgraph_api/src/app/routers/ping/router.py +28 -0
- tenxgraph_api/src/app/routers/setup_router.py +60 -0
- tenxgraph_api/src/app/routers/store/__init__.py +4 -0
- tenxgraph_api/src/app/routers/store/router.py +200 -0
- tenxgraph_api/src/app/routers/store/schemas/__init__.py +28 -0
- tenxgraph_api/src/app/routers/store/schemas/store_schemas.py +200 -0
- tenxgraph_api/src/app/routers/store/services/__init__.py +4 -0
- tenxgraph_api/src/app/routers/store/services/store_service.py +203 -0
- tenxgraph_api/src/app/tasks/__init__.py +0 -0
- tenxgraph_api/src/app/tasks/user_tasks.py +38 -0
- tenxgraph_api/src/app/utils/__init__.py +12 -0
- tenxgraph_api/src/app/utils/callable_helper.py +26 -0
- tenxgraph_api/src/app/utils/media/__init__.py +15 -0
- tenxgraph_api/src/app/utils/media/extractor.py +172 -0
- tenxgraph_api/src/app/utils/media/pipeline.py +83 -0
- tenxgraph_api/src/app/utils/parse_output.py +17 -0
- tenxgraph_api/src/app/utils/response_helper.py +129 -0
- tenxgraph_api/src/app/utils/schemas/__init__.py +11 -0
- tenxgraph_api/src/app/utils/schemas/output_schemas.py +28 -0
- tenxgraph_api/src/app/utils/schemas/user_schemas.py +13 -0
- tenxgraph_api/src/app/utils/snowflake_id_generator.py +90 -0
- tenxgraph_api/src/app/utils/swagger_helper.py +160 -0
- tenxgraph_api/src/app/utils/telemetry_store.py +123 -0
- tenxgraph_api/src/app/utils/thread_name_generator.py +304 -0
- tenxgraph_api/src/app/worker.py +91 -0
|
@@ -0,0 +1,500 @@
|
|
|
1
|
+
# Evaluation
|
|
2
|
+
|
|
3
|
+
Use this when defining eval cases, configuring criteria, running evaluations programmatically, generating reports, or using user simulation. All evaluation types live in `tenxgraph.qa.evaluation`.
|
|
4
|
+
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
## EvalSet and EvalCase
|
|
8
|
+
|
|
9
|
+
Define test cases with `EvalSetBuilder` (fluent) or `EvalCase` directly.
|
|
10
|
+
|
|
11
|
+
```python
|
|
12
|
+
from tenxgraph.qa.evaluation import EvalSetBuilder, EvalSet, EvalCase, ToolCall
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
**EvalSetBuilder — fluent API:**
|
|
16
|
+
|
|
17
|
+
```python
|
|
18
|
+
eval_set = (
|
|
19
|
+
EvalSetBuilder("agent-regression")
|
|
20
|
+
.add_case(
|
|
21
|
+
query="Weather in London?",
|
|
22
|
+
expected="sunny",
|
|
23
|
+
case_id="london", # optional; auto-generated if omitted
|
|
24
|
+
expected_tools=["get_weather"], # list of tool names or ToolCall objects
|
|
25
|
+
expected_node_order=["MAIN", "TOOL", "MAIN"],
|
|
26
|
+
)
|
|
27
|
+
.add_tool_test(
|
|
28
|
+
query="Weather in Tokyo?",
|
|
29
|
+
tool_name="get_weather",
|
|
30
|
+
tool_args={"location": "Tokyo"},
|
|
31
|
+
expected_response="Tokyo",
|
|
32
|
+
)
|
|
33
|
+
.add_multi_turn(
|
|
34
|
+
conversation=[("Hello", "Hi!"), ("Weather?", "Sunny.")],
|
|
35
|
+
expected_tools=["get_weather"],
|
|
36
|
+
)
|
|
37
|
+
.build()
|
|
38
|
+
)
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
Quick builders:
|
|
42
|
+
|
|
43
|
+
```python
|
|
44
|
+
EvalSetBuilder.quick(("Hello", "Hi!"), ("Bye", "Goodbye!"))
|
|
45
|
+
EvalSetBuilder.from_conversations([{"user": "Hello", "assistant": "Hi!"}])
|
|
46
|
+
EvalSetBuilder.from_file("evals/cases.json") # returns builder for modification
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
Save / load:
|
|
50
|
+
|
|
51
|
+
```python
|
|
52
|
+
eval_set.to_file("evals/cases.json")
|
|
53
|
+
eval_set = EvalSet.from_file("evals/cases.json")
|
|
54
|
+
builder = EvalSetBuilder.from_file("evals/cases.json") # load + modify
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
**ToolCall:**
|
|
58
|
+
|
|
59
|
+
```python
|
|
60
|
+
ToolCall(name="get_weather") # name only
|
|
61
|
+
ToolCall(name="get_weather", args={"location": "London"}) # with args (checked when check_args=True)
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
**EvalCase directly:**
|
|
65
|
+
|
|
66
|
+
```python
|
|
67
|
+
case = EvalCase.single_turn(
|
|
68
|
+
eval_id="london",
|
|
69
|
+
user_query="Weather in London?",
|
|
70
|
+
expected_response="sunny",
|
|
71
|
+
expected_tools=[ToolCall(name="get_weather", args={"location": "London"})],
|
|
72
|
+
expected_node_order=["MAIN", "TOOL", "MAIN"],
|
|
73
|
+
)
|
|
74
|
+
case = EvalCase.multi_turn(
|
|
75
|
+
eval_id="multi",
|
|
76
|
+
conversation=[("Hello", "Hi!"), ("Weather?", "Sunny.")],
|
|
77
|
+
expected_tools=[ToolCall(name="get_weather")],
|
|
78
|
+
)
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
---
|
|
82
|
+
|
|
83
|
+
## EvalConfig and CriterionConfig
|
|
84
|
+
|
|
85
|
+
```python
|
|
86
|
+
from tenxgraph.qa.evaluation import EvalConfig, CriterionConfig, MatchType, Rubric
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
**EvalConfig** wraps a dict of criterion key → `CriterionConfig`:
|
|
90
|
+
|
|
91
|
+
```python
|
|
92
|
+
config = EvalConfig(
|
|
93
|
+
criteria={
|
|
94
|
+
"tool_trajectory_avg_score": CriterionConfig.trajectory(threshold=1.0, match_type=MatchType.EXACT, check_args=False),
|
|
95
|
+
"rouge_match": CriterionConfig.rouge_match(threshold=0.5),
|
|
96
|
+
"response_match_score": CriterionConfig.response_match(threshold=0.8),
|
|
97
|
+
},
|
|
98
|
+
parallel=False,
|
|
99
|
+
max_concurrency=4,
|
|
100
|
+
timeout=300.0,
|
|
101
|
+
)
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
**MatchType** (for trajectory and node-order criteria):
|
|
105
|
+
|
|
106
|
+
- `EXACT` — same tools/nodes, same order, same count; extras fail
|
|
107
|
+
- `IN_ORDER` — expected appear in order; extras allowed
|
|
108
|
+
- `ANY_ORDER` — all expected present; order and extras don't matter
|
|
109
|
+
|
|
110
|
+
**Built-in class-method presets on EvalConfig:**
|
|
111
|
+
|
|
112
|
+
```python
|
|
113
|
+
EvalConfig.default() # EXACT trajectory + ROUGE response (threshold 0.8)
|
|
114
|
+
EvalConfig.strict() # EXACT + args + high-threshold LLM judge (threshold 0.9, 5 samples)
|
|
115
|
+
EvalConfig.relaxed() # IN_ORDER trajectory + lower response threshold (0.6)
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
**CriterionConfig factory methods:**
|
|
119
|
+
|
|
120
|
+
| Method | LLM? | Key to use in criteria dict |
|
|
121
|
+
|---|---|---|
|
|
122
|
+
| `CriterionConfig.tool_name_match(threshold)` | No | `tool_name_match_score` |
|
|
123
|
+
| `CriterionConfig.trajectory(threshold, match_type, check_args)` | No | `tool_trajectory_avg_score` |
|
|
124
|
+
| `CriterionConfig.node_order(threshold, match_type)` | No | `node_order_match_score` |
|
|
125
|
+
| `CriterionConfig.rouge_match(threshold)` | No | `rouge_match` |
|
|
126
|
+
| `CriterionConfig.contains_keywords(keywords, threshold)` | No | `contains_keywords` |
|
|
127
|
+
| `CriterionConfig.response_match(threshold, judge_model, num_samples)` | Yes | `response_match_score` |
|
|
128
|
+
| `CriterionConfig.llm_judge(threshold, judge_model, num_samples)` | Yes | `llm_judge` |
|
|
129
|
+
| `CriterionConfig.factual_accuracy(threshold, judge_model, num_samples)` | Yes | `factual_accuracy_v1` |
|
|
130
|
+
| `CriterionConfig.hallucination(threshold, judge_model, num_samples)` | Yes | `hallucinations_v1` |
|
|
131
|
+
| `CriterionConfig.safety(threshold, judge_model, num_samples)` | Yes | `safety_v1` |
|
|
132
|
+
| `CriterionConfig.rubric_based(rubrics, threshold, judge_model)` | Yes | any key |
|
|
133
|
+
|
|
134
|
+
Default judge model: `"gemini-2.5-flash"`. Override per criterion with `judge_model="gpt-4o"`.
|
|
135
|
+
|
|
136
|
+
`num_samples` — number of LLM judge calls; final score is the average (reduces non-determinism noise).
|
|
137
|
+
|
|
138
|
+
**Custom rubrics:**
|
|
139
|
+
|
|
140
|
+
```python
|
|
141
|
+
CriterionConfig.rubric_based(
|
|
142
|
+
rubrics=[
|
|
143
|
+
Rubric(rubric_id="concise", content="Response must be under 100 words.", weight=1.0),
|
|
144
|
+
Rubric(rubric_id="empathetic", content="Acknowledge the user's issue first.", weight=0.5),
|
|
145
|
+
],
|
|
146
|
+
threshold=0.8,
|
|
147
|
+
)
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
Score = weighted average of rubric scores.
|
|
151
|
+
|
|
152
|
+
**Disable / enable a criterion at runtime:**
|
|
153
|
+
|
|
154
|
+
```python
|
|
155
|
+
config.disable_criterion("rouge_match")
|
|
156
|
+
config.enable_criterion("rouge_match", CriterionConfig.rouge_match(0.4))
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
**Save / load:**
|
|
160
|
+
|
|
161
|
+
```python
|
|
162
|
+
config.to_file("eval_config.json")
|
|
163
|
+
config = EvalConfig.from_file("eval_config.json")
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
---
|
|
167
|
+
|
|
168
|
+
## EvalPresets
|
|
169
|
+
|
|
170
|
+
Ready-made `EvalConfig` objects for common scenarios.
|
|
171
|
+
|
|
172
|
+
```python
|
|
173
|
+
from tenxgraph.qa.evaluation import EvalPresets
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
| Preset | LLM? | Criteria included |
|
|
177
|
+
|---|---|---|
|
|
178
|
+
| `EvalPresets.quick_check()` | No | `rouge_match` (0.5) |
|
|
179
|
+
| `EvalPresets.tool_usage(threshold, strict, check_args)` | No | `tool_name_match_score` + `tool_trajectory_avg_score` |
|
|
180
|
+
| `EvalPresets.response_quality(threshold, use_llm_judge, judge_model)` | Yes | `response_match_score` + optional `llm_judge` |
|
|
181
|
+
| `EvalPresets.conversation_flow(threshold, judge_model)` | Yes | `response_match_score` + `tool_trajectory_avg_score` (IN_ORDER) |
|
|
182
|
+
| `EvalPresets.safety_check(threshold, judge_model)` | Yes | `hallucinations_v1` + `safety_v1` |
|
|
183
|
+
| `EvalPresets.comprehensive(threshold, use_llm_judge, judge_model)` | Yes | all no-LLM + all LLM criteria |
|
|
184
|
+
| `EvalPresets.custom(response_threshold, tool_threshold, llm_judge_threshold, ...)` | Both | only criteria whose threshold is not None |
|
|
185
|
+
|
|
186
|
+
Combine presets (later keys override earlier on conflict):
|
|
187
|
+
|
|
188
|
+
```python
|
|
189
|
+
config = EvalPresets.combine(
|
|
190
|
+
EvalPresets.tool_usage(threshold=1.0),
|
|
191
|
+
EvalPresets.safety_check(threshold=0.8),
|
|
192
|
+
)
|
|
193
|
+
```
|
|
194
|
+
|
|
195
|
+
---
|
|
196
|
+
|
|
197
|
+
## AgentEvaluator and QuickEval
|
|
198
|
+
|
|
199
|
+
```python
|
|
200
|
+
from tenxgraph.qa.evaluation import AgentEvaluator, QuickEval
|
|
201
|
+
from tenxgraph.qa.evaluation.collectors.trajectory_collector import TrajectoryCollector
|
|
202
|
+
```
|
|
203
|
+
|
|
204
|
+
**TrajectoryCollector** — must be created before evaluation and passed to `AgentEvaluator`. Captures tool calls, node visits, and LLM outputs through the graph callback system.
|
|
205
|
+
|
|
206
|
+
```python
|
|
207
|
+
collector = TrajectoryCollector(capture_all_events=True)
|
|
208
|
+
```
|
|
209
|
+
|
|
210
|
+
**AgentEvaluator** — runs the full eval set:
|
|
211
|
+
|
|
212
|
+
```python
|
|
213
|
+
evaluator = AgentEvaluator(compiled_graph, collector, config=config)
|
|
214
|
+
report = await evaluator.evaluate(eval_set) # EvalSet object
|
|
215
|
+
report = await evaluator.evaluate("evals/cases.json") # path to JSON
|
|
216
|
+
report = await evaluator.evaluate_case(single_case) # one EvalCase
|
|
217
|
+
```
|
|
218
|
+
|
|
219
|
+
**QuickEval** — one-liners for common patterns:
|
|
220
|
+
|
|
221
|
+
```python
|
|
222
|
+
# Single check
|
|
223
|
+
report = await QuickEval.check(
|
|
224
|
+
graph=app, collector=collector,
|
|
225
|
+
query="Weather in London?",
|
|
226
|
+
expected_response_contains="sunny",
|
|
227
|
+
expected_tools=["get_weather"],
|
|
228
|
+
threshold=0.7,
|
|
229
|
+
)
|
|
230
|
+
|
|
231
|
+
# Preset-based batch
|
|
232
|
+
report = await QuickEval.preset(
|
|
233
|
+
graph=app, collector=collector,
|
|
234
|
+
preset=EvalPresets.tool_usage(),
|
|
235
|
+
eval_set=eval_set,
|
|
236
|
+
)
|
|
237
|
+
|
|
238
|
+
# Query-response pairs
|
|
239
|
+
report = await QuickEval.batch(
|
|
240
|
+
app, collector,
|
|
241
|
+
[("Hello", "Hi!"), ("Bye", "Goodbye!")],
|
|
242
|
+
threshold=0.7,
|
|
243
|
+
)
|
|
244
|
+
|
|
245
|
+
# Tool-specific
|
|
246
|
+
report = await QuickEval.tool_usage(
|
|
247
|
+
app, collector,
|
|
248
|
+
test_cases=[("Weather in London?", "sunny", ["get_weather"])],
|
|
249
|
+
strict=True,
|
|
250
|
+
)
|
|
251
|
+
|
|
252
|
+
# Multi-turn conversation
|
|
253
|
+
report = await QuickEval.conversation_flow(
|
|
254
|
+
app, collector,
|
|
255
|
+
conversation=[("Hello", "Hi!"), ("Weather?", "Sunny.")],
|
|
256
|
+
threshold=0.8,
|
|
257
|
+
)
|
|
258
|
+
|
|
259
|
+
# Builder-based
|
|
260
|
+
report = await QuickEval.from_builder(app, collector, builder=my_builder, config=config)
|
|
261
|
+
|
|
262
|
+
# Synchronous wrapper (for non-async contexts)
|
|
263
|
+
report = QuickEval.run_sync(app, collector, eval_set=eval_set, config=config)
|
|
264
|
+
```
|
|
265
|
+
|
|
266
|
+
---
|
|
267
|
+
|
|
268
|
+
## EvalReport
|
|
269
|
+
|
|
270
|
+
```python
|
|
271
|
+
report.summary.total_cases # int
|
|
272
|
+
report.summary.passed_cases # int
|
|
273
|
+
report.summary.failed_cases # int
|
|
274
|
+
report.summary.pass_rate # float 0.0–1.0
|
|
275
|
+
report.summary.criteria_scores # dict[str, float] — average per criterion
|
|
276
|
+
|
|
277
|
+
for result in report.results:
|
|
278
|
+
result.case_id
|
|
279
|
+
result.passed # bool
|
|
280
|
+
result.criteria_results # dict[str, CriterionResult]
|
|
281
|
+
result.criteria_results["rouge_match"].score # float
|
|
282
|
+
result.criteria_results["rouge_match"].passed # bool
|
|
283
|
+
```
|
|
284
|
+
|
|
285
|
+
Print to console:
|
|
286
|
+
|
|
287
|
+
```python
|
|
288
|
+
from tenxgraph.qa.evaluation import print_report
|
|
289
|
+
print_report(report)
|
|
290
|
+
```
|
|
291
|
+
|
|
292
|
+
---
|
|
293
|
+
|
|
294
|
+
## Reporters
|
|
295
|
+
|
|
296
|
+
```python
|
|
297
|
+
from tenxgraph.qa.evaluation import ReporterManager, ReporterConfig
|
|
298
|
+
|
|
299
|
+
manager = ReporterManager(ReporterConfig(
|
|
300
|
+
output_dir="eval_reports",
|
|
301
|
+
html=True,
|
|
302
|
+
json_report=True,
|
|
303
|
+
junit_xml=False,
|
|
304
|
+
console=True,
|
|
305
|
+
timestamp_files=True,
|
|
306
|
+
include_trajectory=True,
|
|
307
|
+
include_tool_call_details=True,
|
|
308
|
+
))
|
|
309
|
+
output = manager.run_all(report)
|
|
310
|
+
# output.html_path, output.json_path, output.has_errors, output.errors
|
|
311
|
+
```
|
|
312
|
+
|
|
313
|
+
Available reporters: `ConsoleReporter`, `JSONReporter`, `HTMLReporter`, `JUnitXMLReporter`.
|
|
314
|
+
|
|
315
|
+
HTML report — visual dashboard with summary cards, criterion bars, per-case details.
|
|
316
|
+
JSON report — full machine-readable results including trajectory data.
|
|
317
|
+
JUnit XML — for CI test summary tabs (GitHub Actions, Jenkins).
|
|
318
|
+
|
|
319
|
+
---
|
|
320
|
+
|
|
321
|
+
## 10xgraph eval CLI
|
|
322
|
+
|
|
323
|
+
Auto-discovers `*_eval.py` / `eval_*.py` in `evals/` (or configured directory). Each file must expose `get_eval_set()`, optionally `get_eval_config()` or `EVAL_CONFIG`, or a full `run()` function.
|
|
324
|
+
|
|
325
|
+
```bash
|
|
326
|
+
10xgraph eval # scan evals/, write to eval_reports/
|
|
327
|
+
10xgraph eval evals/my_eval.py # single file
|
|
328
|
+
10xgraph eval evals/regression/ # subdirectory
|
|
329
|
+
10xgraph eval --open # open HTML report in browser after run
|
|
330
|
+
10xgraph eval --threshold 0.8 # exit non-zero if pass rate < 0.8
|
|
331
|
+
10xgraph eval --output ci/reports # custom output directory
|
|
332
|
+
10xgraph eval --no-report # console only, no files
|
|
333
|
+
```
|
|
334
|
+
|
|
335
|
+
Eval file entry points (in order of priority):
|
|
336
|
+
|
|
337
|
+
1. `run()` — sync or async function returning `EvalReport`; full control
|
|
338
|
+
2. `get_eval_set()` + optional `get_eval_config()` or `EVAL_CONFIG` — CLI loads agent from `10xgraph.json` and runs with default or provided config
|
|
339
|
+
3. Files without any entry point are skipped with a warning
|
|
340
|
+
|
|
341
|
+
`10xgraph.json` configuration:
|
|
342
|
+
|
|
343
|
+
```json
|
|
344
|
+
{
|
|
345
|
+
"evaluation": {
|
|
346
|
+
"directory": "evals",
|
|
347
|
+
"output_dir": "eval_reports",
|
|
348
|
+
"threshold": 0.8,
|
|
349
|
+
"timestamp_files": true
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
```
|
|
353
|
+
|
|
354
|
+
Exit code: 0 only when pass rate is 100% (or meets threshold with no errors). Always 1 when any case fails.
|
|
355
|
+
|
|
356
|
+
---
|
|
357
|
+
|
|
358
|
+
## User Simulation
|
|
359
|
+
|
|
360
|
+
Use `UserSimulator` when fixed test cases are not enough — the simulator uses an LLM to play the role of a user, drives a real conversation with the agent, and checks whether goals are achieved turn by turn.
|
|
361
|
+
|
|
362
|
+
```python
|
|
363
|
+
from tenxgraph.qa.evaluation import (
|
|
364
|
+
UserSimulator, ConversationScenario, BatchSimulator,
|
|
365
|
+
SimulationGoalsCriterion, CriterionConfig, UserSimulatorConfig,
|
|
366
|
+
)
|
|
367
|
+
```
|
|
368
|
+
|
|
369
|
+
**ConversationScenario** — defines what the simulated user wants to accomplish:
|
|
370
|
+
|
|
371
|
+
```python
|
|
372
|
+
scenario = ConversationScenario(
|
|
373
|
+
scenario_id="travel_planning",
|
|
374
|
+
description="User wants to plan a weekend trip.",
|
|
375
|
+
starting_prompt="I'm thinking of going somewhere warm this weekend.",
|
|
376
|
+
conversation_plan="1. Ask about destinations\n2. Narrow down\n3. Ask about flights",
|
|
377
|
+
goals=[
|
|
378
|
+
"Get weather info for at least one destination",
|
|
379
|
+
"Receive flight or travel suggestions",
|
|
380
|
+
],
|
|
381
|
+
max_turns=8, # hard cap; default 10
|
|
382
|
+
metadata={}, # arbitrary; passed through to results
|
|
383
|
+
)
|
|
384
|
+
```
|
|
385
|
+
|
|
386
|
+
If `starting_prompt` is empty, the simulator LLM generates the first message from `description` and `conversation_plan`.
|
|
387
|
+
|
|
388
|
+
**UserSimulator** — runs the conversation loop:
|
|
389
|
+
|
|
390
|
+
```python
|
|
391
|
+
simulator = UserSimulator(
|
|
392
|
+
model="gemini/gemini-2.5-flash", # "gpt-4o" also supported
|
|
393
|
+
temperature=0.7,
|
|
394
|
+
max_turns=10,
|
|
395
|
+
criteria=[], # BaseCriterion instances scored after the simulation
|
|
396
|
+
)
|
|
397
|
+
|
|
398
|
+
result = await simulator.run(compiled_graph, scenario, config={"configurable": {"thread_id": "sim-1"}})
|
|
399
|
+
```
|
|
400
|
+
|
|
401
|
+
Via `UserSimulatorConfig`:
|
|
402
|
+
|
|
403
|
+
```python
|
|
404
|
+
simulator = UserSimulator(config=UserSimulatorConfig(
|
|
405
|
+
model="gemini/gemini-2.5-flash",
|
|
406
|
+
max_invocations=12,
|
|
407
|
+
temperature=0.5,
|
|
408
|
+
thinking_enabled=False,
|
|
409
|
+
thinking_budget=10240,
|
|
410
|
+
))
|
|
411
|
+
```
|
|
412
|
+
|
|
413
|
+
Model routing: model strings starting with `gemini/` or matching a Google model name use Google GenAI; `anthropic/`, `claude/`, or a `claude-*` name use Anthropic; everything else uses OpenAI. A recognised `provider/` prefix is stripped before the name reaches the SDK. Falls back to the other provider if the primary call fails.
|
|
414
|
+
|
|
415
|
+
**SimulationResult:**
|
|
416
|
+
|
|
417
|
+
```python
|
|
418
|
+
result.scenario_id # str
|
|
419
|
+
result.turns # int — how many turns ran
|
|
420
|
+
result.completed # bool — True when all goals achieved before max_turns
|
|
421
|
+
result.goals_achieved # list[str] — goal strings confirmed by the LLM goal-checker
|
|
422
|
+
result.conversation # list[dict] — [{"role": "user"/"assistant", "content": str}]
|
|
423
|
+
result.error # str | None
|
|
424
|
+
result.criterion_scores # dict[str, float] — only populated when criteria passed to simulator
|
|
425
|
+
result.criterion_details # dict[str, Any] — full criterion output including reasoning
|
|
426
|
+
```
|
|
427
|
+
|
|
428
|
+
**SimulationGoalsCriterion** — LLM judge that scores the full conversation transcript against the goals. Designed exclusively for `UserSimulator`; do NOT add to a regular `EvalConfig` (it expects the full transcript, not a single response).
|
|
429
|
+
|
|
430
|
+
```python
|
|
431
|
+
judge = SimulationGoalsCriterion(
|
|
432
|
+
config=CriterionConfig(threshold=0.7, judge_model="gemini-2.5-flash")
|
|
433
|
+
)
|
|
434
|
+
simulator = UserSimulator(model="gemini/gemini-2.5-flash", criteria=[judge])
|
|
435
|
+
result = await simulator.run(graph, scenario)
|
|
436
|
+
# result.criterion_scores["simulation_goals"] → float 0.0–1.0
|
|
437
|
+
# result.criterion_details["simulation_goals"] → {"achieved_goals": [...], "unachieved_goals": [...], "reasoning": "..."}
|
|
438
|
+
```
|
|
439
|
+
|
|
440
|
+
Score = `achieved_goals / total_goals`.
|
|
441
|
+
|
|
442
|
+
**BatchSimulator** — runs multiple scenarios concurrently; each gets its own thread ID:
|
|
443
|
+
|
|
444
|
+
```python
|
|
445
|
+
batch = BatchSimulator(simulator=simulator, max_concurrency=5)
|
|
446
|
+
results = await batch.run_batch(compiled_graph, scenarios)
|
|
447
|
+
|
|
448
|
+
summary = batch.summary(results)
|
|
449
|
+
# summary["total_scenarios"], ["completed"], ["completion_rate"],
|
|
450
|
+
# ["total_goals_achieved"], ["average_turns"], ["errors"]
|
|
451
|
+
```
|
|
452
|
+
|
|
453
|
+
**Goal-checking mechanism** — after each agent turn, the simulator sends the conversation and each unachieved goal to the judge LLM. Falls back to keyword matching if the LLM call fails. Simulation ends early when all goals are achieved.
|
|
454
|
+
|
|
455
|
+
**When to use simulation vs standard evaluation:**
|
|
456
|
+
|
|
457
|
+
- `AgentEvaluator` + `EvalSet` → regression testing with known queries and expected responses
|
|
458
|
+
- `UserSimulator` → open-ended multi-turn behaviour, goal achievement, conversation quality under dynamic input
|
|
459
|
+
|
|
460
|
+
---
|
|
461
|
+
|
|
462
|
+
## pytest Integration Helpers
|
|
463
|
+
|
|
464
|
+
```python
|
|
465
|
+
from tenxgraph.qa.evaluation import (
|
|
466
|
+
eval_test, parametrize_eval_cases, run_eval,
|
|
467
|
+
assert_eval_passed, assert_criterion_passed,
|
|
468
|
+
EvalTestCase, EvalFixtures, EvalPlugin,
|
|
469
|
+
create_eval_app, create_simple_eval_set,
|
|
470
|
+
)
|
|
471
|
+
```
|
|
472
|
+
|
|
473
|
+
- `eval_test` — decorator to mark a test as an eval test
|
|
474
|
+
- `parametrize_eval_cases(eval_set)` — `@pytest.mark.parametrize` over all eval cases
|
|
475
|
+
- `run_eval(graph, eval_set, config)` — sync helper that calls `asyncio.run`
|
|
476
|
+
- `assert_eval_passed(report)` — raises `AssertionError` if `pass_rate < 1.0`
|
|
477
|
+
- `assert_criterion_passed(result, criterion_key)` — raises if named criterion failed
|
|
478
|
+
|
|
479
|
+
---
|
|
480
|
+
|
|
481
|
+
## Rules
|
|
482
|
+
|
|
483
|
+
- Use no-LLM criteria (`tool_usage`, `quick_check`) in fast CI; add LLM-judge criteria only in slower quality gates.
|
|
484
|
+
- Compile the graph once with `TrajectoryCollector` wired in; reuse across all eval cases.
|
|
485
|
+
- Never add `SimulationGoalsCriterion` to `EvalConfig` — it is only valid inside `UserSimulator.criteria`.
|
|
486
|
+
- Set `threshold` in `10xgraph.json` so CI fails automatically without extra flags.
|
|
487
|
+
- Use `timestamp_files: true` so report files from different runs do not overwrite each other.
|
|
488
|
+
|
|
489
|
+
## Source Map
|
|
490
|
+
|
|
491
|
+
- Evaluation package: https://github.com/10xGraph/10xGraph/tree/main/tenxgraph/qa/evaluation
|
|
492
|
+
- EvalSetBuilder: https://github.com/10xGraph/10xGraph/blob/main/tenxgraph/qa/evaluation/dataset/builder.py
|
|
493
|
+
- EvalConfig / CriterionConfig: https://github.com/10xGraph/10xGraph/blob/main/tenxgraph/qa/evaluation/config/eval_config.py
|
|
494
|
+
- EvalPresets: https://github.com/10xGraph/10xGraph/blob/main/tenxgraph/qa/evaluation/config/presets.py
|
|
495
|
+
- AgentEvaluator: https://github.com/10xGraph/10xGraph/blob/main/tenxgraph/qa/evaluation/evaluator.py
|
|
496
|
+
- QuickEval: https://github.com/10xGraph/10xGraph/blob/main/tenxgraph/qa/evaluation/quick_eval.py
|
|
497
|
+
- UserSimulator / BatchSimulator: https://github.com/10xGraph/10xGraph/blob/main/tenxgraph/qa/evaluation/simulators/user_simulator.py
|
|
498
|
+
- SimulationGoalsCriterion: https://github.com/10xGraph/10xGraph/blob/main/tenxgraph/qa/evaluation/criteria/simulation_goals.py
|
|
499
|
+
- CLI eval command: https://github.com/10xGraph/10xgraph-api/blob/main/tenxgraph_api/cli/commands/eval.py
|
|
500
|
+
- Public exports: https://github.com/10xGraph/10xGraph/blob/main/tenxgraph/qa/__init__.py
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
# ID and Thread Name Generators
|
|
2
|
+
|
|
3
|
+
Use this when changing API ID generation, Snowflake IDs, graph ID generators, or thread display-name generation.
|
|
4
|
+
|
|
5
|
+
## Python Graph ID Generators
|
|
6
|
+
|
|
7
|
+
Core graph ID generators live in `tenxgraph.utils`.
|
|
8
|
+
|
|
9
|
+
Built-ins:
|
|
10
|
+
|
|
11
|
+
- `DefaultIDGenerator`
|
|
12
|
+
- `UUIDGenerator`
|
|
13
|
+
- `BigIntIDGenerator`
|
|
14
|
+
- `TimestampIDGenerator`
|
|
15
|
+
- `IntIDGenerator`
|
|
16
|
+
- `HexIDGenerator`
|
|
17
|
+
- `ShortIDGenerator`
|
|
18
|
+
- `AsyncIDGenerator`
|
|
19
|
+
|
|
20
|
+
Pass to `StateGraph(id_generator=...)`.
|
|
21
|
+
|
|
22
|
+
## API Snowflake ID Generator
|
|
23
|
+
|
|
24
|
+
The API package includes a Snowflake-style ID generator for server-side/generated IDs.
|
|
25
|
+
|
|
26
|
+
Environment/config variables:
|
|
27
|
+
|
|
28
|
+
- `SNOWFLAKE_EPOCH`
|
|
29
|
+
- `SNOWFLAKE_NODE_ID`
|
|
30
|
+
- `SNOWFLAKE_WORKER_ID`
|
|
31
|
+
- `SNOWFLAKE_TIME_BITS`
|
|
32
|
+
- `SNOWFLAKE_NODE_BITS`
|
|
33
|
+
- `SNOWFLAKE_WORKER_BITS`
|
|
34
|
+
|
|
35
|
+
Use Snowflake IDs when deployment needs sortable, distributed, non-UUID identifiers.
|
|
36
|
+
|
|
37
|
+
## Thread Name Generator
|
|
38
|
+
|
|
39
|
+
Configure in `10xgraph.json`:
|
|
40
|
+
|
|
41
|
+
```json
|
|
42
|
+
{
|
|
43
|
+
"thread_name_generator": "graph.thread_name_generator:MyNameGenerator"
|
|
44
|
+
}
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
The API uses the configured generator when creating or naming threads. If no generator is configured, a dummy/default generator may be used.
|
|
48
|
+
|
|
49
|
+
Keep thread name generation:
|
|
50
|
+
|
|
51
|
+
- Fast enough for request paths.
|
|
52
|
+
- Safe for arbitrary user messages.
|
|
53
|
+
- Deterministic enough for tests where needed.
|
|
54
|
+
|
|
55
|
+
## Rules
|
|
56
|
+
|
|
57
|
+
- Keep ID type expectations aligned across Python state, API responses, checkpointers, and TypeScript client types.
|
|
58
|
+
- For multi-worker deployments, avoid process-local counters unless partitioned safely.
|
|
59
|
+
- Validate Snowflake node/worker bit settings before production use.
|
|
60
|
+
- Treat thread names as display labels, not stable identifiers.
|
|
61
|
+
|
|
62
|
+
## Source Map
|
|
63
|
+
|
|
64
|
+
- Python ID generators: https://github.com/10xGraph/10xGraph/blob/main/tenxgraph/utils/id_generator.py
|
|
65
|
+
- API Snowflake generator: https://github.com/10xGraph/10xgraph-api/blob/main/tenxgraph_api/src/app/utils/snowflake_id_generator.py
|
|
66
|
+
- Thread name generator: https://github.com/10xGraph/10xgraph-api/blob/main/tenxgraph_api/src/app/utils/thread_name_generator.py
|
|
67
|
+
- Graph service thread naming: https://github.com/10xGraph/10xgraph-api/blob/main/tenxgraph_api/src/app/routers/graph/services/graph_service.py
|
|
68
|
+
- Settings: https://github.com/10xGraph/10xgraph-api/blob/main/tenxgraph_api/src/app/core/config/settings.py
|
|
69
|
+
- Docs: https://10xgraph.com/
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
# Media and Files
|
|
2
|
+
|
|
3
|
+
Use this when implementing multimodal messages, file upload/download, media stores, or provider media handling.
|
|
4
|
+
|
|
5
|
+
## Message Model
|
|
6
|
+
|
|
7
|
+
Messages can mix text and media blocks. Media bytes are referenced through `MediaRef` rather than stored directly in graph logic.
|
|
8
|
+
|
|
9
|
+
`MediaRef.kind` values:
|
|
10
|
+
|
|
11
|
+
- `"url"`: external URL.
|
|
12
|
+
- `"data"`: inline base64 bytes for small payloads.
|
|
13
|
+
- `"file_id"`: key returned by a media store or upload route.
|
|
14
|
+
|
|
15
|
+
Content blocks include image, audio, video, document, and generic data blocks. See `state-and-messages.md` for the full list.
|
|
16
|
+
|
|
17
|
+
## Media Stores
|
|
18
|
+
|
|
19
|
+
Media stores keep binary data outside messages:
|
|
20
|
+
|
|
21
|
+
- `InMemoryMediaStore`: tests/dev only.
|
|
22
|
+
- `LocalFileMediaStore`: local single-server storage with metadata sidecars.
|
|
23
|
+
- `CloudMediaStore`: cloud object storage with signed URL support where configured.
|
|
24
|
+
|
|
25
|
+
The base store supports storing, retrieving, deleting, existence checks, and metadata reads.
|
|
26
|
+
|
|
27
|
+
## MultimodalConfig
|
|
28
|
+
|
|
29
|
+
Use `MultimodalConfig` on `Agent` to control provider transport:
|
|
30
|
+
|
|
31
|
+
- Image handling: base64, URL, or provider file ID.
|
|
32
|
+
- Document handling: extract text, forward raw, or skip.
|
|
33
|
+
- Size/dimension/type constraints for media safety and provider compatibility.
|
|
34
|
+
|
|
35
|
+
The resolver should pick a supported transport based on provider/model capabilities.
|
|
36
|
+
|
|
37
|
+
## REST and Client
|
|
38
|
+
|
|
39
|
+
API routes:
|
|
40
|
+
|
|
41
|
+
- `POST /v1/files/upload`
|
|
42
|
+
- `GET /v1/files/{file_id}`
|
|
43
|
+
- `GET /v1/files/{file_id}/info`
|
|
44
|
+
- `GET /v1/files/{file_id}/url`
|
|
45
|
+
- `GET /v1/config/multimodal`
|
|
46
|
+
|
|
47
|
+
TypeScript client methods:
|
|
48
|
+
|
|
49
|
+
- `uploadFile`
|
|
50
|
+
- `getFile`
|
|
51
|
+
- `getFileInfo`
|
|
52
|
+
- `getFileAccessUrl`
|
|
53
|
+
- `getMultimodalConfig`
|
|
54
|
+
|
|
55
|
+
## Rules
|
|
56
|
+
|
|
57
|
+
- Use `file_id` for repeated or large media.
|
|
58
|
+
- Inline base64 only for small payloads.
|
|
59
|
+
- Preserve MIME types.
|
|
60
|
+
- Pass `media_store` at compile/runtime boundaries where the graph must dereference `file_id` media.
|
|
61
|
+
|
|
62
|
+
## Source Map
|
|
63
|
+
|
|
64
|
+
- Media config/resolver: https://github.com/10xGraph/10xGraph/tree/main/tenxgraph/storage/media
|
|
65
|
+
- Media router: https://github.com/10xGraph/10xgraph-api/tree/main/tenxgraph_api/src/app/routers/media
|
|
66
|
+
- TS files endpoint: https://github.com/10xHub/agentflow-client/blob/main/src/endpoints/files.ts
|