10xgraph-api 0.7.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (203) hide show
  1. 10xgraph_api-0.7.0.dist-info/METADATA +618 -0
  2. 10xgraph_api-0.7.0.dist-info/RECORD +203 -0
  3. 10xgraph_api-0.7.0.dist-info/WHEEL +5 -0
  4. 10xgraph_api-0.7.0.dist-info/entry_points.txt +3 -0
  5. 10xgraph_api-0.7.0.dist-info/licenses/LICENSE +21 -0
  6. 10xgraph_api-0.7.0.dist-info/top_level.txt +2 -0
  7. agentflow_cli/__init__.py +69 -0
  8. tenxgraph_api/__init__.py +23 -0
  9. tenxgraph_api/cli/__init__.py +6 -0
  10. tenxgraph_api/cli/capabilities.py +127 -0
  11. tenxgraph_api/cli/commands/__init__.py +49 -0
  12. tenxgraph_api/cli/commands/api.py +262 -0
  13. tenxgraph_api/cli/commands/audit.py +213 -0
  14. tenxgraph_api/cli/commands/build.py +305 -0
  15. tenxgraph_api/cli/commands/config.py +71 -0
  16. tenxgraph_api/cli/commands/demo.py +93 -0
  17. tenxgraph_api/cli/commands/eval.py +909 -0
  18. tenxgraph_api/cli/commands/init.py +531 -0
  19. tenxgraph_api/cli/commands/skills.py +532 -0
  20. tenxgraph_api/cli/commands/test.py +113 -0
  21. tenxgraph_api/cli/commands/version.py +47 -0
  22. tenxgraph_api/cli/config_editor/__init__.py +15 -0
  23. tenxgraph_api/cli/config_editor/schema.py +566 -0
  24. tenxgraph_api/cli/config_editor/server.py +240 -0
  25. tenxgraph_api/cli/config_editor/static/app.css +2 -0
  26. tenxgraph_api/cli/config_editor/static/app.js +3 -0
  27. tenxgraph_api/cli/config_editor/static/favicon.svg +11 -0
  28. tenxgraph_api/cli/config_editor/static/index.html +20 -0
  29. tenxgraph_api/cli/config_editor/store.py +120 -0
  30. tenxgraph_api/cli/config_editor/validation.py +377 -0
  31. tenxgraph_api/cli/constants.py +112 -0
  32. tenxgraph_api/cli/context.py +46 -0
  33. tenxgraph_api/cli/core/__init__.py +0 -0
  34. tenxgraph_api/cli/core/animations.py +550 -0
  35. tenxgraph_api/cli/core/config.py +279 -0
  36. tenxgraph_api/cli/core/output.py +625 -0
  37. tenxgraph_api/cli/core/prompts.py +220 -0
  38. tenxgraph_api/cli/core/screen.py +287 -0
  39. tenxgraph_api/cli/core/steps.py +621 -0
  40. tenxgraph_api/cli/core/theme.py +242 -0
  41. tenxgraph_api/cli/core/validation.py +251 -0
  42. tenxgraph_api/cli/exceptions.py +129 -0
  43. tenxgraph_api/cli/logger.py +84 -0
  44. tenxgraph_api/cli/main.py +936 -0
  45. tenxgraph_api/cli/templates/__init__.py +0 -0
  46. tenxgraph_api/cli/templates/defaults.py +844 -0
  47. tenxgraph_api/cli/templates/dev/.env.example +1 -0
  48. tenxgraph_api/cli/templates/dev/10xgraph.json +16 -0
  49. tenxgraph_api/cli/templates/dev/graph/__init__.py +0 -0
  50. tenxgraph_api/cli/templates/dev/graph/agent.py +63 -0
  51. tenxgraph_api/cli/templates/prod/.env.example +75 -0
  52. tenxgraph_api/cli/templates/prod/.pre-commit-config.yaml +50 -0
  53. tenxgraph_api/cli/templates/prod/.python-version +1 -0
  54. tenxgraph_api/cli/templates/prod/10xgraph.json +42 -0
  55. tenxgraph_api/cli/templates/prod/auth/__init__.py +0 -0
  56. tenxgraph_api/cli/templates/prod/auth/agent_auth.py +46 -0
  57. tenxgraph_api/cli/templates/prod/evals/__init__.py +0 -0
  58. tenxgraph_api/cli/templates/prod/evals/user_simulator_eval.py +76 -0
  59. tenxgraph_api/cli/templates/prod/evals/weather_agents_eval.py +53 -0
  60. tenxgraph_api/cli/templates/prod/graph/__init__.py +0 -0
  61. tenxgraph_api/cli/templates/prod/graph/agent.py +50 -0
  62. tenxgraph_api/cli/templates/prod/graph/state.py +5 -0
  63. tenxgraph_api/cli/templates/prod/graph/thread_name_generator.py +7 -0
  64. tenxgraph_api/cli/templates/prod/graph/tools/__init__.py +0 -0
  65. tenxgraph_api/cli/templates/prod/graph/tools/weather_tool.py +43 -0
  66. tenxgraph_api/cli/templates/prod/graph/validators/__init__.py +0 -0
  67. tenxgraph_api/cli/templates/prod/graph/validators/lifecyle.py +107 -0
  68. tenxgraph_api/cli/templates/prod/graph/validators/manager.py +9 -0
  69. tenxgraph_api/cli/templates/prod/graph/validators/validators.py +21 -0
  70. tenxgraph_api/cli/templates/prod/pyproject.toml +79 -0
  71. tenxgraph_api/cli/templates/prod/tests/__init__.py +0 -0
  72. tenxgraph_api/cli/templates/prod/tests/conftest.py +6 -0
  73. tenxgraph_api/cli/templates/prod/tests/test_agent_eval.py +78 -0
  74. tenxgraph_api/cli/templates/prod/tests/test_catalog_tools.py +47 -0
  75. tenxgraph_api/cli/templates/prod/tests/test_graph_nodes.py +57 -0
  76. tenxgraph_api/cli/templates/skills/10xgraph/SKILL.md +92 -0
  77. tenxgraph_api/cli/templates/skills/10xgraph/references/agents-and-tools.md +389 -0
  78. tenxgraph_api/cli/templates/skills/10xgraph/references/api-client.md +61 -0
  79. tenxgraph_api/cli/templates/skills/10xgraph/references/api-configuration.md +75 -0
  80. tenxgraph_api/cli/templates/skills/10xgraph/references/api-settings-and-middleware.md +89 -0
  81. tenxgraph_api/cli/templates/skills/10xgraph/references/architecture.md +91 -0
  82. tenxgraph_api/cli/templates/skills/10xgraph/references/auth-and-authorization.md +149 -0
  83. tenxgraph_api/cli/templates/skills/10xgraph/references/callbacks-and-command.md +283 -0
  84. tenxgraph_api/cli/templates/skills/10xgraph/references/checkpointing-and-threads.md +63 -0
  85. tenxgraph_api/cli/templates/skills/10xgraph/references/cli-commands.md +73 -0
  86. tenxgraph_api/cli/templates/skills/10xgraph/references/client-auth-and-errors.md +94 -0
  87. tenxgraph_api/cli/templates/skills/10xgraph/references/client-messages-invoke-stream.md +115 -0
  88. tenxgraph_api/cli/templates/skills/10xgraph/references/client-threads-memory-files.md +80 -0
  89. tenxgraph_api/cli/templates/skills/10xgraph/references/context-id-background.md +80 -0
  90. tenxgraph_api/cli/templates/skills/10xgraph/references/dependency-injection.md +58 -0
  91. tenxgraph_api/cli/templates/skills/10xgraph/references/evaluation.md +500 -0
  92. tenxgraph_api/cli/templates/skills/10xgraph/references/id-and-thread-name-generators.md +69 -0
  93. tenxgraph_api/cli/templates/skills/10xgraph/references/media-and-files.md +66 -0
  94. tenxgraph_api/cli/templates/skills/10xgraph/references/memory-and-store.md +70 -0
  95. tenxgraph_api/cli/templates/skills/10xgraph/references/prebuilt-agents-and-tools.md +394 -0
  96. tenxgraph_api/cli/templates/skills/10xgraph/references/production-runtime.md +56 -0
  97. tenxgraph_api/cli/templates/skills/10xgraph/references/providers-and-adapters.md +89 -0
  98. tenxgraph_api/cli/templates/skills/10xgraph/references/publishers-and-runtime-protocols.md +84 -0
  99. tenxgraph_api/cli/templates/skills/10xgraph/references/rate-limiting.md +194 -0
  100. tenxgraph_api/cli/templates/skills/10xgraph/references/realtime.md +172 -0
  101. tenxgraph_api/cli/templates/skills/10xgraph/references/remote-tools.md +55 -0
  102. tenxgraph_api/cli/templates/skills/10xgraph/references/rest-api-and-errors.md +97 -0
  103. tenxgraph_api/cli/templates/skills/10xgraph/references/security-and-validators.md +86 -0
  104. tenxgraph_api/cli/templates/skills/10xgraph/references/state-and-messages.md +68 -0
  105. tenxgraph_api/cli/templates/skills/10xgraph/references/state-graph.md +232 -0
  106. tenxgraph_api/cli/templates/skills/10xgraph/references/stream-emitter.md +144 -0
  107. tenxgraph_api/cli/templates/skills/10xgraph/references/streaming.md +53 -0
  108. tenxgraph_api/cli/templates/skills/10xgraph/references/testing-and-evaluation.md +64 -0
  109. tenxgraph_api/cli/templates/skills/10xgraph/references/unit-testing.md +202 -0
  110. tenxgraph_api/cli/templates/skills/copilot/10xgraph.instructions.md +108 -0
  111. tenxgraph_api/py.typed +0 -0
  112. tenxgraph_api/src/__init__.py +0 -0
  113. tenxgraph_api/src/app/__init__.py +13 -0
  114. tenxgraph_api/src/app/core/__init__.py +15 -0
  115. tenxgraph_api/src/app/core/auth/__init__.py +0 -0
  116. tenxgraph_api/src/app/core/auth/auth_backend.py +38 -0
  117. tenxgraph_api/src/app/core/auth/authorization.py +379 -0
  118. tenxgraph_api/src/app/core/auth/base_auth.py +27 -0
  119. tenxgraph_api/src/app/core/auth/jwt_auth.py +122 -0
  120. tenxgraph_api/src/app/core/auth/ownership_resolver.py +170 -0
  121. tenxgraph_api/src/app/core/auth/permissions.py +354 -0
  122. tenxgraph_api/src/app/core/auth/request_config.py +130 -0
  123. tenxgraph_api/src/app/core/auth/route_guard.py +105 -0
  124. tenxgraph_api/src/app/core/config/__init__.py +0 -0
  125. tenxgraph_api/src/app/core/config/graph_config.py +532 -0
  126. tenxgraph_api/src/app/core/config/media_settings.py +72 -0
  127. tenxgraph_api/src/app/core/config/sentry_config.py +60 -0
  128. tenxgraph_api/src/app/core/config/settings.py +202 -0
  129. tenxgraph_api/src/app/core/config/setup_logs.py +65 -0
  130. tenxgraph_api/src/app/core/config/setup_middleware.py +483 -0
  131. tenxgraph_api/src/app/core/config/worker_middleware.py +99 -0
  132. tenxgraph_api/src/app/core/exceptions/__init__.py +17 -0
  133. tenxgraph_api/src/app/core/exceptions/general_exception.py +58 -0
  134. tenxgraph_api/src/app/core/exceptions/handle_errors.py +363 -0
  135. tenxgraph_api/src/app/core/exceptions/resources_exceptions.py +72 -0
  136. tenxgraph_api/src/app/core/exceptions/user_exception.py +58 -0
  137. tenxgraph_api/src/app/core/middleware/__init__.py +25 -0
  138. tenxgraph_api/src/app/core/middleware/rate_limit/__init__.py +15 -0
  139. tenxgraph_api/src/app/core/middleware/rate_limit/base.py +27 -0
  140. tenxgraph_api/src/app/core/middleware/rate_limit/factory.py +98 -0
  141. tenxgraph_api/src/app/core/middleware/rate_limit/keying.py +133 -0
  142. tenxgraph_api/src/app/core/middleware/rate_limit/memory.py +98 -0
  143. tenxgraph_api/src/app/core/middleware/rate_limit/middleware.py +90 -0
  144. tenxgraph_api/src/app/core/middleware/rate_limit/redis.py +115 -0
  145. tenxgraph_api/src/app/core/middleware/request_limits.py +154 -0
  146. tenxgraph_api/src/app/core/middleware/security_headers.py +239 -0
  147. tenxgraph_api/src/app/core/utils/__init__.py +14 -0
  148. tenxgraph_api/src/app/core/utils/log_sanitizer.py +216 -0
  149. tenxgraph_api/src/app/loader.py +516 -0
  150. tenxgraph_api/src/app/main.py +180 -0
  151. tenxgraph_api/src/app/routers/__init__.py +4 -0
  152. tenxgraph_api/src/app/routers/ag_ui/__init__.py +10 -0
  153. tenxgraph_api/src/app/routers/ag_ui/converter.py +195 -0
  154. tenxgraph_api/src/app/routers/ag_ui/event_mapper.py +353 -0
  155. tenxgraph_api/src/app/routers/ag_ui/router.py +90 -0
  156. tenxgraph_api/src/app/routers/ag_ui/service.py +193 -0
  157. tenxgraph_api/src/app/routers/checkpointer/__init__.py +1 -0
  158. tenxgraph_api/src/app/routers/checkpointer/router.py +482 -0
  159. tenxgraph_api/src/app/routers/checkpointer/schemas/__init__.py +1 -0
  160. tenxgraph_api/src/app/routers/checkpointer/schemas/checkpointer_schemas.py +121 -0
  161. tenxgraph_api/src/app/routers/checkpointer/services/__init__.py +1 -0
  162. tenxgraph_api/src/app/routers/checkpointer/services/checkpointer_service.py +300 -0
  163. tenxgraph_api/src/app/routers/evals/__init__.py +6 -0
  164. tenxgraph_api/src/app/routers/evals/router.py +38 -0
  165. tenxgraph_api/src/app/routers/evals/services/__init__.py +1 -0
  166. tenxgraph_api/src/app/routers/evals/services/eval_report_service.py +354 -0
  167. tenxgraph_api/src/app/routers/graph/__init__.py +6 -0
  168. tenxgraph_api/src/app/routers/graph/realtime_guard.py +161 -0
  169. tenxgraph_api/src/app/routers/graph/router.py +821 -0
  170. tenxgraph_api/src/app/routers/graph/schemas/__init__.py +1 -0
  171. tenxgraph_api/src/app/routers/graph/schemas/graph_schemas.py +386 -0
  172. tenxgraph_api/src/app/routers/graph/services/__init__.py +1 -0
  173. tenxgraph_api/src/app/routers/graph/services/graph_service.py +1219 -0
  174. tenxgraph_api/src/app/routers/graph/services/multimodal_preprocessor.py +164 -0
  175. tenxgraph_api/src/app/routers/media/__init__.py +506 -0
  176. tenxgraph_api/src/app/routers/media/router.py +242 -0
  177. tenxgraph_api/src/app/routers/media/schemas.py +39 -0
  178. tenxgraph_api/src/app/routers/ping/__init__.py +4 -0
  179. tenxgraph_api/src/app/routers/ping/router.py +28 -0
  180. tenxgraph_api/src/app/routers/setup_router.py +60 -0
  181. tenxgraph_api/src/app/routers/store/__init__.py +4 -0
  182. tenxgraph_api/src/app/routers/store/router.py +200 -0
  183. tenxgraph_api/src/app/routers/store/schemas/__init__.py +28 -0
  184. tenxgraph_api/src/app/routers/store/schemas/store_schemas.py +200 -0
  185. tenxgraph_api/src/app/routers/store/services/__init__.py +4 -0
  186. tenxgraph_api/src/app/routers/store/services/store_service.py +203 -0
  187. tenxgraph_api/src/app/tasks/__init__.py +0 -0
  188. tenxgraph_api/src/app/tasks/user_tasks.py +38 -0
  189. tenxgraph_api/src/app/utils/__init__.py +12 -0
  190. tenxgraph_api/src/app/utils/callable_helper.py +26 -0
  191. tenxgraph_api/src/app/utils/media/__init__.py +15 -0
  192. tenxgraph_api/src/app/utils/media/extractor.py +172 -0
  193. tenxgraph_api/src/app/utils/media/pipeline.py +83 -0
  194. tenxgraph_api/src/app/utils/parse_output.py +17 -0
  195. tenxgraph_api/src/app/utils/response_helper.py +129 -0
  196. tenxgraph_api/src/app/utils/schemas/__init__.py +11 -0
  197. tenxgraph_api/src/app/utils/schemas/output_schemas.py +28 -0
  198. tenxgraph_api/src/app/utils/schemas/user_schemas.py +13 -0
  199. tenxgraph_api/src/app/utils/snowflake_id_generator.py +90 -0
  200. tenxgraph_api/src/app/utils/swagger_helper.py +160 -0
  201. tenxgraph_api/src/app/utils/telemetry_store.py +123 -0
  202. tenxgraph_api/src/app/utils/thread_name_generator.py +304 -0
  203. tenxgraph_api/src/app/worker.py +91 -0
@@ -0,0 +1,500 @@
1
+ # Evaluation
2
+
3
+ Use this when defining eval cases, configuring criteria, running evaluations programmatically, generating reports, or using user simulation. All evaluation types live in `tenxgraph.qa.evaluation`.
4
+
5
+ ---
6
+
7
+ ## EvalSet and EvalCase
8
+
9
+ Define test cases with `EvalSetBuilder` (fluent) or `EvalCase` directly.
10
+
11
+ ```python
12
+ from tenxgraph.qa.evaluation import EvalSetBuilder, EvalSet, EvalCase, ToolCall
13
+ ```
14
+
15
+ **EvalSetBuilder — fluent API:**
16
+
17
+ ```python
18
+ eval_set = (
19
+ EvalSetBuilder("agent-regression")
20
+ .add_case(
21
+ query="Weather in London?",
22
+ expected="sunny",
23
+ case_id="london", # optional; auto-generated if omitted
24
+ expected_tools=["get_weather"], # list of tool names or ToolCall objects
25
+ expected_node_order=["MAIN", "TOOL", "MAIN"],
26
+ )
27
+ .add_tool_test(
28
+ query="Weather in Tokyo?",
29
+ tool_name="get_weather",
30
+ tool_args={"location": "Tokyo"},
31
+ expected_response="Tokyo",
32
+ )
33
+ .add_multi_turn(
34
+ conversation=[("Hello", "Hi!"), ("Weather?", "Sunny.")],
35
+ expected_tools=["get_weather"],
36
+ )
37
+ .build()
38
+ )
39
+ ```
40
+
41
+ Quick builders:
42
+
43
+ ```python
44
+ EvalSetBuilder.quick(("Hello", "Hi!"), ("Bye", "Goodbye!"))
45
+ EvalSetBuilder.from_conversations([{"user": "Hello", "assistant": "Hi!"}])
46
+ EvalSetBuilder.from_file("evals/cases.json") # returns builder for modification
47
+ ```
48
+
49
+ Save / load:
50
+
51
+ ```python
52
+ eval_set.to_file("evals/cases.json")
53
+ eval_set = EvalSet.from_file("evals/cases.json")
54
+ builder = EvalSetBuilder.from_file("evals/cases.json") # load + modify
55
+ ```
56
+
57
+ **ToolCall:**
58
+
59
+ ```python
60
+ ToolCall(name="get_weather") # name only
61
+ ToolCall(name="get_weather", args={"location": "London"}) # with args (checked when check_args=True)
62
+ ```
63
+
64
+ **EvalCase directly:**
65
+
66
+ ```python
67
+ case = EvalCase.single_turn(
68
+ eval_id="london",
69
+ user_query="Weather in London?",
70
+ expected_response="sunny",
71
+ expected_tools=[ToolCall(name="get_weather", args={"location": "London"})],
72
+ expected_node_order=["MAIN", "TOOL", "MAIN"],
73
+ )
74
+ case = EvalCase.multi_turn(
75
+ eval_id="multi",
76
+ conversation=[("Hello", "Hi!"), ("Weather?", "Sunny.")],
77
+ expected_tools=[ToolCall(name="get_weather")],
78
+ )
79
+ ```
80
+
81
+ ---
82
+
83
+ ## EvalConfig and CriterionConfig
84
+
85
+ ```python
86
+ from tenxgraph.qa.evaluation import EvalConfig, CriterionConfig, MatchType, Rubric
87
+ ```
88
+
89
+ **EvalConfig** wraps a dict of criterion key → `CriterionConfig`:
90
+
91
+ ```python
92
+ config = EvalConfig(
93
+ criteria={
94
+ "tool_trajectory_avg_score": CriterionConfig.trajectory(threshold=1.0, match_type=MatchType.EXACT, check_args=False),
95
+ "rouge_match": CriterionConfig.rouge_match(threshold=0.5),
96
+ "response_match_score": CriterionConfig.response_match(threshold=0.8),
97
+ },
98
+ parallel=False,
99
+ max_concurrency=4,
100
+ timeout=300.0,
101
+ )
102
+ ```
103
+
104
+ **MatchType** (for trajectory and node-order criteria):
105
+
106
+ - `EXACT` — same tools/nodes, same order, same count; extras fail
107
+ - `IN_ORDER` — expected appear in order; extras allowed
108
+ - `ANY_ORDER` — all expected present; order and extras don't matter
109
+
110
+ **Built-in class-method presets on EvalConfig:**
111
+
112
+ ```python
113
+ EvalConfig.default() # EXACT trajectory + ROUGE response (threshold 0.8)
114
+ EvalConfig.strict() # EXACT + args + high-threshold LLM judge (threshold 0.9, 5 samples)
115
+ EvalConfig.relaxed() # IN_ORDER trajectory + lower response threshold (0.6)
116
+ ```
117
+
118
+ **CriterionConfig factory methods:**
119
+
120
+ | Method | LLM? | Key to use in criteria dict |
121
+ |---|---|---|
122
+ | `CriterionConfig.tool_name_match(threshold)` | No | `tool_name_match_score` |
123
+ | `CriterionConfig.trajectory(threshold, match_type, check_args)` | No | `tool_trajectory_avg_score` |
124
+ | `CriterionConfig.node_order(threshold, match_type)` | No | `node_order_match_score` |
125
+ | `CriterionConfig.rouge_match(threshold)` | No | `rouge_match` |
126
+ | `CriterionConfig.contains_keywords(keywords, threshold)` | No | `contains_keywords` |
127
+ | `CriterionConfig.response_match(threshold, judge_model, num_samples)` | Yes | `response_match_score` |
128
+ | `CriterionConfig.llm_judge(threshold, judge_model, num_samples)` | Yes | `llm_judge` |
129
+ | `CriterionConfig.factual_accuracy(threshold, judge_model, num_samples)` | Yes | `factual_accuracy_v1` |
130
+ | `CriterionConfig.hallucination(threshold, judge_model, num_samples)` | Yes | `hallucinations_v1` |
131
+ | `CriterionConfig.safety(threshold, judge_model, num_samples)` | Yes | `safety_v1` |
132
+ | `CriterionConfig.rubric_based(rubrics, threshold, judge_model)` | Yes | any key |
133
+
134
+ Default judge model: `"gemini-2.5-flash"`. Override per criterion with `judge_model="gpt-4o"`.
135
+
136
+ `num_samples` — number of LLM judge calls; final score is the average (reduces non-determinism noise).
137
+
138
+ **Custom rubrics:**
139
+
140
+ ```python
141
+ CriterionConfig.rubric_based(
142
+ rubrics=[
143
+ Rubric(rubric_id="concise", content="Response must be under 100 words.", weight=1.0),
144
+ Rubric(rubric_id="empathetic", content="Acknowledge the user's issue first.", weight=0.5),
145
+ ],
146
+ threshold=0.8,
147
+ )
148
+ ```
149
+
150
+ Score = weighted average of rubric scores.
151
+
152
+ **Disable / enable a criterion at runtime:**
153
+
154
+ ```python
155
+ config.disable_criterion("rouge_match")
156
+ config.enable_criterion("rouge_match", CriterionConfig.rouge_match(0.4))
157
+ ```
158
+
159
+ **Save / load:**
160
+
161
+ ```python
162
+ config.to_file("eval_config.json")
163
+ config = EvalConfig.from_file("eval_config.json")
164
+ ```
165
+
166
+ ---
167
+
168
+ ## EvalPresets
169
+
170
+ Ready-made `EvalConfig` objects for common scenarios.
171
+
172
+ ```python
173
+ from tenxgraph.qa.evaluation import EvalPresets
174
+ ```
175
+
176
+ | Preset | LLM? | Criteria included |
177
+ |---|---|---|
178
+ | `EvalPresets.quick_check()` | No | `rouge_match` (0.5) |
179
+ | `EvalPresets.tool_usage(threshold, strict, check_args)` | No | `tool_name_match_score` + `tool_trajectory_avg_score` |
180
+ | `EvalPresets.response_quality(threshold, use_llm_judge, judge_model)` | Yes | `response_match_score` + optional `llm_judge` |
181
+ | `EvalPresets.conversation_flow(threshold, judge_model)` | Yes | `response_match_score` + `tool_trajectory_avg_score` (IN_ORDER) |
182
+ | `EvalPresets.safety_check(threshold, judge_model)` | Yes | `hallucinations_v1` + `safety_v1` |
183
+ | `EvalPresets.comprehensive(threshold, use_llm_judge, judge_model)` | Yes | all no-LLM + all LLM criteria |
184
+ | `EvalPresets.custom(response_threshold, tool_threshold, llm_judge_threshold, ...)` | Both | only criteria whose threshold is not None |
185
+
186
+ Combine presets (later keys override earlier on conflict):
187
+
188
+ ```python
189
+ config = EvalPresets.combine(
190
+ EvalPresets.tool_usage(threshold=1.0),
191
+ EvalPresets.safety_check(threshold=0.8),
192
+ )
193
+ ```
194
+
195
+ ---
196
+
197
+ ## AgentEvaluator and QuickEval
198
+
199
+ ```python
200
+ from tenxgraph.qa.evaluation import AgentEvaluator, QuickEval
201
+ from tenxgraph.qa.evaluation.collectors.trajectory_collector import TrajectoryCollector
202
+ ```
203
+
204
+ **TrajectoryCollector** — must be created before evaluation and passed to `AgentEvaluator`. Captures tool calls, node visits, and LLM outputs through the graph callback system.
205
+
206
+ ```python
207
+ collector = TrajectoryCollector(capture_all_events=True)
208
+ ```
209
+
210
+ **AgentEvaluator** — runs the full eval set:
211
+
212
+ ```python
213
+ evaluator = AgentEvaluator(compiled_graph, collector, config=config)
214
+ report = await evaluator.evaluate(eval_set) # EvalSet object
215
+ report = await evaluator.evaluate("evals/cases.json") # path to JSON
216
+ report = await evaluator.evaluate_case(single_case) # one EvalCase
217
+ ```
218
+
219
+ **QuickEval** — one-liners for common patterns:
220
+
221
+ ```python
222
+ # Single check
223
+ report = await QuickEval.check(
224
+ graph=app, collector=collector,
225
+ query="Weather in London?",
226
+ expected_response_contains="sunny",
227
+ expected_tools=["get_weather"],
228
+ threshold=0.7,
229
+ )
230
+
231
+ # Preset-based batch
232
+ report = await QuickEval.preset(
233
+ graph=app, collector=collector,
234
+ preset=EvalPresets.tool_usage(),
235
+ eval_set=eval_set,
236
+ )
237
+
238
+ # Query-response pairs
239
+ report = await QuickEval.batch(
240
+ app, collector,
241
+ [("Hello", "Hi!"), ("Bye", "Goodbye!")],
242
+ threshold=0.7,
243
+ )
244
+
245
+ # Tool-specific
246
+ report = await QuickEval.tool_usage(
247
+ app, collector,
248
+ test_cases=[("Weather in London?", "sunny", ["get_weather"])],
249
+ strict=True,
250
+ )
251
+
252
+ # Multi-turn conversation
253
+ report = await QuickEval.conversation_flow(
254
+ app, collector,
255
+ conversation=[("Hello", "Hi!"), ("Weather?", "Sunny.")],
256
+ threshold=0.8,
257
+ )
258
+
259
+ # Builder-based
260
+ report = await QuickEval.from_builder(app, collector, builder=my_builder, config=config)
261
+
262
+ # Synchronous wrapper (for non-async contexts)
263
+ report = QuickEval.run_sync(app, collector, eval_set=eval_set, config=config)
264
+ ```
265
+
266
+ ---
267
+
268
+ ## EvalReport
269
+
270
+ ```python
271
+ report.summary.total_cases # int
272
+ report.summary.passed_cases # int
273
+ report.summary.failed_cases # int
274
+ report.summary.pass_rate # float 0.0–1.0
275
+ report.summary.criteria_scores # dict[str, float] — average per criterion
276
+
277
+ for result in report.results:
278
+ result.case_id
279
+ result.passed # bool
280
+ result.criteria_results # dict[str, CriterionResult]
281
+ result.criteria_results["rouge_match"].score # float
282
+ result.criteria_results["rouge_match"].passed # bool
283
+ ```
284
+
285
+ Print to console:
286
+
287
+ ```python
288
+ from tenxgraph.qa.evaluation import print_report
289
+ print_report(report)
290
+ ```
291
+
292
+ ---
293
+
294
+ ## Reporters
295
+
296
+ ```python
297
+ from tenxgraph.qa.evaluation import ReporterManager, ReporterConfig
298
+
299
+ manager = ReporterManager(ReporterConfig(
300
+ output_dir="eval_reports",
301
+ html=True,
302
+ json_report=True,
303
+ junit_xml=False,
304
+ console=True,
305
+ timestamp_files=True,
306
+ include_trajectory=True,
307
+ include_tool_call_details=True,
308
+ ))
309
+ output = manager.run_all(report)
310
+ # output.html_path, output.json_path, output.has_errors, output.errors
311
+ ```
312
+
313
+ Available reporters: `ConsoleReporter`, `JSONReporter`, `HTMLReporter`, `JUnitXMLReporter`.
314
+
315
+ HTML report — visual dashboard with summary cards, criterion bars, per-case details.
316
+ JSON report — full machine-readable results including trajectory data.
317
+ JUnit XML — for CI test summary tabs (GitHub Actions, Jenkins).
318
+
319
+ ---
320
+
321
+ ## 10xgraph eval CLI
322
+
323
+ Auto-discovers `*_eval.py` / `eval_*.py` in `evals/` (or configured directory). Each file must expose `get_eval_set()`, optionally `get_eval_config()` or `EVAL_CONFIG`, or a full `run()` function.
324
+
325
+ ```bash
326
+ 10xgraph eval # scan evals/, write to eval_reports/
327
+ 10xgraph eval evals/my_eval.py # single file
328
+ 10xgraph eval evals/regression/ # subdirectory
329
+ 10xgraph eval --open # open HTML report in browser after run
330
+ 10xgraph eval --threshold 0.8 # exit non-zero if pass rate < 0.8
331
+ 10xgraph eval --output ci/reports # custom output directory
332
+ 10xgraph eval --no-report # console only, no files
333
+ ```
334
+
335
+ Eval file entry points (in order of priority):
336
+
337
+ 1. `run()` — sync or async function returning `EvalReport`; full control
338
+ 2. `get_eval_set()` + optional `get_eval_config()` or `EVAL_CONFIG` — CLI loads agent from `10xgraph.json` and runs with default or provided config
339
+ 3. Files without any entry point are skipped with a warning
340
+
341
+ `10xgraph.json` configuration:
342
+
343
+ ```json
344
+ {
345
+ "evaluation": {
346
+ "directory": "evals",
347
+ "output_dir": "eval_reports",
348
+ "threshold": 0.8,
349
+ "timestamp_files": true
350
+ }
351
+ }
352
+ ```
353
+
354
+ Exit code: 0 only when pass rate is 100% (or meets threshold with no errors). Always 1 when any case fails.
355
+
356
+ ---
357
+
358
+ ## User Simulation
359
+
360
+ Use `UserSimulator` when fixed test cases are not enough — the simulator uses an LLM to play the role of a user, drives a real conversation with the agent, and checks whether goals are achieved turn by turn.
361
+
362
+ ```python
363
+ from tenxgraph.qa.evaluation import (
364
+ UserSimulator, ConversationScenario, BatchSimulator,
365
+ SimulationGoalsCriterion, CriterionConfig, UserSimulatorConfig,
366
+ )
367
+ ```
368
+
369
+ **ConversationScenario** — defines what the simulated user wants to accomplish:
370
+
371
+ ```python
372
+ scenario = ConversationScenario(
373
+ scenario_id="travel_planning",
374
+ description="User wants to plan a weekend trip.",
375
+ starting_prompt="I'm thinking of going somewhere warm this weekend.",
376
+ conversation_plan="1. Ask about destinations\n2. Narrow down\n3. Ask about flights",
377
+ goals=[
378
+ "Get weather info for at least one destination",
379
+ "Receive flight or travel suggestions",
380
+ ],
381
+ max_turns=8, # hard cap; default 10
382
+ metadata={}, # arbitrary; passed through to results
383
+ )
384
+ ```
385
+
386
+ If `starting_prompt` is empty, the simulator LLM generates the first message from `description` and `conversation_plan`.
387
+
388
+ **UserSimulator** — runs the conversation loop:
389
+
390
+ ```python
391
+ simulator = UserSimulator(
392
+ model="gemini/gemini-2.5-flash", # "gpt-4o" also supported
393
+ temperature=0.7,
394
+ max_turns=10,
395
+ criteria=[], # BaseCriterion instances scored after the simulation
396
+ )
397
+
398
+ result = await simulator.run(compiled_graph, scenario, config={"configurable": {"thread_id": "sim-1"}})
399
+ ```
400
+
401
+ Via `UserSimulatorConfig`:
402
+
403
+ ```python
404
+ simulator = UserSimulator(config=UserSimulatorConfig(
405
+ model="gemini/gemini-2.5-flash",
406
+ max_invocations=12,
407
+ temperature=0.5,
408
+ thinking_enabled=False,
409
+ thinking_budget=10240,
410
+ ))
411
+ ```
412
+
413
+ Model routing: model strings starting with `gemini/` or matching a Google model name use Google GenAI; `anthropic/`, `claude/`, or a `claude-*` name use Anthropic; everything else uses OpenAI. A recognised `provider/` prefix is stripped before the name reaches the SDK. Falls back to the other provider if the primary call fails.
414
+
415
+ **SimulationResult:**
416
+
417
+ ```python
418
+ result.scenario_id # str
419
+ result.turns # int — how many turns ran
420
+ result.completed # bool — True when all goals achieved before max_turns
421
+ result.goals_achieved # list[str] — goal strings confirmed by the LLM goal-checker
422
+ result.conversation # list[dict] — [{"role": "user"/"assistant", "content": str}]
423
+ result.error # str | None
424
+ result.criterion_scores # dict[str, float] — only populated when criteria passed to simulator
425
+ result.criterion_details # dict[str, Any] — full criterion output including reasoning
426
+ ```
427
+
428
+ **SimulationGoalsCriterion** — LLM judge that scores the full conversation transcript against the goals. Designed exclusively for `UserSimulator`; do NOT add to a regular `EvalConfig` (it expects the full transcript, not a single response).
429
+
430
+ ```python
431
+ judge = SimulationGoalsCriterion(
432
+ config=CriterionConfig(threshold=0.7, judge_model="gemini-2.5-flash")
433
+ )
434
+ simulator = UserSimulator(model="gemini/gemini-2.5-flash", criteria=[judge])
435
+ result = await simulator.run(graph, scenario)
436
+ # result.criterion_scores["simulation_goals"] → float 0.0–1.0
437
+ # result.criterion_details["simulation_goals"] → {"achieved_goals": [...], "unachieved_goals": [...], "reasoning": "..."}
438
+ ```
439
+
440
+ Score = `achieved_goals / total_goals`.
441
+
442
+ **BatchSimulator** — runs multiple scenarios concurrently; each gets its own thread ID:
443
+
444
+ ```python
445
+ batch = BatchSimulator(simulator=simulator, max_concurrency=5)
446
+ results = await batch.run_batch(compiled_graph, scenarios)
447
+
448
+ summary = batch.summary(results)
449
+ # summary["total_scenarios"], ["completed"], ["completion_rate"],
450
+ # ["total_goals_achieved"], ["average_turns"], ["errors"]
451
+ ```
452
+
453
+ **Goal-checking mechanism** — after each agent turn, the simulator sends the conversation and each unachieved goal to the judge LLM. Falls back to keyword matching if the LLM call fails. Simulation ends early when all goals are achieved.
454
+
455
+ **When to use simulation vs standard evaluation:**
456
+
457
+ - `AgentEvaluator` + `EvalSet` → regression testing with known queries and expected responses
458
+ - `UserSimulator` → open-ended multi-turn behaviour, goal achievement, conversation quality under dynamic input
459
+
460
+ ---
461
+
462
+ ## pytest Integration Helpers
463
+
464
+ ```python
465
+ from tenxgraph.qa.evaluation import (
466
+ eval_test, parametrize_eval_cases, run_eval,
467
+ assert_eval_passed, assert_criterion_passed,
468
+ EvalTestCase, EvalFixtures, EvalPlugin,
469
+ create_eval_app, create_simple_eval_set,
470
+ )
471
+ ```
472
+
473
+ - `eval_test` — decorator to mark a test as an eval test
474
+ - `parametrize_eval_cases(eval_set)` — `@pytest.mark.parametrize` over all eval cases
475
+ - `run_eval(graph, eval_set, config)` — sync helper that calls `asyncio.run`
476
+ - `assert_eval_passed(report)` — raises `AssertionError` if `pass_rate < 1.0`
477
+ - `assert_criterion_passed(result, criterion_key)` — raises if named criterion failed
478
+
479
+ ---
480
+
481
+ ## Rules
482
+
483
+ - Use no-LLM criteria (`tool_usage`, `quick_check`) in fast CI; add LLM-judge criteria only in slower quality gates.
484
+ - Compile the graph once with `TrajectoryCollector` wired in; reuse across all eval cases.
485
+ - Never add `SimulationGoalsCriterion` to `EvalConfig` — it is only valid inside `UserSimulator.criteria`.
486
+ - Set `threshold` in `10xgraph.json` so CI fails automatically without extra flags.
487
+ - Use `timestamp_files: true` so report files from different runs do not overwrite each other.
488
+
489
+ ## Source Map
490
+
491
+ - Evaluation package: https://github.com/10xGraph/10xGraph/tree/main/tenxgraph/qa/evaluation
492
+ - EvalSetBuilder: https://github.com/10xGraph/10xGraph/blob/main/tenxgraph/qa/evaluation/dataset/builder.py
493
+ - EvalConfig / CriterionConfig: https://github.com/10xGraph/10xGraph/blob/main/tenxgraph/qa/evaluation/config/eval_config.py
494
+ - EvalPresets: https://github.com/10xGraph/10xGraph/blob/main/tenxgraph/qa/evaluation/config/presets.py
495
+ - AgentEvaluator: https://github.com/10xGraph/10xGraph/blob/main/tenxgraph/qa/evaluation/evaluator.py
496
+ - QuickEval: https://github.com/10xGraph/10xGraph/blob/main/tenxgraph/qa/evaluation/quick_eval.py
497
+ - UserSimulator / BatchSimulator: https://github.com/10xGraph/10xGraph/blob/main/tenxgraph/qa/evaluation/simulators/user_simulator.py
498
+ - SimulationGoalsCriterion: https://github.com/10xGraph/10xGraph/blob/main/tenxgraph/qa/evaluation/criteria/simulation_goals.py
499
+ - CLI eval command: https://github.com/10xGraph/10xgraph-api/blob/main/tenxgraph_api/cli/commands/eval.py
500
+ - Public exports: https://github.com/10xGraph/10xGraph/blob/main/tenxgraph/qa/__init__.py
@@ -0,0 +1,69 @@
1
+ # ID and Thread Name Generators
2
+
3
+ Use this when changing API ID generation, Snowflake IDs, graph ID generators, or thread display-name generation.
4
+
5
+ ## Python Graph ID Generators
6
+
7
+ Core graph ID generators live in `tenxgraph.utils`.
8
+
9
+ Built-ins:
10
+
11
+ - `DefaultIDGenerator`
12
+ - `UUIDGenerator`
13
+ - `BigIntIDGenerator`
14
+ - `TimestampIDGenerator`
15
+ - `IntIDGenerator`
16
+ - `HexIDGenerator`
17
+ - `ShortIDGenerator`
18
+ - `AsyncIDGenerator`
19
+
20
+ Pass to `StateGraph(id_generator=...)`.
21
+
22
+ ## API Snowflake ID Generator
23
+
24
+ The API package includes a Snowflake-style ID generator for server-side/generated IDs.
25
+
26
+ Environment/config variables:
27
+
28
+ - `SNOWFLAKE_EPOCH`
29
+ - `SNOWFLAKE_NODE_ID`
30
+ - `SNOWFLAKE_WORKER_ID`
31
+ - `SNOWFLAKE_TIME_BITS`
32
+ - `SNOWFLAKE_NODE_BITS`
33
+ - `SNOWFLAKE_WORKER_BITS`
34
+
35
+ Use Snowflake IDs when deployment needs sortable, distributed, non-UUID identifiers.
36
+
37
+ ## Thread Name Generator
38
+
39
+ Configure in `10xgraph.json`:
40
+
41
+ ```json
42
+ {
43
+ "thread_name_generator": "graph.thread_name_generator:MyNameGenerator"
44
+ }
45
+ ```
46
+
47
+ The API uses the configured generator when creating or naming threads. If no generator is configured, a dummy/default generator may be used.
48
+
49
+ Keep thread name generation:
50
+
51
+ - Fast enough for request paths.
52
+ - Safe for arbitrary user messages.
53
+ - Deterministic enough for tests where needed.
54
+
55
+ ## Rules
56
+
57
+ - Keep ID type expectations aligned across Python state, API responses, checkpointers, and TypeScript client types.
58
+ - For multi-worker deployments, avoid process-local counters unless partitioned safely.
59
+ - Validate Snowflake node/worker bit settings before production use.
60
+ - Treat thread names as display labels, not stable identifiers.
61
+
62
+ ## Source Map
63
+
64
+ - Python ID generators: https://github.com/10xGraph/10xGraph/blob/main/tenxgraph/utils/id_generator.py
65
+ - API Snowflake generator: https://github.com/10xGraph/10xgraph-api/blob/main/tenxgraph_api/src/app/utils/snowflake_id_generator.py
66
+ - Thread name generator: https://github.com/10xGraph/10xgraph-api/blob/main/tenxgraph_api/src/app/utils/thread_name_generator.py
67
+ - Graph service thread naming: https://github.com/10xGraph/10xgraph-api/blob/main/tenxgraph_api/src/app/routers/graph/services/graph_service.py
68
+ - Settings: https://github.com/10xGraph/10xgraph-api/blob/main/tenxgraph_api/src/app/core/config/settings.py
69
+ - Docs: https://10xgraph.com/
@@ -0,0 +1,66 @@
1
+ # Media and Files
2
+
3
+ Use this when implementing multimodal messages, file upload/download, media stores, or provider media handling.
4
+
5
+ ## Message Model
6
+
7
+ Messages can mix text and media blocks. Media bytes are referenced through `MediaRef` rather than stored directly in graph logic.
8
+
9
+ `MediaRef.kind` values:
10
+
11
+ - `"url"`: external URL.
12
+ - `"data"`: inline base64 bytes for small payloads.
13
+ - `"file_id"`: key returned by a media store or upload route.
14
+
15
+ Content blocks include image, audio, video, document, and generic data blocks. See `state-and-messages.md` for the full list.
16
+
17
+ ## Media Stores
18
+
19
+ Media stores keep binary data outside messages:
20
+
21
+ - `InMemoryMediaStore`: tests/dev only.
22
+ - `LocalFileMediaStore`: local single-server storage with metadata sidecars.
23
+ - `CloudMediaStore`: cloud object storage with signed URL support where configured.
24
+
25
+ The base store supports storing, retrieving, deleting, existence checks, and metadata reads.
26
+
27
+ ## MultimodalConfig
28
+
29
+ Use `MultimodalConfig` on `Agent` to control provider transport:
30
+
31
+ - Image handling: base64, URL, or provider file ID.
32
+ - Document handling: extract text, forward raw, or skip.
33
+ - Size/dimension/type constraints for media safety and provider compatibility.
34
+
35
+ The resolver should pick a supported transport based on provider/model capabilities.
36
+
37
+ ## REST and Client
38
+
39
+ API routes:
40
+
41
+ - `POST /v1/files/upload`
42
+ - `GET /v1/files/{file_id}`
43
+ - `GET /v1/files/{file_id}/info`
44
+ - `GET /v1/files/{file_id}/url`
45
+ - `GET /v1/config/multimodal`
46
+
47
+ TypeScript client methods:
48
+
49
+ - `uploadFile`
50
+ - `getFile`
51
+ - `getFileInfo`
52
+ - `getFileAccessUrl`
53
+ - `getMultimodalConfig`
54
+
55
+ ## Rules
56
+
57
+ - Use `file_id` for repeated or large media.
58
+ - Inline base64 only for small payloads.
59
+ - Preserve MIME types.
60
+ - Pass `media_store` at compile/runtime boundaries where the graph must dereference `file_id` media.
61
+
62
+ ## Source Map
63
+
64
+ - Media config/resolver: https://github.com/10xGraph/10xGraph/tree/main/tenxgraph/storage/media
65
+ - Media router: https://github.com/10xGraph/10xgraph-api/tree/main/tenxgraph_api/src/app/routers/media
66
+ - TS files endpoint: https://github.com/10xHub/agentflow-client/blob/main/src/endpoints/files.ts