talkops-opscloud 0.3.0b1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (309) hide show
  1. opscloud/__init__.py +5 -0
  2. opscloud/__main__.py +6 -0
  3. opscloud/_constants.py +53 -0
  4. opscloud/_debug.py +145 -0
  5. opscloud/_debug_buffer.py +250 -0
  6. opscloud/_textual_patches.py +367 -0
  7. opscloud/_version.py +15 -0
  8. opscloud/agent/__init__.py +27 -0
  9. opscloud/agent/config.py +168 -0
  10. opscloud/agent/factory.py +1103 -0
  11. opscloud/approval_mode.py +44 -0
  12. opscloud/backend/__init__.py +49 -0
  13. opscloud/backend/composite.py +89 -0
  14. opscloud/backend/local.py +183 -0
  15. opscloud/backend/registry.py +79 -0
  16. opscloud/backend/sandbox_config.py +107 -0
  17. opscloud/backend/sandbox_factory.py +432 -0
  18. opscloud/built_in_skills/remember/SKILL.md +124 -0
  19. opscloud/built_in_skills/skill-creator/SKILL.md +348 -0
  20. opscloud/built_in_skills/skill-creator/scripts/init_skill.py +371 -0
  21. opscloud/built_in_skills/skill-creator/scripts/quick_validate.py +165 -0
  22. opscloud/cli/__init__.py +1 -0
  23. opscloud/cli/commands/__init__.py +15 -0
  24. opscloud/cli/commands/auth.py +45 -0
  25. opscloud/cli/commands/doctor.py +100 -0
  26. opscloud/cli/commands/mcp.py +137 -0
  27. opscloud/cli/commands/threads.py +51 -0
  28. opscloud/cli/interactive.py +154 -0
  29. opscloud/cli/main.py +903 -0
  30. opscloud/cli/non_interactive.py +425 -0
  31. opscloud/cli/server_manager.py +58 -0
  32. opscloud/commands/__init__.py +19 -0
  33. opscloud/commands/_base.py +110 -0
  34. opscloud/commands/_guards.py +52 -0
  35. opscloud/commands/_router.py +146 -0
  36. opscloud/commands/_types.py +36 -0
  37. opscloud/commands/core/__init__.py +47 -0
  38. opscloud/commands/core/auth.py +94 -0
  39. opscloud/commands/core/bug.py +46 -0
  40. opscloud/commands/core/clear.py +106 -0
  41. opscloud/commands/core/cloud.py +185 -0
  42. opscloud/commands/core/compact.py +128 -0
  43. opscloud/commands/core/config_cmd.py +120 -0
  44. opscloud/commands/core/context.py +220 -0
  45. opscloud/commands/core/cost.py +44 -0
  46. opscloud/commands/core/doctor.py +535 -0
  47. opscloud/commands/core/effort.py +108 -0
  48. opscloud/commands/core/exit_cmd.py +41 -0
  49. opscloud/commands/core/fast.py +148 -0
  50. opscloud/commands/core/help_cmd.py +94 -0
  51. opscloud/commands/core/mcp.py +134 -0
  52. opscloud/commands/core/model.py +109 -0
  53. opscloud/commands/core/permissions.py +196 -0
  54. opscloud/commands/core/plugins.py +92 -0
  55. opscloud/commands/core/pool.py +108 -0
  56. opscloud/commands/core/resume.py +97 -0
  57. opscloud/commands/core/skills.py +178 -0
  58. opscloud/commands/power/__init__.py +51 -0
  59. opscloud/commands/power/agents.py +98 -0
  60. opscloud/commands/power/btw.py +85 -0
  61. opscloud/commands/power/copy.py +97 -0
  62. opscloud/commands/power/goal.py +548 -0
  63. opscloud/commands/power/loop.py +295 -0
  64. opscloud/commands/power/memory.py +287 -0
  65. opscloud/commands/power/review.py +152 -0
  66. opscloud/commands/power/rubric.py +287 -0
  67. opscloud/commands/power/runtime.py +321 -0
  68. opscloud/commands/power/skill_creator.py +70 -0
  69. opscloud/commands/power/skill_invoke.py +165 -0
  70. opscloud/commands/power/tasks.py +209 -0
  71. opscloud/commands/power/trace.py +117 -0
  72. opscloud/commands/power/ui_toggles.py +135 -0
  73. opscloud/commands/power/version.py +52 -0
  74. opscloud/commands/registry.py +176 -0
  75. opscloud/config/__init__.py +114 -0
  76. opscloud/config/adapters/__init__.py +5 -0
  77. opscloud/config/adapters/postgres.py +776 -0
  78. opscloud/config/adapters/sqlite.py +769 -0
  79. opscloud/config/aws.py +248 -0
  80. opscloud/config/cloud_profiles.py +437 -0
  81. opscloud/config/env_vars.py +81 -0
  82. opscloud/config/langsmith.py +319 -0
  83. opscloud/config/manifest.py +817 -0
  84. opscloud/config/metadata.py +205 -0
  85. opscloud/config/paths.py +604 -0
  86. opscloud/config/plugins.py +130 -0
  87. opscloud/config/settings.py +834 -0
  88. opscloud/config/store.py +453 -0
  89. opscloud/config/store_factory.py +97 -0
  90. opscloud/config/toml_config.py +562 -0
  91. opscloud/editor.py +173 -0
  92. opscloud/exceptions.py +65 -0
  93. opscloud/file_ops.py +217 -0
  94. opscloud/hooks/__init__.py +139 -0
  95. opscloud/hooks/env.py +44 -0
  96. opscloud/hooks/interrupt.py +121 -0
  97. opscloud/hooks/models/__init__.py +109 -0
  98. opscloud/hooks/models/adapters.py +19 -0
  99. opscloud/hooks/models/config.py +79 -0
  100. opscloud/hooks/models/domain.py +434 -0
  101. opscloud/hooks/models/transport.py +37 -0
  102. opscloud/hooks/tools.py +231 -0
  103. opscloud/input.py +48 -0
  104. opscloud/integrations/__init__.py +80 -0
  105. opscloud/integrations/base.py +153 -0
  106. opscloud/integrations/content_filter.py +119 -0
  107. opscloud/integrations/event_bus.py +480 -0
  108. opscloud/integrations/hooks.py +238 -0
  109. opscloud/integrations/identity.py +181 -0
  110. opscloud/integrations/notifications.py +134 -0
  111. opscloud/integrations/stream_bridge.py +397 -0
  112. opscloud/integrations/thread_store.py +81 -0
  113. opscloud/json_types.py +20 -0
  114. opscloud/mcp/__init__.py +57 -0
  115. opscloud/mcp/config.py +203 -0
  116. opscloud/mcp/discovery.py +229 -0
  117. opscloud/mcp/mcp_info.py +109 -0
  118. opscloud/mcp/middleware.py +102 -0
  119. opscloud/mcp/preload.py +505 -0
  120. opscloud/mcp/raw_config.py +149 -0
  121. opscloud/mcp/semantic_profiler.py +305 -0
  122. opscloud/mcp/session_manager.py +727 -0
  123. opscloud/mcp/trust.py +105 -0
  124. opscloud/memory/__init__.py +27 -0
  125. opscloud/memory/branch.py +61 -0
  126. opscloud/memory/guard.py +19 -0
  127. opscloud/memory/onboarding.py +120 -0
  128. opscloud/memory/registry.py +134 -0
  129. opscloud/memory/store.py +223 -0
  130. opscloud/middleware/__init__.py +106 -0
  131. opscloud/middleware/_repository_bounds.py +459 -0
  132. opscloud/middleware/ask_user.py +152 -0
  133. opscloud/middleware/auto_mode.py +60 -0
  134. opscloud/middleware/auto_mode_hitl.py +2434 -0
  135. opscloud/middleware/compaction.py +566 -0
  136. opscloud/middleware/configurable_model.py +233 -0
  137. opscloud/middleware/cost_tracking.py +1021 -0
  138. opscloud/middleware/glm_stall_recovery.py +109 -0
  139. opscloud/middleware/goal_criteria.py +1234 -0
  140. opscloud/middleware/goal_state_notice.py +712 -0
  141. opscloud/middleware/goal_tools.py +407 -0
  142. opscloud/middleware/headless_mcp_guard.py +88 -0
  143. opscloud/middleware/jev_model_router.py +1018 -0
  144. opscloud/middleware/local_context.py +460 -0
  145. opscloud/middleware/mcp_context.py +151 -0
  146. opscloud/middleware/mcp_middleware.py +146 -0
  147. opscloud/middleware/memory_guard.py +393 -0
  148. opscloud/middleware/model_retry.py +205 -0
  149. opscloud/middleware/registry.py +137 -0
  150. opscloud/middleware/reliable_rubric.py +709 -0
  151. opscloud/middleware/resume_state.py +91 -0
  152. opscloud/middleware/server_hooks.py +1189 -0
  153. opscloud/middleware/shell_allow_list.py +90 -0
  154. opscloud/middleware/skills.py +382 -0
  155. opscloud/middleware/subagent_artifacts.py +84 -0
  156. opscloud/middleware/subagent_telemetry.py +261 -0
  157. opscloud/middleware/subagents.py +243 -0
  158. opscloud/middleware/tool_filter.py +384 -0
  159. opscloud/middleware/unified_system_message.py +62 -0
  160. opscloud/model/__init__.py +37 -0
  161. opscloud/model/config.py +2992 -0
  162. opscloud/model/factory.py +477 -0
  163. opscloud/model/pool.py +459 -0
  164. opscloud/model/reasoning.py +282 -0
  165. opscloud/offload.py +173 -0
  166. opscloud/output.py +64 -0
  167. opscloud/plugins/__init__.py +189 -0
  168. opscloud/plugins/_json.py +22 -0
  169. opscloud/plugins/adapters/__init__.py +13 -0
  170. opscloud/plugins/adapters/agents.py +112 -0
  171. opscloud/plugins/adapters/commands.py +190 -0
  172. opscloud/plugins/adapters/mcp.py +363 -0
  173. opscloud/plugins/adapters/skills.py +132 -0
  174. opscloud/plugins/commands_cli.py +541 -0
  175. opscloud/plugins/discovery.py +512 -0
  176. opscloud/plugins/manifest.py +335 -0
  177. opscloud/plugins/marketplace.py +893 -0
  178. opscloud/plugins/models.py +334 -0
  179. opscloud/plugins/project_plugins.py +317 -0
  180. opscloud/plugins/protocol.py +31 -0
  181. opscloud/plugins/store.py +905 -0
  182. opscloud/plugins/substitution.py +122 -0
  183. opscloud/project_utils.py +145 -0
  184. opscloud/prompts/__init__.py +37 -0
  185. opscloud/prompts/system.py +483 -0
  186. opscloud/prompts/templates/system_prompt.md +166 -0
  187. opscloud/prompts/types.py +54 -0
  188. opscloud/rubrics/__init__.py +42 -0
  189. opscloud/rubrics/evaluator.py +404 -0
  190. opscloud/rubrics/evidence_extractor.py +229 -0
  191. opscloud/rubrics/generator.py +239 -0
  192. opscloud/rubrics/jev_compiler.py +175 -0
  193. opscloud/rubrics/jev_grader.py +297 -0
  194. opscloud/rubrics/middleware.py +8 -0
  195. opscloud/schema/__init__.py +107 -0
  196. opscloud/schema/interrupts.py +981 -0
  197. opscloud/security/__init__.py +91 -0
  198. opscloud/security/approval_mode.py +528 -0
  199. opscloud/security/approval_mode_source.py +230 -0
  200. opscloud/security/cli_ast_evaluator.py +552 -0
  201. opscloud/security/jev_classifier.py +651 -0
  202. opscloud/security/shell_safety.py +340 -0
  203. opscloud/security/unicode_security.py +439 -0
  204. opscloud/security/url_validation.py +160 -0
  205. opscloud/server/__init__.py +13 -0
  206. opscloud/server/_server_config.py +158 -0
  207. opscloud/server/server.py +242 -0
  208. opscloud/server/server_graph.py +93 -0
  209. opscloud/skills/__init__.py +91 -0
  210. opscloud/skills/commands.py +1099 -0
  211. opscloud/skills/invocation.py +200 -0
  212. opscloud/skills/load.py +388 -0
  213. opscloud/skills/loader.py +23 -0
  214. opscloud/skills/merge.py +65 -0
  215. opscloud/skills/registry.py +263 -0
  216. opscloud/skills/sources.py +174 -0
  217. opscloud/skills/trust.py +417 -0
  218. opscloud/state/__init__.py +142 -0
  219. opscloud/state/base.py +154 -0
  220. opscloud/state/cloud_context.py +262 -0
  221. opscloud/state/goal_channels.py +124 -0
  222. opscloud/state/goal_state_limits.py +175 -0
  223. opscloud/state/resume_state.py +58 -0
  224. opscloud/state/service_context.py +281 -0
  225. opscloud/state/session.py +957 -0
  226. opscloud/state/state_migration.py +86 -0
  227. opscloud/subagents/__init__.py +30 -0
  228. opscloud/subagents/loader.py +331 -0
  229. opscloud/subagents/subagents_parser.py +117 -0
  230. opscloud/subagents/types.py +25 -0
  231. opscloud/tools/__init__.py +36 -0
  232. opscloud/tools/catalog.py +37 -0
  233. opscloud/tools/display.py +24 -0
  234. opscloud/tools/fetch_url.py +176 -0
  235. opscloud/tools/goal_tools.py +423 -0
  236. opscloud/tools/registry.py +105 -0
  237. opscloud/tools/thread.py +32 -0
  238. opscloud/tools/web_search.py +215 -0
  239. opscloud/ui/__init__.py +119 -0
  240. opscloud/ui/app.py +5400 -0
  241. opscloud/ui/app.tcss +479 -0
  242. opscloud/ui/clipboard.py +147 -0
  243. opscloud/ui/command_registry.py +482 -0
  244. opscloud/ui/message_store.py +152 -0
  245. opscloud/ui/permission_store.py +360 -0
  246. opscloud/ui/preferences.py +113 -0
  247. opscloud/ui/prompt_manager.py +96 -0
  248. opscloud/ui/remote_client.py +505 -0
  249. opscloud/ui/textual_adapter.py +1269 -0
  250. opscloud/ui/theme.py +728 -0
  251. opscloud/ui/ui_help.py +415 -0
  252. opscloud/ui/widgets/__init__.py +383 -0
  253. opscloud/ui/widgets/_ask_user_types.py +81 -0
  254. opscloud/ui/widgets/_copy_spans.py +52 -0
  255. opscloud/ui/widgets/_inline_prompt.py +290 -0
  256. opscloud/ui/widgets/_js_eval_display.py +139 -0
  257. opscloud/ui/widgets/_links.py +175 -0
  258. opscloud/ui/widgets/_tool_stream.py +308 -0
  259. opscloud/ui/widgets/agent_selector.py +297 -0
  260. opscloud/ui/widgets/approval.py +741 -0
  261. opscloud/ui/widgets/ask_user.py +539 -0
  262. opscloud/ui/widgets/auth.py +695 -0
  263. opscloud/ui/widgets/auth_manager.py +214 -0
  264. opscloud/ui/widgets/auto_mode_notice.py +146 -0
  265. opscloud/ui/widgets/autocomplete.py +709 -0
  266. opscloud/ui/widgets/chat_input.py +805 -0
  267. opscloud/ui/widgets/cloud_selector.py +528 -0
  268. opscloud/ui/widgets/config_manager.py +652 -0
  269. opscloud/ui/widgets/debug_console.py +1269 -0
  270. opscloud/ui/widgets/devops_renderers.py +184 -0
  271. opscloud/ui/widgets/diff.py +237 -0
  272. opscloud/ui/widgets/effort_selector.py +166 -0
  273. opscloud/ui/widgets/goal_review.py +612 -0
  274. opscloud/ui/widgets/goal_status.py +57 -0
  275. opscloud/ui/widgets/infra_panel.py +141 -0
  276. opscloud/ui/widgets/install_confirm.py +114 -0
  277. opscloud/ui/widgets/loading.py +170 -0
  278. opscloud/ui/widgets/mcp_viewer.py +1204 -0
  279. opscloud/ui/widgets/messages.py +4461 -0
  280. opscloud/ui/widgets/model_selector.py +863 -0
  281. opscloud/ui/widgets/notification_center.py +108 -0
  282. opscloud/ui/widgets/notification_settings.py +119 -0
  283. opscloud/ui/widgets/operation_card.py +106 -0
  284. opscloud/ui/widgets/permissions_manager.py +667 -0
  285. opscloud/ui/widgets/plugin_manager.py +1212 -0
  286. opscloud/ui/widgets/pool_selector.py +376 -0
  287. opscloud/ui/widgets/preamble.py +53 -0
  288. opscloud/ui/widgets/skills_viewer.py +432 -0
  289. opscloud/ui/widgets/status.py +1037 -0
  290. opscloud/ui/widgets/subagent_panel.py +927 -0
  291. opscloud/ui/widgets/theme_selector.py +232 -0
  292. opscloud/ui/widgets/thread_selector.py +358 -0
  293. opscloud/ui/widgets/toast.py +31 -0
  294. opscloud/ui/widgets/tool_display.py +325 -0
  295. opscloud/ui/widgets/tool_renderers.py +234 -0
  296. opscloud/ui/widgets/tool_widgets.py +278 -0
  297. opscloud/ui/widgets/welcome.py +421 -0
  298. opscloud/ui/widgets/welcome_popup.py +102 -0
  299. opscloud/utils/__init__.py +24 -0
  300. opscloud/utils/cost_estimation.py +169 -0
  301. opscloud/utils/git.py +390 -0
  302. opscloud/utils/logger.py +571 -0
  303. opscloud/utils/session_stats.py +115 -0
  304. opscloud/utils/startup_error.py +37 -0
  305. talkops_opscloud-0.3.0b1.dist-info/METADATA +449 -0
  306. talkops_opscloud-0.3.0b1.dist-info/RECORD +309 -0
  307. talkops_opscloud-0.3.0b1.dist-info/WHEEL +4 -0
  308. talkops_opscloud-0.3.0b1.dist-info/entry_points.txt +2 -0
  309. talkops_opscloud-0.3.0b1.dist-info/licenses/LICENSE +175 -0
@@ -0,0 +1,709 @@
1
+ """Rubric middleware with transport retry, event streaming, and isolated grading.
2
+
3
+ ``ReliableRubricMiddleware`` extends the SDK's ``RubricMiddleware`` with:
4
+
5
+ - **Isolated Grader Agent**: Creates a dedicated evaluator subagent configured
6
+ with read-only evidence tools, budget guards, and custom system prompts.
7
+ - **Transport Resilience**: Catches transient HTTP read and remote protocol errors
8
+ (e.g., during long grading responses) and retries the grader invocation once
9
+ without replaying worker operations.
10
+ - **Control Message Sanitization**: Strips internal goal-state notices, continuation
11
+ pings, and past evaluation metadata so the grader evaluates only genuine work.
12
+ - **Streaming Telemetry**: Emits ``rubric_evaluation_start`` and ``rubric_evaluation_end``
13
+ lifecycle events via ``runtime.stream_writer`` for real-time terminal display.
14
+ """
15
+
16
+ from __future__ import annotations
17
+
18
+ from collections.abc import Iterator, Mapping, Sequence
19
+ from contextlib import contextmanager
20
+ from contextvars import ContextVar
21
+ from dataclasses import replace
22
+ from typing import (
23
+ TYPE_CHECKING,
24
+ Annotated,
25
+ Any,
26
+ Callable,
27
+ NotRequired,
28
+ cast,
29
+ )
30
+ import warnings
31
+
32
+ import httpx
33
+ from deepagents.middleware.rubric import (
34
+ RUBRIC_GRADER_MESSAGE_SOURCE,
35
+ GraderResponse,
36
+ RubricMiddleware as BaseRubricMiddleware,
37
+ RubricState,
38
+ )
39
+ import deepagents.middleware.rubric as _rubric_mod
40
+ from langchain.agents.middleware.types import (
41
+ AgentMiddleware,
42
+ AgentState,
43
+ PrivateStateAttr,
44
+ hook_config,
45
+ )
46
+ from langchain_core.messages import AnyMessage, HumanMessage
47
+ from langgraph.errors import GraphBubbleUp
48
+
49
+ from opscloud.middleware.goal_state_notice import is_conversation_control_message
50
+ from opscloud.middleware.registry import register_middleware
51
+ from opscloud.utils.logger import get_logger
52
+
53
+ if TYPE_CHECKING:
54
+ from deepagents.middleware.rubric import RubricEvaluation
55
+ from langchain_core.language_models import BaseChatModel
56
+ from langchain_core.tools import BaseTool
57
+ from langgraph.runtime import Runtime
58
+
59
+ logger = get_logger(__name__)
60
+
61
+ _strategy_from_result = getattr(_rubric_mod, "_strategy_from_result", None)
62
+
63
+ __all__ = ["ReliableRubricMiddleware", "ReliableRubricState", "RubricGraderState", "RubricMiddleware"]
64
+
65
+
66
+ # ---------------------------------------------------------------------------
67
+ # Transport Error Recovery Helpers
68
+ # ---------------------------------------------------------------------------
69
+
70
+
71
+ def _exception_chain(exc: BaseException) -> Iterator[BaseException]:
72
+ """Yield an exception, its causes, context, and group members without cycles."""
73
+ pending = [exc]
74
+ seen: set[int] = set()
75
+ while pending:
76
+ current = pending.pop()
77
+ curr_id = id(current)
78
+ if curr_id in seen:
79
+ continue
80
+ seen.add(curr_id)
81
+ yield current
82
+ if isinstance(current, BaseExceptionGroup):
83
+ pending.extend(current.exceptions)
84
+ if current.__cause__ is not None:
85
+ pending.append(current.__cause__)
86
+ elif current.__context__ is not None:
87
+ pending.append(current.__context__)
88
+
89
+
90
+ def _is_transient_grader_transport_error(exc: BaseException) -> bool:
91
+ """Determine whether an evaluation exception is a retryable network drop."""
92
+ for current in _exception_chain(exc):
93
+ if isinstance(
94
+ current,
95
+ (
96
+ httpx.ReadError,
97
+ httpx.RemoteProtocolError,
98
+ httpx.TimeoutException,
99
+ httpx.ConnectError,
100
+ ConnectionError,
101
+ TimeoutError,
102
+ ),
103
+ ):
104
+ return True
105
+ mod_name = type(current).__module__
106
+ cls_name = type(current).__name__
107
+ if mod_name.startswith("httpcore") and cls_name in {
108
+ "ReadError",
109
+ "RemoteProtocolError",
110
+ "ConnectError",
111
+ "TimeoutException",
112
+ }:
113
+ return True
114
+ if (
115
+ mod_name == "aiohttp.http_exceptions"
116
+ and cls_name == "TransferEncodingError"
117
+ and "Not enough data" in str(current)
118
+ ):
119
+ return True
120
+ err_msg = str(current)
121
+ if any(code in err_msg for code in ("429", "503", "Connection reset by peer")):
122
+ return True
123
+ return False
124
+
125
+
126
+ # ---------------------------------------------------------------------------
127
+ # Message History Sanitization
128
+ # ---------------------------------------------------------------------------
129
+
130
+
131
+ def _without_internal_control_messages(state: RubricState) -> RubricState:
132
+ """Filter out internal goal-state notices so the grader evaluates only real work."""
133
+ messages = state.get("messages", [])
134
+ if not isinstance(messages, list):
135
+ return state
136
+ filtered: list[AnyMessage] = [
137
+ msg for msg in messages if not is_conversation_control_message(msg)
138
+ ]
139
+ if len(filtered) == len(messages):
140
+ return state
141
+ updated = dict(state)
142
+ updated["messages"] = filtered
143
+ return cast("RubricState", updated)
144
+
145
+
146
+ # ---------------------------------------------------------------------------
147
+ # State Schemas
148
+ # ---------------------------------------------------------------------------
149
+
150
+
151
+ class ReliableRubricState(RubricState):
152
+ """Rubric state carrying OpsCloud's private runtime model selections."""
153
+
154
+ _model_spec: Annotated[NotRequired[str], PrivateStateAttr]
155
+ """Active chat model specifier (written by ConfigurableModelMiddleware)."""
156
+
157
+ _model_params: Annotated[NotRequired[dict[str, Any] | None], PrivateStateAttr]
158
+ """Model hyperparameters belonging to _model_spec."""
159
+
160
+ _rubric_model_spec: Annotated[NotRequired[str], PrivateStateAttr]
161
+ """Thread-scoped grader model override selected by the operator."""
162
+
163
+
164
+ class RubricGraderState(AgentState[GraderResponse]):
165
+ """Private state for the isolated nested grader agent."""
166
+
167
+ rubric_grading_operation_id: NotRequired[str]
168
+
169
+
170
+ def _is_smart_mode(context: object = None, store: object = None) -> bool:
171
+ """Synchronously determine whether Smart mode is active."""
172
+ try:
173
+ from opscloud.middleware.jev_model_router import _is_smart_mode_sync
174
+
175
+ return _is_smart_mode_sync(context, store=store)
176
+ except Exception:
177
+ return False
178
+
179
+
180
+ async def _ais_smart_mode(context: object = None, store: object = None) -> bool:
181
+ """Asynchronously determine whether Smart mode is active."""
182
+ try:
183
+ from opscloud.middleware.jev_model_router import _ais_smart_mode as _async_smart
184
+
185
+ return await _async_smart(context, store=store)
186
+ except Exception:
187
+ return False
188
+
189
+
190
+ # ---------------------------------------------------------------------------
191
+ # Middleware Implementation
192
+ # ---------------------------------------------------------------------------
193
+
194
+
195
+ with warnings.catch_warnings():
196
+ warnings.filterwarnings(
197
+ "ignore",
198
+ message="The middleware `RubricMiddleware` is in beta",
199
+ category=Warning,
200
+ )
201
+
202
+ @register_middleware(name="reliable_rubric")
203
+ class ReliableRubricMiddleware(BaseRubricMiddleware):
204
+ """Reliable, isolated rubric grader with transport retry and telemetry streaming."""
205
+
206
+ state_schema = ReliableRubricState
207
+
208
+ def __init__(
209
+ self,
210
+ *,
211
+ model: str | BaseChatModel | None = None,
212
+ system_prompt: str | None = None,
213
+ tools: Sequence[BaseTool] | None = None,
214
+ grader_middleware: Sequence[AgentMiddleware[Any, Any]] | None = None,
215
+ grader_context_schema: type[Any] | None = None,
216
+ max_iterations: int = 3,
217
+ on_evaluation: Callable[[RubricEvaluation], None] | None = None,
218
+ inherit_main_model: bool = False,
219
+ runtime_bootstrap_model: str | BaseChatModel | None = None,
220
+ ) -> None:
221
+ kwargs: dict[str, Any] = {
222
+ "model": model or "anthropic:claude-3-5-haiku-20241022",
223
+ "max_iterations": max_iterations,
224
+ }
225
+ if system_prompt is not None:
226
+ kwargs["system_prompt"] = system_prompt
227
+ if tools is not None:
228
+ kwargs["tools"] = tools
229
+ if on_evaluation is not None:
230
+ kwargs["on_evaluation"] = on_evaluation
231
+
232
+ super().__init__(**kwargs)
233
+ self._grader_model_spec = model or "anthropic:claude-3-5-haiku-20241022"
234
+ self._grader_middleware = list(grader_middleware or ())
235
+ self._grader_context_schema = grader_context_schema
236
+ self._inherit_main_model = inherit_main_model
237
+ self._runtime_bootstrap_model = runtime_bootstrap_model
238
+ self._runtime_grader_model: ContextVar[str | None] = ContextVar(
239
+ "runtime_grader_model",
240
+ default=None,
241
+ )
242
+ self._jev_grader: Any | None = None
243
+
244
+ def _get_jev_grader(self) -> Any:
245
+ """Lazily instantiate JevHybridRubricGrader when needed."""
246
+ if self._jev_grader is None:
247
+ from opscloud.rubrics.jev_grader import JevHybridRubricGrader
248
+
249
+ self._jev_grader = JevHybridRubricGrader()
250
+ return self._jev_grader
251
+
252
+ @contextmanager
253
+ def _runtime_grader_trace(self, model_label: str | None) -> Iterator[None]:
254
+ token = self._runtime_grader_model.set(model_label)
255
+ try:
256
+ yield
257
+ finally:
258
+ self._runtime_grader_model.reset(token)
259
+
260
+ def _ensure_grader(self) -> Any:
261
+ """Instantiate the isolated rubric grader agent graph."""
262
+ if self._grader is not None:
263
+ return self._grader
264
+
265
+ from deepagents._models import resolve_model # noqa: PLC2701
266
+ from langchain.agents import create_agent
267
+
268
+ resolved_model = resolve_model(self._model)
269
+ self._resolved_model = resolved_model
270
+
271
+ logger.debug(
272
+ "ReliableRubric._ensure_grader: building isolated grader agent with middleware: %s",
273
+ [getattr(m, "name", type(m).__name__) for m in self._grader_middleware],
274
+ )
275
+
276
+ self._grader = create_agent(
277
+ model=resolved_model,
278
+ system_prompt=self._system_prompt,
279
+ tools=self._tools,
280
+ middleware=self._grader_middleware,
281
+ name=RUBRIC_GRADER_MESSAGE_SOURCE,
282
+ response_format=GraderResponse,
283
+ state_schema=RubricGraderState,
284
+ context_schema=self._grader_context_schema,
285
+ )
286
+ return self._grader
287
+
288
+ def _emit_stream_event(self, runtime: Runtime[Any], payload: dict[str, Any]) -> None:
289
+ """Safely emit an evaluation event to the live stream writer."""
290
+ writer = getattr(runtime, "stream_writer", None)
291
+ if writer is not None and callable(writer):
292
+ try:
293
+ writer(payload)
294
+ except Exception:
295
+ logger.debug("Failed to emit rubric stream event %s", payload, exc_info=True)
296
+
297
+ def _grader_input(
298
+ self,
299
+ state: RubricState,
300
+ iteration: int,
301
+ correction: str | None = None,
302
+ ) -> dict[str, Any]:
303
+ grading_run_id = state.get("_current_grading_run_id") or "untracked"
304
+ sanitized_state = _without_internal_control_messages(state)
305
+ payload = self._build_grader_payload(sanitized_state, iteration, correction)
306
+ return {
307
+ "messages": [HumanMessage(content=payload)],
308
+ "rubric_grading_operation_id": f"{grading_run_id}:{iteration}",
309
+ }
310
+
311
+ def _grade_once(
312
+ self,
313
+ state: RubricState,
314
+ iteration: int,
315
+ correction: str | None = None,
316
+ *,
317
+ context: object | None = None,
318
+ ) -> GraderResponse:
319
+ grader = self._ensure_grader()
320
+ meta_fn = getattr(self, "_grader_trace_metadata", lambda **kw: {})
321
+ rec_fn = getattr(self, "_record_grader_trace_metadata", lambda m: None)
322
+ cfg_fn = getattr(self, "_grader_invocation_config", lambda m: {})
323
+
324
+ metadata = meta_fn()
325
+ rec_fn(metadata)
326
+ result = grader.invoke(
327
+ self._grader_input(state, iteration, correction),
328
+ config=cfg_fn(metadata),
329
+ context=context,
330
+ )
331
+ strategy = _strategy_from_result(result) if _strategy_from_result is not None else None
332
+ rec_fn(meta_fn(effective_strategy=strategy))
333
+ return self._extract_graded(result)
334
+
335
+ async def _agrade_once(
336
+ self,
337
+ state: RubricState,
338
+ iteration: int,
339
+ correction: str | None = None,
340
+ *,
341
+ context: object | None = None,
342
+ ) -> GraderResponse:
343
+ grader = self._ensure_grader()
344
+ meta_fn = getattr(self, "_grader_trace_metadata", lambda **kw: {})
345
+ rec_fn = getattr(self, "_record_grader_trace_metadata", lambda m: None)
346
+ cfg_fn = getattr(self, "_grader_invocation_config", lambda m: {})
347
+
348
+ metadata = meta_fn()
349
+ rec_fn(metadata)
350
+ result = await grader.ainvoke(
351
+ self._grader_input(state, iteration, correction),
352
+ config=cfg_fn(metadata),
353
+ context=context,
354
+ )
355
+ strategy = _strategy_from_result(result) if _strategy_from_result is not None else None
356
+ rec_fn(meta_fn(effective_strategy=strategy))
357
+ return self._extract_graded(result)
358
+
359
+ @staticmethod
360
+ def _context(context: object | None) -> Any:
361
+ """Copy the parent runtime context into the nested grader schema."""
362
+ from opscloud.agent.config import CLIContextSchema
363
+
364
+ inner = getattr(context, "context", None) if not isinstance(context, (CLIContextSchema, dict)) else context
365
+ target = inner if inner is not None else context
366
+
367
+ if isinstance(target, CLIContextSchema):
368
+ return replace(
369
+ target,
370
+ model_params=dict(target.model_params),
371
+ profile_overrides=dict(target.profile_overrides),
372
+ hooks_server_events=list(target.hooks_server_events),
373
+ )
374
+ if isinstance(target, dict):
375
+ parsed = CLIContextSchema.from_payload(target)
376
+ if parsed is not None:
377
+ return parsed
378
+ return CLIContextSchema()
379
+
380
+ def _grader_context(
381
+ self, state: ReliableRubricState, context: object | None
382
+ ) -> Any:
383
+ """Select the effective grader model without mutating shared middleware."""
384
+ from opscloud.middleware.resume_state import INHERIT_RUBRIC_MODEL, coerce_model_spec
385
+
386
+ grader_context = self._context(context)
387
+ selected = coerce_model_spec(state.get("_rubric_model_spec"))
388
+ inherit = selected == INHERIT_RUBRIC_MODEL or (
389
+ selected is None and self._inherit_main_model
390
+ )
391
+ if inherit:
392
+ main_model = coerce_model_spec(state.get("_model_spec"))
393
+ if main_model is not None and hasattr(grader_context, "model"):
394
+ grader_context.model = main_model
395
+ params = state.get("_model_params")
396
+ if isinstance(params, Mapping) and hasattr(grader_context, "model_params"):
397
+ grader_context.model_params = dict(params)
398
+ else:
399
+ if hasattr(grader_context, "model"):
400
+ grader_context.model = selected
401
+ if hasattr(grader_context, "model_params"):
402
+ grader_context.model_params = {}
403
+ return grader_context
404
+
405
+ def _invoke_grader(
406
+ self,
407
+ state: RubricState,
408
+ iteration: int,
409
+ correction: str | None = None,
410
+ *,
411
+ context: object | None = None,
412
+ ) -> GraderResponse:
413
+ """Invoke the isolated grader with Smart-mode Jev fast-pass and LLM fallback."""
414
+ reliable_state = cast("ReliableRubricState", state)
415
+ grader_context = self._grader_context(reliable_state, context)
416
+
417
+ # ── Smart Mode Path (Jev System One Fast-Pass) ──
418
+ jev_grader = self._get_jev_grader()
419
+ raw_rubric = state.get("rubric") or state.get("_goal_rubric") or state.get("_sticky_rubric") or ""
420
+ store = getattr(context, "store", None)
421
+ is_smart = _is_smart_mode(context, store=store) or _is_smart_mode(grader_context)
422
+ if is_smart and jev_grader.is_available() and raw_rubric:
423
+ try:
424
+ from opscloud.rubrics.jev_compiler import JevCriteriaCompiler
425
+
426
+ objective = str(state.get("_goal_objective") or state.get("goal") or "Fulfill user requirements")
427
+ compiled = JevCriteriaCompiler.compile(objective, str(raw_rubric))
428
+ logger.info("ReliableRubricMiddleware: Smart mode active — evaluating criteria via Jev System One")
429
+ def _targeted_diagnostic(failing: list[str]) -> GraderResponse:
430
+ """Generate concise diagnostic remediation feedback in a single 1-shot LLM completion without tools."""
431
+ failing_text = "\n".join(f"- {item}" for item in failing)
432
+ try:
433
+ from deepagents._models import resolve_model
434
+ from langchain_core.messages import HumanMessage, SystemMessage
435
+
436
+ model_spec = getattr(grader_context, "model", None) or getattr(self, "_grader_model_spec", None) or "anthropic:claude-3-5-haiku-20241022"
437
+ chat_model = resolve_model(model_spec) if isinstance(model_spec, str) else model_spec
438
+
439
+ diagnostic_prompt = [
440
+ SystemMessage(
441
+ content=(
442
+ "You are the OpsCloud Rubric Diagnostic Evaluator. Acceptance criteria evaluation failed. "
443
+ "Write concise, actionable remediation advice for what specific files, configurations, "
444
+ "or commands the agent must execute to satisfy the failing criteria. Do not repeat all criteria."
445
+ )
446
+ ),
447
+ HumanMessage(
448
+ content=(
449
+ f"Objective: {objective}\n\n"
450
+ f"Unsatisfied criteria:\n{failing_text}\n\n"
451
+ "Provide 1-2 concise bullet points explaining exactly what is missing and how to fix it."
452
+ )
453
+ ),
454
+ ]
455
+ res = chat_model.invoke(diagnostic_prompt)
456
+ advice = getattr(res, "content", "") or str(res)
457
+ explanation = (
458
+ f"{len(failing)} of {len(compiled.criteria_map)} criteria not yet satisfied.\n\n"
459
+ f"Remediation Guidance:\n{advice}"
460
+ )
461
+ except Exception as diag_exc:
462
+ logger.warning("Tier 2 diagnostic prompt failed (%s); using default explanation", diag_exc)
463
+ explanation = f"{len(failing)} criteria not yet satisfied: {', '.join(failing)}"
464
+
465
+ return GraderResponse(
466
+ result="needs_revision",
467
+ explanation=explanation,
468
+ criteria=[],
469
+ )
470
+
471
+ return jev_grader.grade(
472
+ compiled,
473
+ evidence=state,
474
+ fallback_llm_fn=_targeted_diagnostic,
475
+ )
476
+ except Exception as jev_exc:
477
+ logger.warning("Jev rubric evaluation failed (%s); falling back to standard LLM grader", jev_exc)
478
+
479
+ # ── Standard Path (Manual, Auto, or non-Jev) ──
480
+ model_label = getattr(grader_context, "model", None)
481
+ with self._runtime_grader_trace(model_label):
482
+ try:
483
+ return self._grade_once(state, iteration, correction, context=grader_context)
484
+ except Exception as exc:
485
+ if not _is_transient_grader_transport_error(exc):
486
+ raise
487
+ logger.warning("Rubric grader transient error; retrying grading invocation once: %s", exc)
488
+ return self._grade_once(state, iteration, correction, context=grader_context)
489
+
490
+ async def _ainvoke_grader(
491
+ self,
492
+ state: RubricState,
493
+ iteration: int,
494
+ correction: str | None = None,
495
+ *,
496
+ context: object | None = None,
497
+ ) -> GraderResponse:
498
+ """Invoke the isolated grader asynchronously with Smart-mode Jev fast-pass and LLM fallback."""
499
+ reliable_state = cast("ReliableRubricState", state)
500
+ grader_context = self._grader_context(reliable_state, context)
501
+
502
+ # ── Smart Mode Path (Jev System One Fast-Pass) ──
503
+ jev_grader = self._get_jev_grader()
504
+ raw_rubric = state.get("rubric") or state.get("_goal_rubric") or state.get("_sticky_rubric") or ""
505
+ store = getattr(context, "store", None)
506
+ is_smart = (await _ais_smart_mode(context, store=store)) or (await _ais_smart_mode(grader_context))
507
+ if is_smart and jev_grader.is_available() and raw_rubric:
508
+ try:
509
+ from opscloud.rubrics.jev_compiler import JevCriteriaCompiler
510
+
511
+ objective = str(state.get("_goal_objective") or state.get("goal") or "Fulfill user requirements")
512
+ compiled = JevCriteriaCompiler.compile(objective, str(raw_rubric))
513
+ logger.info("ReliableRubricMiddleware: Smart mode active — evaluating criteria via Jev System One")
514
+
515
+ async def _atargeted_diagnostic(failing: list[str]) -> GraderResponse:
516
+ """Generate concise diagnostic remediation feedback in a single 1-shot LLM completion without tools."""
517
+ failing_text = "\n".join(f"- {item}" for item in failing)
518
+ try:
519
+ from deepagents._models import resolve_model
520
+ from langchain_core.messages import HumanMessage, SystemMessage
521
+
522
+ model_spec = getattr(grader_context, "model", None) or getattr(self, "_grader_model_spec", None) or "anthropic:claude-3-5-haiku-20241022"
523
+ chat_model = resolve_model(model_spec) if isinstance(model_spec, str) else model_spec
524
+
525
+ diagnostic_prompt = [
526
+ SystemMessage(
527
+ content=(
528
+ "You are the OpsCloud Rubric Diagnostic Evaluator. Acceptance criteria evaluation failed. "
529
+ "Write concise, actionable remediation advice for what specific files, configurations, "
530
+ "or commands the agent must execute to satisfy the failing criteria. Do not repeat all criteria."
531
+ )
532
+ ),
533
+ HumanMessage(
534
+ content=(
535
+ f"Objective: {objective}\n\n"
536
+ f"Unsatisfied criteria:\n{failing_text}\n\n"
537
+ "Provide 1-2 concise bullet points explaining exactly what is missing and how to fix it."
538
+ )
539
+ ),
540
+ ]
541
+ res = await chat_model.ainvoke(diagnostic_prompt)
542
+ advice = getattr(res, "content", "") or str(res)
543
+ explanation = (
544
+ f"{len(failing)} of {len(compiled.criteria_map)} criteria not yet satisfied.\n\n"
545
+ f"Remediation Guidance:\n{advice}"
546
+ )
547
+ except Exception as diag_exc:
548
+ logger.warning("Tier 2 diagnostic prompt failed (%s); using default explanation", diag_exc)
549
+ explanation = f"{len(failing)} criteria not yet satisfied: {', '.join(failing)}"
550
+
551
+ return GraderResponse(
552
+ result="needs_revision",
553
+ explanation=explanation,
554
+ criteria=[],
555
+ )
556
+
557
+ return await jev_grader.agrade(
558
+ compiled,
559
+ evidence=state,
560
+ fallback_llm_fn=_atargeted_diagnostic,
561
+ )
562
+ except Exception as jev_exc:
563
+ logger.warning("Jev async rubric evaluation failed (%s); falling back to standard LLM grader", jev_exc)
564
+
565
+ # ── Standard Path (Manual, Auto, or non-Jev) ──
566
+ model_label = getattr(grader_context, "model", None)
567
+ with self._runtime_grader_trace(model_label):
568
+ try:
569
+ return await self._agrade_once(state, iteration, correction, context=grader_context)
570
+ except Exception as exc:
571
+ if not _is_transient_grader_transport_error(exc):
572
+ raise
573
+ logger.warning("Rubric grader transient async error; retrying grading invocation once: %s", exc)
574
+ return await self._agrade_once(state, iteration, correction, context=grader_context)
575
+
576
+ def _grade(
577
+ self,
578
+ state: RubricState,
579
+ iteration: int,
580
+ *,
581
+ context: object | None = None,
582
+ ) -> GraderResponse:
583
+ return super()._grade(state, iteration, context=context)
584
+
585
+ async def _agrade(
586
+ self,
587
+ state: RubricState,
588
+ iteration: int,
589
+ *,
590
+ context: object | None = None,
591
+ ) -> GraderResponse:
592
+ return await super()._agrade(state, iteration, context=context)
593
+
594
+ def _finalize_evaluation(
595
+ self,
596
+ graded: GraderResponse,
597
+ state: RubricState,
598
+ runtime: Runtime[Any],
599
+ grading_run_id: str,
600
+ iteration: int,
601
+ ) -> dict[str, Any]:
602
+ update = super()._finalize_evaluation(
603
+ graded, state, runtime, grading_run_id, iteration
604
+ )
605
+ # If Jev classified an environmental blocker or ambiguous requirement, mark goal as blocked
606
+ blocker = getattr(graded, "blocker_type", None)
607
+ explanation = getattr(graded, "explanation", "") or getattr(graded, "feedback", "") or ""
608
+ if blocker in {"external_blocker", "ambiguous_requirement"} or "Environmental blocker detected" in str(explanation):
609
+ update["_goal_status"] = "blocked"
610
+ update["_goal_status_note"] = str(explanation)
611
+ return update
612
+
613
+ @hook_config(can_jump_to=["model"])
614
+ def after_agent(
615
+ self,
616
+ state: RubricState,
617
+ runtime: Runtime[Any],
618
+ ) -> dict[str, Any] | None:
619
+ prep = self._prepare_evaluation(state, runtime)
620
+ if prep is None:
621
+ return None
622
+ grading_run_id, iteration = prep
623
+
624
+ try:
625
+ graded = self._grade(
626
+ state,
627
+ iteration,
628
+ context=runtime,
629
+ )
630
+ except GraphBubbleUp:
631
+ raise
632
+ except Exception as exc:
633
+ return self._handle_grader_exception(
634
+ runtime,
635
+ state,
636
+ grading_run_id,
637
+ iteration,
638
+ exc,
639
+ )
640
+
641
+ feedback = getattr(graded, "feedback", None) or getattr(graded, "explanation", "")
642
+
643
+ if "Environmental blocker detected" in str(feedback):
644
+ self._emit_stream_event(
645
+ runtime,
646
+ {
647
+ "type": "rubric_blocker_detected",
648
+ "iteration": iteration,
649
+ "explanation": str(feedback),
650
+ },
651
+ )
652
+
653
+ return self._finalize_evaluation(
654
+ graded,
655
+ state,
656
+ runtime,
657
+ grading_run_id,
658
+ iteration,
659
+ )
660
+
661
+ async def aafter_agent(
662
+ self,
663
+ state: RubricState,
664
+ runtime: Runtime[Any],
665
+ ) -> dict[str, Any] | None:
666
+ prep = self._prepare_evaluation(state, runtime)
667
+ if prep is None:
668
+ return None
669
+ grading_run_id, iteration = prep
670
+
671
+ try:
672
+ graded = await self._agrade(
673
+ state,
674
+ iteration,
675
+ context=runtime,
676
+ )
677
+ except GraphBubbleUp:
678
+ raise
679
+ except Exception as exc:
680
+ return self._handle_grader_exception(
681
+ runtime,
682
+ state,
683
+ grading_run_id,
684
+ iteration,
685
+ exc,
686
+ )
687
+
688
+ feedback = getattr(graded, "feedback", None) or getattr(graded, "explanation", "")
689
+
690
+ if "Environmental blocker detected" in str(feedback):
691
+ self._emit_stream_event(
692
+ runtime,
693
+ {
694
+ "type": "rubric_blocker_detected",
695
+ "iteration": iteration,
696
+ "explanation": str(feedback),
697
+ },
698
+ )
699
+
700
+ return self._finalize_evaluation(
701
+ graded,
702
+ state,
703
+ runtime,
704
+ grading_run_id,
705
+ iteration,
706
+ )
707
+
708
+
709
+ RubricMiddleware = ReliableRubricMiddleware