talkops-opscloud 0.3.0b1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (309) hide show
  1. opscloud/__init__.py +5 -0
  2. opscloud/__main__.py +6 -0
  3. opscloud/_constants.py +53 -0
  4. opscloud/_debug.py +145 -0
  5. opscloud/_debug_buffer.py +250 -0
  6. opscloud/_textual_patches.py +367 -0
  7. opscloud/_version.py +15 -0
  8. opscloud/agent/__init__.py +27 -0
  9. opscloud/agent/config.py +168 -0
  10. opscloud/agent/factory.py +1103 -0
  11. opscloud/approval_mode.py +44 -0
  12. opscloud/backend/__init__.py +49 -0
  13. opscloud/backend/composite.py +89 -0
  14. opscloud/backend/local.py +183 -0
  15. opscloud/backend/registry.py +79 -0
  16. opscloud/backend/sandbox_config.py +107 -0
  17. opscloud/backend/sandbox_factory.py +432 -0
  18. opscloud/built_in_skills/remember/SKILL.md +124 -0
  19. opscloud/built_in_skills/skill-creator/SKILL.md +348 -0
  20. opscloud/built_in_skills/skill-creator/scripts/init_skill.py +371 -0
  21. opscloud/built_in_skills/skill-creator/scripts/quick_validate.py +165 -0
  22. opscloud/cli/__init__.py +1 -0
  23. opscloud/cli/commands/__init__.py +15 -0
  24. opscloud/cli/commands/auth.py +45 -0
  25. opscloud/cli/commands/doctor.py +100 -0
  26. opscloud/cli/commands/mcp.py +137 -0
  27. opscloud/cli/commands/threads.py +51 -0
  28. opscloud/cli/interactive.py +154 -0
  29. opscloud/cli/main.py +903 -0
  30. opscloud/cli/non_interactive.py +425 -0
  31. opscloud/cli/server_manager.py +58 -0
  32. opscloud/commands/__init__.py +19 -0
  33. opscloud/commands/_base.py +110 -0
  34. opscloud/commands/_guards.py +52 -0
  35. opscloud/commands/_router.py +146 -0
  36. opscloud/commands/_types.py +36 -0
  37. opscloud/commands/core/__init__.py +47 -0
  38. opscloud/commands/core/auth.py +94 -0
  39. opscloud/commands/core/bug.py +46 -0
  40. opscloud/commands/core/clear.py +106 -0
  41. opscloud/commands/core/cloud.py +185 -0
  42. opscloud/commands/core/compact.py +128 -0
  43. opscloud/commands/core/config_cmd.py +120 -0
  44. opscloud/commands/core/context.py +220 -0
  45. opscloud/commands/core/cost.py +44 -0
  46. opscloud/commands/core/doctor.py +535 -0
  47. opscloud/commands/core/effort.py +108 -0
  48. opscloud/commands/core/exit_cmd.py +41 -0
  49. opscloud/commands/core/fast.py +148 -0
  50. opscloud/commands/core/help_cmd.py +94 -0
  51. opscloud/commands/core/mcp.py +134 -0
  52. opscloud/commands/core/model.py +109 -0
  53. opscloud/commands/core/permissions.py +196 -0
  54. opscloud/commands/core/plugins.py +92 -0
  55. opscloud/commands/core/pool.py +108 -0
  56. opscloud/commands/core/resume.py +97 -0
  57. opscloud/commands/core/skills.py +178 -0
  58. opscloud/commands/power/__init__.py +51 -0
  59. opscloud/commands/power/agents.py +98 -0
  60. opscloud/commands/power/btw.py +85 -0
  61. opscloud/commands/power/copy.py +97 -0
  62. opscloud/commands/power/goal.py +548 -0
  63. opscloud/commands/power/loop.py +295 -0
  64. opscloud/commands/power/memory.py +287 -0
  65. opscloud/commands/power/review.py +152 -0
  66. opscloud/commands/power/rubric.py +287 -0
  67. opscloud/commands/power/runtime.py +321 -0
  68. opscloud/commands/power/skill_creator.py +70 -0
  69. opscloud/commands/power/skill_invoke.py +165 -0
  70. opscloud/commands/power/tasks.py +209 -0
  71. opscloud/commands/power/trace.py +117 -0
  72. opscloud/commands/power/ui_toggles.py +135 -0
  73. opscloud/commands/power/version.py +52 -0
  74. opscloud/commands/registry.py +176 -0
  75. opscloud/config/__init__.py +114 -0
  76. opscloud/config/adapters/__init__.py +5 -0
  77. opscloud/config/adapters/postgres.py +776 -0
  78. opscloud/config/adapters/sqlite.py +769 -0
  79. opscloud/config/aws.py +248 -0
  80. opscloud/config/cloud_profiles.py +437 -0
  81. opscloud/config/env_vars.py +81 -0
  82. opscloud/config/langsmith.py +319 -0
  83. opscloud/config/manifest.py +817 -0
  84. opscloud/config/metadata.py +205 -0
  85. opscloud/config/paths.py +604 -0
  86. opscloud/config/plugins.py +130 -0
  87. opscloud/config/settings.py +834 -0
  88. opscloud/config/store.py +453 -0
  89. opscloud/config/store_factory.py +97 -0
  90. opscloud/config/toml_config.py +562 -0
  91. opscloud/editor.py +173 -0
  92. opscloud/exceptions.py +65 -0
  93. opscloud/file_ops.py +217 -0
  94. opscloud/hooks/__init__.py +139 -0
  95. opscloud/hooks/env.py +44 -0
  96. opscloud/hooks/interrupt.py +121 -0
  97. opscloud/hooks/models/__init__.py +109 -0
  98. opscloud/hooks/models/adapters.py +19 -0
  99. opscloud/hooks/models/config.py +79 -0
  100. opscloud/hooks/models/domain.py +434 -0
  101. opscloud/hooks/models/transport.py +37 -0
  102. opscloud/hooks/tools.py +231 -0
  103. opscloud/input.py +48 -0
  104. opscloud/integrations/__init__.py +80 -0
  105. opscloud/integrations/base.py +153 -0
  106. opscloud/integrations/content_filter.py +119 -0
  107. opscloud/integrations/event_bus.py +480 -0
  108. opscloud/integrations/hooks.py +238 -0
  109. opscloud/integrations/identity.py +181 -0
  110. opscloud/integrations/notifications.py +134 -0
  111. opscloud/integrations/stream_bridge.py +397 -0
  112. opscloud/integrations/thread_store.py +81 -0
  113. opscloud/json_types.py +20 -0
  114. opscloud/mcp/__init__.py +57 -0
  115. opscloud/mcp/config.py +203 -0
  116. opscloud/mcp/discovery.py +229 -0
  117. opscloud/mcp/mcp_info.py +109 -0
  118. opscloud/mcp/middleware.py +102 -0
  119. opscloud/mcp/preload.py +505 -0
  120. opscloud/mcp/raw_config.py +149 -0
  121. opscloud/mcp/semantic_profiler.py +305 -0
  122. opscloud/mcp/session_manager.py +727 -0
  123. opscloud/mcp/trust.py +105 -0
  124. opscloud/memory/__init__.py +27 -0
  125. opscloud/memory/branch.py +61 -0
  126. opscloud/memory/guard.py +19 -0
  127. opscloud/memory/onboarding.py +120 -0
  128. opscloud/memory/registry.py +134 -0
  129. opscloud/memory/store.py +223 -0
  130. opscloud/middleware/__init__.py +106 -0
  131. opscloud/middleware/_repository_bounds.py +459 -0
  132. opscloud/middleware/ask_user.py +152 -0
  133. opscloud/middleware/auto_mode.py +60 -0
  134. opscloud/middleware/auto_mode_hitl.py +2434 -0
  135. opscloud/middleware/compaction.py +566 -0
  136. opscloud/middleware/configurable_model.py +233 -0
  137. opscloud/middleware/cost_tracking.py +1021 -0
  138. opscloud/middleware/glm_stall_recovery.py +109 -0
  139. opscloud/middleware/goal_criteria.py +1234 -0
  140. opscloud/middleware/goal_state_notice.py +712 -0
  141. opscloud/middleware/goal_tools.py +407 -0
  142. opscloud/middleware/headless_mcp_guard.py +88 -0
  143. opscloud/middleware/jev_model_router.py +1018 -0
  144. opscloud/middleware/local_context.py +460 -0
  145. opscloud/middleware/mcp_context.py +151 -0
  146. opscloud/middleware/mcp_middleware.py +146 -0
  147. opscloud/middleware/memory_guard.py +393 -0
  148. opscloud/middleware/model_retry.py +205 -0
  149. opscloud/middleware/registry.py +137 -0
  150. opscloud/middleware/reliable_rubric.py +709 -0
  151. opscloud/middleware/resume_state.py +91 -0
  152. opscloud/middleware/server_hooks.py +1189 -0
  153. opscloud/middleware/shell_allow_list.py +90 -0
  154. opscloud/middleware/skills.py +382 -0
  155. opscloud/middleware/subagent_artifacts.py +84 -0
  156. opscloud/middleware/subagent_telemetry.py +261 -0
  157. opscloud/middleware/subagents.py +243 -0
  158. opscloud/middleware/tool_filter.py +384 -0
  159. opscloud/middleware/unified_system_message.py +62 -0
  160. opscloud/model/__init__.py +37 -0
  161. opscloud/model/config.py +2992 -0
  162. opscloud/model/factory.py +477 -0
  163. opscloud/model/pool.py +459 -0
  164. opscloud/model/reasoning.py +282 -0
  165. opscloud/offload.py +173 -0
  166. opscloud/output.py +64 -0
  167. opscloud/plugins/__init__.py +189 -0
  168. opscloud/plugins/_json.py +22 -0
  169. opscloud/plugins/adapters/__init__.py +13 -0
  170. opscloud/plugins/adapters/agents.py +112 -0
  171. opscloud/plugins/adapters/commands.py +190 -0
  172. opscloud/plugins/adapters/mcp.py +363 -0
  173. opscloud/plugins/adapters/skills.py +132 -0
  174. opscloud/plugins/commands_cli.py +541 -0
  175. opscloud/plugins/discovery.py +512 -0
  176. opscloud/plugins/manifest.py +335 -0
  177. opscloud/plugins/marketplace.py +893 -0
  178. opscloud/plugins/models.py +334 -0
  179. opscloud/plugins/project_plugins.py +317 -0
  180. opscloud/plugins/protocol.py +31 -0
  181. opscloud/plugins/store.py +905 -0
  182. opscloud/plugins/substitution.py +122 -0
  183. opscloud/project_utils.py +145 -0
  184. opscloud/prompts/__init__.py +37 -0
  185. opscloud/prompts/system.py +483 -0
  186. opscloud/prompts/templates/system_prompt.md +166 -0
  187. opscloud/prompts/types.py +54 -0
  188. opscloud/rubrics/__init__.py +42 -0
  189. opscloud/rubrics/evaluator.py +404 -0
  190. opscloud/rubrics/evidence_extractor.py +229 -0
  191. opscloud/rubrics/generator.py +239 -0
  192. opscloud/rubrics/jev_compiler.py +175 -0
  193. opscloud/rubrics/jev_grader.py +297 -0
  194. opscloud/rubrics/middleware.py +8 -0
  195. opscloud/schema/__init__.py +107 -0
  196. opscloud/schema/interrupts.py +981 -0
  197. opscloud/security/__init__.py +91 -0
  198. opscloud/security/approval_mode.py +528 -0
  199. opscloud/security/approval_mode_source.py +230 -0
  200. opscloud/security/cli_ast_evaluator.py +552 -0
  201. opscloud/security/jev_classifier.py +651 -0
  202. opscloud/security/shell_safety.py +340 -0
  203. opscloud/security/unicode_security.py +439 -0
  204. opscloud/security/url_validation.py +160 -0
  205. opscloud/server/__init__.py +13 -0
  206. opscloud/server/_server_config.py +158 -0
  207. opscloud/server/server.py +242 -0
  208. opscloud/server/server_graph.py +93 -0
  209. opscloud/skills/__init__.py +91 -0
  210. opscloud/skills/commands.py +1099 -0
  211. opscloud/skills/invocation.py +200 -0
  212. opscloud/skills/load.py +388 -0
  213. opscloud/skills/loader.py +23 -0
  214. opscloud/skills/merge.py +65 -0
  215. opscloud/skills/registry.py +263 -0
  216. opscloud/skills/sources.py +174 -0
  217. opscloud/skills/trust.py +417 -0
  218. opscloud/state/__init__.py +142 -0
  219. opscloud/state/base.py +154 -0
  220. opscloud/state/cloud_context.py +262 -0
  221. opscloud/state/goal_channels.py +124 -0
  222. opscloud/state/goal_state_limits.py +175 -0
  223. opscloud/state/resume_state.py +58 -0
  224. opscloud/state/service_context.py +281 -0
  225. opscloud/state/session.py +957 -0
  226. opscloud/state/state_migration.py +86 -0
  227. opscloud/subagents/__init__.py +30 -0
  228. opscloud/subagents/loader.py +331 -0
  229. opscloud/subagents/subagents_parser.py +117 -0
  230. opscloud/subagents/types.py +25 -0
  231. opscloud/tools/__init__.py +36 -0
  232. opscloud/tools/catalog.py +37 -0
  233. opscloud/tools/display.py +24 -0
  234. opscloud/tools/fetch_url.py +176 -0
  235. opscloud/tools/goal_tools.py +423 -0
  236. opscloud/tools/registry.py +105 -0
  237. opscloud/tools/thread.py +32 -0
  238. opscloud/tools/web_search.py +215 -0
  239. opscloud/ui/__init__.py +119 -0
  240. opscloud/ui/app.py +5400 -0
  241. opscloud/ui/app.tcss +479 -0
  242. opscloud/ui/clipboard.py +147 -0
  243. opscloud/ui/command_registry.py +482 -0
  244. opscloud/ui/message_store.py +152 -0
  245. opscloud/ui/permission_store.py +360 -0
  246. opscloud/ui/preferences.py +113 -0
  247. opscloud/ui/prompt_manager.py +96 -0
  248. opscloud/ui/remote_client.py +505 -0
  249. opscloud/ui/textual_adapter.py +1269 -0
  250. opscloud/ui/theme.py +728 -0
  251. opscloud/ui/ui_help.py +415 -0
  252. opscloud/ui/widgets/__init__.py +383 -0
  253. opscloud/ui/widgets/_ask_user_types.py +81 -0
  254. opscloud/ui/widgets/_copy_spans.py +52 -0
  255. opscloud/ui/widgets/_inline_prompt.py +290 -0
  256. opscloud/ui/widgets/_js_eval_display.py +139 -0
  257. opscloud/ui/widgets/_links.py +175 -0
  258. opscloud/ui/widgets/_tool_stream.py +308 -0
  259. opscloud/ui/widgets/agent_selector.py +297 -0
  260. opscloud/ui/widgets/approval.py +741 -0
  261. opscloud/ui/widgets/ask_user.py +539 -0
  262. opscloud/ui/widgets/auth.py +695 -0
  263. opscloud/ui/widgets/auth_manager.py +214 -0
  264. opscloud/ui/widgets/auto_mode_notice.py +146 -0
  265. opscloud/ui/widgets/autocomplete.py +709 -0
  266. opscloud/ui/widgets/chat_input.py +805 -0
  267. opscloud/ui/widgets/cloud_selector.py +528 -0
  268. opscloud/ui/widgets/config_manager.py +652 -0
  269. opscloud/ui/widgets/debug_console.py +1269 -0
  270. opscloud/ui/widgets/devops_renderers.py +184 -0
  271. opscloud/ui/widgets/diff.py +237 -0
  272. opscloud/ui/widgets/effort_selector.py +166 -0
  273. opscloud/ui/widgets/goal_review.py +612 -0
  274. opscloud/ui/widgets/goal_status.py +57 -0
  275. opscloud/ui/widgets/infra_panel.py +141 -0
  276. opscloud/ui/widgets/install_confirm.py +114 -0
  277. opscloud/ui/widgets/loading.py +170 -0
  278. opscloud/ui/widgets/mcp_viewer.py +1204 -0
  279. opscloud/ui/widgets/messages.py +4461 -0
  280. opscloud/ui/widgets/model_selector.py +863 -0
  281. opscloud/ui/widgets/notification_center.py +108 -0
  282. opscloud/ui/widgets/notification_settings.py +119 -0
  283. opscloud/ui/widgets/operation_card.py +106 -0
  284. opscloud/ui/widgets/permissions_manager.py +667 -0
  285. opscloud/ui/widgets/plugin_manager.py +1212 -0
  286. opscloud/ui/widgets/pool_selector.py +376 -0
  287. opscloud/ui/widgets/preamble.py +53 -0
  288. opscloud/ui/widgets/skills_viewer.py +432 -0
  289. opscloud/ui/widgets/status.py +1037 -0
  290. opscloud/ui/widgets/subagent_panel.py +927 -0
  291. opscloud/ui/widgets/theme_selector.py +232 -0
  292. opscloud/ui/widgets/thread_selector.py +358 -0
  293. opscloud/ui/widgets/toast.py +31 -0
  294. opscloud/ui/widgets/tool_display.py +325 -0
  295. opscloud/ui/widgets/tool_renderers.py +234 -0
  296. opscloud/ui/widgets/tool_widgets.py +278 -0
  297. opscloud/ui/widgets/welcome.py +421 -0
  298. opscloud/ui/widgets/welcome_popup.py +102 -0
  299. opscloud/utils/__init__.py +24 -0
  300. opscloud/utils/cost_estimation.py +169 -0
  301. opscloud/utils/git.py +390 -0
  302. opscloud/utils/logger.py +571 -0
  303. opscloud/utils/session_stats.py +115 -0
  304. opscloud/utils/startup_error.py +37 -0
  305. talkops_opscloud-0.3.0b1.dist-info/METADATA +449 -0
  306. talkops_opscloud-0.3.0b1.dist-info/RECORD +309 -0
  307. talkops_opscloud-0.3.0b1.dist-info/WHEEL +4 -0
  308. talkops_opscloud-0.3.0b1.dist-info/entry_points.txt +2 -0
  309. talkops_opscloud-0.3.0b1.dist-info/licenses/LICENSE +175 -0
@@ -0,0 +1,297 @@
1
+ """Jev-powered System One rubric grader and hybrid evaluator.
2
+
3
+ Performs ultra-fast (<200ms) parallel criteria verification using TypeSafe AI Jev,
4
+ eliminating autoregressive LLM judge variance and cost, while providing transparent
5
+ Tier 2 generative escalation and environmental blocker detection.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import asyncio
11
+ from collections.abc import Awaitable, Callable, Mapping, Sequence
12
+ import inspect
13
+ import time
14
+ from typing import Any, cast
15
+
16
+ from deepagents.middleware.rubric import (
17
+ CriterionEval,
18
+ CriterionFail,
19
+ CriterionPass,
20
+ GraderResponse,
21
+ )
22
+ from langchain_core.runnables import RunnableConfig
23
+ from langchain_typesafe import (
24
+ ClassifierRequest,
25
+ ClassifierResponse,
26
+ State,
27
+ TypeSafeClassifier,
28
+ )
29
+
30
+ from opscloud.config.settings import get_settings, resolve_env_var
31
+ from opscloud.rubrics.evidence_extractor import extract_grading_evidence
32
+ from opscloud.rubrics.jev_compiler import CompiledRubric
33
+ from opscloud.utils.logger import get_logger
34
+
35
+ logger = get_logger(__name__)
36
+
37
+ # ── Declarative Dispatch & Threshold Rules ─────────────────────────────────
38
+
39
+ _CRITERION_PASS_THRESHOLD = 0.50
40
+ _MIN_READINESS_SCORE = 1.0
41
+
42
+
43
+ def build_criteria_evaluations(
44
+ criteria_map: Mapping[str, str],
45
+ response: ClassifierResponse,
46
+ pass_threshold: float = _CRITERION_PASS_THRESHOLD,
47
+ ) -> list[CriterionEval]:
48
+ """Map Jev Noul probabilities into deepagents CriterionPass / CriterionFail list."""
49
+ evals: list[CriterionEval] = []
50
+ for cid, text in criteria_map.items():
51
+ noul_answer = response.nouls.get(cid)
52
+ prob = noul_answer.noul if noul_answer is not None else 0.0
53
+ if prob >= pass_threshold:
54
+ evals.append(CriterionPass(name=text, passed=True))
55
+ else:
56
+ evals.append(
57
+ CriterionFail(
58
+ name=text,
59
+ passed=False,
60
+ gap=f"Criterion unsatisfied (confidence {prob:.2f} < {pass_threshold:.2f}): {text}",
61
+ )
62
+ )
63
+ return evals
64
+
65
+
66
+ BlockerHandler = Callable[[ClassifierResponse, list[CriterionEval]], GraderResponse]
67
+
68
+ def _make_blocker_response(
69
+ blocker_type: str,
70
+ explanation: str,
71
+ evals: list[CriterionEval],
72
+ ) -> GraderResponse:
73
+ resp = GraderResponse(
74
+ result="needs_revision",
75
+ explanation=explanation,
76
+ criteria=evals,
77
+ )
78
+ object.__setattr__(resp, "blocker_type", blocker_type)
79
+ return resp
80
+
81
+
82
+ BLOCKER_DISPATCH: dict[str, BlockerHandler] = {
83
+ "external_blocker": lambda resp, evals: _make_blocker_response(
84
+ "external_blocker",
85
+ (
86
+ "Environmental blocker detected by Jev (e.g. AWS service quota, "
87
+ "IAM permissions, or network partition). Proposing dynamic rubric amendment."
88
+ ),
89
+ evals,
90
+ ),
91
+ "ambiguous_requirement": lambda resp, evals: _make_blocker_response(
92
+ "ambiguous_requirement",
93
+ (
94
+ "Criteria cannot be verified because the requirements conflict with "
95
+ "environmental constraints or are ambiguous. Operator clarification required."
96
+ ),
97
+ evals,
98
+ ),
99
+ "progressing": lambda resp, evals: (
100
+ GraderResponse(
101
+ result="satisfied",
102
+ explanation=(
103
+ f"All {len(evals)} criteria successfully verified via Jev System One "
104
+ f"(Operational readiness: {getattr(resp.scores.get('readiness'), 'score', 2.0):.2f})."
105
+ ),
106
+ criteria=evals,
107
+ )
108
+ if all(c.get("passed", False) for c in evals)
109
+ and getattr(resp.scores.get("readiness"), "score", 2.0) >= _MIN_READINESS_SCORE
110
+ else GraderResponse(
111
+ result="needs_revision",
112
+ explanation=(
113
+ f"{sum(1 for c in evals if not c.get('passed', False))} of {len(evals)} criteria "
114
+ "not yet satisfied. Revision required."
115
+ ),
116
+ criteria=evals,
117
+ )
118
+ ),
119
+ }
120
+
121
+
122
+ # ── Jev Hybrid Rubric Grader ───────────────────────────────────────────────
123
+
124
+
125
+ class JevHybridRubricGrader:
126
+ """Two-tier rubric evaluator: Jev System One fast-pass + Frontier LLM diagnostic fallback."""
127
+
128
+ def __init__(
129
+ self,
130
+ api_key: str | None = None,
131
+ timeout_seconds: float = 2.0,
132
+ ) -> None:
133
+ """Initialize Jev rubric grader with API key resolution."""
134
+ if api_key is not None:
135
+ self._api_key = api_key if api_key else None
136
+ self._explicit_key = True
137
+ else:
138
+ settings_key = getattr(get_settings(), "typesafe_api_key", None)
139
+ resolved_key = settings_key or resolve_env_var("TYPESAFE_API_KEY", fallback_names=("JEV_API_KEY",))
140
+ self._api_key = resolved_key
141
+ self._explicit_key = False
142
+
143
+ self.timeout_seconds = timeout_seconds
144
+ self._classifier: TypeSafeClassifier | None = None
145
+
146
+ def _get_classifier(self) -> TypeSafeClassifier:
147
+ """Lazily initialize TypeSafeClassifier instance."""
148
+ if self._classifier is None:
149
+ if self._api_key:
150
+ self._classifier = TypeSafeClassifier(api_key=self._api_key)
151
+ else:
152
+ self._classifier = TypeSafeClassifier()
153
+ return self._classifier
154
+
155
+ def is_available(self) -> bool:
156
+ """Check whether TypeSafe API credentials are configured."""
157
+ if self._explicit_key:
158
+ return bool(self._api_key)
159
+ if self._api_key:
160
+ return True
161
+ settings_key = getattr(get_settings(), "typesafe_api_key", None)
162
+ if settings_key:
163
+ return True
164
+ return bool(resolve_env_var("TYPESAFE_API_KEY", fallback_names=("JEV_API_KEY",)))
165
+
166
+ async def agrade(
167
+ self,
168
+ compiled_rubric: CompiledRubric,
169
+ evidence: dict[str, Any] | Sequence[Any] | str,
170
+ *,
171
+ fallback_llm_fn: Callable[[list[str]], Awaitable[GraderResponse] | GraderResponse] | None = None,
172
+ ) -> GraderResponse:
173
+ """Evaluate criteria against evidence asynchronously using Jev System One.
174
+
175
+ Escalates to `fallback_llm_fn` only when a criterion fails and detailed remediation
176
+ is requested.
177
+ """
178
+ classifier = self._get_classifier()
179
+ start_time = time.monotonic()
180
+
181
+ # Build clean state payload for Jev
182
+ if isinstance(evidence, dict) and "commands_executed" in evidence:
183
+ structured_evidence = evidence
184
+ else:
185
+ structured_evidence = extract_grading_evidence(
186
+ evidence if isinstance(evidence, (Mapping, Sequence)) and not isinstance(evidence, (str, bytes)) else [evidence]
187
+ )
188
+
189
+ state_payload: State = cast(
190
+ State,
191
+ {
192
+ "objective": compiled_rubric.objective,
193
+ "evidence": structured_evidence,
194
+ },
195
+ )
196
+
197
+ request: ClassifierRequest = {
198
+ "state": state_payload,
199
+ "questions": compiled_rubric.questions,
200
+ }
201
+
202
+ run_config: RunnableConfig = {
203
+ "run_name": "jev_rubric_grader",
204
+ "tags": ["opscloud:smart", "opscloud:jev", "opscloud:rubric"],
205
+ "metadata": {
206
+ "objective": compiled_rubric.objective[:100],
207
+ "criteria_count": len(compiled_rubric.criteria_map),
208
+ },
209
+ }
210
+
211
+ try:
212
+ async with asyncio.timeout(self.timeout_seconds):
213
+ response: ClassifierResponse = await classifier.ainvoke(
214
+ request,
215
+ config=run_config,
216
+ )
217
+ except Exception as exc:
218
+ latency_ms = int((time.monotonic() - start_time) * 1000)
219
+ logger.warning("Jev rubric evaluation failed or timed out (%d ms): %s", latency_ms, exc)
220
+ raise
221
+
222
+ latency_ms = int((time.monotonic() - start_time) * 1000)
223
+ criteria_evals = build_criteria_evaluations(compiled_rubric.criteria_map, response)
224
+
225
+ blocker = response.choices.get("blocker_status")
226
+ blocker_key = blocker.choice if blocker else "progressing"
227
+
228
+ handler = BLOCKER_DISPATCH.get(blocker_key, BLOCKER_DISPATCH["progressing"])
229
+ verdict = handler(response, criteria_evals)
230
+
231
+ readiness = getattr(response.scores.get("readiness"), "score", 0.0)
232
+ logger.info(
233
+ "Jev rubric fast-pass completed in %d ms: result=%s, readiness=%.2f, blocker=%s, passed=%d/%d",
234
+ latency_ms,
235
+ verdict.result,
236
+ readiness,
237
+ blocker_key,
238
+ sum(1 for c in criteria_evals if c.get("passed", False)),
239
+ len(criteria_evals),
240
+ )
241
+
242
+ # ── Tier 2 Diagnostic Escalation ────────────────────────────────────
243
+ # When criteria fail under normal execution, escalate to generative LLM
244
+ # to author targeted remediation feedback for failing requirements.
245
+ failed_items = [c.get("name", "") for c in criteria_evals if not c.get("passed", False)]
246
+ if not failed_items and readiness < _MIN_READINESS_SCORE:
247
+ failed_items = [f"Operational readiness score ({readiness:.2f}) below threshold ({_MIN_READINESS_SCORE:.2f})"]
248
+
249
+ if verdict.result == "needs_revision" and failed_items and fallback_llm_fn and blocker_key == "progressing":
250
+ logger.info(
251
+ "Jev fast-pass detected %d failing criteria; invoking Tier 2 LLM diagnostic for remediation advice.",
252
+ len(failed_items),
253
+ )
254
+ try:
255
+ llm_res = fallback_llm_fn(failed_items)
256
+ if inspect.isawaitable(llm_res):
257
+ llm_verdict = await llm_res
258
+ else:
259
+ llm_verdict = llm_res
260
+ if isinstance(llm_verdict, GraderResponse):
261
+ if not llm_verdict.criteria:
262
+ object.__setattr__(llm_verdict, "criteria", criteria_evals)
263
+ return llm_verdict
264
+ except Exception as llm_exc:
265
+ logger.warning("Tier 2 LLM diagnostic fallback failed: %s; returning Jev verdict", llm_exc)
266
+
267
+ return verdict
268
+
269
+ def grade(
270
+ self,
271
+ compiled_rubric: CompiledRubric,
272
+ evidence: dict[str, Any] | Sequence[Any] | str,
273
+ *,
274
+ fallback_llm_fn: Callable[[list[str]], GraderResponse] | None = None,
275
+ ) -> GraderResponse:
276
+ """Synchronous version of grade."""
277
+ try:
278
+ loop = asyncio.get_running_loop()
279
+ except RuntimeError:
280
+ loop = None
281
+
282
+ if loop is not None and loop.is_running():
283
+ import concurrent.futures
284
+
285
+ with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:
286
+ return executor.submit(
287
+ asyncio.run,
288
+ self.agrade(compiled_rubric, evidence, fallback_llm_fn=fallback_llm_fn), # type: ignore[arg-type]
289
+ ).result()
290
+ return asyncio.run(self.agrade(compiled_rubric, evidence, fallback_llm_fn=fallback_llm_fn)) # type: ignore[arg-type]
291
+
292
+
293
+ __all__ = [
294
+ "BLOCKER_DISPATCH",
295
+ "JevHybridRubricGrader",
296
+ "build_criteria_evaluations",
297
+ ]
@@ -0,0 +1,8 @@
1
+ """Re-export RubricMiddleware from middleware module."""
2
+
3
+ from opscloud.middleware.reliable_rubric import (
4
+ ReliableRubricMiddleware,
5
+ RubricMiddleware,
6
+ )
7
+
8
+ __all__ = ["ReliableRubricMiddleware", "RubricMiddleware"]
@@ -0,0 +1,107 @@
1
+ """Schema definitions for OpsCloud."""
2
+
3
+ from opscloud.schema.interrupts import (
4
+ # Tool Approvals / HITL
5
+ ActionRequest,
6
+ ApproveDecision,
7
+ AutoApproveAllDecision,
8
+ EditDecision,
9
+ HITLRequest,
10
+ HitlDecision,
11
+ HitlResumePayload,
12
+ RejectDecision,
13
+ ReviewConfig,
14
+ SmartApproveAllDecision,
15
+ SwitchManualDecision,
16
+ # Ask User Protocol
17
+ ASK_USER_ANSWERED_SUMMARY,
18
+ ASK_USER_CANCELLED_ANSWER,
19
+ ASK_USER_CANCELLED_SUMMARY,
20
+ ASK_USER_ERROR_ANSWER_PREFIX,
21
+ ASK_USER_FAILED_SUMMARY,
22
+ ASK_USER_NOTHING_SELECTED,
23
+ ASK_USER_NO_ANSWER,
24
+ AskUserAnswered,
25
+ AskUserCancelled,
26
+ AskUserRequest,
27
+ AskUserResumePayload,
28
+ AskUserWidgetResult,
29
+ CHOICE_QUESTION_TYPES,
30
+ Choice,
31
+ QUESTION_TYPES,
32
+ Question,
33
+ QuestionType,
34
+ ValidatedQuestion,
35
+ ask_user_answer_is_empty,
36
+ decode_multi_select_answer,
37
+ encode_multi_select_answer,
38
+ format_ask_user_error_answer,
39
+ format_ask_user_transcript,
40
+ render_ask_user_transcript_for_display,
41
+ # Goal & Criteria Review
42
+ GoalReviewRequest,
43
+ GoalReviewResumePayload,
44
+ # Hook Invocation Interrupts
45
+ HOOK_INVOCATION_INTERRUPT_TYPE,
46
+ HookInvocationInterrupt,
47
+ HookInvocationRequest,
48
+ HookInvocationResponse,
49
+ build_hook_interrupt_payload,
50
+ build_hook_resume_value,
51
+ is_hook_interrupt_payload,
52
+ parse_hook_interrupt_payload,
53
+ parse_hook_resume_value,
54
+ )
55
+
56
+ __all__ = [
57
+ # HITL & Tool Approval
58
+ "ActionRequest",
59
+ "ApproveDecision",
60
+ "AutoApproveAllDecision",
61
+ "EditDecision",
62
+ "HITLRequest",
63
+ "HitlDecision",
64
+ "HitlResumePayload",
65
+ "RejectDecision",
66
+ "ReviewConfig",
67
+ "SmartApproveAllDecision",
68
+ "SwitchManualDecision",
69
+ # Ask User Protocol
70
+ "ASK_USER_ANSWERED_SUMMARY",
71
+ "ASK_USER_CANCELLED_ANSWER",
72
+ "ASK_USER_CANCELLED_SUMMARY",
73
+ "ASK_USER_ERROR_ANSWER_PREFIX",
74
+ "ASK_USER_FAILED_SUMMARY",
75
+ "ASK_USER_NOTHING_SELECTED",
76
+ "ASK_USER_NO_ANSWER",
77
+ "AskUserAnswered",
78
+ "AskUserCancelled",
79
+ "AskUserRequest",
80
+ "AskUserResumePayload",
81
+ "AskUserWidgetResult",
82
+ "CHOICE_QUESTION_TYPES",
83
+ "Choice",
84
+ "QUESTION_TYPES",
85
+ "Question",
86
+ "QuestionType",
87
+ "ValidatedQuestion",
88
+ "ask_user_answer_is_empty",
89
+ "decode_multi_select_answer",
90
+ "encode_multi_select_answer",
91
+ "format_ask_user_error_answer",
92
+ "format_ask_user_transcript",
93
+ "render_ask_user_transcript_for_display",
94
+ # Goal & Criteria Review
95
+ "GoalReviewRequest",
96
+ "GoalReviewResumePayload",
97
+ # Hook Invocation Interrupts
98
+ "HOOK_INVOCATION_INTERRUPT_TYPE",
99
+ "HookInvocationInterrupt",
100
+ "HookInvocationRequest",
101
+ "HookInvocationResponse",
102
+ "build_hook_interrupt_payload",
103
+ "build_hook_resume_value",
104
+ "is_hook_interrupt_payload",
105
+ "parse_hook_interrupt_payload",
106
+ "parse_hook_resume_value",
107
+ ]