talkops-opscloud 0.3.0b1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- opscloud/__init__.py +5 -0
- opscloud/__main__.py +6 -0
- opscloud/_constants.py +53 -0
- opscloud/_debug.py +145 -0
- opscloud/_debug_buffer.py +250 -0
- opscloud/_textual_patches.py +367 -0
- opscloud/_version.py +15 -0
- opscloud/agent/__init__.py +27 -0
- opscloud/agent/config.py +168 -0
- opscloud/agent/factory.py +1103 -0
- opscloud/approval_mode.py +44 -0
- opscloud/backend/__init__.py +49 -0
- opscloud/backend/composite.py +89 -0
- opscloud/backend/local.py +183 -0
- opscloud/backend/registry.py +79 -0
- opscloud/backend/sandbox_config.py +107 -0
- opscloud/backend/sandbox_factory.py +432 -0
- opscloud/built_in_skills/remember/SKILL.md +124 -0
- opscloud/built_in_skills/skill-creator/SKILL.md +348 -0
- opscloud/built_in_skills/skill-creator/scripts/init_skill.py +371 -0
- opscloud/built_in_skills/skill-creator/scripts/quick_validate.py +165 -0
- opscloud/cli/__init__.py +1 -0
- opscloud/cli/commands/__init__.py +15 -0
- opscloud/cli/commands/auth.py +45 -0
- opscloud/cli/commands/doctor.py +100 -0
- opscloud/cli/commands/mcp.py +137 -0
- opscloud/cli/commands/threads.py +51 -0
- opscloud/cli/interactive.py +154 -0
- opscloud/cli/main.py +903 -0
- opscloud/cli/non_interactive.py +425 -0
- opscloud/cli/server_manager.py +58 -0
- opscloud/commands/__init__.py +19 -0
- opscloud/commands/_base.py +110 -0
- opscloud/commands/_guards.py +52 -0
- opscloud/commands/_router.py +146 -0
- opscloud/commands/_types.py +36 -0
- opscloud/commands/core/__init__.py +47 -0
- opscloud/commands/core/auth.py +94 -0
- opscloud/commands/core/bug.py +46 -0
- opscloud/commands/core/clear.py +106 -0
- opscloud/commands/core/cloud.py +185 -0
- opscloud/commands/core/compact.py +128 -0
- opscloud/commands/core/config_cmd.py +120 -0
- opscloud/commands/core/context.py +220 -0
- opscloud/commands/core/cost.py +44 -0
- opscloud/commands/core/doctor.py +535 -0
- opscloud/commands/core/effort.py +108 -0
- opscloud/commands/core/exit_cmd.py +41 -0
- opscloud/commands/core/fast.py +148 -0
- opscloud/commands/core/help_cmd.py +94 -0
- opscloud/commands/core/mcp.py +134 -0
- opscloud/commands/core/model.py +109 -0
- opscloud/commands/core/permissions.py +196 -0
- opscloud/commands/core/plugins.py +92 -0
- opscloud/commands/core/pool.py +108 -0
- opscloud/commands/core/resume.py +97 -0
- opscloud/commands/core/skills.py +178 -0
- opscloud/commands/power/__init__.py +51 -0
- opscloud/commands/power/agents.py +98 -0
- opscloud/commands/power/btw.py +85 -0
- opscloud/commands/power/copy.py +97 -0
- opscloud/commands/power/goal.py +548 -0
- opscloud/commands/power/loop.py +295 -0
- opscloud/commands/power/memory.py +287 -0
- opscloud/commands/power/review.py +152 -0
- opscloud/commands/power/rubric.py +287 -0
- opscloud/commands/power/runtime.py +321 -0
- opscloud/commands/power/skill_creator.py +70 -0
- opscloud/commands/power/skill_invoke.py +165 -0
- opscloud/commands/power/tasks.py +209 -0
- opscloud/commands/power/trace.py +117 -0
- opscloud/commands/power/ui_toggles.py +135 -0
- opscloud/commands/power/version.py +52 -0
- opscloud/commands/registry.py +176 -0
- opscloud/config/__init__.py +114 -0
- opscloud/config/adapters/__init__.py +5 -0
- opscloud/config/adapters/postgres.py +776 -0
- opscloud/config/adapters/sqlite.py +769 -0
- opscloud/config/aws.py +248 -0
- opscloud/config/cloud_profiles.py +437 -0
- opscloud/config/env_vars.py +81 -0
- opscloud/config/langsmith.py +319 -0
- opscloud/config/manifest.py +817 -0
- opscloud/config/metadata.py +205 -0
- opscloud/config/paths.py +604 -0
- opscloud/config/plugins.py +130 -0
- opscloud/config/settings.py +834 -0
- opscloud/config/store.py +453 -0
- opscloud/config/store_factory.py +97 -0
- opscloud/config/toml_config.py +562 -0
- opscloud/editor.py +173 -0
- opscloud/exceptions.py +65 -0
- opscloud/file_ops.py +217 -0
- opscloud/hooks/__init__.py +139 -0
- opscloud/hooks/env.py +44 -0
- opscloud/hooks/interrupt.py +121 -0
- opscloud/hooks/models/__init__.py +109 -0
- opscloud/hooks/models/adapters.py +19 -0
- opscloud/hooks/models/config.py +79 -0
- opscloud/hooks/models/domain.py +434 -0
- opscloud/hooks/models/transport.py +37 -0
- opscloud/hooks/tools.py +231 -0
- opscloud/input.py +48 -0
- opscloud/integrations/__init__.py +80 -0
- opscloud/integrations/base.py +153 -0
- opscloud/integrations/content_filter.py +119 -0
- opscloud/integrations/event_bus.py +480 -0
- opscloud/integrations/hooks.py +238 -0
- opscloud/integrations/identity.py +181 -0
- opscloud/integrations/notifications.py +134 -0
- opscloud/integrations/stream_bridge.py +397 -0
- opscloud/integrations/thread_store.py +81 -0
- opscloud/json_types.py +20 -0
- opscloud/mcp/__init__.py +57 -0
- opscloud/mcp/config.py +203 -0
- opscloud/mcp/discovery.py +229 -0
- opscloud/mcp/mcp_info.py +109 -0
- opscloud/mcp/middleware.py +102 -0
- opscloud/mcp/preload.py +505 -0
- opscloud/mcp/raw_config.py +149 -0
- opscloud/mcp/semantic_profiler.py +305 -0
- opscloud/mcp/session_manager.py +727 -0
- opscloud/mcp/trust.py +105 -0
- opscloud/memory/__init__.py +27 -0
- opscloud/memory/branch.py +61 -0
- opscloud/memory/guard.py +19 -0
- opscloud/memory/onboarding.py +120 -0
- opscloud/memory/registry.py +134 -0
- opscloud/memory/store.py +223 -0
- opscloud/middleware/__init__.py +106 -0
- opscloud/middleware/_repository_bounds.py +459 -0
- opscloud/middleware/ask_user.py +152 -0
- opscloud/middleware/auto_mode.py +60 -0
- opscloud/middleware/auto_mode_hitl.py +2434 -0
- opscloud/middleware/compaction.py +566 -0
- opscloud/middleware/configurable_model.py +233 -0
- opscloud/middleware/cost_tracking.py +1021 -0
- opscloud/middleware/glm_stall_recovery.py +109 -0
- opscloud/middleware/goal_criteria.py +1234 -0
- opscloud/middleware/goal_state_notice.py +712 -0
- opscloud/middleware/goal_tools.py +407 -0
- opscloud/middleware/headless_mcp_guard.py +88 -0
- opscloud/middleware/jev_model_router.py +1018 -0
- opscloud/middleware/local_context.py +460 -0
- opscloud/middleware/mcp_context.py +151 -0
- opscloud/middleware/mcp_middleware.py +146 -0
- opscloud/middleware/memory_guard.py +393 -0
- opscloud/middleware/model_retry.py +205 -0
- opscloud/middleware/registry.py +137 -0
- opscloud/middleware/reliable_rubric.py +709 -0
- opscloud/middleware/resume_state.py +91 -0
- opscloud/middleware/server_hooks.py +1189 -0
- opscloud/middleware/shell_allow_list.py +90 -0
- opscloud/middleware/skills.py +382 -0
- opscloud/middleware/subagent_artifacts.py +84 -0
- opscloud/middleware/subagent_telemetry.py +261 -0
- opscloud/middleware/subagents.py +243 -0
- opscloud/middleware/tool_filter.py +384 -0
- opscloud/middleware/unified_system_message.py +62 -0
- opscloud/model/__init__.py +37 -0
- opscloud/model/config.py +2992 -0
- opscloud/model/factory.py +477 -0
- opscloud/model/pool.py +459 -0
- opscloud/model/reasoning.py +282 -0
- opscloud/offload.py +173 -0
- opscloud/output.py +64 -0
- opscloud/plugins/__init__.py +189 -0
- opscloud/plugins/_json.py +22 -0
- opscloud/plugins/adapters/__init__.py +13 -0
- opscloud/plugins/adapters/agents.py +112 -0
- opscloud/plugins/adapters/commands.py +190 -0
- opscloud/plugins/adapters/mcp.py +363 -0
- opscloud/plugins/adapters/skills.py +132 -0
- opscloud/plugins/commands_cli.py +541 -0
- opscloud/plugins/discovery.py +512 -0
- opscloud/plugins/manifest.py +335 -0
- opscloud/plugins/marketplace.py +893 -0
- opscloud/plugins/models.py +334 -0
- opscloud/plugins/project_plugins.py +317 -0
- opscloud/plugins/protocol.py +31 -0
- opscloud/plugins/store.py +905 -0
- opscloud/plugins/substitution.py +122 -0
- opscloud/project_utils.py +145 -0
- opscloud/prompts/__init__.py +37 -0
- opscloud/prompts/system.py +483 -0
- opscloud/prompts/templates/system_prompt.md +166 -0
- opscloud/prompts/types.py +54 -0
- opscloud/rubrics/__init__.py +42 -0
- opscloud/rubrics/evaluator.py +404 -0
- opscloud/rubrics/evidence_extractor.py +229 -0
- opscloud/rubrics/generator.py +239 -0
- opscloud/rubrics/jev_compiler.py +175 -0
- opscloud/rubrics/jev_grader.py +297 -0
- opscloud/rubrics/middleware.py +8 -0
- opscloud/schema/__init__.py +107 -0
- opscloud/schema/interrupts.py +981 -0
- opscloud/security/__init__.py +91 -0
- opscloud/security/approval_mode.py +528 -0
- opscloud/security/approval_mode_source.py +230 -0
- opscloud/security/cli_ast_evaluator.py +552 -0
- opscloud/security/jev_classifier.py +651 -0
- opscloud/security/shell_safety.py +340 -0
- opscloud/security/unicode_security.py +439 -0
- opscloud/security/url_validation.py +160 -0
- opscloud/server/__init__.py +13 -0
- opscloud/server/_server_config.py +158 -0
- opscloud/server/server.py +242 -0
- opscloud/server/server_graph.py +93 -0
- opscloud/skills/__init__.py +91 -0
- opscloud/skills/commands.py +1099 -0
- opscloud/skills/invocation.py +200 -0
- opscloud/skills/load.py +388 -0
- opscloud/skills/loader.py +23 -0
- opscloud/skills/merge.py +65 -0
- opscloud/skills/registry.py +263 -0
- opscloud/skills/sources.py +174 -0
- opscloud/skills/trust.py +417 -0
- opscloud/state/__init__.py +142 -0
- opscloud/state/base.py +154 -0
- opscloud/state/cloud_context.py +262 -0
- opscloud/state/goal_channels.py +124 -0
- opscloud/state/goal_state_limits.py +175 -0
- opscloud/state/resume_state.py +58 -0
- opscloud/state/service_context.py +281 -0
- opscloud/state/session.py +957 -0
- opscloud/state/state_migration.py +86 -0
- opscloud/subagents/__init__.py +30 -0
- opscloud/subagents/loader.py +331 -0
- opscloud/subagents/subagents_parser.py +117 -0
- opscloud/subagents/types.py +25 -0
- opscloud/tools/__init__.py +36 -0
- opscloud/tools/catalog.py +37 -0
- opscloud/tools/display.py +24 -0
- opscloud/tools/fetch_url.py +176 -0
- opscloud/tools/goal_tools.py +423 -0
- opscloud/tools/registry.py +105 -0
- opscloud/tools/thread.py +32 -0
- opscloud/tools/web_search.py +215 -0
- opscloud/ui/__init__.py +119 -0
- opscloud/ui/app.py +5400 -0
- opscloud/ui/app.tcss +479 -0
- opscloud/ui/clipboard.py +147 -0
- opscloud/ui/command_registry.py +482 -0
- opscloud/ui/message_store.py +152 -0
- opscloud/ui/permission_store.py +360 -0
- opscloud/ui/preferences.py +113 -0
- opscloud/ui/prompt_manager.py +96 -0
- opscloud/ui/remote_client.py +505 -0
- opscloud/ui/textual_adapter.py +1269 -0
- opscloud/ui/theme.py +728 -0
- opscloud/ui/ui_help.py +415 -0
- opscloud/ui/widgets/__init__.py +383 -0
- opscloud/ui/widgets/_ask_user_types.py +81 -0
- opscloud/ui/widgets/_copy_spans.py +52 -0
- opscloud/ui/widgets/_inline_prompt.py +290 -0
- opscloud/ui/widgets/_js_eval_display.py +139 -0
- opscloud/ui/widgets/_links.py +175 -0
- opscloud/ui/widgets/_tool_stream.py +308 -0
- opscloud/ui/widgets/agent_selector.py +297 -0
- opscloud/ui/widgets/approval.py +741 -0
- opscloud/ui/widgets/ask_user.py +539 -0
- opscloud/ui/widgets/auth.py +695 -0
- opscloud/ui/widgets/auth_manager.py +214 -0
- opscloud/ui/widgets/auto_mode_notice.py +146 -0
- opscloud/ui/widgets/autocomplete.py +709 -0
- opscloud/ui/widgets/chat_input.py +805 -0
- opscloud/ui/widgets/cloud_selector.py +528 -0
- opscloud/ui/widgets/config_manager.py +652 -0
- opscloud/ui/widgets/debug_console.py +1269 -0
- opscloud/ui/widgets/devops_renderers.py +184 -0
- opscloud/ui/widgets/diff.py +237 -0
- opscloud/ui/widgets/effort_selector.py +166 -0
- opscloud/ui/widgets/goal_review.py +612 -0
- opscloud/ui/widgets/goal_status.py +57 -0
- opscloud/ui/widgets/infra_panel.py +141 -0
- opscloud/ui/widgets/install_confirm.py +114 -0
- opscloud/ui/widgets/loading.py +170 -0
- opscloud/ui/widgets/mcp_viewer.py +1204 -0
- opscloud/ui/widgets/messages.py +4461 -0
- opscloud/ui/widgets/model_selector.py +863 -0
- opscloud/ui/widgets/notification_center.py +108 -0
- opscloud/ui/widgets/notification_settings.py +119 -0
- opscloud/ui/widgets/operation_card.py +106 -0
- opscloud/ui/widgets/permissions_manager.py +667 -0
- opscloud/ui/widgets/plugin_manager.py +1212 -0
- opscloud/ui/widgets/pool_selector.py +376 -0
- opscloud/ui/widgets/preamble.py +53 -0
- opscloud/ui/widgets/skills_viewer.py +432 -0
- opscloud/ui/widgets/status.py +1037 -0
- opscloud/ui/widgets/subagent_panel.py +927 -0
- opscloud/ui/widgets/theme_selector.py +232 -0
- opscloud/ui/widgets/thread_selector.py +358 -0
- opscloud/ui/widgets/toast.py +31 -0
- opscloud/ui/widgets/tool_display.py +325 -0
- opscloud/ui/widgets/tool_renderers.py +234 -0
- opscloud/ui/widgets/tool_widgets.py +278 -0
- opscloud/ui/widgets/welcome.py +421 -0
- opscloud/ui/widgets/welcome_popup.py +102 -0
- opscloud/utils/__init__.py +24 -0
- opscloud/utils/cost_estimation.py +169 -0
- opscloud/utils/git.py +390 -0
- opscloud/utils/logger.py +571 -0
- opscloud/utils/session_stats.py +115 -0
- opscloud/utils/startup_error.py +37 -0
- talkops_opscloud-0.3.0b1.dist-info/METADATA +449 -0
- talkops_opscloud-0.3.0b1.dist-info/RECORD +309 -0
- talkops_opscloud-0.3.0b1.dist-info/WHEEL +4 -0
- talkops_opscloud-0.3.0b1.dist-info/entry_points.txt +2 -0
- talkops_opscloud-0.3.0b1.dist-info/licenses/LICENSE +175 -0
|
@@ -0,0 +1,297 @@
|
|
|
1
|
+
"""Jev-powered System One rubric grader and hybrid evaluator.
|
|
2
|
+
|
|
3
|
+
Performs ultra-fast (<200ms) parallel criteria verification using TypeSafe AI Jev,
|
|
4
|
+
eliminating autoregressive LLM judge variance and cost, while providing transparent
|
|
5
|
+
Tier 2 generative escalation and environmental blocker detection.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import asyncio
|
|
11
|
+
from collections.abc import Awaitable, Callable, Mapping, Sequence
|
|
12
|
+
import inspect
|
|
13
|
+
import time
|
|
14
|
+
from typing import Any, cast
|
|
15
|
+
|
|
16
|
+
from deepagents.middleware.rubric import (
|
|
17
|
+
CriterionEval,
|
|
18
|
+
CriterionFail,
|
|
19
|
+
CriterionPass,
|
|
20
|
+
GraderResponse,
|
|
21
|
+
)
|
|
22
|
+
from langchain_core.runnables import RunnableConfig
|
|
23
|
+
from langchain_typesafe import (
|
|
24
|
+
ClassifierRequest,
|
|
25
|
+
ClassifierResponse,
|
|
26
|
+
State,
|
|
27
|
+
TypeSafeClassifier,
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
from opscloud.config.settings import get_settings, resolve_env_var
|
|
31
|
+
from opscloud.rubrics.evidence_extractor import extract_grading_evidence
|
|
32
|
+
from opscloud.rubrics.jev_compiler import CompiledRubric
|
|
33
|
+
from opscloud.utils.logger import get_logger
|
|
34
|
+
|
|
35
|
+
logger = get_logger(__name__)
|
|
36
|
+
|
|
37
|
+
# ── Declarative Dispatch & Threshold Rules ─────────────────────────────────
|
|
38
|
+
|
|
39
|
+
_CRITERION_PASS_THRESHOLD = 0.50
|
|
40
|
+
_MIN_READINESS_SCORE = 1.0
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def build_criteria_evaluations(
|
|
44
|
+
criteria_map: Mapping[str, str],
|
|
45
|
+
response: ClassifierResponse,
|
|
46
|
+
pass_threshold: float = _CRITERION_PASS_THRESHOLD,
|
|
47
|
+
) -> list[CriterionEval]:
|
|
48
|
+
"""Map Jev Noul probabilities into deepagents CriterionPass / CriterionFail list."""
|
|
49
|
+
evals: list[CriterionEval] = []
|
|
50
|
+
for cid, text in criteria_map.items():
|
|
51
|
+
noul_answer = response.nouls.get(cid)
|
|
52
|
+
prob = noul_answer.noul if noul_answer is not None else 0.0
|
|
53
|
+
if prob >= pass_threshold:
|
|
54
|
+
evals.append(CriterionPass(name=text, passed=True))
|
|
55
|
+
else:
|
|
56
|
+
evals.append(
|
|
57
|
+
CriterionFail(
|
|
58
|
+
name=text,
|
|
59
|
+
passed=False,
|
|
60
|
+
gap=f"Criterion unsatisfied (confidence {prob:.2f} < {pass_threshold:.2f}): {text}",
|
|
61
|
+
)
|
|
62
|
+
)
|
|
63
|
+
return evals
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
BlockerHandler = Callable[[ClassifierResponse, list[CriterionEval]], GraderResponse]
|
|
67
|
+
|
|
68
|
+
def _make_blocker_response(
|
|
69
|
+
blocker_type: str,
|
|
70
|
+
explanation: str,
|
|
71
|
+
evals: list[CriterionEval],
|
|
72
|
+
) -> GraderResponse:
|
|
73
|
+
resp = GraderResponse(
|
|
74
|
+
result="needs_revision",
|
|
75
|
+
explanation=explanation,
|
|
76
|
+
criteria=evals,
|
|
77
|
+
)
|
|
78
|
+
object.__setattr__(resp, "blocker_type", blocker_type)
|
|
79
|
+
return resp
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
BLOCKER_DISPATCH: dict[str, BlockerHandler] = {
|
|
83
|
+
"external_blocker": lambda resp, evals: _make_blocker_response(
|
|
84
|
+
"external_blocker",
|
|
85
|
+
(
|
|
86
|
+
"Environmental blocker detected by Jev (e.g. AWS service quota, "
|
|
87
|
+
"IAM permissions, or network partition). Proposing dynamic rubric amendment."
|
|
88
|
+
),
|
|
89
|
+
evals,
|
|
90
|
+
),
|
|
91
|
+
"ambiguous_requirement": lambda resp, evals: _make_blocker_response(
|
|
92
|
+
"ambiguous_requirement",
|
|
93
|
+
(
|
|
94
|
+
"Criteria cannot be verified because the requirements conflict with "
|
|
95
|
+
"environmental constraints or are ambiguous. Operator clarification required."
|
|
96
|
+
),
|
|
97
|
+
evals,
|
|
98
|
+
),
|
|
99
|
+
"progressing": lambda resp, evals: (
|
|
100
|
+
GraderResponse(
|
|
101
|
+
result="satisfied",
|
|
102
|
+
explanation=(
|
|
103
|
+
f"All {len(evals)} criteria successfully verified via Jev System One "
|
|
104
|
+
f"(Operational readiness: {getattr(resp.scores.get('readiness'), 'score', 2.0):.2f})."
|
|
105
|
+
),
|
|
106
|
+
criteria=evals,
|
|
107
|
+
)
|
|
108
|
+
if all(c.get("passed", False) for c in evals)
|
|
109
|
+
and getattr(resp.scores.get("readiness"), "score", 2.0) >= _MIN_READINESS_SCORE
|
|
110
|
+
else GraderResponse(
|
|
111
|
+
result="needs_revision",
|
|
112
|
+
explanation=(
|
|
113
|
+
f"{sum(1 for c in evals if not c.get('passed', False))} of {len(evals)} criteria "
|
|
114
|
+
"not yet satisfied. Revision required."
|
|
115
|
+
),
|
|
116
|
+
criteria=evals,
|
|
117
|
+
)
|
|
118
|
+
),
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
# ── Jev Hybrid Rubric Grader ───────────────────────────────────────────────
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
class JevHybridRubricGrader:
|
|
126
|
+
"""Two-tier rubric evaluator: Jev System One fast-pass + Frontier LLM diagnostic fallback."""
|
|
127
|
+
|
|
128
|
+
def __init__(
|
|
129
|
+
self,
|
|
130
|
+
api_key: str | None = None,
|
|
131
|
+
timeout_seconds: float = 2.0,
|
|
132
|
+
) -> None:
|
|
133
|
+
"""Initialize Jev rubric grader with API key resolution."""
|
|
134
|
+
if api_key is not None:
|
|
135
|
+
self._api_key = api_key if api_key else None
|
|
136
|
+
self._explicit_key = True
|
|
137
|
+
else:
|
|
138
|
+
settings_key = getattr(get_settings(), "typesafe_api_key", None)
|
|
139
|
+
resolved_key = settings_key or resolve_env_var("TYPESAFE_API_KEY", fallback_names=("JEV_API_KEY",))
|
|
140
|
+
self._api_key = resolved_key
|
|
141
|
+
self._explicit_key = False
|
|
142
|
+
|
|
143
|
+
self.timeout_seconds = timeout_seconds
|
|
144
|
+
self._classifier: TypeSafeClassifier | None = None
|
|
145
|
+
|
|
146
|
+
def _get_classifier(self) -> TypeSafeClassifier:
|
|
147
|
+
"""Lazily initialize TypeSafeClassifier instance."""
|
|
148
|
+
if self._classifier is None:
|
|
149
|
+
if self._api_key:
|
|
150
|
+
self._classifier = TypeSafeClassifier(api_key=self._api_key)
|
|
151
|
+
else:
|
|
152
|
+
self._classifier = TypeSafeClassifier()
|
|
153
|
+
return self._classifier
|
|
154
|
+
|
|
155
|
+
def is_available(self) -> bool:
|
|
156
|
+
"""Check whether TypeSafe API credentials are configured."""
|
|
157
|
+
if self._explicit_key:
|
|
158
|
+
return bool(self._api_key)
|
|
159
|
+
if self._api_key:
|
|
160
|
+
return True
|
|
161
|
+
settings_key = getattr(get_settings(), "typesafe_api_key", None)
|
|
162
|
+
if settings_key:
|
|
163
|
+
return True
|
|
164
|
+
return bool(resolve_env_var("TYPESAFE_API_KEY", fallback_names=("JEV_API_KEY",)))
|
|
165
|
+
|
|
166
|
+
async def agrade(
|
|
167
|
+
self,
|
|
168
|
+
compiled_rubric: CompiledRubric,
|
|
169
|
+
evidence: dict[str, Any] | Sequence[Any] | str,
|
|
170
|
+
*,
|
|
171
|
+
fallback_llm_fn: Callable[[list[str]], Awaitable[GraderResponse] | GraderResponse] | None = None,
|
|
172
|
+
) -> GraderResponse:
|
|
173
|
+
"""Evaluate criteria against evidence asynchronously using Jev System One.
|
|
174
|
+
|
|
175
|
+
Escalates to `fallback_llm_fn` only when a criterion fails and detailed remediation
|
|
176
|
+
is requested.
|
|
177
|
+
"""
|
|
178
|
+
classifier = self._get_classifier()
|
|
179
|
+
start_time = time.monotonic()
|
|
180
|
+
|
|
181
|
+
# Build clean state payload for Jev
|
|
182
|
+
if isinstance(evidence, dict) and "commands_executed" in evidence:
|
|
183
|
+
structured_evidence = evidence
|
|
184
|
+
else:
|
|
185
|
+
structured_evidence = extract_grading_evidence(
|
|
186
|
+
evidence if isinstance(evidence, (Mapping, Sequence)) and not isinstance(evidence, (str, bytes)) else [evidence]
|
|
187
|
+
)
|
|
188
|
+
|
|
189
|
+
state_payload: State = cast(
|
|
190
|
+
State,
|
|
191
|
+
{
|
|
192
|
+
"objective": compiled_rubric.objective,
|
|
193
|
+
"evidence": structured_evidence,
|
|
194
|
+
},
|
|
195
|
+
)
|
|
196
|
+
|
|
197
|
+
request: ClassifierRequest = {
|
|
198
|
+
"state": state_payload,
|
|
199
|
+
"questions": compiled_rubric.questions,
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
run_config: RunnableConfig = {
|
|
203
|
+
"run_name": "jev_rubric_grader",
|
|
204
|
+
"tags": ["opscloud:smart", "opscloud:jev", "opscloud:rubric"],
|
|
205
|
+
"metadata": {
|
|
206
|
+
"objective": compiled_rubric.objective[:100],
|
|
207
|
+
"criteria_count": len(compiled_rubric.criteria_map),
|
|
208
|
+
},
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
try:
|
|
212
|
+
async with asyncio.timeout(self.timeout_seconds):
|
|
213
|
+
response: ClassifierResponse = await classifier.ainvoke(
|
|
214
|
+
request,
|
|
215
|
+
config=run_config,
|
|
216
|
+
)
|
|
217
|
+
except Exception as exc:
|
|
218
|
+
latency_ms = int((time.monotonic() - start_time) * 1000)
|
|
219
|
+
logger.warning("Jev rubric evaluation failed or timed out (%d ms): %s", latency_ms, exc)
|
|
220
|
+
raise
|
|
221
|
+
|
|
222
|
+
latency_ms = int((time.monotonic() - start_time) * 1000)
|
|
223
|
+
criteria_evals = build_criteria_evaluations(compiled_rubric.criteria_map, response)
|
|
224
|
+
|
|
225
|
+
blocker = response.choices.get("blocker_status")
|
|
226
|
+
blocker_key = blocker.choice if blocker else "progressing"
|
|
227
|
+
|
|
228
|
+
handler = BLOCKER_DISPATCH.get(blocker_key, BLOCKER_DISPATCH["progressing"])
|
|
229
|
+
verdict = handler(response, criteria_evals)
|
|
230
|
+
|
|
231
|
+
readiness = getattr(response.scores.get("readiness"), "score", 0.0)
|
|
232
|
+
logger.info(
|
|
233
|
+
"Jev rubric fast-pass completed in %d ms: result=%s, readiness=%.2f, blocker=%s, passed=%d/%d",
|
|
234
|
+
latency_ms,
|
|
235
|
+
verdict.result,
|
|
236
|
+
readiness,
|
|
237
|
+
blocker_key,
|
|
238
|
+
sum(1 for c in criteria_evals if c.get("passed", False)),
|
|
239
|
+
len(criteria_evals),
|
|
240
|
+
)
|
|
241
|
+
|
|
242
|
+
# ── Tier 2 Diagnostic Escalation ────────────────────────────────────
|
|
243
|
+
# When criteria fail under normal execution, escalate to generative LLM
|
|
244
|
+
# to author targeted remediation feedback for failing requirements.
|
|
245
|
+
failed_items = [c.get("name", "") for c in criteria_evals if not c.get("passed", False)]
|
|
246
|
+
if not failed_items and readiness < _MIN_READINESS_SCORE:
|
|
247
|
+
failed_items = [f"Operational readiness score ({readiness:.2f}) below threshold ({_MIN_READINESS_SCORE:.2f})"]
|
|
248
|
+
|
|
249
|
+
if verdict.result == "needs_revision" and failed_items and fallback_llm_fn and blocker_key == "progressing":
|
|
250
|
+
logger.info(
|
|
251
|
+
"Jev fast-pass detected %d failing criteria; invoking Tier 2 LLM diagnostic for remediation advice.",
|
|
252
|
+
len(failed_items),
|
|
253
|
+
)
|
|
254
|
+
try:
|
|
255
|
+
llm_res = fallback_llm_fn(failed_items)
|
|
256
|
+
if inspect.isawaitable(llm_res):
|
|
257
|
+
llm_verdict = await llm_res
|
|
258
|
+
else:
|
|
259
|
+
llm_verdict = llm_res
|
|
260
|
+
if isinstance(llm_verdict, GraderResponse):
|
|
261
|
+
if not llm_verdict.criteria:
|
|
262
|
+
object.__setattr__(llm_verdict, "criteria", criteria_evals)
|
|
263
|
+
return llm_verdict
|
|
264
|
+
except Exception as llm_exc:
|
|
265
|
+
logger.warning("Tier 2 LLM diagnostic fallback failed: %s; returning Jev verdict", llm_exc)
|
|
266
|
+
|
|
267
|
+
return verdict
|
|
268
|
+
|
|
269
|
+
def grade(
|
|
270
|
+
self,
|
|
271
|
+
compiled_rubric: CompiledRubric,
|
|
272
|
+
evidence: dict[str, Any] | Sequence[Any] | str,
|
|
273
|
+
*,
|
|
274
|
+
fallback_llm_fn: Callable[[list[str]], GraderResponse] | None = None,
|
|
275
|
+
) -> GraderResponse:
|
|
276
|
+
"""Synchronous version of grade."""
|
|
277
|
+
try:
|
|
278
|
+
loop = asyncio.get_running_loop()
|
|
279
|
+
except RuntimeError:
|
|
280
|
+
loop = None
|
|
281
|
+
|
|
282
|
+
if loop is not None and loop.is_running():
|
|
283
|
+
import concurrent.futures
|
|
284
|
+
|
|
285
|
+
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:
|
|
286
|
+
return executor.submit(
|
|
287
|
+
asyncio.run,
|
|
288
|
+
self.agrade(compiled_rubric, evidence, fallback_llm_fn=fallback_llm_fn), # type: ignore[arg-type]
|
|
289
|
+
).result()
|
|
290
|
+
return asyncio.run(self.agrade(compiled_rubric, evidence, fallback_llm_fn=fallback_llm_fn)) # type: ignore[arg-type]
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
__all__ = [
|
|
294
|
+
"BLOCKER_DISPATCH",
|
|
295
|
+
"JevHybridRubricGrader",
|
|
296
|
+
"build_criteria_evaluations",
|
|
297
|
+
]
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
"""Schema definitions for OpsCloud."""
|
|
2
|
+
|
|
3
|
+
from opscloud.schema.interrupts import (
|
|
4
|
+
# Tool Approvals / HITL
|
|
5
|
+
ActionRequest,
|
|
6
|
+
ApproveDecision,
|
|
7
|
+
AutoApproveAllDecision,
|
|
8
|
+
EditDecision,
|
|
9
|
+
HITLRequest,
|
|
10
|
+
HitlDecision,
|
|
11
|
+
HitlResumePayload,
|
|
12
|
+
RejectDecision,
|
|
13
|
+
ReviewConfig,
|
|
14
|
+
SmartApproveAllDecision,
|
|
15
|
+
SwitchManualDecision,
|
|
16
|
+
# Ask User Protocol
|
|
17
|
+
ASK_USER_ANSWERED_SUMMARY,
|
|
18
|
+
ASK_USER_CANCELLED_ANSWER,
|
|
19
|
+
ASK_USER_CANCELLED_SUMMARY,
|
|
20
|
+
ASK_USER_ERROR_ANSWER_PREFIX,
|
|
21
|
+
ASK_USER_FAILED_SUMMARY,
|
|
22
|
+
ASK_USER_NOTHING_SELECTED,
|
|
23
|
+
ASK_USER_NO_ANSWER,
|
|
24
|
+
AskUserAnswered,
|
|
25
|
+
AskUserCancelled,
|
|
26
|
+
AskUserRequest,
|
|
27
|
+
AskUserResumePayload,
|
|
28
|
+
AskUserWidgetResult,
|
|
29
|
+
CHOICE_QUESTION_TYPES,
|
|
30
|
+
Choice,
|
|
31
|
+
QUESTION_TYPES,
|
|
32
|
+
Question,
|
|
33
|
+
QuestionType,
|
|
34
|
+
ValidatedQuestion,
|
|
35
|
+
ask_user_answer_is_empty,
|
|
36
|
+
decode_multi_select_answer,
|
|
37
|
+
encode_multi_select_answer,
|
|
38
|
+
format_ask_user_error_answer,
|
|
39
|
+
format_ask_user_transcript,
|
|
40
|
+
render_ask_user_transcript_for_display,
|
|
41
|
+
# Goal & Criteria Review
|
|
42
|
+
GoalReviewRequest,
|
|
43
|
+
GoalReviewResumePayload,
|
|
44
|
+
# Hook Invocation Interrupts
|
|
45
|
+
HOOK_INVOCATION_INTERRUPT_TYPE,
|
|
46
|
+
HookInvocationInterrupt,
|
|
47
|
+
HookInvocationRequest,
|
|
48
|
+
HookInvocationResponse,
|
|
49
|
+
build_hook_interrupt_payload,
|
|
50
|
+
build_hook_resume_value,
|
|
51
|
+
is_hook_interrupt_payload,
|
|
52
|
+
parse_hook_interrupt_payload,
|
|
53
|
+
parse_hook_resume_value,
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
__all__ = [
|
|
57
|
+
# HITL & Tool Approval
|
|
58
|
+
"ActionRequest",
|
|
59
|
+
"ApproveDecision",
|
|
60
|
+
"AutoApproveAllDecision",
|
|
61
|
+
"EditDecision",
|
|
62
|
+
"HITLRequest",
|
|
63
|
+
"HitlDecision",
|
|
64
|
+
"HitlResumePayload",
|
|
65
|
+
"RejectDecision",
|
|
66
|
+
"ReviewConfig",
|
|
67
|
+
"SmartApproveAllDecision",
|
|
68
|
+
"SwitchManualDecision",
|
|
69
|
+
# Ask User Protocol
|
|
70
|
+
"ASK_USER_ANSWERED_SUMMARY",
|
|
71
|
+
"ASK_USER_CANCELLED_ANSWER",
|
|
72
|
+
"ASK_USER_CANCELLED_SUMMARY",
|
|
73
|
+
"ASK_USER_ERROR_ANSWER_PREFIX",
|
|
74
|
+
"ASK_USER_FAILED_SUMMARY",
|
|
75
|
+
"ASK_USER_NOTHING_SELECTED",
|
|
76
|
+
"ASK_USER_NO_ANSWER",
|
|
77
|
+
"AskUserAnswered",
|
|
78
|
+
"AskUserCancelled",
|
|
79
|
+
"AskUserRequest",
|
|
80
|
+
"AskUserResumePayload",
|
|
81
|
+
"AskUserWidgetResult",
|
|
82
|
+
"CHOICE_QUESTION_TYPES",
|
|
83
|
+
"Choice",
|
|
84
|
+
"QUESTION_TYPES",
|
|
85
|
+
"Question",
|
|
86
|
+
"QuestionType",
|
|
87
|
+
"ValidatedQuestion",
|
|
88
|
+
"ask_user_answer_is_empty",
|
|
89
|
+
"decode_multi_select_answer",
|
|
90
|
+
"encode_multi_select_answer",
|
|
91
|
+
"format_ask_user_error_answer",
|
|
92
|
+
"format_ask_user_transcript",
|
|
93
|
+
"render_ask_user_transcript_for_display",
|
|
94
|
+
# Goal & Criteria Review
|
|
95
|
+
"GoalReviewRequest",
|
|
96
|
+
"GoalReviewResumePayload",
|
|
97
|
+
# Hook Invocation Interrupts
|
|
98
|
+
"HOOK_INVOCATION_INTERRUPT_TYPE",
|
|
99
|
+
"HookInvocationInterrupt",
|
|
100
|
+
"HookInvocationRequest",
|
|
101
|
+
"HookInvocationResponse",
|
|
102
|
+
"build_hook_interrupt_payload",
|
|
103
|
+
"build_hook_resume_value",
|
|
104
|
+
"is_hook_interrupt_payload",
|
|
105
|
+
"parse_hook_interrupt_payload",
|
|
106
|
+
"parse_hook_resume_value",
|
|
107
|
+
]
|