talkops-opscloud 0.3.0b1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- opscloud/__init__.py +5 -0
- opscloud/__main__.py +6 -0
- opscloud/_constants.py +53 -0
- opscloud/_debug.py +145 -0
- opscloud/_debug_buffer.py +250 -0
- opscloud/_textual_patches.py +367 -0
- opscloud/_version.py +15 -0
- opscloud/agent/__init__.py +27 -0
- opscloud/agent/config.py +168 -0
- opscloud/agent/factory.py +1103 -0
- opscloud/approval_mode.py +44 -0
- opscloud/backend/__init__.py +49 -0
- opscloud/backend/composite.py +89 -0
- opscloud/backend/local.py +183 -0
- opscloud/backend/registry.py +79 -0
- opscloud/backend/sandbox_config.py +107 -0
- opscloud/backend/sandbox_factory.py +432 -0
- opscloud/built_in_skills/remember/SKILL.md +124 -0
- opscloud/built_in_skills/skill-creator/SKILL.md +348 -0
- opscloud/built_in_skills/skill-creator/scripts/init_skill.py +371 -0
- opscloud/built_in_skills/skill-creator/scripts/quick_validate.py +165 -0
- opscloud/cli/__init__.py +1 -0
- opscloud/cli/commands/__init__.py +15 -0
- opscloud/cli/commands/auth.py +45 -0
- opscloud/cli/commands/doctor.py +100 -0
- opscloud/cli/commands/mcp.py +137 -0
- opscloud/cli/commands/threads.py +51 -0
- opscloud/cli/interactive.py +154 -0
- opscloud/cli/main.py +903 -0
- opscloud/cli/non_interactive.py +425 -0
- opscloud/cli/server_manager.py +58 -0
- opscloud/commands/__init__.py +19 -0
- opscloud/commands/_base.py +110 -0
- opscloud/commands/_guards.py +52 -0
- opscloud/commands/_router.py +146 -0
- opscloud/commands/_types.py +36 -0
- opscloud/commands/core/__init__.py +47 -0
- opscloud/commands/core/auth.py +94 -0
- opscloud/commands/core/bug.py +46 -0
- opscloud/commands/core/clear.py +106 -0
- opscloud/commands/core/cloud.py +185 -0
- opscloud/commands/core/compact.py +128 -0
- opscloud/commands/core/config_cmd.py +120 -0
- opscloud/commands/core/context.py +220 -0
- opscloud/commands/core/cost.py +44 -0
- opscloud/commands/core/doctor.py +535 -0
- opscloud/commands/core/effort.py +108 -0
- opscloud/commands/core/exit_cmd.py +41 -0
- opscloud/commands/core/fast.py +148 -0
- opscloud/commands/core/help_cmd.py +94 -0
- opscloud/commands/core/mcp.py +134 -0
- opscloud/commands/core/model.py +109 -0
- opscloud/commands/core/permissions.py +196 -0
- opscloud/commands/core/plugins.py +92 -0
- opscloud/commands/core/pool.py +108 -0
- opscloud/commands/core/resume.py +97 -0
- opscloud/commands/core/skills.py +178 -0
- opscloud/commands/power/__init__.py +51 -0
- opscloud/commands/power/agents.py +98 -0
- opscloud/commands/power/btw.py +85 -0
- opscloud/commands/power/copy.py +97 -0
- opscloud/commands/power/goal.py +548 -0
- opscloud/commands/power/loop.py +295 -0
- opscloud/commands/power/memory.py +287 -0
- opscloud/commands/power/review.py +152 -0
- opscloud/commands/power/rubric.py +287 -0
- opscloud/commands/power/runtime.py +321 -0
- opscloud/commands/power/skill_creator.py +70 -0
- opscloud/commands/power/skill_invoke.py +165 -0
- opscloud/commands/power/tasks.py +209 -0
- opscloud/commands/power/trace.py +117 -0
- opscloud/commands/power/ui_toggles.py +135 -0
- opscloud/commands/power/version.py +52 -0
- opscloud/commands/registry.py +176 -0
- opscloud/config/__init__.py +114 -0
- opscloud/config/adapters/__init__.py +5 -0
- opscloud/config/adapters/postgres.py +776 -0
- opscloud/config/adapters/sqlite.py +769 -0
- opscloud/config/aws.py +248 -0
- opscloud/config/cloud_profiles.py +437 -0
- opscloud/config/env_vars.py +81 -0
- opscloud/config/langsmith.py +319 -0
- opscloud/config/manifest.py +817 -0
- opscloud/config/metadata.py +205 -0
- opscloud/config/paths.py +604 -0
- opscloud/config/plugins.py +130 -0
- opscloud/config/settings.py +834 -0
- opscloud/config/store.py +453 -0
- opscloud/config/store_factory.py +97 -0
- opscloud/config/toml_config.py +562 -0
- opscloud/editor.py +173 -0
- opscloud/exceptions.py +65 -0
- opscloud/file_ops.py +217 -0
- opscloud/hooks/__init__.py +139 -0
- opscloud/hooks/env.py +44 -0
- opscloud/hooks/interrupt.py +121 -0
- opscloud/hooks/models/__init__.py +109 -0
- opscloud/hooks/models/adapters.py +19 -0
- opscloud/hooks/models/config.py +79 -0
- opscloud/hooks/models/domain.py +434 -0
- opscloud/hooks/models/transport.py +37 -0
- opscloud/hooks/tools.py +231 -0
- opscloud/input.py +48 -0
- opscloud/integrations/__init__.py +80 -0
- opscloud/integrations/base.py +153 -0
- opscloud/integrations/content_filter.py +119 -0
- opscloud/integrations/event_bus.py +480 -0
- opscloud/integrations/hooks.py +238 -0
- opscloud/integrations/identity.py +181 -0
- opscloud/integrations/notifications.py +134 -0
- opscloud/integrations/stream_bridge.py +397 -0
- opscloud/integrations/thread_store.py +81 -0
- opscloud/json_types.py +20 -0
- opscloud/mcp/__init__.py +57 -0
- opscloud/mcp/config.py +203 -0
- opscloud/mcp/discovery.py +229 -0
- opscloud/mcp/mcp_info.py +109 -0
- opscloud/mcp/middleware.py +102 -0
- opscloud/mcp/preload.py +505 -0
- opscloud/mcp/raw_config.py +149 -0
- opscloud/mcp/semantic_profiler.py +305 -0
- opscloud/mcp/session_manager.py +727 -0
- opscloud/mcp/trust.py +105 -0
- opscloud/memory/__init__.py +27 -0
- opscloud/memory/branch.py +61 -0
- opscloud/memory/guard.py +19 -0
- opscloud/memory/onboarding.py +120 -0
- opscloud/memory/registry.py +134 -0
- opscloud/memory/store.py +223 -0
- opscloud/middleware/__init__.py +106 -0
- opscloud/middleware/_repository_bounds.py +459 -0
- opscloud/middleware/ask_user.py +152 -0
- opscloud/middleware/auto_mode.py +60 -0
- opscloud/middleware/auto_mode_hitl.py +2434 -0
- opscloud/middleware/compaction.py +566 -0
- opscloud/middleware/configurable_model.py +233 -0
- opscloud/middleware/cost_tracking.py +1021 -0
- opscloud/middleware/glm_stall_recovery.py +109 -0
- opscloud/middleware/goal_criteria.py +1234 -0
- opscloud/middleware/goal_state_notice.py +712 -0
- opscloud/middleware/goal_tools.py +407 -0
- opscloud/middleware/headless_mcp_guard.py +88 -0
- opscloud/middleware/jev_model_router.py +1018 -0
- opscloud/middleware/local_context.py +460 -0
- opscloud/middleware/mcp_context.py +151 -0
- opscloud/middleware/mcp_middleware.py +146 -0
- opscloud/middleware/memory_guard.py +393 -0
- opscloud/middleware/model_retry.py +205 -0
- opscloud/middleware/registry.py +137 -0
- opscloud/middleware/reliable_rubric.py +709 -0
- opscloud/middleware/resume_state.py +91 -0
- opscloud/middleware/server_hooks.py +1189 -0
- opscloud/middleware/shell_allow_list.py +90 -0
- opscloud/middleware/skills.py +382 -0
- opscloud/middleware/subagent_artifacts.py +84 -0
- opscloud/middleware/subagent_telemetry.py +261 -0
- opscloud/middleware/subagents.py +243 -0
- opscloud/middleware/tool_filter.py +384 -0
- opscloud/middleware/unified_system_message.py +62 -0
- opscloud/model/__init__.py +37 -0
- opscloud/model/config.py +2992 -0
- opscloud/model/factory.py +477 -0
- opscloud/model/pool.py +459 -0
- opscloud/model/reasoning.py +282 -0
- opscloud/offload.py +173 -0
- opscloud/output.py +64 -0
- opscloud/plugins/__init__.py +189 -0
- opscloud/plugins/_json.py +22 -0
- opscloud/plugins/adapters/__init__.py +13 -0
- opscloud/plugins/adapters/agents.py +112 -0
- opscloud/plugins/adapters/commands.py +190 -0
- opscloud/plugins/adapters/mcp.py +363 -0
- opscloud/plugins/adapters/skills.py +132 -0
- opscloud/plugins/commands_cli.py +541 -0
- opscloud/plugins/discovery.py +512 -0
- opscloud/plugins/manifest.py +335 -0
- opscloud/plugins/marketplace.py +893 -0
- opscloud/plugins/models.py +334 -0
- opscloud/plugins/project_plugins.py +317 -0
- opscloud/plugins/protocol.py +31 -0
- opscloud/plugins/store.py +905 -0
- opscloud/plugins/substitution.py +122 -0
- opscloud/project_utils.py +145 -0
- opscloud/prompts/__init__.py +37 -0
- opscloud/prompts/system.py +483 -0
- opscloud/prompts/templates/system_prompt.md +166 -0
- opscloud/prompts/types.py +54 -0
- opscloud/rubrics/__init__.py +42 -0
- opscloud/rubrics/evaluator.py +404 -0
- opscloud/rubrics/evidence_extractor.py +229 -0
- opscloud/rubrics/generator.py +239 -0
- opscloud/rubrics/jev_compiler.py +175 -0
- opscloud/rubrics/jev_grader.py +297 -0
- opscloud/rubrics/middleware.py +8 -0
- opscloud/schema/__init__.py +107 -0
- opscloud/schema/interrupts.py +981 -0
- opscloud/security/__init__.py +91 -0
- opscloud/security/approval_mode.py +528 -0
- opscloud/security/approval_mode_source.py +230 -0
- opscloud/security/cli_ast_evaluator.py +552 -0
- opscloud/security/jev_classifier.py +651 -0
- opscloud/security/shell_safety.py +340 -0
- opscloud/security/unicode_security.py +439 -0
- opscloud/security/url_validation.py +160 -0
- opscloud/server/__init__.py +13 -0
- opscloud/server/_server_config.py +158 -0
- opscloud/server/server.py +242 -0
- opscloud/server/server_graph.py +93 -0
- opscloud/skills/__init__.py +91 -0
- opscloud/skills/commands.py +1099 -0
- opscloud/skills/invocation.py +200 -0
- opscloud/skills/load.py +388 -0
- opscloud/skills/loader.py +23 -0
- opscloud/skills/merge.py +65 -0
- opscloud/skills/registry.py +263 -0
- opscloud/skills/sources.py +174 -0
- opscloud/skills/trust.py +417 -0
- opscloud/state/__init__.py +142 -0
- opscloud/state/base.py +154 -0
- opscloud/state/cloud_context.py +262 -0
- opscloud/state/goal_channels.py +124 -0
- opscloud/state/goal_state_limits.py +175 -0
- opscloud/state/resume_state.py +58 -0
- opscloud/state/service_context.py +281 -0
- opscloud/state/session.py +957 -0
- opscloud/state/state_migration.py +86 -0
- opscloud/subagents/__init__.py +30 -0
- opscloud/subagents/loader.py +331 -0
- opscloud/subagents/subagents_parser.py +117 -0
- opscloud/subagents/types.py +25 -0
- opscloud/tools/__init__.py +36 -0
- opscloud/tools/catalog.py +37 -0
- opscloud/tools/display.py +24 -0
- opscloud/tools/fetch_url.py +176 -0
- opscloud/tools/goal_tools.py +423 -0
- opscloud/tools/registry.py +105 -0
- opscloud/tools/thread.py +32 -0
- opscloud/tools/web_search.py +215 -0
- opscloud/ui/__init__.py +119 -0
- opscloud/ui/app.py +5400 -0
- opscloud/ui/app.tcss +479 -0
- opscloud/ui/clipboard.py +147 -0
- opscloud/ui/command_registry.py +482 -0
- opscloud/ui/message_store.py +152 -0
- opscloud/ui/permission_store.py +360 -0
- opscloud/ui/preferences.py +113 -0
- opscloud/ui/prompt_manager.py +96 -0
- opscloud/ui/remote_client.py +505 -0
- opscloud/ui/textual_adapter.py +1269 -0
- opscloud/ui/theme.py +728 -0
- opscloud/ui/ui_help.py +415 -0
- opscloud/ui/widgets/__init__.py +383 -0
- opscloud/ui/widgets/_ask_user_types.py +81 -0
- opscloud/ui/widgets/_copy_spans.py +52 -0
- opscloud/ui/widgets/_inline_prompt.py +290 -0
- opscloud/ui/widgets/_js_eval_display.py +139 -0
- opscloud/ui/widgets/_links.py +175 -0
- opscloud/ui/widgets/_tool_stream.py +308 -0
- opscloud/ui/widgets/agent_selector.py +297 -0
- opscloud/ui/widgets/approval.py +741 -0
- opscloud/ui/widgets/ask_user.py +539 -0
- opscloud/ui/widgets/auth.py +695 -0
- opscloud/ui/widgets/auth_manager.py +214 -0
- opscloud/ui/widgets/auto_mode_notice.py +146 -0
- opscloud/ui/widgets/autocomplete.py +709 -0
- opscloud/ui/widgets/chat_input.py +805 -0
- opscloud/ui/widgets/cloud_selector.py +528 -0
- opscloud/ui/widgets/config_manager.py +652 -0
- opscloud/ui/widgets/debug_console.py +1269 -0
- opscloud/ui/widgets/devops_renderers.py +184 -0
- opscloud/ui/widgets/diff.py +237 -0
- opscloud/ui/widgets/effort_selector.py +166 -0
- opscloud/ui/widgets/goal_review.py +612 -0
- opscloud/ui/widgets/goal_status.py +57 -0
- opscloud/ui/widgets/infra_panel.py +141 -0
- opscloud/ui/widgets/install_confirm.py +114 -0
- opscloud/ui/widgets/loading.py +170 -0
- opscloud/ui/widgets/mcp_viewer.py +1204 -0
- opscloud/ui/widgets/messages.py +4461 -0
- opscloud/ui/widgets/model_selector.py +863 -0
- opscloud/ui/widgets/notification_center.py +108 -0
- opscloud/ui/widgets/notification_settings.py +119 -0
- opscloud/ui/widgets/operation_card.py +106 -0
- opscloud/ui/widgets/permissions_manager.py +667 -0
- opscloud/ui/widgets/plugin_manager.py +1212 -0
- opscloud/ui/widgets/pool_selector.py +376 -0
- opscloud/ui/widgets/preamble.py +53 -0
- opscloud/ui/widgets/skills_viewer.py +432 -0
- opscloud/ui/widgets/status.py +1037 -0
- opscloud/ui/widgets/subagent_panel.py +927 -0
- opscloud/ui/widgets/theme_selector.py +232 -0
- opscloud/ui/widgets/thread_selector.py +358 -0
- opscloud/ui/widgets/toast.py +31 -0
- opscloud/ui/widgets/tool_display.py +325 -0
- opscloud/ui/widgets/tool_renderers.py +234 -0
- opscloud/ui/widgets/tool_widgets.py +278 -0
- opscloud/ui/widgets/welcome.py +421 -0
- opscloud/ui/widgets/welcome_popup.py +102 -0
- opscloud/utils/__init__.py +24 -0
- opscloud/utils/cost_estimation.py +169 -0
- opscloud/utils/git.py +390 -0
- opscloud/utils/logger.py +571 -0
- opscloud/utils/session_stats.py +115 -0
- opscloud/utils/startup_error.py +37 -0
- talkops_opscloud-0.3.0b1.dist-info/METADATA +449 -0
- talkops_opscloud-0.3.0b1.dist-info/RECORD +309 -0
- talkops_opscloud-0.3.0b1.dist-info/WHEEL +4 -0
- talkops_opscloud-0.3.0b1.dist-info/entry_points.txt +2 -0
- talkops_opscloud-0.3.0b1.dist-info/licenses/LICENSE +175 -0
|
@@ -0,0 +1,709 @@
|
|
|
1
|
+
"""Rubric middleware with transport retry, event streaming, and isolated grading.
|
|
2
|
+
|
|
3
|
+
``ReliableRubricMiddleware`` extends the SDK's ``RubricMiddleware`` with:
|
|
4
|
+
|
|
5
|
+
- **Isolated Grader Agent**: Creates a dedicated evaluator subagent configured
|
|
6
|
+
with read-only evidence tools, budget guards, and custom system prompts.
|
|
7
|
+
- **Transport Resilience**: Catches transient HTTP read and remote protocol errors
|
|
8
|
+
(e.g., during long grading responses) and retries the grader invocation once
|
|
9
|
+
without replaying worker operations.
|
|
10
|
+
- **Control Message Sanitization**: Strips internal goal-state notices, continuation
|
|
11
|
+
pings, and past evaluation metadata so the grader evaluates only genuine work.
|
|
12
|
+
- **Streaming Telemetry**: Emits ``rubric_evaluation_start`` and ``rubric_evaluation_end``
|
|
13
|
+
lifecycle events via ``runtime.stream_writer`` for real-time terminal display.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
from collections.abc import Iterator, Mapping, Sequence
|
|
19
|
+
from contextlib import contextmanager
|
|
20
|
+
from contextvars import ContextVar
|
|
21
|
+
from dataclasses import replace
|
|
22
|
+
from typing import (
|
|
23
|
+
TYPE_CHECKING,
|
|
24
|
+
Annotated,
|
|
25
|
+
Any,
|
|
26
|
+
Callable,
|
|
27
|
+
NotRequired,
|
|
28
|
+
cast,
|
|
29
|
+
)
|
|
30
|
+
import warnings
|
|
31
|
+
|
|
32
|
+
import httpx
|
|
33
|
+
from deepagents.middleware.rubric import (
|
|
34
|
+
RUBRIC_GRADER_MESSAGE_SOURCE,
|
|
35
|
+
GraderResponse,
|
|
36
|
+
RubricMiddleware as BaseRubricMiddleware,
|
|
37
|
+
RubricState,
|
|
38
|
+
)
|
|
39
|
+
import deepagents.middleware.rubric as _rubric_mod
|
|
40
|
+
from langchain.agents.middleware.types import (
|
|
41
|
+
AgentMiddleware,
|
|
42
|
+
AgentState,
|
|
43
|
+
PrivateStateAttr,
|
|
44
|
+
hook_config,
|
|
45
|
+
)
|
|
46
|
+
from langchain_core.messages import AnyMessage, HumanMessage
|
|
47
|
+
from langgraph.errors import GraphBubbleUp
|
|
48
|
+
|
|
49
|
+
from opscloud.middleware.goal_state_notice import is_conversation_control_message
|
|
50
|
+
from opscloud.middleware.registry import register_middleware
|
|
51
|
+
from opscloud.utils.logger import get_logger
|
|
52
|
+
|
|
53
|
+
if TYPE_CHECKING:
|
|
54
|
+
from deepagents.middleware.rubric import RubricEvaluation
|
|
55
|
+
from langchain_core.language_models import BaseChatModel
|
|
56
|
+
from langchain_core.tools import BaseTool
|
|
57
|
+
from langgraph.runtime import Runtime
|
|
58
|
+
|
|
59
|
+
logger = get_logger(__name__)
|
|
60
|
+
|
|
61
|
+
_strategy_from_result = getattr(_rubric_mod, "_strategy_from_result", None)
|
|
62
|
+
|
|
63
|
+
__all__ = ["ReliableRubricMiddleware", "ReliableRubricState", "RubricGraderState", "RubricMiddleware"]
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
# ---------------------------------------------------------------------------
|
|
67
|
+
# Transport Error Recovery Helpers
|
|
68
|
+
# ---------------------------------------------------------------------------
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _exception_chain(exc: BaseException) -> Iterator[BaseException]:
|
|
72
|
+
"""Yield an exception, its causes, context, and group members without cycles."""
|
|
73
|
+
pending = [exc]
|
|
74
|
+
seen: set[int] = set()
|
|
75
|
+
while pending:
|
|
76
|
+
current = pending.pop()
|
|
77
|
+
curr_id = id(current)
|
|
78
|
+
if curr_id in seen:
|
|
79
|
+
continue
|
|
80
|
+
seen.add(curr_id)
|
|
81
|
+
yield current
|
|
82
|
+
if isinstance(current, BaseExceptionGroup):
|
|
83
|
+
pending.extend(current.exceptions)
|
|
84
|
+
if current.__cause__ is not None:
|
|
85
|
+
pending.append(current.__cause__)
|
|
86
|
+
elif current.__context__ is not None:
|
|
87
|
+
pending.append(current.__context__)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _is_transient_grader_transport_error(exc: BaseException) -> bool:
|
|
91
|
+
"""Determine whether an evaluation exception is a retryable network drop."""
|
|
92
|
+
for current in _exception_chain(exc):
|
|
93
|
+
if isinstance(
|
|
94
|
+
current,
|
|
95
|
+
(
|
|
96
|
+
httpx.ReadError,
|
|
97
|
+
httpx.RemoteProtocolError,
|
|
98
|
+
httpx.TimeoutException,
|
|
99
|
+
httpx.ConnectError,
|
|
100
|
+
ConnectionError,
|
|
101
|
+
TimeoutError,
|
|
102
|
+
),
|
|
103
|
+
):
|
|
104
|
+
return True
|
|
105
|
+
mod_name = type(current).__module__
|
|
106
|
+
cls_name = type(current).__name__
|
|
107
|
+
if mod_name.startswith("httpcore") and cls_name in {
|
|
108
|
+
"ReadError",
|
|
109
|
+
"RemoteProtocolError",
|
|
110
|
+
"ConnectError",
|
|
111
|
+
"TimeoutException",
|
|
112
|
+
}:
|
|
113
|
+
return True
|
|
114
|
+
if (
|
|
115
|
+
mod_name == "aiohttp.http_exceptions"
|
|
116
|
+
and cls_name == "TransferEncodingError"
|
|
117
|
+
and "Not enough data" in str(current)
|
|
118
|
+
):
|
|
119
|
+
return True
|
|
120
|
+
err_msg = str(current)
|
|
121
|
+
if any(code in err_msg for code in ("429", "503", "Connection reset by peer")):
|
|
122
|
+
return True
|
|
123
|
+
return False
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
# ---------------------------------------------------------------------------
|
|
127
|
+
# Message History Sanitization
|
|
128
|
+
# ---------------------------------------------------------------------------
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _without_internal_control_messages(state: RubricState) -> RubricState:
|
|
132
|
+
"""Filter out internal goal-state notices so the grader evaluates only real work."""
|
|
133
|
+
messages = state.get("messages", [])
|
|
134
|
+
if not isinstance(messages, list):
|
|
135
|
+
return state
|
|
136
|
+
filtered: list[AnyMessage] = [
|
|
137
|
+
msg for msg in messages if not is_conversation_control_message(msg)
|
|
138
|
+
]
|
|
139
|
+
if len(filtered) == len(messages):
|
|
140
|
+
return state
|
|
141
|
+
updated = dict(state)
|
|
142
|
+
updated["messages"] = filtered
|
|
143
|
+
return cast("RubricState", updated)
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
# ---------------------------------------------------------------------------
|
|
147
|
+
# State Schemas
|
|
148
|
+
# ---------------------------------------------------------------------------
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
class ReliableRubricState(RubricState):
|
|
152
|
+
"""Rubric state carrying OpsCloud's private runtime model selections."""
|
|
153
|
+
|
|
154
|
+
_model_spec: Annotated[NotRequired[str], PrivateStateAttr]
|
|
155
|
+
"""Active chat model specifier (written by ConfigurableModelMiddleware)."""
|
|
156
|
+
|
|
157
|
+
_model_params: Annotated[NotRequired[dict[str, Any] | None], PrivateStateAttr]
|
|
158
|
+
"""Model hyperparameters belonging to _model_spec."""
|
|
159
|
+
|
|
160
|
+
_rubric_model_spec: Annotated[NotRequired[str], PrivateStateAttr]
|
|
161
|
+
"""Thread-scoped grader model override selected by the operator."""
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
class RubricGraderState(AgentState[GraderResponse]):
|
|
165
|
+
"""Private state for the isolated nested grader agent."""
|
|
166
|
+
|
|
167
|
+
rubric_grading_operation_id: NotRequired[str]
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def _is_smart_mode(context: object = None, store: object = None) -> bool:
|
|
171
|
+
"""Synchronously determine whether Smart mode is active."""
|
|
172
|
+
try:
|
|
173
|
+
from opscloud.middleware.jev_model_router import _is_smart_mode_sync
|
|
174
|
+
|
|
175
|
+
return _is_smart_mode_sync(context, store=store)
|
|
176
|
+
except Exception:
|
|
177
|
+
return False
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
async def _ais_smart_mode(context: object = None, store: object = None) -> bool:
|
|
181
|
+
"""Asynchronously determine whether Smart mode is active."""
|
|
182
|
+
try:
|
|
183
|
+
from opscloud.middleware.jev_model_router import _ais_smart_mode as _async_smart
|
|
184
|
+
|
|
185
|
+
return await _async_smart(context, store=store)
|
|
186
|
+
except Exception:
|
|
187
|
+
return False
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
# ---------------------------------------------------------------------------
|
|
191
|
+
# Middleware Implementation
|
|
192
|
+
# ---------------------------------------------------------------------------
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
with warnings.catch_warnings():
|
|
196
|
+
warnings.filterwarnings(
|
|
197
|
+
"ignore",
|
|
198
|
+
message="The middleware `RubricMiddleware` is in beta",
|
|
199
|
+
category=Warning,
|
|
200
|
+
)
|
|
201
|
+
|
|
202
|
+
@register_middleware(name="reliable_rubric")
|
|
203
|
+
class ReliableRubricMiddleware(BaseRubricMiddleware):
|
|
204
|
+
"""Reliable, isolated rubric grader with transport retry and telemetry streaming."""
|
|
205
|
+
|
|
206
|
+
state_schema = ReliableRubricState
|
|
207
|
+
|
|
208
|
+
def __init__(
|
|
209
|
+
self,
|
|
210
|
+
*,
|
|
211
|
+
model: str | BaseChatModel | None = None,
|
|
212
|
+
system_prompt: str | None = None,
|
|
213
|
+
tools: Sequence[BaseTool] | None = None,
|
|
214
|
+
grader_middleware: Sequence[AgentMiddleware[Any, Any]] | None = None,
|
|
215
|
+
grader_context_schema: type[Any] | None = None,
|
|
216
|
+
max_iterations: int = 3,
|
|
217
|
+
on_evaluation: Callable[[RubricEvaluation], None] | None = None,
|
|
218
|
+
inherit_main_model: bool = False,
|
|
219
|
+
runtime_bootstrap_model: str | BaseChatModel | None = None,
|
|
220
|
+
) -> None:
|
|
221
|
+
kwargs: dict[str, Any] = {
|
|
222
|
+
"model": model or "anthropic:claude-3-5-haiku-20241022",
|
|
223
|
+
"max_iterations": max_iterations,
|
|
224
|
+
}
|
|
225
|
+
if system_prompt is not None:
|
|
226
|
+
kwargs["system_prompt"] = system_prompt
|
|
227
|
+
if tools is not None:
|
|
228
|
+
kwargs["tools"] = tools
|
|
229
|
+
if on_evaluation is not None:
|
|
230
|
+
kwargs["on_evaluation"] = on_evaluation
|
|
231
|
+
|
|
232
|
+
super().__init__(**kwargs)
|
|
233
|
+
self._grader_model_spec = model or "anthropic:claude-3-5-haiku-20241022"
|
|
234
|
+
self._grader_middleware = list(grader_middleware or ())
|
|
235
|
+
self._grader_context_schema = grader_context_schema
|
|
236
|
+
self._inherit_main_model = inherit_main_model
|
|
237
|
+
self._runtime_bootstrap_model = runtime_bootstrap_model
|
|
238
|
+
self._runtime_grader_model: ContextVar[str | None] = ContextVar(
|
|
239
|
+
"runtime_grader_model",
|
|
240
|
+
default=None,
|
|
241
|
+
)
|
|
242
|
+
self._jev_grader: Any | None = None
|
|
243
|
+
|
|
244
|
+
def _get_jev_grader(self) -> Any:
|
|
245
|
+
"""Lazily instantiate JevHybridRubricGrader when needed."""
|
|
246
|
+
if self._jev_grader is None:
|
|
247
|
+
from opscloud.rubrics.jev_grader import JevHybridRubricGrader
|
|
248
|
+
|
|
249
|
+
self._jev_grader = JevHybridRubricGrader()
|
|
250
|
+
return self._jev_grader
|
|
251
|
+
|
|
252
|
+
@contextmanager
|
|
253
|
+
def _runtime_grader_trace(self, model_label: str | None) -> Iterator[None]:
|
|
254
|
+
token = self._runtime_grader_model.set(model_label)
|
|
255
|
+
try:
|
|
256
|
+
yield
|
|
257
|
+
finally:
|
|
258
|
+
self._runtime_grader_model.reset(token)
|
|
259
|
+
|
|
260
|
+
def _ensure_grader(self) -> Any:
|
|
261
|
+
"""Instantiate the isolated rubric grader agent graph."""
|
|
262
|
+
if self._grader is not None:
|
|
263
|
+
return self._grader
|
|
264
|
+
|
|
265
|
+
from deepagents._models import resolve_model # noqa: PLC2701
|
|
266
|
+
from langchain.agents import create_agent
|
|
267
|
+
|
|
268
|
+
resolved_model = resolve_model(self._model)
|
|
269
|
+
self._resolved_model = resolved_model
|
|
270
|
+
|
|
271
|
+
logger.debug(
|
|
272
|
+
"ReliableRubric._ensure_grader: building isolated grader agent with middleware: %s",
|
|
273
|
+
[getattr(m, "name", type(m).__name__) for m in self._grader_middleware],
|
|
274
|
+
)
|
|
275
|
+
|
|
276
|
+
self._grader = create_agent(
|
|
277
|
+
model=resolved_model,
|
|
278
|
+
system_prompt=self._system_prompt,
|
|
279
|
+
tools=self._tools,
|
|
280
|
+
middleware=self._grader_middleware,
|
|
281
|
+
name=RUBRIC_GRADER_MESSAGE_SOURCE,
|
|
282
|
+
response_format=GraderResponse,
|
|
283
|
+
state_schema=RubricGraderState,
|
|
284
|
+
context_schema=self._grader_context_schema,
|
|
285
|
+
)
|
|
286
|
+
return self._grader
|
|
287
|
+
|
|
288
|
+
def _emit_stream_event(self, runtime: Runtime[Any], payload: dict[str, Any]) -> None:
|
|
289
|
+
"""Safely emit an evaluation event to the live stream writer."""
|
|
290
|
+
writer = getattr(runtime, "stream_writer", None)
|
|
291
|
+
if writer is not None and callable(writer):
|
|
292
|
+
try:
|
|
293
|
+
writer(payload)
|
|
294
|
+
except Exception:
|
|
295
|
+
logger.debug("Failed to emit rubric stream event %s", payload, exc_info=True)
|
|
296
|
+
|
|
297
|
+
def _grader_input(
|
|
298
|
+
self,
|
|
299
|
+
state: RubricState,
|
|
300
|
+
iteration: int,
|
|
301
|
+
correction: str | None = None,
|
|
302
|
+
) -> dict[str, Any]:
|
|
303
|
+
grading_run_id = state.get("_current_grading_run_id") or "untracked"
|
|
304
|
+
sanitized_state = _without_internal_control_messages(state)
|
|
305
|
+
payload = self._build_grader_payload(sanitized_state, iteration, correction)
|
|
306
|
+
return {
|
|
307
|
+
"messages": [HumanMessage(content=payload)],
|
|
308
|
+
"rubric_grading_operation_id": f"{grading_run_id}:{iteration}",
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
def _grade_once(
|
|
312
|
+
self,
|
|
313
|
+
state: RubricState,
|
|
314
|
+
iteration: int,
|
|
315
|
+
correction: str | None = None,
|
|
316
|
+
*,
|
|
317
|
+
context: object | None = None,
|
|
318
|
+
) -> GraderResponse:
|
|
319
|
+
grader = self._ensure_grader()
|
|
320
|
+
meta_fn = getattr(self, "_grader_trace_metadata", lambda **kw: {})
|
|
321
|
+
rec_fn = getattr(self, "_record_grader_trace_metadata", lambda m: None)
|
|
322
|
+
cfg_fn = getattr(self, "_grader_invocation_config", lambda m: {})
|
|
323
|
+
|
|
324
|
+
metadata = meta_fn()
|
|
325
|
+
rec_fn(metadata)
|
|
326
|
+
result = grader.invoke(
|
|
327
|
+
self._grader_input(state, iteration, correction),
|
|
328
|
+
config=cfg_fn(metadata),
|
|
329
|
+
context=context,
|
|
330
|
+
)
|
|
331
|
+
strategy = _strategy_from_result(result) if _strategy_from_result is not None else None
|
|
332
|
+
rec_fn(meta_fn(effective_strategy=strategy))
|
|
333
|
+
return self._extract_graded(result)
|
|
334
|
+
|
|
335
|
+
async def _agrade_once(
|
|
336
|
+
self,
|
|
337
|
+
state: RubricState,
|
|
338
|
+
iteration: int,
|
|
339
|
+
correction: str | None = None,
|
|
340
|
+
*,
|
|
341
|
+
context: object | None = None,
|
|
342
|
+
) -> GraderResponse:
|
|
343
|
+
grader = self._ensure_grader()
|
|
344
|
+
meta_fn = getattr(self, "_grader_trace_metadata", lambda **kw: {})
|
|
345
|
+
rec_fn = getattr(self, "_record_grader_trace_metadata", lambda m: None)
|
|
346
|
+
cfg_fn = getattr(self, "_grader_invocation_config", lambda m: {})
|
|
347
|
+
|
|
348
|
+
metadata = meta_fn()
|
|
349
|
+
rec_fn(metadata)
|
|
350
|
+
result = await grader.ainvoke(
|
|
351
|
+
self._grader_input(state, iteration, correction),
|
|
352
|
+
config=cfg_fn(metadata),
|
|
353
|
+
context=context,
|
|
354
|
+
)
|
|
355
|
+
strategy = _strategy_from_result(result) if _strategy_from_result is not None else None
|
|
356
|
+
rec_fn(meta_fn(effective_strategy=strategy))
|
|
357
|
+
return self._extract_graded(result)
|
|
358
|
+
|
|
359
|
+
@staticmethod
|
|
360
|
+
def _context(context: object | None) -> Any:
|
|
361
|
+
"""Copy the parent runtime context into the nested grader schema."""
|
|
362
|
+
from opscloud.agent.config import CLIContextSchema
|
|
363
|
+
|
|
364
|
+
inner = getattr(context, "context", None) if not isinstance(context, (CLIContextSchema, dict)) else context
|
|
365
|
+
target = inner if inner is not None else context
|
|
366
|
+
|
|
367
|
+
if isinstance(target, CLIContextSchema):
|
|
368
|
+
return replace(
|
|
369
|
+
target,
|
|
370
|
+
model_params=dict(target.model_params),
|
|
371
|
+
profile_overrides=dict(target.profile_overrides),
|
|
372
|
+
hooks_server_events=list(target.hooks_server_events),
|
|
373
|
+
)
|
|
374
|
+
if isinstance(target, dict):
|
|
375
|
+
parsed = CLIContextSchema.from_payload(target)
|
|
376
|
+
if parsed is not None:
|
|
377
|
+
return parsed
|
|
378
|
+
return CLIContextSchema()
|
|
379
|
+
|
|
380
|
+
def _grader_context(
|
|
381
|
+
self, state: ReliableRubricState, context: object | None
|
|
382
|
+
) -> Any:
|
|
383
|
+
"""Select the effective grader model without mutating shared middleware."""
|
|
384
|
+
from opscloud.middleware.resume_state import INHERIT_RUBRIC_MODEL, coerce_model_spec
|
|
385
|
+
|
|
386
|
+
grader_context = self._context(context)
|
|
387
|
+
selected = coerce_model_spec(state.get("_rubric_model_spec"))
|
|
388
|
+
inherit = selected == INHERIT_RUBRIC_MODEL or (
|
|
389
|
+
selected is None and self._inherit_main_model
|
|
390
|
+
)
|
|
391
|
+
if inherit:
|
|
392
|
+
main_model = coerce_model_spec(state.get("_model_spec"))
|
|
393
|
+
if main_model is not None and hasattr(grader_context, "model"):
|
|
394
|
+
grader_context.model = main_model
|
|
395
|
+
params = state.get("_model_params")
|
|
396
|
+
if isinstance(params, Mapping) and hasattr(grader_context, "model_params"):
|
|
397
|
+
grader_context.model_params = dict(params)
|
|
398
|
+
else:
|
|
399
|
+
if hasattr(grader_context, "model"):
|
|
400
|
+
grader_context.model = selected
|
|
401
|
+
if hasattr(grader_context, "model_params"):
|
|
402
|
+
grader_context.model_params = {}
|
|
403
|
+
return grader_context
|
|
404
|
+
|
|
405
|
+
def _invoke_grader(
|
|
406
|
+
self,
|
|
407
|
+
state: RubricState,
|
|
408
|
+
iteration: int,
|
|
409
|
+
correction: str | None = None,
|
|
410
|
+
*,
|
|
411
|
+
context: object | None = None,
|
|
412
|
+
) -> GraderResponse:
|
|
413
|
+
"""Invoke the isolated grader with Smart-mode Jev fast-pass and LLM fallback."""
|
|
414
|
+
reliable_state = cast("ReliableRubricState", state)
|
|
415
|
+
grader_context = self._grader_context(reliable_state, context)
|
|
416
|
+
|
|
417
|
+
# ── Smart Mode Path (Jev System One Fast-Pass) ──
|
|
418
|
+
jev_grader = self._get_jev_grader()
|
|
419
|
+
raw_rubric = state.get("rubric") or state.get("_goal_rubric") or state.get("_sticky_rubric") or ""
|
|
420
|
+
store = getattr(context, "store", None)
|
|
421
|
+
is_smart = _is_smart_mode(context, store=store) or _is_smart_mode(grader_context)
|
|
422
|
+
if is_smart and jev_grader.is_available() and raw_rubric:
|
|
423
|
+
try:
|
|
424
|
+
from opscloud.rubrics.jev_compiler import JevCriteriaCompiler
|
|
425
|
+
|
|
426
|
+
objective = str(state.get("_goal_objective") or state.get("goal") or "Fulfill user requirements")
|
|
427
|
+
compiled = JevCriteriaCompiler.compile(objective, str(raw_rubric))
|
|
428
|
+
logger.info("ReliableRubricMiddleware: Smart mode active — evaluating criteria via Jev System One")
|
|
429
|
+
def _targeted_diagnostic(failing: list[str]) -> GraderResponse:
|
|
430
|
+
"""Generate concise diagnostic remediation feedback in a single 1-shot LLM completion without tools."""
|
|
431
|
+
failing_text = "\n".join(f"- {item}" for item in failing)
|
|
432
|
+
try:
|
|
433
|
+
from deepagents._models import resolve_model
|
|
434
|
+
from langchain_core.messages import HumanMessage, SystemMessage
|
|
435
|
+
|
|
436
|
+
model_spec = getattr(grader_context, "model", None) or getattr(self, "_grader_model_spec", None) or "anthropic:claude-3-5-haiku-20241022"
|
|
437
|
+
chat_model = resolve_model(model_spec) if isinstance(model_spec, str) else model_spec
|
|
438
|
+
|
|
439
|
+
diagnostic_prompt = [
|
|
440
|
+
SystemMessage(
|
|
441
|
+
content=(
|
|
442
|
+
"You are the OpsCloud Rubric Diagnostic Evaluator. Acceptance criteria evaluation failed. "
|
|
443
|
+
"Write concise, actionable remediation advice for what specific files, configurations, "
|
|
444
|
+
"or commands the agent must execute to satisfy the failing criteria. Do not repeat all criteria."
|
|
445
|
+
)
|
|
446
|
+
),
|
|
447
|
+
HumanMessage(
|
|
448
|
+
content=(
|
|
449
|
+
f"Objective: {objective}\n\n"
|
|
450
|
+
f"Unsatisfied criteria:\n{failing_text}\n\n"
|
|
451
|
+
"Provide 1-2 concise bullet points explaining exactly what is missing and how to fix it."
|
|
452
|
+
)
|
|
453
|
+
),
|
|
454
|
+
]
|
|
455
|
+
res = chat_model.invoke(diagnostic_prompt)
|
|
456
|
+
advice = getattr(res, "content", "") or str(res)
|
|
457
|
+
explanation = (
|
|
458
|
+
f"{len(failing)} of {len(compiled.criteria_map)} criteria not yet satisfied.\n\n"
|
|
459
|
+
f"Remediation Guidance:\n{advice}"
|
|
460
|
+
)
|
|
461
|
+
except Exception as diag_exc:
|
|
462
|
+
logger.warning("Tier 2 diagnostic prompt failed (%s); using default explanation", diag_exc)
|
|
463
|
+
explanation = f"{len(failing)} criteria not yet satisfied: {', '.join(failing)}"
|
|
464
|
+
|
|
465
|
+
return GraderResponse(
|
|
466
|
+
result="needs_revision",
|
|
467
|
+
explanation=explanation,
|
|
468
|
+
criteria=[],
|
|
469
|
+
)
|
|
470
|
+
|
|
471
|
+
return jev_grader.grade(
|
|
472
|
+
compiled,
|
|
473
|
+
evidence=state,
|
|
474
|
+
fallback_llm_fn=_targeted_diagnostic,
|
|
475
|
+
)
|
|
476
|
+
except Exception as jev_exc:
|
|
477
|
+
logger.warning("Jev rubric evaluation failed (%s); falling back to standard LLM grader", jev_exc)
|
|
478
|
+
|
|
479
|
+
# ── Standard Path (Manual, Auto, or non-Jev) ──
|
|
480
|
+
model_label = getattr(grader_context, "model", None)
|
|
481
|
+
with self._runtime_grader_trace(model_label):
|
|
482
|
+
try:
|
|
483
|
+
return self._grade_once(state, iteration, correction, context=grader_context)
|
|
484
|
+
except Exception as exc:
|
|
485
|
+
if not _is_transient_grader_transport_error(exc):
|
|
486
|
+
raise
|
|
487
|
+
logger.warning("Rubric grader transient error; retrying grading invocation once: %s", exc)
|
|
488
|
+
return self._grade_once(state, iteration, correction, context=grader_context)
|
|
489
|
+
|
|
490
|
+
async def _ainvoke_grader(
|
|
491
|
+
self,
|
|
492
|
+
state: RubricState,
|
|
493
|
+
iteration: int,
|
|
494
|
+
correction: str | None = None,
|
|
495
|
+
*,
|
|
496
|
+
context: object | None = None,
|
|
497
|
+
) -> GraderResponse:
|
|
498
|
+
"""Invoke the isolated grader asynchronously with Smart-mode Jev fast-pass and LLM fallback."""
|
|
499
|
+
reliable_state = cast("ReliableRubricState", state)
|
|
500
|
+
grader_context = self._grader_context(reliable_state, context)
|
|
501
|
+
|
|
502
|
+
# ── Smart Mode Path (Jev System One Fast-Pass) ──
|
|
503
|
+
jev_grader = self._get_jev_grader()
|
|
504
|
+
raw_rubric = state.get("rubric") or state.get("_goal_rubric") or state.get("_sticky_rubric") or ""
|
|
505
|
+
store = getattr(context, "store", None)
|
|
506
|
+
is_smart = (await _ais_smart_mode(context, store=store)) or (await _ais_smart_mode(grader_context))
|
|
507
|
+
if is_smart and jev_grader.is_available() and raw_rubric:
|
|
508
|
+
try:
|
|
509
|
+
from opscloud.rubrics.jev_compiler import JevCriteriaCompiler
|
|
510
|
+
|
|
511
|
+
objective = str(state.get("_goal_objective") or state.get("goal") or "Fulfill user requirements")
|
|
512
|
+
compiled = JevCriteriaCompiler.compile(objective, str(raw_rubric))
|
|
513
|
+
logger.info("ReliableRubricMiddleware: Smart mode active — evaluating criteria via Jev System One")
|
|
514
|
+
|
|
515
|
+
async def _atargeted_diagnostic(failing: list[str]) -> GraderResponse:
|
|
516
|
+
"""Generate concise diagnostic remediation feedback in a single 1-shot LLM completion without tools."""
|
|
517
|
+
failing_text = "\n".join(f"- {item}" for item in failing)
|
|
518
|
+
try:
|
|
519
|
+
from deepagents._models import resolve_model
|
|
520
|
+
from langchain_core.messages import HumanMessage, SystemMessage
|
|
521
|
+
|
|
522
|
+
model_spec = getattr(grader_context, "model", None) or getattr(self, "_grader_model_spec", None) or "anthropic:claude-3-5-haiku-20241022"
|
|
523
|
+
chat_model = resolve_model(model_spec) if isinstance(model_spec, str) else model_spec
|
|
524
|
+
|
|
525
|
+
diagnostic_prompt = [
|
|
526
|
+
SystemMessage(
|
|
527
|
+
content=(
|
|
528
|
+
"You are the OpsCloud Rubric Diagnostic Evaluator. Acceptance criteria evaluation failed. "
|
|
529
|
+
"Write concise, actionable remediation advice for what specific files, configurations, "
|
|
530
|
+
"or commands the agent must execute to satisfy the failing criteria. Do not repeat all criteria."
|
|
531
|
+
)
|
|
532
|
+
),
|
|
533
|
+
HumanMessage(
|
|
534
|
+
content=(
|
|
535
|
+
f"Objective: {objective}\n\n"
|
|
536
|
+
f"Unsatisfied criteria:\n{failing_text}\n\n"
|
|
537
|
+
"Provide 1-2 concise bullet points explaining exactly what is missing and how to fix it."
|
|
538
|
+
)
|
|
539
|
+
),
|
|
540
|
+
]
|
|
541
|
+
res = await chat_model.ainvoke(diagnostic_prompt)
|
|
542
|
+
advice = getattr(res, "content", "") or str(res)
|
|
543
|
+
explanation = (
|
|
544
|
+
f"{len(failing)} of {len(compiled.criteria_map)} criteria not yet satisfied.\n\n"
|
|
545
|
+
f"Remediation Guidance:\n{advice}"
|
|
546
|
+
)
|
|
547
|
+
except Exception as diag_exc:
|
|
548
|
+
logger.warning("Tier 2 diagnostic prompt failed (%s); using default explanation", diag_exc)
|
|
549
|
+
explanation = f"{len(failing)} criteria not yet satisfied: {', '.join(failing)}"
|
|
550
|
+
|
|
551
|
+
return GraderResponse(
|
|
552
|
+
result="needs_revision",
|
|
553
|
+
explanation=explanation,
|
|
554
|
+
criteria=[],
|
|
555
|
+
)
|
|
556
|
+
|
|
557
|
+
return await jev_grader.agrade(
|
|
558
|
+
compiled,
|
|
559
|
+
evidence=state,
|
|
560
|
+
fallback_llm_fn=_atargeted_diagnostic,
|
|
561
|
+
)
|
|
562
|
+
except Exception as jev_exc:
|
|
563
|
+
logger.warning("Jev async rubric evaluation failed (%s); falling back to standard LLM grader", jev_exc)
|
|
564
|
+
|
|
565
|
+
# ── Standard Path (Manual, Auto, or non-Jev) ──
|
|
566
|
+
model_label = getattr(grader_context, "model", None)
|
|
567
|
+
with self._runtime_grader_trace(model_label):
|
|
568
|
+
try:
|
|
569
|
+
return await self._agrade_once(state, iteration, correction, context=grader_context)
|
|
570
|
+
except Exception as exc:
|
|
571
|
+
if not _is_transient_grader_transport_error(exc):
|
|
572
|
+
raise
|
|
573
|
+
logger.warning("Rubric grader transient async error; retrying grading invocation once: %s", exc)
|
|
574
|
+
return await self._agrade_once(state, iteration, correction, context=grader_context)
|
|
575
|
+
|
|
576
|
+
def _grade(
|
|
577
|
+
self,
|
|
578
|
+
state: RubricState,
|
|
579
|
+
iteration: int,
|
|
580
|
+
*,
|
|
581
|
+
context: object | None = None,
|
|
582
|
+
) -> GraderResponse:
|
|
583
|
+
return super()._grade(state, iteration, context=context)
|
|
584
|
+
|
|
585
|
+
async def _agrade(
|
|
586
|
+
self,
|
|
587
|
+
state: RubricState,
|
|
588
|
+
iteration: int,
|
|
589
|
+
*,
|
|
590
|
+
context: object | None = None,
|
|
591
|
+
) -> GraderResponse:
|
|
592
|
+
return await super()._agrade(state, iteration, context=context)
|
|
593
|
+
|
|
594
|
+
def _finalize_evaluation(
|
|
595
|
+
self,
|
|
596
|
+
graded: GraderResponse,
|
|
597
|
+
state: RubricState,
|
|
598
|
+
runtime: Runtime[Any],
|
|
599
|
+
grading_run_id: str,
|
|
600
|
+
iteration: int,
|
|
601
|
+
) -> dict[str, Any]:
|
|
602
|
+
update = super()._finalize_evaluation(
|
|
603
|
+
graded, state, runtime, grading_run_id, iteration
|
|
604
|
+
)
|
|
605
|
+
# If Jev classified an environmental blocker or ambiguous requirement, mark goal as blocked
|
|
606
|
+
blocker = getattr(graded, "blocker_type", None)
|
|
607
|
+
explanation = getattr(graded, "explanation", "") or getattr(graded, "feedback", "") or ""
|
|
608
|
+
if blocker in {"external_blocker", "ambiguous_requirement"} or "Environmental blocker detected" in str(explanation):
|
|
609
|
+
update["_goal_status"] = "blocked"
|
|
610
|
+
update["_goal_status_note"] = str(explanation)
|
|
611
|
+
return update
|
|
612
|
+
|
|
613
|
+
@hook_config(can_jump_to=["model"])
|
|
614
|
+
def after_agent(
|
|
615
|
+
self,
|
|
616
|
+
state: RubricState,
|
|
617
|
+
runtime: Runtime[Any],
|
|
618
|
+
) -> dict[str, Any] | None:
|
|
619
|
+
prep = self._prepare_evaluation(state, runtime)
|
|
620
|
+
if prep is None:
|
|
621
|
+
return None
|
|
622
|
+
grading_run_id, iteration = prep
|
|
623
|
+
|
|
624
|
+
try:
|
|
625
|
+
graded = self._grade(
|
|
626
|
+
state,
|
|
627
|
+
iteration,
|
|
628
|
+
context=runtime,
|
|
629
|
+
)
|
|
630
|
+
except GraphBubbleUp:
|
|
631
|
+
raise
|
|
632
|
+
except Exception as exc:
|
|
633
|
+
return self._handle_grader_exception(
|
|
634
|
+
runtime,
|
|
635
|
+
state,
|
|
636
|
+
grading_run_id,
|
|
637
|
+
iteration,
|
|
638
|
+
exc,
|
|
639
|
+
)
|
|
640
|
+
|
|
641
|
+
feedback = getattr(graded, "feedback", None) or getattr(graded, "explanation", "")
|
|
642
|
+
|
|
643
|
+
if "Environmental blocker detected" in str(feedback):
|
|
644
|
+
self._emit_stream_event(
|
|
645
|
+
runtime,
|
|
646
|
+
{
|
|
647
|
+
"type": "rubric_blocker_detected",
|
|
648
|
+
"iteration": iteration,
|
|
649
|
+
"explanation": str(feedback),
|
|
650
|
+
},
|
|
651
|
+
)
|
|
652
|
+
|
|
653
|
+
return self._finalize_evaluation(
|
|
654
|
+
graded,
|
|
655
|
+
state,
|
|
656
|
+
runtime,
|
|
657
|
+
grading_run_id,
|
|
658
|
+
iteration,
|
|
659
|
+
)
|
|
660
|
+
|
|
661
|
+
async def aafter_agent(
|
|
662
|
+
self,
|
|
663
|
+
state: RubricState,
|
|
664
|
+
runtime: Runtime[Any],
|
|
665
|
+
) -> dict[str, Any] | None:
|
|
666
|
+
prep = self._prepare_evaluation(state, runtime)
|
|
667
|
+
if prep is None:
|
|
668
|
+
return None
|
|
669
|
+
grading_run_id, iteration = prep
|
|
670
|
+
|
|
671
|
+
try:
|
|
672
|
+
graded = await self._agrade(
|
|
673
|
+
state,
|
|
674
|
+
iteration,
|
|
675
|
+
context=runtime,
|
|
676
|
+
)
|
|
677
|
+
except GraphBubbleUp:
|
|
678
|
+
raise
|
|
679
|
+
except Exception as exc:
|
|
680
|
+
return self._handle_grader_exception(
|
|
681
|
+
runtime,
|
|
682
|
+
state,
|
|
683
|
+
grading_run_id,
|
|
684
|
+
iteration,
|
|
685
|
+
exc,
|
|
686
|
+
)
|
|
687
|
+
|
|
688
|
+
feedback = getattr(graded, "feedback", None) or getattr(graded, "explanation", "")
|
|
689
|
+
|
|
690
|
+
if "Environmental blocker detected" in str(feedback):
|
|
691
|
+
self._emit_stream_event(
|
|
692
|
+
runtime,
|
|
693
|
+
{
|
|
694
|
+
"type": "rubric_blocker_detected",
|
|
695
|
+
"iteration": iteration,
|
|
696
|
+
"explanation": str(feedback),
|
|
697
|
+
},
|
|
698
|
+
)
|
|
699
|
+
|
|
700
|
+
return self._finalize_evaluation(
|
|
701
|
+
graded,
|
|
702
|
+
state,
|
|
703
|
+
runtime,
|
|
704
|
+
grading_run_id,
|
|
705
|
+
iteration,
|
|
706
|
+
)
|
|
707
|
+
|
|
708
|
+
|
|
709
|
+
RubricMiddleware = ReliableRubricMiddleware
|