talkops-opscloud 0.3.0b1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- opscloud/__init__.py +5 -0
- opscloud/__main__.py +6 -0
- opscloud/_constants.py +53 -0
- opscloud/_debug.py +145 -0
- opscloud/_debug_buffer.py +250 -0
- opscloud/_textual_patches.py +367 -0
- opscloud/_version.py +15 -0
- opscloud/agent/__init__.py +27 -0
- opscloud/agent/config.py +168 -0
- opscloud/agent/factory.py +1103 -0
- opscloud/approval_mode.py +44 -0
- opscloud/backend/__init__.py +49 -0
- opscloud/backend/composite.py +89 -0
- opscloud/backend/local.py +183 -0
- opscloud/backend/registry.py +79 -0
- opscloud/backend/sandbox_config.py +107 -0
- opscloud/backend/sandbox_factory.py +432 -0
- opscloud/built_in_skills/remember/SKILL.md +124 -0
- opscloud/built_in_skills/skill-creator/SKILL.md +348 -0
- opscloud/built_in_skills/skill-creator/scripts/init_skill.py +371 -0
- opscloud/built_in_skills/skill-creator/scripts/quick_validate.py +165 -0
- opscloud/cli/__init__.py +1 -0
- opscloud/cli/commands/__init__.py +15 -0
- opscloud/cli/commands/auth.py +45 -0
- opscloud/cli/commands/doctor.py +100 -0
- opscloud/cli/commands/mcp.py +137 -0
- opscloud/cli/commands/threads.py +51 -0
- opscloud/cli/interactive.py +154 -0
- opscloud/cli/main.py +903 -0
- opscloud/cli/non_interactive.py +425 -0
- opscloud/cli/server_manager.py +58 -0
- opscloud/commands/__init__.py +19 -0
- opscloud/commands/_base.py +110 -0
- opscloud/commands/_guards.py +52 -0
- opscloud/commands/_router.py +146 -0
- opscloud/commands/_types.py +36 -0
- opscloud/commands/core/__init__.py +47 -0
- opscloud/commands/core/auth.py +94 -0
- opscloud/commands/core/bug.py +46 -0
- opscloud/commands/core/clear.py +106 -0
- opscloud/commands/core/cloud.py +185 -0
- opscloud/commands/core/compact.py +128 -0
- opscloud/commands/core/config_cmd.py +120 -0
- opscloud/commands/core/context.py +220 -0
- opscloud/commands/core/cost.py +44 -0
- opscloud/commands/core/doctor.py +535 -0
- opscloud/commands/core/effort.py +108 -0
- opscloud/commands/core/exit_cmd.py +41 -0
- opscloud/commands/core/fast.py +148 -0
- opscloud/commands/core/help_cmd.py +94 -0
- opscloud/commands/core/mcp.py +134 -0
- opscloud/commands/core/model.py +109 -0
- opscloud/commands/core/permissions.py +196 -0
- opscloud/commands/core/plugins.py +92 -0
- opscloud/commands/core/pool.py +108 -0
- opscloud/commands/core/resume.py +97 -0
- opscloud/commands/core/skills.py +178 -0
- opscloud/commands/power/__init__.py +51 -0
- opscloud/commands/power/agents.py +98 -0
- opscloud/commands/power/btw.py +85 -0
- opscloud/commands/power/copy.py +97 -0
- opscloud/commands/power/goal.py +548 -0
- opscloud/commands/power/loop.py +295 -0
- opscloud/commands/power/memory.py +287 -0
- opscloud/commands/power/review.py +152 -0
- opscloud/commands/power/rubric.py +287 -0
- opscloud/commands/power/runtime.py +321 -0
- opscloud/commands/power/skill_creator.py +70 -0
- opscloud/commands/power/skill_invoke.py +165 -0
- opscloud/commands/power/tasks.py +209 -0
- opscloud/commands/power/trace.py +117 -0
- opscloud/commands/power/ui_toggles.py +135 -0
- opscloud/commands/power/version.py +52 -0
- opscloud/commands/registry.py +176 -0
- opscloud/config/__init__.py +114 -0
- opscloud/config/adapters/__init__.py +5 -0
- opscloud/config/adapters/postgres.py +776 -0
- opscloud/config/adapters/sqlite.py +769 -0
- opscloud/config/aws.py +248 -0
- opscloud/config/cloud_profiles.py +437 -0
- opscloud/config/env_vars.py +81 -0
- opscloud/config/langsmith.py +319 -0
- opscloud/config/manifest.py +817 -0
- opscloud/config/metadata.py +205 -0
- opscloud/config/paths.py +604 -0
- opscloud/config/plugins.py +130 -0
- opscloud/config/settings.py +834 -0
- opscloud/config/store.py +453 -0
- opscloud/config/store_factory.py +97 -0
- opscloud/config/toml_config.py +562 -0
- opscloud/editor.py +173 -0
- opscloud/exceptions.py +65 -0
- opscloud/file_ops.py +217 -0
- opscloud/hooks/__init__.py +139 -0
- opscloud/hooks/env.py +44 -0
- opscloud/hooks/interrupt.py +121 -0
- opscloud/hooks/models/__init__.py +109 -0
- opscloud/hooks/models/adapters.py +19 -0
- opscloud/hooks/models/config.py +79 -0
- opscloud/hooks/models/domain.py +434 -0
- opscloud/hooks/models/transport.py +37 -0
- opscloud/hooks/tools.py +231 -0
- opscloud/input.py +48 -0
- opscloud/integrations/__init__.py +80 -0
- opscloud/integrations/base.py +153 -0
- opscloud/integrations/content_filter.py +119 -0
- opscloud/integrations/event_bus.py +480 -0
- opscloud/integrations/hooks.py +238 -0
- opscloud/integrations/identity.py +181 -0
- opscloud/integrations/notifications.py +134 -0
- opscloud/integrations/stream_bridge.py +397 -0
- opscloud/integrations/thread_store.py +81 -0
- opscloud/json_types.py +20 -0
- opscloud/mcp/__init__.py +57 -0
- opscloud/mcp/config.py +203 -0
- opscloud/mcp/discovery.py +229 -0
- opscloud/mcp/mcp_info.py +109 -0
- opscloud/mcp/middleware.py +102 -0
- opscloud/mcp/preload.py +505 -0
- opscloud/mcp/raw_config.py +149 -0
- opscloud/mcp/semantic_profiler.py +305 -0
- opscloud/mcp/session_manager.py +727 -0
- opscloud/mcp/trust.py +105 -0
- opscloud/memory/__init__.py +27 -0
- opscloud/memory/branch.py +61 -0
- opscloud/memory/guard.py +19 -0
- opscloud/memory/onboarding.py +120 -0
- opscloud/memory/registry.py +134 -0
- opscloud/memory/store.py +223 -0
- opscloud/middleware/__init__.py +106 -0
- opscloud/middleware/_repository_bounds.py +459 -0
- opscloud/middleware/ask_user.py +152 -0
- opscloud/middleware/auto_mode.py +60 -0
- opscloud/middleware/auto_mode_hitl.py +2434 -0
- opscloud/middleware/compaction.py +566 -0
- opscloud/middleware/configurable_model.py +233 -0
- opscloud/middleware/cost_tracking.py +1021 -0
- opscloud/middleware/glm_stall_recovery.py +109 -0
- opscloud/middleware/goal_criteria.py +1234 -0
- opscloud/middleware/goal_state_notice.py +712 -0
- opscloud/middleware/goal_tools.py +407 -0
- opscloud/middleware/headless_mcp_guard.py +88 -0
- opscloud/middleware/jev_model_router.py +1018 -0
- opscloud/middleware/local_context.py +460 -0
- opscloud/middleware/mcp_context.py +151 -0
- opscloud/middleware/mcp_middleware.py +146 -0
- opscloud/middleware/memory_guard.py +393 -0
- opscloud/middleware/model_retry.py +205 -0
- opscloud/middleware/registry.py +137 -0
- opscloud/middleware/reliable_rubric.py +709 -0
- opscloud/middleware/resume_state.py +91 -0
- opscloud/middleware/server_hooks.py +1189 -0
- opscloud/middleware/shell_allow_list.py +90 -0
- opscloud/middleware/skills.py +382 -0
- opscloud/middleware/subagent_artifacts.py +84 -0
- opscloud/middleware/subagent_telemetry.py +261 -0
- opscloud/middleware/subagents.py +243 -0
- opscloud/middleware/tool_filter.py +384 -0
- opscloud/middleware/unified_system_message.py +62 -0
- opscloud/model/__init__.py +37 -0
- opscloud/model/config.py +2992 -0
- opscloud/model/factory.py +477 -0
- opscloud/model/pool.py +459 -0
- opscloud/model/reasoning.py +282 -0
- opscloud/offload.py +173 -0
- opscloud/output.py +64 -0
- opscloud/plugins/__init__.py +189 -0
- opscloud/plugins/_json.py +22 -0
- opscloud/plugins/adapters/__init__.py +13 -0
- opscloud/plugins/adapters/agents.py +112 -0
- opscloud/plugins/adapters/commands.py +190 -0
- opscloud/plugins/adapters/mcp.py +363 -0
- opscloud/plugins/adapters/skills.py +132 -0
- opscloud/plugins/commands_cli.py +541 -0
- opscloud/plugins/discovery.py +512 -0
- opscloud/plugins/manifest.py +335 -0
- opscloud/plugins/marketplace.py +893 -0
- opscloud/plugins/models.py +334 -0
- opscloud/plugins/project_plugins.py +317 -0
- opscloud/plugins/protocol.py +31 -0
- opscloud/plugins/store.py +905 -0
- opscloud/plugins/substitution.py +122 -0
- opscloud/project_utils.py +145 -0
- opscloud/prompts/__init__.py +37 -0
- opscloud/prompts/system.py +483 -0
- opscloud/prompts/templates/system_prompt.md +166 -0
- opscloud/prompts/types.py +54 -0
- opscloud/rubrics/__init__.py +42 -0
- opscloud/rubrics/evaluator.py +404 -0
- opscloud/rubrics/evidence_extractor.py +229 -0
- opscloud/rubrics/generator.py +239 -0
- opscloud/rubrics/jev_compiler.py +175 -0
- opscloud/rubrics/jev_grader.py +297 -0
- opscloud/rubrics/middleware.py +8 -0
- opscloud/schema/__init__.py +107 -0
- opscloud/schema/interrupts.py +981 -0
- opscloud/security/__init__.py +91 -0
- opscloud/security/approval_mode.py +528 -0
- opscloud/security/approval_mode_source.py +230 -0
- opscloud/security/cli_ast_evaluator.py +552 -0
- opscloud/security/jev_classifier.py +651 -0
- opscloud/security/shell_safety.py +340 -0
- opscloud/security/unicode_security.py +439 -0
- opscloud/security/url_validation.py +160 -0
- opscloud/server/__init__.py +13 -0
- opscloud/server/_server_config.py +158 -0
- opscloud/server/server.py +242 -0
- opscloud/server/server_graph.py +93 -0
- opscloud/skills/__init__.py +91 -0
- opscloud/skills/commands.py +1099 -0
- opscloud/skills/invocation.py +200 -0
- opscloud/skills/load.py +388 -0
- opscloud/skills/loader.py +23 -0
- opscloud/skills/merge.py +65 -0
- opscloud/skills/registry.py +263 -0
- opscloud/skills/sources.py +174 -0
- opscloud/skills/trust.py +417 -0
- opscloud/state/__init__.py +142 -0
- opscloud/state/base.py +154 -0
- opscloud/state/cloud_context.py +262 -0
- opscloud/state/goal_channels.py +124 -0
- opscloud/state/goal_state_limits.py +175 -0
- opscloud/state/resume_state.py +58 -0
- opscloud/state/service_context.py +281 -0
- opscloud/state/session.py +957 -0
- opscloud/state/state_migration.py +86 -0
- opscloud/subagents/__init__.py +30 -0
- opscloud/subagents/loader.py +331 -0
- opscloud/subagents/subagents_parser.py +117 -0
- opscloud/subagents/types.py +25 -0
- opscloud/tools/__init__.py +36 -0
- opscloud/tools/catalog.py +37 -0
- opscloud/tools/display.py +24 -0
- opscloud/tools/fetch_url.py +176 -0
- opscloud/tools/goal_tools.py +423 -0
- opscloud/tools/registry.py +105 -0
- opscloud/tools/thread.py +32 -0
- opscloud/tools/web_search.py +215 -0
- opscloud/ui/__init__.py +119 -0
- opscloud/ui/app.py +5400 -0
- opscloud/ui/app.tcss +479 -0
- opscloud/ui/clipboard.py +147 -0
- opscloud/ui/command_registry.py +482 -0
- opscloud/ui/message_store.py +152 -0
- opscloud/ui/permission_store.py +360 -0
- opscloud/ui/preferences.py +113 -0
- opscloud/ui/prompt_manager.py +96 -0
- opscloud/ui/remote_client.py +505 -0
- opscloud/ui/textual_adapter.py +1269 -0
- opscloud/ui/theme.py +728 -0
- opscloud/ui/ui_help.py +415 -0
- opscloud/ui/widgets/__init__.py +383 -0
- opscloud/ui/widgets/_ask_user_types.py +81 -0
- opscloud/ui/widgets/_copy_spans.py +52 -0
- opscloud/ui/widgets/_inline_prompt.py +290 -0
- opscloud/ui/widgets/_js_eval_display.py +139 -0
- opscloud/ui/widgets/_links.py +175 -0
- opscloud/ui/widgets/_tool_stream.py +308 -0
- opscloud/ui/widgets/agent_selector.py +297 -0
- opscloud/ui/widgets/approval.py +741 -0
- opscloud/ui/widgets/ask_user.py +539 -0
- opscloud/ui/widgets/auth.py +695 -0
- opscloud/ui/widgets/auth_manager.py +214 -0
- opscloud/ui/widgets/auto_mode_notice.py +146 -0
- opscloud/ui/widgets/autocomplete.py +709 -0
- opscloud/ui/widgets/chat_input.py +805 -0
- opscloud/ui/widgets/cloud_selector.py +528 -0
- opscloud/ui/widgets/config_manager.py +652 -0
- opscloud/ui/widgets/debug_console.py +1269 -0
- opscloud/ui/widgets/devops_renderers.py +184 -0
- opscloud/ui/widgets/diff.py +237 -0
- opscloud/ui/widgets/effort_selector.py +166 -0
- opscloud/ui/widgets/goal_review.py +612 -0
- opscloud/ui/widgets/goal_status.py +57 -0
- opscloud/ui/widgets/infra_panel.py +141 -0
- opscloud/ui/widgets/install_confirm.py +114 -0
- opscloud/ui/widgets/loading.py +170 -0
- opscloud/ui/widgets/mcp_viewer.py +1204 -0
- opscloud/ui/widgets/messages.py +4461 -0
- opscloud/ui/widgets/model_selector.py +863 -0
- opscloud/ui/widgets/notification_center.py +108 -0
- opscloud/ui/widgets/notification_settings.py +119 -0
- opscloud/ui/widgets/operation_card.py +106 -0
- opscloud/ui/widgets/permissions_manager.py +667 -0
- opscloud/ui/widgets/plugin_manager.py +1212 -0
- opscloud/ui/widgets/pool_selector.py +376 -0
- opscloud/ui/widgets/preamble.py +53 -0
- opscloud/ui/widgets/skills_viewer.py +432 -0
- opscloud/ui/widgets/status.py +1037 -0
- opscloud/ui/widgets/subagent_panel.py +927 -0
- opscloud/ui/widgets/theme_selector.py +232 -0
- opscloud/ui/widgets/thread_selector.py +358 -0
- opscloud/ui/widgets/toast.py +31 -0
- opscloud/ui/widgets/tool_display.py +325 -0
- opscloud/ui/widgets/tool_renderers.py +234 -0
- opscloud/ui/widgets/tool_widgets.py +278 -0
- opscloud/ui/widgets/welcome.py +421 -0
- opscloud/ui/widgets/welcome_popup.py +102 -0
- opscloud/utils/__init__.py +24 -0
- opscloud/utils/cost_estimation.py +169 -0
- opscloud/utils/git.py +390 -0
- opscloud/utils/logger.py +571 -0
- opscloud/utils/session_stats.py +115 -0
- opscloud/utils/startup_error.py +37 -0
- talkops_opscloud-0.3.0b1.dist-info/METADATA +449 -0
- talkops_opscloud-0.3.0b1.dist-info/RECORD +309 -0
- talkops_opscloud-0.3.0b1.dist-info/WHEEL +4 -0
- talkops_opscloud-0.3.0b1.dist-info/entry_points.txt +2 -0
- talkops_opscloud-0.3.0b1.dist-info/licenses/LICENSE +175 -0
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
"""Model-node retry middleware for OpsCloud terminal & TUI execution.
|
|
2
|
+
|
|
3
|
+
Wraps model invocation with exponential backoff and jitter for transient errors:
|
|
4
|
+
- Rate limits (HTTP 429, ResourceExhausted)
|
|
5
|
+
- Timeouts (HTTP 408, 504, APITimeoutError)
|
|
6
|
+
- Connection drops and resets (Botocore ConnectionClosedError, Anthropic APIConnectionError, OpenAI APIConnectionError)
|
|
7
|
+
- Server faults (HTTP 500, 502, 503)
|
|
8
|
+
|
|
9
|
+
Emits real-time stream events so the terminal status bar updates with countdown/attempt details.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import asyncio
|
|
15
|
+
from collections.abc import Awaitable, Callable
|
|
16
|
+
import logging
|
|
17
|
+
import math
|
|
18
|
+
import random
|
|
19
|
+
import time
|
|
20
|
+
from typing import TYPE_CHECKING, Any
|
|
21
|
+
import uuid
|
|
22
|
+
|
|
23
|
+
from langchain.agents.middleware.types import (
|
|
24
|
+
AgentMiddleware,
|
|
25
|
+
ModelRequest,
|
|
26
|
+
ModelResponse,
|
|
27
|
+
)
|
|
28
|
+
from langgraph.errors import GraphBubbleUp
|
|
29
|
+
|
|
30
|
+
from opscloud.middleware.registry import register_middleware
|
|
31
|
+
from opscloud.utils.logger import get_logger
|
|
32
|
+
|
|
33
|
+
logger = get_logger(__name__)
|
|
34
|
+
|
|
35
|
+
DEFAULT_MODEL_RETRIES = 3
|
|
36
|
+
MODEL_RETRY_EVENT_TYPE = "model_retry"
|
|
37
|
+
|
|
38
|
+
_INITIAL_DELAY_SECONDS = 0.5
|
|
39
|
+
_BACKOFF_FACTOR = 2.0
|
|
40
|
+
_MAX_DELAY_SECONDS = 10.0
|
|
41
|
+
_JITTER_FRACTION = 0.15
|
|
42
|
+
_RETRYABLE_STATUS_CODES = frozenset({408, 409, 429, 500, 502, 503, 504})
|
|
43
|
+
|
|
44
|
+
_TRANSIENT_SDK_EXC_NAMES = frozenset(
|
|
45
|
+
{
|
|
46
|
+
("anthropic", "APIConnectionError"),
|
|
47
|
+
("anthropic", "APIConnectionTimeoutError"),
|
|
48
|
+
("anthropic", "APITimeoutError"),
|
|
49
|
+
("botocore", "ConnectionClosedError"),
|
|
50
|
+
("botocore", "ConnectTimeoutError"),
|
|
51
|
+
("botocore", "EndpointConnectionError"),
|
|
52
|
+
("botocore", "ReadTimeoutError"),
|
|
53
|
+
("google", "Aborted"),
|
|
54
|
+
("google", "DeadlineExceeded"),
|
|
55
|
+
("google", "ResourceExhausted"),
|
|
56
|
+
("google", "ServiceUnavailable"),
|
|
57
|
+
("openai", "APIConnectionError"),
|
|
58
|
+
("openai", "APITimeoutError"),
|
|
59
|
+
("urllib3", "ConnectTimeoutError"),
|
|
60
|
+
("urllib3", "ReadTimeoutError"),
|
|
61
|
+
("httpx", "ConnectTimeout"),
|
|
62
|
+
("httpx", "ReadTimeout"),
|
|
63
|
+
("httpx", "NetworkError"),
|
|
64
|
+
}
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _extract_status_code(exc: Exception) -> int | None:
|
|
69
|
+
for attr in ("status_code", "status", "http_status", "code"):
|
|
70
|
+
val = getattr(exc, attr, None)
|
|
71
|
+
if isinstance(val, int) and not isinstance(val, bool):
|
|
72
|
+
return val
|
|
73
|
+
# Check response object
|
|
74
|
+
resp = getattr(exc, "response", None)
|
|
75
|
+
if resp is not None:
|
|
76
|
+
val = getattr(resp, "status_code", None)
|
|
77
|
+
if isinstance(val, int) and not isinstance(val, bool):
|
|
78
|
+
return val
|
|
79
|
+
return None
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def is_transient_error(exc: Exception) -> bool:
|
|
83
|
+
"""Return whether the exception represents a transient, retryable failure."""
|
|
84
|
+
if isinstance(exc, (KeyboardInterrupt, SystemExit, GraphBubbleUp)):
|
|
85
|
+
return False
|
|
86
|
+
|
|
87
|
+
code = _extract_status_code(exc)
|
|
88
|
+
if code is not None and code in _RETRYABLE_STATUS_CODES:
|
|
89
|
+
return True
|
|
90
|
+
|
|
91
|
+
# Check known exception class names
|
|
92
|
+
exc_cls = type(exc)
|
|
93
|
+
exc_name = exc_cls.__name__
|
|
94
|
+
module_root = exc_cls.__module__.split(".")[0].lower()
|
|
95
|
+
|
|
96
|
+
for pkg, name in _TRANSIENT_SDK_EXC_NAMES:
|
|
97
|
+
if (pkg == module_root or pkg in exc_cls.__module__.lower()) and exc_name == name:
|
|
98
|
+
return True
|
|
99
|
+
|
|
100
|
+
# Common string signals
|
|
101
|
+
exc_str = str(exc).lower()
|
|
102
|
+
transient_signals = (
|
|
103
|
+
"rate limit",
|
|
104
|
+
"too many requests",
|
|
105
|
+
"overloaded",
|
|
106
|
+
"connection reset",
|
|
107
|
+
"timeout",
|
|
108
|
+
"timed out",
|
|
109
|
+
"time out",
|
|
110
|
+
"service unavailable",
|
|
111
|
+
"throttling",
|
|
112
|
+
)
|
|
113
|
+
return any(sig in exc_str for sig in transient_signals)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def compute_delay(attempt: int) -> float:
|
|
117
|
+
"""Compute exponential backoff delay with jitter."""
|
|
118
|
+
base = _INITIAL_DELAY_SECONDS * (_BACKOFF_FACTOR ** (attempt - 1))
|
|
119
|
+
bounded = min(base, _MAX_DELAY_SECONDS)
|
|
120
|
+
jitter = bounded * _JITTER_FRACTION * (random.random() * 2 - 1)
|
|
121
|
+
return max(0.1, bounded + jitter)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
@register_middleware(name="model_retry")
|
|
125
|
+
class CodeModelRetryMiddleware(AgentMiddleware):
|
|
126
|
+
"""Retries transient LLM failures with exponential backoff and telemetry events."""
|
|
127
|
+
|
|
128
|
+
def __init__(self, max_retries: int = DEFAULT_MODEL_RETRIES) -> None:
|
|
129
|
+
super().__init__()
|
|
130
|
+
self.max_retries = max_retries
|
|
131
|
+
|
|
132
|
+
def _emit_retry_event(
|
|
133
|
+
self,
|
|
134
|
+
request: ModelRequest,
|
|
135
|
+
attempt: int,
|
|
136
|
+
delay: float,
|
|
137
|
+
exc: Exception,
|
|
138
|
+
) -> None:
|
|
139
|
+
state = getattr(request, "state", {}) or {}
|
|
140
|
+
runtime = getattr(request, "runtime", None)
|
|
141
|
+
writer = getattr(runtime, "stream_writer", None) if runtime else None
|
|
142
|
+
code = _extract_status_code(exc)
|
|
143
|
+
status_msg = (
|
|
144
|
+
f"Transient error ({type(exc).__name__}"
|
|
145
|
+
f"{f' {code}' if code else ''}); retrying attempt {attempt}/{self.max_retries} "
|
|
146
|
+
f"in {delay:.1f}s..."
|
|
147
|
+
)
|
|
148
|
+
logger.warning(status_msg)
|
|
149
|
+
if callable(writer):
|
|
150
|
+
try:
|
|
151
|
+
writer(
|
|
152
|
+
{
|
|
153
|
+
"type": MODEL_RETRY_EVENT_TYPE,
|
|
154
|
+
"attempt": attempt,
|
|
155
|
+
"max_retries": self.max_retries,
|
|
156
|
+
"delay": delay,
|
|
157
|
+
"message": status_msg,
|
|
158
|
+
"error": str(exc),
|
|
159
|
+
}
|
|
160
|
+
)
|
|
161
|
+
except Exception:
|
|
162
|
+
pass
|
|
163
|
+
|
|
164
|
+
def wrap_model_call(
|
|
165
|
+
self,
|
|
166
|
+
request: ModelRequest,
|
|
167
|
+
handler: Callable[[ModelRequest], ModelResponse],
|
|
168
|
+
) -> ModelResponse:
|
|
169
|
+
attempt = 0
|
|
170
|
+
while True:
|
|
171
|
+
try:
|
|
172
|
+
return handler(request)
|
|
173
|
+
except Exception as exc:
|
|
174
|
+
if attempt >= self.max_retries or not is_transient_error(exc):
|
|
175
|
+
raise
|
|
176
|
+
attempt += 1
|
|
177
|
+
delay = compute_delay(attempt)
|
|
178
|
+
self._emit_retry_event(request, attempt, delay, exc)
|
|
179
|
+
time.sleep(delay)
|
|
180
|
+
|
|
181
|
+
async def awrap_model_call(
|
|
182
|
+
self,
|
|
183
|
+
request: ModelRequest,
|
|
184
|
+
handler: Callable[[ModelRequest], Awaitable[ModelResponse]],
|
|
185
|
+
) -> ModelResponse:
|
|
186
|
+
attempt = 0
|
|
187
|
+
while True:
|
|
188
|
+
try:
|
|
189
|
+
return await handler(request)
|
|
190
|
+
except Exception as exc:
|
|
191
|
+
if attempt >= self.max_retries or not is_transient_error(exc):
|
|
192
|
+
raise
|
|
193
|
+
attempt += 1
|
|
194
|
+
delay = compute_delay(attempt)
|
|
195
|
+
self._emit_retry_event(request, attempt, delay, exc)
|
|
196
|
+
await asyncio.sleep(delay)
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
__all__ = [
|
|
200
|
+
"DEFAULT_MODEL_RETRIES",
|
|
201
|
+
"MODEL_RETRY_EVENT_TYPE",
|
|
202
|
+
"CodeModelRetryMiddleware",
|
|
203
|
+
"compute_delay",
|
|
204
|
+
"is_transient_error",
|
|
205
|
+
]
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
"""Middleware registry — singleton with ``@register_middleware`` decorator."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Callable
|
|
6
|
+
import contextlib
|
|
7
|
+
import threading
|
|
8
|
+
from typing import Any, TypeVar
|
|
9
|
+
|
|
10
|
+
from langchain.agents.middleware.types import AgentMiddleware
|
|
11
|
+
|
|
12
|
+
BaseAgentMiddleware = AgentMiddleware
|
|
13
|
+
|
|
14
|
+
T = TypeVar("T", bound=type)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class MiddlewareRegistry:
|
|
18
|
+
"""Registry of middleware providers. Preserves insertion order."""
|
|
19
|
+
|
|
20
|
+
_instance: MiddlewareRegistry | None = None
|
|
21
|
+
_lock = threading.Lock()
|
|
22
|
+
|
|
23
|
+
def __init__(self) -> None:
|
|
24
|
+
"""Initialize an empty middleware registry."""
|
|
25
|
+
self._registry: dict[str, tuple[type[AgentMiddleware], dict[str, Any]]] = {}
|
|
26
|
+
|
|
27
|
+
def register(
|
|
28
|
+
self,
|
|
29
|
+
name: str,
|
|
30
|
+
cls: type[AgentMiddleware],
|
|
31
|
+
*,
|
|
32
|
+
default_kwargs: dict[str, Any] | None = None,
|
|
33
|
+
) -> None:
|
|
34
|
+
"""Register a middleware class under *name*."""
|
|
35
|
+
self._registry[name] = (cls, default_kwargs or {})
|
|
36
|
+
|
|
37
|
+
def get(self, name: str) -> type[AgentMiddleware] | None:
|
|
38
|
+
"""Return the middleware class for *name*, or ``None``."""
|
|
39
|
+
entry = self._registry.get(name)
|
|
40
|
+
return entry[0] if entry else None
|
|
41
|
+
|
|
42
|
+
def list_registered(self) -> list[str]:
|
|
43
|
+
"""Return sorted list of registered middleware names."""
|
|
44
|
+
return sorted(self._registry.keys())
|
|
45
|
+
|
|
46
|
+
def build_stack(
|
|
47
|
+
self,
|
|
48
|
+
*,
|
|
49
|
+
exclude: set[str] | None = None,
|
|
50
|
+
**kwargs: Any,
|
|
51
|
+
) -> list[AgentMiddleware]:
|
|
52
|
+
"""Build the full middleware stack, preserving insertion order."""
|
|
53
|
+
items = [(name, item) for name, item in self._registry.items() if not (exclude and name in exclude)]
|
|
54
|
+
stack = []
|
|
55
|
+
for name, (cls, default_kwargs) in items:
|
|
56
|
+
inst_kwargs = {**default_kwargs, **kwargs.get(name, {})}
|
|
57
|
+
stack.append(cls(**inst_kwargs))
|
|
58
|
+
return stack
|
|
59
|
+
|
|
60
|
+
def build_middleware(self, name: str, **kwargs: Any) -> AgentMiddleware:
|
|
61
|
+
"""Instantiate a single registered middleware by name."""
|
|
62
|
+
entry = self._registry.get(name)
|
|
63
|
+
if entry is None:
|
|
64
|
+
raise KeyError(f"Middleware {name!r} not registered")
|
|
65
|
+
cls, default_kwargs = entry
|
|
66
|
+
merged = {**default_kwargs, **kwargs}
|
|
67
|
+
return cls(**merged)
|
|
68
|
+
|
|
69
|
+
@classmethod
|
|
70
|
+
def get_instance(cls) -> MiddlewareRegistry:
|
|
71
|
+
"""Return the singleton instance."""
|
|
72
|
+
if cls._instance is None:
|
|
73
|
+
with cls._lock:
|
|
74
|
+
if cls._instance is None:
|
|
75
|
+
cls._instance = cls()
|
|
76
|
+
return cls._instance
|
|
77
|
+
|
|
78
|
+
@classmethod
|
|
79
|
+
def ensure_discovered(cls) -> MiddlewareRegistry:
|
|
80
|
+
"""Ensure all middleware modules are imported (firing @register_middleware)."""
|
|
81
|
+
inst = cls.get_instance()
|
|
82
|
+
if inst._registry:
|
|
83
|
+
return inst
|
|
84
|
+
|
|
85
|
+
import importlib
|
|
86
|
+
|
|
87
|
+
_modules = [
|
|
88
|
+
"opscloud.middleware.ask_user",
|
|
89
|
+
"opscloud.middleware.auto_mode",
|
|
90
|
+
"opscloud.middleware.auto_mode_hitl",
|
|
91
|
+
"opscloud.middleware.compaction",
|
|
92
|
+
"opscloud.middleware.configurable_model",
|
|
93
|
+
"opscloud.middleware.cost_tracking",
|
|
94
|
+
"opscloud.middleware.glm_stall_recovery",
|
|
95
|
+
"opscloud.middleware.goal_criteria",
|
|
96
|
+
"opscloud.middleware.goal_state_notice",
|
|
97
|
+
"opscloud.middleware.goal_tools",
|
|
98
|
+
"opscloud.middleware.headless_mcp_guard",
|
|
99
|
+
"opscloud.middleware.local_context",
|
|
100
|
+
"opscloud.middleware.mcp_context",
|
|
101
|
+
"opscloud.middleware.mcp_middleware",
|
|
102
|
+
"opscloud.middleware.memory_guard",
|
|
103
|
+
"opscloud.middleware.model_retry",
|
|
104
|
+
"opscloud.middleware.reliable_rubric",
|
|
105
|
+
"opscloud.middleware.resume_state",
|
|
106
|
+
"opscloud.middleware.server_hooks",
|
|
107
|
+
"opscloud.middleware.shell_allow_list",
|
|
108
|
+
"opscloud.middleware.skills",
|
|
109
|
+
"opscloud.middleware.subagents",
|
|
110
|
+
"opscloud.middleware.subagent_artifacts",
|
|
111
|
+
"opscloud.middleware.tool_filter",
|
|
112
|
+
"opscloud.middleware.unified_system_message",
|
|
113
|
+
]
|
|
114
|
+
for mod_name in _modules:
|
|
115
|
+
with contextlib.suppress(ImportError):
|
|
116
|
+
importlib.import_module(mod_name)
|
|
117
|
+
|
|
118
|
+
return inst
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def get_middleware_registry() -> MiddlewareRegistry:
|
|
122
|
+
"""Module-level accessor for the singleton ``MiddlewareRegistry``."""
|
|
123
|
+
return MiddlewareRegistry.ensure_discovered()
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def register_middleware(
|
|
127
|
+
name: str,
|
|
128
|
+
*,
|
|
129
|
+
default_kwargs: dict[str, Any] | None = None,
|
|
130
|
+
) -> Callable[[T], T]:
|
|
131
|
+
"""Decorator to register a middleware class."""
|
|
132
|
+
|
|
133
|
+
def decorator(cls: T) -> T:
|
|
134
|
+
get_middleware_registry().register(name, cls, default_kwargs=default_kwargs) # type: ignore[arg-type]
|
|
135
|
+
return cls
|
|
136
|
+
|
|
137
|
+
return decorator
|