systemr-cli 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- neo/__init__.py +3 -0
- neo/__main__.py +5 -0
- neo/auth.py +192 -0
- neo/cli.py +205 -0
- neo/client.py +64 -0
- neo/commands/__init__.py +0 -0
- neo/commands/auth_commands.py +303 -0
- neo/commands/chat_commands.py +937 -0
- neo/commands/cron_commands.py +179 -0
- neo/commands/doctor_command.py +178 -0
- neo/commands/eval_commands.py +73 -0
- neo/commands/journal_commands.py +197 -0
- neo/commands/plan_commands.py +77 -0
- neo/commands/risk_commands.py +68 -0
- neo/commands/scan_commands.py +62 -0
- neo/commands/size_commands.py +60 -0
- neo/config.py +70 -0
- neo/confirmation.py +311 -0
- neo/credits.py +98 -0
- neo/cron.py +365 -0
- neo/display/__init__.py +0 -0
- neo/display/chat_renderer.py +127 -0
- neo/display/formatters.py +112 -0
- neo/display/tables.py +53 -0
- neo/display/theme.py +154 -0
- neo/hooks.py +170 -0
- neo/logging.py +56 -0
- neo/model_failover.py +193 -0
- neo/orchestrator.py +288 -0
- neo/profile.py +505 -0
- neo/store.py +405 -0
- neo/streaming.py +315 -0
- neo/types.py +109 -0
- systemr_cli-1.0.0.dist-info/METADATA +191 -0
- systemr_cli-1.0.0.dist-info/RECORD +37 -0
- systemr_cli-1.0.0.dist-info/WHEEL +4 -0
- systemr_cli-1.0.0.dist-info/entry_points.txt +3 -0
neo/display/tables.py
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""Rich table helpers — dark, precise, institutional tables.
|
|
2
|
+
|
|
3
|
+
System R brand: green accent on dark background. Labels in green,
|
|
4
|
+
values in white. Matches DESIGN.md §4.3 typography rules.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
from rich.table import Table
|
|
12
|
+
|
|
13
|
+
from neo.display.theme import GREEN, GREEN_DIM, DIM, console
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def kv_table(title: str, data: dict[str, Any], *, value_style: str = f"bold {GREEN}") -> None:
|
|
17
|
+
"""Print a key-value table — labels in green, values in white."""
|
|
18
|
+
table = Table(
|
|
19
|
+
title=f"[bold {GREEN}]{title}[/]",
|
|
20
|
+
show_header=False,
|
|
21
|
+
border_style=DIM,
|
|
22
|
+
padding=(0, 2),
|
|
23
|
+
title_justify="left",
|
|
24
|
+
)
|
|
25
|
+
table.add_column("Key", style=GREEN_DIM, min_width=18)
|
|
26
|
+
table.add_column("Value", style=value_style)
|
|
27
|
+
for key, value in data.items():
|
|
28
|
+
table.add_row(key, str(value))
|
|
29
|
+
console.print(table)
|
|
30
|
+
console.print()
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def data_table(
|
|
34
|
+
title: str,
|
|
35
|
+
columns: list[tuple[str, str]], # (name, style)
|
|
36
|
+
rows: list[list[str]],
|
|
37
|
+
) -> None:
|
|
38
|
+
"""Print a data table with brand-styled columns."""
|
|
39
|
+
table = Table(
|
|
40
|
+
title=f"[bold {GREEN}]{title}[/]",
|
|
41
|
+
border_style=DIM,
|
|
42
|
+
padding=(0, 1),
|
|
43
|
+
title_justify="left",
|
|
44
|
+
row_styles=[f"{GREEN_DIM}", ""], # alternating subtle rows
|
|
45
|
+
)
|
|
46
|
+
for col_name, col_style in columns:
|
|
47
|
+
# Default to green if no style specified
|
|
48
|
+
style = col_style if col_style else GREEN
|
|
49
|
+
table.add_column(col_name, style=style, header_style=f"bold {GREEN}")
|
|
50
|
+
for row in rows:
|
|
51
|
+
table.add_row(*row)
|
|
52
|
+
console.print(table)
|
|
53
|
+
console.print()
|
neo/display/theme.py
ADDED
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
"""System R CLI display theme — dark, precise, institutional.
|
|
2
|
+
|
|
3
|
+
Provides the canonical brand palette (from DESIGN.md §4.2),
|
|
4
|
+
Rich theme, console instance, and print helpers.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from rich.console import Console
|
|
8
|
+
from rich.theme import Theme
|
|
9
|
+
|
|
10
|
+
# ── System R brand palette ──────────────────────────────────────────
|
|
11
|
+
# Canonical colors from systemr.ai. Do not modify without updating
|
|
12
|
+
# DESIGN.md §4.2.
|
|
13
|
+
|
|
14
|
+
WHITE = "#F5F5F5" # Primary text — values, data, emphasis
|
|
15
|
+
GRAY = "#A1A1AA" # Secondary text — labels, descriptions
|
|
16
|
+
DIM = "#52525B" # Tertiary text — timestamps, hints
|
|
17
|
+
MUTED = "#3F3F46" # Borders, separators, disabled
|
|
18
|
+
GREEN = "#3ECF8E" # Brand accent — success, active, emphasis
|
|
19
|
+
GREEN_DIM = "#34B87A" # Secondary accent — hover, borders
|
|
20
|
+
RED = "#EF4444" # Errors, losses, short positions
|
|
21
|
+
AMBER = "#F59E0B" # Warnings, caution states
|
|
22
|
+
BLUE = "#3B82F6" # Info, neutral indicators
|
|
23
|
+
BG = "#09090B" # Terminal background
|
|
24
|
+
|
|
25
|
+
# Legacy aliases (additive-only — old names from Matrix theme still work)
|
|
26
|
+
DARK_BG = BG
|
|
27
|
+
|
|
28
|
+
# ── Rich theme ──────────────────────────────────────────────────────
|
|
29
|
+
|
|
30
|
+
SYSTEM_R_THEME = Theme({
|
|
31
|
+
# Core palette
|
|
32
|
+
"sr.white": f"bold {WHITE}",
|
|
33
|
+
"sr.gray": GRAY,
|
|
34
|
+
"sr.dim": DIM,
|
|
35
|
+
"sr.muted": MUTED,
|
|
36
|
+
"sr.accent": f"bold {GREEN}",
|
|
37
|
+
"sr.accent.dim": GREEN_DIM,
|
|
38
|
+
"sr.value": f"bold {WHITE}",
|
|
39
|
+
"sr.label": GRAY,
|
|
40
|
+
|
|
41
|
+
# Semantic
|
|
42
|
+
"sr.success": f"bold {GREEN}",
|
|
43
|
+
"sr.error": f"bold {RED}",
|
|
44
|
+
"sr.warn": f"bold {AMBER}",
|
|
45
|
+
"sr.info": f"bold {BLUE}",
|
|
46
|
+
|
|
47
|
+
# Trading-specific
|
|
48
|
+
"sr.long": f"bold {GREEN}",
|
|
49
|
+
"sr.short": f"bold {RED}",
|
|
50
|
+
"sr.profit": f"bold {GREEN}",
|
|
51
|
+
"sr.loss": f"bold {RED}",
|
|
52
|
+
|
|
53
|
+
# Legacy aliases (additive-only)
|
|
54
|
+
"neo.green": f"bold {GREEN}",
|
|
55
|
+
"neo.dim": DIM,
|
|
56
|
+
"neo.label": GREEN_DIM,
|
|
57
|
+
"neo.value": f"bold {WHITE}",
|
|
58
|
+
"neo.error": f"bold {RED}",
|
|
59
|
+
"neo.warn": f"bold {AMBER}",
|
|
60
|
+
"neo.header": f"bold {GREEN}",
|
|
61
|
+
"neo.muted": f"dim {GREEN_DIM}",
|
|
62
|
+
})
|
|
63
|
+
|
|
64
|
+
console = Console(theme=SYSTEM_R_THEME)
|
|
65
|
+
|
|
66
|
+
# ── Indicators ──────────────────────────────────────────────────────
|
|
67
|
+
|
|
68
|
+
ICON_THINKING = "◐"
|
|
69
|
+
ICON_SUCCESS = "✓"
|
|
70
|
+
ICON_ERROR = "✗"
|
|
71
|
+
ICON_WARN = "!"
|
|
72
|
+
ICON_INFO = "●"
|
|
73
|
+
ICON_CONNECTED = "●"
|
|
74
|
+
ICON_DISCONNECTED = "○"
|
|
75
|
+
|
|
76
|
+
# ── Legacy banner (kept for backward compat) ────────────────────────
|
|
77
|
+
|
|
78
|
+
NEO_BANNER = rf"""[{GREEN}]
|
|
79
|
+
███╗ ██╗███████╗ ██████╗
|
|
80
|
+
████╗ ██║██╔════╝██╔═══██╗
|
|
81
|
+
██╔██╗ ██║█████╗ ██║ ██║
|
|
82
|
+
██║╚██╗██║██╔══╝ ██║ ██║
|
|
83
|
+
██║ ╚████║███████╗╚██████╔╝
|
|
84
|
+
╚═╝ ╚═══╝╚══════╝ ╚═════╝[/]
|
|
85
|
+
[{GREEN_DIM}] System R AI — Trading Operator[/]
|
|
86
|
+
"""
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
# ── Print helpers ───────────────────────────────────────────────────
|
|
90
|
+
|
|
91
|
+
def print_banner() -> None:
|
|
92
|
+
"""Print the System R home screen header.
|
|
93
|
+
|
|
94
|
+
SYSTEM R AI in brand green bold, version, tagline.
|
|
95
|
+
"""
|
|
96
|
+
from neo import __version__
|
|
97
|
+
console.print()
|
|
98
|
+
console.print(f" [{GREEN}][bold]SYSTEM R AI[/bold][/] [{DIM}]V{__version__}[/]", highlight=False)
|
|
99
|
+
console.print()
|
|
100
|
+
console.print(f" [{GRAY}]Trading operating system for agents[/]")
|
|
101
|
+
console.print()
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def neo_banner() -> str:
|
|
105
|
+
"""Return legacy banner string.
|
|
106
|
+
|
|
107
|
+
Returns:
|
|
108
|
+
Rich-formatted banner string.
|
|
109
|
+
"""
|
|
110
|
+
return f"[{GREEN}]SYSTEM R[/] [{GREEN_DIM}]Trading OS[/]"
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def print_error(msg: str) -> None:
|
|
114
|
+
"""Print an error message.
|
|
115
|
+
|
|
116
|
+
Args:
|
|
117
|
+
msg: Error message text.
|
|
118
|
+
"""
|
|
119
|
+
console.print(f" [{RED}]{ICON_ERROR}[/] [{GRAY}]{msg}[/]")
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def print_success(msg: str) -> None:
|
|
123
|
+
"""Print a success message.
|
|
124
|
+
|
|
125
|
+
Args:
|
|
126
|
+
msg: Success message text.
|
|
127
|
+
"""
|
|
128
|
+
console.print(f" [{GREEN}]{ICON_SUCCESS}[/] [{WHITE}]{msg}[/]")
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def print_warn(msg: str) -> None:
|
|
132
|
+
"""Print a warning message.
|
|
133
|
+
|
|
134
|
+
Args:
|
|
135
|
+
msg: Warning message text.
|
|
136
|
+
"""
|
|
137
|
+
console.print(f" [{AMBER}]{ICON_WARN}[/] [{GRAY}]{msg}[/]")
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def print_info(msg: str) -> None:
|
|
141
|
+
"""Print an info message.
|
|
142
|
+
|
|
143
|
+
Args:
|
|
144
|
+
msg: Info message text.
|
|
145
|
+
"""
|
|
146
|
+
console.print(f" [{BLUE}]{ICON_INFO}[/] [{GRAY}]{msg}[/]")
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def print_separator() -> None:
|
|
150
|
+
"""Print a separator line.
|
|
151
|
+
|
|
152
|
+
Uses 48 thin dashes in MUTED color.
|
|
153
|
+
"""
|
|
154
|
+
console.print(f" [{MUTED}]{'─' * 48}[/]")
|
neo/hooks.py
ADDED
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
"""Hooks system — pre/post event automation.
|
|
2
|
+
|
|
3
|
+
Mirrors Claude Code's hook architecture with trading-specific events.
|
|
4
|
+
Hooks are shell commands configured in ~/.systemr/config.json.
|
|
5
|
+
|
|
6
|
+
Event types:
|
|
7
|
+
PreTrade — before any order is placed (confirmation approved)
|
|
8
|
+
PostTrade — after an order is confirmed executed
|
|
9
|
+
PreSession — when a chat session starts
|
|
10
|
+
PostSession — when a chat session ends
|
|
11
|
+
RuleViolation — when a trading rule is violated
|
|
12
|
+
KillSwitch — when kill switch is activated
|
|
13
|
+
LowBalance — when credits drop below threshold
|
|
14
|
+
Error — when an error occurs
|
|
15
|
+
|
|
16
|
+
Config format in config.json:
|
|
17
|
+
{
|
|
18
|
+
"hooks": {
|
|
19
|
+
"PreTrade": [{"command": "echo 'Trade incoming'"}],
|
|
20
|
+
"PostTrade": [{"command": "say 'Trade executed'"}]
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
import json
|
|
28
|
+
import os
|
|
29
|
+
import subprocess
|
|
30
|
+
from enum import Enum
|
|
31
|
+
from typing import Any
|
|
32
|
+
|
|
33
|
+
import structlog
|
|
34
|
+
|
|
35
|
+
from neo.config import CONFIG_FILE
|
|
36
|
+
|
|
37
|
+
logger = structlog.get_logger(module="hooks")
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class HookEvent(Enum):
|
|
41
|
+
"""Hook event types matching Claude Code's architecture.
|
|
42
|
+
|
|
43
|
+
Extended with OpenClaw-inspired events:
|
|
44
|
+
- BEFORE_TOOL_CALL: inspect tool name + args before execution, can block
|
|
45
|
+
- COMPACT_BEFORE: auto-flush memories before context compaction
|
|
46
|
+
"""
|
|
47
|
+
|
|
48
|
+
PRE_TRADE = "PreTrade"
|
|
49
|
+
POST_TRADE = "PostTrade"
|
|
50
|
+
PRE_SESSION = "PreSession"
|
|
51
|
+
POST_SESSION = "PostSession"
|
|
52
|
+
RULE_VIOLATION = "RuleViolation"
|
|
53
|
+
KILL_SWITCH = "KillSwitch"
|
|
54
|
+
LOW_BALANCE = "LowBalance"
|
|
55
|
+
ERROR = "Error"
|
|
56
|
+
# New: OpenClaw-inspired events
|
|
57
|
+
BEFORE_TOOL_CALL = "BeforeToolCall"
|
|
58
|
+
COMPACT_BEFORE = "CompactBefore"
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
# Maximum time a hook can run before being killed
|
|
62
|
+
_HOOK_TIMEOUT_SECONDS = 10
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def fire_hook(
|
|
66
|
+
event: HookEvent,
|
|
67
|
+
context: dict[str, Any] | None = None,
|
|
68
|
+
) -> list[str]:
|
|
69
|
+
"""Fire all hooks registered for the given event.
|
|
70
|
+
|
|
71
|
+
Hooks are non-blocking — they run with a timeout and failures
|
|
72
|
+
are logged but never propagate to the caller.
|
|
73
|
+
|
|
74
|
+
Args:
|
|
75
|
+
event: The hook event type to fire.
|
|
76
|
+
context: Optional dict passed as SYSTEMR_HOOK_CONTEXT env var.
|
|
77
|
+
|
|
78
|
+
Returns:
|
|
79
|
+
List of hook stdout outputs (empty strings filtered out).
|
|
80
|
+
"""
|
|
81
|
+
hooks = _load_hooks(event)
|
|
82
|
+
if not hooks:
|
|
83
|
+
return []
|
|
84
|
+
|
|
85
|
+
outputs: list[str] = []
|
|
86
|
+
for hook in hooks:
|
|
87
|
+
command = hook.get("command", "")
|
|
88
|
+
if not command:
|
|
89
|
+
continue
|
|
90
|
+
try:
|
|
91
|
+
env = dict(os.environ)
|
|
92
|
+
if context:
|
|
93
|
+
env["SYSTEMR_HOOK_CONTEXT"] = json.dumps(context)
|
|
94
|
+
|
|
95
|
+
result = subprocess.run(
|
|
96
|
+
command,
|
|
97
|
+
shell=True,
|
|
98
|
+
capture_output=True,
|
|
99
|
+
text=True,
|
|
100
|
+
timeout=_HOOK_TIMEOUT_SECONDS,
|
|
101
|
+
env=env,
|
|
102
|
+
)
|
|
103
|
+
if result.stdout.strip():
|
|
104
|
+
outputs.append(result.stdout.strip())
|
|
105
|
+
logger.debug(
|
|
106
|
+
"hook_fired", hook_event=event.value,
|
|
107
|
+
command=command[:50], exit_code=result.returncode,
|
|
108
|
+
)
|
|
109
|
+
except subprocess.TimeoutExpired:
|
|
110
|
+
logger.warning("hook_timeout", hook_event=event.value, command=command[:50])
|
|
111
|
+
except Exception as exc:
|
|
112
|
+
logger.warning("hook_error", hook_event=event.value, error=str(exc))
|
|
113
|
+
|
|
114
|
+
return outputs
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _load_hooks(event: HookEvent) -> list[dict[str, str]]:
|
|
118
|
+
"""Load hooks for a specific event from config.json.
|
|
119
|
+
|
|
120
|
+
Args:
|
|
121
|
+
event: The hook event type.
|
|
122
|
+
|
|
123
|
+
Returns:
|
|
124
|
+
List of hook dicts with "command" key. Empty if no config.
|
|
125
|
+
"""
|
|
126
|
+
if not CONFIG_FILE.exists():
|
|
127
|
+
return []
|
|
128
|
+
try:
|
|
129
|
+
config = json.loads(CONFIG_FILE.read_text())
|
|
130
|
+
hooks = config.get("hooks", {})
|
|
131
|
+
return hooks.get(event.value, [])
|
|
132
|
+
except (json.JSONDecodeError, KeyError):
|
|
133
|
+
return []
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def register_hook(event: HookEvent, command: str) -> None:
|
|
137
|
+
"""Register a new hook for an event in config.json.
|
|
138
|
+
|
|
139
|
+
Args:
|
|
140
|
+
event: The hook event type.
|
|
141
|
+
command: Shell command to execute.
|
|
142
|
+
"""
|
|
143
|
+
config: dict[str, Any] = {}
|
|
144
|
+
if CONFIG_FILE.exists():
|
|
145
|
+
try:
|
|
146
|
+
config = json.loads(CONFIG_FILE.read_text())
|
|
147
|
+
except json.JSONDecodeError:
|
|
148
|
+
config = {}
|
|
149
|
+
|
|
150
|
+
hooks = config.setdefault("hooks", {})
|
|
151
|
+
event_hooks: list[dict[str, str]] = hooks.setdefault(event.value, [])
|
|
152
|
+
event_hooks.append({"command": command})
|
|
153
|
+
|
|
154
|
+
CONFIG_FILE.write_text(json.dumps(config, indent=2))
|
|
155
|
+
logger.info("hook_registered", hook_event=event.value, command=command[:50])
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def list_hooks() -> dict[str, list[dict[str, str]]]:
|
|
159
|
+
"""List all registered hooks.
|
|
160
|
+
|
|
161
|
+
Returns:
|
|
162
|
+
Dict mapping event names to lists of hook dicts.
|
|
163
|
+
"""
|
|
164
|
+
if not CONFIG_FILE.exists():
|
|
165
|
+
return {}
|
|
166
|
+
try:
|
|
167
|
+
config = json.loads(CONFIG_FILE.read_text())
|
|
168
|
+
return config.get("hooks", {})
|
|
169
|
+
except (json.JSONDecodeError, KeyError):
|
|
170
|
+
return {}
|
neo/logging.py
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
"""Structured logging for System R CLI.
|
|
2
|
+
|
|
3
|
+
All production logging goes through structlog. The Rich console
|
|
4
|
+
is used for user-facing display only — not for logging.
|
|
5
|
+
|
|
6
|
+
Usage:
|
|
7
|
+
from neo.logging import get_logger
|
|
8
|
+
logger = get_logger()
|
|
9
|
+
logger.info("trade_executed", symbol="TSLA", shares=160)
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import logging
|
|
15
|
+
import sys
|
|
16
|
+
|
|
17
|
+
import structlog
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def configure_logging(*, debug: bool = False) -> None:
|
|
21
|
+
"""Configure structlog for the CLI.
|
|
22
|
+
|
|
23
|
+
In production mode (debug=False), logs go to stderr at WARNING level
|
|
24
|
+
to keep stdout clean for the user. In debug mode, logs go to stderr
|
|
25
|
+
at DEBUG level with full console rendering.
|
|
26
|
+
|
|
27
|
+
Args:
|
|
28
|
+
debug: If True, log at DEBUG level with full tracebacks.
|
|
29
|
+
If False, log at WARNING level, JSON format, stderr only.
|
|
30
|
+
"""
|
|
31
|
+
structlog.configure(
|
|
32
|
+
processors=[
|
|
33
|
+
structlog.contextvars.merge_contextvars,
|
|
34
|
+
structlog.processors.add_log_level,
|
|
35
|
+
structlog.processors.TimeStamper(fmt="iso"),
|
|
36
|
+
structlog.dev.ConsoleRenderer() if debug else structlog.processors.JSONRenderer(),
|
|
37
|
+
],
|
|
38
|
+
wrapper_class=structlog.make_filtering_bound_logger(
|
|
39
|
+
logging.DEBUG if debug else logging.WARNING,
|
|
40
|
+
),
|
|
41
|
+
context_class=dict,
|
|
42
|
+
logger_factory=structlog.PrintLoggerFactory(),
|
|
43
|
+
cache_logger_on_first_use=True,
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def get_logger(**initial_context: str) -> structlog.stdlib.BoundLogger:
|
|
48
|
+
"""Get a bound structlog logger.
|
|
49
|
+
|
|
50
|
+
Args:
|
|
51
|
+
**initial_context: Key-value pairs to bind to every log entry.
|
|
52
|
+
|
|
53
|
+
Returns:
|
|
54
|
+
A bound structlog logger instance.
|
|
55
|
+
"""
|
|
56
|
+
return structlog.get_logger(**initial_context)
|
neo/model_failover.py
ADDED
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
"""Model failover chain — automatic fallback on stream failure.
|
|
2
|
+
|
|
3
|
+
When the primary model fails (timeout, 429, 5xx), tries the next model
|
|
4
|
+
in the configured fallback chain. Tracks cooldowns per model to avoid
|
|
5
|
+
retrying a model that just failed.
|
|
6
|
+
|
|
7
|
+
Inspired by OpenClaw's two-stage failover with cooldown tracking and
|
|
8
|
+
session pinning. Simplified for System R: no auth rotation, just model
|
|
9
|
+
fallback with cooldown.
|
|
10
|
+
|
|
11
|
+
No bare print() — logging via structlog.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import json
|
|
17
|
+
import time
|
|
18
|
+
from dataclasses import dataclass, field
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
|
|
21
|
+
import structlog
|
|
22
|
+
|
|
23
|
+
from neo.config import CONFIG_FILE, ensure_systemr_home
|
|
24
|
+
|
|
25
|
+
logger = structlog.get_logger(module="model_failover")
|
|
26
|
+
|
|
27
|
+
# Default cooldown: 60 seconds after a failure
|
|
28
|
+
DEFAULT_COOLDOWN_SECONDS: float = 60.0
|
|
29
|
+
|
|
30
|
+
# Maximum cooldown: 15 minutes
|
|
31
|
+
MAX_COOLDOWN_SECONDS: float = 900.0
|
|
32
|
+
|
|
33
|
+
# Default fallback chain
|
|
34
|
+
DEFAULT_FALLBACKS: list[str] = [
|
|
35
|
+
"anthropic.claude-sonnet-4-6",
|
|
36
|
+
"anthropic.claude-haiku-4-5-20251001",
|
|
37
|
+
]
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@dataclass
|
|
41
|
+
class ModelCooldown:
|
|
42
|
+
"""Tracks failure state for a single model.
|
|
43
|
+
|
|
44
|
+
Attributes:
|
|
45
|
+
model: Model identifier.
|
|
46
|
+
failed_at: Unix timestamp of last failure.
|
|
47
|
+
cooldown_until: Unix timestamp when cooldown expires.
|
|
48
|
+
error_count: Consecutive failure count.
|
|
49
|
+
"""
|
|
50
|
+
|
|
51
|
+
model: str
|
|
52
|
+
failed_at: float = 0.0
|
|
53
|
+
cooldown_until: float = 0.0
|
|
54
|
+
error_count: int = 0
|
|
55
|
+
|
|
56
|
+
@property
|
|
57
|
+
def is_cooled_down(self) -> bool:
|
|
58
|
+
"""True if the model is available (cooldown expired)."""
|
|
59
|
+
return time.time() >= self.cooldown_until
|
|
60
|
+
|
|
61
|
+
def record_failure(self) -> None:
|
|
62
|
+
"""Record a failure and extend cooldown with exponential backoff."""
|
|
63
|
+
self.error_count += 1
|
|
64
|
+
self.failed_at = time.time()
|
|
65
|
+
# Exponential backoff: 60s, 120s, 240s, ... capped at 15min
|
|
66
|
+
backoff = min(
|
|
67
|
+
DEFAULT_COOLDOWN_SECONDS * (2 ** (self.error_count - 1)),
|
|
68
|
+
MAX_COOLDOWN_SECONDS,
|
|
69
|
+
)
|
|
70
|
+
self.cooldown_until = self.failed_at + backoff
|
|
71
|
+
logger.info(
|
|
72
|
+
"model_cooldown_set",
|
|
73
|
+
model=self.model,
|
|
74
|
+
error_count=self.error_count,
|
|
75
|
+
cooldown_seconds=backoff,
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
def record_success(self) -> None:
|
|
79
|
+
"""Reset cooldown state after a successful call."""
|
|
80
|
+
if self.error_count > 0:
|
|
81
|
+
logger.info(
|
|
82
|
+
"model_cooldown_reset",
|
|
83
|
+
model=self.model,
|
|
84
|
+
was_error_count=self.error_count,
|
|
85
|
+
)
|
|
86
|
+
self.error_count = 0
|
|
87
|
+
self.cooldown_until = 0.0
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
@dataclass
|
|
91
|
+
class FailoverChain:
|
|
92
|
+
"""Manages model selection with automatic failover.
|
|
93
|
+
|
|
94
|
+
Attributes:
|
|
95
|
+
primary: Primary model to use.
|
|
96
|
+
fallbacks: Ordered list of fallback models.
|
|
97
|
+
cooldowns: Per-model cooldown state.
|
|
98
|
+
pinned_model: Model pinned for current session (sticks until failure).
|
|
99
|
+
"""
|
|
100
|
+
|
|
101
|
+
primary: str = ""
|
|
102
|
+
fallbacks: list[str] = field(default_factory=list)
|
|
103
|
+
cooldowns: dict[str, ModelCooldown] = field(default_factory=dict)
|
|
104
|
+
pinned_model: str | None = None
|
|
105
|
+
|
|
106
|
+
def get_model(self) -> str:
|
|
107
|
+
"""Return the best available model (pinned > primary > fallbacks).
|
|
108
|
+
|
|
109
|
+
Returns:
|
|
110
|
+
Model identifier string.
|
|
111
|
+
"""
|
|
112
|
+
# If pinned and not in cooldown, use it
|
|
113
|
+
if self.pinned_model:
|
|
114
|
+
cd = self.cooldowns.get(self.pinned_model)
|
|
115
|
+
if cd is None or cd.is_cooled_down:
|
|
116
|
+
return self.pinned_model
|
|
117
|
+
|
|
118
|
+
# Try primary
|
|
119
|
+
cd = self.cooldowns.get(self.primary)
|
|
120
|
+
if cd is None or cd.is_cooled_down:
|
|
121
|
+
return self.primary
|
|
122
|
+
|
|
123
|
+
# Try fallbacks in order
|
|
124
|
+
for model in self.fallbacks:
|
|
125
|
+
cd = self.cooldowns.get(model)
|
|
126
|
+
if cd is None or cd.is_cooled_down:
|
|
127
|
+
logger.info("model_failover", from_model=self.primary, to_model=model)
|
|
128
|
+
return model
|
|
129
|
+
|
|
130
|
+
# All models in cooldown — use primary anyway (best effort)
|
|
131
|
+
logger.warning("all_models_in_cooldown", primary=self.primary)
|
|
132
|
+
return self.primary
|
|
133
|
+
|
|
134
|
+
def on_success(self, model: str) -> None:
|
|
135
|
+
"""Record successful call — pin model for session, reset cooldown.
|
|
136
|
+
|
|
137
|
+
Args:
|
|
138
|
+
model: The model that succeeded.
|
|
139
|
+
"""
|
|
140
|
+
self.pinned_model = model
|
|
141
|
+
if model in self.cooldowns:
|
|
142
|
+
self.cooldowns[model].record_success()
|
|
143
|
+
|
|
144
|
+
def on_failure(self, model: str) -> None:
|
|
145
|
+
"""Record failed call — set cooldown, unpin if pinned.
|
|
146
|
+
|
|
147
|
+
Args:
|
|
148
|
+
model: The model that failed.
|
|
149
|
+
"""
|
|
150
|
+
if model not in self.cooldowns:
|
|
151
|
+
self.cooldowns[model] = ModelCooldown(model=model)
|
|
152
|
+
self.cooldowns[model].record_failure()
|
|
153
|
+
|
|
154
|
+
# Unpin so next call tries a different model
|
|
155
|
+
if self.pinned_model == model:
|
|
156
|
+
self.pinned_model = None
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def load_failover_chain() -> FailoverChain:
|
|
160
|
+
"""Load failover configuration from ~/.systemr/config.json.
|
|
161
|
+
|
|
162
|
+
Config format:
|
|
163
|
+
{
|
|
164
|
+
"model": {
|
|
165
|
+
"primary": "anthropic.claude-sonnet-4-6",
|
|
166
|
+
"fallbacks": ["anthropic.claude-haiku-4-5-20251001"]
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
Returns:
|
|
171
|
+
Configured FailoverChain instance.
|
|
172
|
+
"""
|
|
173
|
+
chain = FailoverChain()
|
|
174
|
+
|
|
175
|
+
if CONFIG_FILE.exists():
|
|
176
|
+
try:
|
|
177
|
+
config = json.loads(CONFIG_FILE.read_text())
|
|
178
|
+
model_config = config.get("model", {})
|
|
179
|
+
chain.primary = model_config.get("primary", "")
|
|
180
|
+
chain.fallbacks = model_config.get("fallbacks", DEFAULT_FALLBACKS)
|
|
181
|
+
except (json.JSONDecodeError, TypeError):
|
|
182
|
+
logger.warning("config_parse_error", path=str(CONFIG_FILE))
|
|
183
|
+
|
|
184
|
+
if not chain.primary:
|
|
185
|
+
chain.primary = DEFAULT_FALLBACKS[0] if DEFAULT_FALLBACKS else ""
|
|
186
|
+
chain.fallbacks = DEFAULT_FALLBACKS[1:] if len(DEFAULT_FALLBACKS) > 1 else []
|
|
187
|
+
|
|
188
|
+
logger.info(
|
|
189
|
+
"failover_chain_loaded",
|
|
190
|
+
primary=chain.primary,
|
|
191
|
+
fallbacks=chain.fallbacks,
|
|
192
|
+
)
|
|
193
|
+
return chain
|