steerable-agent-harness 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- steerable_agent_harness-0.2.0/PKG-INFO +12 -0
- steerable_agent_harness-0.2.0/README.md +3 -0
- steerable_agent_harness-0.2.0/pyproject.toml +27 -0
- steerable_agent_harness-0.2.0/setup.cfg +4 -0
- steerable_agent_harness-0.2.0/src/steerable_agent_harness/__init__.py +21 -0
- steerable_agent_harness-0.2.0/src/steerable_agent_harness/budget.py +38 -0
- steerable_agent_harness-0.2.0/src/steerable_agent_harness/completion.py +13 -0
- steerable_agent_harness-0.2.0/src/steerable_agent_harness/generated.py +159 -0
- steerable_agent_harness-0.2.0/src/steerable_agent_harness/policy.py +25 -0
- steerable_agent_harness-0.2.0/src/steerable_agent_harness/retry.py +21 -0
- steerable_agent_harness-0.2.0/src/steerable_agent_harness/tracing.py +20 -0
- steerable_agent_harness-0.2.0/src/steerable_agent_harness.egg-info/PKG-INFO +12 -0
- steerable_agent_harness-0.2.0/src/steerable_agent_harness.egg-info/SOURCES.txt +19 -0
- steerable_agent_harness-0.2.0/src/steerable_agent_harness.egg-info/dependency_links.txt +1 -0
- steerable_agent_harness-0.2.0/src/steerable_agent_harness.egg-info/requires.txt +2 -0
- steerable_agent_harness-0.2.0/src/steerable_agent_harness.egg-info/top_level.txt +1 -0
- steerable_agent_harness-0.2.0/tests/test_budget.py +80 -0
- steerable_agent_harness-0.2.0/tests/test_completion.py +44 -0
- steerable_agent_harness-0.2.0/tests/test_policy.py +52 -0
- steerable_agent_harness-0.2.0/tests/test_retry.py +62 -0
- steerable_agent_harness-0.2.0/tests/test_tracing.py +43 -0
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: steerable-agent-harness
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Steerable harness helpers for Python
|
|
5
|
+
Requires-Python: >=3.10
|
|
6
|
+
Description-Content-Type: text/markdown
|
|
7
|
+
Requires-Dist: pydantic>=2.10.0
|
|
8
|
+
Requires-Dist: steerable-agent-protocol<1.0.0,>=0.1.0
|
|
9
|
+
|
|
10
|
+
# steerable-agent-harness
|
|
11
|
+
|
|
12
|
+
Python harness primitives and policy helpers for Steerable.
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "steerable-agent-harness"
|
|
3
|
+
version = "0.2.0"
|
|
4
|
+
description = "Steerable harness helpers for Python"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
requires-python = ">=3.10"
|
|
7
|
+
# Pre-1.0 inter-package pin uses a `>=X,<1.0.0` range so release-please can
|
|
8
|
+
# bump any single Steerable package without breaking installs of the others.
|
|
9
|
+
# Lockstep across protocol↔harness is enforced separately by
|
|
10
|
+
# scripts/check_lockstep_versions.py (CI gate). Tighten to `~=1.0` post-GA.
|
|
11
|
+
dependencies = [
|
|
12
|
+
"pydantic>=2.10.0",
|
|
13
|
+
"steerable-agent-protocol>=0.1.0,<1.0.0",
|
|
14
|
+
]
|
|
15
|
+
|
|
16
|
+
[build-system]
|
|
17
|
+
requires = ["setuptools>=68", "wheel"]
|
|
18
|
+
build-backend = "setuptools.build_meta"
|
|
19
|
+
|
|
20
|
+
[tool.setuptools]
|
|
21
|
+
package-dir = {"" = "src"}
|
|
22
|
+
|
|
23
|
+
[tool.setuptools.packages.find]
|
|
24
|
+
where = ["src"]
|
|
25
|
+
|
|
26
|
+
[tool.uv.sources]
|
|
27
|
+
steerable-agent-protocol = { workspace = true }
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
from .policy import ToolMode, PolicyDecision, decide_tool_mode
|
|
2
|
+
from .budget import BudgetState, BudgetLimit, consume_budget
|
|
3
|
+
from .retry import RetryPolicy, next_retry_delay_ms
|
|
4
|
+
from .completion import is_terminal_result
|
|
5
|
+
from .tracing import TraceSpan
|
|
6
|
+
|
|
7
|
+
__version__ = "0.2.0"
|
|
8
|
+
|
|
9
|
+
__all__ = [
|
|
10
|
+
"__version__",
|
|
11
|
+
"ToolMode",
|
|
12
|
+
"PolicyDecision",
|
|
13
|
+
"decide_tool_mode",
|
|
14
|
+
"BudgetState",
|
|
15
|
+
"BudgetLimit",
|
|
16
|
+
"consume_budget",
|
|
17
|
+
"RetryPolicy",
|
|
18
|
+
"next_retry_delay_ms",
|
|
19
|
+
"is_terminal_result",
|
|
20
|
+
"TraceSpan",
|
|
21
|
+
]
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
@dataclass(slots=True)
|
|
7
|
+
class BudgetLimit:
|
|
8
|
+
max_tokens: int
|
|
9
|
+
max_steps: int
|
|
10
|
+
max_tool_calls: int
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@dataclass(slots=True)
|
|
14
|
+
class BudgetState:
|
|
15
|
+
tokens_used: int = 0
|
|
16
|
+
steps_used: int = 0
|
|
17
|
+
tool_calls_used: int = 0
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def consume_budget(
|
|
21
|
+
state: BudgetState,
|
|
22
|
+
limits: BudgetLimit,
|
|
23
|
+
*,
|
|
24
|
+
tokens: int = 0,
|
|
25
|
+
step: bool = False,
|
|
26
|
+
tool_call: bool = False,
|
|
27
|
+
) -> tuple[BudgetState, bool]:
|
|
28
|
+
next_state = BudgetState(
|
|
29
|
+
tokens_used=state.tokens_used + max(tokens, 0),
|
|
30
|
+
steps_used=state.steps_used + (1 if step else 0),
|
|
31
|
+
tool_calls_used=state.tool_calls_used + (1 if tool_call else 0),
|
|
32
|
+
)
|
|
33
|
+
exhausted = (
|
|
34
|
+
next_state.tokens_used > limits.max_tokens
|
|
35
|
+
or next_state.steps_used > limits.max_steps
|
|
36
|
+
or next_state.tool_calls_used > limits.max_tool_calls
|
|
37
|
+
)
|
|
38
|
+
return next_state, exhausted
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def is_terminal_result(result: dict[str, Any] | None) -> bool:
|
|
7
|
+
if not result:
|
|
8
|
+
return False
|
|
9
|
+
if result.get("terminal") is True:
|
|
10
|
+
return True
|
|
11
|
+
if result.get("success") is False and result.get("needsFollowup") is not True:
|
|
12
|
+
return True
|
|
13
|
+
return False
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
from pydantic import BaseModel
|
|
5
|
+
|
|
6
|
+
class ChatAgent(BaseModel):
|
|
7
|
+
id: str
|
|
8
|
+
slug: str | None = None
|
|
9
|
+
name: str
|
|
10
|
+
icon: str | None = None
|
|
11
|
+
color: str | None = None
|
|
12
|
+
description: str | None = None
|
|
13
|
+
rolePrompt: str | None = None
|
|
14
|
+
forbiddenPrompt: str | None = None
|
|
15
|
+
skillIds: list[Any] | None = None
|
|
16
|
+
allowExternalSkills: bool | None = None
|
|
17
|
+
isBuiltin: bool | None = None
|
|
18
|
+
isArchived: bool | None = None
|
|
19
|
+
sortOrder: int | None = None
|
|
20
|
+
createdAt: str
|
|
21
|
+
updatedAt: str
|
|
22
|
+
|
|
23
|
+
class ChatMessage(BaseModel):
|
|
24
|
+
id: str
|
|
25
|
+
chatId: str | None = None
|
|
26
|
+
role: str
|
|
27
|
+
content: str
|
|
28
|
+
agentId: str | None = None
|
|
29
|
+
toolCalls: list[Any] | None = None
|
|
30
|
+
toolResult: Any | None = None
|
|
31
|
+
createdAt: str
|
|
32
|
+
updatedAt: str | None = None
|
|
33
|
+
|
|
34
|
+
class SSEEvent(BaseModel):
|
|
35
|
+
type: str
|
|
36
|
+
event: str | None = None
|
|
37
|
+
content: str | None = None
|
|
38
|
+
hint: str | None = None
|
|
39
|
+
message: str | None = None
|
|
40
|
+
code: str | None = None
|
|
41
|
+
orchestrationGroupId: str | None = None
|
|
42
|
+
taskId: str | None = None
|
|
43
|
+
messageId: str | None = None
|
|
44
|
+
payload: dict[str, Any] | None = None
|
|
45
|
+
|
|
46
|
+
class AgentSession(BaseModel):
|
|
47
|
+
id: str | None = None
|
|
48
|
+
sessionId: str
|
|
49
|
+
userId: str
|
|
50
|
+
projectId: Any | None = None
|
|
51
|
+
chatId: str
|
|
52
|
+
currentStage: str
|
|
53
|
+
nextStage: Any | None = None
|
|
54
|
+
scenario: str | None = None
|
|
55
|
+
stageData: Any | None = None
|
|
56
|
+
isActive: bool
|
|
57
|
+
createdAt: str
|
|
58
|
+
updatedAt: str
|
|
59
|
+
|
|
60
|
+
class HarnessTrace(BaseModel):
|
|
61
|
+
traceId: str
|
|
62
|
+
userId: Any | None = None
|
|
63
|
+
chatId: Any | None = None
|
|
64
|
+
sessionId: Any | None = None
|
|
65
|
+
assistantMessageId: Any | None = None
|
|
66
|
+
status: str
|
|
67
|
+
durationMs: Any | None = None
|
|
68
|
+
hadError: bool
|
|
69
|
+
errorMessage: Any | None = None
|
|
70
|
+
eventCount: int
|
|
71
|
+
spanCount: int
|
|
72
|
+
totalTokens: Any | None = None
|
|
73
|
+
modelId: Any | None = None
|
|
74
|
+
startedAtMs: Any | None = None
|
|
75
|
+
createdAt: str
|
|
76
|
+
updatedAt: str
|
|
77
|
+
|
|
78
|
+
class TraceEvent(BaseModel):
|
|
79
|
+
id: str | None = None
|
|
80
|
+
traceId: str
|
|
81
|
+
kind: str
|
|
82
|
+
name: str
|
|
83
|
+
sequence: int
|
|
84
|
+
timestampMs: int
|
|
85
|
+
durationMs: Any | None = None
|
|
86
|
+
status: Any | None = None
|
|
87
|
+
payload: Any | None = None
|
|
88
|
+
createdAt: str | None = None
|
|
89
|
+
|
|
90
|
+
class TraceSpan(BaseModel):
|
|
91
|
+
spanId: str
|
|
92
|
+
traceId: Any | None = None
|
|
93
|
+
parentSpanId: Any | None = None
|
|
94
|
+
name: str
|
|
95
|
+
kind: str | None = None
|
|
96
|
+
startMs: int
|
|
97
|
+
endMs: Any | None = None
|
|
98
|
+
durationMs: Any | None = None
|
|
99
|
+
status: str
|
|
100
|
+
attrs: dict[str, Any] | None = None
|
|
101
|
+
|
|
102
|
+
class CommandSafetyPattern(BaseModel):
|
|
103
|
+
id: str
|
|
104
|
+
label: str
|
|
105
|
+
description: str
|
|
106
|
+
pattern: str
|
|
107
|
+
category: str
|
|
108
|
+
severity: str
|
|
109
|
+
platform: str
|
|
110
|
+
|
|
111
|
+
class SidecarError(BaseModel):
|
|
112
|
+
code: int
|
|
113
|
+
message: str
|
|
114
|
+
data: Any | None = None
|
|
115
|
+
kind: str | None = None
|
|
116
|
+
|
|
117
|
+
class SidecarHealth(BaseModel):
|
|
118
|
+
status: str
|
|
119
|
+
version: str
|
|
120
|
+
protocolVersion: str | None = None
|
|
121
|
+
uptimeMs: int
|
|
122
|
+
pid: int | None = None
|
|
123
|
+
pythonVersion: str | None = None
|
|
124
|
+
platform: str | None = None
|
|
125
|
+
loadedProviders: list[Any] | None = None
|
|
126
|
+
loadedTools: int | None = None
|
|
127
|
+
activeTraces: int | None = None
|
|
128
|
+
checks: dict[str, Any] | None = None
|
|
129
|
+
|
|
130
|
+
class SidecarNotification(BaseModel):
|
|
131
|
+
jsonrpc: str
|
|
132
|
+
method: str
|
|
133
|
+
params: Any | None = None
|
|
134
|
+
|
|
135
|
+
class SidecarRequest(BaseModel):
|
|
136
|
+
jsonrpc: str
|
|
137
|
+
id: Any
|
|
138
|
+
method: str
|
|
139
|
+
params: Any | None = None
|
|
140
|
+
|
|
141
|
+
class SidecarResponse(BaseModel):
|
|
142
|
+
jsonrpc: str
|
|
143
|
+
id: Any
|
|
144
|
+
result: Any | None = None
|
|
145
|
+
error: Any | None = None
|
|
146
|
+
|
|
147
|
+
class ToolCall(BaseModel):
|
|
148
|
+
id: str
|
|
149
|
+
name: str
|
|
150
|
+
arguments: dict[str, Any]
|
|
151
|
+
|
|
152
|
+
class ToolResult(BaseModel):
|
|
153
|
+
success: bool
|
|
154
|
+
terminal: bool | None = None
|
|
155
|
+
needsFollowup: bool | None = None
|
|
156
|
+
nextAction: str | None = None
|
|
157
|
+
message: str | None = None
|
|
158
|
+
error: str | None = None
|
|
159
|
+
data: dict[str, Any] | None = None
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from typing import Literal
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
ToolMode = Literal["read", "safe_write", "destructive", "other"]
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@dataclass(slots=True)
|
|
11
|
+
class PolicyDecision:
|
|
12
|
+
allowed: bool
|
|
13
|
+
tool_mode: ToolMode
|
|
14
|
+
reason: str
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def decide_tool_mode(tool_name: str) -> ToolMode:
|
|
18
|
+
normalized = tool_name.lower()
|
|
19
|
+
if normalized.startswith(("get_", "list_", "read_")):
|
|
20
|
+
return "read"
|
|
21
|
+
if normalized.startswith(("create_", "update_", "set_", "write_", "apply_")):
|
|
22
|
+
return "safe_write"
|
|
23
|
+
if normalized.startswith(("delete_", "drop_", "remove_", "destroy_")):
|
|
24
|
+
return "destructive"
|
|
25
|
+
return "other"
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
import random
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
@dataclass(slots=True)
|
|
8
|
+
class RetryPolicy:
|
|
9
|
+
max_attempts: int = 3
|
|
10
|
+
base_delay_ms: int = 200
|
|
11
|
+
max_delay_ms: int = 5000
|
|
12
|
+
jitter: bool = True
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def next_retry_delay_ms(policy: RetryPolicy, attempt: int) -> int:
|
|
16
|
+
if attempt < 1:
|
|
17
|
+
attempt = 1
|
|
18
|
+
delay = min(policy.base_delay_ms * (2 ** (attempt - 1)), policy.max_delay_ms)
|
|
19
|
+
if policy.jitter:
|
|
20
|
+
delay = int(delay * random.uniform(0.8, 1.2))
|
|
21
|
+
return max(delay, 0)
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass, field
|
|
4
|
+
from datetime import datetime, timezone
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@dataclass(slots=True)
|
|
9
|
+
class TraceSpan:
|
|
10
|
+
span_id: str
|
|
11
|
+
name: str
|
|
12
|
+
start_at: str = field(
|
|
13
|
+
default_factory=lambda: datetime.now(timezone.utc).isoformat()
|
|
14
|
+
)
|
|
15
|
+
end_at: str | None = None
|
|
16
|
+
attrs: dict[str, Any] = field(default_factory=dict)
|
|
17
|
+
|
|
18
|
+
def finish(self) -> None:
|
|
19
|
+
if self.end_at is None:
|
|
20
|
+
self.end_at = datetime.now(timezone.utc).isoformat()
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: steerable-agent-harness
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Steerable harness helpers for Python
|
|
5
|
+
Requires-Python: >=3.10
|
|
6
|
+
Description-Content-Type: text/markdown
|
|
7
|
+
Requires-Dist: pydantic>=2.10.0
|
|
8
|
+
Requires-Dist: steerable-agent-protocol<1.0.0,>=0.1.0
|
|
9
|
+
|
|
10
|
+
# steerable-agent-harness
|
|
11
|
+
|
|
12
|
+
Python harness primitives and policy helpers for Steerable.
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
README.md
|
|
2
|
+
pyproject.toml
|
|
3
|
+
src/steerable_agent_harness/__init__.py
|
|
4
|
+
src/steerable_agent_harness/budget.py
|
|
5
|
+
src/steerable_agent_harness/completion.py
|
|
6
|
+
src/steerable_agent_harness/generated.py
|
|
7
|
+
src/steerable_agent_harness/policy.py
|
|
8
|
+
src/steerable_agent_harness/retry.py
|
|
9
|
+
src/steerable_agent_harness/tracing.py
|
|
10
|
+
src/steerable_agent_harness.egg-info/PKG-INFO
|
|
11
|
+
src/steerable_agent_harness.egg-info/SOURCES.txt
|
|
12
|
+
src/steerable_agent_harness.egg-info/dependency_links.txt
|
|
13
|
+
src/steerable_agent_harness.egg-info/requires.txt
|
|
14
|
+
src/steerable_agent_harness.egg-info/top_level.txt
|
|
15
|
+
tests/test_budget.py
|
|
16
|
+
tests/test_completion.py
|
|
17
|
+
tests/test_policy.py
|
|
18
|
+
tests/test_retry.py
|
|
19
|
+
tests/test_tracing.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
steerable_agent_harness
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from steerable_agent_harness import budget
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def _limits(**overrides) -> budget.BudgetLimit:
|
|
7
|
+
base = {"max_tokens": 1000, "max_steps": 10, "max_tool_calls": 5}
|
|
8
|
+
base.update(overrides)
|
|
9
|
+
return budget.BudgetLimit(**base)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def test_budget_consume_returns_new_state() -> None:
|
|
13
|
+
state = budget.BudgetState()
|
|
14
|
+
next_state, exhausted = budget.consume_budget(
|
|
15
|
+
state,
|
|
16
|
+
_limits(),
|
|
17
|
+
tokens=200,
|
|
18
|
+
step=True,
|
|
19
|
+
tool_call=True,
|
|
20
|
+
)
|
|
21
|
+
assert next_state.tokens_used == 200
|
|
22
|
+
assert next_state.steps_used == 1
|
|
23
|
+
assert next_state.tool_calls_used == 1
|
|
24
|
+
assert exhausted is False
|
|
25
|
+
assert state.tokens_used == 0 # original state untouched
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def test_budget_token_overflow_exhausted() -> None:
|
|
29
|
+
state = budget.BudgetState(tokens_used=900)
|
|
30
|
+
_, exhausted = budget.consume_budget(state, _limits(), tokens=200)
|
|
31
|
+
assert exhausted is True
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def test_budget_step_overflow_exhausted() -> None:
|
|
35
|
+
state = budget.BudgetState(steps_used=10)
|
|
36
|
+
_, exhausted = budget.consume_budget(state, _limits(), step=True)
|
|
37
|
+
assert exhausted is True
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def test_budget_tool_call_overflow_exhausted() -> None:
|
|
41
|
+
state = budget.BudgetState(tool_calls_used=5)
|
|
42
|
+
_, exhausted = budget.consume_budget(state, _limits(), tool_call=True)
|
|
43
|
+
assert exhausted is True
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def test_negative_tokens_clamp_to_zero() -> None:
|
|
47
|
+
state = budget.BudgetState(tokens_used=10)
|
|
48
|
+
next_state, exhausted = budget.consume_budget(state, _limits(), tokens=-50)
|
|
49
|
+
assert next_state.tokens_used == 10
|
|
50
|
+
assert exhausted is False
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def test_idle_call_does_not_increment() -> None:
|
|
54
|
+
state = budget.BudgetState(tokens_used=5, steps_used=1, tool_calls_used=1)
|
|
55
|
+
next_state, exhausted = budget.consume_budget(state, _limits())
|
|
56
|
+
assert next_state == state
|
|
57
|
+
assert exhausted is False
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def test_budget_golden(assert_golden) -> None:
|
|
61
|
+
state = budget.BudgetState()
|
|
62
|
+
timeline = []
|
|
63
|
+
for i in range(3):
|
|
64
|
+
state, exhausted = budget.consume_budget(
|
|
65
|
+
state,
|
|
66
|
+
_limits(max_tokens=400),
|
|
67
|
+
tokens=150,
|
|
68
|
+
step=True,
|
|
69
|
+
tool_call=(i % 2 == 0),
|
|
70
|
+
)
|
|
71
|
+
timeline.append(
|
|
72
|
+
{
|
|
73
|
+
"step": i + 1,
|
|
74
|
+
"tokens_used": state.tokens_used,
|
|
75
|
+
"steps_used": state.steps_used,
|
|
76
|
+
"tool_calls_used": state.tool_calls_used,
|
|
77
|
+
"exhausted": exhausted,
|
|
78
|
+
}
|
|
79
|
+
)
|
|
80
|
+
assert_golden("budget_timeline", timeline)
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from steerable_agent_harness import completion
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def test_terminal_when_explicit_flag() -> None:
|
|
7
|
+
assert completion.is_terminal_result({"terminal": True}) is True
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def test_terminal_when_failure_without_followup() -> None:
|
|
11
|
+
assert completion.is_terminal_result({"success": False}) is True
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def test_not_terminal_when_failure_needs_followup() -> None:
|
|
15
|
+
assert (
|
|
16
|
+
completion.is_terminal_result({"success": False, "needsFollowup": True})
|
|
17
|
+
is False
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def test_not_terminal_for_pure_success() -> None:
|
|
22
|
+
assert completion.is_terminal_result({"success": True}) is False
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def test_empty_or_none_not_terminal() -> None:
|
|
26
|
+
assert completion.is_terminal_result(None) is False
|
|
27
|
+
assert completion.is_terminal_result({}) is False
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def test_completion_golden(assert_golden) -> None:
|
|
31
|
+
cases = [
|
|
32
|
+
{"success": True},
|
|
33
|
+
{"success": False},
|
|
34
|
+
{"success": False, "needsFollowup": True},
|
|
35
|
+
{"terminal": True},
|
|
36
|
+
{"success": False, "terminal": True},
|
|
37
|
+
{},
|
|
38
|
+
None,
|
|
39
|
+
]
|
|
40
|
+
payload = [
|
|
41
|
+
{"input": case, "is_terminal": completion.is_terminal_result(case)}
|
|
42
|
+
for case in cases
|
|
43
|
+
]
|
|
44
|
+
assert_golden("completion_decisions", payload)
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
|
|
5
|
+
from steerable_agent_harness import policy
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@pytest.mark.parametrize(
|
|
9
|
+
"tool_name,expected",
|
|
10
|
+
[
|
|
11
|
+
("get_user", "read"),
|
|
12
|
+
("list_files", "read"),
|
|
13
|
+
("read_file", "read"),
|
|
14
|
+
("create_event", "safe_write"),
|
|
15
|
+
("update_chat", "safe_write"),
|
|
16
|
+
("set_config", "safe_write"),
|
|
17
|
+
("write_file", "safe_write"),
|
|
18
|
+
("apply_patch", "safe_write"),
|
|
19
|
+
("delete_event", "destructive"),
|
|
20
|
+
("drop_table", "destructive"),
|
|
21
|
+
("remove_user", "destructive"),
|
|
22
|
+
("destroy_session", "destructive"),
|
|
23
|
+
("compute_score", "other"),
|
|
24
|
+
("orchestrate_steps", "other"),
|
|
25
|
+
("", "other"),
|
|
26
|
+
],
|
|
27
|
+
)
|
|
28
|
+
def test_decide_tool_mode(tool_name: str, expected: str) -> None:
|
|
29
|
+
assert policy.decide_tool_mode(tool_name) == expected
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def test_decide_tool_mode_is_case_insensitive() -> None:
|
|
33
|
+
assert policy.decide_tool_mode("DELETE_USER") == "destructive"
|
|
34
|
+
assert policy.decide_tool_mode("Get_User") == "read"
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def test_policy_decision_dataclass_round_trip() -> None:
|
|
38
|
+
decision = policy.PolicyDecision(allowed=True, tool_mode="read", reason="auto")
|
|
39
|
+
assert decision.allowed is True
|
|
40
|
+
assert decision.tool_mode == "read"
|
|
41
|
+
assert decision.reason == "auto"
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def test_policy_decision_golden(assert_golden) -> None:
|
|
45
|
+
samples = [
|
|
46
|
+
("get_x", policy.decide_tool_mode("get_x")),
|
|
47
|
+
("create_x", policy.decide_tool_mode("create_x")),
|
|
48
|
+
("delete_x", policy.decide_tool_mode("delete_x")),
|
|
49
|
+
("frobnicate", policy.decide_tool_mode("frobnicate")),
|
|
50
|
+
]
|
|
51
|
+
payload = {name: mode for name, mode in samples}
|
|
52
|
+
assert_golden("policy_decisions", payload)
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import random
|
|
4
|
+
|
|
5
|
+
from steerable_agent_harness import retry
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def test_default_policy_attempts() -> None:
|
|
9
|
+
policy = retry.RetryPolicy()
|
|
10
|
+
assert policy.max_attempts == 3
|
|
11
|
+
assert policy.base_delay_ms == 200
|
|
12
|
+
assert policy.max_delay_ms == 5000
|
|
13
|
+
assert policy.jitter is True
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def test_retry_delay_doubling_without_jitter() -> None:
|
|
17
|
+
policy = retry.RetryPolicy(base_delay_ms=100, max_delay_ms=1_000_000, jitter=False)
|
|
18
|
+
delays = [retry.next_retry_delay_ms(policy, attempt) for attempt in range(1, 6)]
|
|
19
|
+
assert delays == [100, 200, 400, 800, 1600]
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def test_retry_delay_caps_at_max() -> None:
|
|
23
|
+
policy = retry.RetryPolicy(base_delay_ms=1000, max_delay_ms=2500, jitter=False)
|
|
24
|
+
delays = [retry.next_retry_delay_ms(policy, attempt) for attempt in range(1, 6)]
|
|
25
|
+
assert delays == [1000, 2000, 2500, 2500, 2500]
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def test_retry_delay_jitter_within_bounds() -> None:
|
|
29
|
+
policy = retry.RetryPolicy(base_delay_ms=200, max_delay_ms=5000, jitter=True)
|
|
30
|
+
rng = random.Random(42)
|
|
31
|
+
samples: list[int] = []
|
|
32
|
+
state = random.getstate()
|
|
33
|
+
try:
|
|
34
|
+
random.seed(42)
|
|
35
|
+
for attempt in range(1, 6):
|
|
36
|
+
samples.append(retry.next_retry_delay_ms(policy, attempt))
|
|
37
|
+
finally:
|
|
38
|
+
random.setstate(state)
|
|
39
|
+
for attempt, value in zip(range(1, 6), samples, strict=True):
|
|
40
|
+
base = min(200 * (2 ** (attempt - 1)), 5000)
|
|
41
|
+
assert int(base * 0.8) <= value <= int(base * 1.2) + 1
|
|
42
|
+
assert rng # silence unused warning if helper kept
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def test_attempt_clamped_to_one() -> None:
|
|
46
|
+
policy = retry.RetryPolicy(base_delay_ms=100, max_delay_ms=10_000, jitter=False)
|
|
47
|
+
assert retry.next_retry_delay_ms(policy, 0) == 100
|
|
48
|
+
assert retry.next_retry_delay_ms(policy, -1) == 100
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def test_retry_golden_no_jitter(assert_golden) -> None:
|
|
52
|
+
policy = retry.RetryPolicy(base_delay_ms=100, max_delay_ms=2_000, jitter=False)
|
|
53
|
+
payload = {
|
|
54
|
+
"policy": {
|
|
55
|
+
"max_attempts": policy.max_attempts,
|
|
56
|
+
"base_delay_ms": policy.base_delay_ms,
|
|
57
|
+
"max_delay_ms": policy.max_delay_ms,
|
|
58
|
+
"jitter": policy.jitter,
|
|
59
|
+
},
|
|
60
|
+
"delays": [retry.next_retry_delay_ms(policy, a) for a in range(1, 6)],
|
|
61
|
+
}
|
|
62
|
+
assert_golden("retry_no_jitter", payload)
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from steerable_agent_harness import tracing
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def test_trace_span_starts_open() -> None:
|
|
7
|
+
span = tracing.TraceSpan(span_id="step_1", name="llm.generate")
|
|
8
|
+
assert span.start_at # ISO timestamp present
|
|
9
|
+
assert span.end_at is None
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def test_trace_span_finish_sets_end_at_once() -> None:
|
|
13
|
+
span = tracing.TraceSpan(span_id="step_2", name="tool.exec")
|
|
14
|
+
span.finish()
|
|
15
|
+
assert span.end_at is not None
|
|
16
|
+
first_end = span.end_at
|
|
17
|
+
span.finish()
|
|
18
|
+
assert span.end_at == first_end # no double-overwrite
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def test_trace_span_attrs_default_to_empty_dict() -> None:
|
|
22
|
+
span = tracing.TraceSpan(span_id="step_3", name="planning")
|
|
23
|
+
assert span.attrs == {}
|
|
24
|
+
span.attrs["model"] = "gpt-4o"
|
|
25
|
+
assert span.attrs == {"model": "gpt-4o"}
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def test_trace_span_golden(assert_golden) -> None:
|
|
29
|
+
span = tracing.TraceSpan(
|
|
30
|
+
span_id="step_1",
|
|
31
|
+
name="llm.generate",
|
|
32
|
+
start_at="2026-01-01T00:00:00+00:00",
|
|
33
|
+
attrs={"model": "gpt-4o"},
|
|
34
|
+
)
|
|
35
|
+
span.end_at = "2026-01-01T00:00:01+00:00"
|
|
36
|
+
payload = {
|
|
37
|
+
"span_id": span.span_id,
|
|
38
|
+
"name": span.name,
|
|
39
|
+
"start_at": span.start_at,
|
|
40
|
+
"end_at": span.end_at,
|
|
41
|
+
"attrs": span.attrs,
|
|
42
|
+
}
|
|
43
|
+
assert_golden("trace_span_basic", payload)
|