openreflex 0.9.0__tar.gz → 0.9.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {openreflex-0.9.0/src/openreflex.egg-info → openreflex-0.9.2}/PKG-INFO +15 -6
- {openreflex-0.9.0 → openreflex-0.9.2}/README.md +14 -5
- {openreflex-0.9.0 → openreflex-0.9.2}/pyproject.toml +1 -1
- {openreflex-0.9.0 → openreflex-0.9.2}/src/openreflex/__init__.py +1 -1
- {openreflex-0.9.0 → openreflex-0.9.2}/src/openreflex/decision.py +7 -33
- {openreflex-0.9.0 → openreflex-0.9.2}/src/openreflex/engine.py +3 -2
- {openreflex-0.9.0 → openreflex-0.9.2}/src/openreflex/entrypoint.py +178 -0
- {openreflex-0.9.0 → openreflex-0.9.2}/src/openreflex/hooks.py +3 -15
- {openreflex-0.9.0 → openreflex-0.9.2/src/openreflex.egg-info}/PKG-INFO +15 -6
- {openreflex-0.9.0 → openreflex-0.9.2}/src/openreflex.egg-info/SOURCES.txt +1 -0
- {openreflex-0.9.0 → openreflex-0.9.2}/tests/test_completion_recap_semantics.py +21 -22
- {openreflex-0.9.0 → openreflex-0.9.2}/tests/test_install_cli.py +61 -2
- openreflex-0.9.2/tests/test_live_eval.py +83 -0
- {openreflex-0.9.0 → openreflex-0.9.2}/tests/test_outcome_closure.py +14 -13
- {openreflex-0.9.0 → openreflex-0.9.2}/tests/test_policy_recap.py +45 -8
- {openreflex-0.9.0 → openreflex-0.9.2}/tests/test_quality_hardening.py +13 -7
- {openreflex-0.9.0 → openreflex-0.9.2}/tests/test_work_modes_tokens.py +8 -6
- {openreflex-0.9.0 → openreflex-0.9.2}/LICENSE +0 -0
- {openreflex-0.9.0 → openreflex-0.9.2}/setup.cfg +0 -0
- {openreflex-0.9.0 → openreflex-0.9.2}/src/openreflex/__main__.py +0 -0
- {openreflex-0.9.0 → openreflex-0.9.2}/src/openreflex/benchmark.py +0 -0
- {openreflex-0.9.0 → openreflex-0.9.2}/src/openreflex/claude_ui.py +0 -0
- {openreflex-0.9.0 → openreflex-0.9.2}/src/openreflex/cli.py +0 -0
- {openreflex-0.9.0 → openreflex-0.9.2}/src/openreflex/control.py +0 -0
- {openreflex-0.9.0 → openreflex-0.9.2}/src/openreflex/detect.py +0 -0
- {openreflex-0.9.0 → openreflex-0.9.2}/src/openreflex/execution_state.py +0 -0
- {openreflex-0.9.0 → openreflex-0.9.2}/src/openreflex/install.py +0 -0
- {openreflex-0.9.0 → openreflex-0.9.2}/src/openreflex/integrations/opencode.ts +0 -0
- {openreflex-0.9.0 → openreflex-0.9.2}/src/openreflex/learning.py +0 -0
- {openreflex-0.9.0 → openreflex-0.9.2}/src/openreflex/mcp_contracts.py +0 -0
- {openreflex-0.9.0 → openreflex-0.9.2}/src/openreflex/mcp_server.py +0 -0
- {openreflex-0.9.0 → openreflex-0.9.2}/src/openreflex/metrics.py +0 -0
- {openreflex-0.9.0 → openreflex-0.9.2}/src/openreflex/models.py +0 -0
- {openreflex-0.9.0 → openreflex-0.9.2}/src/openreflex/policies/default.toml +0 -0
- {openreflex-0.9.0 → openreflex-0.9.2}/src/openreflex/policy.py +0 -0
- {openreflex-0.9.0 → openreflex-0.9.2}/src/openreflex/privacy.py +0 -0
- {openreflex-0.9.0 → openreflex-0.9.2}/src/openreflex/project.py +0 -0
- {openreflex-0.9.0 → openreflex-0.9.2}/src/openreflex/project_map.py +0 -0
- {openreflex-0.9.0 → openreflex-0.9.2}/src/openreflex/project_memory.py +0 -0
- {openreflex-0.9.0 → openreflex-0.9.2}/src/openreflex/reflex_index.py +0 -0
- {openreflex-0.9.0 → openreflex-0.9.2}/src/openreflex/reflexes.py +0 -0
- {openreflex-0.9.0 → openreflex-0.9.2}/src/openreflex/routing.py +0 -0
- {openreflex-0.9.0 → openreflex-0.9.2}/src/openreflex/store.py +0 -0
- {openreflex-0.9.0 → openreflex-0.9.2}/src/openreflex/tui.py +0 -0
- {openreflex-0.9.0 → openreflex-0.9.2}/src/openreflex/updater.py +0 -0
- {openreflex-0.9.0 → openreflex-0.9.2}/src/openreflex/usage.py +0 -0
- {openreflex-0.9.0 → openreflex-0.9.2}/src/openreflex.egg-info/dependency_links.txt +0 -0
- {openreflex-0.9.0 → openreflex-0.9.2}/src/openreflex.egg-info/entry_points.txt +0 -0
- {openreflex-0.9.0 → openreflex-0.9.2}/src/openreflex.egg-info/requires.txt +0 -0
- {openreflex-0.9.0 → openreflex-0.9.2}/src/openreflex.egg-info/top_level.txt +0 -0
- {openreflex-0.9.0 → openreflex-0.9.2}/tests/test_engine.py +0 -0
- {openreflex-0.9.0 → openreflex-0.9.2}/tests/test_execution_state.py +0 -0
- {openreflex-0.9.0 → openreflex-0.9.2}/tests/test_hooks.py +0 -0
- {openreflex-0.9.0 → openreflex-0.9.2}/tests/test_mcp_metrics_benchmark.py +0 -0
- {openreflex-0.9.0 → openreflex-0.9.2}/tests/test_primer_background.py +0 -0
- {openreflex-0.9.0 → openreflex-0.9.2}/tests/test_project_map.py +0 -0
- {openreflex-0.9.0 → openreflex-0.9.2}/tests/test_project_memory.py +0 -0
- {openreflex-0.9.0 → openreflex-0.9.2}/tests/test_project_reflexes.py +0 -0
- {openreflex-0.9.0 → openreflex-0.9.2}/tests/test_project_scope.py +0 -0
- {openreflex-0.9.0 → openreflex-0.9.2}/tests/test_reflex_index.py +0 -0
- {openreflex-0.9.0 → openreflex-0.9.2}/tests/test_release_consistency.py +0 -0
- {openreflex-0.9.0 → openreflex-0.9.2}/tests/test_release_pipeline.py +0 -0
- {openreflex-0.9.0 → openreflex-0.9.2}/tests/test_reliability_lifecycle.py +0 -0
- {openreflex-0.9.0 → openreflex-0.9.2}/tests/test_routing_control.py +0 -0
- {openreflex-0.9.0 → openreflex-0.9.2}/tests/test_stress.py +0 -0
- {openreflex-0.9.0 → openreflex-0.9.2}/tests/test_token_receiver.py +0 -0
- {openreflex-0.9.0 → openreflex-0.9.2}/tests/test_tokens_cli.py +0 -0
- {openreflex-0.9.0 → openreflex-0.9.2}/tests/test_tui_memory.py +0 -0
- {openreflex-0.9.0 → openreflex-0.9.2}/tests/test_update_cli.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: openreflex
|
|
3
|
-
Version: 0.9.
|
|
3
|
+
Version: 0.9.2
|
|
4
4
|
Summary: Ambient project muscle memory for AI coding agents: learns and reuses project Reflexes automatically
|
|
5
5
|
Author: vishnu-77
|
|
6
6
|
License-Expression: MIT
|
|
@@ -97,19 +97,21 @@ OpenTelemetry export to a loopback-only OpenReflex receiver and stores counts on
|
|
|
97
97
|
|
|
98
98
|
Requires Python 3.11 or newer.
|
|
99
99
|
|
|
100
|
-
**Claude Code — recommended:** install the plugin, reload it,
|
|
100
|
+
**Claude Code — recommended:** install the plugin, reload it, inspect the current state, then explicitly enable
|
|
101
|
+
project memory.
|
|
101
102
|
|
|
102
103
|
```text
|
|
103
104
|
/plugin marketplace add vishnu-77/openreflex
|
|
104
105
|
/plugin install openreflex@openreflex
|
|
105
106
|
/reload-plugins
|
|
106
107
|
/openreflex
|
|
108
|
+
/openreflex approve
|
|
107
109
|
```
|
|
108
110
|
|
|
109
111
|
Claude may display the canonical plugin namespace as `/openreflex:openreflex`; both refer to the same user-invoked
|
|
110
|
-
OpenReflex control surface when the bare alias is available.
|
|
111
|
-
for the current project
|
|
112
|
-
|
|
112
|
+
OpenReflex control surface when the bare alias is available. The no-argument command is read-only and compact.
|
|
113
|
+
`approve` explicitly enables local memory for the current project. The plugin prepares an exact-version runtime
|
|
114
|
+
under `~/.openreflex/runtime/` and normal agent work remains ambient after approval.
|
|
113
115
|
|
|
114
116
|
The Claude plugin does **not** execute a global `openreflex` from PATH. An older pipx/uv/pip installation can coexist
|
|
115
117
|
without taking over the plugin runtime.
|
|
@@ -146,7 +148,12 @@ OpenReflex installs per project with `openreflex install <agent>`, or as a plugi
|
|
|
146
148
|
| **Cursor** | Lifecycle hooks | `openreflex install cursor` | Protocol and fuzz tests |
|
|
147
149
|
| **OpenCode** | Local lifecycle plugin | `openreflex install opencode` | Protocol and fuzz tests |
|
|
148
150
|
|
|
149
|
-
With the Claude plugin,
|
|
151
|
+
With the Claude plugin, `/openreflex` shows a compact read-only state view. Use `/openreflex approve` to enable
|
|
152
|
+
local memory for the current project, then work normally. Other explicit controls include `status`, `reflexes`,
|
|
153
|
+
`memory`, `doctor`, `why`, `trace`, and `revoke`. OpenReflex control turns are not learned as project
|
|
154
|
+
experiences and do not trigger normal task verification. The control skill is user-invoked only and does not enable
|
|
155
|
+
MCP diagnostics or participate in normal model routing. Claude may show the canonical namespaced form
|
|
156
|
+
`/openreflex:openreflex`. Per-agent guides:
|
|
150
157
|
[Claude Code](https://openreflex.cc/claude-code), [Codex](https://openreflex.cc/codex), [Cursor](https://openreflex.cc/cursor),
|
|
151
158
|
[OpenCode](https://openreflex.cc/opencode).
|
|
152
159
|
|
|
@@ -259,6 +266,8 @@ retrieval, evidence-aware path selection, budget-aware execution, work-mode-spec
|
|
|
259
266
|
path comparison. The [Researcher view](https://openreflex.cc/?view=research) on the site explains the idea,
|
|
260
267
|
lets you step through one reflex forming in the graph, and places it next to related work.
|
|
261
268
|
|
|
269
|
+
The reproducible evaluation protocol, pinned real-agent corpus, exact prompts, and claim boundaries are documented in [docs/evaluation.md](docs/evaluation.md).
|
|
270
|
+
|
|
262
271
|
## Community & Contributing
|
|
263
272
|
|
|
264
273
|
- **Issues and ideas:** [GitHub Issues](https://github.com/vishnu-77/openreflex/issues)
|
|
@@ -65,19 +65,21 @@ OpenTelemetry export to a loopback-only OpenReflex receiver and stores counts on
|
|
|
65
65
|
|
|
66
66
|
Requires Python 3.11 or newer.
|
|
67
67
|
|
|
68
|
-
**Claude Code — recommended:** install the plugin, reload it,
|
|
68
|
+
**Claude Code — recommended:** install the plugin, reload it, inspect the current state, then explicitly enable
|
|
69
|
+
project memory.
|
|
69
70
|
|
|
70
71
|
```text
|
|
71
72
|
/plugin marketplace add vishnu-77/openreflex
|
|
72
73
|
/plugin install openreflex@openreflex
|
|
73
74
|
/reload-plugins
|
|
74
75
|
/openreflex
|
|
76
|
+
/openreflex approve
|
|
75
77
|
```
|
|
76
78
|
|
|
77
79
|
Claude may display the canonical plugin namespace as `/openreflex:openreflex`; both refer to the same user-invoked
|
|
78
|
-
OpenReflex control surface when the bare alias is available.
|
|
79
|
-
for the current project
|
|
80
|
-
|
|
80
|
+
OpenReflex control surface when the bare alias is available. The no-argument command is read-only and compact.
|
|
81
|
+
`approve` explicitly enables local memory for the current project. The plugin prepares an exact-version runtime
|
|
82
|
+
under `~/.openreflex/runtime/` and normal agent work remains ambient after approval.
|
|
81
83
|
|
|
82
84
|
The Claude plugin does **not** execute a global `openreflex` from PATH. An older pipx/uv/pip installation can coexist
|
|
83
85
|
without taking over the plugin runtime.
|
|
@@ -114,7 +116,12 @@ OpenReflex installs per project with `openreflex install <agent>`, or as a plugi
|
|
|
114
116
|
| **Cursor** | Lifecycle hooks | `openreflex install cursor` | Protocol and fuzz tests |
|
|
115
117
|
| **OpenCode** | Local lifecycle plugin | `openreflex install opencode` | Protocol and fuzz tests |
|
|
116
118
|
|
|
117
|
-
With the Claude plugin,
|
|
119
|
+
With the Claude plugin, `/openreflex` shows a compact read-only state view. Use `/openreflex approve` to enable
|
|
120
|
+
local memory for the current project, then work normally. Other explicit controls include `status`, `reflexes`,
|
|
121
|
+
`memory`, `doctor`, `why`, `trace`, and `revoke`. OpenReflex control turns are not learned as project
|
|
122
|
+
experiences and do not trigger normal task verification. The control skill is user-invoked only and does not enable
|
|
123
|
+
MCP diagnostics or participate in normal model routing. Claude may show the canonical namespaced form
|
|
124
|
+
`/openreflex:openreflex`. Per-agent guides:
|
|
118
125
|
[Claude Code](https://openreflex.cc/claude-code), [Codex](https://openreflex.cc/codex), [Cursor](https://openreflex.cc/cursor),
|
|
119
126
|
[OpenCode](https://openreflex.cc/opencode).
|
|
120
127
|
|
|
@@ -227,6 +234,8 @@ retrieval, evidence-aware path selection, budget-aware execution, work-mode-spec
|
|
|
227
234
|
path comparison. The [Researcher view](https://openreflex.cc/?view=research) on the site explains the idea,
|
|
228
235
|
lets you step through one reflex forming in the graph, and places it next to related work.
|
|
229
236
|
|
|
237
|
+
The reproducible evaluation protocol, pinned real-agent corpus, exact prompts, and claim boundaries are documented in [docs/evaluation.md](docs/evaluation.md).
|
|
238
|
+
|
|
230
239
|
## Community & Contributing
|
|
231
240
|
|
|
232
241
|
- **Issues and ideas:** [GitHub Issues](https://github.com/vishnu-77/openreflex/issues)
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "openreflex"
|
|
7
|
-
version = "0.9.
|
|
7
|
+
version = "0.9.2"
|
|
8
8
|
description = "Ambient project muscle memory for AI coding agents: learns and reuses project Reflexes automatically"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = "MIT"
|
|
@@ -147,41 +147,15 @@ def make_snapshot(*, now: float, phase: str, action: str, best: CandidatePath |
|
|
|
147
147
|
|
|
148
148
|
def render_recap(snapshot: DecisionSnapshot, previous: DecisionSnapshot | None = None) -> str:
|
|
149
149
|
if snapshot.phase == "complete":
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
summary.append(snapshot.reflex_name)
|
|
155
|
-
|
|
156
|
-
cost = []
|
|
157
|
-
if snapshot.actual_tool_calls is not None:
|
|
158
|
-
cost.append(f"{snapshot.actual_tool_calls} calls")
|
|
159
|
-
if snapshot.model_tokens is not None and snapshot.model_tokens > 0:
|
|
160
|
-
cost.append(f"{snapshot.model_tokens / 1000:.1f}k model tokens")
|
|
161
|
-
elif snapshot.actual_tokens is not None and snapshot.actual_tokens > 0:
|
|
162
|
-
cost.append(f"{snapshot.actual_tokens / 1000:.1f}k tool-output est.")
|
|
163
|
-
if snapshot.elapsed_seconds is not None:
|
|
164
|
-
cost.append(f"{snapshot.elapsed_seconds / 60:.1f}m")
|
|
150
|
+
# Missing completion evidence is an internal epistemic state, not a user-facing failure.
|
|
151
|
+
# Keep it available in status/trace, but stay silent in the ambient Stop surface.
|
|
152
|
+
if snapshot.outcome == "unknown":
|
|
153
|
+
return ""
|
|
165
154
|
|
|
166
|
-
|
|
167
|
-
if snapshot.outcome != "unknown":
|
|
168
|
-
if snapshot.expected_regret is not None and snapshot.next_best_strategy:
|
|
169
|
-
comparison = "Path check · another approach may be better"
|
|
170
|
-
else:
|
|
171
|
-
comparison = "Path check · no better option proven"
|
|
172
|
-
|
|
173
|
-
state = {
|
|
174
|
-
"success": "COMPLETE",
|
|
175
|
-
"failure": "FAILED",
|
|
176
|
-
"unknown": "UNVERIFIED",
|
|
177
|
-
}.get(snapshot.outcome, "FINISHED")
|
|
155
|
+
state = "COMPLETE" if snapshot.outcome == "success" else "FAILED"
|
|
178
156
|
lines = [f"↺ OpenReflex · {state}"]
|
|
179
|
-
if
|
|
180
|
-
lines.append(
|
|
181
|
-
if cost:
|
|
182
|
-
lines.append(" · ".join(cost))
|
|
183
|
-
if comparison:
|
|
184
|
-
lines.append(comparison)
|
|
157
|
+
if snapshot.reflex_name:
|
|
158
|
+
lines.append(snapshot.reflex_name)
|
|
185
159
|
return "\n".join(lines)
|
|
186
160
|
|
|
187
161
|
delta = ""
|
|
@@ -508,8 +508,9 @@ class Engine:
|
|
|
508
508
|
actual_tokens=actual_tokens, elapsed_seconds=elapsed_seconds, outcome=outcome,
|
|
509
509
|
expected_regret=expected_regret, model_tokens=model_tokens, token_usage=token_usage,
|
|
510
510
|
reflex_name=reflex_name)
|
|
511
|
-
if not task.substantial:
|
|
512
|
-
#
|
|
511
|
+
if not task.substantial or (phase == "complete" and outcome == "unknown"):
|
|
512
|
+
# Trivial activity and completion without observable evidence remain available in
|
|
513
|
+
# trace/status, but must not become alarming ambient user-facing recaps.
|
|
513
514
|
snapshot = DecisionSnapshot.from_dict({**snapshot.as_dict(), "visibility": "ambient"})
|
|
514
515
|
execution.decision_history.append(snapshot.as_dict())
|
|
515
516
|
self.store.put(execution)
|
|
@@ -49,6 +49,67 @@ ACTIVITY_LABELS = {
|
|
|
49
49
|
"other": "TOOL",
|
|
50
50
|
}
|
|
51
51
|
|
|
52
|
+
CONTROL_SESSION_TTL_SECONDS = 600
|
|
53
|
+
CONTROL_PROMPT = re.compile(r"^\s*/openreflex(?::openreflex)?(?:\s|$)", re.I)
|
|
54
|
+
CONTROL_NATURAL = re.compile(
|
|
55
|
+
r"^\s*(?:please\s+)?(?:enable|disable)\s+openreflex(?:\s+for)?\s+(?:this|the|current)\s+project[.!]?\s*$"
|
|
56
|
+
r"|^\s*(?:please\s+)?(?:run\s+)?openreflex\s+(?:approve|revoke|status|doctor|why|trace|reflexes|memory)[.!]?\s*$",
|
|
57
|
+
re.I,
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _is_openreflex_control_prompt(prompt: str) -> bool:
|
|
62
|
+
text = prompt.strip()
|
|
63
|
+
return bool(
|
|
64
|
+
text
|
|
65
|
+
and (
|
|
66
|
+
CONTROL_PROMPT.search(text)
|
|
67
|
+
or CONTROL_NATURAL.fullmatch(text)
|
|
68
|
+
or "OPENREFLEX_CONTROL_REQUEST:" in text
|
|
69
|
+
)
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _is_openreflex_control_tool(payload: dict) -> bool:
|
|
74
|
+
if str(payload.get("tool_name") or "").lower() not in {"bash", "shell", "run"}:
|
|
75
|
+
return False
|
|
76
|
+
arguments = payload.get("tool_input")
|
|
77
|
+
if not isinstance(arguments, dict):
|
|
78
|
+
return False
|
|
79
|
+
command = str(arguments.get("command") or "")
|
|
80
|
+
if not command:
|
|
81
|
+
return False
|
|
82
|
+
return bool(
|
|
83
|
+
re.search(r"runtime[/\\]launcher\.cjs[^\n]*\bcontrol\b", command, re.I)
|
|
84
|
+
or re.search(r"(^|[;&|]\s*)openreflex\s+(?:approve|revoke|status|doctor|why|trace|reflexes|memory)\b", command, re.I)
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _control_sessions(project: Path) -> dict[str, float]:
|
|
89
|
+
now = time.time()
|
|
90
|
+
raw = read_state(project).get("control_sessions")
|
|
91
|
+
sessions = raw if isinstance(raw, dict) else {}
|
|
92
|
+
return {
|
|
93
|
+
str(key): float(value)
|
|
94
|
+
for key, value in sessions.items()
|
|
95
|
+
if isinstance(value, (int, float)) and now - float(value) < CONTROL_SESSION_TTL_SECONDS
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _set_control_session(project: Path, session: str, active: bool) -> None:
|
|
100
|
+
if not session:
|
|
101
|
+
return
|
|
102
|
+
sessions = _control_sessions(project)
|
|
103
|
+
if active:
|
|
104
|
+
sessions[session] = round(time.time(), 3)
|
|
105
|
+
else:
|
|
106
|
+
sessions.pop(session, None)
|
|
107
|
+
update_state(project, control_sessions=sessions)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _control_session_active(project: Path, session: str) -> bool:
|
|
111
|
+
return bool(session and session in _control_sessions(project))
|
|
112
|
+
|
|
52
113
|
|
|
53
114
|
def _best_effort(label: str, call: Callable[[], T], default: T) -> T:
|
|
54
115
|
try:
|
|
@@ -298,6 +359,23 @@ def _hook(argv: list[str]) -> int:
|
|
|
298
359
|
cwd = payload.get("cwd") if isinstance(payload.get("cwd"), str) else None
|
|
299
360
|
session = str(payload.get("session_id") or payload.get("conversation_id") or "")
|
|
300
361
|
project = project_root(cwd, hook_session=session)
|
|
362
|
+
|
|
363
|
+
# OpenReflex's own control surface is operational metadata, not project work.
|
|
364
|
+
# Never learn it as an experience, count its shell call, or ask it for code verification.
|
|
365
|
+
if event in PROMPT_EVENTS:
|
|
366
|
+
prompt = payload.get("prompt") if isinstance(payload.get("prompt"), str) else ""
|
|
367
|
+
if _is_openreflex_control_prompt(prompt):
|
|
368
|
+
_best_effort("mark control turn", lambda: _set_control_session(project, session, True), None)
|
|
369
|
+
return 0
|
|
370
|
+
_best_effort("clear control turn", lambda: _set_control_session(project, session, False), None)
|
|
371
|
+
elif _control_session_active(project, session):
|
|
372
|
+
if event in STOP_EVENTS:
|
|
373
|
+
_best_effort("clear control turn", lambda: _set_control_session(project, session, False), None)
|
|
374
|
+
return 0
|
|
375
|
+
elif event in TOOL_START_EVENTS | TOOL_END_EVENTS and _is_openreflex_control_tool(payload):
|
|
376
|
+
_best_effort("mark control turn", lambda: _set_control_session(project, session, True), None)
|
|
377
|
+
return 0
|
|
378
|
+
|
|
301
379
|
enabled = approval(project) is not None
|
|
302
380
|
phase = None
|
|
303
381
|
|
|
@@ -365,6 +443,104 @@ def _tui(argv: list[str]) -> int:
|
|
|
365
443
|
return 0
|
|
366
444
|
|
|
367
445
|
|
|
446
|
+
def _control_request(argv: list[str]) -> str:
|
|
447
|
+
for index, item in enumerate(argv):
|
|
448
|
+
if item == "--request" and index + 1 < len(argv):
|
|
449
|
+
return argv[index + 1].strip()
|
|
450
|
+
if item.startswith("--request="):
|
|
451
|
+
return item.partition("=")[2].strip()
|
|
452
|
+
return ""
|
|
453
|
+
|
|
454
|
+
|
|
455
|
+
def _compact_overview(project: Path, plugin_root: str | None = None) -> int:
|
|
456
|
+
enabled = approval(project) is not None
|
|
457
|
+
if enabled:
|
|
458
|
+
_best_effort("control counts", lambda: sync_counts(project), {})
|
|
459
|
+
state = read_state(project)
|
|
460
|
+
plugin_version = (
|
|
461
|
+
_plugin_manifest_version(plugin_root)
|
|
462
|
+
or str(state.get("plugin_version") or "").strip()
|
|
463
|
+
or os.environ.get("OPENREFLEX_PLUGIN_VERSION")
|
|
464
|
+
or __version__
|
|
465
|
+
)
|
|
466
|
+
runtime_version = str(state.get("hook_runtime_version") or __version__)
|
|
467
|
+
aligned = plugin_version == runtime_version == __version__
|
|
468
|
+
task = state.get("task") if isinstance(state.get("task"), dict) else {}
|
|
469
|
+
reflex_name = str(task.get("reflex") or "").strip()
|
|
470
|
+
|
|
471
|
+
reflex_count = 0
|
|
472
|
+
if enabled:
|
|
473
|
+
try:
|
|
474
|
+
from .reflexes import visible_reflexes
|
|
475
|
+
from .store import Store, database_path
|
|
476
|
+
store = Store(database_path(project))
|
|
477
|
+
try:
|
|
478
|
+
reflex_count = len(visible_reflexes(store))
|
|
479
|
+
finally:
|
|
480
|
+
store.close()
|
|
481
|
+
except Exception: # noqa: BLE001 - overview must remain available if optional detail fails
|
|
482
|
+
reflex_count = 0
|
|
483
|
+
|
|
484
|
+
state_label = "READY" if enabled and aligned else "SETUP" if not enabled else "ATTENTION"
|
|
485
|
+
print(f"OPENREFLEX {__version__} {state_label}")
|
|
486
|
+
print("")
|
|
487
|
+
print(f"Project {project.name}")
|
|
488
|
+
if enabled:
|
|
489
|
+
experiences = int(state.get("experiences", 0) or 0)
|
|
490
|
+
print(f"Memory {experiences} experiences · {reflex_count} Reflexes")
|
|
491
|
+
print(f"Current {reflex_name or ('learning' if experiences else 'learning project')}")
|
|
492
|
+
else:
|
|
493
|
+
print("Memory not enabled")
|
|
494
|
+
print(f"Runtime {'aligned' if aligned else f'plugin {plugin_version} / runtime {runtime_version}'}")
|
|
495
|
+
if not enabled:
|
|
496
|
+
print("")
|
|
497
|
+
print("Run /openreflex:openreflex approve to start learning.")
|
|
498
|
+
return 0
|
|
499
|
+
|
|
500
|
+
|
|
501
|
+
def _control(argv: list[str]) -> int:
|
|
502
|
+
project = project_root(_project_arg(argv))
|
|
503
|
+
plugin_root = _plugin_root_arg(argv)
|
|
504
|
+
request = _control_request(argv).strip().lower()
|
|
505
|
+
|
|
506
|
+
if request in {"", "show", "dashboard"}:
|
|
507
|
+
return _compact_overview(project, plugin_root)
|
|
508
|
+
|
|
509
|
+
if request == "approve":
|
|
510
|
+
approve(project, source="claude-plugin:control")
|
|
511
|
+
_record_hook_runtime(project, ["control", "--plugin-root", plugin_root or ""])
|
|
512
|
+
configure_statusline(project)
|
|
513
|
+
_best_effort("control primer", lambda: ensure_background_refresh(project), "cold")
|
|
514
|
+
_best_effort("control counts", lambda: sync_counts(project), {})
|
|
515
|
+
update_state(project, onboarding="ready", project_enabled=True)
|
|
516
|
+
print("OPENREFLEX READY")
|
|
517
|
+
print("")
|
|
518
|
+
print(f"Project {project.name}")
|
|
519
|
+
print("Memory enabled · local")
|
|
520
|
+
print("Runtime aligned")
|
|
521
|
+
return 0
|
|
522
|
+
|
|
523
|
+
if request == "revoke":
|
|
524
|
+
return cli.main(["revoke", "--project", str(project)])
|
|
525
|
+
if request == "status":
|
|
526
|
+
return cli.main(["status", "--project", str(project)])
|
|
527
|
+
if request == "doctor":
|
|
528
|
+
return cli.main(["doctor", "--project", str(project)])
|
|
529
|
+
if request == "why":
|
|
530
|
+
return cli.main(["why", "--project", str(project)])
|
|
531
|
+
if request == "trace":
|
|
532
|
+
return cli.main(["trace", "--project", str(project)])
|
|
533
|
+
if request == "reflexes":
|
|
534
|
+
return _reflexes(["--project", str(project)])
|
|
535
|
+
if request == "memory":
|
|
536
|
+
return _memory(["--project", str(project)])
|
|
537
|
+
|
|
538
|
+
print("OPENREFLEX / COMMAND")
|
|
539
|
+
print(f" unknown {request or '-'}")
|
|
540
|
+
print(" available approve · status · reflexes · memory · doctor · why · trace · revoke")
|
|
541
|
+
return 2
|
|
542
|
+
|
|
543
|
+
|
|
368
544
|
def _onboard(argv: list[str]) -> int:
|
|
369
545
|
"""Explicit Claude-plugin onboarding: enable one project, pin UI, then show the dashboard."""
|
|
370
546
|
project = project_root(_project_arg(argv))
|
|
@@ -542,6 +718,8 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
542
718
|
return 0
|
|
543
719
|
if argv[0] == "statusline":
|
|
544
720
|
return _statusline()
|
|
721
|
+
if argv[0] == "control":
|
|
722
|
+
return _control(argv[1:])
|
|
545
723
|
if argv[0] == "onboard":
|
|
546
724
|
return _onboard(argv[1:])
|
|
547
725
|
if argv[0] == "tui":
|
|
@@ -220,19 +220,6 @@ def render(agent: str, name: str, context: str | None, notice: str | None = None
|
|
|
220
220
|
return ""
|
|
221
221
|
|
|
222
222
|
|
|
223
|
-
def _closure_context(agent: str, name: str, payload: dict, outcome) -> str | None:
|
|
224
|
-
"""Ask Claude Code for one verification pass after edits, never loop or block read-only work."""
|
|
225
|
-
if agent != "claude-code" or name != "Stop" or outcome is None or payload.get("stop_hook_active") is True:
|
|
226
|
-
return None
|
|
227
|
-
if outcome.status != "unknown" or outcome.evidence != "no verification observed after the last edit":
|
|
228
|
-
return None
|
|
229
|
-
return (
|
|
230
|
-
"OpenReflex cannot verify this changed task yet. Before finishing, run the most relevant test, lint, or "
|
|
231
|
-
"build check. If verification is not possible, explain that limitation and finish; OpenReflex will retain "
|
|
232
|
-
"the execution as unverified rather than requiring an MCP call."
|
|
233
|
-
)
|
|
234
|
-
|
|
235
|
-
|
|
236
223
|
def handle(agent: str, name: str, payload: dict, engine_factory=Engine) -> str:
|
|
237
224
|
agent = detect_agent(agent, payload)
|
|
238
225
|
name = name or payload.get("hook_event_name") or payload.get("event") or ""
|
|
@@ -286,8 +273,9 @@ def handle(agent: str, name: str, payload: dict, engine_factory=Engine) -> str:
|
|
|
286
273
|
detail.append(f"{event.pending_crons} scheduled")
|
|
287
274
|
notice = f"↺ OpenReflex · WAITING\n{total} pending · " + " · ".join(detail)
|
|
288
275
|
else:
|
|
289
|
-
|
|
290
|
-
|
|
276
|
+
engine.stop(agent, event.session, assistant_completed=event.assistant_response_substantial)
|
|
277
|
+
# Missing verification is retained as internal evidence state. Ambient Stop never
|
|
278
|
+
# nags the agent or user merely because OpenReflex did not observe a post-change check.
|
|
291
279
|
if agent == "claude-code":
|
|
292
280
|
notice = engine.take_notice(agent, event.session)
|
|
293
281
|
return render(agent, name, context, notice)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: openreflex
|
|
3
|
-
Version: 0.9.
|
|
3
|
+
Version: 0.9.2
|
|
4
4
|
Summary: Ambient project muscle memory for AI coding agents: learns and reuses project Reflexes automatically
|
|
5
5
|
Author: vishnu-77
|
|
6
6
|
License-Expression: MIT
|
|
@@ -97,19 +97,21 @@ OpenTelemetry export to a loopback-only OpenReflex receiver and stores counts on
|
|
|
97
97
|
|
|
98
98
|
Requires Python 3.11 or newer.
|
|
99
99
|
|
|
100
|
-
**Claude Code — recommended:** install the plugin, reload it,
|
|
100
|
+
**Claude Code — recommended:** install the plugin, reload it, inspect the current state, then explicitly enable
|
|
101
|
+
project memory.
|
|
101
102
|
|
|
102
103
|
```text
|
|
103
104
|
/plugin marketplace add vishnu-77/openreflex
|
|
104
105
|
/plugin install openreflex@openreflex
|
|
105
106
|
/reload-plugins
|
|
106
107
|
/openreflex
|
|
108
|
+
/openreflex approve
|
|
107
109
|
```
|
|
108
110
|
|
|
109
111
|
Claude may display the canonical plugin namespace as `/openreflex:openreflex`; both refer to the same user-invoked
|
|
110
|
-
OpenReflex control surface when the bare alias is available.
|
|
111
|
-
for the current project
|
|
112
|
-
|
|
112
|
+
OpenReflex control surface when the bare alias is available. The no-argument command is read-only and compact.
|
|
113
|
+
`approve` explicitly enables local memory for the current project. The plugin prepares an exact-version runtime
|
|
114
|
+
under `~/.openreflex/runtime/` and normal agent work remains ambient after approval.
|
|
113
115
|
|
|
114
116
|
The Claude plugin does **not** execute a global `openreflex` from PATH. An older pipx/uv/pip installation can coexist
|
|
115
117
|
without taking over the plugin runtime.
|
|
@@ -146,7 +148,12 @@ OpenReflex installs per project with `openreflex install <agent>`, or as a plugi
|
|
|
146
148
|
| **Cursor** | Lifecycle hooks | `openreflex install cursor` | Protocol and fuzz tests |
|
|
147
149
|
| **OpenCode** | Local lifecycle plugin | `openreflex install opencode` | Protocol and fuzz tests |
|
|
148
150
|
|
|
149
|
-
With the Claude plugin,
|
|
151
|
+
With the Claude plugin, `/openreflex` shows a compact read-only state view. Use `/openreflex approve` to enable
|
|
152
|
+
local memory for the current project, then work normally. Other explicit controls include `status`, `reflexes`,
|
|
153
|
+
`memory`, `doctor`, `why`, `trace`, and `revoke`. OpenReflex control turns are not learned as project
|
|
154
|
+
experiences and do not trigger normal task verification. The control skill is user-invoked only and does not enable
|
|
155
|
+
MCP diagnostics or participate in normal model routing. Claude may show the canonical namespaced form
|
|
156
|
+
`/openreflex:openreflex`. Per-agent guides:
|
|
150
157
|
[Claude Code](https://openreflex.cc/claude-code), [Codex](https://openreflex.cc/codex), [Cursor](https://openreflex.cc/cursor),
|
|
151
158
|
[OpenCode](https://openreflex.cc/opencode).
|
|
152
159
|
|
|
@@ -259,6 +266,8 @@ retrieval, evidence-aware path selection, budget-aware execution, work-mode-spec
|
|
|
259
266
|
path comparison. The [Researcher view](https://openreflex.cc/?view=research) on the site explains the idea,
|
|
260
267
|
lets you step through one reflex forming in the graph, and places it next to related work.
|
|
261
268
|
|
|
269
|
+
The reproducible evaluation protocol, pinned real-agent corpus, exact prompts, and claim boundaries are documented in [docs/evaluation.md](docs/evaluation.md).
|
|
270
|
+
|
|
262
271
|
## Community & Contributing
|
|
263
272
|
|
|
264
273
|
- **Issues and ideas:** [GitHub Issues](https://github.com/vishnu-77/openreflex/issues)
|
|
@@ -24,39 +24,35 @@ def _snapshot(*, outcome: str, expected_regret: float | None = 0.0) -> DecisionS
|
|
|
24
24
|
context_tokens=0,
|
|
25
25
|
budget_used=0.1,
|
|
26
26
|
budget_pressure=0.1,
|
|
27
|
-
actual_tool_calls=
|
|
28
|
-
actual_tokens=
|
|
29
|
-
elapsed_seconds=
|
|
27
|
+
actual_tool_calls=9,
|
|
28
|
+
actual_tokens=7900,
|
|
29
|
+
elapsed_seconds=84.0,
|
|
30
30
|
outcome=outcome,
|
|
31
31
|
reflex_name="Helm values change",
|
|
32
32
|
)
|
|
33
33
|
|
|
34
34
|
|
|
35
|
-
def
|
|
35
|
+
def test_unknown_completion_is_silent_in_ambient_ui():
|
|
36
36
|
recap = render_recap(_snapshot(outcome="unknown", expected_regret=0.0))
|
|
37
|
-
assert recap
|
|
38
|
-
assert "unknown · Helm values change" in recap
|
|
39
|
-
assert "regret" not in recap
|
|
40
|
-
assert "vs test-first" not in recap
|
|
37
|
+
assert recap == ""
|
|
41
38
|
|
|
42
39
|
|
|
43
|
-
def
|
|
40
|
+
def test_success_completion_is_compact_and_evidence_backed():
|
|
44
41
|
recap = render_recap(_snapshot(outcome="success", expected_regret=0.25))
|
|
45
|
-
assert recap
|
|
46
|
-
assert "
|
|
47
|
-
assert "
|
|
48
|
-
assert "
|
|
42
|
+
assert recap == "↺ OpenReflex · COMPLETE\nHelm values change"
|
|
43
|
+
assert "calls" not in recap
|
|
44
|
+
assert "tokens" not in recap
|
|
45
|
+
assert "Path check" not in recap
|
|
49
46
|
|
|
50
47
|
|
|
51
|
-
def
|
|
48
|
+
def test_failure_completion_is_compact_and_evidence_backed():
|
|
52
49
|
recap = render_recap(_snapshot(outcome="failure", expected_regret=0.4))
|
|
53
|
-
assert recap
|
|
54
|
-
assert "
|
|
55
|
-
assert "
|
|
56
|
-
assert "regret" not in recap.lower()
|
|
50
|
+
assert recap == "↺ OpenReflex · FAILED\nHelm values change"
|
|
51
|
+
assert "calls" not in recap
|
|
52
|
+
assert "tokens" not in recap
|
|
57
53
|
|
|
58
54
|
|
|
59
|
-
def
|
|
55
|
+
def test_unknown_stop_stays_internal_then_upgrades_to_visible_verified_completion(engine, clock):
|
|
60
56
|
engine.prompt("claude-code", "s", "Fix the Helm deployment values validation failure please")
|
|
61
57
|
engine.take_notice("claude-code", "s") # consume the start recap
|
|
62
58
|
|
|
@@ -67,8 +63,12 @@ def test_unverified_stop_can_upgrade_to_verified_completion(engine, clock):
|
|
|
67
63
|
|
|
68
64
|
first = engine.stop("claude-code", "s")
|
|
69
65
|
assert first.status == "unknown"
|
|
70
|
-
|
|
71
|
-
|
|
66
|
+
assert engine.take_notice("claude-code", "s") is None
|
|
67
|
+
|
|
68
|
+
completions = [item for item in engine.store.latest("claude-code", "s").decision_history
|
|
69
|
+
if item.get("phase") == "complete"]
|
|
70
|
+
assert completions[-1]["outcome"] == "unknown"
|
|
71
|
+
assert completions[-1]["visibility"] == "ambient"
|
|
72
72
|
|
|
73
73
|
clock.advance(1)
|
|
74
74
|
verification = ("Bash", {"command": "helm lint charts/app"})
|
|
@@ -80,7 +80,6 @@ def test_unverified_stop_can_upgrade_to_verified_completion(engine, clock):
|
|
|
80
80
|
assert second.status == "success"
|
|
81
81
|
second_recap = engine.take_notice("claude-code", "s")
|
|
82
82
|
assert second_recap.startswith("↺ OpenReflex · COMPLETE")
|
|
83
|
-
assert "success" in second_recap
|
|
84
83
|
|
|
85
84
|
completions = [item for item in engine.store.latest("claude-code", "s").decision_history
|
|
86
85
|
if item.get("phase") == "complete"]
|
|
@@ -87,9 +87,12 @@ def test_plugin_hook_files_match_installer_definitions():
|
|
|
87
87
|
assert "name: openreflex" in skill
|
|
88
88
|
assert "user-invocable: true" in skill
|
|
89
89
|
assert "disable-model-invocation: true" in skill
|
|
90
|
-
assert
|
|
90
|
+
assert 'argument-hint: "[approve|status|reflexes|memory|doctor|why|trace|revoke]"' in skill
|
|
91
|
+
assert "OPENREFLEX_CONTROL_REQUEST: $ARGUMENTS" in skill
|
|
92
|
+
assert "launcher.cjs" in skill and " control " in skill and "--request" in skill
|
|
91
93
|
assert "global `openreflex` executable" in skill
|
|
92
|
-
assert "
|
|
94
|
+
assert "Do not enable MCP diagnostics" in skill
|
|
95
|
+
assert "tui" not in skill.lower(), "the user-facing skill must not leak the internal TUI command"
|
|
93
96
|
|
|
94
97
|
launcher = (PLUGIN / "runtime" / "launcher.cjs").read_text(encoding="utf-8")
|
|
95
98
|
assert "openreflex==" in launcher
|
|
@@ -111,6 +114,62 @@ def test_cli_hook_never_blocks_even_with_bad_arguments(isolated_home, project):
|
|
|
111
114
|
assert result.returncode == 0, (args, result.stderr)
|
|
112
115
|
|
|
113
116
|
|
|
117
|
+
def test_control_surface_is_compact_and_approve_is_explicit(isolated_home, project):
|
|
118
|
+
before = _run(["control", "--request", "", "--project", str(project)], env_home=isolated_home, cwd=project)
|
|
119
|
+
text = before.stdout.decode()
|
|
120
|
+
assert before.returncode == 0
|
|
121
|
+
assert "Memory not enabled" in text
|
|
122
|
+
assert "tui" not in text.lower()
|
|
123
|
+
assert approval(project) is None
|
|
124
|
+
|
|
125
|
+
enabled = _run(["control", "--request", "approve", "--project", str(project)], env_home=isolated_home, cwd=project)
|
|
126
|
+
text = enabled.stdout.decode()
|
|
127
|
+
assert enabled.returncode == 0
|
|
128
|
+
assert "OPENREFLEX" in text and "READY" in text
|
|
129
|
+
assert "Memory enabled" in text
|
|
130
|
+
assert "tui" not in text.lower()
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
@pytest.mark.parametrize("prompt", [
|
|
134
|
+
"/openreflex:openreflex status",
|
|
135
|
+
"enable openreflex for this project",
|
|
136
|
+
])
|
|
137
|
+
def test_openreflex_control_turn_is_not_learned_or_verified(isolated_home, project, prompt):
|
|
138
|
+
connected = _run(["install", "claude-code", "--project", str(project)], env_home=isolated_home, cwd=project)
|
|
139
|
+
assert connected.returncode == 0
|
|
140
|
+
|
|
141
|
+
base = {"session_id": "control", "cwd": str(project)}
|
|
142
|
+
events = [
|
|
143
|
+
("UserPromptSubmit", {**base, "prompt": prompt}),
|
|
144
|
+
("PreToolUse", {
|
|
145
|
+
**base,
|
|
146
|
+
"tool_name": "Bash",
|
|
147
|
+
"tool_input": {"command": "node /plugin/runtime/launcher.cjs control --request status"},
|
|
148
|
+
"tool_use_id": "control-1",
|
|
149
|
+
}),
|
|
150
|
+
("PostToolUse", {
|
|
151
|
+
**base,
|
|
152
|
+
"tool_name": "Bash",
|
|
153
|
+
"tool_input": {"command": "node /plugin/runtime/launcher.cjs control --request status"},
|
|
154
|
+
"tool_use_id": "control-1",
|
|
155
|
+
"tool_response": {"stdout": "OPENREFLEX READY", "stderr": ""},
|
|
156
|
+
}),
|
|
157
|
+
("Stop", base),
|
|
158
|
+
]
|
|
159
|
+
outputs = []
|
|
160
|
+
for event, payload in events:
|
|
161
|
+
result = _run(["hook", "claude-code", event], stdin=json.dumps(payload),
|
|
162
|
+
env_home=isolated_home, cwd=project)
|
|
163
|
+
assert result.returncode == 0 and not result.stderr, result.stderr
|
|
164
|
+
outputs.append(result.stdout.decode())
|
|
165
|
+
|
|
166
|
+
status = _run(["status", "--json", "--project", str(project)], env_home=isolated_home, cwd=project)
|
|
167
|
+
data = json.loads(status.stdout)
|
|
168
|
+
assert data["engagement"]["tasks"] == 0
|
|
169
|
+
assert data["engagement"]["experiences"] == 0
|
|
170
|
+
assert all("cannot verify" not in output.lower() for output in outputs)
|
|
171
|
+
|
|
172
|
+
|
|
114
173
|
def test_cli_end_to_end_capture_via_subprocess(isolated_home, project):
|
|
115
174
|
connected = _run(["install", "claude-code", "--project", str(project)], env_home=isolated_home)
|
|
116
175
|
assert connected.returncode == 0, connected.stderr
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import importlib.util
|
|
2
|
+
import json
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
ROOT = Path(__file__).resolve().parents[1]
|
|
6
|
+
SCRIPT = ROOT / "scripts" / "live_cold_start_eval.py"
|
|
7
|
+
SPEC = importlib.util.spec_from_file_location("openreflex_live_eval", SCRIPT)
|
|
8
|
+
LIVE_EVAL = importlib.util.module_from_spec(SPEC)
|
|
9
|
+
assert SPEC and SPEC.loader
|
|
10
|
+
SPEC.loader.exec_module(LIVE_EVAL)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def test_click_live_eval_manifest_is_pinned_and_unambiguous():
|
|
14
|
+
manifest = LIVE_EVAL._load_manifest(ROOT / "evals" / "cold_start" / "click.json")
|
|
15
|
+
assert manifest["name"] == "click-cold-start-v1"
|
|
16
|
+
assert manifest["repository"]["name"] == "pallets/click"
|
|
17
|
+
assert len(manifest["repository"]["commit"]) == 40
|
|
18
|
+
assert manifest["claude_code_version"] == "2.1.278"
|
|
19
|
+
assert manifest["model"] == "claude-sonnet-4-6"
|
|
20
|
+
|
|
21
|
+
tasks = manifest["tasks"]
|
|
22
|
+
assert len(tasks) == 3
|
|
23
|
+
assert len({task["name"] for task in tasks}) == 3
|
|
24
|
+
assert {task["target"] for task in tasks} == {
|
|
25
|
+
"src/click/shell_completion.py",
|
|
26
|
+
"src/click/types.py",
|
|
27
|
+
"src/click/formatting.py",
|
|
28
|
+
}
|
|
29
|
+
for task in tasks:
|
|
30
|
+
assert task["target"] not in task["prompt"], "the prompt must not reveal the answer path"
|
|
31
|
+
assert "Without editing any files" in task["prompt"]
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def test_usage_tokens_counts_all_reported_model_token_classes():
|
|
35
|
+
usage = {
|
|
36
|
+
"input_tokens": 100,
|
|
37
|
+
"output_tokens": 20,
|
|
38
|
+
"cache_creation_input_tokens": 30,
|
|
39
|
+
"cache_read_input_tokens": 50,
|
|
40
|
+
}
|
|
41
|
+
assert LIVE_EVAL._usage_tokens(usage) == 200
|
|
42
|
+
assert LIVE_EVAL._usage_tokens({}) is None
|
|
43
|
+
assert LIVE_EVAL._usage_tokens(None) is None
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def test_paired_summary_reports_real_deltas_without_requiring_a_positive_result():
|
|
47
|
+
manifest = json.loads((ROOT / "evals" / "cold_start" / "click.json").read_text())
|
|
48
|
+
results = [
|
|
49
|
+
{
|
|
50
|
+
"task": "shell-completion-dispatch",
|
|
51
|
+
"rep": 1,
|
|
52
|
+
"condition": "baseline",
|
|
53
|
+
"code": 0,
|
|
54
|
+
"observed_model_tokens": 1000,
|
|
55
|
+
"exploration_calls": 4,
|
|
56
|
+
"total_tool_calls": 4,
|
|
57
|
+
"seconds": 20.0,
|
|
58
|
+
"total_cost_usd": 0.02,
|
|
59
|
+
"found_correct_file": True,
|
|
60
|
+
"answer_mentions_target": True,
|
|
61
|
+
"openreflex_context_delivered": False,
|
|
62
|
+
},
|
|
63
|
+
{
|
|
64
|
+
"task": "shell-completion-dispatch",
|
|
65
|
+
"rep": 1,
|
|
66
|
+
"condition": "primed",
|
|
67
|
+
"code": 0,
|
|
68
|
+
"observed_model_tokens": 1200,
|
|
69
|
+
"exploration_calls": 5,
|
|
70
|
+
"total_tool_calls": 5,
|
|
71
|
+
"seconds": 24.0,
|
|
72
|
+
"total_cost_usd": 0.024,
|
|
73
|
+
"found_correct_file": True,
|
|
74
|
+
"answer_mentions_target": True,
|
|
75
|
+
"openreflex_context_delivered": True,
|
|
76
|
+
},
|
|
77
|
+
]
|
|
78
|
+
summary = LIVE_EVAL._paired_summary(results, manifest, "claude-sonnet-4-6")
|
|
79
|
+
assert summary["pairs_valid"] == 1
|
|
80
|
+
assert summary["paired_relative_change"]["observed_model_tokens"] == 0.2
|
|
81
|
+
assert summary["paired_relative_change"]["total_tool_calls"] == 0.25
|
|
82
|
+
assert summary["missing_treatment_context"] == []
|
|
83
|
+
assert "pilot only" in summary["claim_scope"]
|
|
@@ -31,24 +31,25 @@ def _successful_tool(factory, project, tool, arguments, tool_id):
|
|
|
31
31
|
_call(factory, project, "PostToolUse", {**payload, "tool_response": {"stdout": "ok"}})
|
|
32
32
|
|
|
33
33
|
|
|
34
|
-
def
|
|
34
|
+
def test_first_stop_after_unverified_edit_is_silent(tmp_path, project, clock):
|
|
35
35
|
approve(project)
|
|
36
36
|
factory = _factory(tmp_path, clock)
|
|
37
37
|
_call(factory, project, "UserPromptSubmit", {"prompt": "Fix the Helm values bug causing invalid deployment configuration"})
|
|
38
38
|
_successful_tool(factory, project, "Edit", {"file_path": "charts/app/values.yaml"}, "e1")
|
|
39
39
|
|
|
40
|
-
first =
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
40
|
+
first = _call(factory, project, "Stop", {})
|
|
41
|
+
assert first == ""
|
|
42
|
+
|
|
43
|
+
engine = factory(project)
|
|
44
|
+
try:
|
|
45
|
+
execution = engine.store.latest("claude-code", "s")
|
|
46
|
+
outcome = engine.store.find("Outcome", execution_id=execution.id)[0]
|
|
47
|
+
assert outcome.status == "unknown"
|
|
48
|
+
assert outcome.evidence == "no verification observed after the last edit"
|
|
49
|
+
finally:
|
|
50
|
+
engine.close()
|
|
51
|
+
|
|
52
|
+
assert _call(factory, project, "Stop", {"stop_hook_active": True}) == ""
|
|
52
53
|
|
|
53
54
|
|
|
54
55
|
def test_read_only_exploration_can_finish_unknown_without_continuation(tmp_path, project, clock):
|
|
@@ -79,7 +79,7 @@ def test_recap_reports_retrieved_experience_and_context_cost(engine, clock):
|
|
|
79
79
|
assert "1 experiences" in notice and f"+{snapshot.context_tokens} context tokens" in notice
|
|
80
80
|
|
|
81
81
|
|
|
82
|
-
def
|
|
82
|
+
def test_completion_recap_is_compact_and_verified(engine, clock):
|
|
83
83
|
outcome, _ = run_task(
|
|
84
84
|
engine,
|
|
85
85
|
clock,
|
|
@@ -90,11 +90,10 @@ def test_completion_recap_shows_realised_path_cost_and_comparison(engine, clock)
|
|
|
90
90
|
snapshot = engine.decision_snapshots(outcome.execution_id)[-1]
|
|
91
91
|
recap = render_recap(snapshot)
|
|
92
92
|
assert recap.startswith("↺ OpenReflex · COMPLETE")
|
|
93
|
-
assert "success" in recap
|
|
94
93
|
assert "test-first" not in recap
|
|
95
|
-
assert "
|
|
96
|
-
assert "
|
|
97
|
-
assert "
|
|
94
|
+
assert "calls" not in recap
|
|
95
|
+
assert "tokens" not in recap
|
|
96
|
+
assert "Path check" not in recap
|
|
98
97
|
|
|
99
98
|
|
|
100
99
|
def test_claude_hook_surfaces_recap_as_system_message(project, tmp_path, clock):
|
|
@@ -113,9 +112,47 @@ def test_claude_hook_surfaces_recap_as_system_message(project, tmp_path, clock):
|
|
|
113
112
|
assert start["systemMessage"].startswith("↺ OpenReflex · ")
|
|
114
113
|
assert "authentication refresh" not in start["systemMessage"]
|
|
115
114
|
|
|
116
|
-
complete =
|
|
117
|
-
|
|
118
|
-
assert complete
|
|
115
|
+
complete = hooks.handle("claude-code", "Stop", {"session_id": "s", "cwd": str(project)},
|
|
116
|
+
engine_factory=factory)
|
|
117
|
+
assert complete == ""
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def test_claude_unverified_edit_stop_is_silent(project, tmp_path, clock):
|
|
121
|
+
approve(project)
|
|
122
|
+
database = tmp_path / "quiet-unverified.sqlite3"
|
|
123
|
+
|
|
124
|
+
def factory(root):
|
|
125
|
+
return Engine(root, Store(database), clock=clock)
|
|
126
|
+
|
|
127
|
+
start_payload = {
|
|
128
|
+
"session_id": "quiet",
|
|
129
|
+
"cwd": str(project),
|
|
130
|
+
"prompt": "Fix the Helm deployment values validation failure please",
|
|
131
|
+
}
|
|
132
|
+
hooks.handle("claude-code", "UserPromptSubmit", start_payload, engine_factory=factory)
|
|
133
|
+
|
|
134
|
+
edit_payload = {
|
|
135
|
+
"session_id": "quiet",
|
|
136
|
+
"cwd": str(project),
|
|
137
|
+
"tool_name": "Edit",
|
|
138
|
+
"tool_use_id": "e1",
|
|
139
|
+
"tool_input": {"file_path": str(project / "charts/app/values.yaml")},
|
|
140
|
+
}
|
|
141
|
+
hooks.handle("claude-code", "PreToolUse", edit_payload, engine_factory=factory)
|
|
142
|
+
hooks.handle(
|
|
143
|
+
"claude-code",
|
|
144
|
+
"PostToolUse",
|
|
145
|
+
{**edit_payload, "tool_response": {"success": True}},
|
|
146
|
+
engine_factory=factory,
|
|
147
|
+
)
|
|
148
|
+
|
|
149
|
+
result = hooks.handle(
|
|
150
|
+
"claude-code",
|
|
151
|
+
"Stop",
|
|
152
|
+
{"session_id": "quiet", "cwd": str(project), "last_assistant_message": "Implemented the requested change."},
|
|
153
|
+
engine_factory=factory,
|
|
154
|
+
)
|
|
155
|
+
assert result == ""
|
|
119
156
|
|
|
120
157
|
|
|
121
158
|
def test_codex_remains_non_chatty_on_stop(project, tmp_path, clock):
|
|
@@ -20,7 +20,7 @@ def _call(factory, agent, event, payload):
|
|
|
20
20
|
|
|
21
21
|
|
|
22
22
|
def test_claude_round_trip_counts_tools_and_output_tokens(project, tmp_path, clock):
|
|
23
|
-
"""A real prompt -> tools -> stop lifecycle
|
|
23
|
+
"""A real prompt -> tools -> stop lifecycle keeps accounting internally while the recap stays compact."""
|
|
24
24
|
approve(project)
|
|
25
25
|
factory = _factory(tmp_path / "quality-hooks.sqlite3", clock)
|
|
26
26
|
base = {"session_id": "quality", "cwd": str(project)}
|
|
@@ -41,8 +41,9 @@ def test_claude_round_trip_counts_tools_and_output_tokens(project, tmp_path, clo
|
|
|
41
41
|
clock.advance(5)
|
|
42
42
|
rendered = json.loads(_call(factory, "claude-code", "Stop", base))
|
|
43
43
|
recap = rendered["systemMessage"]
|
|
44
|
-
assert "
|
|
45
|
-
assert "
|
|
44
|
+
assert recap.startswith("↺ OpenReflex · COMPLETE")
|
|
45
|
+
assert "calls" not in recap
|
|
46
|
+
assert "tokens" not in recap
|
|
46
47
|
|
|
47
48
|
engine = factory(project)
|
|
48
49
|
try:
|
|
@@ -63,10 +64,15 @@ def test_unknown_completion_does_not_publish_predicted_regret(project, tmp_path,
|
|
|
63
64
|
_call(factory, "claude-code", "UserPromptSubmit",
|
|
64
65
|
{**base, "prompt": "Compare two possible designs for the cache invalidation mechanism"})
|
|
65
66
|
clock.advance(30)
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
67
|
+
assert _call(factory, "claude-code", "Stop", base) == ""
|
|
68
|
+
engine = factory(project)
|
|
69
|
+
try:
|
|
70
|
+
execution = engine.store.latest("claude-code", "unknown")
|
|
71
|
+
outcome = engine.store.find("Outcome", execution_id=execution.id)[0]
|
|
72
|
+
assert outcome.status == "unknown"
|
|
73
|
+
assert outcome.estimated_regret is None
|
|
74
|
+
finally:
|
|
75
|
+
engine.close()
|
|
70
76
|
|
|
71
77
|
|
|
72
78
|
def test_machine_readable_tools_publish_real_output_schemas(project):
|
|
@@ -93,9 +93,9 @@ def test_zero_tool_investigation_does_not_fake_recommended_path(project):
|
|
|
93
93
|
recap = render_recap(snapshots[-1])
|
|
94
94
|
assert "COMPLETE" in recap
|
|
95
95
|
assert "source-first" not in recap
|
|
96
|
-
assert "
|
|
96
|
+
assert "tokens" not in recap
|
|
97
97
|
assert "regret" not in recap.lower()
|
|
98
|
-
assert "Path check
|
|
98
|
+
assert "Path check" not in recap
|
|
99
99
|
finally:
|
|
100
100
|
engine.close()
|
|
101
101
|
|
|
@@ -264,7 +264,7 @@ def test_otlp_token_usage_is_correlated_deduped_and_saved(project):
|
|
|
264
264
|
engine.close()
|
|
265
265
|
|
|
266
266
|
|
|
267
|
-
def
|
|
267
|
+
def test_completion_accounting_keeps_real_model_tokens_out_of_ambient_recap(project):
|
|
268
268
|
engine = Engine(project)
|
|
269
269
|
try:
|
|
270
270
|
engine.prompt("claude-code", "recap",
|
|
@@ -273,9 +273,11 @@ def test_completion_recap_prefers_real_model_tokens(project):
|
|
|
273
273
|
ingest_otlp_logs(_otlp_api_request("recap", input_tokens=2200, output_tokens=800,
|
|
274
274
|
cache_read=1000, cache_creation=0))
|
|
275
275
|
outcome = engine.stop("claude-code", "recap", assistant_completed=True)
|
|
276
|
-
|
|
277
|
-
|
|
276
|
+
snapshot = engine.decision_snapshots(outcome.execution_id)[-1]
|
|
277
|
+
recap = render_recap(snapshot)
|
|
278
|
+
assert outcome.model_tokens == 4000
|
|
279
|
+
assert snapshot.model_tokens == 4000
|
|
280
|
+
assert "model tokens" not in recap
|
|
278
281
|
assert "tool-output est." not in recap
|
|
279
|
-
assert "regret" not in recap.lower()
|
|
280
282
|
finally:
|
|
281
283
|
engine.close()
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|