hermes-blind 0.1.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hermes_blind-0.1.3/CHANGELOG.md +76 -0
- hermes_blind-0.1.3/EVALUATION.md +125 -0
- hermes_blind-0.1.3/LICENSE +21 -0
- hermes_blind-0.1.3/MANIFEST.in +3 -0
- hermes_blind-0.1.3/PKG-INFO +178 -0
- hermes_blind-0.1.3/README.md +150 -0
- hermes_blind-0.1.3/evaluation/anchor-fidelity-20260718/results.json +83 -0
- hermes_blind-0.1.3/pyproject.toml +53 -0
- hermes_blind-0.1.3/setup.cfg +4 -0
- hermes_blind-0.1.3/src/hermes_blind/__init__.py +49 -0
- hermes_blind-0.1.3/src/hermes_blind/__main__.py +3 -0
- hermes_blind-0.1.3/src/hermes_blind/apply.py +364 -0
- hermes_blind-0.1.3/src/hermes_blind/cli.py +45 -0
- hermes_blind-0.1.3/src/hermes_blind/preambles.py +184 -0
- hermes_blind-0.1.3/src/hermes_blind/scaffold.py +156 -0
- hermes_blind-0.1.3/src/hermes_blind.egg-info/PKG-INFO +178 -0
- hermes_blind-0.1.3/src/hermes_blind.egg-info/SOURCES.txt +25 -0
- hermes_blind-0.1.3/src/hermes_blind.egg-info/dependency_links.txt +1 -0
- hermes_blind-0.1.3/src/hermes_blind.egg-info/entry_points.txt +2 -0
- hermes_blind-0.1.3/src/hermes_blind.egg-info/requires.txt +6 -0
- hermes_blind-0.1.3/src/hermes_blind.egg-info/top_level.txt +1 -0
- hermes_blind-0.1.3/tests/test_apply.py +308 -0
- hermes_blind-0.1.3/tests/test_cli.py +15 -0
- hermes_blind-0.1.3/tests/test_core.py +67 -0
- hermes_blind-0.1.3/tests/test_graceful_degrade.py +112 -0
- hermes_blind-0.1.3/tests/test_preambles.py +136 -0
- hermes_blind-0.1.3/tests/test_scaffold.py +163 -0
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to `hermes-blind` will be documented here.
|
|
4
|
+
|
|
5
|
+
Format: [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
|
|
6
|
+
Versioning: [SemVer](https://semver.org/). The 0.x line remains experimental;
|
|
7
|
+
minor versions may change the public surface before 1.0.
|
|
8
|
+
|
|
9
|
+
## [0.1.3] — 2026-07-19
|
|
10
|
+
|
|
11
|
+
0.1.3 is the first public 0.1.x release. The public comparison is
|
|
12
|
+
**0.0.6 → 0.1.3**. Versions 0.1.0 through 0.1.2 were unpublished development
|
|
13
|
+
candidates.
|
|
14
|
+
|
|
15
|
+
### Added since public 0.0.6
|
|
16
|
+
|
|
17
|
+
- A dependency-free CLI: `hermes-blind apply`.
|
|
18
|
+
- Deterministic recovery-scaffold generation from Claude Code and Codex JSONL.
|
|
19
|
+
- `goals`, `first-sentence`, and `full` anchor modes.
|
|
20
|
+
- Intent-debias and target-scope preambles used by hermes-rubric.
|
|
21
|
+
- `placebo` and `gate-only` mechanism-isolation variants.
|
|
22
|
+
|
|
23
|
+
### Changed from the unpublished 0.1.2 candidate
|
|
24
|
+
|
|
25
|
+
- Goal-set extraction replaces first-sentence truncation as the default.
|
|
26
|
+
- Recovery output records the source filename instead of its absolute path.
|
|
27
|
+
- Recovery output refuses to overwrite its input and preserves existing output
|
|
28
|
+
files unless `--force` is explicit.
|
|
29
|
+
- Public docs and metadata now separate tested mechanics from unproven
|
|
30
|
+
behavioral efficacy.
|
|
31
|
+
- Added a privacy-safe anchor-fidelity report with sanitized per-session
|
|
32
|
+
counts, statistical context, receipt commitments, and public synthetic
|
|
33
|
+
mechanics fixtures; the report and sanitized data ship in the source
|
|
34
|
+
distribution.
|
|
35
|
+
- The public CLI excludes unfinished experiment and analysis harnesses tied to
|
|
36
|
+
sibling repositories.
|
|
37
|
+
- Removed the staged placeholder seal; it is not a release requirement.
|
|
38
|
+
|
|
39
|
+
### Evidence boundary
|
|
40
|
+
|
|
41
|
+
- A frozen 66-goal audit represented 40 pre-listed goals with goal-set
|
|
42
|
+
extraction, versus 7 with the previous first-sentence heuristic: a 50.0
|
|
43
|
+
percentage-point increase, with improvement in 7 of 9 sessions and ties in
|
|
44
|
+
2. This supports substantially better mission representation in the
|
|
45
|
+
recovery artifact.
|
|
46
|
+
- Bias reduction, downstream model adherence or task outcomes, automatic
|
|
47
|
+
drift detection, and optimal timing remain unproven.
|
|
48
|
+
|
|
49
|
+
## [0.0.6] — 2026-04-24
|
|
50
|
+
|
|
51
|
+
### Added
|
|
52
|
+
- Initial release. Five scaffold variants (`null`, `micro`, `short`, `v1`, `full`)
|
|
53
|
+
exposed via `wrap(prompt, variant)`.
|
|
54
|
+
- `extract_disclosure(response)` for parsing the model's disclosure line from
|
|
55
|
+
a completed response.
|
|
56
|
+
- `token_estimate(variant)` rough token count per variant.
|
|
57
|
+
- 19 unit tests covering: variant correctness, caller-prompt preservation,
|
|
58
|
+
length-ordering invariant, disclosure extraction (seven shapes incl. edge
|
|
59
|
+
cases), scaffold structural invariants.
|
|
60
|
+
- `INTENT.md`, `PLAN-v1.md`, `PLAN-v2.md` — design trail.
|
|
61
|
+
- Three hermes-rubric audit runs on the plan under `rubric-runs/`:
|
|
62
|
+
phase1-claude.json (5.4/10 on v1), phase1-v2.json (5.7/10 after iteration),
|
|
63
|
+
phase3-adversarial.json (5.5/10 — low aggregate = useful adversarial
|
|
64
|
+
signal; three real gaps surfaced: self-referential evaluation loop,
|
|
65
|
+
length-vs-content confound, backend-generalization runtime check).
|
|
66
|
+
|
|
67
|
+
### Known not-yet-validated
|
|
68
|
+
- Empirical variance-reduction effect of the scaffold. Phase 4 ablation
|
|
69
|
+
study not yet executed. Do not treat this release as a proven debiaser.
|
|
70
|
+
- Cross-model convergence across Opus / Sonnet / Haiku / Ollama qwen3.5.
|
|
71
|
+
- Behavior on long targets (>10k tokens) or multi-turn scoring.
|
|
72
|
+
|
|
73
|
+
### Not in this release
|
|
74
|
+
- CLI (library-only for v0.0.x)
|
|
75
|
+
- Integration into `hermes-rubric` backends (gated on Phase 4 pass)
|
|
76
|
+
- Non-English scaffold variants
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
# Recovery-anchor fidelity evaluation
|
|
2
|
+
|
|
3
|
+
This is the public, privacy-safe report for the anchor-fidelity evidence cited
|
|
4
|
+
by hermes-blind 0.1.3. It covers one question only:
|
|
5
|
+
|
|
6
|
+
> When a long first user turn contains several goals, does the default
|
|
7
|
+
> `goals` recovery anchor represent more of those goals than the previous
|
|
8
|
+
> first-sentence anchor?
|
|
9
|
+
|
|
10
|
+
It does **not** evaluate the prompt-wrapping primitive, automatic drift
|
|
11
|
+
detection, downstream model adherence, or final task outcomes.
|
|
12
|
+
|
|
13
|
+
## What “recovery” means
|
|
14
|
+
|
|
15
|
+
Recovery is the package's second main surface, not a third mode. Given a
|
|
16
|
+
Claude Code or Codex JSONL session, `hermes-blind apply --session ...` reads
|
|
17
|
+
the first user turn and creates a markdown anchor that can be reviewed and
|
|
18
|
+
reinserted into a long session.
|
|
19
|
+
|
|
20
|
+
That recovery surface has three anchor modes:
|
|
21
|
+
|
|
22
|
+
- `goals` (default): extracts up to 12 goal-carrying sentences;
|
|
23
|
+
- `first-sentence`: retains the legacy compact anchor; and
|
|
24
|
+
- `full`: includes up to 4,000 characters of the first turn.
|
|
25
|
+
|
|
26
|
+
The package does not decide when to inject the anchor, inject it
|
|
27
|
+
automatically, or make a model call.
|
|
28
|
+
|
|
29
|
+
## Method
|
|
30
|
+
|
|
31
|
+
The design was frozen before corpus mining. Fresh-context model reviewers
|
|
32
|
+
were separated by role:
|
|
33
|
+
|
|
34
|
+
1. one reviewer saw only each turn-one mission seed and pre-listed its goals;
|
|
35
|
+
2. hermes-blind generated a legacy `first-sentence` anchor and a `goals`
|
|
36
|
+
anchor for the same seed; and
|
|
37
|
+
3. a separate Haiku reviewer marked each pre-listed goal `REPRESENTED` or
|
|
38
|
+
`ABSENT` in each anchor.
|
|
39
|
+
|
|
40
|
+
The evaluated extraction implementation was commit
|
|
41
|
+
`ca433597e8b3683d38dcd656fa7a35ad1c985114`.
|
|
42
|
+
|
|
43
|
+
The frozen plan called for 10 qualifying sessions. The completed receipt
|
|
44
|
+
contains 9 sessions and 66 goals, and does not record why the tenth session
|
|
45
|
+
was absent. The figures below use the observed denominator; no missing result
|
|
46
|
+
was imputed.
|
|
47
|
+
|
|
48
|
+
## Results
|
|
49
|
+
|
|
50
|
+
| Anchor | Goals represented | Coverage | Wilson 95% CI |
|
|
51
|
+
|---|---:|---:|---:|
|
|
52
|
+
| Legacy `first-sentence` | 7 / 66 | 10.6% | 5.2–20.3% |
|
|
53
|
+
| Default `goals` | 40 / 66 | 60.6% | 48.5–71.5% |
|
|
54
|
+
|
|
55
|
+
The default anchor represented 33 additional goals: a **50.0 percentage-point
|
|
56
|
+
increase** and **5.7×** the legacy coverage. At the paired goal level, all 7
|
|
57
|
+
goals represented by the legacy anchor remained represented; 33 previously
|
|
58
|
+
absent goals became represented; and none regressed.
|
|
59
|
+
|
|
60
|
+
Across sessions, `goals` improved coverage in 7 of 9 sessions and tied in 2.
|
|
61
|
+
An exploratory two-sided exact sign test over the 7 non-tied sessions gives
|
|
62
|
+
`p = 0.015625`. This test was calculated after the frozen evaluation rather
|
|
63
|
+
than pre-registered, and the sessions come from one private operating
|
|
64
|
+
environment, so it should be read as corroborating evidence rather than a
|
|
65
|
+
population-wide effect estimate.
|
|
66
|
+
|
|
67
|
+
The sanitized per-session counts and exact calculations are in
|
|
68
|
+
[`evaluation/anchor-fidelity-20260718/results.json`](evaluation/anchor-fidelity-20260718/results.json).
|
|
69
|
+
|
|
70
|
+
## What this establishes
|
|
71
|
+
|
|
72
|
+
For these nine long mission seeds, the 0.1.3 `goals` extractor preserved
|
|
73
|
+
substantially more of the original mission in the generated recovery artifact
|
|
74
|
+
than the legacy first-sentence heuristic. Mission representation is a
|
|
75
|
+
necessary first step for recovery: a goal absent from the artifact cannot be
|
|
76
|
+
restored by that artifact.
|
|
77
|
+
|
|
78
|
+
## What this does not establish
|
|
79
|
+
|
|
80
|
+
This evaluation does not show that reinserting the artifact makes a model
|
|
81
|
+
follow the mission, improves the final answer, prevents drift, or identifies
|
|
82
|
+
the right intervention time. Those are separate links in the causal chain and
|
|
83
|
+
need separate evaluations. The goal listing and representation judgments were
|
|
84
|
+
model-produced, not human-adjudicated, and the private corpus is too small and
|
|
85
|
+
too concentrated to support a general claim across users, languages, or
|
|
86
|
+
session types.
|
|
87
|
+
|
|
88
|
+
## Privacy and auditability
|
|
89
|
+
|
|
90
|
+
The source material consists of real private session openings that may contain
|
|
91
|
+
personal or unpublished project context. Raw text, local paths, complete
|
|
92
|
+
session identifiers, and individual judgments are therefore not published.
|
|
93
|
+
The public JSON contains only relabeled per-session counts, aggregate results,
|
|
94
|
+
method metadata, and SHA-256 commitments to the frozen internal receipts.
|
|
95
|
+
|
|
96
|
+
This protects the operator's private context, but it also means an outside
|
|
97
|
+
reader cannot independently audit the goal labels from this packet alone. A
|
|
98
|
+
future public or consented corpus is required for full external replication.
|
|
99
|
+
|
|
100
|
+
## Reproduce the public mechanics
|
|
101
|
+
|
|
102
|
+
The repository's synthetic fixtures are small, fabricated Claude Code and
|
|
103
|
+
Codex JSONL records—not sanitized private conversations. They verify the
|
|
104
|
+
reproducible software mechanics: parsing both log shapes, extracting multiple
|
|
105
|
+
goals, preserving a later goal after context-only prose, using safe markdown
|
|
106
|
+
fences, refusing unsafe overwrites, omitting parent paths from errors, and
|
|
107
|
+
deduplicating Codex's paired event shapes.
|
|
108
|
+
|
|
109
|
+
Run them with:
|
|
110
|
+
|
|
111
|
+
```bash
|
|
112
|
+
pytest -q tests/test_apply.py
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
These fixtures reproduce the mechanism and regression boundary. They do not
|
|
116
|
+
reproduce the private 40/66 behavioral audit.
|
|
117
|
+
|
|
118
|
+
## Receipt commitments
|
|
119
|
+
|
|
120
|
+
| Internal receipt | SHA-256 |
|
|
121
|
+
|---|---|
|
|
122
|
+
| Frozen specification | `05b4afc471eb47a15212fd3dbf225321ca61eeb698252b247594890ef3956555` |
|
|
123
|
+
| Source results | `376a5fbd69accb644171bac0f0363cca09e93d54f298f1adcc20ee4b9ca6fbcf` |
|
|
124
|
+
| Results report | `4774264dd9b8ea16b13fb63a2a81a73c240b68a9783b264c586e104173e92d2a` |
|
|
125
|
+
| Private corpus commitment file | `a241c04d94c572ae109e2e99f2e22e1c9cf3ae1df42c649010882685b8820c71` |
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Hermes Labs
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: hermes-blind
|
|
3
|
+
Version: 0.1.3
|
|
4
|
+
Summary: Deterministic prompt and session-recovery scaffolds for LLM workflows
|
|
5
|
+
Author-email: Hermes Labs <roli@hermes-labs.ai>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Source, https://github.com/hermes-labs-ai/hermes-blind
|
|
8
|
+
Project-URL: Issues, https://github.com/hermes-labs-ai/hermes-blind/issues
|
|
9
|
+
Project-URL: Changelog, https://github.com/hermes-labs-ai/hermes-blind/blob/main/CHANGELOG.md
|
|
10
|
+
Project-URL: Evaluation, https://github.com/hermes-labs-ai/hermes-blind/blob/main/EVALUATION.md
|
|
11
|
+
Keywords: llm,prompt-engineering,session-recovery,scaffold,evaluation
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
19
|
+
Requires-Python: >=3.10
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
License-File: LICENSE
|
|
22
|
+
Provides-Extra: dev
|
|
23
|
+
Requires-Dist: build>=1.2; extra == "dev"
|
|
24
|
+
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
25
|
+
Requires-Dist: ruff>=0.4; extra == "dev"
|
|
26
|
+
Requires-Dist: twine>=5.0; extra == "dev"
|
|
27
|
+
Dynamic: license-file
|
|
28
|
+
|
|
29
|
+
# hermes-blind
|
|
30
|
+
|
|
31
|
+
**Deterministic prompt and session-recovery scaffolds for LLM workflows.**
|
|
32
|
+
|
|
33
|
+
[](https://pypi.org/project/hermes-blind/)
|
|
34
|
+
[](https://pypi.org/project/hermes-blind/)
|
|
35
|
+
[](LICENSE)
|
|
36
|
+
[](#evidence-and-limits)
|
|
37
|
+
|
|
38
|
+
hermes-blind provides two small, standard-library-only primitives:
|
|
39
|
+
|
|
40
|
+
1. prepend an evidence-gating scaffold to an evaluation prompt; and
|
|
41
|
+
2. recover a compact turn-one goal anchor from a Claude Code or Codex session log.
|
|
42
|
+
|
|
43
|
+
It makes no model calls, sends no network requests, and does not claim to
|
|
44
|
+
detect drift automatically.
|
|
45
|
+
|
|
46
|
+
## Install
|
|
47
|
+
|
|
48
|
+
```bash
|
|
49
|
+
pip install hermes-blind
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
Python 3.10+.
|
|
53
|
+
|
|
54
|
+
## Recover a long session
|
|
55
|
+
|
|
56
|
+
```bash
|
|
57
|
+
hermes-blind apply \
|
|
58
|
+
--session /path/to/rollout.jsonl \
|
|
59
|
+
--format auto \
|
|
60
|
+
--anchor-mode goals \
|
|
61
|
+
--turn 9 \
|
|
62
|
+
--out recovery.md
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
The default `goals` mode scans the first user turn for goal-carrying
|
|
66
|
+
sentences and preserves up to 12 of them. Use `first-sentence` for the
|
|
67
|
+
legacy compact behavior or `full` to include up to 4,000 characters.
|
|
68
|
+
`--format auto` recognizes Claude Code and Codex JSONL shapes.
|
|
69
|
+
|
|
70
|
+
The `--turn` value is output metadata. It is not an automatic trigger and
|
|
71
|
+
does not imply that turn 9 is an empirically optimal intervention point.
|
|
72
|
+
Existing output files are preserved unless `--force` is passed; the input
|
|
73
|
+
session file can never be used as the output path.
|
|
74
|
+
|
|
75
|
+
Recovery output includes user-authored text. It records only the session
|
|
76
|
+
filename, not its absolute path, but you should still inspect the markdown
|
|
77
|
+
before sharing it.
|
|
78
|
+
|
|
79
|
+
## Wrap an evaluation prompt
|
|
80
|
+
|
|
81
|
+
```python
|
|
82
|
+
from hermes_blind import wrap
|
|
83
|
+
|
|
84
|
+
prompt = wrap(
|
|
85
|
+
"Rate this paper on novelty from 0 to 10 and cite the target text.",
|
|
86
|
+
variant="v1",
|
|
87
|
+
)
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
Or from the CLI:
|
|
91
|
+
|
|
92
|
+
```bash
|
|
93
|
+
hermes-blind apply --variant v1 --prompt "Score this artifact from quoted evidence."
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
Available variants are `null`, `micro`, `short`, `v1`, `full`,
|
|
97
|
+
`placebo`, and `gate-only`. The `null` variant is an exact no-op for
|
|
98
|
+
controlled comparisons.
|
|
99
|
+
|
|
100
|
+
## Rubric framing helpers
|
|
101
|
+
|
|
102
|
+
The package also exposes the dependency-free intent and scope preambles used
|
|
103
|
+
by hermes-rubric:
|
|
104
|
+
|
|
105
|
+
```python
|
|
106
|
+
from hermes_blind import compose_intent
|
|
107
|
+
|
|
108
|
+
framed = compose_intent(
|
|
109
|
+
"Evaluate whether this release is ready.",
|
|
110
|
+
scope_class="results-bundle",
|
|
111
|
+
intent_debias=True,
|
|
112
|
+
)
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
## Evidence and limits
|
|
116
|
+
|
|
117
|
+
Validated mechanics for 0.1.3:
|
|
118
|
+
|
|
119
|
+
- deterministic prompt wrapping and disclosure parsing;
|
|
120
|
+
- Claude Code and Codex JSONL parsing;
|
|
121
|
+
- three recovery anchor modes;
|
|
122
|
+
- prompt preservation and scaffold-family invariants;
|
|
123
|
+
- package build, clean installation, CLI invocation, and unit tests.
|
|
124
|
+
|
|
125
|
+
A frozen 66-goal extraction audit found that goal-set extraction represented
|
|
126
|
+
40 of 66 pre-listed goals, compared with 7 of 66 for the previous
|
|
127
|
+
first-sentence heuristic: a 50.0 percentage-point increase, with improvement
|
|
128
|
+
in 7 of 9 sessions and ties in 2. This demonstrates substantially better
|
|
129
|
+
**mission representation in the generated recovery artifact**, a necessary
|
|
130
|
+
first step for recovery. It does not establish that reinserting the artifact
|
|
131
|
+
causes downstream model adherence or better task outcomes. See the
|
|
132
|
+
[privacy-safe evaluation report](https://github.com/hermes-labs-ai/hermes-blind/blob/main/EVALUATION.md)
|
|
133
|
+
for the method, sanitized
|
|
134
|
+
per-session results, statistical context, limitations, and receipt hashes.
|
|
135
|
+
|
|
136
|
+
Public synthetic fixtures in `tests/test_apply.py` reproduce the parsing,
|
|
137
|
+
extraction, safety, and output mechanics with fabricated Claude Code and Codex
|
|
138
|
+
JSONL. They do not reproduce the private 40/66 audit.
|
|
139
|
+
|
|
140
|
+
Not established:
|
|
141
|
+
|
|
142
|
+
- reliable bias reduction from the evaluation prefix;
|
|
143
|
+
- successful behavioral recovery after inserting a generated anchor;
|
|
144
|
+
- automatic drift detection or an optimal intervention turn;
|
|
145
|
+
- adversarial prompt-injection resistance;
|
|
146
|
+
- non-English behavior.
|
|
147
|
+
|
|
148
|
+
Earlier single-shot experiments did not establish the original bias-reduction
|
|
149
|
+
hypothesis. A later multi-turn research harness remains internal and is not
|
|
150
|
+
part of the public runtime because its efficacy study is incomplete.
|
|
151
|
+
|
|
152
|
+
Treat the output as a transparent scaffold for a human or agent to inspect,
|
|
153
|
+
not as a security boundary or independent evaluator.
|
|
154
|
+
|
|
155
|
+
## Development
|
|
156
|
+
|
|
157
|
+
```bash
|
|
158
|
+
python -m venv .venv
|
|
159
|
+
source .venv/bin/activate
|
|
160
|
+
pip install -e ".[dev]"
|
|
161
|
+
ruff check src tests
|
|
162
|
+
pytest -q
|
|
163
|
+
python -m build
|
|
164
|
+
twine check dist/*
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
## Version note
|
|
168
|
+
|
|
169
|
+
0.1.3 is the first public 0.1.x release. The public comparison is therefore
|
|
170
|
+
**0.0.6 → 0.1.3**. Versions 0.1.0 through 0.1.2 were internal development
|
|
171
|
+
candidates; 0.1.3 is a patch over the unpublished 0.1.2 candidate that adds
|
|
172
|
+
multi-goal extraction and aligns the public package surface.
|
|
173
|
+
|
|
174
|
+
## License
|
|
175
|
+
|
|
176
|
+
MIT. See [LICENSE](LICENSE).
|
|
177
|
+
|
|
178
|
+
Part of [Hermes Labs](https://hermes-labs.ai).
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
# hermes-blind
|
|
2
|
+
|
|
3
|
+
**Deterministic prompt and session-recovery scaffolds for LLM workflows.**
|
|
4
|
+
|
|
5
|
+
[](https://pypi.org/project/hermes-blind/)
|
|
6
|
+
[](https://pypi.org/project/hermes-blind/)
|
|
7
|
+
[](LICENSE)
|
|
8
|
+
[](#evidence-and-limits)
|
|
9
|
+
|
|
10
|
+
hermes-blind provides two small, standard-library-only primitives:
|
|
11
|
+
|
|
12
|
+
1. prepend an evidence-gating scaffold to an evaluation prompt; and
|
|
13
|
+
2. recover a compact turn-one goal anchor from a Claude Code or Codex session log.
|
|
14
|
+
|
|
15
|
+
It makes no model calls, sends no network requests, and does not claim to
|
|
16
|
+
detect drift automatically.
|
|
17
|
+
|
|
18
|
+
## Install
|
|
19
|
+
|
|
20
|
+
```bash
|
|
21
|
+
pip install hermes-blind
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
Python 3.10+.
|
|
25
|
+
|
|
26
|
+
## Recover a long session
|
|
27
|
+
|
|
28
|
+
```bash
|
|
29
|
+
hermes-blind apply \
|
|
30
|
+
--session /path/to/rollout.jsonl \
|
|
31
|
+
--format auto \
|
|
32
|
+
--anchor-mode goals \
|
|
33
|
+
--turn 9 \
|
|
34
|
+
--out recovery.md
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
The default `goals` mode scans the first user turn for goal-carrying
|
|
38
|
+
sentences and preserves up to 12 of them. Use `first-sentence` for the
|
|
39
|
+
legacy compact behavior or `full` to include up to 4,000 characters.
|
|
40
|
+
`--format auto` recognizes Claude Code and Codex JSONL shapes.
|
|
41
|
+
|
|
42
|
+
The `--turn` value is output metadata. It is not an automatic trigger and
|
|
43
|
+
does not imply that turn 9 is an empirically optimal intervention point.
|
|
44
|
+
Existing output files are preserved unless `--force` is passed; the input
|
|
45
|
+
session file can never be used as the output path.
|
|
46
|
+
|
|
47
|
+
Recovery output includes user-authored text. It records only the session
|
|
48
|
+
filename, not its absolute path, but you should still inspect the markdown
|
|
49
|
+
before sharing it.
|
|
50
|
+
|
|
51
|
+
## Wrap an evaluation prompt
|
|
52
|
+
|
|
53
|
+
```python
|
|
54
|
+
from hermes_blind import wrap
|
|
55
|
+
|
|
56
|
+
prompt = wrap(
|
|
57
|
+
"Rate this paper on novelty from 0 to 10 and cite the target text.",
|
|
58
|
+
variant="v1",
|
|
59
|
+
)
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
Or from the CLI:
|
|
63
|
+
|
|
64
|
+
```bash
|
|
65
|
+
hermes-blind apply --variant v1 --prompt "Score this artifact from quoted evidence."
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
Available variants are `null`, `micro`, `short`, `v1`, `full`,
|
|
69
|
+
`placebo`, and `gate-only`. The `null` variant is an exact no-op for
|
|
70
|
+
controlled comparisons.
|
|
71
|
+
|
|
72
|
+
## Rubric framing helpers
|
|
73
|
+
|
|
74
|
+
The package also exposes the dependency-free intent and scope preambles used
|
|
75
|
+
by hermes-rubric:
|
|
76
|
+
|
|
77
|
+
```python
|
|
78
|
+
from hermes_blind import compose_intent
|
|
79
|
+
|
|
80
|
+
framed = compose_intent(
|
|
81
|
+
"Evaluate whether this release is ready.",
|
|
82
|
+
scope_class="results-bundle",
|
|
83
|
+
intent_debias=True,
|
|
84
|
+
)
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
## Evidence and limits
|
|
88
|
+
|
|
89
|
+
Validated mechanics for 0.1.3:
|
|
90
|
+
|
|
91
|
+
- deterministic prompt wrapping and disclosure parsing;
|
|
92
|
+
- Claude Code and Codex JSONL parsing;
|
|
93
|
+
- three recovery anchor modes;
|
|
94
|
+
- prompt preservation and scaffold-family invariants;
|
|
95
|
+
- package build, clean installation, CLI invocation, and unit tests.
|
|
96
|
+
|
|
97
|
+
A frozen 66-goal extraction audit found that goal-set extraction represented
|
|
98
|
+
40 of 66 pre-listed goals, compared with 7 of 66 for the previous
|
|
99
|
+
first-sentence heuristic: a 50.0 percentage-point increase, with improvement
|
|
100
|
+
in 7 of 9 sessions and ties in 2. This demonstrates substantially better
|
|
101
|
+
**mission representation in the generated recovery artifact**, a necessary
|
|
102
|
+
first step for recovery. It does not establish that reinserting the artifact
|
|
103
|
+
causes downstream model adherence or better task outcomes. See the
|
|
104
|
+
[privacy-safe evaluation report](https://github.com/hermes-labs-ai/hermes-blind/blob/main/EVALUATION.md)
|
|
105
|
+
for the method, sanitized
|
|
106
|
+
per-session results, statistical context, limitations, and receipt hashes.
|
|
107
|
+
|
|
108
|
+
Public synthetic fixtures in `tests/test_apply.py` reproduce the parsing,
|
|
109
|
+
extraction, safety, and output mechanics with fabricated Claude Code and Codex
|
|
110
|
+
JSONL. They do not reproduce the private 40/66 audit.
|
|
111
|
+
|
|
112
|
+
Not established:
|
|
113
|
+
|
|
114
|
+
- reliable bias reduction from the evaluation prefix;
|
|
115
|
+
- successful behavioral recovery after inserting a generated anchor;
|
|
116
|
+
- automatic drift detection or an optimal intervention turn;
|
|
117
|
+
- adversarial prompt-injection resistance;
|
|
118
|
+
- non-English behavior.
|
|
119
|
+
|
|
120
|
+
Earlier single-shot experiments did not establish the original bias-reduction
|
|
121
|
+
hypothesis. A later multi-turn research harness remains internal and is not
|
|
122
|
+
part of the public runtime because its efficacy study is incomplete.
|
|
123
|
+
|
|
124
|
+
Treat the output as a transparent scaffold for a human or agent to inspect,
|
|
125
|
+
not as a security boundary or independent evaluator.
|
|
126
|
+
|
|
127
|
+
## Development
|
|
128
|
+
|
|
129
|
+
```bash
|
|
130
|
+
python -m venv .venv
|
|
131
|
+
source .venv/bin/activate
|
|
132
|
+
pip install -e ".[dev]"
|
|
133
|
+
ruff check src tests
|
|
134
|
+
pytest -q
|
|
135
|
+
python -m build
|
|
136
|
+
twine check dist/*
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
## Version note
|
|
140
|
+
|
|
141
|
+
0.1.3 is the first public 0.1.x release. The public comparison is therefore
|
|
142
|
+
**0.0.6 → 0.1.3**. Versions 0.1.0 through 0.1.2 were internal development
|
|
143
|
+
candidates; 0.1.3 is a patch over the unpublished 0.1.2 candidate that adds
|
|
144
|
+
multi-goal extraction and aligns the public package surface.
|
|
145
|
+
|
|
146
|
+
## License
|
|
147
|
+
|
|
148
|
+
MIT. See [LICENSE](LICENSE).
|
|
149
|
+
|
|
150
|
+
Part of [Hermes Labs](https://hermes-labs.ai).
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
{
|
|
2
|
+
"evaluation_id": "anchor-fidelity-20260718",
|
|
3
|
+
"release": "0.1.3",
|
|
4
|
+
"question": "Does goals-mode represent more pre-listed turn-one goals than the legacy first-sentence anchor?",
|
|
5
|
+
"scope": "recovery-artifact mission representation only",
|
|
6
|
+
"evaluated_commit": "ca433597e8b3683d38dcd656fa7a35ad1c985114",
|
|
7
|
+
"method": {
|
|
8
|
+
"design_frozen_before_corpus_mining": true,
|
|
9
|
+
"planned_sessions": 10,
|
|
10
|
+
"observed_sessions": 9,
|
|
11
|
+
"protocol_deviation": "The completed receipt contains nine sessions and does not record why the tenth planned session was absent.",
|
|
12
|
+
"goals": 66,
|
|
13
|
+
"goal_listing": "A fresh-context model reviewer saw only each turn-one seed and pre-listed its distinct goals.",
|
|
14
|
+
"representation_check": "A separate Haiku reviewer marked every pre-listed goal REPRESENTED or ABSENT in both anchors.",
|
|
15
|
+
"human_adjudication": false
|
|
16
|
+
},
|
|
17
|
+
"privacy": {
|
|
18
|
+
"source": "private real-world session openings from one operating environment",
|
|
19
|
+
"raw_text_published": false,
|
|
20
|
+
"local_paths_published": false,
|
|
21
|
+
"complete_session_ids_published": false,
|
|
22
|
+
"individual_goal_judgments_published": false,
|
|
23
|
+
"external_label_audit_possible_from_public_packet": false
|
|
24
|
+
},
|
|
25
|
+
"per_session": [
|
|
26
|
+
{"session": "S01", "goals": 12, "legacy_represented": 0, "goals_mode_represented": 7},
|
|
27
|
+
{"session": "S02", "goals": 10, "legacy_represented": 0, "goals_mode_represented": 4},
|
|
28
|
+
{"session": "S03", "goals": 3, "legacy_represented": 1, "goals_mode_represented": 1},
|
|
29
|
+
{"session": "S04", "goals": 6, "legacy_represented": 1, "goals_mode_represented": 5},
|
|
30
|
+
{"session": "S05", "goals": 4, "legacy_represented": 3, "goals_mode_represented": 3},
|
|
31
|
+
{"session": "S06", "goals": 12, "legacy_represented": 0, "goals_mode_represented": 5},
|
|
32
|
+
{"session": "S07", "goals": 5, "legacy_represented": 2, "goals_mode_represented": 4},
|
|
33
|
+
{"session": "S08", "goals": 12, "legacy_represented": 0, "goals_mode_represented": 9},
|
|
34
|
+
{"session": "S09", "goals": 2, "legacy_represented": 0, "goals_mode_represented": 2}
|
|
35
|
+
],
|
|
36
|
+
"pooled": {
|
|
37
|
+
"legacy": {
|
|
38
|
+
"represented": 7,
|
|
39
|
+
"total": 66,
|
|
40
|
+
"coverage": 0.106,
|
|
41
|
+
"wilson_95_ci": [0.052, 0.203]
|
|
42
|
+
},
|
|
43
|
+
"goals_mode": {
|
|
44
|
+
"represented": 40,
|
|
45
|
+
"total": 66,
|
|
46
|
+
"coverage": 0.606,
|
|
47
|
+
"wilson_95_ci": [0.485, 0.715]
|
|
48
|
+
},
|
|
49
|
+
"additional_goals_represented": 33,
|
|
50
|
+
"absolute_coverage_increase": 0.5,
|
|
51
|
+
"coverage_ratio": 5.714285714285714,
|
|
52
|
+
"paired_goal_transitions": {
|
|
53
|
+
"absent_to_absent": 26,
|
|
54
|
+
"absent_to_represented": 33,
|
|
55
|
+
"represented_to_absent": 0,
|
|
56
|
+
"represented_to_represented": 7
|
|
57
|
+
}
|
|
58
|
+
},
|
|
59
|
+
"session_level": {
|
|
60
|
+
"improved": 7,
|
|
61
|
+
"tied": 2,
|
|
62
|
+
"regressed": 0,
|
|
63
|
+
"exploratory_two_sided_exact_sign_test": {
|
|
64
|
+
"non_tied_sessions": 7,
|
|
65
|
+
"positive_differences": 7,
|
|
66
|
+
"p_value": 0.015625,
|
|
67
|
+
"pre_registered": false
|
|
68
|
+
}
|
|
69
|
+
},
|
|
70
|
+
"limits": [
|
|
71
|
+
"Does not evaluate downstream model adherence or task outcomes.",
|
|
72
|
+
"Does not evaluate automatic drift detection or intervention timing.",
|
|
73
|
+
"Model-produced goal labels and representation checks were not human-adjudicated.",
|
|
74
|
+
"The private corpus is not included, so external readers cannot audit labels from this packet alone.",
|
|
75
|
+
"Nine sessions from one operating environment do not support population-wide generalization."
|
|
76
|
+
],
|
|
77
|
+
"source_commitments": {
|
|
78
|
+
"frozen_spec_sha256": "05b4afc471eb47a15212fd3dbf225321ca61eeb698252b247594890ef3956555",
|
|
79
|
+
"source_e3_results_sha256": "376a5fbd69accb644171bac0f0363cca09e93d54f298f1adcc20ee4b9ca6fbcf",
|
|
80
|
+
"source_results_report_sha256": "4774264dd9b8ea16b13fb63a2a81a73c240b68a9783b264c586e104173e92d2a",
|
|
81
|
+
"private_corpus_commitment_file_sha256": "a241c04d94c572ae109e2e99f2e22e1c9cf3ae1df42c649010882685b8820c71"
|
|
82
|
+
}
|
|
83
|
+
}
|