poisson-break 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- poisson_break-0.1.0/.gitignore +46 -0
- poisson_break-0.1.0/CHANGELOG.md +495 -0
- poisson_break-0.1.0/LICENSE +21 -0
- poisson_break-0.1.0/NOTICE +125 -0
- poisson_break-0.1.0/PKG-INFO +270 -0
- poisson_break-0.1.0/README.md +232 -0
- poisson_break-0.1.0/docs/catalog.md +142 -0
- poisson_break-0.1.0/docs/claims.md +93 -0
- poisson_break-0.1.0/docs/cli-look.md +275 -0
- poisson_break-0.1.0/docs/export/langfuse.md +93 -0
- poisson_break-0.1.0/docs/export/langsmith.md +85 -0
- poisson_break-0.1.0/docs/export/phoenix.md +75 -0
- poisson_break-0.1.0/docs/export/samples/langfuse-export.json +42 -0
- poisson_break-0.1.0/docs/export/samples/langfuse.jsonl +4 -0
- poisson_break-0.1.0/docs/export/samples/langsmith-export.json +30 -0
- poisson_break-0.1.0/docs/export/samples/langsmith.jsonl +4 -0
- poisson_break-0.1.0/docs/export/samples/phoenix-export.csv +6 -0
- poisson_break-0.1.0/docs/export/samples/phoenix.jsonl +4 -0
- poisson_break-0.1.0/docs/extending/check.md +143 -0
- poisson_break-0.1.0/docs/extending/fault.md +103 -0
- poisson_break-0.1.0/docs/extending/strategy.md +172 -0
- poisson_break-0.1.0/docs/extending/way-in.md +128 -0
- poisson_break-0.1.0/docs/findings.md +233 -0
- poisson_break-0.1.0/docs/recheck.md +111 -0
- poisson_break-0.1.0/docs/reference.md +857 -0
- poisson_break-0.1.0/docs/repo-hygiene.md +101 -0
- poisson_break-0.1.0/docs/ways-in/rollout.md +256 -0
- poisson_break-0.1.0/examples/README.md +243 -0
- poisson_break-0.1.0/examples/desk/break.toml +34 -0
- poisson_break-0.1.0/examples/desk/cases.jsonl +6 -0
- poisson_break-0.1.0/examples/desk/desk_agent.py +75 -0
- poisson_break-0.1.0/examples/full-loop/agent.py +46 -0
- poisson_break-0.1.0/examples/full-loop/agent_fixed.py +46 -0
- poisson_break-0.1.0/examples/full-loop/break.toml +34 -0
- poisson_break-0.1.0/examples/full-loop/cases.jsonl +6 -0
- poisson_break-0.1.0/examples/own-check/break.toml +34 -0
- poisson_break-0.1.0/examples/own-check/cases.jsonl +6 -0
- poisson_break-0.1.0/examples/own-check/checks.py +14 -0
- poisson_break-0.1.0/examples/own-check/desk.py +47 -0
- poisson_break-0.1.0/examples/own-fault/break.toml +35 -0
- poisson_break-0.1.0/examples/own-fault/cases.jsonl +6 -0
- poisson_break-0.1.0/examples/own-fault/faults.py +25 -0
- poisson_break-0.1.0/examples/own-fault/shop_agent.py +50 -0
- poisson_break-0.1.0/examples/stand-in/break.toml +37 -0
- poisson_break-0.1.0/examples/stand-in/cases.jsonl +6 -0
- poisson_break-0.1.0/examples/stand-in/payments_fake.py +9 -0
- poisson_break-0.1.0/examples/stand-in/refund_desk.py +64 -0
- poisson_break-0.1.0/pyproject.toml +119 -0
- poisson_break-0.1.0/scripts/clean_install.py +701 -0
- poisson_break-0.1.0/src/poisson_break/__init__.py +36 -0
- poisson_break-0.1.0/src/poisson_break/__main__.py +5 -0
- poisson_break-0.1.0/src/poisson_break/_pretty.py +296 -0
- poisson_break-0.1.0/src/poisson_break/apply.py +994 -0
- poisson_break-0.1.0/src/poisson_break/changes.py +1253 -0
- poisson_break-0.1.0/src/poisson_break/checks.py +817 -0
- poisson_break-0.1.0/src/poisson_break/cli.py +2242 -0
- poisson_break-0.1.0/src/poisson_break/confirm.py +1577 -0
- poisson_break-0.1.0/src/poisson_break/hunt.py +1549 -0
- poisson_break-0.1.0/src/poisson_break/interfaces.py +167 -0
- poisson_break-0.1.0/src/poisson_break/labels.toml +24 -0
- poisson_break-0.1.0/src/poisson_break/look.py +1003 -0
- poisson_break-0.1.0/src/poisson_break/py.typed +0 -0
- poisson_break-0.1.0/src/poisson_break/records.py +351 -0
- poisson_break-0.1.0/src/poisson_break/search.py +1392 -0
- poisson_break-0.1.0/src/poisson_break/settings.py +1733 -0
- poisson_break-0.1.0/tests/__init__.py +0 -0
- poisson_break-0.1.0/tests/_guard/sitecustomize.py +29 -0
- poisson_break-0.1.0/tests/agents/__init__.py +36 -0
- poisson_break-0.1.0/tests/agents/plain.py +140 -0
- poisson_break-0.1.0/tests/agents/way_in.py +76 -0
- poisson_break-0.1.0/tests/apply_support.py +164 -0
- poisson_break-0.1.0/tests/apply_support_standin.py +34 -0
- poisson_break-0.1.0/tests/changes_support.py +164 -0
- poisson_break-0.1.0/tests/checks_own.py +102 -0
- poisson_break-0.1.0/tests/checks_support.py +314 -0
- poisson_break-0.1.0/tests/commands_child.py +38 -0
- poisson_break-0.1.0/tests/commands_corpus.py +115 -0
- poisson_break-0.1.0/tests/commands_dialogue.py +204 -0
- poisson_break-0.1.0/tests/commands_hunts.py +169 -0
- poisson_break-0.1.0/tests/commands_lines.py +64 -0
- poisson_break-0.1.0/tests/commands_own_fault.py +39 -0
- poisson_break-0.1.0/tests/commands_spies.py +144 -0
- poisson_break-0.1.0/tests/commands_support.py +450 -0
- poisson_break-0.1.0/tests/commands_targets.py +147 -0
- poisson_break-0.1.0/tests/confirm_support.py +378 -0
- poisson_break-0.1.0/tests/confirm_targets.py +102 -0
- poisson_break-0.1.0/tests/conftest.py +107 -0
- poisson_break-0.1.0/tests/hunt_broken_module.py +3 -0
- poisson_break-0.1.0/tests/hunt_support.py +288 -0
- poisson_break-0.1.0/tests/hunt_targets.py +233 -0
- poisson_break-0.1.0/tests/label_sources/nist-ai-100-2e2025-section-3-4.txt +1 -0
- poisson_break-0.1.0/tests/label_sources/owasp-llm01-2025.txt +1 -0
- poisson_break-0.1.0/tests/label_sources/sources.toml +30 -0
- poisson_break-0.1.0/tests/look_child.py +55 -0
- poisson_break-0.1.0/tests/look_conformance.py +387 -0
- poisson_break-0.1.0/tests/look_corpus.py +104 -0
- poisson_break-0.1.0/tests/look_results.py +179 -0
- poisson_break-0.1.0/tests/look_support.py +448 -0
- poisson_break-0.1.0/tests/release_docs.py +428 -0
- poisson_break-0.1.0/tests/release_probe/sitecustomize.py +122 -0
- poisson_break-0.1.0/tests/search_support.py +346 -0
- poisson_break-0.1.0/tests/settings_support.py +208 -0
- poisson_break-0.1.0/tests/strategies.py +116 -0
- poisson_break-0.1.0/tests/test_adding_a_fault.py +498 -0
- poisson_break-0.1.0/tests/test_apply_adding_a_fault.py +250 -0
- poisson_break-0.1.0/tests/test_apply_case.py +170 -0
- poisson_break-0.1.0/tests/test_apply_failures.py +401 -0
- poisson_break-0.1.0/tests/test_apply_launch_faults.py +384 -0
- poisson_break-0.1.0/tests/test_apply_previous.py +220 -0
- poisson_break-0.1.0/tests/test_apply_session.py +604 -0
- poisson_break-0.1.0/tests/test_apply_tool_classes.py +358 -0
- poisson_break-0.1.0/tests/test_apply_two_faults.py +216 -0
- poisson_break-0.1.0/tests/test_apply_wrap.py +313 -0
- poisson_break-0.1.0/tests/test_apply_wrapped_while.py +114 -0
- poisson_break-0.1.0/tests/test_catalog.py +286 -0
- poisson_break-0.1.0/tests/test_changes_applies.py +226 -0
- poisson_break-0.1.0/tests/test_changes_names.py +439 -0
- poisson_break-0.1.0/tests/test_changes_operators.py +313 -0
- poisson_break-0.1.0/tests/test_checks_kinds.py +640 -0
- poisson_break-0.1.0/tests/test_checks_no_model.py +138 -0
- poisson_break-0.1.0/tests/test_checks_order.py +398 -0
- poisson_break-0.1.0/tests/test_checks_own.py +545 -0
- poisson_break-0.1.0/tests/test_checks_ruled.py +646 -0
- poisson_break-0.1.0/tests/test_checks_signals.py +316 -0
- poisson_break-0.1.0/tests/test_command.py +232 -0
- poisson_break-0.1.0/tests/test_commands_catalog.py +113 -0
- poisson_break-0.1.0/tests/test_commands_cli.py +293 -0
- poisson_break-0.1.0/tests/test_commands_confirm.py +596 -0
- poisson_break-0.1.0/tests/test_commands_ctrl_c_reference.py +117 -0
- poisson_break-0.1.0/tests/test_commands_dialogue.py +753 -0
- poisson_break-0.1.0/tests/test_commands_doctor.py +422 -0
- poisson_break-0.1.0/tests/test_commands_findings.py +521 -0
- poisson_break-0.1.0/tests/test_commands_hunt.py +919 -0
- poisson_break-0.1.0/tests/test_commands_init.py +335 -0
- poisson_break-0.1.0/tests/test_commands_json_fields.py +649 -0
- poisson_break-0.1.0/tests/test_commands_one.py +74 -0
- poisson_break-0.1.0/tests/test_commands_one_detail.py +222 -0
- poisson_break-0.1.0/tests/test_commands_printed.py +598 -0
- poisson_break-0.1.0/tests/test_commands_read_stages.py +437 -0
- poisson_break-0.1.0/tests/test_commands_recheck.py +373 -0
- poisson_break-0.1.0/tests/test_commands_renderings.py +228 -0
- poisson_break-0.1.0/tests/test_commands_resume_dry_run.py +53 -0
- poisson_break-0.1.0/tests/test_commands_resume_estimate.py +101 -0
- poisson_break-0.1.0/tests/test_commands_several_unclassed.py +122 -0
- poisson_break-0.1.0/tests/test_commands_stored.py +186 -0
- poisson_break-0.1.0/tests/test_confirm_design.py +313 -0
- poisson_break-0.1.0/tests/test_confirm_findings.py +275 -0
- poisson_break-0.1.0/tests/test_confirm_findings_list.py +252 -0
- poisson_break-0.1.0/tests/test_confirm_finished_stage.py +103 -0
- poisson_break-0.1.0/tests/test_confirm_names.py +104 -0
- poisson_break-0.1.0/tests/test_confirm_null.py +37 -0
- poisson_break-0.1.0/tests/test_confirm_read_stages.py +333 -0
- poisson_break-0.1.0/tests/test_confirm_real_trace.py +64 -0
- poisson_break-0.1.0/tests/test_confirm_recheck.py +284 -0
- poisson_break-0.1.0/tests/test_confirm_replayed_error.py +65 -0
- poisson_break-0.1.0/tests/test_confirm_stage_settings.py +271 -0
- poisson_break-0.1.0/tests/test_confirm_trace.py +214 -0
- poisson_break-0.1.0/tests/test_confirm_verdicts.py +468 -0
- poisson_break-0.1.0/tests/test_dependencies.py +82 -0
- poisson_break-0.1.0/tests/test_hunt_control.py +295 -0
- poisson_break-0.1.0/tests/test_hunt_names.py +132 -0
- poisson_break-0.1.0/tests/test_hunt_open.py +539 -0
- poisson_break-0.1.0/tests/test_hunt_reopened.py +73 -0
- poisson_break-0.1.0/tests/test_hunt_rows.py +963 -0
- poisson_break-0.1.0/tests/test_hunt_run_once.py +170 -0
- poisson_break-0.1.0/tests/test_hunt_stops.py +658 -0
- poisson_break-0.1.0/tests/test_hunt_stored.py +184 -0
- poisson_break-0.1.0/tests/test_hunt_way_in.py +296 -0
- poisson_break-0.1.0/tests/test_interfaces.py +184 -0
- poisson_break-0.1.0/tests/test_launch_faults.py +548 -0
- poisson_break-0.1.0/tests/test_look_ascii.py +126 -0
- poisson_break-0.1.0/tests/test_look_colour.py +189 -0
- poisson_break-0.1.0/tests/test_look_conformance.py +305 -0
- poisson_break-0.1.0/tests/test_look_no_rich.py +121 -0
- poisson_break-0.1.0/tests/test_look_numbers.py +167 -0
- poisson_break-0.1.0/tests/test_look_problems.py +458 -0
- poisson_break-0.1.0/tests/test_look_questions.py +302 -0
- poisson_break-0.1.0/tests/test_look_renderings.py +344 -0
- poisson_break-0.1.0/tests/test_look_show.py +180 -0
- poisson_break-0.1.0/tests/test_look_stand_ins.py +44 -0
- poisson_break-0.1.0/tests/test_look_switch.py +286 -0
- poisson_break-0.1.0/tests/test_look_wrap.py +305 -0
- poisson_break-0.1.0/tests/test_package.py +221 -0
- poisson_break-0.1.0/tests/test_records.py +350 -0
- poisson_break-0.1.0/tests/test_release_commands.py +415 -0
- poisson_break-0.1.0/tests/test_release_docs_set.py +134 -0
- poisson_break-0.1.0/tests/test_release_examples.py +311 -0
- poisson_break-0.1.0/tests/test_release_export.py +78 -0
- poisson_break-0.1.0/tests/test_release_extending.py +138 -0
- poisson_break-0.1.0/tests/test_release_init_example.py +193 -0
- poisson_break-0.1.0/tests/test_release_json_examples.py +174 -0
- poisson_break-0.1.0/tests/test_release_labels.py +335 -0
- poisson_break-0.1.0/tests/test_release_own_fault.py +146 -0
- poisson_break-0.1.0/tests/test_release_readme.py +260 -0
- poisson_break-0.1.0/tests/test_release_recipes.py +170 -0
- poisson_break-0.1.0/tests/test_release_same_command.py +100 -0
- poisson_break-0.1.0/tests/test_release_transcripts.py +252 -0
- poisson_break-0.1.0/tests/test_release_wheel.py +141 -0
- poisson_break-0.1.0/tests/test_release_words.py +341 -0
- poisson_break-0.1.0/tests/test_scripted_agents.py +565 -0
- poisson_break-0.1.0/tests/test_search_coverage.py +295 -0
- poisson_break-0.1.0/tests/test_search_interface.py +442 -0
- poisson_break-0.1.0/tests/test_search_load.py +295 -0
- poisson_break-0.1.0/tests/test_search_random.py +434 -0
- poisson_break-0.1.0/tests/test_search_record.py +102 -0
- poisson_break-0.1.0/tests/test_search_score.py +275 -0
- poisson_break-0.1.0/tests/test_search_screen.py +505 -0
- poisson_break-0.1.0/tests/test_search_shrink.py +388 -0
- poisson_break-0.1.0/tests/test_search_stage.py +912 -0
- poisson_break-0.1.0/tests/test_settings_cases.py +338 -0
- poisson_break-0.1.0/tests/test_settings_init.py +328 -0
- poisson_break-0.1.0/tests/test_settings_install_command.py +134 -0
- poisson_break-0.1.0/tests/test_settings_plan.py +190 -0
- poisson_break-0.1.0/tests/test_settings_pure.py +294 -0
- poisson_break-0.1.0/tests/test_settings_read.py +350 -0
- poisson_break-0.1.0/tests/test_settings_refusals.py +623 -0
- poisson_break-0.1.0/tests/trace_recorded/fix_check_7e6fef6.jsonl +64 -0
- poisson_break-0.1.0/tests/trace_stand_in/poisson_trace/__init__.py +10 -0
- poisson_break-0.1.0/tests/trace_stand_in/poisson_trace/__main__.py +169 -0
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# Byte-compiled / optimized / DLL files
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*$py.class
|
|
5
|
+
|
|
6
|
+
# Distribution / packaging
|
|
7
|
+
build/
|
|
8
|
+
dist/
|
|
9
|
+
*.egg-info/
|
|
10
|
+
*.egg
|
|
11
|
+
.eggs/
|
|
12
|
+
|
|
13
|
+
# Virtual environments
|
|
14
|
+
.venv/
|
|
15
|
+
venv/
|
|
16
|
+
env/
|
|
17
|
+
|
|
18
|
+
# Testing / tooling caches
|
|
19
|
+
.pytest_cache/
|
|
20
|
+
.hypothesis/
|
|
21
|
+
.mypy_cache/
|
|
22
|
+
.ruff_cache/
|
|
23
|
+
.coverage
|
|
24
|
+
htmlcov/
|
|
25
|
+
|
|
26
|
+
# What a hunt writes: the default output folder of the settings file.
|
|
27
|
+
break-runs/
|
|
28
|
+
|
|
29
|
+
# Prove run stores a local check might leave behind.
|
|
30
|
+
prove-runs/
|
|
31
|
+
|
|
32
|
+
# Local working folders (scripts, run outputs). The sdist is built from this file, so a folder
|
|
33
|
+
# left out only in .git/info/exclude would ship in it.
|
|
34
|
+
.work/
|
|
35
|
+
|
|
36
|
+
# Editors / OS
|
|
37
|
+
.vscode/
|
|
38
|
+
.idea/
|
|
39
|
+
.DS_Store
|
|
40
|
+
|
|
41
|
+
# Devlogs are kept outside the repository (docs/repo-hygiene.md, section 2.7)
|
|
42
|
+
devlog.md
|
|
43
|
+
|
|
44
|
+
# Local secrets. Never committed. Break reads no key, and nothing here needs one.
|
|
45
|
+
.env
|
|
46
|
+
.env.local
|
|
@@ -0,0 +1,495 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## 0.1.0 (2026-11-13)
|
|
4
|
+
|
|
5
|
+
The first release, an alpha. Break runs an agent many times while changing how its tools behave
|
|
6
|
+
and what the user asks, keeps the conditions that fail more than a control, confirms them on
|
|
7
|
+
fresh runs with Poisson Trace's test, and checks a fix against them later. This version has one
|
|
8
|
+
way in, `rollout` (an agent Break calls as a Python function), ten tool faults, five input
|
|
9
|
+
operators, two search strategies that do not learn, and eight commands.
|
|
10
|
+
|
|
11
|
+
What a first user gets:
|
|
12
|
+
|
|
13
|
+
- `pip install poisson-break`, then `poisson-break init --example desk`, `hunt --yes` and
|
|
14
|
+
`confirm --yes` end with three findings on a help desk agent, with no key and no network.
|
|
15
|
+
- Five examples, each written into a folder by `init --example NAME` and each run offline
|
|
16
|
+
against a scripted model: `desk`, `full-loop` (hunt, confirm, a fix, recheck), `stand-in` (a
|
|
17
|
+
write tool that is never really called), `own-check` (a failure rule of your own) and
|
|
18
|
+
`own-fault` (a fault of your own). They ride in the wheel.
|
|
19
|
+
- The docs: the catalogue, reading a finding, checking a fix, the rollout way in, recipes for
|
|
20
|
+
making cases from LangSmith, Langfuse and Phoenix exports, what Break does not claim, and one
|
|
21
|
+
page for each of the four extension points, and a reference for the commands, the settings
|
|
22
|
+
file, the stored runs and the library. Every command the README and the docs show is
|
|
23
|
+
run in CI from the installed wheel, and every program they show is run there too.
|
|
24
|
+
- `catalog --labels` shows, beside a change, the labels it carries from the OWASP Top 10 for
|
|
25
|
+
LLM Applications 2025 and NIST AI 100-2 E2025, where the table has a row for it. Each row
|
|
26
|
+
quotes the published text it rests on.
|
|
27
|
+
|
|
28
|
+
Changed since the last development build:
|
|
29
|
+
|
|
30
|
+
- The installed `poisson-break` puts the folder it runs in first on the import path, as
|
|
31
|
+
`python -m poisson_break` does, so an agent, a stand-in, a check or a strategy in a module
|
|
32
|
+
beside `break.toml` loads under both.
|
|
33
|
+
- A tool fault of your own can be named in `[vary] tools`. Reading `break.toml` takes a name
|
|
34
|
+
the catalogue does not hold; `doctor`, `hunt` and `open_hunt` refuse one that is still
|
|
35
|
+
unknown once the agent's module is loaded, before any run, naming it
|
|
36
|
+
(`changes.check_registered`, `changes.UnknownFault`).
|
|
37
|
+
- Every detail row and estimate line says one in the singular ("1 run left", "1 case × 3
|
|
38
|
+
repeats"), as the verdict sentences already did.
|
|
39
|
+
- A hunt taken up with no prices set estimates over the runs it has left, and once its control
|
|
40
|
+
has run, from the control's seconds a run.
|
|
41
|
+
- `none`, given to `--outcomes` or `--own-check` or answered to their questions, means no
|
|
42
|
+
outcome counts as a failure and there is no check of your own. It was written as an outcome
|
|
43
|
+
named "none", and refused as a check's import path.
|
|
44
|
+
- `init` asks only about what the way in can reach, read from the way in itself: a `rollout`
|
|
45
|
+
target is not asked whether Break may change the model's calls or the conversation, and
|
|
46
|
+
`--yes` varies neither for it. `break.toml`'s comments list only the kind and the strategies
|
|
47
|
+
this version runs, and its `conversation` and `model` lines say they need a way in this
|
|
48
|
+
version does not have. A `[vary]` line the way in cannot make is named, in `doctor`, `hunt`,
|
|
49
|
+
`confirm` and `recheck` alike, with what to write there, and the command printed under it is
|
|
50
|
+
the `init` command that writes the file with that line emptied.
|
|
51
|
+
- `init` asks three things where it asked up to nine: where the agent is, where the cases are,
|
|
52
|
+
and for each tool one yes or no on whether calling it changes anything, even in a test
|
|
53
|
+
environment, in place of a `read`, `write` or `off` menu about changes outside it. The kind,
|
|
54
|
+
what fails a run, what Break may change, the runs, the prices and the cap take their defaults
|
|
55
|
+
without a question; `init` lists them after `wrote` and says `break.toml` is where to change
|
|
56
|
+
them. A tool that changes things, is not safe to repeat and has no stand-in is refused with
|
|
57
|
+
the `init` command that leaves its stand-in to fill in, and never written `off`. Every flag
|
|
58
|
+
still sets its value.
|
|
59
|
+
- `init` takes only the kind `rollout`, and `break.toml` too, wherever it is read: any other
|
|
60
|
+
kind is refused before anything is written or run, in one line saying this version has only
|
|
61
|
+
the rollout way in, with the same `init` command and `--kind rollout` to run instead. No
|
|
62
|
+
`pip install` line is printed for an extra, since this release publishes none:
|
|
63
|
+
`settings.install_command` hands back `None` for every kind until a release ships one.
|
|
64
|
+
- `hunt` ends with a `wrote` row naming `hunt.json`, `proposals.jsonl`, `candidates.json` and
|
|
65
|
+
every run in `store/`, and its `--json` names them under `wrote`. A hunt that found more
|
|
66
|
+
failing conditions than it keeps says how many it left out and that `candidates` under
|
|
67
|
+
`[confirm]` raises the limit, and its `--json` lists them under `left_out`. The search line
|
|
68
|
+
counts the cases each condition ran on, not the setting.
|
|
69
|
+
- `confirm` asked again about a stage already finished runs nothing and writes nothing: it says
|
|
70
|
+
the stage is already finished and shows the stored result with a `findings` row. Every
|
|
71
|
+
`confirm --json` carries `already_finished`, and `confirm.finished_stage(hunt)` is the
|
|
72
|
+
finished stage, or `None`.
|
|
73
|
+
- `recheck` shows its findings as a table under the headers `condition`, `verdict` and
|
|
74
|
+
`numbers`, as `confirm` does.
|
|
75
|
+
- A mistake in a verb's own options prints that verb's help as the command to run next, such as
|
|
76
|
+
`poisson-break confirm --help`; a mistake before any verb still prints `poisson-break --help`.
|
|
77
|
+
- The README says how Break changes a tool, what a hunt ran and what `break-runs/` holds, and
|
|
78
|
+
what the `wrap` line costs, and its first screen points to running Break on an agent of your
|
|
79
|
+
own. The rollout page has a whole agent and its cases to copy.
|
|
80
|
+
- The package's version is 0.1.0, its status Alpha. An sdist is built beside the wheel and
|
|
81
|
+
holds only the source, the tests, the docs, the examples, the scripts and the project's own
|
|
82
|
+
files.
|
|
83
|
+
|
|
84
|
+
Everything in the release, module by module:
|
|
85
|
+
|
|
86
|
+
- The commands: `init`, `doctor`, `hunt`, `confirm`, `findings`, `replay`, `recheck` and
|
|
87
|
+
`catalog`, each with its own `--help`. The command computes nothing: every number it prints
|
|
88
|
+
is the settings module's, the runs module's, the search module's or the confirm module's, and
|
|
89
|
+
what is in `poisson_break.cli` is the command line, the words on screen and the order of the
|
|
90
|
+
calls. Every verb prints one result through the look, takes `--json` for one strict document
|
|
91
|
+
named `poisson_break.<verb>-result/1`, and keeps the shared exit codes. A verb that spends
|
|
92
|
+
shows its plan and asks `start?` first; `--yes` answers, `--dry-run` runs nothing and writes
|
|
93
|
+
nothing, and no terminal without `--yes` is the plan and a refusal in one line. After every
|
|
94
|
+
plan, and in `init`'s and `doctor`'s output, the note that Break calls your tools for real.
|
|
95
|
+
- `init` asks only what Break cannot find out for itself: where the agent is, where the cases
|
|
96
|
+
are, and for each tool whether calling it changes anything, with a yes followed by whether it
|
|
97
|
+
is safe to repeat and, if not, where its stand-in is; given none, it writes nothing and
|
|
98
|
+
refuses, with the command to fill the stand-in in. The tools of a `rollout` target are the
|
|
99
|
+
ones its module wraps while it is imported (never run). Every other setting takes its default,
|
|
100
|
+
and `init` lists what it chose and says `break.toml` is where to change it. Every setting has a
|
|
101
|
+
flag; `--yes` takes every default and needs `--entry`. It writes `break.toml`, refuses to
|
|
102
|
+
write over one without `--force`, writes `cases.jsonl` only when the starter file was asked
|
|
103
|
+
for, and ends with the one command that writes the same file, which runs as printed with no
|
|
104
|
+
terminal. A line it prints back (`--force`, `--entry MODULE:NAME`) is the user's own words.
|
|
105
|
+
- `doctor` finds the first problem of the settings, the cases, the target, the strategy, the
|
|
106
|
+
changes the way in cannot make, each tool line and stand-in and your own check, and says
|
|
107
|
+
`not ready`, exit 4, in one line with the command to run next: `init` when there is no
|
|
108
|
+
settings file, `doctor` once the line it names is mended. `--run` runs the first case once
|
|
109
|
+
with no change, asked first, and stores nothing; a tool the agent calls that nobody classed
|
|
110
|
+
is `not ready` naming it, with the `init` command that writes the settings back with that
|
|
111
|
+
tool `off`.
|
|
112
|
+
- `hunt` runs the control, the search and the shrink stage, writes the candidates best first,
|
|
113
|
+
and shows each with how many of its runs failed and the kind most of them had; a hunt that
|
|
114
|
+
finds nothing prints what it did not look at first. A cap's stop and Ctrl-C say what was
|
|
115
|
+
kept and print `hunt --hunt NAME`, which goes on with the same hunt and runs no stored cell
|
|
116
|
+
again. A control that fails more than half its runs is a `note:` carrying its rate.
|
|
117
|
+
- `confirm` plans from the control stage's measured cost, runs the stage or finishes the one on
|
|
118
|
+
disk under its written design (its plan then says what is left), asks Trace about each
|
|
119
|
+
candidate and writes the findings; `--repeats N` asks again about the candidates too small to
|
|
120
|
+
tell, and is refused before any plan when there is nothing to ask again. `findings` lists them
|
|
121
|
+
with each case that failed in the control under a `control` heading, and `findings N` shows
|
|
122
|
+
one with its numbers as the look prints them, the command that replays it and Trace's
|
|
123
|
+
command for its full report. `replay` asks nothing before its one run. `recheck` prints one
|
|
124
|
+
of Trace's four phrases a finding with the rates before and now, the gap in days since the
|
|
125
|
+
reference runs, and `--check` exits 4 on a finding that fails more, `--strict` adding too
|
|
126
|
+
small to tell. `catalog` needs no settings file.
|
|
127
|
+
- A stage stopped by a tool nobody classed names it, and prints first the `init` command that
|
|
128
|
+
writes the settings back as they are with every such tool classed `off`, then the resume.
|
|
129
|
+
A refusal over a finding file that cannot be read names the file by a path that leads to it,
|
|
130
|
+
and prints no `next:`.
|
|
131
|
+
- `poisson_break.apply`: `wrapped_while(import_path)` names the tools wrapped while a module is
|
|
132
|
+
imported, in order, each once. `poisson_break.hunt`: `run_once(settings, target=, case=)`
|
|
133
|
+
runs one case once with no change, stored nowhere; a tool nobody classed raises
|
|
134
|
+
`ToolRefused`. `poisson_break.settings`: `install_command(kind)` is the `pip install` line
|
|
135
|
+
for a kind whose way in comes as an extra a release publishes and is not installed, or
|
|
136
|
+
`None`, which is every kind in this release; `way_in_installed(kind)` says whether a kind's
|
|
137
|
+
way in is on the machine. `poisson_break.hunt`: `Hunt.stored()` is every arm the hunt's store holds, by its word, with
|
|
138
|
+
the rows Prove counts in it. `poisson_break.confirm`: `read_stages(hunt)` is each confirm
|
|
139
|
+
stage written, a `Stage` with its arms' rows; `Replayed.error` carries a run not counted's
|
|
140
|
+
one line of error; `read_findings(hunt)` reads every finding file in number order and
|
|
141
|
+
refuses the whole call on
|
|
142
|
+
one it cannot read; each confirm stage in `confirm.json` carries the `q` and
|
|
143
|
+
`max_difference` it was begun under, and is judged under those. `poisson_break.look`:
|
|
144
|
+
`show(result)` prints a result the way the current run prints one, and plain outside a run.
|
|
145
|
+
- The command reads the confirm stages through `read_stages` and the store through
|
|
146
|
+
`Hunt.stored`, never `confirm.json` or Prove's store itself; a `confirm.json` that cannot be
|
|
147
|
+
read is refused in one line. Each `--json` document's fields are fixed: a plan is
|
|
148
|
+
`poisson_break.<verb>-plan/1` with `dollars` beside its estimate and `runs` what it may make
|
|
149
|
+
from here; `init` names `cases_file`; every `not ready` carries `key`, `line` and
|
|
150
|
+
`unclassed`; each tested candidate and rechecked finding carries `no_outcome` and `left_out`;
|
|
151
|
+
`confirm` names the files it `wrote`; the listing's `control` is null while the control is
|
|
152
|
+
not finished; a Ctrl-C document carries a stop's keys. A dry run of `hunt --hunt NAME` plans
|
|
153
|
+
what the hunt has left, and a hunt taken up after its control, with prices, is priced at the
|
|
154
|
+
control's cost of one run. A verdict sentence says a count of one in the singular.
|
|
155
|
+
|
|
156
|
+
- `poisson_break.confirm`: confirming findings. `write_candidates(hunt, conditions)` keeps a
|
|
157
|
+
hunt's candidates, best first, in its `candidates.json`, and `read_candidates(hunt)` reads
|
|
158
|
+
them back; no condition at all, a condition twice in any spelling, the empty condition, more
|
|
159
|
+
than the settings' `candidates`, and writing again once a confirm stage is on disk are each
|
|
160
|
+
refused. `design(candidates, settings, control)` works out the confirm stage and runs
|
|
161
|
+
nothing: one control arm and one arm for each candidate, every arm on every case the control
|
|
162
|
+
did not set aside, the settings' `repeats` times each, with the run counts `plan_confirm`
|
|
163
|
+
gives and their cost at what one control run cost.
|
|
164
|
+
- `confirm(hunt)` writes the design to the hunt's `confirm.json` before the stage's first run,
|
|
165
|
+
runs the arms on seeds no earlier stage used, and hands back a `Confirmation`: the stage, its
|
|
166
|
+
design, a `Tested` for every candidate and a `Finding` for each one confirmed. A stage that
|
|
167
|
+
stopped is taken up again under its written design, whatever the settings say by then, and a
|
|
168
|
+
finished stage asked again runs nothing and gives the same answer.
|
|
169
|
+
- Every number is Trace's or Prove's. Each candidate's arm is put to Trace once, against the
|
|
170
|
+
shared control arm, as `python -m poisson_trace fix-check OLD NEW --inputs all --event failed
|
|
171
|
+
--max-mde X --store STORE --json`, run with the Python Break runs on, in the hunt's folder,
|
|
172
|
+
and what is read is the JSON it prints, whatever its exit code (`ask_trace`). The candidates'
|
|
173
|
+
p-values are adjusted together with Prove's `bh` at the settings' `q`. A failure share is two
|
|
174
|
+
of Trace's counts divided, and the module computes nothing else.
|
|
175
|
+
- The verdicts: `confirmed` when Trace says *different*, the candidate's arm failed more and
|
|
176
|
+
the adjusted p-value is at or under `q`; `too small to tell` when Trace says so, with its
|
|
177
|
+
count of repeats that would decide; `not confirmed` otherwise, with the note `failed less`,
|
|
178
|
+
`not after the correction` or `nothing to compare` where one applies. A candidate with
|
|
179
|
+
nothing to compare has no numbers and is left out of the adjustment. `no_outcome` names the
|
|
180
|
+
cases with no run that has an outcome in one of the two arms, beside Trace's own count of
|
|
181
|
+
what it left out.
|
|
182
|
+
- `confirm(hunt, repeats=N)` asks again after a *too small to tell*: a new stage of only the
|
|
183
|
+
candidates the latest stage called *too small to tell*, with its own control arm, on new
|
|
184
|
+
seeds, read on its own and never pooled with the earlier stage. It is refused while the
|
|
185
|
+
latest stage has runs missing and when nothing was too small.
|
|
186
|
+
- A finding is numbered within the hunt from 1 in the order confirmed and written to
|
|
187
|
+
`findings/<n>.toml` (`write_finding`), plain TOML with its schema name first and a value that
|
|
188
|
+
is none left out; `read_finding` reads it back equal. It holds the condition by name, the two
|
|
189
|
+
failure shares, Trace's difference, interval and p-value, the adjusted p-value and `q`, the
|
|
190
|
+
failure kinds and signals counted over its arm, whether a failed run was judged by your own
|
|
191
|
+
check, what Trace flagged and left out, its cases, repeats, stage and arms, its reference
|
|
192
|
+
arm, the command that replays it and the Trace command for its full report.
|
|
193
|
+
- Each finding gets a reference arm, `reference-<n>`: the same agent under the finding's
|
|
194
|
+
condition, every case of its stage again, on fresh seeds, run at the end of `confirm` with
|
|
195
|
+
the finding's file written first. `reference(hunt)` runs what is missing of them.
|
|
196
|
+
- `replay(hunt, finding)` runs one finding once and stores nothing: on the first of its cases
|
|
197
|
+
with the most failed runs in its confirm arm, on the seed of that case's first failed run,
|
|
198
|
+
or on the case given, on the seed of its first run. It hands back a `Replayed`: the outcome,
|
|
199
|
+
the kind and every call in order with the change applied to it.
|
|
200
|
+
- `recheck(hunt)` runs every finding on the agent as it is now, over the cases and repeats of
|
|
201
|
+
its reference arm, in arms `recheck-<n>-<finding>`, asks Trace about the reference arm and
|
|
202
|
+
the new one, adjusts across the findings with `bh`, and hands back a `Rechecked` for each
|
|
203
|
+
with one of four phrases read from Trace's verdict and direction: `fails less`, `fails
|
|
204
|
+
more`, `no significant difference`, `too small to tell`. *Different* stands only at an
|
|
205
|
+
adjusted p-value at or under `q`. `gap_days` is Trace's time between the two runs. With
|
|
206
|
+
`old`, a way in over the agent as it was, both are run now (`recheck-<n>-<finding>-old`) and
|
|
207
|
+
compared with each other. A recheck that stopped is finished before a new one is begun.
|
|
208
|
+
- Trace is required at one version, `TRACE_VERSION`. Before any run of `confirm` or `recheck`
|
|
209
|
+
Break asks Trace for its version, then for a check of two runs of one row each that it
|
|
210
|
+
writes in a temporary folder. Trace missing, at another version or printing another schema
|
|
211
|
+
is refused there with nothing run. Every refusal is a `ConfirmError` with a `reason`:
|
|
212
|
+
`trace missing`, `trace version`, `trace schema`, `trace answer`, `no candidates`,
|
|
213
|
+
`no control`, `nothing to ask again`, `unfinished` or `unknown finding`. `trace answer` is an
|
|
214
|
+
answer Break could not read after the arms ran, and the arms are kept.
|
|
215
|
+
- Break now depends on Trace, pinned at 0.1.0a1, a pre-release on PyPI, and its Prove pin
|
|
216
|
+
moves to 0.1.3, the version that Trace pins.
|
|
217
|
+
- `poisson_break.search`: the search without learning, and the door every strategy comes
|
|
218
|
+
through. A strategy is anything with `propose(seen, budget, *, rng)` that returns a list of
|
|
219
|
+
`Proposal`: a condition, the reason in one line, and the probability the strategy gave that
|
|
220
|
+
condition, above 0 and at most 1. `Seen` is all it is shown, read-only: the control's map, every
|
|
221
|
+
run so far (`SeenRun`: condition, case, seed and an outcome of `ok`, `failed` or `not
|
|
222
|
+
counted`), every proposal so far, the runs left, what the settings allow, the cases set aside
|
|
223
|
+
and how many cases a condition runs on.
|
|
224
|
+
- `Screen` proposes every single change the control's map makes reachable, each once, in a fixed
|
|
225
|
+
order, at probability 1: every tool and every fault once in as few proposals as that takes,
|
|
226
|
+
then every input operator and every listed value of a setting, then the rest of the first call
|
|
227
|
+
of each tool, then each later call up to `max_call`. `Random` draws single changes and pairs of
|
|
228
|
+
them as one set, uniformly and never twice, at 1 over the size of the set; a pair on one call of
|
|
229
|
+
one tool, two values of one setting and a pair no one case can hold are not in it. Neither reads
|
|
230
|
+
an outcome, and neither proposes a rate, a fault on every call, or a change to the conversation
|
|
231
|
+
or the model's own calls.
|
|
232
|
+
- `allowed(vary=, tools=, capabilities=)` gives what the settings allow as a frozen `Allowed`:
|
|
233
|
+
each tool that is not `off` with the faults a session takes on it, the input operators, each
|
|
234
|
+
setting's values, `max_call` and the faults turned off. It asks a real session, so no second
|
|
235
|
+
copy of the tool-class rule exists.
|
|
236
|
+
- `search_stage(strategy, name=, seen=, rng=, run=, log=None, reserve=0)` runs a stage and returns
|
|
237
|
+
a `Stage`: the last `Seen`, the log and why it stopped or `None`. `run` is a callable that runs
|
|
238
|
+
each entry's condition on its cases and returns the rows, so the module runs no agent and
|
|
239
|
+
stores nothing. No condition runs twice: a repeat, a second spelling of a condition or the
|
|
240
|
+
control proposed again is logged `dropped` and costs no run, and a batch with nothing new ends
|
|
241
|
+
the stage. Every proposal is logged with its strategy, reason and probability before its run,
|
|
242
|
+
as a `LogEntry` whose `selection` is what a run's record carries, and with `log` given as one
|
|
243
|
+
JSON line in that file, which `read_log` reads back. A proposal with no probability or one
|
|
244
|
+
outside (0, 1], a reason that is not one line of text, a return that is not a list, a condition
|
|
245
|
+
the catalogue could not have written, or a change outside what the settings allow (named in the
|
|
246
|
+
refusal) refuses its whole batch before any of it runs. A strategy that raises stops the stage
|
|
247
|
+
with what it had; Ctrl-C and anything `run` raises pass through. A condition runs whole or not
|
|
248
|
+
at all: what the runs left cannot pay for is logged `not run` and ends the stage, and
|
|
249
|
+
`reserve` holds runs back for shrinking.
|
|
250
|
+
- `cases_for(seen, condition, rng=)`: the cases a condition runs on, drawn from those not set
|
|
251
|
+
aside on which the map says every tool change can land; where none fits, from those that call
|
|
252
|
+
each tool named at all, so a change at a call the control never saw has somewhere to run. A
|
|
253
|
+
condition no case can take is logged `not run`, spends nothing and does not end the stage.
|
|
254
|
+
- `score(seen, condition)`: failed runs over counted runs under the condition, less the same for
|
|
255
|
+
the control on those cases, each side pooled. A condition with a run on a case set aside, or
|
|
256
|
+
with no counted run, is refused.
|
|
257
|
+
- `shrink(condition, failures=, calls=, tries=None)`: one change taken out at a time, the smaller
|
|
258
|
+
condition kept when it fails at least as often as the candidate did, less one run, the line
|
|
259
|
+
staying at the candidate's own count. A fault on every call is tried as each single call the
|
|
260
|
+
tool reaches. A candidate of one change that is not on every call costs no run.
|
|
261
|
+
- `coverage(seen)`: what was tried and what was not looked at, as a frozen `Coverage`: reachable,
|
|
262
|
+
tried and untried for each tool and each fault, the tools never called, the tools called past
|
|
263
|
+
`max_call`, the faults turned off, the cases set aside, and how many reachable single changes
|
|
264
|
+
and pairs have no run.
|
|
265
|
+
- `load_strategy(name_or_path)`: `screen`, `random`, or `module:Name`, imported; a class is made
|
|
266
|
+
with no arguments and anything else with a `propose` is used as it is. Whatever goes wrong is
|
|
267
|
+
one `SearchError` of one line, before anything runs. `adaptive` is not in this version and is
|
|
268
|
+
refused the same way.
|
|
269
|
+
- `poisson_break.interfaces`: a strategy's `propose` is typed as returning a list, which is what
|
|
270
|
+
the stage holds it to.
|
|
271
|
+
- `poisson_break.hunt`: runs stored on Prove. `open_hunt(settings, target=, cases=, name=None,
|
|
272
|
+
existing=False)` makes a hunt's folder and its `hunt.json`, or opens one that is there, and
|
|
273
|
+
runs nothing. A `Hunt` is read-only: `name`, `folder`, `store`, `proposals`, `settings`,
|
|
274
|
+
`cases`, `target`, `plan`, `reserve` and `budget()`. `latest_hunt(settings)` names the hunt
|
|
275
|
+
made last.
|
|
276
|
+
- A hunt is named as given, if the name is letters, digits, `.`, `_` and `-`, starts with a
|
|
277
|
+
letter or a digit and is at most 64 characters; one that is not, or is taken, is refused and
|
|
278
|
+
never swapped for a number. With no name it is numbered from the last part of the target's
|
|
279
|
+
entry, one above the highest number there.
|
|
280
|
+
- Refused when a hunt is opened, before anything is made: a control stage that does not fit
|
|
281
|
+
(`SettingsError`), a tool the way in lists with no line in `[tools]` and a check that cannot be
|
|
282
|
+
loaded or called (`HuntError`), a `[vary]` entry the way in cannot make (`NeedsAdapter`,
|
|
283
|
+
naming the change), a stand-in that cannot be loaded (`ToolRefused`).
|
|
284
|
+
- `run_arm(hunt, arm, cases, condition, repeats=)` runs one condition on its cases as the Prove
|
|
285
|
+
run `<hunt>-<arm>`; the arm's first word is its stage (`control`, `confirm`, `reference`,
|
|
286
|
+
`recheck`). `run_search(hunt, entries, stage="search")` runs what a search hands over, each
|
|
287
|
+
entry's condition on its own cases, into `<hunt>-search` or, for `stage="shrink"`,
|
|
288
|
+
`<hunt>-shrink`. Each hands back one `RunRow` per run: `condition`, `case`, `seed`, `outcome`
|
|
289
|
+
(`ok`, `failed`, `not counted`) and `kind`. `on_progress(done, total)` is told after each run.
|
|
290
|
+
- `control(hunt)` runs every case `control_repeats` times with no change and returns a
|
|
291
|
+
`ControlSummary` read from the store: `runs`, `failed`, `not_counted`, `rate`, `per_case`,
|
|
292
|
+
`calls` (for each case, each tool with the most calls of it in one run), `set_aside` (the
|
|
293
|
+
cases that fail every counted run), `high_base_rate`, `dollars_per_run`, `seconds_per_run` and
|
|
294
|
+
`rows`.
|
|
295
|
+
- Break decides every row. A row's outcome is `ok` or `failed`; an agent that raises and a run
|
|
296
|
+
past its limit are `failed` rows with their kind, and only a run that could not be counted has
|
|
297
|
+
no outcome, with one line of error. Every row carries its record under `poisson_break`, a run
|
|
298
|
+
not counted included, beside `case_id` and nothing of the target's own. A tool that never
|
|
299
|
+
returns is cut off at `run_seconds`, on the main thread of a program on macOS or Linux.
|
|
300
|
+
- An arm's cell is `{"case": <id>}`, so arms line up case for case; a search or shrink row's is
|
|
301
|
+
the case and the condition's name. Seeds are Prove's, from a salt of `[search] seed` and the
|
|
302
|
+
arm's own word: no two arms share one, and the same settings give the same seeds under any
|
|
303
|
+
hunt name. An arm's rows carry the settings' strategy at probability 1, a search row the
|
|
304
|
+
selection of the entry that produced it, and a shrink row the entry's strategy at
|
|
305
|
+
probability 1.
|
|
306
|
+
- `Budget` (`runs`, `runs_spent`, `runs_left`, `dollars`, `dollars_spent`, `hours`,
|
|
307
|
+
`seconds_spent`) is read from the store. `runs` counts the control, search and shrink rows.
|
|
308
|
+
With prices, the next run is priced at the dearest run stored and a stage stops before a run
|
|
309
|
+
that could pass `dollars`; the first run of a hunt always runs. `hours` is the stored runs'
|
|
310
|
+
seconds, summed. `Hunt.reserve` is the number of runs a search leaves unspent for shrinking.
|
|
311
|
+
- `HuntStopped` is a stage that stopped short with every finished run stored: `reason` (`runs`,
|
|
312
|
+
`dollars`, `hours`, `not counted`, `refused`, `tools`), `arm`, `kept`, `count`, `cause` and
|
|
313
|
+
`tools`. More than a tenth of an arm's runs not counted stops it, judged after every run once
|
|
314
|
+
ten are stored, over the runs stored since the hunt was opened, so a hunt opened again after
|
|
315
|
+
the cause is mended goes on. A tool nobody classed stops it and that run is not stored. A tool the control
|
|
316
|
+
saw called that the way in did not list stops the hunt after the control. Ctrl-C passes
|
|
317
|
+
through as it is. `HuntError` is the module's own refusal before any run.
|
|
318
|
+
- A run is stored the moment it finishes, and a cell is run once in an arm: asked for again,
|
|
319
|
+
its stored row is handed back. A hunt opened again with `existing=True`, under its own seed
|
|
320
|
+
and cases, runs exactly what is missing on the same calls, under the settings handed in.
|
|
321
|
+
- `hunt.json` holds `schema` (`poisson_break.hunt/1`), `name`, `created`, `settings` as
|
|
322
|
+
resolved, `cases`, `stages` and `arms`, each arm with its `stage`, `condition`, `cases` and
|
|
323
|
+
`repeats`. It is rewritten whole when an arm starts.
|
|
324
|
+
- `RolloutTarget(fn, tools=)` is the way in for a plain Prove target, and `load_target(settings)`
|
|
325
|
+
makes one from `target.entry`. It reads `reply`, `tokens_in`, `tokens_out` and `ended` from
|
|
326
|
+
the episode's `meta`, each optional. `Limits(run_seconds)` is what a way in is handed.
|
|
327
|
+
`Target`, `ToolInfo`, `RunResult` and `TARGET_INTERFACE_VERSION` are re-exported.
|
|
328
|
+
- Prove's planner gives the seeds and its store takes the rows, one row as each run finishes.
|
|
329
|
+
Prove's runner is not used, and nothing is taken from a private module of Prove.
|
|
330
|
+
|
|
331
|
+
- `poisson_break.look`: one result, two renderings. A verb builds a `Result` (a verdict word,
|
|
332
|
+
its sentence, the detail as `Row`, `Line` and `Table`, the commands to run next) or a `Plan`
|
|
333
|
+
(a result whose verdict is `plan`, with an estimate and its source), and `render` prints it
|
|
334
|
+
plain or pretty. Plain is text only, with no colour code and no box character, and needs
|
|
335
|
+
nothing but the standard library: one line per fact on a pipe, wrapped at the terminal's width
|
|
336
|
+
with a hanging indent on a terminal, the verdict line and a command never wrapped. Pretty
|
|
337
|
+
carries the same facts in the same order, drawn with Rich.
|
|
338
|
+
- `mode_for` chooses: pretty only when stdout and stderr are both terminals, `NO_COLOR` is
|
|
339
|
+
unset or empty, `CI` is unset, `TERM` is not `dumb`, stdout is UTF-8, and neither `--json` nor
|
|
340
|
+
`--plain` stands before a `--`. Anywhere else, and wherever Rich cannot be imported, plain
|
|
341
|
+
runs, byte for byte, and nothing is said about it.
|
|
342
|
+
- Colour has four meanings: green for a command that did what was asked (`found`, `confirmed`,
|
|
343
|
+
`wrote`, `ready`, `rechecked`, `fails less`), yellow for one that stopped short or cannot say
|
|
344
|
+
(`stopped`, `nothing found`, `not confirmed`, `too small to tell`, `did not fail`, `note:`),
|
|
345
|
+
red for an error or a failed check (`error:`, `not ready`, `failed`, `fails more`), dim for
|
|
346
|
+
secondary text. Any other verdict word is bold with no colour. A number is never coloured,
|
|
347
|
+
and only the terminal's sixteen named colours, bold and dim are used.
|
|
348
|
+
- A table too wide for the terminal folds in pretty: a unit every cell of a column repeats
|
|
349
|
+
moves into the header, and past that each row is printed as a record. Nothing is cut.
|
|
350
|
+
- `run(tool, argv, body)` runs a command under the look and returns its exit code, from `EXIT`:
|
|
351
|
+
0 done, 1 refused, 2 usage, 3 stopped, 4 a check did not pass, 130 interrupted. `Refused` and
|
|
352
|
+
`Usage` are one `error:` line on stderr with their `next:` commands under it; `Stopped`
|
|
353
|
+
prints the result it carries; Ctrl-C prints `stopped interrupted: ...`; anything unexpected
|
|
354
|
+
is one line naming `~/.cache/poisson-break/errors.log`, where the traceback is appended, and
|
|
355
|
+
`--debug` prints the traceback too. `error`, `note`, `out` and `err` write the lines.
|
|
356
|
+
- `ask(question, default, choices=None)` asks on stderr: `[y/N]` for a yes or no, `[default]`
|
|
357
|
+
for text, a menu for `choices` (questionary's in pretty, a numbered list in plain). `--yes`
|
|
358
|
+
answers without asking, and without it a stdin that is not a terminal refuses the question.
|
|
359
|
+
`Progress(label, total)` shows work on stderr as one line, removed when the work ends.
|
|
360
|
+
- `as_json(result)`: the `--json` document, `schema`, `verdict` and `sentence` first, never a
|
|
361
|
+
`NaN`.
|
|
362
|
+
- Numbers read the same everywhere: `fmt_rate` (`0.150`, and `+0.870` for a difference),
|
|
363
|
+
`fmt_p` (`0.049`, `2.3e-08`), `fmt_dollars` (`$1.60`, `$0.0052`), `fmt_count` (`4,200`),
|
|
364
|
+
`fmt_duration` (`1 min 40 s`, `1 h 36 min`), `fmt_time` (UTC with `Z`) and `fmt_hash` (12
|
|
365
|
+
characters). `word` quotes a value for a shell and `command` joins a line that is never
|
|
366
|
+
wrapped.
|
|
367
|
+
- A stream that cannot encode `·`, `–` or `×` gets `-`, `-` and `x`, and any other character it
|
|
368
|
+
cannot encode is written as its escape, so two names never read as one. A control character
|
|
369
|
+
in a value is shown as text and never acted on.
|
|
370
|
+
- The command goes through the look. It takes `--plain` and `--debug`, a mistake on the command
|
|
371
|
+
line is drawn in the terminal's red on a terminal, and a failure inside the command is one
|
|
372
|
+
line and exit 3.
|
|
373
|
+
- Rich (14.2.0 or later), questionary (2.1.1 or later) and prompt_toolkit (3.0.53 or later) are
|
|
374
|
+
ordinary dependencies. A plain run loads none of them.
|
|
375
|
+
- The clean-install check also confirms that the three came with the wheel, and runs the
|
|
376
|
+
command on a terminal, with colour and with `--plain`.
|
|
377
|
+
|
|
378
|
+
- `poisson_break.settings`: the settings file `break.toml`. `loads` reads its text and `load` a
|
|
379
|
+
file into a frozen `Settings`, compared by what it holds; `dump` writes the file `init` writes,
|
|
380
|
+
with every setting that has a value on its own line and each one that is off commented out,
|
|
381
|
+
and `loads(dump(s)) == s`. Only `version`, `target.kind` and `target.entry` must be given; the
|
|
382
|
+
defaults are 400 runs, 5 control repeats, 4 cases per condition, 8 candidates, 5 confirm
|
|
383
|
+
repeats, a difference of 0.20, `q` 0.05, the first 3 calls, 120 seconds a run, 2 hours, the
|
|
384
|
+
ten tool faults and five input operators on, the conversation and model calls off.
|
|
385
|
+
- The file is strict. An unknown key or table is refused with its dotted path, its line and the
|
|
386
|
+
nearest known key within two edits. So is a value that means nothing: a kind, fault, operator,
|
|
387
|
+
move, check or signal that is not one, read from the live catalogue so a fault registered from
|
|
388
|
+
outside is taken; a count below 1 or not a whole number; a share outside 0 to 1; a `write`
|
|
389
|
+
tool whose line does not say how it is made safe to repeat; a stand-in, a check of your own or a strategy
|
|
390
|
+
not written `module:name`; a cap in dollars without both prices, or one price alone; `hours =
|
|
391
|
+
inf` with no cap; a file with no `version` or one from a newer Break. Every refusal is a
|
|
392
|
+
`SettingsError`, one line, with `.key`, `.line` and `.suggestion`. Two prices with no cap read
|
|
393
|
+
as a cap of $5.00.
|
|
394
|
+
- Reading, checking and writing settings run nobody's code: no framework is imported, and
|
|
395
|
+
neither is the target, the strategy, a stand-in or your own check. Nothing here asks what a
|
|
396
|
+
way in can do; that is refused before any run by what holds the target.
|
|
397
|
+
- This version has one way in, `rollout`. A `kind` whose way in will come as an optional extra
|
|
398
|
+
(`openai-agents`, `langgraph`, `crewai`, `mcp`, `command`) parses with `loads`, and `load`
|
|
399
|
+
refuses it in one line saying this version has only the rollout way in, until its module is
|
|
400
|
+
there, found without importing it. The `adaptive` strategy parses and is refused by `load`
|
|
401
|
+
the same way.
|
|
402
|
+
- `read_cases`: the case file, JSON Lines. A case needs only `input`, non-empty text; it may
|
|
403
|
+
carry an `id`, an `expect` table (`reply_contains`, `calls`, `never_calls`), `variants` and
|
|
404
|
+
`turns`, and any key of your target's own, which is kept. A case with no `id` takes its place
|
|
405
|
+
among the cases, counted from 1, as text. A blank line is skipped and still counted. A line
|
|
406
|
+
that is refused is named by its number.
|
|
407
|
+
- `INIT_FLAGS`, `settings_from_init_args` and `equivalent_init_command`: every setting has one
|
|
408
|
+
flag, with `--tool` and `--env` repeating for a tool's line and a setting of your own, and the
|
|
409
|
+
one command a shell can run that writes the same file byte for byte. Flags are written whole.
|
|
410
|
+
- `plan_hunt` and `plan_confirm`: the run counts a spending command shows before it starts, as
|
|
411
|
+
frozen results. A hunt on 12 cases with the defaults is 60 control runs and up to 340 for
|
|
412
|
+
search and shrink; confirming is 540 runs and up to 480 more for reference. A control stage
|
|
413
|
+
that leaves no room for one condition is refused, naming the smallest `runs` that fits.
|
|
414
|
+
- `poisson_break.checks`: deciding whether a run failed, from what the agent did. No model
|
|
415
|
+
grades a run. `watch(run_agent, *, case, session, fails=None)` runs the target once, catches
|
|
416
|
+
what escapes it (an exception is a crash, an agent's `SystemExit` among them; running out of
|
|
417
|
+
time is a timeout; `KeyboardInterrupt` passes through), measures the seconds and hands back a
|
|
418
|
+
frozen `FinishedRun`. `judge(run)` returns a `Verdict`: `ok`, `failed` with one kind, or
|
|
419
|
+
`not counted`, with the outcome a stored run carries (`ok` or `failed`), who judged it and,
|
|
420
|
+
for a run not counted, one line saying why. The kinds, the first that applies: `crashed`,
|
|
421
|
+
`hung` (120 seconds by default), `ran_out`, `forbidden_call`, `followed_plant`,
|
|
422
|
+
`duplicate_write`, `failed` (an outcome of the target's that `outcomes` lists), `wrong` (the
|
|
423
|
+
case's `reply_contains`, `calls`, `never_calls`), then a promoted signal's own word, then
|
|
424
|
+
`own:<word>` from your check. `checks` in the settings turns any of the seven words of
|
|
425
|
+
`CHECKS` off and nothing else. `signals(run)` counts `gave_up`, `retry_storm` (more than
|
|
426
|
+
`retry_storm_after` calls of one tool in a row, 5 by default) and `over_budget`
|
|
427
|
+
(`max_tool_calls`, `max_tokens`); a signal is a failure only when `also` names it. A run the
|
|
428
|
+
session stopped for an instrument error or a refused tool is not counted, whatever else it
|
|
429
|
+
met.
|
|
430
|
+
- Your own check, named in `own` by its import path, is called with one read-only `RunView` and
|
|
431
|
+
nothing else, only on a run everything before it found ok. It returns `None` for ok or a
|
|
432
|
+
failure kind, a word of ASCII letters, digits, `_` and `-`. Anything else it returns or
|
|
433
|
+
raises makes the run not counted, never failed. The view shows the case as the run saw it,
|
|
434
|
+
the condition and whether each change applied, every tool call with its arguments, what the
|
|
435
|
+
agent saw and whether the real tool ran, the reply, the target's outcome, tokens, seconds, and
|
|
436
|
+
the kinds Break's checks give with every one on. What it holds is a private copy, so a check
|
|
437
|
+
cannot change a run.
|
|
438
|
+
- `poisson_break.apply`: `Session.seen` keeps each call's arguments and what the agent was
|
|
439
|
+
handed back, apart from the two records, which still hold no content; `Session.tools` is each
|
|
440
|
+
tool's line as given.
|
|
441
|
+
- `poisson_break.records`: a `Failure`'s kind may be a signal's own word, for one the settings
|
|
442
|
+
promote.
|
|
443
|
+
- `poisson_break.apply`: `Session`, one run's applier. A way in routes every tool call through
|
|
444
|
+
`session.call(tool, args, real)`, which returns what the agent should see or raises
|
|
445
|
+
`ToolFailure` (a `TimeoutError` too when its kind is `timeout`). Calls are numbered per tool;
|
|
446
|
+
the first change in the condition's order that applies to a call lands on it, and any other
|
|
447
|
+
change on that call does nothing. Under the control the agent gets the tool's own object and
|
|
448
|
+
the tool's own exception. `session.calls` and `session.applied` are the two records a stored
|
|
449
|
+
run carries: every tool call with the change that landed and whether the real tool ran, and
|
|
450
|
+
every change with whether it applied and the first call it landed on. A fault's `previous` is
|
|
451
|
+
what the real tool last returned, held as a copy. Each tool is classed `read`, `write` or
|
|
452
|
+
`off`: a `write` tool must be `repeatable` or have a stand-in, `lost_reply` is taken only on a
|
|
453
|
+
tool safe to repeat, an `off` tool is never touched, a stand-in replaces its tool in every
|
|
454
|
+
run, and each refusal is `ToolRefused` naming the tool, when the session is made or, for a
|
|
455
|
+
tool with no line, on the call. Break's own failure, a fault's or a stand-in's is an
|
|
456
|
+
`InstrumentError` held in `session.stopped`, after which every call raises the same error and
|
|
457
|
+
runs nothing; the real tool's own exception passes to the agent untouched. The session sleeps
|
|
458
|
+
the delay a result states, and `run_seconds` is the run's wall-clock limit: past it,
|
|
459
|
+
`RunOutOfTime`, which an agent's `except Exception` does not catch. `session.case(case)` gives
|
|
460
|
+
the case with the condition's input operators and conversation moves applied. `wrap(tool_name,
|
|
461
|
+
fn)` is the one call a Prove target adds per tool, and `with session:` opens the run around
|
|
462
|
+
the agent; an `async def` tool is refused. `ToolInfo` and `RunResult` are what a way in hands
|
|
463
|
+
the core.
|
|
464
|
+
- Five scripted agents for the tests, in `tests/agents/`: one retries, one gives up, one follows
|
|
465
|
+
planted text, one repeats a write, and one speaks only the way-in protocol. No model and no
|
|
466
|
+
framework.
|
|
467
|
+
- `poisson_break.changes`: changes and conditions by name (`parse`, `name`), canonical and
|
|
468
|
+
round-tripping, with a bad name refused and the nearest valid name offered when one is within
|
|
469
|
+
two edits; whether a change applies to a call (`applies`): a schedule by the call number alone,
|
|
470
|
+
a rate by a keyed draw of the seed, the tool and the call; the five input operators and the four
|
|
471
|
+
conversation moves as pure functions of a case and a seed; the catalogue (`CATALOG`) of the ten
|
|
472
|
+
tool faults, five input operators, four conversation moves and five model faults, each with a
|
|
473
|
+
one-line description and its defaults (`delay` 5 s, `wait` 30 s, `length` 200 characters,
|
|
474
|
+
`typos` share 5%); the ten launch faults, each a `Fault` that is given a frozen `CallContext`
|
|
475
|
+
and returns a `Result` or a `ToolError`; `register` for a fault written outside the package;
|
|
476
|
+
`run_fault`, which counts the real tool's calls against the fault's declaration and turns a
|
|
477
|
+
fault's mistake into an `InstrumentError`; `check_capabilities`, which refuses before any run a
|
|
478
|
+
condition that asks of a way in something it cannot do (`NeedsAdapter`); and `marker`, the
|
|
479
|
+
planted word derived from the hunt and the case.
|
|
480
|
+
- The package `poisson-break`, importing as `poisson_break`, with the command `poisson-break`
|
|
481
|
+
(`python -m poisson_break` is the same). `--version` prints one line with the versions of
|
|
482
|
+
Break, Prove, Trace and Python; `--help` gives a description and an example; a mistake on the
|
|
483
|
+
command line is one `error:` line on stderr and exit 2, and abbreviated options are refused.
|
|
484
|
+
- `poisson_break.records`: the record each stored run carries under the key `poisson_break` in
|
|
485
|
+
the run's extra facts, in the published shape `poisson_break.run/1`: the hunt, the stage, the
|
|
486
|
+
arm, the case, the condition with each of its changes and whether and where it applied, the
|
|
487
|
+
failure kind and the three signals, the strategy and the probability it gave the condition,
|
|
488
|
+
every tool call, who judged the run and the target's own outcome. Frozen types that round-trip
|
|
489
|
+
through JSON and refuse a field the shape does not name or a value outside its closed set.
|
|
490
|
+
- `poisson_break.interfaces`: the four extension points, each at version 1: a fault, a way in,
|
|
491
|
+
a strategy and your own failure check.
|
|
492
|
+
- Prove pinned to the exact version Trace pins, and Trace pinned exactly, both from PyPI.
|
|
493
|
+
- CI: lint, type check and the tests on Python 3.11 to 3.13, under a guard that fails any
|
|
494
|
+
connection and a check that no statistic is computed locally; a clean install of the built
|
|
495
|
+
wheel into a fresh environment outside the checkout that runs the command.
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Poisson Labs
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|