hpflow 0.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
hpflow/__init__.py ADDED
@@ -0,0 +1,4 @@
1
+ """hpflow: run agent workers through Puppetmaster and take Python functions up the evidence ladder."""
2
+ from __future__ import annotations
3
+
4
+ __version__ = "0.3.0"
hpflow/__main__.py ADDED
@@ -0,0 +1,7 @@
1
+ from __future__ import annotations
2
+
3
+ import sys
4
+
5
+ from .cli import main
6
+
7
+ sys.exit(main())
hpflow/briefs.py ADDED
@@ -0,0 +1,74 @@
1
+ """The briefs of the workers. The texts come from the study scripts in `hard-proof/v2`."""
2
+ from __future__ import annotations
3
+
4
+ REVIEW = """The folder `project/` has the Python source of a tool. Please review one function: `{function}` in `project/{file}`, with the helper functions that it calls.
5
+
6
+ Look for inputs for which this function gives a wrong result: a result that conflicts with its docstring, its comments, the documentation in `project/docs` or `project/README.md`, the name of the function, or what its callers in the project need. Only report a problem that you are sure about. If you find no problem, report none. Do not report style, speed, or missing features.
7
+
8
+ Write your report into the current folder:
9
+ - `findings.json`: a JSON list, the most certain finding first, at most 5 entries. Each entry: {{"n": 1, "claim": "what is wrong, in one or two sentences", "expected": "the result that a correct function gives for the input of the repro", "confidence": "high | medium"}}. An empty list when you find no problem.
10
+ - For each finding `n`, a file `finding_<n>.py` that defines `run()` with no arguments. `run()` imports the function from the project (the folder `project/` is the first entry of `sys.path`, so `import {top}` works), calls it on the input of your finding, and returns a short string that shows the result (use `repr`). If the function needs small state, build it inside `run()` with the standard library only. `run()` must be deterministic: no real time, no random values, no network, no paths outside a `tempfile.TemporaryDirectory()`. If the result is an exception, catch it and return its class name.
11
+
12
+ {arm_text}Do not change `project/`. End with VERDICT: PASS when the report is written."""
13
+
14
+ REVIEW_ARM = {"plain": "You have no shell in this task. Read the code.\n\n",
15
+ "shell": "You have a shell. You can run Python on the project to check a finding, for example `cd project && python3 -c \"...\"`.\n\n"}
16
+ REVIEW_CHECK = "python3 -c \"import json;d=json.load(open('findings.json'));assert isinstance(d,list) and len(d)<=5\""
17
+
18
+ VERIFY = """The folder `project/` has the current source of the product `{repo}` (Python package, tests and documentation). A code review made {n} claims about one function: `{function}` in `project/{file}`. Some claims can be wrong, and some can be the same claim two times.
19
+
20
+ `claims.json` lists the claims. For claim `k`, the file `claim_<k>.py` defines `run()`, which calls the function on the input of the claim, and `claims.json` has the result that `run()` gives on the current source (`result_now`). You can run a claim again: `cd project && python3 -c "import sys; sys.path.insert(0, '.'); exec(open('../claim_1.py').read()); print(run())"`.
21
+
22
+ Your work: decide for each claim what it is. Read the function, its docstring and comments, the documentation, the tests in `project/tests` that cover it, and each caller of the function in the product (use grep). Then give one verdict:
23
+
24
+ - `real`: the result is wrong, and the product can hit it. You must name a caller (file and line) that can pass such an input in normal use, and say in one sentence what goes wrong for the user or for the product.
25
+ - `real_unreachable`: the result conflicts with the docstring, the documentation or the clear contract of the function, but no caller in the product can pass such an input today.
26
+ - `intended`: the behavior is by design. You must quote the comment, the documentation, the test, or the caller that depends on it.
27
+ - `unclear`: the evidence does not decide it. Say what decision the owner must make.
28
+
29
+ Be strict. A claim is `real` only when you can show the path from normal use of the product to the wrong result. A test in `project/tests` that asserts the current behavior is strong evidence for `intended`, unless the test itself conflicts with the documentation.
30
+
31
+ Write one file, `verdicts.json`, in the current folder: a JSON list with one entry for each claim:
32
+ {{"k": 1, "verdict": "real | real_unreachable | intended | unclear", "duplicate_of": null or the k of an earlier claim that states the same fault, "what_goes_wrong": "one sentence, or null", "caller": "path:line of the caller that reaches it, or null", "evidence": [{{"file": "path", "line": 12, "quote": "exact text"}}], "severity": "high | medium | low", "fix_idea": "one or two sentences, or null"}}
33
+
34
+ Do not change `project/`. Do not write other files, except scratch files in `.lake/`. End with `VERDICT: PASS` when `verdicts.json` has one entry for each claim."""
35
+
36
+ VERIFY_CHECK = "python3 -c \"import json;d=json.load(open('verdicts.json'));c=json.load(open('claims.json'));assert {x['k'] for x in d}=={x['k'] for x in c};assert all(x['verdict'] in ('real','real_unreachable','intended','unclear') for x in d)\""
37
+ VERDICTS = ("real", "real_unreachable", "intended", "unclear")
38
+
39
+ DEFAULT_RULES = ("Keep the style of the repository. Do not add a dependency. "
40
+ "Tests have no network.")
41
+
42
+ BUILD = """This folder is a git worktree of the product `{repo}`, at the commit `{base}`. A code review found {n} bugs in `{file}`. Two independent reviewers confirmed each one on this source: the result is wrong, and a caller in the product can reach it.
43
+
44
+ The bugs:
45
+
46
+ {bugs}
47
+
48
+ For bug `<id>` and claim `<k>`, the file `{info}/<id>_claim_<k>.py` defines `run()`, which shows the wrong result on this source (the listed "result now"). Read these files with `cat`.
49
+
50
+ Your work:
51
+ 1. Read the function, its callers and its tests. Make sure that you understand each bug. If you find that one is not a bug (the behavior is by design), do not change the code for it: say so in your report with the evidence.
52
+ 2. Fix each bug with the smallest change that corrects it. Do not change behavior that the bugs do not need. Do not refactor. Keep each existing test green: if an existing test asserts the wrong behavior, name it in your report and correct that one assertion.
53
+ 3. Write regression tests in the new file `{test_file}`: for each bug at least one test that fails on the old code and passes on the fixed code, and one test for a near input that must keep its old result.
54
+ 4. Run `{test_run}` and the existing test files that cover this module.
55
+
56
+ Rules of this repository: {rules} No emojis. Comments and docstrings in short plain sentences; a comment says why, not what. Edit `{file}` and `{test_file}`. Edit a second source file only when the same fault is in a sibling code path that must change together with it (for example a second caller that builds the same value in the same wrong way), and name each such file in your report. Edit an existing test file only when it asserts the wrong behavior. Do not commit.
57
+
58
+ The check after your work: `{check}` (each repro must give a new result, the new tests and the related tests must pass). Run it yourself.
59
+
60
+ Report: for each bug, the cause in one sentence and the change in one sentence. End with `VERDICT: PASS` when each bug is fixed or shown to be by design, else `VERDICT: PARTIAL - <what is open>`."""
61
+
62
+ JUDGE = """Review the uncommitted change in this git worktree of `{repo}` (`git diff` and the new file `{test_file}`). The change must fix these confirmed bugs in `{file}`:
63
+
64
+ {bugs}
65
+
66
+ Rubric. Give FAIL when one item fails, and list each problem as `path:line - what`:
67
+ 1. Each bug is fixed at its cause, or the report shows with evidence that it is by design. A fix that only hides the repro input is a failure.
68
+ 2. The change is minimal: no refactor, no unrelated edit, no new dependency.
69
+ 3. No behavior changes for inputs that the bugs do not cover. Check the callers of the function.
70
+ 4. Each new test asserts the correct behavior and fails on the old code (read the test and the old code, or run it against `git stash`-free means: `git show HEAD:{file}`).
71
+ 5. The code follows the rules of the repository: {rules} No emojis.
72
+ 6. No existing test was weakened, except an assertion that tested the wrong behavior of one of these bugs.
73
+
74
+ You can run `{test_run}`. Do not edit files."""
hpflow/cli.py ADDED
@@ -0,0 +1,148 @@
1
+ """The command line of hpflow. Each command prints one JSON report."""
2
+ from __future__ import annotations
3
+
4
+ import argparse
5
+ import json
6
+ import sys
7
+ from pathlib import Path
8
+ from typing import Any, Callable, Dict, List, Optional
9
+
10
+ from . import commands as c
11
+ from .common import DEFAULT_CONCURRENCY, StepError, UsageError
12
+ from .runner import PuppetmasterRunner, Runner
13
+
14
+
15
+ class Parser(argparse.ArgumentParser):
16
+ def error(self, message: str) -> None: # type: ignore[override]
17
+ raise UsageError(message)
18
+
19
+
20
+ def add_common(p: argparse.ArgumentParser) -> None:
21
+ p.add_argument("--out", required=True, help="the audit folder")
22
+ p.add_argument("--redo", action="store_true", help="run the workers again, also when a result is stored")
23
+ p.add_argument("--concurrency", type=int, default=DEFAULT_CONCURRENCY,
24
+ help=f"the most worker flows that run at one time (default {DEFAULT_CONCURRENCY})")
25
+
26
+
27
+ def add_init(p: argparse.ArgumentParser) -> None:
28
+ p.add_argument("--repo", required=True)
29
+ p.add_argument("--ref", required=True)
30
+ p.add_argument("--paths", nargs="+")
31
+ p.add_argument("--python")
32
+ p.add_argument("--test-cmd", help="template with {python} and {tests}")
33
+
34
+
35
+ def add_targets(p: argparse.ArgumentParser, required: bool = True) -> None:
36
+ g = p.add_mutually_exclusive_group(required=required)
37
+ g.add_argument("--changed", metavar="A..B")
38
+ g.add_argument("--function", nargs="+", metavar="FILE:NAME", dest="functions")
39
+ g.add_argument("--file", nargs="+", metavar="FILE", dest="files")
40
+ p.add_argument("--max-lines", type=int, default=150)
41
+ p.add_argument("--max", type=int, dest="max_n")
42
+
43
+
44
+ def add_review(p: argparse.ArgumentParser) -> None:
45
+ p.add_argument("--model", default="sonnet")
46
+ p.add_argument("--shell", dest="shell", action="store_true", default=True)
47
+ p.add_argument("--no-shell", dest="shell", action="store_false")
48
+ p.add_argument("--passes", type=int, default=1)
49
+ p.add_argument("--repro-timeout", type=float, default=60)
50
+
51
+
52
+ def add_verify(p: argparse.ArgumentParser) -> None:
53
+ p.add_argument("--models", nargs="+", default=["opus", "sonnet"])
54
+
55
+
56
+ def add_fix(p: argparse.ArgumentParser) -> None:
57
+ p.add_argument("--base")
58
+ p.add_argument("--only", nargs="+")
59
+ p.add_argument("--model", default="opus", dest="fix_model")
60
+ p.add_argument("--judge-model", default="sonnet")
61
+ p.add_argument("--rules", help="the rules of the repository for the fix worker and the judge")
62
+
63
+
64
+ def add_waive(p: argparse.ArgumentParser) -> None:
65
+ p.add_argument("--bug", required=True, help="the id of a confirmed bug")
66
+ p.add_argument("--reason", required=True, help="why the behavior stays as it is")
67
+ p.add_argument("--by", help="who decided")
68
+
69
+
70
+ def add_gate(p: argparse.ArgumentParser) -> None:
71
+ p.add_argument("--ref", help="the commit to check, for example the release candidate")
72
+ p.add_argument("--repro-timeout", type=float, default=60)
73
+
74
+
75
+ def build_parser() -> Parser:
76
+ parser = Parser(prog="hpflow", description="Take Python functions up the evidence ladder with agent workers.")
77
+ sub = parser.add_subparsers(dest="command", parser_class=Parser)
78
+ for name, adders in (("init", [add_init]), ("targets", [add_targets]), ("review", [add_review]),
79
+ ("verify", [add_verify]), ("fix", [add_fix]), ("status", []),
80
+ ("waive", [add_waive]), ("gate", [add_gate]),
81
+ ("audit", [add_init, add_targets, add_review, add_verify])):
82
+ p = sub.add_parser(name)
83
+ add_common(p)
84
+ for add in adders:
85
+ add(p)
86
+ return parser
87
+
88
+
89
+ def run_command(args: argparse.Namespace, ctx: c.Context) -> Dict[str, Any]:
90
+ cmd = args.command
91
+ if cmd == "init":
92
+ return c.cmd_init(ctx, args.repo, args.ref, args.paths, args.python, args.test_cmd)
93
+ if cmd == "targets":
94
+ return c.cmd_targets(ctx, args.changed, args.functions, args.files, args.max_lines, args.max_n)
95
+ if cmd == "review":
96
+ if args.passes < 1:
97
+ raise UsageError("--passes must be 1 or more")
98
+ return c.cmd_review(ctx, args.model, args.shell, args.passes, args.repro_timeout)
99
+ if cmd == "verify":
100
+ return c.cmd_verify(ctx, args.models)
101
+ if cmd == "fix":
102
+ return c.cmd_fix(ctx, args.base, args.only, args.fix_model, args.judge_model, args.rules)
103
+ if cmd == "status":
104
+ return c.cmd_status(ctx)
105
+ if cmd == "waive":
106
+ return c.cmd_waive(ctx, args.bug, args.reason, args.by)
107
+ if cmd == "gate":
108
+ return c.cmd_gate(ctx, args.ref, args.repro_timeout)
109
+ if cmd == "audit":
110
+ steps: List[Dict[str, Any]] = []
111
+ calls: List[Callable[[], Dict[str, Any]]] = [
112
+ lambda: c.cmd_init(ctx, args.repo, args.ref, args.paths, args.python, args.test_cmd),
113
+ lambda: c.cmd_targets(ctx, args.changed, args.functions, args.files, args.max_lines, args.max_n),
114
+ lambda: c.cmd_review(ctx, args.model, args.shell, args.passes, args.repro_timeout),
115
+ lambda: c.cmd_verify(ctx, args.models)]
116
+ for call in calls:
117
+ r = call()
118
+ steps.append(r)
119
+ if not r.get("ok"):
120
+ break
121
+ return {"command": "audit", "ok": all(s.get("ok") for s in steps) and len(steps) == len(calls), "steps": steps}
122
+ raise UsageError("give a command: init, targets, review, verify, fix, audit, status, waive or gate")
123
+
124
+
125
+ def main(argv: Optional[List[str]] = None, runner: Optional[Runner] = None) -> int:
126
+ try:
127
+ args = build_parser().parse_args(argv)
128
+ if not args.command:
129
+ raise UsageError("give a command: init, targets, review, verify, fix, audit, status, waive or gate")
130
+ if args.command == "audit" and not (args.changed or args.functions or args.files):
131
+ raise UsageError("audit needs --changed, --function or --file")
132
+ if args.concurrency < 1:
133
+ raise UsageError("--concurrency must be 1 or more")
134
+ out = Path(args.out).expanduser().resolve()
135
+ runner = runner or PuppetmasterRunner(out / "graphs", concurrency=args.concurrency)
136
+ ctx = c.Context(out, runner, redo=args.redo, concurrency=args.concurrency)
137
+ report = run_command(args, ctx)
138
+ code = 0 if report.get("ok") else 1
139
+ except UsageError as e:
140
+ report, code = {"ok": False, "usage_error": str(e)}, 2
141
+ except StepError as e:
142
+ report, code = {"ok": False, "error": str(e)}, 1
143
+ print(json.dumps(report, indent=1))
144
+ return code
145
+
146
+
147
+ if __name__ == "__main__":
148
+ sys.exit(main())