orchestra-review 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. orchestra_review-0.2.0/LICENSE +21 -0
  2. orchestra_review-0.2.0/PKG-INFO +108 -0
  3. orchestra_review-0.2.0/README.md +82 -0
  4. orchestra_review-0.2.0/orchestra_review/__init__.py +1 -0
  5. orchestra_review-0.2.0/orchestra_review/__main__.py +5 -0
  6. orchestra_review-0.2.0/orchestra_review/claude_code.py +79 -0
  7. orchestra_review-0.2.0/orchestra_review/cli.py +183 -0
  8. orchestra_review-0.2.0/orchestra_review/codex.py +110 -0
  9. orchestra_review-0.2.0/orchestra_review/detectors.py +34 -0
  10. orchestra_review-0.2.0/orchestra_review/facts.py +70 -0
  11. orchestra_review-0.2.0/orchestra_review/fixes.py +198 -0
  12. orchestra_review-0.2.0/orchestra_review/hooks.py +92 -0
  13. orchestra_review-0.2.0/orchestra_review/instructions.py +82 -0
  14. orchestra_review-0.2.0/orchestra_review/model.py +83 -0
  15. orchestra_review-0.2.0/orchestra_review/opencode.py +71 -0
  16. orchestra_review-0.2.0/orchestra_review/redact.py +21 -0
  17. orchestra_review-0.2.0/orchestra_review/report.py +99 -0
  18. orchestra_review-0.2.0/orchestra_review/share.py +95 -0
  19. orchestra_review-0.2.0/orchestra_review/state.py +43 -0
  20. orchestra_review-0.2.0/orchestra_review/sync.py +101 -0
  21. orchestra_review-0.2.0/orchestra_review.egg-info/PKG-INFO +108 -0
  22. orchestra_review-0.2.0/orchestra_review.egg-info/SOURCES.txt +31 -0
  23. orchestra_review-0.2.0/orchestra_review.egg-info/dependency_links.txt +1 -0
  24. orchestra_review-0.2.0/orchestra_review.egg-info/entry_points.txt +2 -0
  25. orchestra_review-0.2.0/orchestra_review.egg-info/requires.txt +3 -0
  26. orchestra_review-0.2.0/orchestra_review.egg-info/top_level.txt +1 -0
  27. orchestra_review-0.2.0/pyproject.toml +43 -0
  28. orchestra_review-0.2.0/setup.cfg +4 -0
  29. orchestra_review-0.2.0/tests/test_codex.py +64 -0
  30. orchestra_review-0.2.0/tests/test_fixes.py +206 -0
  31. orchestra_review-0.2.0/tests/test_review.py +115 -0
  32. orchestra_review-0.2.0/tests/test_share.py +69 -0
  33. orchestra_review-0.2.0/tests/test_sync.py +150 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 VarunTx
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,108 @@
1
+ Metadata-Version: 2.4
2
+ Name: orchestra-review
3
+ Version: 0.2.0
4
+ Summary: Finds the mistakes your coding agents repeat across sessions, from local logs, fixes each with one instruction line, and blocks them live
5
+ Author: VarunTx
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://github.com/VarunTx/orchestra-review
8
+ Project-URL: Issues, https://github.com/VarunTx/orchestra-review/issues
9
+ Keywords: claude-code,codex,opencode,coding-agents,agents,developer-tools,llm
10
+ Classifier: Development Status :: 3 - Alpha
11
+ Classifier: Environment :: Console
12
+ Classifier: Intended Audience :: Developers
13
+ Classifier: Operating System :: MacOS
14
+ Classifier: Operating System :: POSIX :: Linux
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Programming Language :: Python :: 3.11
17
+ Classifier: Programming Language :: Python :: 3.12
18
+ Classifier: Programming Language :: Python :: 3.13
19
+ Classifier: Topic :: Software Development :: Quality Assurance
20
+ Requires-Python: >=3.11
21
+ Description-Content-Type: text/markdown
22
+ License-File: LICENSE
23
+ Provides-Extra: dev
24
+ Requires-Dist: pytest>=8; extra == "dev"
25
+ Dynamic: license-file
26
+
27
+ # Orchestra Review
28
+
29
+ ![orchestra-review on the author's own sessions, run with --public so paths and names are scrubbed](docs/demo.gif)
30
+
31
+ **Your coding agent keeps making the same mistakes, because every session starts with no memory of your machine or its tools.** It calls a program that isn't installed, imports a module from the wrong Python, or trips over the same shell quirk, week after week.
32
+
33
+ Orchestra Review reads the session logs that Claude Code, Codex CLI and OpenCode already keep on your computer. It finds the mistakes that recur across sessions and fixes each with one instruction line. It can also block a known mistake before it runs, and it measures whether a fix actually worked. Nothing leaves your machine.
34
+
35
+ ```sh
36
+ pipx install orchestra-review
37
+ orchestra-review
38
+ ```
39
+
40
+ ## What it found
41
+
42
+ **In 67,074 public agent runs** (OpenHands with Qwen3-Coder on real GitHub issues from 1,823 repositories):
43
+ - **41% of the failures with an identifiable cause repeated a mistake seen in another repository.**
44
+ - **62% of runs** hit at least one of those mistakes.
45
+ - **The largest was asking to view lines past the end of a file:** 21,780 runs across 1,509 repositories.
46
+ - **Second was piping after `find -exec ... \;`,** which that shell tool rejects: 5,406 runs.
47
+ - **Built-in rules cover 59%** of those repeated failures, and the nine rules mined from this data ship with the tool.
48
+ - **Runs that hit a repeated mistake resolved their issue less often,** 44.5% against 53.6%. That's a correlation, not a measured effect.
49
+
50
+ Details and limits: [`research/`](research/).
51
+
52
+ **In the author's own logs** (as of 2026-11-16): 74 sessions across Claude Code, Codex and OpenCode, with 188 failed tool calls. **51 of those failures (27%) repeated a mistake from an earlier session.** The mistakes were all about the machine:
53
+ - `pdftoppm` missing for PDF reads (8 sessions);
54
+ - zsh treating `echo ====` as a command (7 sessions);
55
+ - `No module named 'fitz'` (7 sessions, across Claude Code and Codex);
56
+ - zsh erroring on a glob that matches nothing (5 sessions).
57
+
58
+ Six lines went into `~/.claude/CLAUDE.md` on 2026-11-13. Each line is judged only on sessions that **tried** the action it covers:
59
+
60
+ | Fix | Sessions that hit it / tried it, before | After | Verdict |
61
+ | --- | --- | --- | --- |
62
+ | zsh: a glob that matches nothing is an error | 5 / 15 | 0 / 7 | stopped |
63
+ | zsh: quote words starting with `=` | 5 / 14 | 2 / 4 | too early, still happening |
64
+ | `pdftoppm` missing for PDF reads | 8 / 9 | 0 / 0 | too early |
65
+ | `fitz`, `pypdf`, `docx` not importable | 5 / 10, 2 / 10, 2 / 4 | 0 / 0 | too early |
66
+
67
+ "Too early" means fewer than five sessions have tried the action since the line was added. One person's sessions are a small sample.
68
+
69
+ ## Usage
70
+
71
+ ```sh
72
+ orchestra-review # what your agents keep getting wrong, with a fix line for each
73
+ orchestra-review apply 6eab0c6e --to ~/.claude/CLAUDE.md # add one line, with a timed marker
74
+ orchestra-review sync # add rule-based lines for mistakes in 3+ sessions, retire stale ones
75
+ orchestra-review undo 6eab0c6e # remove a line; it will never be added back automatically
76
+ orchestra-review hook install # Claude Code: block known mistakes live, and sync when a session ends
77
+ orchestra-review --public # scrub paths, file names, URLs and emails, for screenshots
78
+ orchestra-review --json # everything, as JSON
79
+ ```
80
+
81
+ ## How it works
82
+
83
+ - **Finding mistakes.** A recurring mistake is a failure in two or more sessions that shares the same last error line, ignoring numbers and ids. Some failures are left out:
84
+ - test runs that fail, because the agent is checking its work;
85
+ - calls refused by a permission rule or interrupted by you, which are counted as blocked;
86
+ - error lines that say nothing about the cause, such as a bare exit code or a path.
87
+ - **Fix lines.** Lines come from built-in rules: missing programs and Python modules, zsh `=` and glob quirks, old-Python syntax, and misused file-viewing and editing tools. For other causes, a local model you run yourself (Ollama, `--model`) can draft a line. A draft is kept only if it names the cause and contains no URL or command that downloads or deletes anything, because log text is untrusted and the line ends up in a file your agent obeys. Drafted lines are never added automatically.
88
+ - **`sync`.** It adds a rule-based line once a mistake has hit three sessions. If the mistake was seen in only one project, the line goes into that project's instruction file, but only if the file already exists; otherwise it goes into the agent's global file. Before keeping a line, `sync` re-checks its fact on this machine: is the program still missing, can the module still not be imported, is the shell still zsh? When a fact stops being true, for example after you install `poppler`, the line is removed and logged. A line you delete by hand, or remove with `undo`, is never added back.
89
+ - **The hook.** `hook install` adds two Claude Code hooks to `~/.claude/settings.json`, keeping a `.bak` copy of the original:
90
+ - **Before a tool call** that would repeat a known mistake (running a missing program, importing a missing module, an unquoted zsh `=word`, or reading a PDF without `pdftoppm`), the hook re-checks the fact and blocks the call with the fix as the reason. The agent corrects course without the failed step. Glob mistakes are never blocked, because whether a glob matches depends on the directory at run time.
91
+ - **When a session ends,** the other hook runs `sync`.
92
+
93
+ Any error in either hook lets the call through.
94
+ - **Measurement.** Before-and-after counts only sessions that tried the relevant action, so a change of task can't look like a cure.
95
+ - **Privacy.** Everything is read locally. Nothing is sent anywhere unless you point `--model` at a model server. `--public` and `share` drop paths, file names, project names and commands.
96
+
97
+ ## Scope
98
+
99
+ - **Reads:**
100
+ - Claude Code, `~/.claude/projects`, including subagents;
101
+ - Codex CLI, `~/.codex/sessions`;
102
+ - OpenCode, through `opencode session export`, for a project directory.
103
+
104
+ Cursor is not supported.
105
+ - **Runs on:** macOS and Linux with Python 3.11 or later, with no dependencies. Windows is untested.
106
+ - **Maintained:** best-effort. Issues are welcome; there are no promised response times.
107
+
108
+ Changes: [`CHANGELOG.md`](CHANGELOG.md). Design notes: [`docs/design.md`](docs/design.md). MIT licensed.
@@ -0,0 +1,82 @@
1
+ # Orchestra Review
2
+
3
+ ![orchestra-review on the author's own sessions, run with --public so paths and names are scrubbed](docs/demo.gif)
4
+
5
+ **Your coding agent keeps making the same mistakes, because every session starts with no memory of your machine or its tools.** It calls a program that isn't installed, imports a module from the wrong Python, or trips over the same shell quirk, week after week.
6
+
7
+ Orchestra Review reads the session logs that Claude Code, Codex CLI and OpenCode already keep on your computer. It finds the mistakes that recur across sessions and fixes each with one instruction line. It can also block a known mistake before it runs, and it measures whether a fix actually worked. Nothing leaves your machine.
8
+
9
+ ```sh
10
+ pipx install orchestra-review
11
+ orchestra-review
12
+ ```
13
+
14
+ ## What it found
15
+
16
+ **In 67,074 public agent runs** (OpenHands with Qwen3-Coder on real GitHub issues from 1,823 repositories):
17
+ - **41% of the failures with an identifiable cause repeated a mistake seen in another repository.**
18
+ - **62% of runs** hit at least one of those mistakes.
19
+ - **The largest was asking to view lines past the end of a file:** 21,780 runs across 1,509 repositories.
20
+ - **Second was piping after `find -exec ... \;`,** which that shell tool rejects: 5,406 runs.
21
+ - **Built-in rules cover 59%** of those repeated failures, and the nine rules mined from this data ship with the tool.
22
+ - **Runs that hit a repeated mistake resolved their issue less often,** 44.5% against 53.6%. That's a correlation, not a measured effect.
23
+
24
+ Details and limits: [`research/`](research/).
25
+
26
+ **In the author's own logs** (as of 2026-11-16): 74 sessions across Claude Code, Codex and OpenCode, with 188 failed tool calls. **51 of those failures (27%) repeated a mistake from an earlier session.** The mistakes were all about the machine:
27
+ - `pdftoppm` missing for PDF reads (8 sessions);
28
+ - zsh treating `echo ====` as a command (7 sessions);
29
+ - `No module named 'fitz'` (7 sessions, across Claude Code and Codex);
30
+ - zsh erroring on a glob that matches nothing (5 sessions).
31
+
32
+ Six lines went into `~/.claude/CLAUDE.md` on 2026-11-13. Each line is judged only on sessions that **tried** the action it covers:
33
+
34
+ | Fix | Sessions that hit it / tried it, before | After | Verdict |
35
+ | --- | --- | --- | --- |
36
+ | zsh: a glob that matches nothing is an error | 5 / 15 | 0 / 7 | stopped |
37
+ | zsh: quote words starting with `=` | 5 / 14 | 2 / 4 | too early, still happening |
38
+ | `pdftoppm` missing for PDF reads | 8 / 9 | 0 / 0 | too early |
39
+ | `fitz`, `pypdf`, `docx` not importable | 5 / 10, 2 / 10, 2 / 4 | 0 / 0 | too early |
40
+
41
+ "Too early" means fewer than five sessions have tried the action since the line was added. One person's sessions are a small sample.
42
+
43
+ ## Usage
44
+
45
+ ```sh
46
+ orchestra-review # what your agents keep getting wrong, with a fix line for each
47
+ orchestra-review apply 6eab0c6e --to ~/.claude/CLAUDE.md # add one line, with a timed marker
48
+ orchestra-review sync # add rule-based lines for mistakes in 3+ sessions, retire stale ones
49
+ orchestra-review undo 6eab0c6e # remove a line; it will never be added back automatically
50
+ orchestra-review hook install # Claude Code: block known mistakes live, and sync when a session ends
51
+ orchestra-review --public # scrub paths, file names, URLs and emails, for screenshots
52
+ orchestra-review --json # everything, as JSON
53
+ ```
54
+
55
+ ## How it works
56
+
57
+ - **Finding mistakes.** A recurring mistake is a failure in two or more sessions that shares the same last error line, ignoring numbers and ids. Some failures are left out:
58
+ - test runs that fail, because the agent is checking its work;
59
+ - calls refused by a permission rule or interrupted by you, which are counted as blocked;
60
+ - error lines that say nothing about the cause, such as a bare exit code or a path.
61
+ - **Fix lines.** Lines come from built-in rules: missing programs and Python modules, zsh `=` and glob quirks, old-Python syntax, and misused file-viewing and editing tools. For other causes, a local model you run yourself (Ollama, `--model`) can draft a line. A draft is kept only if it names the cause and contains no URL or command that downloads or deletes anything, because log text is untrusted and the line ends up in a file your agent obeys. Drafted lines are never added automatically.
62
+ - **`sync`.** It adds a rule-based line once a mistake has hit three sessions. If the mistake was seen in only one project, the line goes into that project's instruction file, but only if the file already exists; otherwise it goes into the agent's global file. Before keeping a line, `sync` re-checks its fact on this machine: is the program still missing, can the module still not be imported, is the shell still zsh? When a fact stops being true, for example after you install `poppler`, the line is removed and logged. A line you delete by hand, or remove with `undo`, is never added back.
63
+ - **The hook.** `hook install` adds two Claude Code hooks to `~/.claude/settings.json`, keeping a `.bak` copy of the original:
64
+ - **Before a tool call** that would repeat a known mistake (running a missing program, importing a missing module, an unquoted zsh `=word`, or reading a PDF without `pdftoppm`), the hook re-checks the fact and blocks the call with the fix as the reason. The agent corrects course without the failed step. Glob mistakes are never blocked, because whether a glob matches depends on the directory at run time.
65
+ - **When a session ends,** the other hook runs `sync`.
66
+
67
+ Any error in either hook lets the call through.
68
+ - **Measurement.** Before-and-after counts only sessions that tried the relevant action, so a change of task can't look like a cure.
69
+ - **Privacy.** Everything is read locally. Nothing is sent anywhere unless you point `--model` at a model server. `--public` and `share` drop paths, file names, project names and commands.
70
+
71
+ ## Scope
72
+
73
+ - **Reads:**
74
+ - Claude Code, `~/.claude/projects`, including subagents;
75
+ - Codex CLI, `~/.codex/sessions`;
76
+ - OpenCode, through `opencode session export`, for a project directory.
77
+
78
+ Cursor is not supported.
79
+ - **Runs on:** macOS and Linux with Python 3.11 or later, with no dependencies. Windows is untested.
80
+ - **Maintained:** best-effort. Issues are welcome; there are no promised response times.
81
+
82
+ Changes: [`CHANGELOG.md`](CHANGELOG.md). Design notes: [`docs/design.md`](docs/design.md). MIT licensed.
@@ -0,0 +1 @@
1
+ """Finds the mistakes your coding agent repeats across sessions, from local logs."""
@@ -0,0 +1,5 @@
1
+ import sys
2
+
3
+ from .cli import main
4
+
5
+ sys.exit(main())
@@ -0,0 +1,79 @@
1
+ """Claude Code keeps one JSONL transcript per session (and per subagent) under ~/.claude/projects."""
2
+ import json
3
+ import os
4
+ from pathlib import Path
5
+ import re
6
+
7
+ from .model import Session,Step,digest,iso_time,kind_of,tail,target_of
8
+
9
+ EXIT=re.compile(r'^Exit code (\d+)')
10
+ # Refusals by the harness or the user are not the agent failing at its task.
11
+ BLOCKED=('<tool_use_error>Blocked','Permission for this command was denied','The server-side auto mode classifier',
12
+ "The user doesn't want to proceed")
13
+
14
+
15
+ def default_root():
16
+ base=os.environ.get('CLAUDE_CONFIG_DIR')
17
+ return Path(base)/'projects' if base else Path.home()/'.claude'/'projects'
18
+
19
+
20
+ def load(root):
21
+ root=Path(root)
22
+ return [read(path) for path in sorted(root.rglob('*.jsonl'))] if root.is_dir() else []
23
+
24
+
25
+ def result_text(content):
26
+ if isinstance(content,str): return content
27
+ if isinstance(content,list):
28
+ return '\n'.join(p.get('text','') for p in content if isinstance(p,dict) and p.get('type')=='text')
29
+ return None
30
+
31
+
32
+ def read(path):
33
+ path=Path(path)
34
+ # Subagents write their own file; keep them apart so their steps never interleave with the parent's.
35
+ sid=path.stem if path.parent.name!='subagents' else path.parents[1].name+'/'+path.stem
36
+ s=Session(source='claude-code',id=sid)
37
+ pending={}
38
+ with path.open(encoding='utf-8',errors='replace') as lines:
39
+ for line in lines:
40
+ try: entry=json.loads(line)
41
+ except ValueError: s.skipped_lines+=1;continue
42
+ if not isinstance(entry,dict): s.skipped_lines+=1;continue
43
+ kind=entry.get('type');at=iso_time(entry.get('timestamp'))
44
+ if kind in ('assistant','user'):
45
+ s.seen(at)
46
+ if entry.get('cwd') and not s.project: s.project=entry['cwd']
47
+ message=entry.get('message') if isinstance(entry.get('message'),dict) else {}
48
+ if kind=='assistant':
49
+ model=message.get('model')
50
+ if model and model!='<synthetic>': s.model(model)
51
+ for part in message.get('content') or []:
52
+ if isinstance(part,dict) and part.get('type')=='tool_use':
53
+ tool=str(part.get('name',''))
54
+ step=Step(index=len(s.steps)+1,tool=tool,kind=kind_of(tool),target=target_of(tool,part.get('input')),at=at)
55
+ s.steps.append(step);pending[part.get('id')]=step
56
+ elif kind=='user':
57
+ content=message.get('content')
58
+ if isinstance(content,str):
59
+ if not entry.get('isMeta') and not entry.get('isCompactSummary'): s.prompts+=1
60
+ continue
61
+ for part in content or []:
62
+ if not isinstance(part,dict): continue
63
+ if part.get('type')=='tool_result' and part.get('tool_use_id') in pending:
64
+ step=pending.pop(part['tool_use_id'])
65
+ text=result_text(part.get('content'))
66
+ step.digest=digest(text)
67
+ result=entry.get('toolUseResult') if isinstance(entry.get('toolUseResult'),dict) else {}
68
+ if entry.get('toolDenialKind') or result.get('interrupted') or (text or '').startswith(BLOCKED):
69
+ step.blocked=True;continue
70
+ step.ok=not part.get('is_error')
71
+ if not step.ok:
72
+ step.error=tail(text)
73
+ code=EXIT.match(text or '')
74
+ if code: step.exit_code=int(code.group(1))
75
+ elif part.get('type')=='text' and not entry.get('isMeta') and not entry.get('toolUseResult'):
76
+ s.prompts+=1
77
+ elif kind=='custom-title' and entry.get('customTitle'):
78
+ s.title=entry['customTitle']
79
+ return s
@@ -0,0 +1,183 @@
1
+ """Find the mistakes your coding agents repeat across sessions. Nothing leaves this machine.
2
+
3
+ Reads Claude Code transcripts (~/.claude/projects), Codex CLI rollouts (~/.codex/sessions) and OpenCode
4
+ sessions for the given project directories (default: the current directory).
5
+
6
+ orchestra-review report, with a suggested instruction line per recurring mistake
7
+ orchestra-review apply ID --to FILE append those lines to an instruction file, dated for later measurement
8
+ orchestra-review sync add rule-based lines for mistakes in 3+ sessions, retire lines whose fact changed
9
+ orchestra-review undo ID remove a line and never add it back automatically
10
+ orchestra-review hook install block known mistakes live in Claude Code, and sync when a session ends
11
+ orchestra-review share > mine.json counts and scrubbed causes only, safe to send to someone else
12
+ orchestra-review combine a.json b.json which mistakes recur for how many people"""
13
+ import argparse
14
+ import importlib.metadata
15
+ from datetime import datetime,timedelta,timezone
16
+ import json
17
+ import os
18
+ import re
19
+ import sys
20
+
21
+ from . import claude_code,codex,opencode
22
+ from .detectors import recurring
23
+ from .fixes import suggest
24
+ from . import hooks,share
25
+ from . import state as st
26
+ from .instructions import apply
27
+ from .sync import record,sync,undo
28
+ from .report import build,render
29
+
30
+
31
+ def version():
32
+ try: return importlib.metadata.version('orchestra-review')
33
+ except importlib.metadata.PackageNotFoundError: return 'unknown'
34
+
35
+
36
+ def since(value):
37
+ if value=='all': return None
38
+ match=re.fullmatch(r'(\d+)([dh])',value)
39
+ if not match: raise argparse.ArgumentTypeError('use a number of days or hours, such as 30d or 12h, or all')
40
+ amount=int(match.group(1))
41
+ return datetime.now(timezone.utc)-(timedelta(days=amount) if match.group(2)=='d' else timedelta(hours=amount))
42
+
43
+
44
+ def collect(args):
45
+ sessions=[]
46
+ if args.source in ('all','claude-code'): sessions+=claude_code.load(args.claude_dir)
47
+ if args.source in ('all','codex'): sessions+=codex.load(args.codex_dir)
48
+ if args.source in ('all','opencode'):
49
+ for path in args.opencode_export: sessions+=opencode.load_exports(path)
50
+ if not args.opencode_export:
51
+ for directory in args.opencode_project or ['.']: sessions+=opencode.load_project(directory)
52
+ if args.since: sessions=[s for s in sessions if s.ended and s.ended>=args.since]
53
+ if args.session: sessions=[s for s in sessions if s.id.startswith(args.session)]
54
+ return [s for s in sessions if s.steps]
55
+
56
+
57
+ def sources(parser):
58
+ parser.add_argument('--source',choices=['all','claude-code','codex','opencode'],default='all')
59
+ parser.add_argument('--since',type=since,default=None,help='only sessions active within this window, such as 30d (default: all)')
60
+ parser.add_argument('--claude-dir',default=claude_code.default_root(),help='Claude Code projects directory')
61
+ parser.add_argument('--codex-dir',default=codex.default_root(),help='Codex CLI sessions directory')
62
+ parser.add_argument('--opencode-project',action='append',help='project directory whose OpenCode sessions to read; repeatable')
63
+ parser.add_argument('--opencode-export',action='append',default=[],help='OpenCode export file or directory of them; repeatable')
64
+ parser.add_argument('--session',help='only sessions whose id starts with this')
65
+ parser.add_argument('--model',help='local Ollama model that drafts a line when no rule matches, such as qwen3:4b')
66
+ parser.add_argument('--model-url',default=os.environ.get('OLLAMA_HOST','http://127.0.0.1:11434'))
67
+
68
+
69
+ def main(argv=None):
70
+ argv=sys.argv[1:] if argv is None else argv
71
+ if argv[:1]==['apply']: return apply_main(argv[1:])
72
+ if argv[:1]==['share']: return share_main(argv[1:])
73
+ if argv[:1]==['combine']: return combine_main(argv[1:])
74
+ if argv[:1]==['sync']: return sync_main(argv[1:])
75
+ if argv[:1]==['undo']: return undo_main(argv[1:])
76
+ if argv[:1]==['hook']: return hook_main(argv[1:])
77
+ parser=argparse.ArgumentParser(prog='orchestra-review',description=__doc__,formatter_class=argparse.RawDescriptionHelpFormatter)
78
+ parser.add_argument('--version',action='version',version=f'orchestra-review {version()}')
79
+ sources(parser)
80
+ parser.add_argument('--instructions',action='append',default=[],help='extra instruction file to measure fixes in; repeatable')
81
+ parser.add_argument('--top',type=int,default=10)
82
+ parser.add_argument('--json',action='store_true',help='print the full result as JSON')
83
+ parser.add_argument('--public',action='store_true',help='scrub paths, file names, URLs and emails from the output, for screenshots')
84
+ args=parser.parse_args(argv)
85
+ data=build(collect(args),top=args.top,model=args.model,model_url=args.model_url,instruction_files=args.instructions,public=args.public)
86
+ print(json.dumps(data,indent=2) if args.json else render(data))
87
+ return 0
88
+
89
+
90
+ def apply_main(argv):
91
+ parser=argparse.ArgumentParser(prog='orchestra-review apply',
92
+ description='Append suggested lines to an instruction file, each with a dated marker.')
93
+ parser.add_argument('ids',nargs='+',help='fix ids from the report')
94
+ parser.add_argument('--to',required=True,help='instruction file, such as ~/.claude/CLAUDE.md or ./AGENTS.md')
95
+ parser.add_argument('--include-drafted',action='store_true',
96
+ help='also write lines drafted by a model; by default only rule-based lines are written')
97
+ sources(parser)
98
+ args=parser.parse_args(argv)
99
+ fixes,keys={},{}
100
+ for key,hits in recurring(collect(args)):
101
+ fix=suggest(key,hits,args.model,args.model_url)
102
+ fixes[fix.id]=fix;keys[fix.id]=key
103
+ chosen=[]
104
+ for fid in args.ids:
105
+ fix=fixes.get(fid)
106
+ if not fix or not fix.line: print(f'{fid}: no suggested line for this id',file=sys.stderr);return 1
107
+ if fix.source=='model' and not args.include_drafted:
108
+ print(f'{fid}: drafted by a model; review it and pass --include-drafted to write it',file=sys.stderr);return 1
109
+ chosen.append(fix)
110
+ path=os.path.expanduser(args.to)
111
+ written=apply(path,chosen)
112
+ state=st.load()
113
+ for fix in chosen:
114
+ record(state,fix,keys[fix.id],path,'manual')
115
+ if fix.id in written: st.log(state,'added',fix.id,path,fix.line)
116
+ st.save(state)
117
+ for fix in chosen: print(('added ' if fix.id in written else 'present ')+f'[{fix.id}] {fix.line}')
118
+ print(f'→ {path}')
119
+ return 0
120
+
121
+
122
+ def sync_main(argv):
123
+ parser=argparse.ArgumentParser(prog='orchestra-review sync',description='Add rule-based lines for mistakes seen in 3 or more '
124
+ 'sessions, retire lines whose fact no longer holds, and respect lines you removed.')
125
+ parser.add_argument('--dry-run',action='store_true',help='show what would change without writing anything')
126
+ sources(parser)
127
+ args=parser.parse_args(argv)
128
+ changes=sync(collect(args),dry_run=args.dry_run)
129
+ for action,fid,path,detail in changes: print(f'{action:<16} [{fid}] {path}' + (f' — {detail}' if detail else ''))
130
+ if not changes: print('Nothing to change.')
131
+ return 0
132
+
133
+
134
+ def undo_main(argv):
135
+ parser=argparse.ArgumentParser(prog='orchestra-review undo',description='Remove a line and never add it back automatically.')
136
+ parser.add_argument('id')
137
+ args=parser.parse_args(argv)
138
+ removed=undo(args.id)
139
+ print(f'removed [{args.id}] from {", ".join(removed)}' if removed else f'[{args.id}] was not in any known instruction file; it will not be added automatically')
140
+ return 0
141
+
142
+
143
+ def hook_main(argv):
144
+ parser=argparse.ArgumentParser(prog='orchestra-review hook',description='Claude Code hooks.')
145
+ parser.add_argument('action',choices=['install','uninstall',hooks.PRE,hooks.END])
146
+ parser.add_argument('--settings',default=os.path.expanduser('~/.claude/settings.json'),help='Claude Code settings file')
147
+ sources(parser)
148
+ args=parser.parse_args(argv)
149
+ if args.action==hooks.PRE: return hooks.pre_tool_use()
150
+ if args.action==hooks.END: return hooks.session_end(lambda:collect(args))
151
+ if args.action=='install':
152
+ hooks.install(args.settings)
153
+ print(f'Installed the PreToolUse and SessionEnd hooks in {args.settings} (original saved as {args.settings}.bak).')
154
+ print('Run `orchestra-review sync` once so the hook knows which mistakes to block.')
155
+ return 0
156
+ print('Removed the orchestra-review hooks.' if hooks.uninstall(args.settings) else 'No orchestra-review hooks were installed.')
157
+ return 0
158
+
159
+
160
+
161
+ def share_main(argv):
162
+ parser=argparse.ArgumentParser(prog='orchestra-review share',
163
+ description='Print a share (JSON) on stdout and a preview of everything in it on stderr.')
164
+ sources(parser)
165
+ args=parser.parse_args(argv)
166
+ data=share.summary(collect(args))
167
+ print(share.preview(data),file=sys.stderr)
168
+ print(json.dumps(data,indent=2))
169
+ return 0
170
+
171
+
172
+ def combine_main(argv):
173
+ parser=argparse.ArgumentParser(prog='orchestra-review combine',description='Combine shares from several people.')
174
+ parser.add_argument('files',nargs='+')
175
+ parser.add_argument('--json',action='store_true')
176
+ args=parser.parse_args(argv)
177
+ try: data=share.combine(share.load(args.files))
178
+ except (OSError,ValueError) as error: print(error,file=sys.stderr);return 1
179
+ print(json.dumps(data,indent=2) if args.json else share.render_combined(data))
180
+ return 0
181
+
182
+
183
+ if __name__=='__main__': sys.exit(main())
@@ -0,0 +1,110 @@
1
+ """Codex CLI keeps one JSONL rollout per session under ~/.codex/sessions/YYYY/MM/DD.
2
+
3
+ Shell calls appear in two shapes, depending on the Codex version: a `function_call` (shell, shell_command,
4
+ exec_command) whose output carries an exit code, or a `custom_tool_call` named `exec` whose input is a short
5
+ script calling tools.exec_command({cmd: ...}) and whose output holds JSON chunks with an exit_code."""
6
+ import json
7
+ import os
8
+ from pathlib import Path
9
+ import re
10
+
11
+ from .model import Session,Step,digest,iso_time,tail
12
+
13
+ SHELL_FUNCTIONS={'shell','shell_command','exec_command','local_shell','container.exec'}
14
+ CMD=re.compile(r'\bcmd\s*:\s*"((?:[^"\\]|\\.)*)"')
15
+ EXIT=re.compile(r'"exit_code"\s*:\s*(-?\d+)|Exit code:?\s*(-?\d+)|"exit_code\\?"\s*:\s*(-?\d+)')
16
+
17
+
18
+ def default_root():
19
+ return Path(os.environ.get('CODEX_HOME') or Path.home()/'.codex')/'sessions'
20
+
21
+
22
+ def load(root):
23
+ root=Path(root)
24
+ return [read(path) for path in sorted(root.rglob('*.jsonl'))] if root.is_dir() else []
25
+
26
+
27
+ def text_of(output):
28
+ """Outputs are a string, a JSON string, or a list of {type: input_text, text} parts."""
29
+ if isinstance(output,list): return '\n'.join(p.get('text','') for p in output if isinstance(p,dict))
30
+ if isinstance(output,dict): return json.dumps(output)
31
+ return str(output or '')
32
+
33
+
34
+ def command_of(name,arguments):
35
+ try: data=json.loads(arguments) if isinstance(arguments,str) else (arguments or {})
36
+ except ValueError: return None
37
+ command=data.get('command') if isinstance(data,dict) else None
38
+ if command is None and isinstance(data,dict): command=data.get('cmd')
39
+ if isinstance(command,list):
40
+ # ["bash", "-lc", "real command"] is the common wrapper; keep the real command.
41
+ command=command[-1] if len(command)>=3 and command[1] in ('-lc','-c') else ' '.join(map(str,command))
42
+ return ' '.join(str(command).split()) if command else None
43
+
44
+
45
+ def outcome(text):
46
+ """ok and exit code from every exit code in the output: any non-zero means the call failed."""
47
+ codes=[int(next(g for g in m.groups() if g is not None)) for m in EXIT.finditer(text)]
48
+ if not codes: return None,None
49
+ bad=[c for c in codes if c!=0]
50
+ return (False,bad[0]) if bad else (True,0)
51
+
52
+
53
+ def chunks(text):
54
+ """JSON chunks from an `exec` output: each sits on its own line after the "Output:" header."""
55
+ for line in text.splitlines():
56
+ if line.startswith('{') and '"exit_code"' in line:
57
+ try: data=json.loads(line)
58
+ except ValueError: continue # truncated by Codex
59
+ # Parallel calls arrive wrapped as {"i": n, "status": ..., "value": {chunk}}.
60
+ yield data.get('value') if isinstance(data.get('value'),dict) else data
61
+
62
+
63
+ def error_text(text):
64
+ """The `output` field of the failing chunk, or of a JSON output; otherwise the raw tail."""
65
+ for chunk in chunks(text):
66
+ if chunk.get('exit_code') not in (0,None): return tail(str(chunk.get('output','')))
67
+ if '"chunk_id"' in text: return None # the failing chunk was truncated: no cause rather than a wrong one
68
+ try:
69
+ data=json.loads(text)
70
+ if isinstance(data,dict) and 'output' in data: return tail(str(data['output']))
71
+ except ValueError: pass
72
+ return tail(text)
73
+
74
+
75
+ def read(path):
76
+ path=Path(path)
77
+ s=Session(source='codex',id=path.stem.split('-',6)[-1] if path.stem.startswith('rollout-') else path.stem)
78
+ pending={}
79
+ with path.open(encoding='utf-8',errors='replace') as lines:
80
+ for line in lines:
81
+ try: entry=json.loads(line)
82
+ except ValueError: s.skipped_lines+=1;continue
83
+ if not isinstance(entry,dict): s.skipped_lines+=1;continue
84
+ at=iso_time(entry.get('timestamp'));payload=entry.get('payload')
85
+ if not isinstance(payload,dict): continue
86
+ kind=entry.get('type');ptype=payload.get('type')
87
+ if kind=='session_meta':
88
+ s.project=s.project or payload.get('cwd');s.id=str(payload.get('id') or s.id)
89
+ elif kind=='turn_context':
90
+ s.model(payload.get('model'));s.project=s.project or payload.get('cwd')
91
+ if kind!='response_item': continue
92
+ s.seen(at)
93
+ if ptype=='message' and payload.get('role')=='user': s.prompts+=1
94
+ elif ptype in ('function_call','custom_tool_call'):
95
+ name=str(payload.get('name',''))
96
+ if ptype=='custom_tool_call' and name=='exec':
97
+ cmds=[bytes(c,'utf-8').decode('unicode_escape') for c in CMD.findall(str(payload.get('input','')))]
98
+ command=' ; '.join(' '.join(c.split()) for c in cmds) or None
99
+ else: command=command_of(name,payload.get('arguments')) if name in SHELL_FUNCTIONS else None
100
+ step=Step(index=len(s.steps)+1,tool=name,kind='shell' if command else ('edit' if name=='apply_patch' else 'other'),
101
+ target=command or name,at=at)
102
+ s.steps.append(step);pending[payload.get('call_id')]=step
103
+ elif ptype in ('function_call_output','custom_tool_call_output') and payload.get('call_id') in pending:
104
+ step=pending.pop(payload['call_id'])
105
+ text=text_of(payload.get('output'))
106
+ step.digest=digest(text)
107
+ if step.kind!='shell': continue # only shell calls report an outcome Codex records reliably
108
+ step.ok,step.exit_code=outcome(text)
109
+ if step.ok is False: step.error=error_text(text)
110
+ return s
@@ -0,0 +1,34 @@
1
+ """Groups failures into mistakes: two failures are the same mistake when their last error line matches."""
2
+ from collections import defaultdict
3
+ import re
4
+
5
+ # Test runs that fail are the agent checking its work, not a mistake: reproduce scripts fail on purpose.
6
+ TEST_OUTCOME=re.compile(r'^(?:AssertionError\b.*|FAILED \(.*|=* ?# failed\b.*|.*\btests? failed\b.*|.*Issue reproduced.*|KeyboardInterrupt|Ran # tests? in .*|# deselected in .*|E +.*)$',re.I)
7
+ VOLATILE=re.compile(r'[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}|\d+',re.I)
8
+
9
+
10
+ def signature(step):
11
+ """Two failures are the same mistake when their last error line matches, ignoring ids and numbers.
12
+ The last line is where shells and tracebacks put the actual cause."""
13
+ last=next((line.strip() for line in reversed((step.error or '').splitlines()) if line.strip()),'')
14
+ last=re.sub(r'={2,}','==',VOLATILE.sub('#',last.lstrip('…')))[:160]
15
+ if len(last)<12 or re.fullmatch(r'Exit(ed with)? code #',last): return None # says nothing about the cause
16
+ if re.fullmatch(r'[~/]\S*',last): return None # a bare path is output that happened to come last, not a cause
17
+ if TEST_OUTCOME.match(last): return None
18
+ return (step.kind,last)
19
+
20
+
21
+ def by_signature(sessions):
22
+ """Every failed step with a usable signature, grouped by signature and then by session."""
23
+ seen=defaultdict(lambda:defaultdict(list))
24
+ for s in sessions:
25
+ for step in s.steps:
26
+ key=signature(step) if step.ok is False else None
27
+ if key: seen[key][(s.source,s.id)].append(step)
28
+ return seen
29
+
30
+
31
+ def recurring(sessions,minimum=2):
32
+ """Mistakes seen in at least `minimum` sessions, with every failed step that matched them."""
33
+ return sorted(((key,hits) for key,hits in by_signature(sessions).items() if len(hits)>=minimum),
34
+ key=lambda item:(-len(item[1]),-sum(map(len,item[1].values()))))