debrief-cli 0.1.1__tar.gz → 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. {debrief_cli-0.1.1 → debrief_cli-0.2.0}/PKG-INFO +20 -4
  2. {debrief_cli-0.1.1 → debrief_cli-0.2.0}/README.md +19 -3
  3. {debrief_cli-0.1.1 → debrief_cli-0.2.0}/debrief/cli.py +2 -0
  4. {debrief_cli-0.1.1 → debrief_cli-0.2.0}/debrief/export_site.py +1 -87
  5. debrief_cli-0.2.0/debrief/push.py +237 -0
  6. debrief_cli-0.2.0/debrief/redact.py +195 -0
  7. {debrief_cli-0.1.1 → debrief_cli-0.2.0}/pyproject.toml +1 -1
  8. debrief_cli-0.2.0/tests/test_push.py +288 -0
  9. debrief_cli-0.2.0/tests/test_redact.py +233 -0
  10. debrief_cli-0.1.1/tests/test_export_site.py +0 -101
  11. {debrief_cli-0.1.1 → debrief_cli-0.2.0}/.gitignore +0 -0
  12. {debrief_cli-0.1.1 → debrief_cli-0.2.0}/LICENSE +0 -0
  13. {debrief_cli-0.1.1 → debrief_cli-0.2.0}/debrief/__init__.py +0 -0
  14. {debrief_cli-0.1.1 → debrief_cli-0.2.0}/debrief/build_artifact.py +0 -0
  15. {debrief_cli-0.1.1 → debrief_cli-0.2.0}/debrief/classify.py +0 -0
  16. {debrief_cli-0.1.1 → debrief_cli-0.2.0}/debrief/compare_definitions.py +0 -0
  17. {debrief_cli-0.1.1 → debrief_cli-0.2.0}/debrief/compare_llm.py +0 -0
  18. {debrief_cli-0.1.1 → debrief_cli-0.2.0}/debrief/config.py +0 -0
  19. {debrief_cli-0.1.1 → debrief_cli-0.2.0}/debrief/db.py +0 -0
  20. {debrief_cli-0.1.1 → debrief_cli-0.2.0}/debrief/export.py +0 -0
  21. {debrief_cli-0.1.1 → debrief_cli-0.2.0}/debrief/github.py +0 -0
  22. {debrief_cli-0.1.1 → debrief_cli-0.2.0}/debrief/normalize.py +0 -0
  23. {debrief_cli-0.1.1 → debrief_cli-0.2.0}/debrief/paths.py +0 -0
  24. {debrief_cli-0.1.1 → debrief_cli-0.2.0}/debrief/prune.py +0 -0
  25. {debrief_cli-0.1.1 → debrief_cli-0.2.0}/debrief/recommend.py +0 -0
  26. {debrief_cli-0.1.1 → debrief_cli-0.2.0}/debrief/relay.py +0 -0
  27. {debrief_cli-0.1.1 → debrief_cli-0.2.0}/debrief/runner.py +0 -0
  28. {debrief_cli-0.1.1 → debrief_cli-0.2.0}/debrief/scrape.py +0 -0
  29. {debrief_cli-0.1.1 → debrief_cli-0.2.0}/debrief/site/app.js +0 -0
  30. {debrief_cli-0.1.1 → debrief_cli-0.2.0}/debrief/site/index.html +0 -0
  31. {debrief_cli-0.1.1 → debrief_cli-0.2.0}/debrief/site/method.js +0 -0
  32. {debrief_cli-0.1.1 → debrief_cli-0.2.0}/debrief/site/style.css +0 -0
  33. {debrief_cli-0.1.1 → debrief_cli-0.2.0}/debrief/skills.py +0 -0
  34. {debrief_cli-0.1.1 → debrief_cli-0.2.0}/debrief/sources.py +0 -0
  35. {debrief_cli-0.1.1 → debrief_cli-0.2.0}/debrief/synthesize.py +0 -0
  36. {debrief_cli-0.1.1 → debrief_cli-0.2.0}/debrief/verify_evidence.py +0 -0
  37. {debrief_cli-0.1.1 → debrief_cli-0.2.0}/tests/conftest.py +0 -0
  38. {debrief_cli-0.1.1 → debrief_cli-0.2.0}/tests/test_classify.py +0 -0
  39. {debrief_cli-0.1.1 → debrief_cli-0.2.0}/tests/test_cli.py +0 -0
  40. {debrief_cli-0.1.1 → debrief_cli-0.2.0}/tests/test_config.py +0 -0
  41. {debrief_cli-0.1.1 → debrief_cli-0.2.0}/tests/test_export_site_db.py +0 -0
  42. {debrief_cli-0.1.1 → debrief_cli-0.2.0}/tests/test_github.py +0 -0
  43. {debrief_cli-0.1.1 → debrief_cli-0.2.0}/tests/test_paths.py +0 -0
  44. {debrief_cli-0.1.1 → debrief_cli-0.2.0}/tests/test_relay.py +0 -0
  45. {debrief_cli-0.1.1 → debrief_cli-0.2.0}/tests/test_sources.py +0 -0
  46. {debrief_cli-0.1.1 → debrief_cli-0.2.0}/tests/test_synthesize.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: debrief-cli
3
- Version: 0.1.1
3
+ Version: 0.2.0
4
4
  Summary: Review your coding-agent sessions: what went wrong, and what to change.
5
5
  Project-URL: Homepage, https://github.com/mvwi/debrief
6
6
  Project-URL: Issues, https://github.com/mvwi/debrief/issues
@@ -56,6 +56,7 @@ logged-in `claude` CLI. Without one, pick OpenRouter and save a key on the Setti
56
56
  debrief sync --dry-run # ingest logs, then say how many model calls labeling would take
57
57
  debrief sync # ingest, label, find patterns, build the report
58
58
  debrief open # serve the report at http://127.0.0.1:8765 and open it
59
+ debrief push --dry-run # write what an upload to a team server would hold; sends nothing
59
60
 
60
61
  Useful flags for `sync`:
61
62
 
@@ -102,9 +103,24 @@ Unticking it on the Projects page does the same.
102
103
  logged-in `gh` CLI and read-only. It sends repo names, PR numbers, branch names and commit
103
104
  SHAs, and nothing from your transcripts. `--no-github`, or `[github] enabled = false` in
104
105
  config.toml, turns it off; without `gh` it is skipped.
105
- - The report holds your transcripts. `debrief open` serves it on 127.0.0.1 only. Token-shaped
106
- strings (GitHub, Anthropic, OpenAI, AWS, Slack, Stripe, private keys and others) are replaced
107
- with `[redacted]` in the report, but the redaction is pattern-based: read before you share.
106
+ - `debrief push --dry-run` writes a folder (`~/.local/share/debrief/push`, or `--out DIR`) that
107
+ holds transcript text from your picked projects, and sends nothing. No upload exists yet:
108
+ `debrief push` without `--dry-run` refuses. [docs/upload-format.md](docs/upload-format.md)
109
+ says what the folder holds. Upload only what you have the right to share: transcripts hold
110
+ your employer's or clients' code and other people's names.
111
+ - The report holds your transcripts. `debrief open` serves it on 127.0.0.1 only.
112
+ - Secrets are replaced with `[redacted]` in the report and in anything debrief uploads. The text
113
+ sent to the model for labeling and synthesis is not redacted. Three things are caught: known
114
+ token shapes (GitHub, Anthropic, OpenAI, AWS key ids, Slack, Stripe, Google, Resend and other
115
+ vendor prefixes, JWTs, private keys, bearer tokens, cookies, passwords in URLs); values of 12 or
116
+ more characters given to a name that sounds secret (`API_KEY=`, `"client_secret":`,
117
+ `--token`); and the exact values of your saved OpenRouter key and of environment variables
118
+ whose names contain KEY, TOKEN, SECRET, PASSWORD or CREDENTIAL. It is pattern matching, not a
119
+ guarantee. It misses a secret with no known shape and no telling name beside it: a password
120
+ in a sentence, a token in a URL parameter called `t`, a short password. It does not remove
121
+ IP addresses, host names or people's names; uploads replace email addresses and your home
122
+ folder path, the report does not. Findings and ideas are text a model wrote, and nothing
123
+ checks that text for the names of projects you did not pick. Read before you share.
108
124
 
109
125
  ## Which labeler
110
126
 
@@ -37,6 +37,7 @@ logged-in `claude` CLI. Without one, pick OpenRouter and save a key on the Setti
37
37
  debrief sync --dry-run # ingest logs, then say how many model calls labeling would take
38
38
  debrief sync # ingest, label, find patterns, build the report
39
39
  debrief open # serve the report at http://127.0.0.1:8765 and open it
40
+ debrief push --dry-run # write what an upload to a team server would hold; sends nothing
40
41
 
41
42
  Useful flags for `sync`:
42
43
 
@@ -83,9 +84,24 @@ Unticking it on the Projects page does the same.
83
84
  logged-in `gh` CLI and read-only. It sends repo names, PR numbers, branch names and commit
84
85
  SHAs, and nothing from your transcripts. `--no-github`, or `[github] enabled = false` in
85
86
  config.toml, turns it off; without `gh` it is skipped.
86
- - The report holds your transcripts. `debrief open` serves it on 127.0.0.1 only. Token-shaped
87
- strings (GitHub, Anthropic, OpenAI, AWS, Slack, Stripe, private keys and others) are replaced
88
- with `[redacted]` in the report, but the redaction is pattern-based: read before you share.
87
+ - `debrief push --dry-run` writes a folder (`~/.local/share/debrief/push`, or `--out DIR`) that
88
+ holds transcript text from your picked projects, and sends nothing. No upload exists yet:
89
+ `debrief push` without `--dry-run` refuses. [docs/upload-format.md](docs/upload-format.md)
90
+ says what the folder holds. Upload only what you have the right to share: transcripts hold
91
+ your employer's or clients' code and other people's names.
92
+ - The report holds your transcripts. `debrief open` serves it on 127.0.0.1 only.
93
+ - Secrets are replaced with `[redacted]` in the report and in anything debrief uploads. The text
94
+ sent to the model for labeling and synthesis is not redacted. Three things are caught: known
95
+ token shapes (GitHub, Anthropic, OpenAI, AWS key ids, Slack, Stripe, Google, Resend and other
96
+ vendor prefixes, JWTs, private keys, bearer tokens, cookies, passwords in URLs); values of 12 or
97
+ more characters given to a name that sounds secret (`API_KEY=`, `"client_secret":`,
98
+ `--token`); and the exact values of your saved OpenRouter key and of environment variables
99
+ whose names contain KEY, TOKEN, SECRET, PASSWORD or CREDENTIAL. It is pattern matching, not a
100
+ guarantee. It misses a secret with no known shape and no telling name beside it: a password
101
+ in a sentence, a token in a URL parameter called `t`, a short password. It does not remove
102
+ IP addresses, host names or people's names; uploads replace email addresses and your home
103
+ folder path, the report does not. Findings and ideas are text a model wrote, and nothing
104
+ checks that text for the names of projects you did not pick. Read before you share.
89
105
 
90
106
  ## Which labeler
91
107
 
@@ -6,6 +6,7 @@
6
6
  debrief sync --dry-run ingest, then say what labeling would cost; no model calls
7
7
  debrief open serve the report on localhost and open it in the browser;
8
8
  the page can also start a sync, show its progress and stop it
9
+ debrief push --dry-run build what an upload to a team server would hold; sends nothing
9
10
  debrief <stage> [args] run one stage; `debrief <stage> --help` lists its flags
10
11
 
11
12
  Everything is kept under DEBRIEF_HOME (default ~/.local/share/debrief).
@@ -43,6 +44,7 @@ STAGES = {
43
44
  "github": "github",
44
45
  "prune": "prune",
45
46
  "config": "config",
47
+ "push": "push",
46
48
  }
47
49
 
48
50
 
@@ -29,6 +29,7 @@ from debrief import config, skills
29
29
  from debrief.db import connect, repo_projects, session_projects
30
30
  from debrief.paths import SITE_OUT, SITE_SRC
31
31
  from debrief.recommend import LEVERS
32
+ from debrief.redact import SECRETS, redact, redact_damaged, redact_tree # noqa: F401 (old import path)
32
33
  from debrief.relay import is_relayed
33
34
  from debrief.synthesize import ref_project
34
35
 
@@ -50,93 +51,6 @@ SIGNALS = {
50
51
  }
51
52
  CORRECTING = ("correction", "reversal", "defect") # together, the site's "Agent corrected"
52
53
 
53
- # Secret-shaped strings, redacted from every string before it is serialized. Redacting the
54
- # serialized JSON instead misses a secret that follows an escape: in "\nghp_..." the "n" of
55
- # the escape is a word character, so \b never matches.
56
- SECRETS = [
57
- (re.compile(r"\b(?:gh[pousr]_[A-Za-z0-9]{36,}|github_pat_[A-Za-z0-9_]{40,})"), "[redacted]"),
58
- (re.compile(r"\bsk-(?=[A-Za-z0-9_-]*\d)(?:ant-|proj-|or-v1-)?[A-Za-z0-9_-]{20,}"), "[redacted]"), # needs a digit
59
- (re.compile(r"\b(?:sk|rk|pk)_(?:live|test)_[A-Za-z0-9]{20,}"), "[redacted]"), # Stripe
60
- (re.compile(r"\b(?:AKIA|ASIA)[A-Z0-9]{16}\b"), "[redacted]"),
61
- (re.compile(r"\bxox[abeprs]-[A-Za-z0-9-]{10,}"), "[redacted]"),
62
- (re.compile(r"\bglpat-[A-Za-z0-9_-]{20,}"), "[redacted]"),
63
- (re.compile(r"\bAIza[A-Za-z0-9_-]{35}"), "[redacted]"), # Google API key
64
- (re.compile(r"\b(?:npm|hf)_[A-Za-z0-9]{30,}"), "[redacted]"),
65
- (re.compile(r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}"), "[redacted]"),
66
- (re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----.*?(?:-----END [A-Z ]*PRIVATE KEY-----|\Z)", re.DOTALL), "[redacted]"),
67
- (re.compile(r"(?i)\b(bearer\s+)[A-Za-z0-9._~+/-]{24,}=*"), r"\1[redacted]"),
68
- (re.compile(r"(https?://)[^/\s:@\"\\]+:[^/\s@\"\\]+@"), r"\1[redacted]@"),
69
- ]
70
-
71
-
72
- def redact(text: str) -> str:
73
- """Redact secrets in one plain (not JSON-encoded) string."""
74
- for pattern, repl in SECRETS:
75
- text = pattern.sub(repl, text)
76
- return text
77
-
78
-
79
- # A run of non-space text that holds U+FFFD and is at least this long without it is blanked whole.
80
- # Shorter than any token pattern above can match, so no secret touching a mark survives.
81
- DAMAGED_MIN = 16
82
- DAMAGED_RUN = re.compile(r"\S*\ufffd\S*")
83
-
84
-
85
- # In damaged text a private key's header may not survive well enough to match, so any text
86
- # that spells PRIVATE KEY once marks and spaces are ignored is cut from its first dashes on.
87
- DAMAGED_KEY = re.compile(r"P\W*R\W*I\W*V\W*A\W*T\W*E\W*K\W*E\W*Y")
88
- DAMAGED_KEY_END = re.compile(r"E\W*N\W*D\b[^\n]{0,60}?-{3,}")
89
-
90
-
91
- def redact_damaged(text: str) -> str:
92
- """Redact a string holding U+FFFD, which marks bytes the source could not decode, then drop it.
93
-
94
- A mark can sit inside a secret, or be the only break before or after one, in any mix, so
95
- no single way of reading the marks finds every secret. Three steps, each on what the
96
- last one left:
97
- 1. blank what the patterns find with the marks removed, and with the marks as spaces
98
- (this is what catches a secret of several words, such as a private key or a bearer
99
- token, whose first word a mark touches);
100
- 2. blank any long run of text that still touches a mark, secret or not;
101
- 3. the normal pass.
102
- """
103
- key = DAMAGED_KEY.search(text)
104
- if key:
105
- start = max(text.rfind("---", 0, key.start()), 0)
106
- start = len(text[:start].rstrip("-")) if start else 0
107
- ends = [m.end() for m in DAMAGED_KEY_END.finditer(text, key.end())]
108
- text = text[:start] + "[redacted]" + (text[ends[-1]:].lstrip("-") if ends else "")
109
- keep = [i for i, c in enumerate(text) if c != "\ufffd"]
110
- joined = "".join(text[i] for i in keep)
111
- spaced = text.replace("\ufffd", " ")
112
- spans = []
113
- for pattern, _ in SECRETS:
114
- spans += [(keep[m.start()], keep[m.end() - 1] + 1) for m in pattern.finditer(joined) if m.end() > m.start()]
115
- spans += [m.span() for m in pattern.finditer(spaced)]
116
- out, at = [], 0
117
- for start, end in sorted(spans):
118
- if start >= at:
119
- out += [text[at:start], "[redacted]"]
120
- at = max(at, end)
121
- text = "".join([*out, text[at:]])
122
-
123
- def blank(m: re.Match) -> str:
124
- rest = m.group().replace("\ufffd", "")
125
- return "[redacted]" if len(rest) >= DAMAGED_MIN else rest
126
- return redact(DAMAGED_RUN.sub(blank, text))
127
-
128
-
129
- def redact_tree(obj):
130
- """Redact every string, keys included, in a JSON-like structure, and drop U+FFFD (the artifact host rejects it)."""
131
- if isinstance(obj, str):
132
- return redact_damaged(obj) if "\ufffd" in obj else redact(obj)
133
- if isinstance(obj, (list, tuple)):
134
- return [redact_tree(v) for v in obj]
135
- if isinstance(obj, dict):
136
- return {redact_tree(k): redact_tree(v) for k, v in obj.items()}
137
- return obj
138
-
139
-
140
54
  def excerpt(text, n: int) -> str:
141
55
  text = re.sub(r"\s+", " ", str(text or "")).strip()
142
56
  return text if len(text) <= n else text[: n - 1] + "…"
@@ -0,0 +1,237 @@
1
+ """Build what an upload to a team server would hold. There is no server yet, so only --dry-run works.
2
+
3
+ debrief push --dry-run write the bundle under DEBRIEF_HOME/push and say what is in it
4
+ debrief push --dry-run --out DIR write it to DIR
5
+ debrief push --dry-run --prev DIR send only what changed since the bundle in DIR
6
+
7
+ Only picked projects (`[projects] include` in config.toml) contribute anything. The rest are a
8
+ count. docs/upload-format.md describes the files.
9
+ """
10
+
11
+ import argparse
12
+ import hashlib
13
+ import json
14
+ import re
15
+ import sys
16
+ import uuid
17
+ from collections import Counter
18
+ from importlib.metadata import version
19
+ from pathlib import Path
20
+
21
+ from debrief import config
22
+ from debrief.db import connect, repo_projects, session_projects
23
+ from debrief.export_site import (
24
+ ci_section,
25
+ findings_and_recs,
26
+ labels_by_turn,
27
+ projects_section,
28
+ prs_section,
29
+ review_section,
30
+ session_file,
31
+ sessions_section,
32
+ )
33
+ from debrief.paths import HOME, NAME
34
+ from debrief.redact import VERSION, scrub
35
+ from debrief.sources import CLAUDE_DIR, CODEX_DIR
36
+ from debrief.synthesize import ref_project
37
+
38
+ FORMAT = 1
39
+ MACHINE_PATH = HOME / "machine_id"
40
+ DEFAULT_OUT = HOME / "push"
41
+ SAFE_NAME = re.compile(r"[A-Za-z0-9_-][\w.-]*", re.ASCII) # a session id that can be a file name as it is
42
+ NO_SERVER = (f"{NAME}: there is no server to push to yet. "
43
+ f"`{NAME} push --dry-run` builds what would be sent and sends nothing.")
44
+
45
+
46
+ def machine_id() -> tuple[str, bool]:
47
+ """This machine's id, a random UUID made on first use, and whether this call made it."""
48
+ if MACHINE_PATH.exists():
49
+ return MACHINE_PATH.read_text().strip(), False
50
+ HOME.mkdir(parents=True, exist_ok=True)
51
+ MACHINE_PATH.write_text(new := str(uuid.uuid4()))
52
+ return new, True
53
+
54
+
55
+ def object_path(session_id: str) -> str:
56
+ """Where a session's object goes. An id that is not a plain file name (logs can be imported,
57
+ so it may hold a slash) is named by its hash."""
58
+ name = session_id if SAFE_NAME.fullmatch(session_id) else hashlib.sha256(session_id.encode()).hexdigest()[:32]
59
+ return f"sessions/{name}.json"
60
+
61
+
62
+ def build(conn, picked: set[str], machine: str, have: dict[str, str] | None = None) -> tuple[dict, dict[str, bytes], dict]:
63
+ """(manifest, {path: bytes} for the objects to send, stats for the summary).
64
+
65
+ `have` is {path: sha256} of what the server already holds; an object with the same hash is
66
+ listed in the manifest but not sent.
67
+ """
68
+ have = have or {}
69
+ labels = labels_by_turn(conn)
70
+ run_id = conn.execute("SELECT MAX(run_id) FROM recommendations").fetchone()[0] \
71
+ or conn.execute("SELECT MAX(run_id) FROM findings").fetchone()[0]
72
+ rows = projects_section(conn, picked)
73
+ repo_of = {}
74
+ for repo, path in sorted(repo_projects(conn).items(), reverse=True):
75
+ repo_of[path] = repo # a folder tied to several repos takes the first by name
76
+ key = {r["path"]: repo_of.get(r["path"]) or "local:" + hashlib.sha256(f"{machine}:{r['path']}".encode()).hexdigest()[:12]
77
+ for r in rows if r["picked"]}
78
+
79
+ raw: dict[str, object] = {}
80
+ every, _ = sessions_section(conn, labels, "", picked)
81
+ # A session with an author came from Entire and may be a teammate's; a server would file it
82
+ # under whoever pushed. Local logs name no author.
83
+ mine = [s for s in every if not s["author"]]
84
+ # Nor is a log read from anywhere but the agents' own log folders on this machine: `--import`
85
+ # takes any folder, and a teammate's log names no author. The loaded row is the file the
86
+ # session's rows were read from.
87
+ local = {sid for sid, path in conn.execute("SELECT session_id, path FROM source_logs WHERE status = 'loaded'")
88
+ if ".." not in Path(path).parts and any(Path(path).is_relative_to(root) for root in (CLAUDE_DIR, CODEX_DIR))}
89
+ sessions = [s for s in mine if s["id"] in local]
90
+ for s in sessions:
91
+ raw[object_path(s["id"])] = {**s, "project": key[s["project"]], **session_file(conn, labels, s["id"])}
92
+ ids = {s["id"] for s in sessions}
93
+
94
+ findings, recs, refs = findings_and_recs(conn, labels, run_id, picked)
95
+ found, ideas = len(findings), len(recs)
96
+ findings, recs, refs = only_picked(conn, findings, recs, refs, key, ids)
97
+ left_out = {"findings": found - len(findings), "ideas": ideas - len(recs), "authored": len(every) - len(mine), "imported": len(mine) - len(sessions)}
98
+ if findings or recs:
99
+ for item in (*findings.values(), *recs):
100
+ item["project"] = item["project"] and key[item["project"]]
101
+ for ref in refs.values():
102
+ if "sessions" in ref: # a CI or review event names every session on its PR; keep the bundled ones
103
+ ref["sessions"] = [sid for sid in ref["sessions"] if sid in ids]
104
+ raw["findings.json"] = {"run_id": run_id, "findings": findings, "recommendations": recs, "refs": refs}
105
+
106
+ linked = {p["n"] for s in sessions for p in s["prs"]}
107
+ prs = {k: v for k, v in prs_section(conn).items() if k in linked}
108
+ if prs:
109
+ ci = ci_section(conn, "")
110
+ raw["github.json"] = {
111
+ "prs": prs,
112
+ "ci": {"runs_by_pr": {k: v for k, v in ci["runs_by_pr"].items() if k in linked},
113
+ "failures": [f for f in ci["failures"] if f["pr"] in linked]},
114
+ "reviews": [r for r in review_section(conn) if r["pr"] in linked]}
115
+
116
+ counts, objects, send = Counter(), {}, {}
117
+ for path, obj in raw.items():
118
+ clean, n = scrub(obj, home_paths=True, emails=True)
119
+ counts.update(n)
120
+ body = json.dumps(clean, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode()
121
+ objects[path] = {"sha256": hashlib.sha256(body).hexdigest(), "bytes": len(body)}
122
+ if have.get(path) != objects[path]["sha256"]:
123
+ send[path] = body
124
+ count, turns = Counter(s["project"] for s in sessions), Counter()
125
+ for s in sessions:
126
+ turns[s["project"]] += s["human_turns"]
127
+ projects, n = scrub([{"key": key[r["path"]], "name": r["name"], "sessions": count[r["path"]], "turns": turns[r["path"]],
128
+ "analysed_at": r["analysed_at"]} for r in rows if r["picked"]], home_paths=True, emails=True)
129
+ counts.update(n)
130
+ manifest = {"format": FORMAT, "tool": f"{NAME} {version('debrief-cli')}", "machine": machine,
131
+ "redaction": VERSION,
132
+ "projects": projects, "unpicked": sum(not r["picked"] for r in rows),
133
+ "objects": objects, "sent": sorted(send)}
134
+ stats = {
135
+ "sessions": len(sessions), "sessions_sent": sum(p.startswith("sessions/") for p in send),
136
+ "turns": sum(s["human_turns"] for s in sessions),
137
+ "messages": sum(len(raw[object_path(s["id"])]["messages"]) for s in sessions),
138
+ "secrets": counts["secrets"], "home_paths": counts["home_paths"], "emails": counts["emails"],
139
+ "repos": sorted({k for k in key.values() if not k.startswith("local:")} | {k.rsplit("#", 1)[0] for k in prs}),
140
+ "removed": sorted(set(have) - set(objects)), "left_out": left_out,
141
+ }
142
+ return manifest, send, stats
143
+
144
+
145
+ def only_picked(conn, findings: dict, recs: list, refs: dict, key: dict[str, str], ids: set[str]) -> tuple[dict, list, dict]:
146
+ """Cut the report's findings and ideas down to what an upload may hold. The report's own rule
147
+ keeps a finding when some of its evidence is in a picked project, and an idea when some of
148
+ its findings are kept; a model wrote each text from all of them, so that is too loose to send.
149
+
150
+ Kept: a finding whose stored evidence is all in the bundle (turns of sessions `ids`, events
151
+ of a picked project's repo), and an idea whose stored findings are all kept. Either must be
152
+ about a project with a key, or about all of them.
153
+ """
154
+ project, repos = session_projects(conn), repo_projects(conn)
155
+
156
+ def in_bundle(ref: dict) -> bool:
157
+ return ref["session_id"] in ids if "session_id" in ref else ref_project(conn, project, repos, ref) in key
158
+
159
+ def stored(table: str, column: str, id_column: str, row_id: int) -> list:
160
+ return json.loads(conn.execute(f"SELECT {column} FROM {table} WHERE {id_column} = ?", (row_id,)).fetchone()[0] or "[]")
161
+
162
+ findings = {i: f for i, f in findings.items() if f["project"] in (None, *key)
163
+ and all(in_bundle(e) for e in stored("findings", "evidence", "finding_id", i))}
164
+ recs = [r for r in recs if r["project"] in (None, *key)
165
+ and all(i in findings for i in stored("recommendations", "finding_ids", "rec_id", r["id"]))]
166
+ cited = {e for f in findings.values() for e in f["evidence"]}
167
+ return findings, recs, {i: ref for i, ref in refs.items() if i in cited}
168
+
169
+
170
+ def write(out: Path, manifest: dict, send: dict[str, bytes]) -> int:
171
+ """Write the bundle to `out`, replacing an earlier bundle there. Returns the bytes written."""
172
+ if out.exists() and any(out.iterdir()) and not (out / "manifest.json").exists():
173
+ sys.exit(f"{NAME}: {out} is not empty and is not an earlier bundle; name another folder")
174
+ out.mkdir(parents=True, exist_ok=True, mode=0o700) # it holds transcripts
175
+ (out / "sessions").mkdir(exist_ok=True)
176
+ for old in (*out.glob("sessions/*.json"), out / "findings.json", out / "github.json"):
177
+ old.unlink(missing_ok=True)
178
+ files = {**send, "manifest.json": json.dumps(manifest, ensure_ascii=False, indent=1).encode()}
179
+ for path, body in files.items():
180
+ (out / path).write_bytes(body)
181
+ return sum(len(b) for b in files.values())
182
+
183
+
184
+ def main(argv: list[str] | None = None) -> None:
185
+ p = argparse.ArgumentParser(prog=f"{NAME} push", description=__doc__,
186
+ formatter_class=argparse.RawDescriptionHelpFormatter)
187
+ p.add_argument("--dry-run", action="store_true", help="Build the bundle and write it to a folder; send nothing")
188
+ p.add_argument("--out", metavar="DIR", type=Path, default=DEFAULT_OUT, help=f"Where to write it (default {DEFAULT_OUT})")
189
+ p.add_argument("--prev", metavar="DIR", type=Path,
190
+ help="An earlier bundle, taken as what the server holds: only objects that changed since are written")
191
+ args = p.parse_args(argv)
192
+ if not args.dry_run:
193
+ sys.exit(NO_SERVER)
194
+ have = None
195
+ if args.prev:
196
+ try:
197
+ have = {path: o["sha256"] for path, o in json.loads((args.prev / "manifest.json").read_text())["objects"].items()}
198
+ except (OSError, ValueError, KeyError, TypeError, AttributeError) as e:
199
+ sys.exit(f"{NAME}: {args.prev} holds no bundle manifest ({e})")
200
+ machine, made = machine_id()
201
+ manifest, send, stats = build(connect(), set(config.projects()), machine, have)
202
+ size = write(args.out, manifest, send)
203
+
204
+ others = manifest["unpicked"]
205
+ print(f"Dry run: nothing was sent. The bundle is in {args.out} ({len(send) + 1} files, {size:,} bytes).")
206
+ print(f" projects {len(manifest['projects'])} picked")
207
+ for r in manifest["projects"]:
208
+ print(f" {r['name']} ({r['key']}; {r['sessions']} sessions, {r['turns']} turns)")
209
+ print(f" {others} other project{'' if others == 1 else 's'} on this machine "
210
+ f"{'is' if others == 1 else 'are'} not picked and nothing about them is sent")
211
+ print(f" sessions {stats['sessions']}, of which {stats['sessions_sent']} sent"
212
+ f"{' (the rest have not changed)' if have is not None else ''}")
213
+ print(f" turns {stats['turns']} of yours, {stats['messages']} messages in all")
214
+ print(f" redactions {stats['secrets']} secrets and {stats['emails']} email addresses replaced, "
215
+ f"{stats['home_paths']} home-folder paths shortened to ~")
216
+ print(f" repo names {', '.join(stats['repos']) or 'none'}")
217
+ if stats["removed"]:
218
+ gone = sum(path.startswith("sessions/") for path in stats["removed"])
219
+ print(f" This push removes {gone} session{'' if gone == 1 else 's'} the server holds "
220
+ f"({len(stats['removed'])} object{'' if len(stats['removed']) == 1 else 's'} no longer in a picked project).")
221
+ out = stats["left_out"]
222
+ if out["authored"]:
223
+ print(f" Left out {out['authored']} sessions: they are other people's (they name an author).")
224
+ if out["imported"]:
225
+ print(f" Left out {out['imported']} sessions: they were imported from another folder.")
226
+ if out["findings"] or out["ideas"]:
227
+ print(f" Left out {out['findings']} findings and {out['ideas']} ideas: "
228
+ "they draw on projects you have not picked or on sessions that are not sent.")
229
+ if made:
230
+ print(f" Made an id for this machine, a random number kept in {MACHINE_PATH}.")
231
+ print("Redaction is pattern-based: read the bundle before you trust it.")
232
+ print("Upload only what you have the right to share: transcripts hold your employer's or clients' code "
233
+ "and other people's names.")
234
+
235
+
236
+ if __name__ == "__main__":
237
+ main()
@@ -0,0 +1,195 @@
1
+ """Replace secret-shaped strings with [redacted] before text leaves the database for the report or an upload.
2
+
3
+ redact(text) one string
4
+ redact_tree(obj) nested JSON-like data
5
+ scrub(obj, home_paths=False, emails=False) either, plus counts; for callers that upload
6
+
7
+ Three kinds of rule, in the order they run: exact values this machine already knows (debrief's
8
+ saved OpenRouter key and secret-named environment variables), known token shapes (SECRETS),
9
+ and values given to a secret-sounding name (NAMED). It is pattern matching: a secret with
10
+ no known shape and no telling name beside it gets through.
11
+ """
12
+
13
+ import os
14
+ import re
15
+ from functools import cache
16
+ from pathlib import Path
17
+
18
+ from debrief import config
19
+
20
+ MARK = "[redacted]"
21
+ EMAIL_MARK = "[email]"
22
+ VERSION = 2 # bump when a change to the patterns or scrub rules changes what comes out; uploads record it
23
+
24
+ # Redacting serialized JSON instead of each string misses a secret that follows an escape:
25
+ # in "\nghp_..." the "n" of the escape is a word character, so \b never matches.
26
+ SECRETS = [
27
+ (re.compile(r"\b(?:gh[pousr]_[A-Za-z0-9]{36,}|github_pat_[A-Za-z0-9_]{40,})"), MARK),
28
+ (re.compile(r"\bsk-(?=[A-Za-z0-9_-]*\d)(?:ant-|proj-|or-v1-)?[A-Za-z0-9_-]{20,}"), MARK), # needs a digit
29
+ (re.compile(r"\b(?:sk|rk|pk)_(?:live|test)_[A-Za-z0-9]{20,}"), MARK), # Stripe
30
+ (re.compile(r"\b(?:AKIA|ASIA)[A-Z0-9]{16}\b"), MARK),
31
+ (re.compile(r"\bxox[abeprs]-[A-Za-z0-9-]{10,}"), MARK),
32
+ (re.compile(r"\bglpat-[A-Za-z0-9_-]{20,}"), MARK),
33
+ (re.compile(r"\bAIza[A-Za-z0-9_-]{35}"), MARK), # Google API key
34
+ (re.compile(r"\b(?:npm|hf)_[A-Za-z0-9]{30,}"), MARK),
35
+ (re.compile(r"\bre_[A-Za-z0-9]{8}_[A-Za-z0-9]{20,}"), MARK), # Resend
36
+ # Other vendors' prefixes. The digit keeps snake_case names (r8_some_long_function_name) out.
37
+ (re.compile(r"\b(?:gsk|r8|phx|sbp|shpat|figd|ntn|dop_v1|lin_api|whsec|sntrys|sb_secret)_"
38
+ r"(?=[A-Za-z0-9_-]*\d)[A-Za-z0-9_-]{20,}"), MARK),
39
+ (re.compile(r"\b(?:ya29\.|GOCSPX-|SG\.|pypi-AgE|xai-|pplx-|tvly-)(?=[A-Za-z0-9._-]*\d)[A-Za-z0-9._-]{20,}"), MARK),
40
+ (re.compile(r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}"), MARK),
41
+ (re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----.*?(?:-----END [A-Z ]*PRIVATE KEY-----|\Z)", re.DOTALL), MARK),
42
+ (re.compile(r"(?i)\b(bearer\s+)[A-Za-z0-9._~+/-]{24,}=*"), rf"\1{MARK}"),
43
+ # user:password@ in any URL: https, postgres, mongodb+srv, redis, amqp ...
44
+ (re.compile(r"\b([a-z][a-z0-9+.-]{1,15}://)[^/\s:@\"\\]*:[^/\s@\"\\]+@"), rf"\1{MARK}@"),
45
+ (re.compile(r"(?i)\b((?:set-)?cookie[\"']?:\s*[\"']?)[^\s=;\"']+=[^\n\"']{8,}"), rf"\1{MARK}"),
46
+ ]
47
+
48
+ # A value given to a secret-sounding name: API_KEY=..., "client_secret": "...", token: ..., --token ...
49
+ # The name stays, the value goes. The value must be 12+ characters with a digit and a letter,
50
+ # which leaves out code (`token = options.authToken`) and prose. Also left alone: UUIDs (ids,
51
+ # not secrets), paths, the AWS4-HMAC-SHA256 scheme name and PostHog's public phc_ project keys.
52
+ # "auth" does not match "author".
53
+ NAME = r"(?:api[_-]?key|access[_-]?key|private[_-]?key|secret|token|passw(?:or)?d|passwd|pwd|credential|auth(?!or(?!iz)))"
54
+ VALUE = (r"(?![0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b|[.~]{0,2}/|AWS4-|phc_)"
55
+ r"(?=[A-Za-z0-9+/_.~=-]*\d)(?=[0-9+/_.~=-]*[A-Za-z])[A-Za-z0-9+/_.~=-]{12,}")
56
+ NAMED = [
57
+ (re.compile(rf"(?i)(\b[\w.-]*{NAME}[\w-]*\\?[\"']?[ \t]*[:=][ \t]*(?:(?:basic|bearer|token)[ \t]+)?\\?[\"']?){VALUE}"),
58
+ rf"\1{MARK}"),
59
+ (re.compile(rf"(?i)(--[\w-]*{NAME}[\w-]*[ \t]+\\?[\"']?){VALUE}"), rf"\1{MARK}"), # a command-line flag: --token abc
60
+ ]
61
+
62
+ HOME = re.compile(r"(?<![\w.~-])(?:/(?:Users|home)/|[A-Za-z]:\\{1,2}Users\\{1,2})[^/\\\s\"'`:;,<>|()\[\]{}]+")
63
+
64
+ # Claude Code names a project's log folder after its path, every other character a dash: -Users-name-Dev-x.
65
+ # Only for a home under /Users or /home: the dashed form of / or /root would match ordinary text.
66
+ HOME_DASHED = re.sub(r"[^A-Za-z0-9]", "-", str(Path.home())) if Path.home().parent.name in ("Users", "home") else None
67
+
68
+ # An email address. Left alone: git@host, user@host:path (ssh, scp, git remotes), user@host inside a
69
+ # URL, pkg@1.2.3 and @scope/pkg (the domain must start with a letter and end in a letter-only label).
70
+ # An address right after a colon (mailto:a@b.com, "email":a@b.com) is one; scheme://user:pass@host is
71
+ # safe only because the secret rules blank it first.
72
+ # A colon after it marks user@host:path only when text follows at once ("sent to a@b.com: done" is prose).
73
+ EMAIL = re.compile(r"(?<![\w.+/@-])(?!git@)[A-Za-z0-9][\w.+-]*@[A-Za-z][A-Za-z0-9-]*(?:\.[A-Za-z0-9-]+)*\.[A-Za-z]{2,}\b(?![@\w]|:\S|\.[A-Za-z0-9-])")
74
+
75
+ KNOWN_NAME = re.compile(r"(?i)KEY|TOKEN|SECRET|PASSWORD|PASSWD|CREDENTIAL")
76
+ KNOWN_MIN = 12 # a shorter value could blank ordinary words
77
+
78
+
79
+ @cache
80
+ def known_secrets() -> tuple[str, ...]:
81
+ """Exact secret values this machine holds: debrief's saved key and secret-named environment variables."""
82
+ values = {v for k, v in os.environ.items() if KNOWN_NAME.search(k)}
83
+ values.add(config.dotenv_values(config.ENV_PATH).get(config.KEY))
84
+ keep = (v for v in values if v and len(v) >= KNOWN_MIN and not v.startswith(("/", "~"))) # not a path to a key file
85
+ return tuple(sorted(keep, key=len, reverse=True))
86
+
87
+
88
+ def clean(text: str) -> tuple[str, int]:
89
+ """Redact one string that holds no U+FFFD; return it and the number of replacements."""
90
+ count = 0
91
+ for secret in known_secrets():
92
+ count += text.count(secret)
93
+ text = text.replace(secret, MARK)
94
+ for pattern, repl in [*SECRETS, *NAMED]:
95
+ text, n = pattern.subn(repl, text)
96
+ count += n
97
+ return text, count
98
+
99
+
100
+ # A run of non-space text that holds U+FFFD and is at least this long without it is blanked whole.
101
+ # Shorter than any token pattern above can match, so no secret touching a mark survives.
102
+ DAMAGED_MIN = 16
103
+ DAMAGED_RUN = re.compile(r"\S*\ufffd\S*")
104
+
105
+
106
+ # In damaged text a private key's header may not survive well enough to match, so any text
107
+ # that spells PRIVATE KEY once marks and spaces are ignored is cut from its first dashes on.
108
+ DAMAGED_KEY = re.compile(r"P\W*R\W*I\W*V\W*A\W*T\W*E\W*K\W*E\W*Y")
109
+ DAMAGED_KEY_END = re.compile(r"E\W*N\W*D\b[^\n]{0,60}?-{3,}")
110
+
111
+
112
+ def redact_damaged(text: str) -> str:
113
+ """Redact a string holding U+FFFD, which marks bytes the source could not decode, then drop it.
114
+
115
+ A mark can sit inside a secret, or be the only break before or after one, in any mix, so
116
+ no single way of reading the marks finds every secret. Three steps, each on what the
117
+ last one left:
118
+ 1. blank what the patterns find with the marks removed, and with the marks as spaces
119
+ (this is what catches a secret of several words, such as a private key or a bearer
120
+ token, whose first word a mark touches);
121
+ 2. blank any long run of text that still touches a mark, secret or not;
122
+ 3. the normal pass.
123
+ """
124
+ key = DAMAGED_KEY.search(text)
125
+ if key:
126
+ start = max(text.rfind("---", 0, key.start()), 0)
127
+ start = len(text[:start].rstrip("-")) if start else 0
128
+ ends = [m.end() for m in DAMAGED_KEY_END.finditer(text, key.end())]
129
+ text = text[:start] + MARK + (text[ends[-1]:].lstrip("-") if ends else "")
130
+ keep = [i for i, c in enumerate(text) if c != "\ufffd"]
131
+ joined = "".join(text[i] for i in keep)
132
+ spaced = text.replace("\ufffd", " ")
133
+ spans = []
134
+ for pattern, _ in SECRETS:
135
+ spans += [(keep[m.start()], keep[m.end() - 1] + 1) for m in pattern.finditer(joined) if m.end() > m.start()]
136
+ spans += [m.span() for m in pattern.finditer(spaced)]
137
+ out, at = [], 0
138
+ for start, end in sorted(spans):
139
+ if start >= at:
140
+ out += [text[at:start], MARK]
141
+ at = max(at, end)
142
+ text = "".join([*out, text[at:]])
143
+
144
+ def blank(m: re.Match) -> str:
145
+ rest = m.group().replace("\ufffd", "")
146
+ return MARK if len(rest) >= DAMAGED_MIN else rest
147
+ return clean(DAMAGED_RUN.sub(blank, text))[0]
148
+
149
+
150
+ def scrub(obj, *, home_paths: bool = False, emails: bool = False):
151
+ """Return (scrubbed copy, {"secrets": n, "home_paths": n, "emails": n}) for a string or JSON-like data.
152
+
153
+ Every string is redacted, dict keys included, and U+FFFD is dropped (the artifact host
154
+ rejects it). With home_paths, /Users/<name>, /home/<name> and C:\\Users\\<name> become ~,
155
+ and so does this user's home folder in Claude's dashed form (-Users-<name>). With emails,
156
+ each email address becomes [email]; GitHub logins, @mentions and owner/name stay.
157
+ """
158
+ counts = {"secrets": 0, "home_paths": 0, "emails": 0}
159
+
160
+ def walk(o):
161
+ if isinstance(o, str):
162
+ if "\ufffd" in o:
163
+ # Counted by marks gained: the damaged pass merges overlapping finds.
164
+ text = redact_damaged(o)
165
+ counts["secrets"] += max(text.count(MARK) - o.count(MARK), 0)
166
+ else:
167
+ text, n = clean(o)
168
+ counts["secrets"] += n
169
+ if home_paths:
170
+ text, n = HOME.subn("~", text)
171
+ counts["home_paths"] += n
172
+ if HOME_DASHED:
173
+ counts["home_paths"] += text.count(HOME_DASHED)
174
+ text = text.replace(HOME_DASHED, "~")
175
+ if emails:
176
+ text, n = EMAIL.subn(EMAIL_MARK, text)
177
+ counts["emails"] += n
178
+ return text
179
+ if isinstance(o, (list, tuple)):
180
+ return [walk(v) for v in o]
181
+ if isinstance(o, dict):
182
+ return {walk(k): walk(v) for k, v in o.items()}
183
+ return o
184
+
185
+ return walk(obj), counts
186
+
187
+
188
+ def redact(text: str) -> str:
189
+ """Redact secrets in one plain (not JSON-encoded) string."""
190
+ return scrub(text)[0]
191
+
192
+
193
+ def redact_tree(obj):
194
+ """Redact every string, keys included, in a JSON-like structure, and drop U+FFFD."""
195
+ return scrub(obj)[0]
@@ -1,7 +1,7 @@
1
1
  [project]
2
2
  # "debrief" is taken on PyPI. The command and the import package are still `debrief`.
3
3
  name = "debrief-cli"
4
- version = "0.1.1"
4
+ version = "0.2.0"
5
5
  description = "Review your coding-agent sessions: what went wrong, and what to change."
6
6
  readme = "README.md"
7
7
  requires-python = ">=3.13"