debrief-cli 0.1.0__tar.gz → 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. {debrief_cli-0.1.0 → debrief_cli-0.2.0}/PKG-INFO +20 -4
  2. {debrief_cli-0.1.0 → debrief_cli-0.2.0}/README.md +19 -3
  3. {debrief_cli-0.1.0 → debrief_cli-0.2.0}/debrief/cli.py +6 -0
  4. {debrief_cli-0.1.0 → debrief_cli-0.2.0}/debrief/export_site.py +10 -91
  5. debrief_cli-0.2.0/debrief/push.py +237 -0
  6. debrief_cli-0.2.0/debrief/redact.py +195 -0
  7. {debrief_cli-0.1.0 → debrief_cli-0.2.0}/pyproject.toml +1 -1
  8. {debrief_cli-0.1.0 → debrief_cli-0.2.0}/tests/test_cli.py +3 -0
  9. debrief_cli-0.2.0/tests/test_push.py +288 -0
  10. debrief_cli-0.2.0/tests/test_redact.py +233 -0
  11. debrief_cli-0.1.0/tests/test_export_site.py +0 -101
  12. {debrief_cli-0.1.0 → debrief_cli-0.2.0}/.gitignore +0 -0
  13. {debrief_cli-0.1.0 → debrief_cli-0.2.0}/LICENSE +0 -0
  14. {debrief_cli-0.1.0 → debrief_cli-0.2.0}/debrief/__init__.py +0 -0
  15. {debrief_cli-0.1.0 → debrief_cli-0.2.0}/debrief/build_artifact.py +0 -0
  16. {debrief_cli-0.1.0 → debrief_cli-0.2.0}/debrief/classify.py +0 -0
  17. {debrief_cli-0.1.0 → debrief_cli-0.2.0}/debrief/compare_definitions.py +0 -0
  18. {debrief_cli-0.1.0 → debrief_cli-0.2.0}/debrief/compare_llm.py +0 -0
  19. {debrief_cli-0.1.0 → debrief_cli-0.2.0}/debrief/config.py +0 -0
  20. {debrief_cli-0.1.0 → debrief_cli-0.2.0}/debrief/db.py +0 -0
  21. {debrief_cli-0.1.0 → debrief_cli-0.2.0}/debrief/export.py +0 -0
  22. {debrief_cli-0.1.0 → debrief_cli-0.2.0}/debrief/github.py +0 -0
  23. {debrief_cli-0.1.0 → debrief_cli-0.2.0}/debrief/normalize.py +0 -0
  24. {debrief_cli-0.1.0 → debrief_cli-0.2.0}/debrief/paths.py +0 -0
  25. {debrief_cli-0.1.0 → debrief_cli-0.2.0}/debrief/prune.py +0 -0
  26. {debrief_cli-0.1.0 → debrief_cli-0.2.0}/debrief/recommend.py +0 -0
  27. {debrief_cli-0.1.0 → debrief_cli-0.2.0}/debrief/relay.py +0 -0
  28. {debrief_cli-0.1.0 → debrief_cli-0.2.0}/debrief/runner.py +0 -0
  29. {debrief_cli-0.1.0 → debrief_cli-0.2.0}/debrief/scrape.py +0 -0
  30. {debrief_cli-0.1.0 → debrief_cli-0.2.0}/debrief/site/app.js +0 -0
  31. {debrief_cli-0.1.0 → debrief_cli-0.2.0}/debrief/site/index.html +0 -0
  32. {debrief_cli-0.1.0 → debrief_cli-0.2.0}/debrief/site/method.js +0 -0
  33. {debrief_cli-0.1.0 → debrief_cli-0.2.0}/debrief/site/style.css +0 -0
  34. {debrief_cli-0.1.0 → debrief_cli-0.2.0}/debrief/skills.py +0 -0
  35. {debrief_cli-0.1.0 → debrief_cli-0.2.0}/debrief/sources.py +0 -0
  36. {debrief_cli-0.1.0 → debrief_cli-0.2.0}/debrief/synthesize.py +0 -0
  37. {debrief_cli-0.1.0 → debrief_cli-0.2.0}/debrief/verify_evidence.py +0 -0
  38. {debrief_cli-0.1.0 → debrief_cli-0.2.0}/tests/conftest.py +0 -0
  39. {debrief_cli-0.1.0 → debrief_cli-0.2.0}/tests/test_classify.py +0 -0
  40. {debrief_cli-0.1.0 → debrief_cli-0.2.0}/tests/test_config.py +0 -0
  41. {debrief_cli-0.1.0 → debrief_cli-0.2.0}/tests/test_export_site_db.py +0 -0
  42. {debrief_cli-0.1.0 → debrief_cli-0.2.0}/tests/test_github.py +0 -0
  43. {debrief_cli-0.1.0 → debrief_cli-0.2.0}/tests/test_paths.py +0 -0
  44. {debrief_cli-0.1.0 → debrief_cli-0.2.0}/tests/test_relay.py +0 -0
  45. {debrief_cli-0.1.0 → debrief_cli-0.2.0}/tests/test_sources.py +0 -0
  46. {debrief_cli-0.1.0 → debrief_cli-0.2.0}/tests/test_synthesize.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: debrief-cli
3
- Version: 0.1.0
3
+ Version: 0.2.0
4
4
  Summary: Review your coding-agent sessions: what went wrong, and what to change.
5
5
  Project-URL: Homepage, https://github.com/mvwi/debrief
6
6
  Project-URL: Issues, https://github.com/mvwi/debrief/issues
@@ -56,6 +56,7 @@ logged-in `claude` CLI. Without one, pick OpenRouter and save a key on the Setti
56
56
  debrief sync --dry-run # ingest logs, then say how many model calls labeling would take
57
57
  debrief sync # ingest, label, find patterns, build the report
58
58
  debrief open # serve the report at http://127.0.0.1:8765 and open it
59
+ debrief push --dry-run # write what an upload to a team server would hold; sends nothing
59
60
 
60
61
  Useful flags for `sync`:
61
62
 
@@ -102,9 +103,24 @@ Unticking it on the Projects page does the same.
102
103
  logged-in `gh` CLI and read-only. It sends repo names, PR numbers, branch names and commit
103
104
  SHAs, and nothing from your transcripts. `--no-github`, or `[github] enabled = false` in
104
105
  config.toml, turns it off; without `gh` it is skipped.
105
- - The report holds your transcripts. `debrief open` serves it on 127.0.0.1 only. Token-shaped
106
- strings (GitHub, Anthropic, OpenAI, AWS, Slack, Stripe, private keys and others) are replaced
107
- with `[redacted]` in the report, but the redaction is pattern-based: read before you share.
106
+ - `debrief push --dry-run` writes a folder (`~/.local/share/debrief/push`, or `--out DIR`) that
107
+ holds transcript text from your picked projects, and sends nothing. No upload exists yet:
108
+ `debrief push` without `--dry-run` refuses. [docs/upload-format.md](docs/upload-format.md)
109
+ says what the folder holds. Upload only what you have the right to share: transcripts hold
110
+ your employer's or clients' code and other people's names.
111
+ - The report holds your transcripts. `debrief open` serves it on 127.0.0.1 only.
112
+ - Secrets are replaced with `[redacted]` in the report and in anything debrief uploads. The text
113
+ sent to the model for labeling and synthesis is not redacted. Three things are caught: known
114
+ token shapes (GitHub, Anthropic, OpenAI, AWS key ids, Slack, Stripe, Google, Resend and other
115
+ vendor prefixes, JWTs, private keys, bearer tokens, cookies, passwords in URLs); values of 12 or
116
+ more characters given to a name that sounds secret (`API_KEY=`, `"client_secret":`,
117
+ `--token`); and the exact values of your saved OpenRouter key and of environment variables
118
+ whose names contain KEY, TOKEN, SECRET, PASSWORD or CREDENTIAL. It is pattern matching, not a
119
+ guarantee. It misses a secret with no known shape and no telling name beside it: a password
120
+ in a sentence, a token in a URL parameter called `t`, a short password. It does not remove
121
+ IP addresses, host names or people's names; uploads replace email addresses and your home
122
+ folder path, the report does not. Findings and ideas are text a model wrote, and nothing
123
+ checks that text for the names of projects you did not pick. Read before you share.
108
124
 
109
125
  ## Which labeler
110
126
 
@@ -37,6 +37,7 @@ logged-in `claude` CLI. Without one, pick OpenRouter and save a key on the Setti
37
37
  debrief sync --dry-run # ingest logs, then say how many model calls labeling would take
38
38
  debrief sync # ingest, label, find patterns, build the report
39
39
  debrief open # serve the report at http://127.0.0.1:8765 and open it
40
+ debrief push --dry-run # write what an upload to a team server would hold; sends nothing
40
41
 
41
42
  Useful flags for `sync`:
42
43
 
@@ -83,9 +84,24 @@ Unticking it on the Projects page does the same.
83
84
  logged-in `gh` CLI and read-only. It sends repo names, PR numbers, branch names and commit
84
85
  SHAs, and nothing from your transcripts. `--no-github`, or `[github] enabled = false` in
85
86
  config.toml, turns it off; without `gh` it is skipped.
86
- - The report holds your transcripts. `debrief open` serves it on 127.0.0.1 only. Token-shaped
87
- strings (GitHub, Anthropic, OpenAI, AWS, Slack, Stripe, private keys and others) are replaced
88
- with `[redacted]` in the report, but the redaction is pattern-based: read before you share.
87
+ - `debrief push --dry-run` writes a folder (`~/.local/share/debrief/push`, or `--out DIR`) that
88
+ holds transcript text from your picked projects, and sends nothing. No upload exists yet:
89
+ `debrief push` without `--dry-run` refuses. [docs/upload-format.md](docs/upload-format.md)
90
+ says what the folder holds. Upload only what you have the right to share: transcripts hold
91
+ your employer's or clients' code and other people's names.
92
+ - The report holds your transcripts. `debrief open` serves it on 127.0.0.1 only.
93
+ - Secrets are replaced with `[redacted]` in the report and in anything debrief uploads. The text
94
+ sent to the model for labeling and synthesis is not redacted. Three things are caught: known
95
+ token shapes (GitHub, Anthropic, OpenAI, AWS key ids, Slack, Stripe, Google, Resend and other
96
+ vendor prefixes, JWTs, private keys, bearer tokens, cookies, passwords in URLs); values of 12 or
97
+ more characters given to a name that sounds secret (`API_KEY=`, `"client_secret":`,
98
+ `--token`); and the exact values of your saved OpenRouter key and of environment variables
99
+ whose names contain KEY, TOKEN, SECRET, PASSWORD or CREDENTIAL. It is pattern matching, not a
100
+ guarantee. It misses a secret with no known shape and no telling name beside it: a password
101
+ in a sentence, a token in a URL parameter called `t`, a short password. It does not remove
102
+ IP addresses, host names or people's names; uploads replace email addresses and your home
103
+ folder path, the report does not. Findings and ideas are text a model wrote, and nothing
104
+ checks that text for the names of projects you did not pick. Read before you share.
89
105
 
90
106
  ## Which labeler
91
107
 
@@ -6,6 +6,7 @@
6
6
  debrief sync --dry-run ingest, then say what labeling would cost; no model calls
7
7
  debrief open serve the report on localhost and open it in the browser;
8
8
  the page can also start a sync, show its progress and stop it
9
+ debrief push --dry-run build what an upload to a team server would hold; sends nothing
9
10
  debrief <stage> [args] run one stage; `debrief <stage> --help` lists its flags
10
11
 
11
12
  Everything is kept under DEBRIEF_HOME (default ~/.local/share/debrief).
@@ -43,6 +44,7 @@ STAGES = {
43
44
  "github": "github",
44
45
  "prune": "prune",
45
46
  "config": "config",
47
+ "push": "push",
46
48
  }
47
49
 
48
50
 
@@ -251,6 +253,10 @@ def serve(argv: list[str]) -> None:
251
253
  # connection open for as long as the server lives, and its lock would fail those runs.
252
254
  for name in ("ingest", "export"):
253
255
  subprocess.run([sys.executable, "-m", f"{__package__}.cli", name], check=True)
256
+ else:
257
+ # The report keeps a copy of the page from its last build; after an upgrade that copy is old.
258
+ from debrief.export_site import copy_static
259
+ copy_static(SITE_OUT)
254
260
  signal.signal(signal.SIGTERM, signal.default_int_handler) # `kill` stops a run too, as Ctrl-C does
255
261
  url = f"http://127.0.0.1:{server.server_port}/"
256
262
  url += f"?t={ReportHandler.token}" # the page keeps the token for this tab and drops it from the address
@@ -29,6 +29,7 @@ from debrief import config, skills
29
29
  from debrief.db import connect, repo_projects, session_projects
30
30
  from debrief.paths import SITE_OUT, SITE_SRC
31
31
  from debrief.recommend import LEVERS
32
+ from debrief.redact import SECRETS, redact, redact_damaged, redact_tree # noqa: F401 (old import path)
32
33
  from debrief.relay import is_relayed
33
34
  from debrief.synthesize import ref_project
34
35
 
@@ -50,93 +51,6 @@ SIGNALS = {
50
51
  }
51
52
  CORRECTING = ("correction", "reversal", "defect") # together, the site's "Agent corrected"
52
53
 
53
- # Secret-shaped strings, redacted from every string before it is serialized. Redacting the
54
- # serialized JSON instead misses a secret that follows an escape: in "\nghp_..." the "n" of
55
- # the escape is a word character, so \b never matches.
56
- SECRETS = [
57
- (re.compile(r"\b(?:gh[pousr]_[A-Za-z0-9]{36,}|github_pat_[A-Za-z0-9_]{40,})"), "[redacted]"),
58
- (re.compile(r"\bsk-(?=[A-Za-z0-9_-]*\d)(?:ant-|proj-|or-v1-)?[A-Za-z0-9_-]{20,}"), "[redacted]"), # needs a digit
59
- (re.compile(r"\b(?:sk|rk|pk)_(?:live|test)_[A-Za-z0-9]{20,}"), "[redacted]"), # Stripe
60
- (re.compile(r"\b(?:AKIA|ASIA)[A-Z0-9]{16}\b"), "[redacted]"),
61
- (re.compile(r"\bxox[abeprs]-[A-Za-z0-9-]{10,}"), "[redacted]"),
62
- (re.compile(r"\bglpat-[A-Za-z0-9_-]{20,}"), "[redacted]"),
63
- (re.compile(r"\bAIza[A-Za-z0-9_-]{35}"), "[redacted]"), # Google API key
64
- (re.compile(r"\b(?:npm|hf)_[A-Za-z0-9]{30,}"), "[redacted]"),
65
- (re.compile(r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}"), "[redacted]"),
66
- (re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----.*?(?:-----END [A-Z ]*PRIVATE KEY-----|\Z)", re.DOTALL), "[redacted]"),
67
- (re.compile(r"(?i)\b(bearer\s+)[A-Za-z0-9._~+/-]{24,}=*"), r"\1[redacted]"),
68
- (re.compile(r"(https?://)[^/\s:@\"\\]+:[^/\s@\"\\]+@"), r"\1[redacted]@"),
69
- ]
70
-
71
-
72
- def redact(text: str) -> str:
73
- """Redact secrets in one plain (not JSON-encoded) string."""
74
- for pattern, repl in SECRETS:
75
- text = pattern.sub(repl, text)
76
- return text
77
-
78
-
79
- # A run of non-space text that holds U+FFFD and is at least this long without it is blanked whole.
80
- # Shorter than any token pattern above can match, so no secret touching a mark survives.
81
- DAMAGED_MIN = 16
82
- DAMAGED_RUN = re.compile(r"\S*\ufffd\S*")
83
-
84
-
85
- # In damaged text a private key's header may not survive well enough to match, so any text
86
- # that spells PRIVATE KEY once marks and spaces are ignored is cut from its first dashes on.
87
- DAMAGED_KEY = re.compile(r"P\W*R\W*I\W*V\W*A\W*T\W*E\W*K\W*E\W*Y")
88
- DAMAGED_KEY_END = re.compile(r"E\W*N\W*D\b[^\n]{0,60}?-{3,}")
89
-
90
-
91
- def redact_damaged(text: str) -> str:
92
- """Redact a string holding U+FFFD, which marks bytes the source could not decode, then drop it.
93
-
94
- A mark can sit inside a secret, or be the only break before or after one, in any mix, so
95
- no single way of reading the marks finds every secret. Three steps, each on what the
96
- last one left:
97
- 1. blank what the patterns find with the marks removed, and with the marks as spaces
98
- (this is what catches a secret of several words, such as a private key or a bearer
99
- token, whose first word a mark touches);
100
- 2. blank any long run of text that still touches a mark, secret or not;
101
- 3. the normal pass.
102
- """
103
- key = DAMAGED_KEY.search(text)
104
- if key:
105
- start = max(text.rfind("---", 0, key.start()), 0)
106
- start = len(text[:start].rstrip("-")) if start else 0
107
- ends = [m.end() for m in DAMAGED_KEY_END.finditer(text, key.end())]
108
- text = text[:start] + "[redacted]" + (text[ends[-1]:].lstrip("-") if ends else "")
109
- keep = [i for i, c in enumerate(text) if c != "\ufffd"]
110
- joined = "".join(text[i] for i in keep)
111
- spaced = text.replace("\ufffd", " ")
112
- spans = []
113
- for pattern, _ in SECRETS:
114
- spans += [(keep[m.start()], keep[m.end() - 1] + 1) for m in pattern.finditer(joined) if m.end() > m.start()]
115
- spans += [m.span() for m in pattern.finditer(spaced)]
116
- out, at = [], 0
117
- for start, end in sorted(spans):
118
- if start >= at:
119
- out += [text[at:start], "[redacted]"]
120
- at = max(at, end)
121
- text = "".join([*out, text[at:]])
122
-
123
- def blank(m: re.Match) -> str:
124
- rest = m.group().replace("\ufffd", "")
125
- return "[redacted]" if len(rest) >= DAMAGED_MIN else rest
126
- return redact(DAMAGED_RUN.sub(blank, text))
127
-
128
-
129
- def redact_tree(obj):
130
- """Redact every string, keys included, in a JSON-like structure, and drop U+FFFD (the artifact host rejects it)."""
131
- if isinstance(obj, str):
132
- return redact_damaged(obj) if "\ufffd" in obj else redact(obj)
133
- if isinstance(obj, (list, tuple)):
134
- return [redact_tree(v) for v in obj]
135
- if isinstance(obj, dict):
136
- return {redact_tree(k): redact_tree(v) for k, v in obj.items()}
137
- return obj
138
-
139
-
140
54
  def excerpt(text, n: int) -> str:
141
55
  text = re.sub(r"\s+", " ", str(text or "")).strip()
142
56
  return text if len(text) <= n else text[: n - 1] + "…"
@@ -587,6 +501,14 @@ def repo_of(prs: dict) -> str | None:
587
501
  return repos.pop() if len(repos) == 1 else None
588
502
 
589
503
 
504
+ def copy_static(out: Path) -> None:
505
+ """Put the page's own files (HTML, CSS, JS) in the site folder."""
506
+ out.mkdir(parents=True, exist_ok=True)
507
+ for static in SITE_SRC.iterdir():
508
+ if static.is_file():
509
+ shutil.copy(static, out / static.name)
510
+
511
+
590
512
  def main(argv: list[str] | None = None) -> None:
591
513
  parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
592
514
  parser.add_argument("--run", help="Findings run_id (default: latest with recommendations)")
@@ -631,10 +553,7 @@ def main(argv: list[str] | None = None) -> None:
631
553
  "ci": ci, "reviews": [r for r in review_section(conn) if r["pr"] in linked], "prs": prs,
632
554
  "skills": skills_section(conn, set(authors)),
633
555
  }
634
- SITE_OUT.mkdir(parents=True, exist_ok=True)
635
- for static in SITE_SRC.iterdir():
636
- if static.is_file():
637
- shutil.copy(static, SITE_OUT / static.name)
556
+ copy_static(SITE_OUT)
638
557
  out = SITE_OUT / "data"
639
558
  shard_dir = out / "sessions"
640
559
  shard_dir.mkdir(parents=True, exist_ok=True)
@@ -0,0 +1,237 @@
1
+ """Build what an upload to a team server would hold. There is no server yet, so only --dry-run works.
2
+
3
+ debrief push --dry-run write the bundle under DEBRIEF_HOME/push and say what is in it
4
+ debrief push --dry-run --out DIR write it to DIR
5
+ debrief push --dry-run --prev DIR send only what changed since the bundle in DIR
6
+
7
+ Only picked projects (`[projects] include` in config.toml) contribute anything. The rest are a
8
+ count. docs/upload-format.md describes the files.
9
+ """
10
+
11
+ import argparse
12
+ import hashlib
13
+ import json
14
+ import re
15
+ import sys
16
+ import uuid
17
+ from collections import Counter
18
+ from importlib.metadata import version
19
+ from pathlib import Path
20
+
21
+ from debrief import config
22
+ from debrief.db import connect, repo_projects, session_projects
23
+ from debrief.export_site import (
24
+ ci_section,
25
+ findings_and_recs,
26
+ labels_by_turn,
27
+ projects_section,
28
+ prs_section,
29
+ review_section,
30
+ session_file,
31
+ sessions_section,
32
+ )
33
+ from debrief.paths import HOME, NAME
34
+ from debrief.redact import VERSION, scrub
35
+ from debrief.sources import CLAUDE_DIR, CODEX_DIR
36
+ from debrief.synthesize import ref_project
37
+
38
+ FORMAT = 1
39
+ MACHINE_PATH = HOME / "machine_id"
40
+ DEFAULT_OUT = HOME / "push"
41
+ SAFE_NAME = re.compile(r"[A-Za-z0-9_-][\w.-]*", re.ASCII) # a session id that can be a file name as it is
42
+ NO_SERVER = (f"{NAME}: there is no server to push to yet. "
43
+ f"`{NAME} push --dry-run` builds what would be sent and sends nothing.")
44
+
45
+
46
+ def machine_id() -> tuple[str, bool]:
47
+ """This machine's id, a random UUID made on first use, and whether this call made it."""
48
+ if MACHINE_PATH.exists():
49
+ return MACHINE_PATH.read_text().strip(), False
50
+ HOME.mkdir(parents=True, exist_ok=True)
51
+ MACHINE_PATH.write_text(new := str(uuid.uuid4()))
52
+ return new, True
53
+
54
+
55
+ def object_path(session_id: str) -> str:
56
+ """Where a session's object goes. An id that is not a plain file name (logs can be imported,
57
+ so it may hold a slash) is named by its hash."""
58
+ name = session_id if SAFE_NAME.fullmatch(session_id) else hashlib.sha256(session_id.encode()).hexdigest()[:32]
59
+ return f"sessions/{name}.json"
60
+
61
+
62
+ def build(conn, picked: set[str], machine: str, have: dict[str, str] | None = None) -> tuple[dict, dict[str, bytes], dict]:
63
+ """(manifest, {path: bytes} for the objects to send, stats for the summary).
64
+
65
+ `have` is {path: sha256} of what the server already holds; an object with the same hash is
66
+ listed in the manifest but not sent.
67
+ """
68
+ have = have or {}
69
+ labels = labels_by_turn(conn)
70
+ run_id = conn.execute("SELECT MAX(run_id) FROM recommendations").fetchone()[0] \
71
+ or conn.execute("SELECT MAX(run_id) FROM findings").fetchone()[0]
72
+ rows = projects_section(conn, picked)
73
+ repo_of = {}
74
+ for repo, path in sorted(repo_projects(conn).items(), reverse=True):
75
+ repo_of[path] = repo # a folder tied to several repos takes the first by name
76
+ key = {r["path"]: repo_of.get(r["path"]) or "local:" + hashlib.sha256(f"{machine}:{r['path']}".encode()).hexdigest()[:12]
77
+ for r in rows if r["picked"]}
78
+
79
+ raw: dict[str, object] = {}
80
+ every, _ = sessions_section(conn, labels, "", picked)
81
+ # A session with an author came from Entire and may be a teammate's; a server would file it
82
+ # under whoever pushed. Local logs name no author.
83
+ mine = [s for s in every if not s["author"]]
84
+ # Nor is a log read from anywhere but the agents' own log folders on this machine: `--import`
85
+ # takes any folder, and a teammate's log names no author. The loaded row is the file the
86
+ # session's rows were read from.
87
+ local = {sid for sid, path in conn.execute("SELECT session_id, path FROM source_logs WHERE status = 'loaded'")
88
+ if ".." not in Path(path).parts and any(Path(path).is_relative_to(root) for root in (CLAUDE_DIR, CODEX_DIR))}
89
+ sessions = [s for s in mine if s["id"] in local]
90
+ for s in sessions:
91
+ raw[object_path(s["id"])] = {**s, "project": key[s["project"]], **session_file(conn, labels, s["id"])}
92
+ ids = {s["id"] for s in sessions}
93
+
94
+ findings, recs, refs = findings_and_recs(conn, labels, run_id, picked)
95
+ found, ideas = len(findings), len(recs)
96
+ findings, recs, refs = only_picked(conn, findings, recs, refs, key, ids)
97
+ left_out = {"findings": found - len(findings), "ideas": ideas - len(recs), "authored": len(every) - len(mine), "imported": len(mine) - len(sessions)}
98
+ if findings or recs:
99
+ for item in (*findings.values(), *recs):
100
+ item["project"] = item["project"] and key[item["project"]]
101
+ for ref in refs.values():
102
+ if "sessions" in ref: # a CI or review event names every session on its PR; keep the bundled ones
103
+ ref["sessions"] = [sid for sid in ref["sessions"] if sid in ids]
104
+ raw["findings.json"] = {"run_id": run_id, "findings": findings, "recommendations": recs, "refs": refs}
105
+
106
+ linked = {p["n"] for s in sessions for p in s["prs"]}
107
+ prs = {k: v for k, v in prs_section(conn).items() if k in linked}
108
+ if prs:
109
+ ci = ci_section(conn, "")
110
+ raw["github.json"] = {
111
+ "prs": prs,
112
+ "ci": {"runs_by_pr": {k: v for k, v in ci["runs_by_pr"].items() if k in linked},
113
+ "failures": [f for f in ci["failures"] if f["pr"] in linked]},
114
+ "reviews": [r for r in review_section(conn) if r["pr"] in linked]}
115
+
116
+ counts, objects, send = Counter(), {}, {}
117
+ for path, obj in raw.items():
118
+ clean, n = scrub(obj, home_paths=True, emails=True)
119
+ counts.update(n)
120
+ body = json.dumps(clean, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode()
121
+ objects[path] = {"sha256": hashlib.sha256(body).hexdigest(), "bytes": len(body)}
122
+ if have.get(path) != objects[path]["sha256"]:
123
+ send[path] = body
124
+ count, turns = Counter(s["project"] for s in sessions), Counter()
125
+ for s in sessions:
126
+ turns[s["project"]] += s["human_turns"]
127
+ projects, n = scrub([{"key": key[r["path"]], "name": r["name"], "sessions": count[r["path"]], "turns": turns[r["path"]],
128
+ "analysed_at": r["analysed_at"]} for r in rows if r["picked"]], home_paths=True, emails=True)
129
+ counts.update(n)
130
+ manifest = {"format": FORMAT, "tool": f"{NAME} {version('debrief-cli')}", "machine": machine,
131
+ "redaction": VERSION,
132
+ "projects": projects, "unpicked": sum(not r["picked"] for r in rows),
133
+ "objects": objects, "sent": sorted(send)}
134
+ stats = {
135
+ "sessions": len(sessions), "sessions_sent": sum(p.startswith("sessions/") for p in send),
136
+ "turns": sum(s["human_turns"] for s in sessions),
137
+ "messages": sum(len(raw[object_path(s["id"])]["messages"]) for s in sessions),
138
+ "secrets": counts["secrets"], "home_paths": counts["home_paths"], "emails": counts["emails"],
139
+ "repos": sorted({k for k in key.values() if not k.startswith("local:")} | {k.rsplit("#", 1)[0] for k in prs}),
140
+ "removed": sorted(set(have) - set(objects)), "left_out": left_out,
141
+ }
142
+ return manifest, send, stats
143
+
144
+
145
+ def only_picked(conn, findings: dict, recs: list, refs: dict, key: dict[str, str], ids: set[str]) -> tuple[dict, list, dict]:
146
+ """Cut the report's findings and ideas down to what an upload may hold. The report's own rule
147
+ keeps a finding when some of its evidence is in a picked project, and an idea when some of
148
+ its findings are kept; a model wrote each text from all of them, so that is too loose to send.
149
+
150
+ Kept: a finding whose stored evidence is all in the bundle (turns of sessions `ids`, events
151
+ of a picked project's repo), and an idea whose stored findings are all kept. Either must be
152
+ about a project with a key, or about all of them.
153
+ """
154
+ project, repos = session_projects(conn), repo_projects(conn)
155
+
156
+ def in_bundle(ref: dict) -> bool:
157
+ return ref["session_id"] in ids if "session_id" in ref else ref_project(conn, project, repos, ref) in key
158
+
159
+ def stored(table: str, column: str, id_column: str, row_id: int) -> list:
160
+ return json.loads(conn.execute(f"SELECT {column} FROM {table} WHERE {id_column} = ?", (row_id,)).fetchone()[0] or "[]")
161
+
162
+ findings = {i: f for i, f in findings.items() if f["project"] in (None, *key)
163
+ and all(in_bundle(e) for e in stored("findings", "evidence", "finding_id", i))}
164
+ recs = [r for r in recs if r["project"] in (None, *key)
165
+ and all(i in findings for i in stored("recommendations", "finding_ids", "rec_id", r["id"]))]
166
+ cited = {e for f in findings.values() for e in f["evidence"]}
167
+ return findings, recs, {i: ref for i, ref in refs.items() if i in cited}
168
+
169
+
170
+ def write(out: Path, manifest: dict, send: dict[str, bytes]) -> int:
171
+ """Write the bundle to `out`, replacing an earlier bundle there. Returns the bytes written."""
172
+ if out.exists() and any(out.iterdir()) and not (out / "manifest.json").exists():
173
+ sys.exit(f"{NAME}: {out} is not empty and is not an earlier bundle; name another folder")
174
+ out.mkdir(parents=True, exist_ok=True, mode=0o700) # it holds transcripts
175
+ (out / "sessions").mkdir(exist_ok=True)
176
+ for old in (*out.glob("sessions/*.json"), out / "findings.json", out / "github.json"):
177
+ old.unlink(missing_ok=True)
178
+ files = {**send, "manifest.json": json.dumps(manifest, ensure_ascii=False, indent=1).encode()}
179
+ for path, body in files.items():
180
+ (out / path).write_bytes(body)
181
+ return sum(len(b) for b in files.values())
182
+
183
+
184
+ def main(argv: list[str] | None = None) -> None:
185
+ p = argparse.ArgumentParser(prog=f"{NAME} push", description=__doc__,
186
+ formatter_class=argparse.RawDescriptionHelpFormatter)
187
+ p.add_argument("--dry-run", action="store_true", help="Build the bundle and write it to a folder; send nothing")
188
+ p.add_argument("--out", metavar="DIR", type=Path, default=DEFAULT_OUT, help=f"Where to write it (default {DEFAULT_OUT})")
189
+ p.add_argument("--prev", metavar="DIR", type=Path,
190
+ help="An earlier bundle, taken as what the server holds: only objects that changed since are written")
191
+ args = p.parse_args(argv)
192
+ if not args.dry_run:
193
+ sys.exit(NO_SERVER)
194
+ have = None
195
+ if args.prev:
196
+ try:
197
+ have = {path: o["sha256"] for path, o in json.loads((args.prev / "manifest.json").read_text())["objects"].items()}
198
+ except (OSError, ValueError, KeyError, TypeError, AttributeError) as e:
199
+ sys.exit(f"{NAME}: {args.prev} holds no bundle manifest ({e})")
200
+ machine, made = machine_id()
201
+ manifest, send, stats = build(connect(), set(config.projects()), machine, have)
202
+ size = write(args.out, manifest, send)
203
+
204
+ others = manifest["unpicked"]
205
+ print(f"Dry run: nothing was sent. The bundle is in {args.out} ({len(send) + 1} files, {size:,} bytes).")
206
+ print(f" projects {len(manifest['projects'])} picked")
207
+ for r in manifest["projects"]:
208
+ print(f" {r['name']} ({r['key']}; {r['sessions']} sessions, {r['turns']} turns)")
209
+ print(f" {others} other project{'' if others == 1 else 's'} on this machine "
210
+ f"{'is' if others == 1 else 'are'} not picked and nothing about them is sent")
211
+ print(f" sessions {stats['sessions']}, of which {stats['sessions_sent']} sent"
212
+ f"{' (the rest have not changed)' if have is not None else ''}")
213
+ print(f" turns {stats['turns']} of yours, {stats['messages']} messages in all")
214
+ print(f" redactions {stats['secrets']} secrets and {stats['emails']} email addresses replaced, "
215
+ f"{stats['home_paths']} home-folder paths shortened to ~")
216
+ print(f" repo names {', '.join(stats['repos']) or 'none'}")
217
+ if stats["removed"]:
218
+ gone = sum(path.startswith("sessions/") for path in stats["removed"])
219
+ print(f" This push removes {gone} session{'' if gone == 1 else 's'} the server holds "
220
+ f"({len(stats['removed'])} object{'' if len(stats['removed']) == 1 else 's'} no longer in a picked project).")
221
+ out = stats["left_out"]
222
+ if out["authored"]:
223
+ print(f" Left out {out['authored']} sessions: they are other people's (they name an author).")
224
+ if out["imported"]:
225
+ print(f" Left out {out['imported']} sessions: they were imported from another folder.")
226
+ if out["findings"] or out["ideas"]:
227
+ print(f" Left out {out['findings']} findings and {out['ideas']} ideas: "
228
+ "they draw on projects you have not picked or on sessions that are not sent.")
229
+ if made:
230
+ print(f" Made an id for this machine, a random number kept in {MACHINE_PATH}.")
231
+ print("Redaction is pattern-based: read the bundle before you trust it.")
232
+ print("Upload only what you have the right to share: transcripts hold your employer's or clients' code "
233
+ "and other people's names.")
234
+
235
+
236
+ if __name__ == "__main__":
237
+ main()