sensorium-trace 0.18.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (135) hide show
  1. sensorium/__init__.py +0 -0
  2. sensorium/__main__.py +4 -0
  3. sensorium/_c/rt/sensorium_c_rt.c +1446 -0
  4. sensorium/_typescript/package-lock.json +68 -0
  5. sensorium/_typescript/package.json +39 -0
  6. sensorium/_typescript/src/bindings.mjs +792 -0
  7. sensorium/_typescript/src/cap.mjs +227 -0
  8. sensorium/_typescript/src/capabilities.mjs +40 -0
  9. sensorium/_typescript/src/dbg.mjs +348 -0
  10. sensorium/_typescript/src/escape.mjs +351 -0
  11. sensorium/_typescript/src/focus.mjs +135 -0
  12. sensorium/_typescript/src/hook.mjs +127 -0
  13. sensorium/_typescript/src/index.mjs +1 -0
  14. sensorium/_typescript/src/naming.mjs +63 -0
  15. sensorium/_typescript/src/positions.mjs +37 -0
  16. sensorium/_typescript/src/probe.mjs +246 -0
  17. sensorium/_typescript/src/qualname.mjs +236 -0
  18. sensorium/_typescript/src/redact.mjs +711 -0
  19. sensorium/_typescript/src/register.mjs +15 -0
  20. sensorium/_typescript/src/resolve.mjs +260 -0
  21. sensorium/_typescript/src/rt.mjs +798 -0
  22. sensorium/_typescript/src/setup.mjs +11 -0
  23. sensorium/_typescript/src/tally.mjs +164 -0
  24. sensorium/_typescript/src/tasks.mjs +175 -0
  25. sensorium/_typescript/src/transform.mjs +800 -0
  26. sensorium/_typescript/src/vite.mjs +99 -0
  27. sensorium/c/__init__.py +8 -0
  28. sensorium/c/cc.py +612 -0
  29. sensorium/c/cli.py +60 -0
  30. sensorium/c/convert.py +669 -0
  31. sensorium/c/decode.py +300 -0
  32. sensorium/c/driver.py +365 -0
  33. sensorium/c/dwarf.py +706 -0
  34. sensorium/c/format.py +271 -0
  35. sensorium/c/pkg.py +187 -0
  36. sensorium/c/plan.py +277 -0
  37. sensorium/cli.py +140 -0
  38. sensorium/driver.py +47 -0
  39. sensorium/exit.py +33 -0
  40. sensorium/invocations.py +83 -0
  41. sensorium/limits.py +253 -0
  42. sensorium/mcp/__init__.py +0 -0
  43. sensorium/mcp/audit.py +162 -0
  44. sensorium/mcp/child.py +240 -0
  45. sensorium/mcp/cmd.py +101 -0
  46. sensorium/mcp/jsonrpc.py +172 -0
  47. sensorium/mcp/result.py +187 -0
  48. sensorium/mcp/schema.py +390 -0
  49. sensorium/mcp/server.py +493 -0
  50. sensorium/mcp/tools.py +299 -0
  51. sensorium/paths.py +80 -0
  52. sensorium/query/__init__.py +0 -0
  53. sensorium/query/caps.py +150 -0
  54. sensorium/query/dbg_dialects.py +64 -0
  55. sensorium/query/diff_cmd.py +748 -0
  56. sensorium/query/diff_notes.py +117 -0
  57. sensorium/query/exceptions_c.py +263 -0
  58. sensorium/query/exceptions_cmd.py +788 -0
  59. sensorium/query/exceptions_group.py +571 -0
  60. sensorium/query/exceptions_invocation.py +487 -0
  61. sensorium/query/exceptions_rust.py +683 -0
  62. sensorium/query/exceptions_typescript.py +688 -0
  63. sensorium/query/expr.py +606 -0
  64. sensorium/query/failures_cmd.py +306 -0
  65. sensorium/query/flow_cmd.py +778 -0
  66. sensorium/query/flow_report.py +128 -0
  67. sensorium/query/flow_values.py +195 -0
  68. sensorium/query/fmt.py +280 -0
  69. sensorium/query/frame_cmd.py +237 -0
  70. sensorium/query/grep_cmd.py +156 -0
  71. sensorium/query/help.py +18 -0
  72. sensorium/query/info_cmd.py +527 -0
  73. sensorium/query/info_rust.py +214 -0
  74. sensorium/query/info_typescript.py +261 -0
  75. sensorium/query/js_inspect.py +394 -0
  76. sensorium/query/moves.py +254 -0
  77. sensorium/query/redact_cmd.py +304 -0
  78. sensorium/query/refocus_cmd.py +661 -0
  79. sensorium/query/refocus_env.py +493 -0
  80. sensorium/query/refocus_facts.py +269 -0
  81. sensorium/query/refocus_licence.py +187 -0
  82. sensorium/query/refocus_report.py +327 -0
  83. sensorium/query/refocus_rust.py +537 -0
  84. sensorium/query/refocus_threads.py +197 -0
  85. sensorium/query/refocus_typescript.py +756 -0
  86. sensorium/query/refocus_world.py +666 -0
  87. sensorium/query/runs_cmd.py +213 -0
  88. sensorium/query/rust_debug.py +176 -0
  89. sensorium/query/sites.py +171 -0
  90. sensorium/query/tree_cmd.py +511 -0
  91. sensorium/query/tree_threads.py +168 -0
  92. sensorium/query/vocab.py +723 -0
  93. sensorium/query/watch_cmd.py +657 -0
  94. sensorium/record/__init__.py +0 -0
  95. sensorium/record/boot.py +797 -0
  96. sensorium/record/boot_io.py +141 -0
  97. sensorium/record/budget.py +135 -0
  98. sensorium/record/capture.py +433 -0
  99. sensorium/record/children.py +755 -0
  100. sensorium/record/children_merge.py +179 -0
  101. sensorium/record/elide.py +55 -0
  102. sensorium/record/fingerprint.py +26 -0
  103. sensorium/record/thread_starts.py +261 -0
  104. sensorium/record/tracer.py +798 -0
  105. sensorium/record/tracer_exc.py +199 -0
  106. sensorium/record/tracer_frames.py +618 -0
  107. sensorium/redact.py +304 -0
  108. sensorium/redact_content.py +156 -0
  109. sensorium/redact_key.py +317 -0
  110. sensorium/redact_store.py +502 -0
  111. sensorium/redact_values.py +433 -0
  112. sensorium/store/__init__.py +0 -0
  113. sensorium/store/db.py +284 -0
  114. sensorium/store/gc.py +360 -0
  115. sensorium/store/payloads.py +69 -0
  116. sensorium/store/reader.py +438 -0
  117. sensorium/store/writer.py +283 -0
  118. sensorium/ts/__init__.py +10 -0
  119. sensorium/ts/build.py +783 -0
  120. sensorium/ts/cli.py +130 -0
  121. sensorium/ts/driver.py +472 -0
  122. sensorium/ts/focus.py +146 -0
  123. sensorium/ts/harness.py +314 -0
  124. sensorium/ts/ingest.py +380 -0
  125. sensorium/ts/invocation.py +252 -0
  126. sensorium/ts/pkg.py +167 -0
  127. sensorium/ts/redaction.py +64 -0
  128. sensorium/ts/spool.py +168 -0
  129. sensorium/ts/truncation.py +81 -0
  130. sensorium/ts/wrapper.py +255 -0
  131. sensorium_trace-0.18.2.dist-info/METADATA +245 -0
  132. sensorium_trace-0.18.2.dist-info/RECORD +135 -0
  133. sensorium_trace-0.18.2.dist-info/WHEEL +4 -0
  134. sensorium_trace-0.18.2.dist-info/entry_points.txt +3 -0
  135. sensorium_trace-0.18.2.dist-info/licenses/LICENSE +21 -0
sensorium/redact.py ADDED
@@ -0,0 +1,304 @@
1
+ """Rule v1: which NAMES hold a secret, and the knobs that amend that
2
+ judgement.
3
+
4
+ Every recorder stored the whole process environment in plaintext until this
5
+ slice, so a trace of any program launched from this box's shell held a live
6
+ `CLAUDE_CODE_MESSAGING_TOKEN` at 0644. Rule v1 replaces such a value with
7
+ `<redacted>` at the WRITER, before anything reaches disk.
8
+
9
+ The digest that stands in its place, and the per-store key the digests are
10
+ taken under, are `redact_key`'s -- re-exported below, so `redact.Key` and
11
+ `redact.KEY_VAR` still resolve here.
12
+
13
+ WHAT THIS IS NOT
14
+ ----------------
15
+ **Not a secret scanner.** The rule fires on a name's SEGMENTS, so a secret
16
+ in a variable called `x`, or one this set has never heard of, is stored as
17
+ typed; the content rule (patterns inside a value, whatever the name) is
18
+ PR B's and is a floor too. `SENSORIUM_REDACT_NAMES` is the remedy for a
19
+ name this misses and `SENSORIUM_REDACT_ALLOW` for one it wrongly fires on,
20
+ and both are recorded so a reader sees what the recording was made under.
21
+ Matching is segment-exact, never substring: substring `KEY` would fire on
22
+ `MONKEY_PATCH` and substring `PASS` on `BYPASS_CACHE`. The false positives
23
+ that do fire (`SSH_AUTH_SOCK`, `GPG_KEY_ID`: a path and an id) cost one
24
+ plaintext path in a trace, which the digest still makes comparable.
25
+
26
+ THE FIXTURE
27
+ -----------
28
+ `docs/trace-format/redaction-v1.json` holds every `split` and `fires` case,
29
+ and `sensorium-rt`'s and the TypeScript recorder's suites read the SAME
30
+ file. A case belongs there, not in a test module: three implementations of
31
+ one rule agree only on what all three are asked.
32
+
33
+ This module is pure -- it imports nothing from `record` or `query` -- and
34
+ never raises.
35
+ """
36
+ import re
37
+ from collections.abc import Mapping
38
+ from dataclasses import dataclass
39
+
40
+ # The store key, split out at this module's own seam: the name rule
41
+ # above and the content rule beside it both grow in here, and the key is
42
+ # neither of them. Re-exported so `redact.Key` and `redact.KEY_VAR` keep
43
+ # resolving: these are one rule's internals across two files, not
44
+ # separate modules with surfaces of their own.
45
+ from sensorium.redact_key import ( # noqa: F401
46
+ KEY_BYTES, KEY_FILE, KEY_MODE, KEY_VAR, Key)
47
+
48
+ #: The rule's identity, stamped in every trace it touches. A later rule is
49
+ #: v2; v1 is never amended, because a trace that says `v1` has to mean one
50
+ #: thing forever.
51
+ RULE = "v1"
52
+
53
+ #: What a whole-redacted value reads as. The spelling this repo's renderers
54
+ #: already use for a withheld thing (`<unread>`, `<not in scope>`).
55
+ REDACTED = "<redacted>"
56
+
57
+ OFF_VAR = "SENSORIUM_NO_REDACT"
58
+ NAMES_VAR = "SENSORIUM_REDACT_NAMES"
59
+ ALLOW_VAR = "SENSORIUM_REDACT_ALLOW"
60
+
61
+ #: §2.1's set, plus `PWD` (ruling R10, below). A segment here fires the rule;
62
+ #: every one of them has a firing case in the fixture, and `test_the_fixture_
63
+ #: gives_every_segment_of_the_rule_a_firing_case` is what keeps that true. The
64
+ #: four compounds (`APIKEY`, `ACCESSKEY`, `SECRETKEY`, `AUTHTOKEN`) exist
65
+ #: because a name written as one word -- `apikey` -- splits into one segment
66
+ #: that none of `API`, `KEY` matches.
67
+ SEGMENTS = frozenset("""
68
+ KEY APIKEY TOKEN SECRET SECRETS PASSWORD PASSWD PASSPHRASE PASS
69
+ AUTH AUTHORIZATION CREDENTIAL CREDENTIALS CREDS PRIVATE
70
+ COOKIE COOKIES SIGNATURE BEARER JWT DSN ACCESSKEY SECRETKEY AUTHTOKEN
71
+ PWD
72
+ """.split())
73
+
74
+ #: The segments that fire only as part of a LONGER name (ruling B28, for
75
+ #: `KEY`). `PWD`: `MYSQL_PWD` and `DB_PWD` are passwords, and `PWD` and
76
+ #: `OLDPWD` are the shell's working directory, on every machine that has ever
77
+ #: run a shell. `KEY`: a bare `key` is a cache key, a dict key, a lookup key
78
+ #: on almost every function that iterates a mapping, and redacting it by
79
+ #: default would blind `watch` on the commonest local in the language, while
80
+ #: every compound spelling (`api_key`, `apiKey`, `secret_key`, `build_key`,
81
+ #: `KEY_FILE`) still fires and `SENSORIUM_REDACT_NAMES=key` restores it per
82
+ #: run. §2.1 names both `PWD` and `OLDPWD` as non-firing and they still are --
83
+ #: one segment, so this condition holds them out while the two-segment
84
+ #: database variables come in.
85
+ _SOLO_EXEMPT = frozenset({"PWD", "KEY"})
86
+
87
+ #: Whole NORMALISED names that fire whatever their segments say. Segment-exact
88
+ #: matching cannot see inside a compound word: `PGPASSWORD` splits into the
89
+ #: single segment `PGPASSWORD`, which is neither `PG` nor `PASSWORD`, and
90
+ #: libpq's standard password variable was stored in plaintext by rule v1 until
91
+ #: the census of `tests/fixtures/benign-env-names.txt` said so. Kept apart
92
+ #: from `SEGMENTS` because it is a different question -- "is this word one of
93
+ #: the few known one-word secrets" rather than "does this name have a
94
+ #: secret-shaped part" -- and because a name dropped in here silently would
95
+ #: otherwise widen the segment rule for every name that contains it. Short by
96
+ #: design: a name earns a place only when a census or a report shows it
97
+ #: matters, never on imagination.
98
+ EXACT: frozenset[str] = frozenset({"PGPASSWORD"})
99
+
100
+ #: The two camelCase boundaries, in this order: an uppercase RUN followed by
101
+ #: Upper+lower (`HTTPToken` -> `HTTP Token`), then lower-or-digit followed by
102
+ #: upper (`apiKey` -> `api Key`). Verbatim from §2.1, because three
103
+ #: implementers would otherwise spell them three ways.
104
+ _CAMEL_RUN = re.compile(r"([A-Z]+)([A-Z][a-z])")
105
+ _CAMEL_STEP = re.compile(r"([a-z0-9])([A-Z])")
106
+ _SEPARATOR = re.compile(r"[^A-Za-z0-9]+")
107
+
108
+
109
+ def split(name: str) -> list[str]:
110
+ """`name`'s uppercased segments: `apiKey` -> `[API, KEY]`,
111
+ `SSH_AUTH_SOCK` -> `[SSH, AUTH, SOCK]`, `XAUTHORITY` -> `[XAUTHORITY]`.
112
+
113
+ Empty segments are dropped, so `__init__` is `[INIT]` and `""` is `[]`.
114
+ """
115
+ spaced = _CAMEL_STEP.sub(r"\1 \2", _CAMEL_RUN.sub(r"\1 \2", name))
116
+ return [part.upper() for part in _SEPARATOR.split(spaced) if part]
117
+
118
+
119
+ def normalise(name: str) -> str:
120
+ """The comparison form the two knobs use: the segments, joined. So
121
+ `myco_dsn`, `MYCO_DSN` and `mycoDsn` are one name to a user's list."""
122
+ return "".join(split(name))
123
+
124
+
125
+ @dataclass(frozen=True)
126
+ class Knobs:
127
+ """The three environment variables, as read once at record time.
128
+
129
+ `names` and `allow` are held NORMALISED, so a comparison against a
130
+ variable's own normalised form is exact and case-blind at once.
131
+ """
132
+ off: bool
133
+ names: frozenset[str]
134
+ allow: frozenset[str]
135
+
136
+ @classmethod
137
+ def from_environ(cls, environ: Mapping[str, str]) -> "Knobs":
138
+ """`SENSORIUM_NO_REDACT` reads like `SENSORIUM_NO_INVOCATION_LOG`
139
+ (`invocations._log_disabled`): any non-empty value other than `"0"`
140
+ turns the rule off, and `=0`, empty and unset all leave it on."""
141
+ off = environ.get(OFF_VAR)
142
+ return cls(bool(off) and off != "0",
143
+ _listed(environ, NAMES_VAR), _listed(environ, ALLOW_VAR))
144
+
145
+ def meta(self) -> dict:
146
+ return {"names": sorted(self.names), "allow": sorted(self.allow)}
147
+
148
+
149
+ def _listed(environ: Mapping[str, str], var: str) -> frozenset[str]:
150
+ """A comma list, stripped, empties dropped, each entry normalised. An
151
+ entry that normalises to nothing (`","`, `"-"`) is dropped too -- an
152
+ empty string in the set would match a nameless value."""
153
+ raw = (environ.get(var) or "").split(",")
154
+ return frozenset(n for n in (normalise(part.strip()) for part in raw) if n)
155
+
156
+
157
+ def fires(name: str, knobs: Knobs) -> bool:
158
+ """Whether the name rule fires on `name`.
159
+
160
+ Allow first, and it WINS over `names` -- a user's statement about their
161
+ own variable outranks their own list (§2.3). Then `EXACT`, then the
162
+ segments. `knobs.off` is not read here: turning the whole rule off is
163
+ `env`'s business, one level up, so that this function stays the answer to
164
+ "is this a secret-shaped name".
165
+ """
166
+ segments = split(name)
167
+ normalised = "".join(segments)
168
+ if normalised in knobs.allow:
169
+ return False
170
+ if normalised in knobs.names:
171
+ return True
172
+ if normalised in EXACT:
173
+ return True
174
+ multi = len(segments) > 1
175
+ return any(segment in SEGMENTS and (multi or segment not in _SOLO_EXEMPT)
176
+ for segment in segments)
177
+
178
+
179
+ def env(environ: Mapping[str, str], key: Key,
180
+ knobs: Knobs) -> tuple[dict[str, str], dict[str, str | None]]:
181
+ """(the environment to STORE, the name -> digest table for `redaction`).
182
+
183
+ A NEW dict either way -- the caller's mapping is never touched -- minus
184
+ `KEY_VAR`. With the rule off, that deletion is the only thing that
185
+ happens, and `meta` says `mode: "off"` so a reader knows the plaintext
186
+ was recorded on purpose. Applied to an ALREADY redacted environment this
187
+ would digest the marker; §5.4 is why that never happens -- a converter
188
+ meeting a `mode: "on"` header trusts it and does not re-run the rule.
189
+
190
+ The table's keys are SORTED (R13) so that one environment gives one
191
+ table, byte for byte, whichever language wrote it -- Rust's `BTreeMap`
192
+ and the TypeScript implementation sort too. The stored environment keeps
193
+ the order it arrived in; only the table is ordered.
194
+ """
195
+ stored = {name: value for name, value in environ.items()
196
+ if name != KEY_VAR}
197
+ if knobs.off:
198
+ return stored, {}
199
+ table: dict[str, str | None] = {}
200
+ for name in sorted(stored):
201
+ if fires(name, knobs):
202
+ table[name] = key.digest(stored[name])
203
+ stored[name] = REDACTED
204
+ return stored, table
205
+
206
+
207
+ def meta(key: Key, knobs: Knobs, table: dict[str, str | None],
208
+ by: str = "recorder") -> dict:
209
+ """The trace's `redaction` key (§4.3). `by` is the LAST hand that applied
210
+ the rule: `recorder`, `converter` or `retrofit`."""
211
+ if knobs.off:
212
+ return {"rule": RULE, "mode": "off"}
213
+ return {"rule": RULE, "mode": "on", "keyed": key.keyed,
214
+ "key_id": key.key_id, "env": table, **knobs.meta(), "by": by}
215
+
216
+
217
+ #: One side's account of one name: its digest, the key that digest was taken
218
+ #: under, and whether that side had a key at all. `None` in place of a Side
219
+ #: means the side did not redact the name -- its value is plaintext.
220
+ Side = tuple[str | None, str | None, bool]
221
+
222
+
223
+ def compare(before: str | None, after: str | None, was: Side | None,
224
+ now: Side | None, key: Key) -> bool | None:
225
+ """§6.2's table, once, for one name: True equal, False differs, None
226
+ unverifiable.
227
+
228
+ `before`/`after` are the two sides' stored values -- read only on a side
229
+ that did NOT redact, where the value is the plaintext itself. A value
230
+ missing from the plaintext side is a DIFFERENCE and a verified one (the
231
+ variable vanished), never an unverifiability.
232
+
233
+ Called only for a name at least one side redacted; plain equality is
234
+ the caller's answer for every other name, as it always was.
235
+ """
236
+ if was is not None and now is not None:
237
+ (mine, my_key, my_keyed), (theirs, their_key, their_keyed) = was, now
238
+ if not (my_keyed and their_keyed) or my_key != their_key:
239
+ return None
240
+ if mine is None or theirs is None:
241
+ return None
242
+ return mine == theirs
243
+ if was is None and now is None:
244
+ return before == after
245
+ side, plain = (was, after) if was is not None else (now, before)
246
+ digest, under, keyed = side
247
+ if digest is None or not keyed or not key.keyed or key.key_id != under:
248
+ return None
249
+ if plain is None:
250
+ return False
251
+ return key.digest(plain) == digest
252
+
253
+
254
+ def uncomparable(was_meta: dict, now_meta: dict | None,
255
+ key: Key) -> tuple[list[str], str | None]:
256
+ """The redacted names whose comparison `compare` cannot decide, sorted,
257
+ and the reason word -- `([], None)` when there are none.
258
+
259
+ `now_meta is None` is a live plaintext side with no table of its own
260
+ (the Python re-run): its names are verifiable under the store's key,
261
+ and appear here only when that key is gone or is not the one that wrote
262
+ the digests.
263
+ """
264
+ was_table, was_env = _table(was_meta), _stored(was_meta)
265
+ now_table, now_env = _table(now_meta), _stored(now_meta)
266
+ names, unkeyed = [], False
267
+ for name in sorted(set(was_table) | set(now_table)):
268
+ mine, theirs = _side(was_meta, name), _side(now_meta, name)
269
+ if compare(was_env.get(name), now_env.get(name),
270
+ mine, theirs, key) is not None:
271
+ continue
272
+ names.append(name)
273
+ unkeyed = unkeyed or _unkeyed(mine) or _unkeyed(theirs)
274
+ if mine is None or theirs is None:
275
+ # One side is plaintext, so the STORE's key is the one doing the
276
+ # verifying and its absence is the reason.
277
+ unkeyed = unkeyed or not key.keyed
278
+ if not names:
279
+ return [], None
280
+ return names, "unkeyed" if unkeyed else "different keys"
281
+
282
+
283
+ def _redaction(meta_dict: dict | None) -> dict:
284
+ return ((meta_dict or {}).get("redaction") or {})
285
+
286
+
287
+ def _table(meta_dict: dict | None) -> dict:
288
+ return _redaction(meta_dict).get("env") or {}
289
+
290
+
291
+ def _stored(meta_dict: dict | None) -> dict:
292
+ return (meta_dict or {}).get("env") or {}
293
+
294
+
295
+ def _side(meta_dict: dict | None, name: str) -> Side | None:
296
+ table = _table(meta_dict)
297
+ if name not in table:
298
+ return None
299
+ redaction = _redaction(meta_dict)
300
+ return table[name], redaction.get("key_id"), bool(redaction.get("keyed"))
301
+
302
+
303
+ def _unkeyed(side: Side | None) -> bool:
304
+ return side is not None and (not side[2] or side[0] is None)
@@ -0,0 +1,156 @@
1
+ """Rule v1's CONTENT half (§2.2): a fixed list of patterns over a value's
2
+ TEXT, whatever its name. Beside `redact.py`'s name rule, which decides
3
+ whether a whole value is worth redacting at all; this module decides which
4
+ SPANS inside a value's text look like a secret regardless of what it was
5
+ called.
6
+
7
+ WHAT THIS IS NOT
8
+ ----------------
9
+ **Not a secret scanner.** The list is a floor: nineteen shapes, each with a
10
+ minimum length so a short benign string cannot fire. A secret that matches
11
+ none of them is stored as typed.
12
+
13
+ THE FIXTURE
14
+ -----------
15
+ `docs/trace-format/redaction-v1.json`'s `content` list holds every case, and
16
+ `sensorium-rt`'s converter (`redact_content.rs`, `regex` crate -- the
17
+ runtime stays dependency-free, §5.2) and the TypeScript recorder's suite
18
+ read the SAME file. The three implementations are the same nineteen
19
+ patterns, textually, save for the one place the three engines cannot agree
20
+ on a spelling: end-of-text in the PEM row's truncated form (`\\Z` here,
21
+ `\\z` in Rust, `$` under JavaScript's `s` flag) and inline case/dotall flags
22
+ (`(?i)`/`(?s)` here and in Rust; the `i`/`s` RegExp flags in JavaScript,
23
+ which has no inline spelling for either).
24
+
25
+ TWO OPERATIONS, ONE MODULE
26
+ ---------------------------
27
+ §2.3: a name hit redacts the WHOLE value (`redact.py`); a content hit
28
+ replaces the matched SPAN and keeps everything around it
29
+ (`postgres://u:<redacted>@h/db`) -- a repr or a log line is not a value
30
+ with an identity, it is text that happened to contain one, and a partial
31
+ cannot honestly carry a digest of the whole. `hit` below means exactly
32
+ "the text changed": a pattern that matches a span already holding the
33
+ marker (an env value the name rule already redacted, then rendered
34
+ somewhere) is not a second hit, and `content` must say so, because the
35
+ converters (Task 6/7) count `values` from this flag.
36
+
37
+ This module is pure -- it imports nothing but `redact.REDACTED` -- and never
38
+ raises.
39
+ """
40
+ import re
41
+
42
+ from sensorium.redact import REDACTED
43
+
44
+ #: (row name, compiled pattern, replaced group: 0 = whole match). Verbatim
45
+ #: from §2.2's table, in the order that table gives -- table order matters
46
+ #: exactly once, for `sk-ant` before `sk`: both match an Anthropic key, the
47
+ #: whole match is replaced either way so the two rows' `after` never differs,
48
+ #: and the order is only for which row NAME a test or a report attributes
49
+ #: the hit to.
50
+ PATTERNS: tuple[tuple[str, "re.Pattern[str]", int], ...] = (
51
+ ("url-userinfo",
52
+ re.compile(r"://[^/\s:@]{1,64}:([^@\s/]{1,256})@"), 1),
53
+ # The PEM body, through the matching END line or to end of text when
54
+ # truncated -- `\Z` is this engine's spelling of that end-of-text
55
+ # alternative; Rust's `redact_content.rs` spells it `\z` and JavaScript's
56
+ # spells it `$` under the `s` flag. `(?s)` is inline DOTALL: the body
57
+ # spans real newlines.
58
+ ("pem",
59
+ re.compile(r"(?s)-----BEGIN [A-Z ]*PRIVATE KEY-----(.*?)"
60
+ r"(?:-----END [A-Z ]*PRIVATE KEY-----|\Z)"), 1),
61
+ ("authorization-header",
62
+ re.compile(r"(?i)\b(Bearer|Basic)\s+([A-Za-z0-9._~+/=-]{16,})"), 2),
63
+ ("jwt",
64
+ re.compile(r"eyJ[A-Za-z0-9_-]{8,}\.eyJ[A-Za-z0-9_-]{8,}"
65
+ r"\.[A-Za-z0-9_-]{8,}"), 0),
66
+ ("sk-ant", re.compile(r"\bsk-ant-[A-Za-z0-9_-]{20,}"), 0),
67
+ ("sk", re.compile(r"\bsk-[A-Za-z0-9_-]{20,}"), 0),
68
+ ("stripe", re.compile(r"\b(sk|rk)_(live|test)_[A-Za-z0-9]{16,}"), 0),
69
+ ("github", re.compile(r"\bgh[pousr]_[A-Za-z0-9]{20,}"), 0),
70
+ ("github-pat", re.compile(r"\bgithub_pat_[A-Za-z0-9_]{20,}"), 0),
71
+ ("gitlab", re.compile(r"\bglpat-[A-Za-z0-9_-]{20,}"), 0),
72
+ ("slack", re.compile(r"\bxox[abprs]-[A-Za-z0-9-]{10,}"), 0),
73
+ ("aws", re.compile(r"\b(AKIA|ASIA)[0-9A-Z]{16}\b"), 0),
74
+ ("google", re.compile(r"\bAIza[0-9A-Za-z_-]{35}\b"), 0),
75
+ ("huggingface", re.compile(r"\bhf_[A-Za-z0-9]{20,}"), 0),
76
+ ("npm", re.compile(r"\bnpm_[A-Za-z0-9]{20,}"), 0),
77
+ ("pypi", re.compile(r"\bpypi-[A-Za-z0-9_-]{20,}"), 0),
78
+ ("digitalocean", re.compile(r"\bdop_v1_[a-f0-9]{20,}"), 0),
79
+ ("shopify", re.compile(r"\bshpat_[a-f0-9]{20,}"), 0),
80
+ ("sendgrid",
81
+ re.compile(r"\bSG\.[A-Za-z0-9_-]{16,}\.[A-Za-z0-9_-]{16,}"), 0),
82
+ )
83
+
84
+ #: B18's pre-check (§12's mitigation, built in rather than waited for): the
85
+ #: literal prefix every CASE-SENSITIVE pattern above starts with, as one
86
+ #: alternation. A string that matches neither this nor `_TRIGGER_CI` cannot
87
+ #: match any of the nineteen patterns either, so `content` skips the whole
88
+ #: table rather than running twenty regexes over every string a recorder
89
+ #: ever touches.
90
+ _TRIGGER = re.compile("|".join(re.escape(p) for p in (
91
+ "://", "-----BEGIN", "eyJ", "sk-", "sk_", "rk_", "gh", "github_pat_",
92
+ "glpat-", "xox", "AKIA", "ASIA", "AIza", "hf_", "npm_", "pypi-",
93
+ "dop_v1_", "shpat_", "SG.",
94
+ )))
95
+
96
+ #: The same pre-check for the one pattern above that is itself `(?i)`.
97
+ #: `authorization-header` accepts every case spelling of each word, and a
98
+ #: literal alternation of `Bearer|Basic|bearer|basic` stood in front of it
99
+ #: covering two of each: `Authorization: BEARER <token>` matched no literal,
100
+ #: skipped the table, and was stored in plaintext. A pre-check NARROWER than
101
+ #: pattern it guards is a leak and not an optimisation (ruling R21). Its own
102
+ #: object rather than an inline `(?i:...)` group because the TypeScript twin
103
+ #: has no inline-flag spelling, and the three implementations have to agree
104
+ #: in behaviour rather than in syntax.
105
+ _TRIGGER_CI = re.compile("Bearer|Basic", re.IGNORECASE)
106
+
107
+
108
+ def triggers(text: str) -> bool:
109
+ """Whether any §2.2 pattern could match `text` at all: the union of the
110
+ two pre-checks, and the one thing `content` consults.
111
+
112
+ The union is what a test may pin. Either half alone is a pre-check for
113
+ part of the table, and pinning one of them proves nothing about the
114
+ shapes the other stands in front of.
115
+ """
116
+ return bool(_TRIGGER.search(text) or _TRIGGER_CI.search(text))
117
+
118
+
119
+ def content(text: str) -> tuple[str, bool]:
120
+ """`text` with every matched span replaced by `REDACTED`, and whether
121
+ the text CHANGED -- never whether some pattern merely matched.
122
+
123
+ That distinction is the whole of the contract: `url-userinfo` matches
124
+ `postgres://u:<redacted>@h/db` (its group already reads as the marker),
125
+ and replacing it with itself is not a hit. Comparing the WHOLE result to
126
+ the input, once, at the end, is what makes that true without a special
127
+ case for it -- the converters that count `values` from this flag rely on
128
+ it (Task 6/7).
129
+
130
+ Applied left to right, pattern by pattern in table order, over the
131
+ CURRENT text -- so a later pattern sees an earlier pattern's markers,
132
+ never the original secret twice.
133
+ """
134
+ if not text or not triggers(text):
135
+ return text, False
136
+ out = text
137
+ for _, pattern, group in PATTERNS:
138
+ out = pattern.sub(_replacer(group), out)
139
+ return out, out != text
140
+
141
+
142
+ def _replacer(group: int):
143
+ """One pattern's substitution function: the whole match for group 0, or
144
+ the match with only `group`'s span swapped for `REDACTED` -- never the
145
+ whole match when a narrower group was asked for (§2.3's span operation:
146
+ everything around the secret is kept)."""
147
+ if group == 0:
148
+ return lambda m: REDACTED
149
+
150
+ def _swap(m: "re.Match[str]") -> str:
151
+ whole = m.group(0)
152
+ start = m.start(group) - m.start(0)
153
+ end = m.end(group) - m.start(0)
154
+ return whole[:start] + REDACTED + whole[end:]
155
+
156
+ return _swap