sensorium-trace 0.18.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sensorium/__init__.py +0 -0
- sensorium/__main__.py +4 -0
- sensorium/_c/rt/sensorium_c_rt.c +1446 -0
- sensorium/_typescript/package-lock.json +68 -0
- sensorium/_typescript/package.json +39 -0
- sensorium/_typescript/src/bindings.mjs +792 -0
- sensorium/_typescript/src/cap.mjs +227 -0
- sensorium/_typescript/src/capabilities.mjs +40 -0
- sensorium/_typescript/src/dbg.mjs +348 -0
- sensorium/_typescript/src/escape.mjs +351 -0
- sensorium/_typescript/src/focus.mjs +135 -0
- sensorium/_typescript/src/hook.mjs +127 -0
- sensorium/_typescript/src/index.mjs +1 -0
- sensorium/_typescript/src/naming.mjs +63 -0
- sensorium/_typescript/src/positions.mjs +37 -0
- sensorium/_typescript/src/probe.mjs +246 -0
- sensorium/_typescript/src/qualname.mjs +236 -0
- sensorium/_typescript/src/redact.mjs +711 -0
- sensorium/_typescript/src/register.mjs +15 -0
- sensorium/_typescript/src/resolve.mjs +260 -0
- sensorium/_typescript/src/rt.mjs +798 -0
- sensorium/_typescript/src/setup.mjs +11 -0
- sensorium/_typescript/src/tally.mjs +164 -0
- sensorium/_typescript/src/tasks.mjs +175 -0
- sensorium/_typescript/src/transform.mjs +800 -0
- sensorium/_typescript/src/vite.mjs +99 -0
- sensorium/c/__init__.py +8 -0
- sensorium/c/cc.py +612 -0
- sensorium/c/cli.py +60 -0
- sensorium/c/convert.py +669 -0
- sensorium/c/decode.py +300 -0
- sensorium/c/driver.py +365 -0
- sensorium/c/dwarf.py +706 -0
- sensorium/c/format.py +271 -0
- sensorium/c/pkg.py +187 -0
- sensorium/c/plan.py +277 -0
- sensorium/cli.py +140 -0
- sensorium/driver.py +47 -0
- sensorium/exit.py +33 -0
- sensorium/invocations.py +83 -0
- sensorium/limits.py +253 -0
- sensorium/mcp/__init__.py +0 -0
- sensorium/mcp/audit.py +162 -0
- sensorium/mcp/child.py +240 -0
- sensorium/mcp/cmd.py +101 -0
- sensorium/mcp/jsonrpc.py +172 -0
- sensorium/mcp/result.py +187 -0
- sensorium/mcp/schema.py +390 -0
- sensorium/mcp/server.py +493 -0
- sensorium/mcp/tools.py +299 -0
- sensorium/paths.py +80 -0
- sensorium/query/__init__.py +0 -0
- sensorium/query/caps.py +150 -0
- sensorium/query/dbg_dialects.py +64 -0
- sensorium/query/diff_cmd.py +748 -0
- sensorium/query/diff_notes.py +117 -0
- sensorium/query/exceptions_c.py +263 -0
- sensorium/query/exceptions_cmd.py +788 -0
- sensorium/query/exceptions_group.py +571 -0
- sensorium/query/exceptions_invocation.py +487 -0
- sensorium/query/exceptions_rust.py +683 -0
- sensorium/query/exceptions_typescript.py +688 -0
- sensorium/query/expr.py +606 -0
- sensorium/query/failures_cmd.py +306 -0
- sensorium/query/flow_cmd.py +778 -0
- sensorium/query/flow_report.py +128 -0
- sensorium/query/flow_values.py +195 -0
- sensorium/query/fmt.py +280 -0
- sensorium/query/frame_cmd.py +237 -0
- sensorium/query/grep_cmd.py +156 -0
- sensorium/query/help.py +18 -0
- sensorium/query/info_cmd.py +527 -0
- sensorium/query/info_rust.py +214 -0
- sensorium/query/info_typescript.py +261 -0
- sensorium/query/js_inspect.py +394 -0
- sensorium/query/moves.py +254 -0
- sensorium/query/redact_cmd.py +304 -0
- sensorium/query/refocus_cmd.py +661 -0
- sensorium/query/refocus_env.py +493 -0
- sensorium/query/refocus_facts.py +269 -0
- sensorium/query/refocus_licence.py +187 -0
- sensorium/query/refocus_report.py +327 -0
- sensorium/query/refocus_rust.py +537 -0
- sensorium/query/refocus_threads.py +197 -0
- sensorium/query/refocus_typescript.py +756 -0
- sensorium/query/refocus_world.py +666 -0
- sensorium/query/runs_cmd.py +213 -0
- sensorium/query/rust_debug.py +176 -0
- sensorium/query/sites.py +171 -0
- sensorium/query/tree_cmd.py +511 -0
- sensorium/query/tree_threads.py +168 -0
- sensorium/query/vocab.py +723 -0
- sensorium/query/watch_cmd.py +657 -0
- sensorium/record/__init__.py +0 -0
- sensorium/record/boot.py +797 -0
- sensorium/record/boot_io.py +141 -0
- sensorium/record/budget.py +135 -0
- sensorium/record/capture.py +433 -0
- sensorium/record/children.py +755 -0
- sensorium/record/children_merge.py +179 -0
- sensorium/record/elide.py +55 -0
- sensorium/record/fingerprint.py +26 -0
- sensorium/record/thread_starts.py +261 -0
- sensorium/record/tracer.py +798 -0
- sensorium/record/tracer_exc.py +199 -0
- sensorium/record/tracer_frames.py +618 -0
- sensorium/redact.py +304 -0
- sensorium/redact_content.py +156 -0
- sensorium/redact_key.py +317 -0
- sensorium/redact_store.py +502 -0
- sensorium/redact_values.py +433 -0
- sensorium/store/__init__.py +0 -0
- sensorium/store/db.py +284 -0
- sensorium/store/gc.py +360 -0
- sensorium/store/payloads.py +69 -0
- sensorium/store/reader.py +438 -0
- sensorium/store/writer.py +283 -0
- sensorium/ts/__init__.py +10 -0
- sensorium/ts/build.py +783 -0
- sensorium/ts/cli.py +130 -0
- sensorium/ts/driver.py +472 -0
- sensorium/ts/focus.py +146 -0
- sensorium/ts/harness.py +314 -0
- sensorium/ts/ingest.py +380 -0
- sensorium/ts/invocation.py +252 -0
- sensorium/ts/pkg.py +167 -0
- sensorium/ts/redaction.py +64 -0
- sensorium/ts/spool.py +168 -0
- sensorium/ts/truncation.py +81 -0
- sensorium/ts/wrapper.py +255 -0
- sensorium_trace-0.18.2.dist-info/METADATA +245 -0
- sensorium_trace-0.18.2.dist-info/RECORD +135 -0
- sensorium_trace-0.18.2.dist-info/WHEEL +4 -0
- sensorium_trace-0.18.2.dist-info/entry_points.txt +3 -0
- sensorium_trace-0.18.2.dist-info/licenses/LICENSE +21 -0
sensorium/redact.py
ADDED
|
@@ -0,0 +1,304 @@
|
|
|
1
|
+
"""Rule v1: which NAMES hold a secret, and the knobs that amend that
|
|
2
|
+
judgement.
|
|
3
|
+
|
|
4
|
+
Every recorder stored the whole process environment in plaintext until this
|
|
5
|
+
slice, so a trace of any program launched from this box's shell held a live
|
|
6
|
+
`CLAUDE_CODE_MESSAGING_TOKEN` at 0644. Rule v1 replaces such a value with
|
|
7
|
+
`<redacted>` at the WRITER, before anything reaches disk.
|
|
8
|
+
|
|
9
|
+
The digest that stands in its place, and the per-store key the digests are
|
|
10
|
+
taken under, are `redact_key`'s -- re-exported below, so `redact.Key` and
|
|
11
|
+
`redact.KEY_VAR` still resolve here.
|
|
12
|
+
|
|
13
|
+
WHAT THIS IS NOT
|
|
14
|
+
----------------
|
|
15
|
+
**Not a secret scanner.** The rule fires on a name's SEGMENTS, so a secret
|
|
16
|
+
in a variable called `x`, or one this set has never heard of, is stored as
|
|
17
|
+
typed; the content rule (patterns inside a value, whatever the name) is
|
|
18
|
+
PR B's and is a floor too. `SENSORIUM_REDACT_NAMES` is the remedy for a
|
|
19
|
+
name this misses and `SENSORIUM_REDACT_ALLOW` for one it wrongly fires on,
|
|
20
|
+
and both are recorded so a reader sees what the recording was made under.
|
|
21
|
+
Matching is segment-exact, never substring: substring `KEY` would fire on
|
|
22
|
+
`MONKEY_PATCH` and substring `PASS` on `BYPASS_CACHE`. The false positives
|
|
23
|
+
that do fire (`SSH_AUTH_SOCK`, `GPG_KEY_ID`: a path and an id) cost one
|
|
24
|
+
plaintext path in a trace, which the digest still makes comparable.
|
|
25
|
+
|
|
26
|
+
THE FIXTURE
|
|
27
|
+
-----------
|
|
28
|
+
`docs/trace-format/redaction-v1.json` holds every `split` and `fires` case,
|
|
29
|
+
and `sensorium-rt`'s and the TypeScript recorder's suites read the SAME
|
|
30
|
+
file. A case belongs there, not in a test module: three implementations of
|
|
31
|
+
one rule agree only on what all three are asked.
|
|
32
|
+
|
|
33
|
+
This module is pure -- it imports nothing from `record` or `query` -- and
|
|
34
|
+
never raises.
|
|
35
|
+
"""
|
|
36
|
+
import re
|
|
37
|
+
from collections.abc import Mapping
|
|
38
|
+
from dataclasses import dataclass
|
|
39
|
+
|
|
40
|
+
# The store key, split out at this module's own seam: the name rule
|
|
41
|
+
# above and the content rule beside it both grow in here, and the key is
|
|
42
|
+
# neither of them. Re-exported so `redact.Key` and `redact.KEY_VAR` keep
|
|
43
|
+
# resolving: these are one rule's internals across two files, not
|
|
44
|
+
# separate modules with surfaces of their own.
|
|
45
|
+
from sensorium.redact_key import ( # noqa: F401
|
|
46
|
+
KEY_BYTES, KEY_FILE, KEY_MODE, KEY_VAR, Key)
|
|
47
|
+
|
|
48
|
+
#: The rule's identity, stamped in every trace it touches. A later rule is
|
|
49
|
+
#: v2; v1 is never amended, because a trace that says `v1` has to mean one
|
|
50
|
+
#: thing forever.
|
|
51
|
+
RULE = "v1"
|
|
52
|
+
|
|
53
|
+
#: What a whole-redacted value reads as. The spelling this repo's renderers
|
|
54
|
+
#: already use for a withheld thing (`<unread>`, `<not in scope>`).
|
|
55
|
+
REDACTED = "<redacted>"
|
|
56
|
+
|
|
57
|
+
OFF_VAR = "SENSORIUM_NO_REDACT"
|
|
58
|
+
NAMES_VAR = "SENSORIUM_REDACT_NAMES"
|
|
59
|
+
ALLOW_VAR = "SENSORIUM_REDACT_ALLOW"
|
|
60
|
+
|
|
61
|
+
#: §2.1's set, plus `PWD` (ruling R10, below). A segment here fires the rule;
|
|
62
|
+
#: every one of them has a firing case in the fixture, and `test_the_fixture_
|
|
63
|
+
#: gives_every_segment_of_the_rule_a_firing_case` is what keeps that true. The
|
|
64
|
+
#: four compounds (`APIKEY`, `ACCESSKEY`, `SECRETKEY`, `AUTHTOKEN`) exist
|
|
65
|
+
#: because a name written as one word -- `apikey` -- splits into one segment
|
|
66
|
+
#: that none of `API`, `KEY` matches.
|
|
67
|
+
SEGMENTS = frozenset("""
|
|
68
|
+
KEY APIKEY TOKEN SECRET SECRETS PASSWORD PASSWD PASSPHRASE PASS
|
|
69
|
+
AUTH AUTHORIZATION CREDENTIAL CREDENTIALS CREDS PRIVATE
|
|
70
|
+
COOKIE COOKIES SIGNATURE BEARER JWT DSN ACCESSKEY SECRETKEY AUTHTOKEN
|
|
71
|
+
PWD
|
|
72
|
+
""".split())
|
|
73
|
+
|
|
74
|
+
#: The segments that fire only as part of a LONGER name (ruling B28, for
|
|
75
|
+
#: `KEY`). `PWD`: `MYSQL_PWD` and `DB_PWD` are passwords, and `PWD` and
|
|
76
|
+
#: `OLDPWD` are the shell's working directory, on every machine that has ever
|
|
77
|
+
#: run a shell. `KEY`: a bare `key` is a cache key, a dict key, a lookup key
|
|
78
|
+
#: on almost every function that iterates a mapping, and redacting it by
|
|
79
|
+
#: default would blind `watch` on the commonest local in the language, while
|
|
80
|
+
#: every compound spelling (`api_key`, `apiKey`, `secret_key`, `build_key`,
|
|
81
|
+
#: `KEY_FILE`) still fires and `SENSORIUM_REDACT_NAMES=key` restores it per
|
|
82
|
+
#: run. §2.1 names both `PWD` and `OLDPWD` as non-firing and they still are --
|
|
83
|
+
#: one segment, so this condition holds them out while the two-segment
|
|
84
|
+
#: database variables come in.
|
|
85
|
+
_SOLO_EXEMPT = frozenset({"PWD", "KEY"})
|
|
86
|
+
|
|
87
|
+
#: Whole NORMALISED names that fire whatever their segments say. Segment-exact
|
|
88
|
+
#: matching cannot see inside a compound word: `PGPASSWORD` splits into the
|
|
89
|
+
#: single segment `PGPASSWORD`, which is neither `PG` nor `PASSWORD`, and
|
|
90
|
+
#: libpq's standard password variable was stored in plaintext by rule v1 until
|
|
91
|
+
#: the census of `tests/fixtures/benign-env-names.txt` said so. Kept apart
|
|
92
|
+
#: from `SEGMENTS` because it is a different question -- "is this word one of
|
|
93
|
+
#: the few known one-word secrets" rather than "does this name have a
|
|
94
|
+
#: secret-shaped part" -- and because a name dropped in here silently would
|
|
95
|
+
#: otherwise widen the segment rule for every name that contains it. Short by
|
|
96
|
+
#: design: a name earns a place only when a census or a report shows it
|
|
97
|
+
#: matters, never on imagination.
|
|
98
|
+
EXACT: frozenset[str] = frozenset({"PGPASSWORD"})
|
|
99
|
+
|
|
100
|
+
#: The two camelCase boundaries, in this order: an uppercase RUN followed by
|
|
101
|
+
#: Upper+lower (`HTTPToken` -> `HTTP Token`), then lower-or-digit followed by
|
|
102
|
+
#: upper (`apiKey` -> `api Key`). Verbatim from §2.1, because three
|
|
103
|
+
#: implementers would otherwise spell them three ways.
|
|
104
|
+
_CAMEL_RUN = re.compile(r"([A-Z]+)([A-Z][a-z])")
|
|
105
|
+
_CAMEL_STEP = re.compile(r"([a-z0-9])([A-Z])")
|
|
106
|
+
_SEPARATOR = re.compile(r"[^A-Za-z0-9]+")
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def split(name: str) -> list[str]:
|
|
110
|
+
"""`name`'s uppercased segments: `apiKey` -> `[API, KEY]`,
|
|
111
|
+
`SSH_AUTH_SOCK` -> `[SSH, AUTH, SOCK]`, `XAUTHORITY` -> `[XAUTHORITY]`.
|
|
112
|
+
|
|
113
|
+
Empty segments are dropped, so `__init__` is `[INIT]` and `""` is `[]`.
|
|
114
|
+
"""
|
|
115
|
+
spaced = _CAMEL_STEP.sub(r"\1 \2", _CAMEL_RUN.sub(r"\1 \2", name))
|
|
116
|
+
return [part.upper() for part in _SEPARATOR.split(spaced) if part]
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def normalise(name: str) -> str:
|
|
120
|
+
"""The comparison form the two knobs use: the segments, joined. So
|
|
121
|
+
`myco_dsn`, `MYCO_DSN` and `mycoDsn` are one name to a user's list."""
|
|
122
|
+
return "".join(split(name))
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
@dataclass(frozen=True)
|
|
126
|
+
class Knobs:
|
|
127
|
+
"""The three environment variables, as read once at record time.
|
|
128
|
+
|
|
129
|
+
`names` and `allow` are held NORMALISED, so a comparison against a
|
|
130
|
+
variable's own normalised form is exact and case-blind at once.
|
|
131
|
+
"""
|
|
132
|
+
off: bool
|
|
133
|
+
names: frozenset[str]
|
|
134
|
+
allow: frozenset[str]
|
|
135
|
+
|
|
136
|
+
@classmethod
|
|
137
|
+
def from_environ(cls, environ: Mapping[str, str]) -> "Knobs":
|
|
138
|
+
"""`SENSORIUM_NO_REDACT` reads like `SENSORIUM_NO_INVOCATION_LOG`
|
|
139
|
+
(`invocations._log_disabled`): any non-empty value other than `"0"`
|
|
140
|
+
turns the rule off, and `=0`, empty and unset all leave it on."""
|
|
141
|
+
off = environ.get(OFF_VAR)
|
|
142
|
+
return cls(bool(off) and off != "0",
|
|
143
|
+
_listed(environ, NAMES_VAR), _listed(environ, ALLOW_VAR))
|
|
144
|
+
|
|
145
|
+
def meta(self) -> dict:
|
|
146
|
+
return {"names": sorted(self.names), "allow": sorted(self.allow)}
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def _listed(environ: Mapping[str, str], var: str) -> frozenset[str]:
|
|
150
|
+
"""A comma list, stripped, empties dropped, each entry normalised. An
|
|
151
|
+
entry that normalises to nothing (`","`, `"-"`) is dropped too -- an
|
|
152
|
+
empty string in the set would match a nameless value."""
|
|
153
|
+
raw = (environ.get(var) or "").split(",")
|
|
154
|
+
return frozenset(n for n in (normalise(part.strip()) for part in raw) if n)
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def fires(name: str, knobs: Knobs) -> bool:
|
|
158
|
+
"""Whether the name rule fires on `name`.
|
|
159
|
+
|
|
160
|
+
Allow first, and it WINS over `names` -- a user's statement about their
|
|
161
|
+
own variable outranks their own list (§2.3). Then `EXACT`, then the
|
|
162
|
+
segments. `knobs.off` is not read here: turning the whole rule off is
|
|
163
|
+
`env`'s business, one level up, so that this function stays the answer to
|
|
164
|
+
"is this a secret-shaped name".
|
|
165
|
+
"""
|
|
166
|
+
segments = split(name)
|
|
167
|
+
normalised = "".join(segments)
|
|
168
|
+
if normalised in knobs.allow:
|
|
169
|
+
return False
|
|
170
|
+
if normalised in knobs.names:
|
|
171
|
+
return True
|
|
172
|
+
if normalised in EXACT:
|
|
173
|
+
return True
|
|
174
|
+
multi = len(segments) > 1
|
|
175
|
+
return any(segment in SEGMENTS and (multi or segment not in _SOLO_EXEMPT)
|
|
176
|
+
for segment in segments)
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def env(environ: Mapping[str, str], key: Key,
|
|
180
|
+
knobs: Knobs) -> tuple[dict[str, str], dict[str, str | None]]:
|
|
181
|
+
"""(the environment to STORE, the name -> digest table for `redaction`).
|
|
182
|
+
|
|
183
|
+
A NEW dict either way -- the caller's mapping is never touched -- minus
|
|
184
|
+
`KEY_VAR`. With the rule off, that deletion is the only thing that
|
|
185
|
+
happens, and `meta` says `mode: "off"` so a reader knows the plaintext
|
|
186
|
+
was recorded on purpose. Applied to an ALREADY redacted environment this
|
|
187
|
+
would digest the marker; §5.4 is why that never happens -- a converter
|
|
188
|
+
meeting a `mode: "on"` header trusts it and does not re-run the rule.
|
|
189
|
+
|
|
190
|
+
The table's keys are SORTED (R13) so that one environment gives one
|
|
191
|
+
table, byte for byte, whichever language wrote it -- Rust's `BTreeMap`
|
|
192
|
+
and the TypeScript implementation sort too. The stored environment keeps
|
|
193
|
+
the order it arrived in; only the table is ordered.
|
|
194
|
+
"""
|
|
195
|
+
stored = {name: value for name, value in environ.items()
|
|
196
|
+
if name != KEY_VAR}
|
|
197
|
+
if knobs.off:
|
|
198
|
+
return stored, {}
|
|
199
|
+
table: dict[str, str | None] = {}
|
|
200
|
+
for name in sorted(stored):
|
|
201
|
+
if fires(name, knobs):
|
|
202
|
+
table[name] = key.digest(stored[name])
|
|
203
|
+
stored[name] = REDACTED
|
|
204
|
+
return stored, table
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def meta(key: Key, knobs: Knobs, table: dict[str, str | None],
|
|
208
|
+
by: str = "recorder") -> dict:
|
|
209
|
+
"""The trace's `redaction` key (§4.3). `by` is the LAST hand that applied
|
|
210
|
+
the rule: `recorder`, `converter` or `retrofit`."""
|
|
211
|
+
if knobs.off:
|
|
212
|
+
return {"rule": RULE, "mode": "off"}
|
|
213
|
+
return {"rule": RULE, "mode": "on", "keyed": key.keyed,
|
|
214
|
+
"key_id": key.key_id, "env": table, **knobs.meta(), "by": by}
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
#: One side's account of one name: its digest, the key that digest was taken
|
|
218
|
+
#: under, and whether that side had a key at all. `None` in place of a Side
|
|
219
|
+
#: means the side did not redact the name -- its value is plaintext.
|
|
220
|
+
Side = tuple[str | None, str | None, bool]
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def compare(before: str | None, after: str | None, was: Side | None,
|
|
224
|
+
now: Side | None, key: Key) -> bool | None:
|
|
225
|
+
"""§6.2's table, once, for one name: True equal, False differs, None
|
|
226
|
+
unverifiable.
|
|
227
|
+
|
|
228
|
+
`before`/`after` are the two sides' stored values -- read only on a side
|
|
229
|
+
that did NOT redact, where the value is the plaintext itself. A value
|
|
230
|
+
missing from the plaintext side is a DIFFERENCE and a verified one (the
|
|
231
|
+
variable vanished), never an unverifiability.
|
|
232
|
+
|
|
233
|
+
Called only for a name at least one side redacted; plain equality is
|
|
234
|
+
the caller's answer for every other name, as it always was.
|
|
235
|
+
"""
|
|
236
|
+
if was is not None and now is not None:
|
|
237
|
+
(mine, my_key, my_keyed), (theirs, their_key, their_keyed) = was, now
|
|
238
|
+
if not (my_keyed and their_keyed) or my_key != their_key:
|
|
239
|
+
return None
|
|
240
|
+
if mine is None or theirs is None:
|
|
241
|
+
return None
|
|
242
|
+
return mine == theirs
|
|
243
|
+
if was is None and now is None:
|
|
244
|
+
return before == after
|
|
245
|
+
side, plain = (was, after) if was is not None else (now, before)
|
|
246
|
+
digest, under, keyed = side
|
|
247
|
+
if digest is None or not keyed or not key.keyed or key.key_id != under:
|
|
248
|
+
return None
|
|
249
|
+
if plain is None:
|
|
250
|
+
return False
|
|
251
|
+
return key.digest(plain) == digest
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def uncomparable(was_meta: dict, now_meta: dict | None,
|
|
255
|
+
key: Key) -> tuple[list[str], str | None]:
|
|
256
|
+
"""The redacted names whose comparison `compare` cannot decide, sorted,
|
|
257
|
+
and the reason word -- `([], None)` when there are none.
|
|
258
|
+
|
|
259
|
+
`now_meta is None` is a live plaintext side with no table of its own
|
|
260
|
+
(the Python re-run): its names are verifiable under the store's key,
|
|
261
|
+
and appear here only when that key is gone or is not the one that wrote
|
|
262
|
+
the digests.
|
|
263
|
+
"""
|
|
264
|
+
was_table, was_env = _table(was_meta), _stored(was_meta)
|
|
265
|
+
now_table, now_env = _table(now_meta), _stored(now_meta)
|
|
266
|
+
names, unkeyed = [], False
|
|
267
|
+
for name in sorted(set(was_table) | set(now_table)):
|
|
268
|
+
mine, theirs = _side(was_meta, name), _side(now_meta, name)
|
|
269
|
+
if compare(was_env.get(name), now_env.get(name),
|
|
270
|
+
mine, theirs, key) is not None:
|
|
271
|
+
continue
|
|
272
|
+
names.append(name)
|
|
273
|
+
unkeyed = unkeyed or _unkeyed(mine) or _unkeyed(theirs)
|
|
274
|
+
if mine is None or theirs is None:
|
|
275
|
+
# One side is plaintext, so the STORE's key is the one doing the
|
|
276
|
+
# verifying and its absence is the reason.
|
|
277
|
+
unkeyed = unkeyed or not key.keyed
|
|
278
|
+
if not names:
|
|
279
|
+
return [], None
|
|
280
|
+
return names, "unkeyed" if unkeyed else "different keys"
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
def _redaction(meta_dict: dict | None) -> dict:
|
|
284
|
+
return ((meta_dict or {}).get("redaction") or {})
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
def _table(meta_dict: dict | None) -> dict:
|
|
288
|
+
return _redaction(meta_dict).get("env") or {}
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
def _stored(meta_dict: dict | None) -> dict:
|
|
292
|
+
return (meta_dict or {}).get("env") or {}
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def _side(meta_dict: dict | None, name: str) -> Side | None:
|
|
296
|
+
table = _table(meta_dict)
|
|
297
|
+
if name not in table:
|
|
298
|
+
return None
|
|
299
|
+
redaction = _redaction(meta_dict)
|
|
300
|
+
return table[name], redaction.get("key_id"), bool(redaction.get("keyed"))
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
def _unkeyed(side: Side | None) -> bool:
|
|
304
|
+
return side is not None and (not side[2] or side[0] is None)
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
"""Rule v1's CONTENT half (§2.2): a fixed list of patterns over a value's
|
|
2
|
+
TEXT, whatever its name. Beside `redact.py`'s name rule, which decides
|
|
3
|
+
whether a whole value is worth redacting at all; this module decides which
|
|
4
|
+
SPANS inside a value's text look like a secret regardless of what it was
|
|
5
|
+
called.
|
|
6
|
+
|
|
7
|
+
WHAT THIS IS NOT
|
|
8
|
+
----------------
|
|
9
|
+
**Not a secret scanner.** The list is a floor: nineteen shapes, each with a
|
|
10
|
+
minimum length so a short benign string cannot fire. A secret that matches
|
|
11
|
+
none of them is stored as typed.
|
|
12
|
+
|
|
13
|
+
THE FIXTURE
|
|
14
|
+
-----------
|
|
15
|
+
`docs/trace-format/redaction-v1.json`'s `content` list holds every case, and
|
|
16
|
+
`sensorium-rt`'s converter (`redact_content.rs`, `regex` crate -- the
|
|
17
|
+
runtime stays dependency-free, §5.2) and the TypeScript recorder's suite
|
|
18
|
+
read the SAME file. The three implementations are the same nineteen
|
|
19
|
+
patterns, textually, save for the one place the three engines cannot agree
|
|
20
|
+
on a spelling: end-of-text in the PEM row's truncated form (`\\Z` here,
|
|
21
|
+
`\\z` in Rust, `$` under JavaScript's `s` flag) and inline case/dotall flags
|
|
22
|
+
(`(?i)`/`(?s)` here and in Rust; the `i`/`s` RegExp flags in JavaScript,
|
|
23
|
+
which has no inline spelling for either).
|
|
24
|
+
|
|
25
|
+
TWO OPERATIONS, ONE MODULE
|
|
26
|
+
---------------------------
|
|
27
|
+
§2.3: a name hit redacts the WHOLE value (`redact.py`); a content hit
|
|
28
|
+
replaces the matched SPAN and keeps everything around it
|
|
29
|
+
(`postgres://u:<redacted>@h/db`) -- a repr or a log line is not a value
|
|
30
|
+
with an identity, it is text that happened to contain one, and a partial
|
|
31
|
+
cannot honestly carry a digest of the whole. `hit` below means exactly
|
|
32
|
+
"the text changed": a pattern that matches a span already holding the
|
|
33
|
+
marker (an env value the name rule already redacted, then rendered
|
|
34
|
+
somewhere) is not a second hit, and `content` must say so, because the
|
|
35
|
+
converters (Task 6/7) count `values` from this flag.
|
|
36
|
+
|
|
37
|
+
This module is pure -- it imports nothing but `redact.REDACTED` -- and never
|
|
38
|
+
raises.
|
|
39
|
+
"""
|
|
40
|
+
import re
|
|
41
|
+
|
|
42
|
+
from sensorium.redact import REDACTED
|
|
43
|
+
|
|
44
|
+
#: (row name, compiled pattern, replaced group: 0 = whole match). Verbatim
|
|
45
|
+
#: from §2.2's table, in the order that table gives -- table order matters
|
|
46
|
+
#: exactly once, for `sk-ant` before `sk`: both match an Anthropic key, the
|
|
47
|
+
#: whole match is replaced either way so the two rows' `after` never differs,
|
|
48
|
+
#: and the order is only for which row NAME a test or a report attributes
|
|
49
|
+
#: the hit to.
|
|
50
|
+
PATTERNS: tuple[tuple[str, "re.Pattern[str]", int], ...] = (
|
|
51
|
+
("url-userinfo",
|
|
52
|
+
re.compile(r"://[^/\s:@]{1,64}:([^@\s/]{1,256})@"), 1),
|
|
53
|
+
# The PEM body, through the matching END line or to end of text when
|
|
54
|
+
# truncated -- `\Z` is this engine's spelling of that end-of-text
|
|
55
|
+
# alternative; Rust's `redact_content.rs` spells it `\z` and JavaScript's
|
|
56
|
+
# spells it `$` under the `s` flag. `(?s)` is inline DOTALL: the body
|
|
57
|
+
# spans real newlines.
|
|
58
|
+
("pem",
|
|
59
|
+
re.compile(r"(?s)-----BEGIN [A-Z ]*PRIVATE KEY-----(.*?)"
|
|
60
|
+
r"(?:-----END [A-Z ]*PRIVATE KEY-----|\Z)"), 1),
|
|
61
|
+
("authorization-header",
|
|
62
|
+
re.compile(r"(?i)\b(Bearer|Basic)\s+([A-Za-z0-9._~+/=-]{16,})"), 2),
|
|
63
|
+
("jwt",
|
|
64
|
+
re.compile(r"eyJ[A-Za-z0-9_-]{8,}\.eyJ[A-Za-z0-9_-]{8,}"
|
|
65
|
+
r"\.[A-Za-z0-9_-]{8,}"), 0),
|
|
66
|
+
("sk-ant", re.compile(r"\bsk-ant-[A-Za-z0-9_-]{20,}"), 0),
|
|
67
|
+
("sk", re.compile(r"\bsk-[A-Za-z0-9_-]{20,}"), 0),
|
|
68
|
+
("stripe", re.compile(r"\b(sk|rk)_(live|test)_[A-Za-z0-9]{16,}"), 0),
|
|
69
|
+
("github", re.compile(r"\bgh[pousr]_[A-Za-z0-9]{20,}"), 0),
|
|
70
|
+
("github-pat", re.compile(r"\bgithub_pat_[A-Za-z0-9_]{20,}"), 0),
|
|
71
|
+
("gitlab", re.compile(r"\bglpat-[A-Za-z0-9_-]{20,}"), 0),
|
|
72
|
+
("slack", re.compile(r"\bxox[abprs]-[A-Za-z0-9-]{10,}"), 0),
|
|
73
|
+
("aws", re.compile(r"\b(AKIA|ASIA)[0-9A-Z]{16}\b"), 0),
|
|
74
|
+
("google", re.compile(r"\bAIza[0-9A-Za-z_-]{35}\b"), 0),
|
|
75
|
+
("huggingface", re.compile(r"\bhf_[A-Za-z0-9]{20,}"), 0),
|
|
76
|
+
("npm", re.compile(r"\bnpm_[A-Za-z0-9]{20,}"), 0),
|
|
77
|
+
("pypi", re.compile(r"\bpypi-[A-Za-z0-9_-]{20,}"), 0),
|
|
78
|
+
("digitalocean", re.compile(r"\bdop_v1_[a-f0-9]{20,}"), 0),
|
|
79
|
+
("shopify", re.compile(r"\bshpat_[a-f0-9]{20,}"), 0),
|
|
80
|
+
("sendgrid",
|
|
81
|
+
re.compile(r"\bSG\.[A-Za-z0-9_-]{16,}\.[A-Za-z0-9_-]{16,}"), 0),
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
#: B18's pre-check (§12's mitigation, built in rather than waited for): the
|
|
85
|
+
#: literal prefix every CASE-SENSITIVE pattern above starts with, as one
|
|
86
|
+
#: alternation. A string that matches neither this nor `_TRIGGER_CI` cannot
|
|
87
|
+
#: match any of the nineteen patterns either, so `content` skips the whole
|
|
88
|
+
#: table rather than running twenty regexes over every string a recorder
|
|
89
|
+
#: ever touches.
|
|
90
|
+
_TRIGGER = re.compile("|".join(re.escape(p) for p in (
|
|
91
|
+
"://", "-----BEGIN", "eyJ", "sk-", "sk_", "rk_", "gh", "github_pat_",
|
|
92
|
+
"glpat-", "xox", "AKIA", "ASIA", "AIza", "hf_", "npm_", "pypi-",
|
|
93
|
+
"dop_v1_", "shpat_", "SG.",
|
|
94
|
+
)))
|
|
95
|
+
|
|
96
|
+
#: The same pre-check for the one pattern above that is itself `(?i)`.
|
|
97
|
+
#: `authorization-header` accepts every case spelling of each word, and a
|
|
98
|
+
#: literal alternation of `Bearer|Basic|bearer|basic` stood in front of it
|
|
99
|
+
#: covering two of each: `Authorization: BEARER <token>` matched no literal,
|
|
100
|
+
#: skipped the table, and was stored in plaintext. A pre-check NARROWER than
|
|
101
|
+
#: pattern it guards is a leak and not an optimisation (ruling R21). Its own
|
|
102
|
+
#: object rather than an inline `(?i:...)` group because the TypeScript twin
|
|
103
|
+
#: has no inline-flag spelling, and the three implementations have to agree
|
|
104
|
+
#: in behaviour rather than in syntax.
|
|
105
|
+
_TRIGGER_CI = re.compile("Bearer|Basic", re.IGNORECASE)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def triggers(text: str) -> bool:
|
|
109
|
+
"""Whether any §2.2 pattern could match `text` at all: the union of the
|
|
110
|
+
two pre-checks, and the one thing `content` consults.
|
|
111
|
+
|
|
112
|
+
The union is what a test may pin. Either half alone is a pre-check for
|
|
113
|
+
part of the table, and pinning one of them proves nothing about the
|
|
114
|
+
shapes the other stands in front of.
|
|
115
|
+
"""
|
|
116
|
+
return bool(_TRIGGER.search(text) or _TRIGGER_CI.search(text))
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def content(text: str) -> tuple[str, bool]:
|
|
120
|
+
"""`text` with every matched span replaced by `REDACTED`, and whether
|
|
121
|
+
the text CHANGED -- never whether some pattern merely matched.
|
|
122
|
+
|
|
123
|
+
That distinction is the whole of the contract: `url-userinfo` matches
|
|
124
|
+
`postgres://u:<redacted>@h/db` (its group already reads as the marker),
|
|
125
|
+
and replacing it with itself is not a hit. Comparing the WHOLE result to
|
|
126
|
+
the input, once, at the end, is what makes that true without a special
|
|
127
|
+
case for it -- the converters that count `values` from this flag rely on
|
|
128
|
+
it (Task 6/7).
|
|
129
|
+
|
|
130
|
+
Applied left to right, pattern by pattern in table order, over the
|
|
131
|
+
CURRENT text -- so a later pattern sees an earlier pattern's markers,
|
|
132
|
+
never the original secret twice.
|
|
133
|
+
"""
|
|
134
|
+
if not text or not triggers(text):
|
|
135
|
+
return text, False
|
|
136
|
+
out = text
|
|
137
|
+
for _, pattern, group in PATTERNS:
|
|
138
|
+
out = pattern.sub(_replacer(group), out)
|
|
139
|
+
return out, out != text
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def _replacer(group: int):
|
|
143
|
+
"""One pattern's substitution function: the whole match for group 0, or
|
|
144
|
+
the match with only `group`'s span swapped for `REDACTED` -- never the
|
|
145
|
+
whole match when a narrower group was asked for (§2.3's span operation:
|
|
146
|
+
everything around the secret is kept)."""
|
|
147
|
+
if group == 0:
|
|
148
|
+
return lambda m: REDACTED
|
|
149
|
+
|
|
150
|
+
def _swap(m: "re.Match[str]") -> str:
|
|
151
|
+
whole = m.group(0)
|
|
152
|
+
start = m.start(group) - m.start(0)
|
|
153
|
+
end = m.end(group) - m.start(0)
|
|
154
|
+
return whole[:start] + REDACTED + whole[end:]
|
|
155
|
+
|
|
156
|
+
return _swap
|