agent-postmortem 0.3.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_postmortem/__init__.py +9 -0
- agent_postmortem/argerrors.py +160 -0
- agent_postmortem/claims.py +43 -0
- agent_postmortem/cli.py +2278 -0
- agent_postmortem/commands.py +1543 -0
- agent_postmortem/config.py +435 -0
- agent_postmortem/demo/-home-user-project/00000000-0000-4000-8000-000000000000/subagents/workflows/wf_00000000-d30/agent-ac200000000000001.jsonl +5 -0
- agent_postmortem/demo/-home-user-project/00000000-0000-4000-8000-000000000000/subagents/workflows/wf_00000000-d30/agent-ac200000000000001.meta.json +8 -0
- agent_postmortem/demo/-home-user-project/00000000-0000-4000-8000-000000000000/subagents/workflows/wf_00000000-d30/agent-ac200000000000002.jsonl +2 -0
- agent_postmortem/demo/-home-user-project/00000000-0000-4000-8000-000000000000/subagents/workflows/wf_00000000-d30/agent-ac200000000000002.meta.json +8 -0
- agent_postmortem/demo/-home-user-project/00000000-0000-4000-8000-000000000000/subagents/workflows/wf_00000000-d30/agent-ac200000000000003.jsonl +5 -0
- agent_postmortem/demo/-home-user-project/00000000-0000-4000-8000-000000000000/subagents/workflows/wf_00000000-d30/agent-ac200000000000003.meta.json +8 -0
- agent_postmortem/demo/-home-user-project/00000000-0000-4000-8000-000000000000/subagents/workflows/wf_00000000-d30/agent-ac200000000000004.jsonl +2 -0
- agent_postmortem/demo/-home-user-project/00000000-0000-4000-8000-000000000000/subagents/workflows/wf_00000000-d30/agent-ac200000000000004.meta.json +8 -0
- agent_postmortem/demo/-home-user-project/00000000-0000-4000-8000-000000000000/subagents/workflows/wf_00000000-d30/journal.jsonl +7 -0
- agent_postmortem/demo/-home-user-project/00000000-0000-4000-8000-000000000000.jsonl +4 -0
- agent_postmortem/detectors/__init__.py +98 -0
- agent_postmortem/detectors/attempt_failed.py +261 -0
- agent_postmortem/detectors/base.py +319 -0
- agent_postmortem/detectors/bash_backgrounded.py +72 -0
- agent_postmortem/detectors/bash_timeout.py +94 -0
- agent_postmortem/detectors/claim_contradiction.py +251 -0
- agent_postmortem/detectors/classifier_unavailable.py +73 -0
- agent_postmortem/detectors/codex_unavailable.py +102 -0
- agent_postmortem/detectors/codex_write_denied.py +100 -0
- agent_postmortem/detectors/git_cwd_mismatch.py +74 -0
- agent_postmortem/detectors/mislabeled_kill.py +164 -0
- agent_postmortem/detectors/missing_artifact.py +159 -0
- agent_postmortem/detectors/permission_denied.py +99 -0
- agent_postmortem/detectors/silent_retry.py +163 -0
- agent_postmortem/detectors/structured_output_failure.py +91 -0
- agent_postmortem/detectors/tool_json_error.py +77 -0
- agent_postmortem/detectors/wasted_time.py +125 -0
- agent_postmortem/detectors/workflow_failed.py +62 -0
- agent_postmortem/disclosure.py +931 -0
- agent_postmortem/hooks/model_allocation_guard.mjs +248 -0
- agent_postmortem/hooks_cli.py +1269 -0
- agent_postmortem/ledger.py +624 -0
- agent_postmortem/markers.py +505 -0
- agent_postmortem/models.py +815 -0
- agent_postmortem/observe.py +792 -0
- agent_postmortem/reader.py +4611 -0
- agent_postmortem/redact.py +1981 -0
- agent_postmortem/report.py +3010 -0
- agent_postmortem/rescue.py +2096 -0
- agent_postmortem/returns.py +499 -0
- agent_postmortem/rules.py +627 -0
- agent_postmortem/severity.py +275 -0
- agent_postmortem/state.py +1261 -0
- agent_postmortem/trend.py +391 -0
- agent_postmortem/watch.py +973 -0
- agent_postmortem-0.3.4.dist-info/METADATA +264 -0
- agent_postmortem-0.3.4.dist-info/RECORD +56 -0
- agent_postmortem-0.3.4.dist-info/WHEEL +4 -0
- agent_postmortem-0.3.4.dist-info/entry_points.txt +3 -0
- agent_postmortem-0.3.4.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,1981 @@
|
|
|
1
|
+
"""Redact strings before they reach a report, a JSON export, or a terminal.
|
|
2
|
+
|
|
3
|
+
redact() is the single entry point; every other module that prints text a
|
|
4
|
+
stranger might read calls it (or the Redactor wrapper) instead of formatting
|
|
5
|
+
raw paths or ids itself. See docs/DESIGN.md section 4.8 for the exact rules.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import bisect
|
|
11
|
+
import hashlib
|
|
12
|
+
import itertools
|
|
13
|
+
import json
|
|
14
|
+
import os
|
|
15
|
+
import re
|
|
16
|
+
import unicodedata
|
|
17
|
+
from collections.abc import Callable
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
|
|
20
|
+
from . import disclosure
|
|
21
|
+
from .disclosure import DisclosureContext
|
|
22
|
+
|
|
23
|
+
_UUID_RE = re.compile(
|
|
24
|
+
r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b"
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
# Claude Code project slugs are a run of hyphen-joined path segments derived
|
|
28
|
+
# from an absolute path under /Users/<name>/... (macOS) or /home/<name>/...
|
|
29
|
+
# (Linux), for example "-Users-alice-Documents-demo" or
|
|
30
|
+
# "-home-alice-project". A segment may hold anything the source path could
|
|
31
|
+
# (spaces, apostrophes, non-ASCII characters); matching on whitespace rather
|
|
32
|
+
# than a fixed character class means a segment we did not anticipate still
|
|
33
|
+
# gets swept into the hash instead of silently ending the match early and
|
|
34
|
+
# leaking the rest of the slug. Agent ids (17 hex chars starting with "a")
|
|
35
|
+
# never match this pattern, so they pass through untouched.
|
|
36
|
+
_PROJECT_SLUG_RE = re.compile(r"-(?:Users|home)-\S+")
|
|
37
|
+
|
|
38
|
+
# Email addresses appear in captured git output ("Author: Name <x@y>").
|
|
39
|
+
# F fix 9 (W9-6): linear. A match can only start where the run of local-part characters starts (the
|
|
40
|
+
# lookbehind), and the run is taken once, possessively; without both, a long word with no `@` was
|
|
41
|
+
# scanned again from every one of its characters (100,000 characters took about 30 seconds).
|
|
42
|
+
_EMAIL_RE = re.compile(r"(?<![A-Za-z0-9._%+-])[A-Za-z0-9._%+-]++@[A-Za-z0-9.-]+\.[A-Za-z]{2,}")
|
|
43
|
+
|
|
44
|
+
# docs/DESIGN.md 5.1 (M8 review D-A): a home-relative path still names every
|
|
45
|
+
# directory between "~/" and the file itself verbatim -- "~/Documents/
|
|
46
|
+
# <client>/<sub>/notes.md" leaks the client name even after the home
|
|
47
|
+
# directory itself becomes "~". Collapsed to "~/.../<basename>" so only the
|
|
48
|
+
# file being pointed at survives, never the directories that name what it
|
|
49
|
+
# is about. A directory segment is "one or more non-slash characters
|
|
50
|
+
# immediately followed by a slash" -- spaces, apostrophes, and non-ASCII
|
|
51
|
+
# characters all pass through fine, since only the slash itself is a
|
|
52
|
+
# delimiter; the trailing basename stops at the next slash or whitespace
|
|
53
|
+
# (whichever bounds it in ordinary prose). Collapsing only fires with at
|
|
54
|
+
# least one directory segment to hide: a bare "~/notes.md" (nothing between
|
|
55
|
+
# "~/" and the file) already shows only its last segment and is left alone,
|
|
56
|
+
# and a bare "~" with no slash at all (shell shorthand, not a path) never
|
|
57
|
+
# matches "~/" in the first place -- so does a URL's "~user" spelling, since
|
|
58
|
+
# nothing there is followed immediately by a slash. (N3b: the collapsed form
|
|
59
|
+
# itself, "~/.../<basename>" followed by whitespace or the end, is a fixed
|
|
60
|
+
# point and is left alone, so a list of paths a sink already collapsed is not
|
|
61
|
+
# swallowed whole by the greedy directory run when it is redacted a second
|
|
62
|
+
# time. "~/.../a/b" is not that form and still collapses.)
|
|
63
|
+
_HOME_PATH_COLLAPSE_RE = re.compile(
|
|
64
|
+
r"~/(?!\.\.\./[^/\s]+(?:\s|$))(?:[^/\n]+/)+([^/\s]+)"
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
# The same collapse for a /tmp or /private/tmp scratch path, but keeping its
|
|
68
|
+
# first segment (the scratch directory itself, for example a fixture or run
|
|
69
|
+
# id) rather than hiding everything: unlike a home directory, /tmp's first
|
|
70
|
+
# segment is not routinely a project or client name, and several of this
|
|
71
|
+
# tool's own detectors and fixtures point straight at
|
|
72
|
+
# "/tmp/<fixed-name>/<file>" with nothing to hide there at all. Only a
|
|
73
|
+
# deeper path -- something beyond that first segment -- gets collapsed.
|
|
74
|
+
# (N3b C6: "/tmp" must begin a path token, not sit inside one, and a segment
|
|
75
|
+
# never crosses whitespace, so an interior ".../tmp/..." in a longer path and
|
|
76
|
+
# the prose after a path are both left alone.)
|
|
77
|
+
# (F fix 9, W9-6: a segment never crosses a shell separator, but a no-break or ideographic space
|
|
78
|
+
# is an ordinary character of a name, so `\s` is not the class.)
|
|
79
|
+
_TMP_SEG = r"[^/ \t\n\r\f\v]"
|
|
80
|
+
_TMP_PATH_COLLAPSE_RE = re.compile(
|
|
81
|
+
r"(?<![\w./~-])(/(?:private/)?tmp)/(" + _TMP_SEG + r"+)/(?:" + _TMP_SEG + r"+/)+("
|
|
82
|
+
+ _TMP_SEG + "+)"
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
# N3b P19 (S4, S10), F fix 4: what STARTS an absolute path in free text. A `/` that is not
|
|
86
|
+
# preceded by a word character, a slash, a dot, a tilde, a dollar sign, or an at sign, so
|
|
87
|
+
# "and/or", "3/4" and "https://host/x" are left alone. A `/` after `:`, `=`, or `-` counts
|
|
88
|
+
# (PYTHONPATH=/a:/b, R-X-/a/b), as does one glued to a short option (`-C/a/b`), and a `file://`
|
|
89
|
+
# URL counts whole (the scheme and a `localhost` host are dropped from the replacement). To keep
|
|
90
|
+
# ordinary prose out, a
|
|
91
|
+
# run after `:` or `-` must hold two segments or a file extension. Where the path ENDS is not
|
|
92
|
+
# decided here: the scanner below finds a start inside a complete shell word and renders the
|
|
93
|
+
# whole rest of the word (quotes, escapes and spaces inside it included) as one path.
|
|
94
|
+
_PATH_CHARS = r"\w.@+=,%-"
|
|
95
|
+
# F fix 9 (W9-1), F fix 10 (W10-1): a script written WITHOUT spaces between words (Chinese,
|
|
96
|
+
# Japanese, Korean, Thai, Lao, Tibetan, Myanmar, Khmer), and `\w` matches its characters, so a path
|
|
97
|
+
# typed right after a word (`无法读取/Users/x`, `อ่าน/srv/x`) would be "glued to a word". A character
|
|
98
|
+
# of that class directly before a start is a boundary instead. One class for every start lookbehind
|
|
99
|
+
# (the cost: `和/或` with two segments or an extension is a start, see `_start_counts`). A script
|
|
100
|
+
# written WITH spaces (Latin, Cyrillic, Arabic, Greek) is not in it: a word glued to a path there is
|
|
101
|
+
# the same construction as English `read/srv/x`, a recorded limit.
|
|
102
|
+
_UNSPACED = (
|
|
103
|
+
"\u0e00-\u0fff" # Thai, Lao, Tibetan
|
|
104
|
+
"\u1000-\u109f" # Myanmar
|
|
105
|
+
"\u1100-\u11ff\u1780-\u17ff\u19e0-\u19ff" # hangul jamo, Khmer, Khmer symbols
|
|
106
|
+
"\u2e80-\u9fff\ua960-\ua97f\ua9e0-\ua9ff\uaa60-\uaa7f\uac00-\ud7ff\uf900-\ufaff"
|
|
107
|
+
"\uff00-\uffef"
|
|
108
|
+
"\U00020000-\U0003ffff"
|
|
109
|
+
)
|
|
110
|
+
_UNSPACED_BEFORE = "(?<=[" + _UNSPACED + "])"
|
|
111
|
+
_UNSPACED_RE = re.compile("[" + _UNSPACED + "]")
|
|
112
|
+
# M3-1: an apostrophe with a word character on both sides is a segment character too
|
|
113
|
+
# (`O'Neil`); at a segment edge or next to a space or a slash it is not.
|
|
114
|
+
_SEG_RUN = "(?:[" + _PATH_CHARS + "]|(?<=\\w)'(?=\\w))+"
|
|
115
|
+
# FA1: the one alphabet of a path token. A segment is a run of _PATH_CHARS, the separator
|
|
116
|
+
# is `/`; Pseudonyms.apply_paths scans tokens with exactly this class (there is no second,
|
|
117
|
+
# narrower one).
|
|
118
|
+
_PATH_TOKEN_CHAR_RE = re.compile("[/" + _PATH_CHARS + "]") # `-` last: never a range
|
|
119
|
+
# W6-8: a `file://` URL's scheme is case-insensitive and may carry the host `localhost`; extra
|
|
120
|
+
# slashes (`file:////srv/x`) are one place.
|
|
121
|
+
_FILE_URL_RE = re.compile("(?i:file://(?:localhost)?)(?=/)")
|
|
122
|
+
# F fix 7 (W7-3): a `file://` URL with any host (`file://server/share/x`) is a path too: the host
|
|
123
|
+
# is part of what the URL names. `_FILE_URL_RE` is the local form whose scheme and host are dropped.
|
|
124
|
+
_FILE_URL_ANY_RE = re.compile(r"(?i:file://[\w.@:+-]*)(?=/)")
|
|
125
|
+
# F fix 7 (W7-3): a Windows path starts at a drive letter (`C:\`, `c:/`: one ASCII letter that no
|
|
126
|
+
# word character precedes) or at a UNC share (`\\server\share`: two backslashes that no word
|
|
127
|
+
# character or backslash precedes, a host, a backslash and a name character).
|
|
128
|
+
# F fix 8 (W8-8): a Windows namespace path (`\\?\UNC\server\share\x`, `\\?\C:\x`, `\\.\C:\x`)
|
|
129
|
+
# starts at its two backslashes, a `?` or `.` and a backslash.
|
|
130
|
+
_WIN_PATH_RE = re.compile(
|
|
131
|
+
r"(?:(?<!\w)|" + _UNSPACED_BEFORE + r")[A-Za-z]:[\\/]"
|
|
132
|
+
r"|(?:(?<![\\\w])|" + _UNSPACED_BEFORE + r")\\\\[\w.$-]+\\[^\s\\]"
|
|
133
|
+
r"|(?:(?<![\\\w])|" + _UNSPACED_BEFORE + r")\\\\[?.]\\[^\s\\]"
|
|
134
|
+
)
|
|
135
|
+
_ABS_PATH_RE = re.compile(
|
|
136
|
+
"(?i:file://[\\w.@:+-]*)(?=/)(?:/+" + _SEG_RUN + ")+/?"
|
|
137
|
+
# N3b R2-2: a leading `//` (POSIX keeps two slashes; they name the same
|
|
138
|
+
# place as one). Not after `:`, so a URL's `://` is never read as one.
|
|
139
|
+
r"|(?:(?<![\w/.~$@:])|" + _UNSPACED_BEFORE + r")//+(?:" + _SEG_RUN + "/)*" + _SEG_RUN
|
|
140
|
+
# F fix 8 (W8-4): `@` is not in the lookbehind: `curl -d @/srv/x` and `file=@/srv/x` upload a
|
|
141
|
+
# path, and `@scope/pkg` or an address have a word character before the slash anyway
|
|
142
|
+
+ r"|(?:(?<![\w/.~$])|(?<=(?<!\w)-[A-Za-z])|" + _UNSPACED_BEFORE + r")/(?:" + _SEG_RUN + "/)*"
|
|
143
|
+
+ _SEG_RUN
|
|
144
|
+
# N3b R2-2: the bare root `/` as a token of its own. A `/` with a space
|
|
145
|
+
# on both sides is a separator in prose ("checkout / restore", "read /
|
|
146
|
+
# write"), not a path, and is left alone.
|
|
147
|
+
+ r"|(?<![\w/.~$@:-])(?<!\s)/(?![\w/.@+=,%~$-])"
|
|
148
|
+
r"|(?<=\s)/(?![\w/.@+=,%~$ -])"
|
|
149
|
+
# (F fix 9, W9-4: a lone `/` before a space is never the root of a path whose first segment
|
|
150
|
+
# starts with a space; the W8-8 recognizer hid prose such as `Pass / Fail rate is 3/4` and is a
|
|
151
|
+
# recorded limit in docs/DESIGN.md 5.1.)
|
|
152
|
+
)
|
|
153
|
+
# N3b S9: C0 controls (and DEL and C1) never reach a terminal inside a name.
|
|
154
|
+
_CONTROL_RE = re.compile(r"[\x00-\x1f\x7f-\x9f]")
|
|
155
|
+
# W6-8: a home anchor at the start of a word: `~/`, `~user/` (another account's home) and the
|
|
156
|
+
# shell variable spellings of the home directory.
|
|
157
|
+
_HOME_START_RE = re.compile(r"~[\w.-]*/|\$HOME/|\$\{HOME\}/")
|
|
158
|
+
# F fix 8 (W8-2, W8-8): every anchor of a variable or another account's home, searched wherever a
|
|
159
|
+
# path start is searched (inside a quote, after a word that is not a proven prefix). The working
|
|
160
|
+
# directory is an absolute place in all but spelling: `$PWD/x`, `${PWD}/x`, `$OLDPWD/x` and
|
|
161
|
+
# `${OLDPWD}/x` are path starts too. Such a text is a relative spelling with no base, so every
|
|
162
|
+
# level shows it as a path-N (a span with whitespace is opaque like any other).
|
|
163
|
+
# F fix 9 (W9-2, W9-6): an `@` does not block the anchor (`curl -d @$HOME/x`), a CJK character is a
|
|
164
|
+
# boundary, and the project directory of a hook command (`$CLAUDE_PROJECT_DIR/`) is a working
|
|
165
|
+
# directory too.
|
|
166
|
+
_VAR_ANCHOR_RE = re.compile(
|
|
167
|
+
r"(?:(?<![\w./~$])|" + _UNSPACED_BEFORE + r")(?:~[\w.-]*|\$HOME|\$\{HOME\}"
|
|
168
|
+
r"|\$(?:\{(?:OLD)?PWD\}|(?:OLD)?PWD|\{CLAUDE_PROJECT_DIR\}|CLAUDE_PROJECT_DIR))/"
|
|
169
|
+
)
|
|
170
|
+
# F fix 8 (W8-3), F fix 9 (W9-5): a terminal control sequence. One parser for every use: `ESC [`,
|
|
171
|
+
# parameter bytes (`0-9 : ; < = > ?`, so colon-form 24-bit colour is one), intermediate bytes (space
|
|
172
|
+
# to `.`; a `/` is never one, so `ESC[` before a path never eats the path start) and a final byte;
|
|
173
|
+
# or the same colour text with the ESC byte already stripped (`[38:2::255:0:0m`: digits, `:` and
|
|
174
|
+
# `;` then `m`, which no word character may precede). A final byte is a word character, so a
|
|
175
|
+
# sequence glued to a path would hide the start: the start grammar sees the sequence blanked, and
|
|
176
|
+
# the sequence is a kept prefix and suffix.
|
|
177
|
+
_CSI = r"\x1b\[[0-?]*[ -.]*[@-~]|(?<!\w)\[[0-9:;]*m"
|
|
178
|
+
_CSI_RE = re.compile(_CSI)
|
|
179
|
+
_CSI_FIRST = frozenset("\x1b[")
|
|
180
|
+
_STRICT_PATH_RE = re.compile(r"/[^/]+/|[\w-]\.\w+$")
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def _start_counts(before: str, value: str, after: str) -> bool:
|
|
184
|
+
"""Whether the POSIX start `value` (what `_ABS_PATH_RE` matched) counts as a path start given
|
|
185
|
+
`before`, the character in front of it, and `after`, the character right behind it ("" at the
|
|
186
|
+
start or end of the text). After `:`, `-` or a WORD character of a script written without
|
|
187
|
+
spaces (F fix 10, W10-2; F fix 11, W11-2: its punctuation and spaces are ordinary boundaries;
|
|
188
|
+
F fix 12, W12-1: a combining mark of the class, such as a Thai tone mark, ends a word like a
|
|
189
|
+
letter does) a run counts only with two segments, a file extension (decided with the trailing
|
|
190
|
+
`.` and `,` of the match removed, F fix 11, W11-1) or a `/` right after it (a segment the path
|
|
191
|
+
alphabet cannot read), so `and/or`, `read-/write`, `测试通过/失败 共 12 项`, `读/写` and
|
|
192
|
+
`ใช่/ไม่ใช่` stay prose; a `file://` URL and every other position count. The one rule for the
|
|
193
|
+
raw text (`_abs_starts`) and for the decoded text of an escaped word (`_decoded_starts`)."""
|
|
194
|
+
if _FILE_URL_ANY_RE.match(value):
|
|
195
|
+
return True
|
|
196
|
+
strict = before in (":", "-") or (
|
|
197
|
+
before != ""
|
|
198
|
+
and (_is_word_char(before) or unicodedata.category(before).startswith("M"))
|
|
199
|
+
and _UNSPACED_RE.match(before) is not None
|
|
200
|
+
)
|
|
201
|
+
if strict:
|
|
202
|
+
core = _one_leading_slash(value).rstrip(".,")
|
|
203
|
+
return after == "/" or _STRICT_PATH_RE.search(core) is not None
|
|
204
|
+
return True
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
# --- the free-text path scanner (docs/DESIGN.md 5.1, F fix 4) -------------------------------------
|
|
208
|
+
#
|
|
209
|
+
# A path in free text is found inside a complete shell word, and once a word holds a path the
|
|
210
|
+
# whole rest of the word is rendered as one path. No regular expression splits a path at a quote,
|
|
211
|
+
# an escape or a space: three review rounds each found one more spelling that a character class
|
|
212
|
+
# or a quoted-span regex cut in two, leaving a raw private tail. The scanner is
|
|
213
|
+
# _lex the text into words and boundaries, with the raw span of every decoded character;
|
|
214
|
+
# _PathLookup the registered path values, matched in NFC form, indexed by first character;
|
|
215
|
+
# _rewrite_word one word that holds a path, rewritten as prefix + rendered path + raw suffix;
|
|
216
|
+
# scan_paths the whole pass, the one entry every free-text output reaches.
|
|
217
|
+
|
|
218
|
+
_OPERATORS = frozenset(";|&<>()")
|
|
219
|
+
# The shell separates words at a space, a tab and a newline (and a carriage return in a CRLF
|
|
220
|
+
# log); a no-break space or an ideographic space is an ordinary character of a name.
|
|
221
|
+
_SPACE = frozenset(" \t\n\r\f\v")
|
|
222
|
+
_BREAKS = _OPERATORS | _SPACE
|
|
223
|
+
_LDQ = "\N{LEFT DOUBLE QUOTATION MARK}"
|
|
224
|
+
_RDQ = "\N{RIGHT DOUBLE QUOTATION MARK}"
|
|
225
|
+
_LSQ = "\N{LEFT SINGLE QUOTATION MARK}"
|
|
226
|
+
_RSQ = "\N{RIGHT SINGLE QUOTATION MARK}"
|
|
227
|
+
_DLQ = "\N{DOUBLE LOW-9 QUOTATION MARK}"
|
|
228
|
+
# Paired display wrappers (docs/DESIGN.md 5.1): the opener maps to the characters that close its
|
|
229
|
+
# fragment. A fragment is literal (no escapes) and may hold spaces; an unpaired opener takes the
|
|
230
|
+
# rest of its line (fail closed). The backquote is a markdown code span, not a shell command
|
|
231
|
+
# substitution; U+201D opens a fragment only where a closing partner follows (`_rdq_opens`).
|
|
232
|
+
_WRAPPER_CLOSERS = {
|
|
233
|
+
"`": "`",
|
|
234
|
+
_LDQ: _RDQ + _LDQ,
|
|
235
|
+
_RDQ: _RDQ + _LDQ,
|
|
236
|
+
_DLQ: _LDQ + _RDQ,
|
|
237
|
+
_LSQ: _RSQ,
|
|
238
|
+
"\u00ab": "\u00bb", # guillemets
|
|
239
|
+
"\u2039": "\u203a", # single guillemets
|
|
240
|
+
"\u300c": "\u300d", # CJK corner brackets
|
|
241
|
+
"\u300e": "\u300f", # CJK white corner brackets
|
|
242
|
+
}
|
|
243
|
+
# A closing wrapper that ends a word ends what the forward join may extend over.
|
|
244
|
+
_CLOSING_WRAPPERS = frozenset("".join(_WRAPPER_CLOSERS.values())) - {"`"}
|
|
245
|
+
_HIDDEN_MARK = " [text hidden]"
|
|
246
|
+
# The only text kept in front of a path: a proven syntactic prefix. `NAME=` (a shell
|
|
247
|
+
# identifier), `--option=`, a short option glued to the path (`-C/x`), a rule id (`R-NOPE-`), and
|
|
248
|
+
# an opening bracket or backquote. Anything else before the path start is part of the path
|
|
249
|
+
# (fail closed).
|
|
250
|
+
_PREFIX_RE = re.compile(
|
|
251
|
+
r"(?:" + _CSI + r")*[\[{`]*"
|
|
252
|
+
r"(?:[A-Za-z_][A-Za-z0-9_]*=|--[A-Za-z0-9][A-Za-z0-9_-]*=|-[A-Za-z](?=/)"
|
|
253
|
+
# a rule id (R-NO-LONG-BASH-) glued to a path is a name, never part of it
|
|
254
|
+
r"|R(?:-[A-Z0-9]+)+-(?=/))?"
|
|
255
|
+
)
|
|
256
|
+
# The raw suffix kept after a path: an evidence pointer (`:12`, `:12:3`), then sentence
|
|
257
|
+
# punctuation, a closing bracket, quote or wrapper, or an escaped trailing space.
|
|
258
|
+
_SUFFIX_WINDOW = 512
|
|
259
|
+
_SUFFIX_RE = re.compile(
|
|
260
|
+
r"(?:(?::[0-9]+){1,2})?(?:" + _CSI + r"|[.,;:)\]}!?`"
|
|
261
|
+
+ re.escape("".join(sorted(_CLOSING_WRAPPERS))) + r" \t\n\r\f\v])*\Z"
|
|
262
|
+
)
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
# F fix 10 (W10-4): an interior run of slashes names the same place as one (`/home/user/Docs//Acme/
|
|
266
|
+
# plan.md`); the structural collapses of `redact()` stop at an empty segment, so the token is
|
|
267
|
+
# collapsed first, and the structural form must then be one of the shapes the contract shows
|
|
268
|
+
# (`~`, `~/name`, `~/.../name`, `/tmp`, `/tmp/first`, `/tmp/first/name`, `/tmp/first/.../name`,
|
|
269
|
+
# each with `/private` in front of `/tmp` and an optional trailing slash), else the token is its
|
|
270
|
+
# own path-N.
|
|
271
|
+
_STRUCTURAL_SHAPE_RE = re.compile(
|
|
272
|
+
r"~(?:/(?:\.\.\./)?[^/]+)?/?|(?:/private)?/tmp(?:/[^/]+(?:/(?:\.\.\./)?[^/]+)?)?/?"
|
|
273
|
+
)
|
|
274
|
+
_SLASH_RUN_RE = re.compile(r"(?<=.)/{2,}", re.DOTALL)
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def _collapse_slash_runs(token: str) -> str:
|
|
278
|
+
return _SLASH_RUN_RE.sub("/", token)
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
def _nfc(text: str) -> str:
|
|
282
|
+
return text if text.isascii() else unicodedata.normalize("NFC", text)
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def _is_word_char(ch: str) -> bool:
|
|
286
|
+
return ch.isalnum() or ch == "_"
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def _inword_apostrophe(text: str, at: int) -> bool:
|
|
290
|
+
"""text[at] is an apostrophe with a word character on both sides (`it's`, `O'Neil`): an
|
|
291
|
+
ordinary character, not a quote (the path alphabet counts it as token text too)."""
|
|
292
|
+
return (
|
|
293
|
+
text[at] == "'" and 0 < at < len(text) - 1
|
|
294
|
+
and _is_word_char(text[at - 1]) and _is_word_char(text[at + 1])
|
|
295
|
+
)
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
def _inword_quote(text: str, at: int) -> bool:
|
|
299
|
+
"""text[at] is an apostrophe or a U+2019 with a word character on both sides: an ordinary
|
|
300
|
+
character, never a closing quote."""
|
|
301
|
+
return (
|
|
302
|
+
text[at] in ("'", _RSQ) and 0 < at < len(text) - 1
|
|
303
|
+
and _is_word_char(text[at - 1]) and _is_word_char(text[at + 1])
|
|
304
|
+
)
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def _one_leading_slash(value: str) -> str:
|
|
308
|
+
"""N3b R2-2: `//a/b` and `/a/b` name the same place; place one spelling."""
|
|
309
|
+
return "/" + value.lstrip("/") if value.startswith("//") else value
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
class _Word:
|
|
313
|
+
"""One logical shell word (or one run of boundary characters): its raw span `[start, end)`,
|
|
314
|
+
its decoded text, the raw span `[rstart[k], rend[k])` of every decoded character k, and the
|
|
315
|
+
quote fragments it opened, each `[opening index, closing index or -1]` in the raw text."""
|
|
316
|
+
|
|
317
|
+
__slots__ = (
|
|
318
|
+
"start", "end", "boundary", "chars", "rstart", "rend", "frags", "text", "escaped",
|
|
319
|
+
)
|
|
320
|
+
|
|
321
|
+
def __init__(self, start: int, boundary: bool = False) -> None:
|
|
322
|
+
self.start = start
|
|
323
|
+
self.end = start
|
|
324
|
+
self.boundary = boundary
|
|
325
|
+
self.chars: list[str] = []
|
|
326
|
+
self.rstart: list[int] = []
|
|
327
|
+
self.rend: list[int] = []
|
|
328
|
+
self.frags: list[list[int]] = []
|
|
329
|
+
self.text = ""
|
|
330
|
+
self.escaped = False # a JSON escape was decoded inside a double-quoted fragment
|
|
331
|
+
|
|
332
|
+
def add(self, ch: str, start: int, end: int) -> None:
|
|
333
|
+
self.chars.append(ch)
|
|
334
|
+
self.rstart.append(start)
|
|
335
|
+
self.rend.append(end)
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
_JSON_SIMPLE_ESCAPES = {"/": "/", "n": "\n", "t": "\t", "r": "\r", "b": "\b", "f": "\f"}
|
|
339
|
+
_HEX4_RE = re.compile(r"[0-9a-fA-F]{4}")
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
def _json_escape(text: str, at: int) -> tuple[str, int] | None:
|
|
343
|
+
"""The JSON escape that starts at text[at] (a backslash): its character and raw length, or
|
|
344
|
+
None. The escapes of a slash, n, t, r, b, f and uXXXX are decoded (a surrogate pair is one
|
|
345
|
+
character; a lone surrogate is not decoded, it stays as its escape text)."""
|
|
346
|
+
letter = text[at + 1:at + 2]
|
|
347
|
+
if letter in _JSON_SIMPLE_ESCAPES:
|
|
348
|
+
return _JSON_SIMPLE_ESCAPES[letter], 2
|
|
349
|
+
if letter != "u" or not _HEX4_RE.fullmatch(text[at + 2:at + 6]):
|
|
350
|
+
return None
|
|
351
|
+
code = int(text[at + 2:at + 6], 16)
|
|
352
|
+
if 0xD800 <= code < 0xDC00:
|
|
353
|
+
low = text[at + 6:at + 12]
|
|
354
|
+
if low[:2] == "\\u" and _HEX4_RE.fullmatch(low[2:]) and 0xDC00 <= int(low[2:], 16) < 0xE000:
|
|
355
|
+
return chr(0x10000 + ((code - 0xD800) << 10) + (int(low[2:], 16) - 0xDC00)), 12
|
|
356
|
+
return None
|
|
357
|
+
if 0xDC00 <= code < 0xE000:
|
|
358
|
+
return None
|
|
359
|
+
return chr(code), 6
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
def _lex_double(text: str, at: int, word: _Word) -> int:
|
|
363
|
+
"""A double-quoted fragment opened at text[at]; the index after it. A backslash escapes `"`,
|
|
364
|
+
a backslash, `$`, a backquote and a newline, and the JSON escapes (see `_json_escape`) are
|
|
365
|
+
decoded too (a truncated JSON string is still a path); before any other character it stays
|
|
366
|
+
literal. An unescaped newline ends an unclosed fragment (fail closed: the rest of the line
|
|
367
|
+
was inside)."""
|
|
368
|
+
fragment = [at, -1]
|
|
369
|
+
word.frags.append(fragment)
|
|
370
|
+
size = len(text)
|
|
371
|
+
i = at + 1
|
|
372
|
+
while i < size:
|
|
373
|
+
ch = text[i]
|
|
374
|
+
if ch == '"':
|
|
375
|
+
fragment[1] = i
|
|
376
|
+
return i + 1
|
|
377
|
+
if ch == "\n":
|
|
378
|
+
return i
|
|
379
|
+
if ch == "\\" and i + 1 < size:
|
|
380
|
+
following = text[i + 1]
|
|
381
|
+
if following == "\n":
|
|
382
|
+
i += 2
|
|
383
|
+
continue
|
|
384
|
+
if following == "\r" and text[i + 2:i + 3] == "\n":
|
|
385
|
+
i += 3
|
|
386
|
+
continue
|
|
387
|
+
if following in '"\\$`':
|
|
388
|
+
word.add(following, i, i + 2)
|
|
389
|
+
if following == "\\":
|
|
390
|
+
word.escaped = True # a Windows path is found on the decoded text too
|
|
391
|
+
i += 2
|
|
392
|
+
continue
|
|
393
|
+
decoded = _json_escape(text, i)
|
|
394
|
+
if decoded is not None:
|
|
395
|
+
word.add(decoded[0], i, i + decoded[1])
|
|
396
|
+
word.escaped = True
|
|
397
|
+
i += decoded[1]
|
|
398
|
+
continue
|
|
399
|
+
word.add(ch, i, i + 1)
|
|
400
|
+
i += 1
|
|
401
|
+
return i
|
|
402
|
+
|
|
403
|
+
|
|
404
|
+
def _lex_literal(
|
|
405
|
+
text: str, at: int, closers: str, word: _Word, registered: "_LineRegistered | None" = None,
|
|
406
|
+
) -> int:
|
|
407
|
+
"""A fragment with no escapes (single quotes, typographic quotes, display wrappers), opened at
|
|
408
|
+
text[at] and closed by any character of `closers`; the index after it. A U+2019 between two
|
|
409
|
+
word characters is an apostrophe, not a closer. A fragment opened by U+2018 that is
|
|
410
|
+
path-bearing (`_path_bearing`) closes at the last U+2019 of its line before the next U+2018
|
|
411
|
+
(F fix 10, W10-3), or runs to the end of the line after a cut-off possessive."""
|
|
412
|
+
fragment = [at, -1]
|
|
413
|
+
word.frags.append(fragment)
|
|
414
|
+
size = len(text)
|
|
415
|
+
i = at + 1
|
|
416
|
+
while i < size:
|
|
417
|
+
ch = text[i]
|
|
418
|
+
if ch in closers and not _inword_quote(text, i):
|
|
419
|
+
if text[at] == _LSQ and _path_bearing(text, at, i, registered):
|
|
420
|
+
fragment[1], i = _extend_to_last_quote(text, i, word, _RSQ, _LSQ)
|
|
421
|
+
return i
|
|
422
|
+
fragment[1] = i
|
|
423
|
+
return i + 1
|
|
424
|
+
if ch == "\n":
|
|
425
|
+
return i
|
|
426
|
+
word.add(ch, i, i + 1)
|
|
427
|
+
i += 1
|
|
428
|
+
return i
|
|
429
|
+
|
|
430
|
+
|
|
431
|
+
def _lex_single(
|
|
432
|
+
text: str, at: int, word: _Word, registered: "_LineRegistered | None" = None,
|
|
433
|
+
) -> int:
|
|
434
|
+
"""A single-quoted fragment opened at text[at]: `\\'` and `\\\\` inside it are the quote and
|
|
435
|
+
the backslash, the union of the shell rule and the Python repr rule (a repr of a string that
|
|
436
|
+
holds both quote kinds is one fragment at every sink; the union can only widen a span), any
|
|
437
|
+
other backslash stays literal. An in-word apostrophe (`Bob's`) is name text and never closes
|
|
438
|
+
the fragment (F fix 8, W8-1, made fail-closed in F fix 8b: Node, git and GNU tools print a name
|
|
439
|
+
in single quotes with no escaping, and an excerpt can be cut off before the real closing
|
|
440
|
+
quote). A fragment that holds a path start closes at the LAST quote of its line that is not an
|
|
441
|
+
in-word apostrophe (F fix 9, W9-3, fail closed): a possessive apostrophe before a space
|
|
442
|
+
(`'/srv/James' Notes/x.md'`) is indistinguishable from a closing quote, so the first such quote
|
|
443
|
+
never ends the span; a later quoted phrase on the line is hidden with the path. An unescaped
|
|
444
|
+
newline ends an unclosed fragment, so the rest of the line is hidden.
|
|
445
|
+
|
|
446
|
+
F fix 10 (W10-3): a fragment is path-bearing (`_path_bearing`) when its text holds an absolute
|
|
447
|
+
start or a registered value occurs between its opener and the end of its line; the quote that
|
|
448
|
+
closes it is the last of its line as above, and not even that one when it is a possessive of a
|
|
449
|
+
path cut off before its real closing quote (`_cut_off_possessive`): the fragment then runs to
|
|
450
|
+
the end of the line."""
|
|
451
|
+
fragment = [at, -1]
|
|
452
|
+
word.frags.append(fragment)
|
|
453
|
+
size = len(text)
|
|
454
|
+
i = at + 1
|
|
455
|
+
while i < size:
|
|
456
|
+
ch = text[i]
|
|
457
|
+
if ch == "\\" and i + 1 < size and text[i + 1] in "'\\":
|
|
458
|
+
word.add(text[i + 1], i, i + 2)
|
|
459
|
+
i += 2
|
|
460
|
+
continue
|
|
461
|
+
if ch == "'":
|
|
462
|
+
if _inword_apostrophe(text, i):
|
|
463
|
+
word.add(ch, i, i + 1) # `Bob's` inside `'/srv/Bob's Docs/x'` is name text
|
|
464
|
+
i += 1
|
|
465
|
+
continue
|
|
466
|
+
if _path_bearing(text, at, i, registered):
|
|
467
|
+
fragment[1], i = _extend_to_last_quote(text, i, word, "'")
|
|
468
|
+
return i
|
|
469
|
+
fragment[1] = i
|
|
470
|
+
return i + 1
|
|
471
|
+
if ch == "\n":
|
|
472
|
+
return i
|
|
473
|
+
word.add(ch, i, i + 1)
|
|
474
|
+
i += 1
|
|
475
|
+
return i
|
|
476
|
+
|
|
477
|
+
|
|
478
|
+
def _path_bearing(text: str, at: int, close: int, registered: "_LineRegistered | None") -> bool:
|
|
479
|
+
"""Whether the quote fragment opened at text[at] whose first closing candidate is text[close]
|
|
480
|
+
is path-bearing (F fix 10, W10-3): its text holds an absolute path start (a bare `'/'` names no
|
|
481
|
+
directory and does not count), or a registered value occurs between its opener and the end of
|
|
482
|
+
its line."""
|
|
483
|
+
quoted = text[at + 1:close]
|
|
484
|
+
if quoted.strip() != "/" and _abs_starts(quoted):
|
|
485
|
+
return True
|
|
486
|
+
return registered is not None and registered.after(at)
|
|
487
|
+
|
|
488
|
+
|
|
489
|
+
_AFTER_QUOTE_RE = re.compile(r"[ \t]+(\S*)")
|
|
490
|
+
_NAME_DOT_RE = re.compile(r"\.\w")
|
|
491
|
+
|
|
492
|
+
|
|
493
|
+
def _cut_off_possessive(text: str, quote: int) -> bool:
|
|
494
|
+
"""Whether the quote text[quote] may be a possessive of a path cut off before its real closing
|
|
495
|
+
quote (F fix 10, W10-3, rule b): it follows a word character and is followed by one or more
|
|
496
|
+
spaces or tabs and then a token (the characters up to the next whitespace) that holds `/`,
|
|
497
|
+
`\\` or a `.` followed by a word character (`open '/srv/James' Notes/x`, `... Notes.md`)."""
|
|
498
|
+
if quote == 0 or not _is_word_char(text[quote - 1]):
|
|
499
|
+
return False
|
|
500
|
+
after = _AFTER_QUOTE_RE.match(text, quote + 1)
|
|
501
|
+
if after is None:
|
|
502
|
+
return False
|
|
503
|
+
token = after.group(1)
|
|
504
|
+
return "/" in token or "\\" in token or _NAME_DOT_RE.search(token) is not None
|
|
505
|
+
|
|
506
|
+
|
|
507
|
+
def _extend_to_last_quote(
|
|
508
|
+
text: str, close: int, word: _Word, quote: str, opener: str | None = None,
|
|
509
|
+
) -> tuple[int, int]:
|
|
510
|
+
"""(the raw index of the quote that closes a path-bearing fragment, the index to lex on from)
|
|
511
|
+
for a fragment whose first closing candidate is text[close]. The closing quote is the last
|
|
512
|
+
`quote` of the line of `text[close]` that is not an in-word apostrophe (at least `close`
|
|
513
|
+
itself; for a U+2018 fragment it also stops before the next `opener` of the line), F fix 9,
|
|
514
|
+
W9-3; the text between is added to the word as name text. When that quote is a possessive of a
|
|
515
|
+
cut-off path (`_cut_off_possessive`) the fragment is not closed there: the index is -1 and the
|
|
516
|
+
word takes the rest of the line (fail closed)."""
|
|
517
|
+
line_end = text.find("\n", close)
|
|
518
|
+
line_end = len(text) if line_end < 0 else line_end
|
|
519
|
+
limit = line_end
|
|
520
|
+
if opener is not None:
|
|
521
|
+
following = text.find(opener, close, line_end)
|
|
522
|
+
if following >= 0:
|
|
523
|
+
limit = following
|
|
524
|
+
last = text.rfind(quote, close, limit)
|
|
525
|
+
while last > close and _inword_quote(text, last):
|
|
526
|
+
last = text.rfind(quote, close, last)
|
|
527
|
+
cut = _cut_off_possessive(text, last)
|
|
528
|
+
stop = line_end if cut else last
|
|
529
|
+
k = close
|
|
530
|
+
while k < stop:
|
|
531
|
+
if quote == "'" and text[k] == "\\" and k + 1 < stop and text[k + 1] in "'\\":
|
|
532
|
+
word.add(text[k + 1], k, k + 2) # the escapes of `_lex_single`, in the text between
|
|
533
|
+
k += 2
|
|
534
|
+
else:
|
|
535
|
+
word.add(text[k], k, k + 1)
|
|
536
|
+
k += 1
|
|
537
|
+
return (-1, line_end) if cut else (last, last + 1)
|
|
538
|
+
|
|
539
|
+
|
|
540
|
+
class _LineRegistered:
|
|
541
|
+
"""Whether a registered value occurs between a quote opener and the end of its line (F fix 10,
|
|
542
|
+
W10-3). The line is analysed once, however many fragments it holds: the raw line gives the
|
|
543
|
+
index of the last registered value in it; the line with `\\'` and `\\\\` decoded and the NFC
|
|
544
|
+
form of the line (and of the decoded line) give no index, so any value that hits one of them
|
|
545
|
+
makes every fragment on the line path-bearing (fail closed instead of mapping indexes back).
|
|
546
|
+
Cost: O(L) probes of the lookup per line of L characters per spelling."""
|
|
547
|
+
|
|
548
|
+
__slots__ = ("_lookup", "_text", "_start", "_end", "_last", "_wide")
|
|
549
|
+
|
|
550
|
+
def __init__(self, lookup: "_PathLookup", text: str) -> None:
|
|
551
|
+
self._lookup = lookup
|
|
552
|
+
self._text = text
|
|
553
|
+
self._start = self._end = -1
|
|
554
|
+
self._last = -1
|
|
555
|
+
self._wide = False
|
|
556
|
+
|
|
557
|
+
def after(self, opener: int) -> bool:
|
|
558
|
+
if not self._start <= opener < self._end:
|
|
559
|
+
self._analyse(opener)
|
|
560
|
+
return self._wide or self._last >= opener
|
|
561
|
+
|
|
562
|
+
def _holds(self, line: str) -> int:
|
|
563
|
+
"""The index of the last registered value in `line`, or -1."""
|
|
564
|
+
first = self._lookup._by_first
|
|
565
|
+
for at in range(len(line) - 1, -1, -1):
|
|
566
|
+
if line[at] in first and self._lookup.longest_at(line, at) is not None:
|
|
567
|
+
return at
|
|
568
|
+
return -1
|
|
569
|
+
|
|
570
|
+
def _analyse(self, opener: int) -> None:
|
|
571
|
+
text = self._text
|
|
572
|
+
start = text.rfind("\n", 0, opener) + 1
|
|
573
|
+
end = text.find("\n", opener)
|
|
574
|
+
end = len(text) if end < 0 else end
|
|
575
|
+
line = text[start:end]
|
|
576
|
+
last = self._holds(line)
|
|
577
|
+
self._start, self._end = start, end
|
|
578
|
+
self._last = start + last if last >= 0 else -1
|
|
579
|
+
spellings = []
|
|
580
|
+
normal = _nfc(line)
|
|
581
|
+
if normal != line:
|
|
582
|
+
spellings.append(normal)
|
|
583
|
+
if "\\" in line:
|
|
584
|
+
decoded = _SINGLE_ESCAPE_RE.sub(r"\1", line)
|
|
585
|
+
spellings.append(decoded)
|
|
586
|
+
if _nfc(decoded) != decoded:
|
|
587
|
+
spellings.append(_nfc(decoded))
|
|
588
|
+
self._wide = any(self._holds(spelling) >= 0 for spelling in spellings)
|
|
589
|
+
|
|
590
|
+
|
|
591
|
+
_SINGLE_ESCAPE_RE = re.compile(r"\\(['\\])")
|
|
592
|
+
|
|
593
|
+
|
|
594
|
+
def _rdq_opens(text: str, at: int) -> bool:
|
|
595
|
+
"""U+201D is a closing quote far more often than an opener: it opens a display fragment only
|
|
596
|
+
where a closing partner follows on the same line, a U+201D after a word character
|
|
597
|
+
(`Client” Secret.txt”`, the smart-quote spelling of a half-quoted name) and a U+201D or a
|
|
598
|
+
U+201C otherwise. A lone one glued to a path stays a suffix."""
|
|
599
|
+
line_end = text.find("\n", at + 1)
|
|
600
|
+
line_end = len(text) if line_end < 0 else line_end
|
|
601
|
+
if text.find(_RDQ, at + 1, line_end) >= 0:
|
|
602
|
+
return True
|
|
603
|
+
if at and _is_word_char(text[at - 1]):
|
|
604
|
+
return False
|
|
605
|
+
return text.find(_LDQ, at + 1, line_end) >= 0
|
|
606
|
+
|
|
607
|
+
|
|
608
|
+
def _lex_word(
|
|
609
|
+
text: str, at: int, word: _Word, registered: "_LineRegistered | None" = None,
|
|
610
|
+
) -> int:
|
|
611
|
+
size = len(text)
|
|
612
|
+
i = at
|
|
613
|
+
while i < size:
|
|
614
|
+
ch = text[i]
|
|
615
|
+
if ch in _BREAKS:
|
|
616
|
+
break
|
|
617
|
+
if ch == "\\":
|
|
618
|
+
if i + 1 >= size:
|
|
619
|
+
word.add(ch, i, i + 1)
|
|
620
|
+
i += 1
|
|
621
|
+
elif text[i + 1] == "\n":
|
|
622
|
+
i += 2 # a line continuation joins the next line into this word
|
|
623
|
+
elif text[i + 1] == "\r" and text[i + 2:i + 3] == "\n":
|
|
624
|
+
i += 3
|
|
625
|
+
else:
|
|
626
|
+
word.add(text[i + 1], i, i + 2)
|
|
627
|
+
i += 2
|
|
628
|
+
elif ch == '"':
|
|
629
|
+
i = _lex_double(text, i, word)
|
|
630
|
+
elif ch == "'":
|
|
631
|
+
if _inword_apostrophe(text, i):
|
|
632
|
+
word.add(ch, i, i + 1)
|
|
633
|
+
i += 1
|
|
634
|
+
else:
|
|
635
|
+
i = _lex_single(text, i, word, registered)
|
|
636
|
+
elif ch in _CSI_FIRST and (csi := _CSI_RE.match(text, i)) is not None:
|
|
637
|
+
# a control sequence is name text of the word, `;` `<` `>` and all (F fix 8, W8-3)
|
|
638
|
+
for k in range(i, csi.end()):
|
|
639
|
+
word.add(text[k], k, k + 1)
|
|
640
|
+
i = csi.end()
|
|
641
|
+
elif ch in _WRAPPER_CLOSERS and (ch != _RDQ or _rdq_opens(text, i)):
|
|
642
|
+
i = _lex_literal(text, i, _WRAPPER_CLOSERS[ch], word, registered)
|
|
643
|
+
else:
|
|
644
|
+
word.add(ch, i, i + 1)
|
|
645
|
+
i += 1
|
|
646
|
+
return i
|
|
647
|
+
|
|
648
|
+
|
|
649
|
+
def _lex(text: str, lookup: "_PathLookup | None" = None) -> list[_Word]:
|
|
650
|
+
"""The text as alternating words and boundaries (docs/DESIGN.md 5.1). Outside quotes a word
|
|
651
|
+
ends at unescaped whitespace or at one of `; | & < > ( )` (a run of them is one boundary,
|
|
652
|
+
copied back unchanged); an unescaped newline always ends a word and no quote crosses it. With a
|
|
653
|
+
registered `lookup` a single-quote or U+2018 fragment is also path-bearing when a registered
|
|
654
|
+
value follows its opener on its line (F fix 10, W10-3); `_holds_start` passes none."""
|
|
655
|
+
size = len(text)
|
|
656
|
+
registered = _LineRegistered(lookup, text) if lookup else None
|
|
657
|
+
segments: list[_Word] = []
|
|
658
|
+
i = 0
|
|
659
|
+
while i < size:
|
|
660
|
+
ch = text[i]
|
|
661
|
+
if ch in _BREAKS:
|
|
662
|
+
j = i + 1
|
|
663
|
+
while j < size and text[j] in _BREAKS:
|
|
664
|
+
j += 1
|
|
665
|
+
boundary = _Word(i, boundary=True)
|
|
666
|
+
boundary.end = j
|
|
667
|
+
boundary.text = text[i:j]
|
|
668
|
+
segments.append(boundary)
|
|
669
|
+
i = j
|
|
670
|
+
continue
|
|
671
|
+
word = _Word(i)
|
|
672
|
+
i = _lex_word(text, i, word, registered)
|
|
673
|
+
word.end = i
|
|
674
|
+
word.text = "".join(word.chars)
|
|
675
|
+
segments.append(word)
|
|
676
|
+
return segments
|
|
677
|
+
|
|
678
|
+
|
|
679
|
+
class _PathLookup:
|
|
680
|
+
"""The registered report_path values, each stored once in NFC form (an NFD spelling matches
|
|
681
|
+
an NFC registration and the reverse; no case folding), indexed by first character.
|
|
682
|
+
|
|
683
|
+
Bound: one entry per distinct registered value (a run registers the distinct paths its
|
|
684
|
+
claims, observations and ledgers name), memory O(total bytes of the values), kept in step
|
|
685
|
+
with the path namespace incrementally (it only grows). A scan costs O(L * k) for L characters
|
|
686
|
+
of text and k, the number of registered values sharing a first character at a position
|
|
687
|
+
(every absolute value shares `/`); the value is tried longest first with `startswith`."""
|
|
688
|
+
|
|
689
|
+
__slots__ = ("_by_nfc", "_by_first", "_seen", "_aliases")
|
|
690
|
+
|
|
691
|
+
def __init__(self) -> None:
|
|
692
|
+
self._aliases: set[str] = set()
|
|
693
|
+
self._by_nfc: dict[str, str] = {}
|
|
694
|
+
self._by_first: dict[str, list[str]] = {}
|
|
695
|
+
self._seen = 0
|
|
696
|
+
|
|
697
|
+
def __bool__(self) -> bool:
|
|
698
|
+
return bool(self._by_nfc)
|
|
699
|
+
|
|
700
|
+
def sync(self, mapping: dict[str, str]) -> None:
|
|
701
|
+
if len(mapping) == self._seen:
|
|
702
|
+
return
|
|
703
|
+
for value in itertools.islice(mapping, self._seen, None):
|
|
704
|
+
if not value or value == "/":
|
|
705
|
+
continue # the root is registered but is a substring of every path
|
|
706
|
+
# a value that ends in a slash also matches without it (`/srv/my work/` and
|
|
707
|
+
# `/srv/my work`), as a name the directory is called by
|
|
708
|
+
keys = [(_nfc(value), False)]
|
|
709
|
+
if len(value) > 1 and value.endswith("/") and value.rstrip("/"):
|
|
710
|
+
keys.append((_nfc(value.rstrip("/")), True))
|
|
711
|
+
for key, alias in keys:
|
|
712
|
+
if key in self._by_nfc:
|
|
713
|
+
if not alias and key in self._aliases:
|
|
714
|
+
# a genuine registration outranks the slash-less spelling of another
|
|
715
|
+
self._by_nfc[key] = value
|
|
716
|
+
self._aliases.discard(key)
|
|
717
|
+
continue
|
|
718
|
+
self._by_nfc[key] = value
|
|
719
|
+
if alias:
|
|
720
|
+
self._aliases.add(key)
|
|
721
|
+
bisect.insort(
|
|
722
|
+
self._by_first.setdefault(key[0], []), key, key=lambda item: -len(item),
|
|
723
|
+
)
|
|
724
|
+
self._seen = len(mapping)
|
|
725
|
+
|
|
726
|
+
def exact(self, text: str) -> str | None:
|
|
727
|
+
"""The registered value that `text` is, in NFC form, or None."""
|
|
728
|
+
return self._by_nfc.get(_nfc(text))
|
|
729
|
+
|
|
730
|
+
def lead(self, text: str) -> tuple[str, int] | None:
|
|
731
|
+
"""(the registered value that `text` starts with, the length of that value and any
|
|
732
|
+
sentence punctuation right after it) when whitespace follows (the first word of a span
|
|
733
|
+
that holds more); None otherwise. Only for a text that is already in NFC form, so the
|
|
734
|
+
length is a length in `text` too."""
|
|
735
|
+
normal = _nfc(text)
|
|
736
|
+
if normal != text:
|
|
737
|
+
return None
|
|
738
|
+
key = self.longest_at(normal, 0) if normal else None
|
|
739
|
+
if key is None:
|
|
740
|
+
return None
|
|
741
|
+
end = len(key)
|
|
742
|
+
while end < len(normal) and normal[end] in ".,;:!?":
|
|
743
|
+
end += 1 # sentence punctuation right after the value belongs to the sentence
|
|
744
|
+
if end < len(normal) and normal[end] in _SPACE:
|
|
745
|
+
return self._by_nfc[key], end
|
|
746
|
+
return None
|
|
747
|
+
|
|
748
|
+
def longest_at(self, text: str, at: int) -> str | None:
|
|
749
|
+
for key in self._by_first.get(text[at], ()):
|
|
750
|
+
if text.startswith(key, at):
|
|
751
|
+
return key
|
|
752
|
+
return None
|
|
753
|
+
|
|
754
|
+
def hits(self, parts: list[str]) -> list[tuple[int, int]]:
|
|
755
|
+
"""(first part, last part) of every registered value in the NFC form of the parts
|
|
756
|
+
joined; only a value that spans parts is wider than its own part."""
|
|
757
|
+
normal = [_nfc(part) for part in parts]
|
|
758
|
+
joined = "".join(normal)
|
|
759
|
+
ends = list(itertools.accumulate(len(part) for part in normal))
|
|
760
|
+
found: list[tuple[int, int]] = []
|
|
761
|
+
first = self._by_first
|
|
762
|
+
for at, ch in enumerate(joined):
|
|
763
|
+
if ch not in first:
|
|
764
|
+
continue
|
|
765
|
+
key = self.longest_at(joined, at)
|
|
766
|
+
if key is not None:
|
|
767
|
+
found.append((
|
|
768
|
+
bisect.bisect_right(ends, at), bisect.bisect_right(ends, at + len(key) - 1),
|
|
769
|
+
))
|
|
770
|
+
return found
|
|
771
|
+
|
|
772
|
+
|
|
773
|
+
def _blank_csi(text: str) -> str:
|
|
774
|
+
"""`text` with every terminal control sequence replaced by spaces of the same length (indexes
|
|
775
|
+
are unchanged): the start grammar then sees a boundary where its final byte would be a word
|
|
776
|
+
character (F fix 8, W8-3; F fix 9, W9-5)."""
|
|
777
|
+
if "[" not in text:
|
|
778
|
+
return text
|
|
779
|
+
return _CSI_RE.sub(lambda match: " " * len(match.group(0)), text)
|
|
780
|
+
|
|
781
|
+
|
|
782
|
+
def _abs_starts(text: str) -> list[int]:
|
|
783
|
+
"""Every index of `text` where the structural grammar `_ABS_PATH_RE` starts an absolute path
|
|
784
|
+
(a run after `:` or `-` counts only with two segments or a file extension, so `and/or` and
|
|
785
|
+
`read-/write` stay prose), `_WIN_PATH_RE` starts a Windows path, or `_VAR_ANCHOR_RE` starts a
|
|
786
|
+
home or working-directory anchor (`~/`, `~user/`, `$HOME/`, `$PWD/`, wherever it stands). Found
|
|
787
|
+
on the raw text, in its real context: a lone `/` between spaces is prose, the same `/` inside
|
|
788
|
+
quotes is the root. A colour sequence is blanked first. One pass for the whole text."""
|
|
789
|
+
text = _blank_csi(text)
|
|
790
|
+
found = []
|
|
791
|
+
for match in _ABS_PATH_RE.finditer(text):
|
|
792
|
+
if _start_counts(
|
|
793
|
+
text[match.start() - 1] if match.start() else "", match.group(0),
|
|
794
|
+
text[match.end()] if match.end() < len(text) else "",
|
|
795
|
+
):
|
|
796
|
+
found.append(match.start())
|
|
797
|
+
extra = [match.start() for match in _WIN_PATH_RE.finditer(text)]
|
|
798
|
+
extra.extend(match.start() for match in _VAR_ANCHOR_RE.finditer(text))
|
|
799
|
+
return sorted(set(found + extra)) if extra else found
|
|
800
|
+
|
|
801
|
+
|
|
802
|
+
def _decoded_starts(word: _Word) -> list[int]:
|
|
803
|
+
"""The raw index of every path start in the DECODED text of a word that decoded an escape
|
|
804
|
+
inside a double-quoted fragment (`"\\u002fsrv\\u002fx y"` has no literal `/`, and a quoted
|
|
805
|
+
`"\\\\server\\x"` no unescaped backslash, so the raw text cannot show them): an absolute
|
|
806
|
+
start, a home anchor or a Windows start. Empty when the decoded text holds none."""
|
|
807
|
+
decoded = _blank_csi(word.text)
|
|
808
|
+
found = []
|
|
809
|
+
for match in _ABS_PATH_RE.finditer(decoded):
|
|
810
|
+
if _start_counts(
|
|
811
|
+
decoded[match.start() - 1] if match.start() else "", match.group(0),
|
|
812
|
+
decoded[match.end()] if match.end() < len(decoded) else "",
|
|
813
|
+
):
|
|
814
|
+
found.append(word.rstart[match.start()])
|
|
815
|
+
for pattern in (_VAR_ANCHOR_RE, _WIN_PATH_RE):
|
|
816
|
+
found.extend(word.rstart[match.start()] for match in pattern.finditer(decoded))
|
|
817
|
+
return found
|
|
818
|
+
|
|
819
|
+
|
|
820
|
+
def _anchors_in(word: _Word, starts: list[int]) -> list[int]:
|
|
821
|
+
"""The raw index of every path start inside the word (`starts`: every start of the whole
|
|
822
|
+
text), a leading home anchor (`~/`, `~user/`, `$HOME/`, `${HOME}/`) after the proven prefix,
|
|
823
|
+
and, for a word that decoded an escape, every start in its decoded text; empty when it holds
|
|
824
|
+
none."""
|
|
825
|
+
decoded = word.text
|
|
826
|
+
prefix = _PREFIX_RE.match(decoded).end()
|
|
827
|
+
at = bisect.bisect_left(starts, word.start)
|
|
828
|
+
found = []
|
|
829
|
+
while at < len(starts) and starts[at] < word.end:
|
|
830
|
+
found.append(starts[at])
|
|
831
|
+
at += 1
|
|
832
|
+
if prefix < len(decoded) and _HOME_START_RE.match(decoded, prefix):
|
|
833
|
+
found.append(word.rstart[prefix])
|
|
834
|
+
if word.escaped:
|
|
835
|
+
found.extend(_decoded_starts(word))
|
|
836
|
+
return found
|
|
837
|
+
|
|
838
|
+
|
|
839
|
+
def _abs_start_in(word: _Word, starts: list[int]) -> int | None:
|
|
840
|
+
"""The raw index of the first path start inside the word (see `_anchors_in`); None when it
|
|
841
|
+
holds none."""
|
|
842
|
+
found = _anchors_in(word, starts)
|
|
843
|
+
return min(found) if found else None
|
|
844
|
+
|
|
845
|
+
|
|
846
|
+
def _rewrite_word(
|
|
847
|
+
word: _Word, text: str, lookup: _PathLookup, flagged: bool, names: "Pseudonyms",
|
|
848
|
+
render: Callable[[str, bool], str], starts: list[int],
|
|
849
|
+
render_hidden: Callable[[str, bool], str] | None = None,
|
|
850
|
+
) -> str | None:
|
|
851
|
+
"""The replacement for the raw span of one word that holds a path, or None when it holds
|
|
852
|
+
none. The path starts at the start of the word, moved right only past a proven prefix, and
|
|
853
|
+
runs to the end of the word minus a raw suffix; it is rendered once. A span whose decoded
|
|
854
|
+
text holds whitespace may be more than a path: it is rendered as an opaque path-N
|
|
855
|
+
(`render_hidden`, never a structural or repo-relative form) and followed by the marker."""
|
|
856
|
+
decoded = word.text
|
|
857
|
+
size = len(decoded)
|
|
858
|
+
prefix = _PREFIX_RE.match(decoded).end()
|
|
859
|
+
if not flagged and _abs_start_in(word, starts) is None:
|
|
860
|
+
return None
|
|
861
|
+
start = prefix if prefix < size else 0
|
|
862
|
+
if start and lookup:
|
|
863
|
+
# a registered value that overlaps the prefix is part of the path
|
|
864
|
+
normal = _nfc(decoded)
|
|
865
|
+
for at in range(start):
|
|
866
|
+
key = lookup.longest_at(normal, at) if normal[at] in lookup._by_first else None
|
|
867
|
+
if key is not None and at + len(key) > start:
|
|
868
|
+
start = 0
|
|
869
|
+
break
|
|
870
|
+
tail = decoded[start:]
|
|
871
|
+
exact = lookup.exact(tail) if lookup else None
|
|
872
|
+
path_text = tail
|
|
873
|
+
if exact is None:
|
|
874
|
+
# F fix 10 (W10-5): the search window is the last 512 characters (a lookbehind still sees
|
|
875
|
+
# the characters before it): a longer run of suffix characters is shortened, so the extra
|
|
876
|
+
# characters stay inside the hidden path (fail closed), and the search stays linear
|
|
877
|
+
found = _SUFFIX_RE.search(tail, max(0, len(tail) - _SUFFIX_WINDOW))
|
|
878
|
+
if found is not None and found.start() > 0:
|
|
879
|
+
path_text = tail[:found.start()]
|
|
880
|
+
if lookup:
|
|
881
|
+
exact = lookup.exact(path_text)
|
|
882
|
+
end = start + len(path_text)
|
|
883
|
+
spaced = any(ch in _SPACE for ch in path_text)
|
|
884
|
+
# the path as the raw text spells it (quotes and backslashes in place): a registered value
|
|
885
|
+
# that holds a quote character or a backslash matches its raw spelling, and a Windows path is
|
|
886
|
+
# recognized by it (an unquoted backslash is a shell escape, so the decoded text lost it)
|
|
887
|
+
raw_path = text[word.rstart[start]:word.rend[end - 1]] if end > start else ""
|
|
888
|
+
if exact is None and lookup and raw_path != path_text:
|
|
889
|
+
exact = lookup.exact(raw_path)
|
|
890
|
+
lead = lookup.lead(path_text) if lookup and exact is None and spaced else None
|
|
891
|
+
if lead is not None:
|
|
892
|
+
# only a plain separator (not an escaped or quoted space, which belongs to a longer name)
|
|
893
|
+
at = start + lead[1]
|
|
894
|
+
if not (
|
|
895
|
+
word.rend[at] - word.rstart[at] == 1 and not _in_fragment(word, word.rstart[at])
|
|
896
|
+
):
|
|
897
|
+
lead = None
|
|
898
|
+
if lead is None and lookup and exact is None and spaced and raw_path != path_text:
|
|
899
|
+
raw_lead = lookup.lead(raw_path)
|
|
900
|
+
if raw_lead is not None:
|
|
901
|
+
at = word.rstart[start] + raw_lead[1]
|
|
902
|
+
if text[at] in _SPACE and text[at - 1] != "\\" and not _in_fragment(word, at):
|
|
903
|
+
lead = raw_lead
|
|
904
|
+
if exact is not None:
|
|
905
|
+
shown = names.registered_display(exact)
|
|
906
|
+
elif lead is not None:
|
|
907
|
+
# a span that starts with a registered value followed by hidden text keeps that value's
|
|
908
|
+
# pseudonym (the same path in two places is one pseudonym), plus the marker
|
|
909
|
+
shown = names.registered_display(lead[0])
|
|
910
|
+
else:
|
|
911
|
+
candidate = path_text
|
|
912
|
+
file_url = _FILE_URL_RE.match(candidate)
|
|
913
|
+
if file_url is not None:
|
|
914
|
+
candidate = candidate[file_url.end():].rstrip("/") or "/"
|
|
915
|
+
candidate = _one_leading_slash(candidate)
|
|
916
|
+
if candidate.startswith("${HOME}/"):
|
|
917
|
+
candidate = "~" + candidate[len("${HOME}"):]
|
|
918
|
+
elif candidate.startswith("$HOME/"):
|
|
919
|
+
candidate = "~" + candidate[len("$HOME"):]
|
|
920
|
+
# a Windows path (or a `file://` URL naming a host) is never a repo-relative or a home
|
|
921
|
+
# form: it is always its own opaque path-N
|
|
922
|
+
foreign = (
|
|
923
|
+
_WIN_PATH_RE.match(raw_path) is not None or _WIN_PATH_RE.match(path_text) is not None
|
|
924
|
+
or (_FILE_URL_ANY_RE.match(path_text) is not None and file_url is None)
|
|
925
|
+
)
|
|
926
|
+
if (spaced or foreign) and render_hidden is not None:
|
|
927
|
+
shown = render_hidden(candidate, flagged)
|
|
928
|
+
else:
|
|
929
|
+
shown = render(candidate, flagged)
|
|
930
|
+
path_start = word.rstart[start]
|
|
931
|
+
hidden_to = word.rend[end - 1]
|
|
932
|
+
dropped = set()
|
|
933
|
+
for fragment in word.frags:
|
|
934
|
+
if fragment[0] >= path_start:
|
|
935
|
+
dropped.add(fragment[0])
|
|
936
|
+
if fragment[1] >= 0:
|
|
937
|
+
dropped.add(fragment[1])
|
|
938
|
+
elif path_start <= fragment[1] < hidden_to:
|
|
939
|
+
# a quote opened before the path start whose closing quote is inside the hidden span
|
|
940
|
+
# (`PATH="/usr/bin":/srv/a b`): neither delimiter is kept, so none is left unpaired
|
|
941
|
+
dropped.add(fragment[0])
|
|
942
|
+
prefix_text = "".join(text[k] for k in range(word.start, path_start) if k not in dropped)
|
|
943
|
+
kept_tail = "".join(text[k] for k in range(hidden_to, word.end) if k not in dropped)
|
|
944
|
+
return prefix_text + shown + (_HIDDEN_MARK if spaced else "") + kept_tail
|
|
945
|
+
|
|
946
|
+
|
|
947
|
+
def _join(segments: list[_Word], text: str, first: int, last: int) -> _Word:
|
|
948
|
+
"""The words first..last (and the boundaries between them) as one word: a registered value
|
|
949
|
+
that holds whitespace or an operator spans them."""
|
|
950
|
+
joined = _Word(segments[first].start)
|
|
951
|
+
joined.end = segments[last].end
|
|
952
|
+
for segment in segments[first:last + 1]:
|
|
953
|
+
if segment.boundary:
|
|
954
|
+
for k in range(segment.start, segment.end):
|
|
955
|
+
joined.add(text[k], k, k + 1)
|
|
956
|
+
else:
|
|
957
|
+
joined.chars.extend(segment.chars)
|
|
958
|
+
joined.rstart.extend(segment.rstart)
|
|
959
|
+
joined.rend.extend(segment.rend)
|
|
960
|
+
joined.frags.extend(segment.frags)
|
|
961
|
+
joined.text = "".join(joined.chars)
|
|
962
|
+
return joined
|
|
963
|
+
|
|
964
|
+
|
|
965
|
+
# The end-of-line rule (docs/DESIGN.md 5.1, F fix 7). A path start in unquoted context hides
|
|
966
|
+
# everything to the end of its line. There is NO other stop: text that came from a transcript, a
|
|
967
|
+
# file, an exception or the environment is untrusted, and no operator, closing wrapper or
|
|
968
|
+
# line-number shape in it is proof that the path ended (every one of them is legal inside a
|
|
969
|
+
# POSIX file name). The span ends only at the end of the line or at the closing delimiter of a
|
|
970
|
+
# fragment that was opened before the path start, which the lexer has already made one word.
|
|
971
|
+
# The display forms this tool prints for a collapsed path (`~/.../name`, `/tmp/dir/.../name`) are
|
|
972
|
+
# a fixed point of the scanner: a text that already holds one (a line scanned a second time) keeps
|
|
973
|
+
# the words after it, because the `...` segment says the directories were hidden already.
|
|
974
|
+
_DISPLAY_FORM_RE = re.compile(r"(?:~|/(?:private/)?tmp/[^/\s]+)/\.\.\./[^/\s]+")
|
|
975
|
+
|
|
976
|
+
|
|
977
|
+
def _in_fragment(word: _Word, raw: int) -> bool:
|
|
978
|
+
"""Whether the raw index `raw` lies inside a quote or display-wrapper fragment of the word."""
|
|
979
|
+
return any(f[0] < raw and (f[1] < 0 or raw < f[1]) for f in word.frags)
|
|
980
|
+
|
|
981
|
+
|
|
982
|
+
def _in_closed_fragment(word: _Word, openers: list[int], raw: int) -> bool:
|
|
983
|
+
"""Whether the raw index `raw` lies inside a fragment of the word that has been closed.
|
|
984
|
+
Fragments never nest and come in order, so the one that can hold `raw` is the last that
|
|
985
|
+
opened before it (`openers`: the opening index of every fragment of the word, in order)."""
|
|
986
|
+
at = bisect.bisect_left(openers, raw) - 1
|
|
987
|
+
return at >= 0 and word.frags[at][1] >= 0 and raw < word.frags[at][1]
|
|
988
|
+
|
|
989
|
+
|
|
990
|
+
def _line_span(
|
|
991
|
+
segments: list[_Word], k: int, starts: list[int],
|
|
992
|
+
) -> tuple[int, int | None]:
|
|
993
|
+
"""(index of the last segment of the span that starts at the word k, the raw index where
|
|
994
|
+
the span ends inside that segment, or None when it ends with the segment). A path start in
|
|
995
|
+
unquoted context hides everything to the end of its line (docs/DESIGN.md 5.1): the span runs
|
|
996
|
+
over every following word and separator of the same line. A word whose every path start sits
|
|
997
|
+
inside a closed quote ends with the quote: its word is not extended."""
|
|
998
|
+
first = segments[k]
|
|
999
|
+
anchors = _anchors_in(first, starts)
|
|
1000
|
+
if not anchors:
|
|
1001
|
+
anchors = [first.rstart[0] if first.rstart else first.start]
|
|
1002
|
+
openers = [fragment[0] for fragment in first.frags]
|
|
1003
|
+
if all(_in_closed_fragment(first, openers, anchor) for anchor in anchors):
|
|
1004
|
+
return k, None
|
|
1005
|
+
from_char = bisect.bisect_left(first.rstart, min(anchors))
|
|
1006
|
+
shown = _DISPLAY_FORM_RE.match(first.text, from_char)
|
|
1007
|
+
if shown is not None and _SUFFIX_RE.fullmatch(first.text[shown.end():]):
|
|
1008
|
+
return k, None
|
|
1009
|
+
last = k
|
|
1010
|
+
j = k + 1
|
|
1011
|
+
while j < len(segments):
|
|
1012
|
+
segment = segments[j]
|
|
1013
|
+
if segment.boundary:
|
|
1014
|
+
line_end = segment.text.find("\n")
|
|
1015
|
+
if line_end >= 0:
|
|
1016
|
+
cut = segment.start + line_end
|
|
1017
|
+
return (j if cut > segment.start else last), cut
|
|
1018
|
+
last = j
|
|
1019
|
+
j += 1
|
|
1020
|
+
return last, None
|
|
1021
|
+
|
|
1022
|
+
|
|
1023
|
+
def _truncate(word: _Word, end: int) -> None:
|
|
1024
|
+
"""Cut a joined word at the raw index `end` (a stop inside its last segment)."""
|
|
1025
|
+
count = bisect.bisect_right(word.rend, end)
|
|
1026
|
+
del word.chars[count:], word.rstart[count:], word.rend[count:]
|
|
1027
|
+
word.frags = [[f[0], f[1] if f[1] < end else -1] for f in word.frags if f[0] < end]
|
|
1028
|
+
word.end = end
|
|
1029
|
+
word.text = "".join(word.chars)
|
|
1030
|
+
|
|
1031
|
+
|
|
1032
|
+
_JSON_STRING_RE = re.compile(r'"(?:[^"\\]|\\.)*"', re.DOTALL)
|
|
1033
|
+
# Where a JSON document can plausibly start: an object opens with a key or closes at once, an
|
|
1034
|
+
# array with a value or closes at once. `[Errno 13]`, `{not json}` and a run of `{` never match,
|
|
1035
|
+
# so they cost no failed parse.
|
|
1036
|
+
_JSON_OPEN_RE = re.compile(r'\{\s*["}]|\[\s*["\-\d\[{tfn\]]')
|
|
1037
|
+
# A bracket span is a JSON document only as its own token (docs/DESIGN.md 5.1): after the start of
|
|
1038
|
+
# the text, whitespace, `:`, `,`, `[` or `{`, and before the end of the text, whitespace, `,`, `]`
|
|
1039
|
+
# or `}`. A span glued to a word character is part of a name.
|
|
1040
|
+
_JSON_BEFORE = frozenset(" \t\n\r\f\v:,[{")
|
|
1041
|
+
_JSON_AFTER = frozenset(" \t\n\r\f\v,]}")
|
|
1042
|
+
|
|
1043
|
+
|
|
1044
|
+
def _json_spans(text: str) -> list[tuple[int, int]]:
|
|
1045
|
+
"""The maximal `{...}` and `[...]` substrings of `text` that parse as JSON, left to right.
|
|
1046
|
+
A failed attempt is charged the distance to its error plus the cost of the error object
|
|
1047
|
+
(the decoder counts newlines up to the error position), and the total is capped at a small
|
|
1048
|
+
multiple of the text, so a hostile run of brackets cannot make the pass quadratic (a text
|
|
1049
|
+
past the cap keeps the fail-closed free-text path)."""
|
|
1050
|
+
spans: list[tuple[int, int]] = []
|
|
1051
|
+
decoder = json.JSONDecoder()
|
|
1052
|
+
budget = 4 * len(text) + 10_000
|
|
1053
|
+
position = 0
|
|
1054
|
+
while budget > 0:
|
|
1055
|
+
opener = _JSON_OPEN_RE.search(text, position)
|
|
1056
|
+
if opener is None:
|
|
1057
|
+
break
|
|
1058
|
+
start = opener.start()
|
|
1059
|
+
if start and text[start - 1] not in _JSON_BEFORE:
|
|
1060
|
+
position = start + 1 # glued to a word: part of a name (`Acme[2024]/Q1`)
|
|
1061
|
+
budget -= 1
|
|
1062
|
+
continue
|
|
1063
|
+
try:
|
|
1064
|
+
_value, end = decoder.raw_decode(text, start)
|
|
1065
|
+
except json.JSONDecodeError as exc:
|
|
1066
|
+
budget -= max(exc.pos - start, 0) + 1 + exc.pos // 16
|
|
1067
|
+
position = start + 1
|
|
1068
|
+
except RecursionError:
|
|
1069
|
+
budget -= 1000
|
|
1070
|
+
position = start + 1
|
|
1071
|
+
else:
|
|
1072
|
+
budget -= end - start
|
|
1073
|
+
if end < len(text) and text[end] not in _JSON_AFTER:
|
|
1074
|
+
position = start + 1 # glued to a word after the span
|
|
1075
|
+
continue
|
|
1076
|
+
spans.append((start, end))
|
|
1077
|
+
position = end
|
|
1078
|
+
return spans
|
|
1079
|
+
|
|
1080
|
+
|
|
1081
|
+
def _scan_json_span(span: str, scan: Callable[[str], str]) -> str:
|
|
1082
|
+
"""One JSON document, in place: every string literal (a key or a value) goes through `scan`
|
|
1083
|
+
and keeps its exact spelling when it comes back unchanged; everything else is kept."""
|
|
1084
|
+
def one(match: re.Match) -> str:
|
|
1085
|
+
literal = match.group(0)
|
|
1086
|
+
value = json.loads(literal)
|
|
1087
|
+
shown = scan(value)
|
|
1088
|
+
# ensure_ascii: every non-ASCII character and every lone surrogate is written as an ASCII
|
|
1089
|
+
# escape, so the result always encodes and prints
|
|
1090
|
+
return literal if shown == value else json.dumps(shown)
|
|
1091
|
+
|
|
1092
|
+
return _JSON_STRING_RE.sub(one, span)
|
|
1093
|
+
|
|
1094
|
+
|
|
1095
|
+
def _holds_start(chunk: str) -> bool:
|
|
1096
|
+
"""Whether `chunk` holds a path start by the word scanner's own predicate: a start in the raw
|
|
1097
|
+
text (`_abs_starts`) or, for a word that decodes an escape inside a double-quoted fragment, a
|
|
1098
|
+
start in its decoded text (`_decoded_starts`). The words of the chunk are lexed only when it
|
|
1099
|
+
holds a backslash, the one character a decoded escape needs."""
|
|
1100
|
+
if _abs_starts(chunk):
|
|
1101
|
+
return True
|
|
1102
|
+
if "\\" in chunk:
|
|
1103
|
+
return any(
|
|
1104
|
+
word.escaped and _decoded_starts(word) for word in _lex(chunk) if not word.boundary
|
|
1105
|
+
)
|
|
1106
|
+
return False
|
|
1107
|
+
|
|
1108
|
+
|
|
1109
|
+
def _json_spans_clear_of_paths(
|
|
1110
|
+
text: str, spans: list[tuple[int, int]], lookup: _PathLookup,
|
|
1111
|
+
) -> list[tuple[int, int]]:
|
|
1112
|
+
"""The JSON spans that no path on the same line precedes. A path hides the rest of its line
|
|
1113
|
+
(docs/DESIGN.md 5.1), so a bracket span after a path, on the same line, is part of what the
|
|
1114
|
+
path hides, not a document to read value by value: it is left to the word scanner.
|
|
1115
|
+
|
|
1116
|
+
Linear (F fix 7, W7-5): the text between the lower bound of a span (the end of the last kept
|
|
1117
|
+
span or the start of its line) and the span is read once. A span that follows a path on its
|
|
1118
|
+
line makes every later span of that line follow it too, so those are skipped without a new
|
|
1119
|
+
scan; a kept span moves the lower bound to its own end, so no text is read twice. "A path
|
|
1120
|
+
precedes" is the predicate of the word scanner (`_holds_start`: an absolute, Windows, UNC,
|
|
1121
|
+
namespace or `file://` start, a home or working-directory anchor, a start in the decoded text
|
|
1122
|
+
of an escaped word, a registered value), not the presence of a slash (F fix 8, W8-9)."""
|
|
1123
|
+
kept: list[tuple[int, int]] = []
|
|
1124
|
+
low = -1 # the lower bound the verdict below holds for
|
|
1125
|
+
dirty = False # a path precedes every span from `low` on
|
|
1126
|
+
line_start = 0
|
|
1127
|
+
previous = 0
|
|
1128
|
+
for start, end in spans:
|
|
1129
|
+
newline = text.rfind("\n", previous, start)
|
|
1130
|
+
if newline >= 0:
|
|
1131
|
+
line_start = newline + 1
|
|
1132
|
+
previous = start
|
|
1133
|
+
bound = max(kept[-1][1] if kept else 0, line_start)
|
|
1134
|
+
if bound != low:
|
|
1135
|
+
low, dirty = bound, False
|
|
1136
|
+
chunk = text[low:start]
|
|
1137
|
+
if _holds_start(chunk):
|
|
1138
|
+
dirty = True # a path start precedes the span
|
|
1139
|
+
elif lookup and lookup.hits([chunk]):
|
|
1140
|
+
dirty = True # a registered value precedes the span
|
|
1141
|
+
if dirty:
|
|
1142
|
+
continue
|
|
1143
|
+
kept.append((start, end))
|
|
1144
|
+
return kept
|
|
1145
|
+
|
|
1146
|
+
|
|
1147
|
+
def scan_paths(
|
|
1148
|
+
text: str, names: "Pseudonyms", render: Callable[[str, bool], str], *, absolute: bool = True,
|
|
1149
|
+
render_hidden: Callable[[str, bool], str] | None = None,
|
|
1150
|
+
) -> str:
|
|
1151
|
+
"""The one free-text path scanner (docs/DESIGN.md 5.1). Every word that holds a registered
|
|
1152
|
+
path value of `names` (decoded or raw spelling, NFC form), or, with `absolute`, an absolute
|
|
1153
|
+
path, is rewritten as its prefix, the path rendered once by `render(path text, whether a
|
|
1154
|
+
registered value was found)` and its raw suffix. A word with neither keeps its bytes. A span
|
|
1155
|
+
that holds whitespace is rendered by `render_hidden` (default `render`) and gets a marker.
|
|
1156
|
+
JSON (the whole text, or a maximal `{...}` or `[...]` substring that stands as its own token)
|
|
1157
|
+
is scanned string by string."""
|
|
1158
|
+
if not text:
|
|
1159
|
+
return text
|
|
1160
|
+
lookup = names.path_lookup()
|
|
1161
|
+
if not lookup and (
|
|
1162
|
+
not absolute or ("/" not in text and "~" not in text and "\\" not in text)
|
|
1163
|
+
):
|
|
1164
|
+
return text
|
|
1165
|
+
|
|
1166
|
+
def again(piece: str) -> str:
|
|
1167
|
+
return scan_paths(piece, names, render, absolute=absolute, render_hidden=render_hidden)
|
|
1168
|
+
|
|
1169
|
+
if "{" in text or "[" in text:
|
|
1170
|
+
spans = _json_spans_clear_of_paths(text, _json_spans(text), lookup)
|
|
1171
|
+
if spans:
|
|
1172
|
+
out: list[str] = []
|
|
1173
|
+
position = 0
|
|
1174
|
+
for start, end in spans:
|
|
1175
|
+
out.append(again(text[position:start]) if start > position else "")
|
|
1176
|
+
out.append(_scan_json_span(text[start:end], again))
|
|
1177
|
+
position = end
|
|
1178
|
+
out.append(again(text[position:]) if position < len(text) else "")
|
|
1179
|
+
shown = "".join(out)
|
|
1180
|
+
return text if shown == text else shown
|
|
1181
|
+
return _scan_words(text, names, render, lookup, absolute, render_hidden)
|
|
1182
|
+
|
|
1183
|
+
|
|
1184
|
+
def _scan_words(
|
|
1185
|
+
text: str, names: "Pseudonyms", render: Callable[[str, bool], str], lookup: _PathLookup,
|
|
1186
|
+
absolute: bool, render_hidden: Callable[[str, bool], str] | None,
|
|
1187
|
+
) -> str:
|
|
1188
|
+
segments = _lex(text, lookup)
|
|
1189
|
+
flags = [False] * len(segments)
|
|
1190
|
+
if lookup:
|
|
1191
|
+
parts = [segment.text for segment in segments]
|
|
1192
|
+
ranges = lookup.hits(parts)
|
|
1193
|
+
raw_parts = [text[segment.start:segment.end] for segment in segments]
|
|
1194
|
+
if raw_parts != parts:
|
|
1195
|
+
ranges += lookup.hits(raw_parts)
|
|
1196
|
+
spans: list[list[int]] = []
|
|
1197
|
+
for first, last in sorted(set(ranges)):
|
|
1198
|
+
while first <= last and segments[first].boundary:
|
|
1199
|
+
first += 1
|
|
1200
|
+
while last >= first and segments[last].boundary:
|
|
1201
|
+
last -= 1
|
|
1202
|
+
if first > last:
|
|
1203
|
+
continue
|
|
1204
|
+
if spans and first <= spans[-1][1]:
|
|
1205
|
+
spans[-1][1] = max(spans[-1][1], last)
|
|
1206
|
+
else:
|
|
1207
|
+
spans.append([first, last])
|
|
1208
|
+
if spans:
|
|
1209
|
+
rebuilt: list[_Word] = []
|
|
1210
|
+
flags = []
|
|
1211
|
+
k = 0
|
|
1212
|
+
pending = iter(spans)
|
|
1213
|
+
span = next(pending, None)
|
|
1214
|
+
while k < len(segments):
|
|
1215
|
+
if span is not None and k == span[0]:
|
|
1216
|
+
first, last = span
|
|
1217
|
+
rebuilt.append(
|
|
1218
|
+
segments[first] if first == last else _join(segments, text, first, last)
|
|
1219
|
+
)
|
|
1220
|
+
flags.append(True)
|
|
1221
|
+
k = last + 1
|
|
1222
|
+
span = next(pending, None)
|
|
1223
|
+
else:
|
|
1224
|
+
rebuilt.append(segments[k])
|
|
1225
|
+
flags.append(False)
|
|
1226
|
+
k += 1
|
|
1227
|
+
segments = rebuilt
|
|
1228
|
+
out: list[str] = []
|
|
1229
|
+
position = 0
|
|
1230
|
+
starts = _abs_starts(text) if absolute else []
|
|
1231
|
+
k = 0
|
|
1232
|
+
while k < len(segments):
|
|
1233
|
+
segment, flagged = segments[k], flags[k]
|
|
1234
|
+
k += 1
|
|
1235
|
+
if segment.boundary or not segment.text:
|
|
1236
|
+
continue
|
|
1237
|
+
if flagged or _abs_start_in(segment, starts) is not None:
|
|
1238
|
+
end, cut = _line_span(segments, k - 1, starts)
|
|
1239
|
+
if end > k - 1 or cut is not None:
|
|
1240
|
+
segment = _join(segments, text, k - 1, end)
|
|
1241
|
+
if cut is not None and cut < segment.end:
|
|
1242
|
+
_truncate(segment, cut)
|
|
1243
|
+
k = end + 1
|
|
1244
|
+
rewritten = _rewrite_word(
|
|
1245
|
+
segment, text, lookup, flagged, names, render, starts, render_hidden,
|
|
1246
|
+
)
|
|
1247
|
+
if rewritten is None:
|
|
1248
|
+
continue
|
|
1249
|
+
out.append(text[position:segment.start])
|
|
1250
|
+
out.append(rewritten)
|
|
1251
|
+
position = segment.end
|
|
1252
|
+
if not out:
|
|
1253
|
+
return text
|
|
1254
|
+
out.append(text[position:])
|
|
1255
|
+
return "".join(out)
|
|
1256
|
+
|
|
1257
|
+
|
|
1258
|
+
# The account name shows up on its own in ls owner columns, ps rows and
|
|
1259
|
+
# pytest temp dirs. Shorter names would collide with ordinary words.
|
|
1260
|
+
_MIN_BARE_USER_LEN = 3
|
|
1261
|
+
|
|
1262
|
+
# docs/DESIGN.md 5.1: agent types this tool itself defines. They identify a
|
|
1263
|
+
# role, not something a user typed, so Pseudonyms.type() leaves them alone.
|
|
1264
|
+
BUILT_IN_AGENT_TYPES = frozenset({
|
|
1265
|
+
"general-purpose", "Explore", "Plan", "code-writer", "code-reviewer",
|
|
1266
|
+
"test-writer", "test-runner", "debugger", "doc-scribe",
|
|
1267
|
+
})
|
|
1268
|
+
|
|
1269
|
+
|
|
1270
|
+
def _redact_uuid(match: re.Match) -> str:
|
|
1271
|
+
value = match.group(0)
|
|
1272
|
+
return f"uuid:{value[-4:]}"
|
|
1273
|
+
|
|
1274
|
+
|
|
1275
|
+
def _redact_email(match: re.Match) -> str:
|
|
1276
|
+
digest = hashlib.sha1(match.group(0).encode("utf-8")).hexdigest()[:6]
|
|
1277
|
+
return f"email:{digest}"
|
|
1278
|
+
|
|
1279
|
+
|
|
1280
|
+
def _redact_project_slug(match: re.Match) -> str:
|
|
1281
|
+
value = match.group(0)
|
|
1282
|
+
digest = hashlib.sha1(value.encode("utf-8")).hexdigest()[:6]
|
|
1283
|
+
return f"project:{digest}"
|
|
1284
|
+
|
|
1285
|
+
|
|
1286
|
+
def _collapse_home_path(match: re.Match) -> str:
|
|
1287
|
+
return f"~/.../{match.group(1)}"
|
|
1288
|
+
|
|
1289
|
+
|
|
1290
|
+
def _collapse_tmp_path(match: re.Match) -> str:
|
|
1291
|
+
root, first_segment, basename = match.group(1), match.group(2), match.group(3)
|
|
1292
|
+
return f"{root}/{first_segment}/.../{basename}"
|
|
1293
|
+
|
|
1294
|
+
|
|
1295
|
+
def _home_pattern(home_str: str) -> re.Pattern:
|
|
1296
|
+
"""Regex for the home directory, including truncated spellings of it.
|
|
1297
|
+
|
|
1298
|
+
Transcripts quote commands cut at a byte limit, so a home of
|
|
1299
|
+
/Users/alice can appear as "/Users/ali". Every non-empty prefix of the
|
|
1300
|
+
user segment matches, but only at a boundary: end of text, a path
|
|
1301
|
+
separator, or a character that cannot continue a user name. That keeps
|
|
1302
|
+
/Users/alicedoe (a different account) and /Users/bob untouched.
|
|
1303
|
+
"""
|
|
1304
|
+
home = Path(home_str)
|
|
1305
|
+
user = home.name
|
|
1306
|
+
parent = str(home.parent)
|
|
1307
|
+
if not user or parent == home_str:
|
|
1308
|
+
return re.compile(re.escape(home_str))
|
|
1309
|
+
prefixes = [re.escape(user[:i]) for i in range(len(user), 0, -1)]
|
|
1310
|
+
return re.compile(
|
|
1311
|
+
re.escape(parent) + "/(?:" + "|".join(prefixes) + r")(?=$|/|[^A-Za-z0-9._-])"
|
|
1312
|
+
)
|
|
1313
|
+
|
|
1314
|
+
|
|
1315
|
+
def redact(text: str, home: Path | None = None) -> str:
|
|
1316
|
+
"""Replace the home directory, UUIDs, and project slugs with placeholders.
|
|
1317
|
+
|
|
1318
|
+
- The home directory (default Path.home()) becomes "~".
|
|
1319
|
+
- Any UUID (8-4-4-4-12 hex) becomes "uuid:" plus its last 4 hex chars.
|
|
1320
|
+
- Any "-Users-<name>-..." project slug becomes "project:" plus a 6-char
|
|
1321
|
+
hash of the slug.
|
|
1322
|
+
- The bare account name (3+ chars, whole word) becomes "user".
|
|
1323
|
+
- Any email address becomes "email:" plus a 6-char hash.
|
|
1324
|
+
- A home-relative path beyond "~/" and a /tmp or /private/tmp path
|
|
1325
|
+
beyond its first segment collapse to ".../<basename>" (docs/
|
|
1326
|
+
DESIGN.md 5.1's D-A): only the file being pointed at survives, never
|
|
1327
|
+
the directory names in between.
|
|
1328
|
+
Nothing else changes. Agent ids are left alone on purpose.
|
|
1329
|
+
"""
|
|
1330
|
+
if not text:
|
|
1331
|
+
return text
|
|
1332
|
+
result = text
|
|
1333
|
+
home_str = str(home) if home is not None else str(Path.home())
|
|
1334
|
+
# Order matters: slugs, emails and UUIDs are hashed whole first, so the
|
|
1335
|
+
# bare-name rule below cannot rewrite a token inside them (a home segment
|
|
1336
|
+
# such as "home" would otherwise break "-home-alice-project").
|
|
1337
|
+
result = _PROJECT_SLUG_RE.sub(_redact_project_slug, result)
|
|
1338
|
+
result = _EMAIL_RE.sub(_redact_email, result)
|
|
1339
|
+
result = _UUID_RE.sub(_redact_uuid, result)
|
|
1340
|
+
if home_str and home_str != "/":
|
|
1341
|
+
result = _home_pattern(home_str).sub("~", result)
|
|
1342
|
+
user = Path(home_str).name
|
|
1343
|
+
if len(user) >= _MIN_BARE_USER_LEN:
|
|
1344
|
+
result = re.sub(
|
|
1345
|
+
r"(?<![A-Za-z0-9_])" + re.escape(user) + r"(?![A-Za-z0-9_])", "user", result
|
|
1346
|
+
)
|
|
1347
|
+
# Collapse the directories of a home-relative or /tmp path (D-A): run
|
|
1348
|
+
# after the home substitution above, so a path that started out as an
|
|
1349
|
+
# absolute home directory is covered too, not only one already spelled
|
|
1350
|
+
# with "~/" in the source text.
|
|
1351
|
+
result = _HOME_PATH_COLLAPSE_RE.sub(_collapse_home_path, result)
|
|
1352
|
+
result = _TMP_PATH_COLLAPSE_RE.sub(_collapse_tmp_path, result)
|
|
1353
|
+
return result
|
|
1354
|
+
|
|
1355
|
+
|
|
1356
|
+
class RenderedCommit(str):
|
|
1357
|
+
"""A commit id already rendered through the commit mapping. `report.redact_value` passes it
|
|
1358
|
+
through unchanged, so a JSON commit list is rendered exactly once (DESIGN 4.13.5, A1)."""
|
|
1359
|
+
|
|
1360
|
+
|
|
1361
|
+
class Pseudonyms:
|
|
1362
|
+
"""Per-render mapping from a user-authored name to a stable, ordinal
|
|
1363
|
+
pseudonym (docs/DESIGN.md 5.1).
|
|
1364
|
+
|
|
1365
|
+
An attempt label maps to agent-N, a meta description to desc-N, a phase
|
|
1366
|
+
to phase-N, a non-built-in agent_type to type-N, and a trend rule target
|
|
1367
|
+
to target-N, each in its own namespace and its own order of first
|
|
1368
|
+
appearance (M8 review D-D): one counter per field family, not shared
|
|
1369
|
+
across them, so a label and a differently-worded description for the
|
|
1370
|
+
same attempt never consume each other's ordinals and leave a gap in,
|
|
1371
|
+
say, an AGENT table that only ever prints labels. A description that is
|
|
1372
|
+
literally identical to its attempt's label is still naming the same
|
|
1373
|
+
thing, though, so description() special-cases that: it returns the
|
|
1374
|
+
label's own agent-N pseudonym rather than minting a new desc-N for text
|
|
1375
|
+
that is not actually a second name. Built-in agent types
|
|
1376
|
+
(BUILT_IN_AGENT_TYPES) are not user-authored and pass through unchanged.
|
|
1377
|
+
|
|
1378
|
+
A caller registers a name with agent()/description()/phase()/type()/
|
|
1379
|
+
target(), which also returns its pseudonym; apply() then substitutes
|
|
1380
|
+
every name registered so far wherever it occurs as a whole word inside a
|
|
1381
|
+
larger piece of text (docs/DESIGN.md 5.1/5.2: Finding.title only --
|
|
1382
|
+
never evidence, a diagnostic, or a metrics value, which are a faithful
|
|
1383
|
+
quote of transcript text, not a rendered sentence built from a label),
|
|
1384
|
+
longest name first so one name that is a substring of another is never
|
|
1385
|
+
partially clobbered, and a word-boundary anchor
|
|
1386
|
+
((?<![A-Za-z0-9_])name(?![A-Za-z0-9_])) so a short registered name (for
|
|
1387
|
+
example an agent_type "cli") never matches as a fragment of an unrelated
|
|
1388
|
+
word ("client").
|
|
1389
|
+
|
|
1390
|
+
One instance belongs to one render (docs/DESIGN.md 5.1: "rebuilt for the
|
|
1391
|
+
next [report]"), so two reports never share a mapping and cannot be
|
|
1392
|
+
linked by a repeated pseudonym. enabled=False (--no-redact) makes every
|
|
1393
|
+
method a pass-through.
|
|
1394
|
+
"""
|
|
1395
|
+
|
|
1396
|
+
# docs/DESIGN.md 5.1: "path" is a different kind of category from the
|
|
1397
|
+
# other five (agent/desc/phase/type/target). Those five name something a
|
|
1398
|
+
# detector wrote a sentence about (a label, a phase) and are confined to
|
|
1399
|
+
# Finding.title (5.1/5.2: never evidence, a diagnostic, or a metrics
|
|
1400
|
+
# value, which are meant to stay a faithful quote). A report_path is
|
|
1401
|
+
# itself a metrics/reported value, not a title ingredient, and the whole
|
|
1402
|
+
# point of pseudonymizing it (M1 review finding 1) is that it must never
|
|
1403
|
+
# appear raw in ANY of those places by default -- so it is substituted by
|
|
1404
|
+
# the plain Redactor call too (see Redactor.__call__ and apply_paths
|
|
1405
|
+
# below), not only by title(). _NAME_CATEGORIES keeps the original five
|
|
1406
|
+
# separate from "path" so apply() (title-only) never touches a path and
|
|
1407
|
+
# apply_paths() (used everywhere) never touches a name.
|
|
1408
|
+
_NAME_CATEGORIES = ("agent", "desc", "phase", "type", "target")
|
|
1409
|
+
# docs/DESIGN.md 4.11/5.1 (F7 hardening): "check" is a third kind of
|
|
1410
|
+
# category, alongside the five names and "path" -- a declared check
|
|
1411
|
+
# command is neither a label a detector wrote a sentence about, nor a
|
|
1412
|
+
# path this tool ever needs to substring-match inside other prose. It is
|
|
1413
|
+
# only ever substituted at the single point rescue.py renders it (a
|
|
1414
|
+
# direct `names.check(value)` call, the same pattern `path` already uses
|
|
1415
|
+
# for a report_path claim or an observation path), so it needs no
|
|
1416
|
+
# apply()/apply_paths()-style substring pass of its own.
|
|
1417
|
+
# docs/DESIGN.md 5.3 (G-B1, GB-8): "key" is a result key name in `reported.keys`: a
|
|
1418
|
+
# user-authored identifier like a label, pseudonymized at share only (key-N), substituted at
|
|
1419
|
+
# the single point the export builds that list, so it needs no substring pass.
|
|
1420
|
+
_CATEGORIES = _NAME_CATEGORIES + ("path", "check", "key", "commit")
|
|
1421
|
+
|
|
1422
|
+
def __init__(
|
|
1423
|
+
self,
|
|
1424
|
+
enabled: bool = True,
|
|
1425
|
+
*,
|
|
1426
|
+
real_names: bool = False,
|
|
1427
|
+
resolver: Callable[[str, str | None, Callable[[str], str]], str] | None = None,
|
|
1428
|
+
) -> None:
|
|
1429
|
+
self.enabled = enabled
|
|
1430
|
+
# N3b P10 (local): real_names makes the five name categories pass
|
|
1431
|
+
# through (a label, description, phase, agent type, and rule target
|
|
1432
|
+
# print as written), and `resolver` (the P19 path function) decides
|
|
1433
|
+
# what a registered path prints as, instead of path-N. Only the run
|
|
1434
|
+
# sinks turn either on; every other sink keeps share's pseudonyms
|
|
1435
|
+
# until its own part lands (D-l).
|
|
1436
|
+
self.real_names = real_names
|
|
1437
|
+
self._resolver = resolver
|
|
1438
|
+
self._outside_paths: dict[str, str] = {}
|
|
1439
|
+
self._maps: dict[str, dict[str, str]] = {category: {} for category in self._CATEGORIES}
|
|
1440
|
+
self._lookup: _PathLookup | None = None
|
|
1441
|
+
|
|
1442
|
+
def _outside_path(self, value: str) -> str:
|
|
1443
|
+
if value not in self._outside_paths:
|
|
1444
|
+
self._outside_paths[value] = f"path-{len(self._outside_paths) + 1}"
|
|
1445
|
+
return self._outside_paths[value]
|
|
1446
|
+
|
|
1447
|
+
def _pseudonym(self, category: str, value: str | None) -> str | None:
|
|
1448
|
+
if not self.enabled or not value:
|
|
1449
|
+
return value
|
|
1450
|
+
if self.real_names and category in self._NAME_CATEGORIES:
|
|
1451
|
+
return self.local_text(value)
|
|
1452
|
+
if self.real_names and category == "commit":
|
|
1453
|
+
# DESIGN 4.13.5 (M9, S1): a commit id prints as written but scanned like a name
|
|
1454
|
+
return self.local_text(value)
|
|
1455
|
+
if self.real_names and category in ("check", "key"):
|
|
1456
|
+
return value
|
|
1457
|
+
mapping = self._maps[category]
|
|
1458
|
+
if value not in mapping:
|
|
1459
|
+
mapping[value] = f"{category}-{len(mapping) + 1}"
|
|
1460
|
+
return mapping[value]
|
|
1461
|
+
|
|
1462
|
+
def local_text(self, value: str) -> str:
|
|
1463
|
+
"""N3b S9, C3: a real name at local. Control characters become a
|
|
1464
|
+
space, an absolute path goes through the one P19 path function, and a
|
|
1465
|
+
home-relative path collapses to ~/.../<basename>: a name is real but
|
|
1466
|
+
never an absolute path and never a terminal escape."""
|
|
1467
|
+
text = scan_paths(
|
|
1468
|
+
value, self, lambda found, _hit: self.path(found) or "",
|
|
1469
|
+
render_hidden=lambda found, _hit: self.opaque_path(found) or "",
|
|
1470
|
+
)
|
|
1471
|
+
text = _CONTROL_RE.sub(" ", text)
|
|
1472
|
+
return _HOME_PATH_COLLAPSE_RE.sub(_collapse_home_path, text)
|
|
1473
|
+
|
|
1474
|
+
def agent(self, value: str | None) -> str | None:
|
|
1475
|
+
"""An attempt label."""
|
|
1476
|
+
return self._pseudonym("agent", value)
|
|
1477
|
+
|
|
1478
|
+
def description(self, value: str | None, *, label: str | None = None) -> str | None:
|
|
1479
|
+
"""A meta description: its own "desc" namespace, except when it is
|
|
1480
|
+
literally identical to the attempt's own label (still naming the
|
|
1481
|
+
same agent, not a second name to count), in which case it reuses
|
|
1482
|
+
the label's already-registered agent-N pseudonym (D-D)."""
|
|
1483
|
+
if not self.enabled or not value:
|
|
1484
|
+
return value
|
|
1485
|
+
if label is not None and value == label:
|
|
1486
|
+
return self.agent(label)
|
|
1487
|
+
return self._pseudonym("desc", value)
|
|
1488
|
+
|
|
1489
|
+
def phase(self, value: str | None) -> str | None:
|
|
1490
|
+
return self._pseudonym("phase", value)
|
|
1491
|
+
|
|
1492
|
+
def type(self, value: str | None) -> str | None:
|
|
1493
|
+
if not self.enabled or not value or value in BUILT_IN_AGENT_TYPES:
|
|
1494
|
+
return value
|
|
1495
|
+
return self._pseudonym("type", value)
|
|
1496
|
+
|
|
1497
|
+
def target(self, value: str | None) -> str | None:
|
|
1498
|
+
"""A trend rule target (docs/DESIGN.md 4.7/5.1): the file a rule is
|
|
1499
|
+
proposed for, for example an instruction-file path that can embed a
|
|
1500
|
+
client or project name. Rendered by the caller as 'target-N
|
|
1501
|
+
(basename)', never the raw path, unless --no-redact."""
|
|
1502
|
+
return self._pseudonym("target", value)
|
|
1503
|
+
|
|
1504
|
+
def path(
|
|
1505
|
+
self, value: str | None, base: str | None = None, *, resolve_from: str | None = None,
|
|
1506
|
+
) -> str | None:
|
|
1507
|
+
"""A report_path claim value (docs/DESIGN.md 5.1, M1 review finding
|
|
1508
|
+
1): its own "path" namespace, numbered in order of first appearance
|
|
1509
|
+
like every other category. Unlike agent/desc/phase/type/target, a
|
|
1510
|
+
registered path is substituted everywhere the plain Redactor renders
|
|
1511
|
+
text (see apply_paths, used by Redactor.__call__), not only inside
|
|
1512
|
+
Finding.title -- a relative path or an absolute path outside home
|
|
1513
|
+
and /tmp must never survive structural redact() unchanged the way it
|
|
1514
|
+
used to.
|
|
1515
|
+
|
|
1516
|
+
At local (N3b P19) a resolver is attached: the display is the path
|
|
1517
|
+
function's answer (repo-relative, ~/.../name, or path-N), made
|
|
1518
|
+
absolute against `base` when the value is relative. `resolve_from`
|
|
1519
|
+
names the original spelling the path function should place when
|
|
1520
|
+
`value` is a display form of it (rescue's repo-relative key): the
|
|
1521
|
+
mapping is still keyed by `value`, the text that will appear."""
|
|
1522
|
+
if self._resolver is not None and self.enabled and value:
|
|
1523
|
+
mapping = self._maps["path"]
|
|
1524
|
+
if value not in mapping:
|
|
1525
|
+
source = resolve_from if resolve_from else value
|
|
1526
|
+
mapping[value] = self._resolver(source, base, self._outside_path)
|
|
1527
|
+
return mapping[value]
|
|
1528
|
+
return self._pseudonym("path", value)
|
|
1529
|
+
|
|
1530
|
+
def opaque_path(self, value: str | None) -> str | None:
|
|
1531
|
+
"""A span that holds whitespace (docs/DESIGN.md 5.1): the span may be more than a path,
|
|
1532
|
+
so it is always its own path-N, never a structural, repo-relative or home form. At local
|
|
1533
|
+
with the P19 resolver it is numbered with the paths P19 places outside the repository."""
|
|
1534
|
+
if not self.enabled or not value:
|
|
1535
|
+
return value
|
|
1536
|
+
if self._resolver is not None:
|
|
1537
|
+
return self._outside_path(value)
|
|
1538
|
+
return self._pseudonym("path", value)
|
|
1539
|
+
|
|
1540
|
+
def check(self, value: str | None) -> str | None:
|
|
1541
|
+
"""A declared check command (docs/DESIGN.md 4.11/5.1, F7 hardening):
|
|
1542
|
+
its own "check" namespace, numbered in order of first appearance.
|
|
1543
|
+
rescue.py calls this directly at every point it would otherwise
|
|
1544
|
+
print a declared check's own text (the CHECKS/STEPS lines of
|
|
1545
|
+
`rescue plan`, the 'last counted run' line of `rescue status`) so
|
|
1546
|
+
the full command -- which can carry arbitrary flags, arguments, or
|
|
1547
|
+
an embedded path -- is replaced wholesale by default, not merely
|
|
1548
|
+
redacted in place."""
|
|
1549
|
+
return self._pseudonym("check", value)
|
|
1550
|
+
|
|
1551
|
+
def key(self, value: str | None) -> str | None:
|
|
1552
|
+
"""A result key name of `reported.keys` (docs/DESIGN.md 5.3, G-B1): its own "key"
|
|
1553
|
+
namespace, `key-1`.. in order of first appearance within the render; the real name at
|
|
1554
|
+
local and raw."""
|
|
1555
|
+
return self._pseudonym("key", value)
|
|
1556
|
+
|
|
1557
|
+
def commit(self, value: str | None) -> str | None:
|
|
1558
|
+
"""A reported commit id (docs/DESIGN.md 4.13.5, 5.1): a short hash of a public
|
|
1559
|
+
repository can identify the repository and its author, so its own "commit"
|
|
1560
|
+
namespace, `commit-1`.. in order of first appearance within the render, the same
|
|
1561
|
+
id the same number; the real id at local and raw."""
|
|
1562
|
+
shown = self._pseudonym("commit", value)
|
|
1563
|
+
# DESIGN 4.13.5 (M9, A1): already rendered; a structured sink never scans it as a path
|
|
1564
|
+
return RenderedCommit(shown) if shown else shown
|
|
1565
|
+
|
|
1566
|
+
def apply(self, text: str) -> str:
|
|
1567
|
+
"""Replace every name (agent/desc/phase/type/target -- never path,
|
|
1568
|
+
see apply_paths) registered so far with its pseudonym wherever it
|
|
1569
|
+
occurs as a whole word inside `text`. A name never registered
|
|
1570
|
+
(nothing has called agent()/phase()/type()/target() with it yet) is
|
|
1571
|
+
left untouched: apply() only substitutes, it never registers."""
|
|
1572
|
+
if not self.enabled or not text:
|
|
1573
|
+
return text
|
|
1574
|
+
result = text
|
|
1575
|
+
pairs = [
|
|
1576
|
+
(original, pseudonym)
|
|
1577
|
+
for category in self._NAME_CATEGORIES
|
|
1578
|
+
for original, pseudonym in self._maps[category].items()
|
|
1579
|
+
]
|
|
1580
|
+
for original, pseudonym in sorted(pairs, key=lambda pair: len(pair[0]), reverse=True):
|
|
1581
|
+
if not original:
|
|
1582
|
+
continue
|
|
1583
|
+
pattern = re.compile(r"(?<![A-Za-z0-9_])" + re.escape(original) + r"(?![A-Za-z0-9_])")
|
|
1584
|
+
result = pattern.sub(pseudonym, result)
|
|
1585
|
+
return result
|
|
1586
|
+
|
|
1587
|
+
def registered_paths_in(self, token: str) -> list[str]:
|
|
1588
|
+
"""Every registered report_path value (the root `/` aside) that occurs inside
|
|
1589
|
+
`token`: the values a token renders as its own path-N must not leave behind."""
|
|
1590
|
+
return [
|
|
1591
|
+
v for v in self._maps["path"]
|
|
1592
|
+
if v and v != "/" and (v in token or (v.endswith("/") and v.rstrip("/") in token))
|
|
1593
|
+
]
|
|
1594
|
+
|
|
1595
|
+
def path_lookup(self) -> _PathLookup:
|
|
1596
|
+
"""The registered path values as the scanner's NFC lookup, in step with the namespace."""
|
|
1597
|
+
if self._lookup is None:
|
|
1598
|
+
self._lookup = _PathLookup()
|
|
1599
|
+
self._lookup.sync(self._maps["path"])
|
|
1600
|
+
return self._lookup
|
|
1601
|
+
|
|
1602
|
+
def registered_path(self, value: str) -> str | None:
|
|
1603
|
+
"""What the registered path `value` prints as (its pseudonym; at local the P19 display),
|
|
1604
|
+
or None when it was never registered."""
|
|
1605
|
+
return self._maps["path"].get(value)
|
|
1606
|
+
|
|
1607
|
+
def registered_display(self, value: str) -> str:
|
|
1608
|
+
"""What a registered path value prints as (its pseudonym; at local the P19 display)."""
|
|
1609
|
+
return self._maps["path"][value]
|
|
1610
|
+
|
|
1611
|
+
def alias_path(self, value: str, like: str) -> None:
|
|
1612
|
+
"""Register `value` (another spelling of the path `like`) so it prints exactly as `like`
|
|
1613
|
+
does: rescue lists a path by its repo-relative key and the transcript spells it absolute."""
|
|
1614
|
+
if not self.enabled or not value or not like:
|
|
1615
|
+
return
|
|
1616
|
+
mapping = self._maps["path"]
|
|
1617
|
+
if value not in mapping:
|
|
1618
|
+
mapping[value] = self.path(like) or ""
|
|
1619
|
+
|
|
1620
|
+
def apply_paths(self, text: str, render: Callable[[str], str] | None = None) -> str:
|
|
1621
|
+
"""Replace every report_path value registered so far (the "path"
|
|
1622
|
+
namespace) wherever it occurs in `text`, by shell WORD, not by substring
|
|
1623
|
+
(docs/DESIGN.md 5.1, FA1 and F fix 4). Each word that holds a registered value
|
|
1624
|
+
(decoded or raw spelling, NFC form; a value that holds a space or an operator is
|
|
1625
|
+
found across words) is rendered whole from the path start to its end: a word that
|
|
1626
|
+
is exactly a registered value becomes its pseudonym; a longer one is rendered whole
|
|
1627
|
+
by `render` (the single-path renderer of the active level; default `self.path`, the
|
|
1628
|
+
word's own path-N), so neither the registered value nor its continuation survives
|
|
1629
|
+
and no pseudonym is glued to a raw tail. One pass over the original text, so a
|
|
1630
|
+
pseudonym is never matched again by a shorter registered value. Registered values
|
|
1631
|
+
only: an unregistered absolute path is `scan_paths`'s `absolute` pass, which
|
|
1632
|
+
Redactor.__call__ also runs."""
|
|
1633
|
+
if not self.enabled or not text:
|
|
1634
|
+
return text
|
|
1635
|
+
if render is None:
|
|
1636
|
+
def show(token: str, _hit: bool) -> str:
|
|
1637
|
+
return self.path(token) or ""
|
|
1638
|
+
else:
|
|
1639
|
+
def show(token: str, _hit: bool) -> str:
|
|
1640
|
+
return render(token)
|
|
1641
|
+
|
|
1642
|
+
def show_hidden(token: str, _hit: bool) -> str:
|
|
1643
|
+
return self.opaque_path(token) or ""
|
|
1644
|
+
|
|
1645
|
+
return scan_paths(text, self, show, absolute=False, render_hidden=show_hidden)
|
|
1646
|
+
|
|
1647
|
+
|
|
1648
|
+
class Redactor:
|
|
1649
|
+
"""Callable wrapper around redact() with a no-op mode for --no-redact.
|
|
1650
|
+
|
|
1651
|
+
Passing an instance around (instead of a bare function) lets callers ask
|
|
1652
|
+
once "am I redacting?" and lets report.py treat --redact/--no-redact as
|
|
1653
|
+
one flag instead of branching at every call site. Every Redactor carries
|
|
1654
|
+
its own Pseudonyms instance (`names`).
|
|
1655
|
+
|
|
1656
|
+
Calling the Redactor itself (__call__) runs structural redact() only --
|
|
1657
|
+
home directory, UUIDs, project slugs, emails, the bare account name.
|
|
1658
|
+
This is what evidence lines, diagnostics, and metrics values go through
|
|
1659
|
+
(docs/DESIGN.md 4.5/5.2: evidence is a faithful quote of transcript
|
|
1660
|
+
text, and a pseudonym substitution has no business rewriting prose that
|
|
1661
|
+
only happens to share a word with a registered name). `title()` is the
|
|
1662
|
+
one place pseudonym substitution also applies, on top of structural
|
|
1663
|
+
redact(): docs/DESIGN.md 5.1/5.2 scope pseudonyms to Finding.title and
|
|
1664
|
+
to the structured name fields (label, phase, meta description,
|
|
1665
|
+
agent_type) a renderer registers directly through `names`.
|
|
1666
|
+
"""
|
|
1667
|
+
|
|
1668
|
+
def __init__(
|
|
1669
|
+
self,
|
|
1670
|
+
enabled: bool = True,
|
|
1671
|
+
home: Path | None = None,
|
|
1672
|
+
*,
|
|
1673
|
+
context: DisclosureContext | None = None,
|
|
1674
|
+
) -> None:
|
|
1675
|
+
# N3b D-a: a command builds one DisclosureContext from its parsed
|
|
1676
|
+
# level and passes it here; enabled is then level != "raw" and the
|
|
1677
|
+
# context's home is the one redact() uses. Redactor(enabled=...)
|
|
1678
|
+
# keeps working for every existing caller: it builds the matching
|
|
1679
|
+
# context itself (share when enabled, raw when not, no book).
|
|
1680
|
+
if context is None:
|
|
1681
|
+
context = DisclosureContext(
|
|
1682
|
+
level="share" if enabled else "raw",
|
|
1683
|
+
home=home if home is not None else Path.home(),
|
|
1684
|
+
)
|
|
1685
|
+
self.home = home
|
|
1686
|
+
else:
|
|
1687
|
+
self.home = context.home
|
|
1688
|
+
self.context = context
|
|
1689
|
+
self.enabled = context.enabled
|
|
1690
|
+
self.names = Pseudonyms(enabled=self.enabled)
|
|
1691
|
+
self._scrub_paths = False
|
|
1692
|
+
self._leftover_names: Pseudonyms | None = None
|
|
1693
|
+
self._fallback: disclosure.AliasBook | None = None
|
|
1694
|
+
self._identity_patterns: list[tuple[str, re.Pattern, dict[str, str]]] | None = None
|
|
1695
|
+
|
|
1696
|
+
@property
|
|
1697
|
+
def level(self) -> str:
|
|
1698
|
+
return self.context.level
|
|
1699
|
+
|
|
1700
|
+
def _local_resolver(
|
|
1701
|
+
self, value: str, base: str | None, outside: Callable[[str], str]
|
|
1702
|
+
) -> str:
|
|
1703
|
+
return disclosure.local_path(
|
|
1704
|
+
value, repo_root=self.context.repo_root, home=self.home, base=base,
|
|
1705
|
+
outside=outside,
|
|
1706
|
+
)
|
|
1707
|
+
|
|
1708
|
+
def fresh_names(self, *, detailed: bool = False) -> Pseudonyms:
|
|
1709
|
+
"""A new per-render Pseudonyms for this redactor. `detailed` is the
|
|
1710
|
+
run sinks' request for the local level's real names and P19 paths
|
|
1711
|
+
(N3b); at share and raw it changes nothing, and any other sink never
|
|
1712
|
+
sets it."""
|
|
1713
|
+
local = detailed and self.context.level == "local"
|
|
1714
|
+
self._scrub_paths = local
|
|
1715
|
+
self.names = Pseudonyms(
|
|
1716
|
+
enabled=self.enabled, real_names=local,
|
|
1717
|
+
resolver=self._local_resolver if local else None,
|
|
1718
|
+
)
|
|
1719
|
+
return self.names
|
|
1720
|
+
|
|
1721
|
+
def _render_path_text(self, token: str, hit: bool, *, leftover: bool = False) -> str:
|
|
1722
|
+
"""FA1, the single-path renderer for the path text of a word (docs/DESIGN.md 5.1) at the
|
|
1723
|
+
active level. `hit`: the word held a registered value (its path-N is in the run's own
|
|
1724
|
+
namespace); otherwise the path is unregistered. local with the P19 resolver (the run
|
|
1725
|
+
sinks) or a hit: the P19 path function through the path namespace (repo-relative,
|
|
1726
|
+
`~/.../name`, else path-N). share, and local without a resolver: a token inside the home
|
|
1727
|
+
directory or under /tmp or /private/tmp gets the structural form (identities first, then
|
|
1728
|
+
home `~` and the `/tmp` collapse); every other token, relative ones included, becomes its
|
|
1729
|
+
own path-N. `leftover` (scrub_leftover_paths): an unregistered path is numbered in the
|
|
1730
|
+
leftover namespace, which `display_path` shares, and a /tmp path is path-N."""
|
|
1731
|
+
names = self._leftover() if leftover and not hit else self.names
|
|
1732
|
+
if self.level == "local" and (hit or names._resolver is not None):
|
|
1733
|
+
return names.path(token) or ""
|
|
1734
|
+
token = _collapse_slash_runs(token)
|
|
1735
|
+
home = str(self.context.home).rstrip("/")
|
|
1736
|
+
inside_home = token.startswith("~/") or (
|
|
1737
|
+
bool(home) and (token == home or token.startswith(home + "/"))
|
|
1738
|
+
)
|
|
1739
|
+
under_tmp = (hit or not leftover) and (
|
|
1740
|
+
token in ("/tmp", "/private/tmp") or token.startswith(("/tmp/", "/private/tmp/"))
|
|
1741
|
+
)
|
|
1742
|
+
if inside_home or under_tmp:
|
|
1743
|
+
shown = redact(self._substitute_identities(token), home=self.home)
|
|
1744
|
+
if _STRUCTURAL_SHAPE_RE.fullmatch(shown) and not self._keeps_registered(
|
|
1745
|
+
token, shown, home,
|
|
1746
|
+
):
|
|
1747
|
+
return shown
|
|
1748
|
+
return names.path(token) or ""
|
|
1749
|
+
|
|
1750
|
+
def _render_hidden_text(self, token: str, hit: bool, *, leftover: bool = False) -> str:
|
|
1751
|
+
"""A span that holds whitespace (docs/DESIGN.md 5.1): always its own opaque path-N, in the
|
|
1752
|
+
namespace `_render_path_text` would use, never a structural or repo-relative form."""
|
|
1753
|
+
names = self._leftover() if leftover and not hit else self.names
|
|
1754
|
+
return names.opaque_path(token) or ""
|
|
1755
|
+
|
|
1756
|
+
def _keeps_registered(self, token: str, shown: str, home: str) -> bool:
|
|
1757
|
+
"""S1 (docs/DESIGN.md 5.1): whether the structural form `shown` of `token` still
|
|
1758
|
+
holds, verbatim, a registered value the token encloses, or the part a collapse
|
|
1759
|
+
keeps of one (its last two segments; its `~`-relative spelling inside home)."""
|
|
1760
|
+
for value in self.names.registered_paths_in(token):
|
|
1761
|
+
segments = [part for part in value.split("/") if part]
|
|
1762
|
+
needles = {value, "/".join(segments[-2:])}
|
|
1763
|
+
if home and value.startswith(home + "/"):
|
|
1764
|
+
needles.add("~" + value[len(home):])
|
|
1765
|
+
if any(needle and needle in shown for needle in needles):
|
|
1766
|
+
return True
|
|
1767
|
+
return False
|
|
1768
|
+
|
|
1769
|
+
def _scrub_absolute_paths(self, text: str) -> str:
|
|
1770
|
+
"""P19 at local: every absolute path in free text goes through the
|
|
1771
|
+
one path function, so none is printed as is. Registered in the same
|
|
1772
|
+
path namespace as report paths, so one path has one display."""
|
|
1773
|
+
if not self._scrub_paths:
|
|
1774
|
+
return text
|
|
1775
|
+
return scan_paths(
|
|
1776
|
+
text, self.names, self._render_path_text, render_hidden=self._render_hidden_text,
|
|
1777
|
+
)
|
|
1778
|
+
|
|
1779
|
+
def scrub_local_line(self, text: str) -> str:
|
|
1780
|
+
"""C3: a fully assembled local line (an AGENT table row, a
|
|
1781
|
+
reproduction line) through the P19 scrub, whatever field carried a
|
|
1782
|
+
path. Only the run sinks that asked for local detail scrub; share and
|
|
1783
|
+
raw lines pass through unchanged."""
|
|
1784
|
+
return self._scrub_absolute_paths(text)
|
|
1785
|
+
|
|
1786
|
+
def scrub_leftover_paths(self, text: str) -> str:
|
|
1787
|
+
"""__call__, with an unregistered absolute path numbered in the leftover namespace (the
|
|
1788
|
+
one `display_path` uses): a location outside home and /tmp (a config or state file, an
|
|
1789
|
+
installed package) becomes path-N at share and the P19 answer at local (outside home
|
|
1790
|
+
and the repository: path-N). A path inside home keeps its `~/...` form. raw is
|
|
1791
|
+
unchanged. For a sink whose text is built from a path the command did not read out of a
|
|
1792
|
+
run (config warnings, a state-file error, a hook template location, a debug
|
|
1793
|
+
traceback)."""
|
|
1794
|
+
if not self.enabled or not text:
|
|
1795
|
+
return text
|
|
1796
|
+
text = scan_paths(
|
|
1797
|
+
text, self.names,
|
|
1798
|
+
lambda token, hit: self._render_path_text(token, hit, leftover=True),
|
|
1799
|
+
render_hidden=lambda token, hit: self._render_hidden_text(token, hit, leftover=True),
|
|
1800
|
+
)
|
|
1801
|
+
text = self._substitute_identities(text)
|
|
1802
|
+
return redact(text, home=self.home)
|
|
1803
|
+
|
|
1804
|
+
def _leftover(self) -> Pseudonyms:
|
|
1805
|
+
if self._leftover_names is None:
|
|
1806
|
+
local = self.context.level == "local"
|
|
1807
|
+
self._leftover_names = Pseudonyms(
|
|
1808
|
+
enabled=True, real_names=local,
|
|
1809
|
+
resolver=self._local_resolver if local else None,
|
|
1810
|
+
)
|
|
1811
|
+
return self._leftover_names
|
|
1812
|
+
|
|
1813
|
+
def display_path(self, value: object) -> str:
|
|
1814
|
+
"""N3b C4: a path the tool itself knows and is about to interpolate
|
|
1815
|
+
into a message (a config file, a state file, a template). It is
|
|
1816
|
+
redacted as one value, never by regex on the composed message, so a
|
|
1817
|
+
space in it leaks no tail. raw: as is. share: the structural home
|
|
1818
|
+
form (`~/.../name`) inside home, else path-N. local: the P19 answer."""
|
|
1819
|
+
text = os.fspath(value) if isinstance(value, os.PathLike) else str(value)
|
|
1820
|
+
if self.level == "raw" or not text:
|
|
1821
|
+
return text
|
|
1822
|
+
if self.level == "share":
|
|
1823
|
+
home = str(self.context.home).rstrip("/")
|
|
1824
|
+
if home and (text == home or text.startswith(home + "/")):
|
|
1825
|
+
parts = text[len(home):].strip("/").split("/")
|
|
1826
|
+
if parts == [""]:
|
|
1827
|
+
return "~"
|
|
1828
|
+
return f"~/{parts[0]}" if len(parts) == 1 else f"~/.../{parts[-1]}"
|
|
1829
|
+
return self._leftover().path(text) or ""
|
|
1830
|
+
|
|
1831
|
+
def exception_text(self, exc: BaseException) -> str:
|
|
1832
|
+
"""str(exc) with the paths the exception itself names (an OSError's
|
|
1833
|
+
filenames) replaced by display_path before anything else, then the
|
|
1834
|
+
leftover-path scrub for what remains."""
|
|
1835
|
+
text = str(exc)
|
|
1836
|
+
if self.level == "raw":
|
|
1837
|
+
return text
|
|
1838
|
+
if isinstance(exc, OSError):
|
|
1839
|
+
# each filename is rendered once, as one value, and the message around it is scanned
|
|
1840
|
+
# piece by piece: the finished text is not scanned again (docs/DESIGN.md 5.1)
|
|
1841
|
+
spellings = sorted(
|
|
1842
|
+
{
|
|
1843
|
+
repr(name): name for name in (exc.filename, exc.filename2)
|
|
1844
|
+
if isinstance(name, str) and name
|
|
1845
|
+
}.items(),
|
|
1846
|
+
key=lambda item: len(item[0]), reverse=True,
|
|
1847
|
+
)
|
|
1848
|
+
if spellings:
|
|
1849
|
+
splitter = re.compile("(" + "|".join(re.escape(r) for r, _ in spellings) + ")")
|
|
1850
|
+
shown = dict(spellings)
|
|
1851
|
+
return "".join(
|
|
1852
|
+
repr(self.display_path(shown[piece])) if index % 2
|
|
1853
|
+
else self.scrub_leftover_paths(piece)
|
|
1854
|
+
for index, piece in enumerate(splitter.split(text))
|
|
1855
|
+
)
|
|
1856
|
+
return self.scrub_leftover_paths(text)
|
|
1857
|
+
|
|
1858
|
+
@property
|
|
1859
|
+
def book(self) -> disclosure.AliasBook:
|
|
1860
|
+
"""The alias book a share sink asks. A command that registered its
|
|
1861
|
+
identities attached one to the context; a caller that did not (a
|
|
1862
|
+
direct render call, a sink Part 2 has not wired yet) gets a
|
|
1863
|
+
positional-only book of its own, so no sink can ever print a stable
|
|
1864
|
+
or real name by lacking one."""
|
|
1865
|
+
if self.context.book is not None:
|
|
1866
|
+
return self.context.book
|
|
1867
|
+
if self._fallback is None:
|
|
1868
|
+
self._fallback = disclosure.fallback_book()
|
|
1869
|
+
return self._fallback
|
|
1870
|
+
|
|
1871
|
+
def identity_text(self, kind: str, real_id: str) -> str:
|
|
1872
|
+
"""P10 rendered form of an identity at share: a:<alias>,
|
|
1873
|
+
s:<alias>, project:<alias>; positional (agent-N) on fallback."""
|
|
1874
|
+
return disclosure.rendered_form(kind, self.book.alias(kind, real_id))
|
|
1875
|
+
|
|
1876
|
+
def _identity_regexes(self):
|
|
1877
|
+
if self._identity_patterns is None:
|
|
1878
|
+
by_kind: dict[str, dict[str, str]] = {}
|
|
1879
|
+
for kind, real_id in self.context.identities:
|
|
1880
|
+
by_kind.setdefault(kind, {})[real_id] = kind
|
|
1881
|
+
patterns = []
|
|
1882
|
+
# Project slugs first: they are the longest and contain hyphens
|
|
1883
|
+
# the other kinds' boundaries would otherwise split on.
|
|
1884
|
+
for kind in ("project", "session", "agent"):
|
|
1885
|
+
ids = by_kind.get(kind)
|
|
1886
|
+
if not ids:
|
|
1887
|
+
continue
|
|
1888
|
+
alternation = "|".join(
|
|
1889
|
+
re.escape(real_id) for real_id in sorted(ids, key=len, reverse=True)
|
|
1890
|
+
)
|
|
1891
|
+
if kind == "project":
|
|
1892
|
+
pattern = re.compile(
|
|
1893
|
+
r"(?<![A-Za-z0-9_])(?:" + alternation + r")(?![A-Za-z0-9_-])"
|
|
1894
|
+
)
|
|
1895
|
+
else:
|
|
1896
|
+
pattern = re.compile(
|
|
1897
|
+
r"(?<![0-9A-Za-z])(?:" + alternation + r")(?![0-9A-Za-z])"
|
|
1898
|
+
)
|
|
1899
|
+
patterns.append((kind, pattern, ids))
|
|
1900
|
+
self._identity_patterns = patterns
|
|
1901
|
+
return self._identity_patterns
|
|
1902
|
+
|
|
1903
|
+
def _substitute_identities(self, text: str) -> str:
|
|
1904
|
+
"""P10/P11 at share: every collected identity quoted in free text
|
|
1905
|
+
becomes its rendered form (the run's own project slug before
|
|
1906
|
+
structural redaction, the order apply_paths already uses). Any other
|
|
1907
|
+
id or slug keeps the structural backstop."""
|
|
1908
|
+
if self.context.level != "share" or not self.context.identities:
|
|
1909
|
+
return text
|
|
1910
|
+
for kind, pattern, _ids in self._identity_regexes():
|
|
1911
|
+
text = pattern.sub(
|
|
1912
|
+
lambda match, kind=kind: self.identity_text(kind, match.group(0)), text
|
|
1913
|
+
)
|
|
1914
|
+
return text
|
|
1915
|
+
|
|
1916
|
+
def __call__(self, text: str) -> str:
|
|
1917
|
+
"""Structural redact(), plus (docs/DESIGN.md 5.1, M1 review finding
|
|
1918
|
+
1) substituting any report_path value already registered on `names`
|
|
1919
|
+
-- run BEFORE redact() itself, not after, so a report_path under the
|
|
1920
|
+
home directory is replaced by its own path-N pseudonym wholesale
|
|
1921
|
+
rather than first being collapsed by redact()'s home/tmp rules into
|
|
1922
|
+
a "~/.../basename" shape that no longer contains the raw substring
|
|
1923
|
+
apply_paths needs to match. This is the one exception to "evidence/
|
|
1924
|
+
diagnostics/metrics never see a pseudonym" (5.2): report_path is a
|
|
1925
|
+
self-reported claim value, not a faithful transcript quote, and must
|
|
1926
|
+
never appear raw by default wherever it is rendered."""
|
|
1927
|
+
if not self.enabled or not text:
|
|
1928
|
+
return text
|
|
1929
|
+
text = scan_paths(
|
|
1930
|
+
text, self.names, self._render_path_text, render_hidden=self._render_hidden_text,
|
|
1931
|
+
)
|
|
1932
|
+
text = self._substitute_identities(text)
|
|
1933
|
+
return redact(text, home=self.home)
|
|
1934
|
+
|
|
1935
|
+
def check_text(self, value: str | None) -> str | None:
|
|
1936
|
+
"""A declared check command as a sink prints it (docs/DESIGN.md 4.11, 5.1): `check-N`
|
|
1937
|
+
at share, as is at raw, and at local the real command after the one free-text path
|
|
1938
|
+
scanner (so no absolute path survives in `checks[]`, `steps[]` or the status line)."""
|
|
1939
|
+
shown = self.names.check(value)
|
|
1940
|
+
if self.level == "local" and shown:
|
|
1941
|
+
return self(shown)
|
|
1942
|
+
return shown
|
|
1943
|
+
|
|
1944
|
+
def title_parts(self, parts) -> str:
|
|
1945
|
+
"""A title built from structured parts (docs/DESIGN.md 4.13.4, 5.1; F fix 7, W7-2): the
|
|
1946
|
+
fixed template with its one `{path}` placeholder filled by the path rendered ONCE, never
|
|
1947
|
+
scanned again. `parts` has `.template` and `.path` (`models.TitleParts`). raw: the real
|
|
1948
|
+
path. share and local: the path's registered pseudonym (at local the P19 display), else
|
|
1949
|
+
what the plain call renders for the path alone, which is what `metrics.paths` and
|
|
1950
|
+
`metrics.claims.report_path` print, so the title's `path-N` is theirs. The words around
|
|
1951
|
+
the placeholder are the tool's own and are never given to the scanner."""
|
|
1952
|
+
head, _sep, tail = parts.template.partition("{path}")
|
|
1953
|
+
if not self.enabled:
|
|
1954
|
+
return head + parts.path + tail
|
|
1955
|
+
shown = self.names.registered_path(parts.path)
|
|
1956
|
+
if shown is None:
|
|
1957
|
+
shown = self(parts.path)
|
|
1958
|
+
if shown.endswith(_HIDDEN_MARK):
|
|
1959
|
+
# the value is one path, so nothing but a path was hidden: no marker
|
|
1960
|
+
shown = shown[:-len(_HIDDEN_MARK)]
|
|
1961
|
+
if self._scrub_paths:
|
|
1962
|
+
shown = _CONTROL_RE.sub(" ", shown)
|
|
1963
|
+
return head + shown + tail
|
|
1964
|
+
|
|
1965
|
+
def title(self, text: str) -> str:
|
|
1966
|
+
"""Finding.title only (docs/DESIGN.md 5.1/5.2): everything __call__
|
|
1967
|
+
already does (structural redact() plus path substitution), followed
|
|
1968
|
+
by pseudonym substitution over every name `names` has registered so
|
|
1969
|
+
far. Never call this on evidence, a diagnostic, or a metrics value --
|
|
1970
|
+
see the class docstring."""
|
|
1971
|
+
if not self.enabled or not text:
|
|
1972
|
+
return text
|
|
1973
|
+
# N3b S1: registered names first, so a label that itself holds a
|
|
1974
|
+
# UUID, email, or project-slug shape is replaced whole before the
|
|
1975
|
+
# structural pass rewrites part of it.
|
|
1976
|
+
shown = self(self.names.apply(text))
|
|
1977
|
+
if self._scrub_paths:
|
|
1978
|
+
# N3b S9: a title is one line; no control character survives it. Flattened after
|
|
1979
|
+
# the scan (F fix 5), so a backslash and a CRLF stay a line continuation for it.
|
|
1980
|
+
shown = _CONTROL_RE.sub(" ", shown)
|
|
1981
|
+
return shown
|