phiward 0.2.0b1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. phiward/__init__.py +8 -0
  2. phiward/__main__.py +5 -0
  3. phiward/_toml.py +211 -0
  4. phiward/adapters/__init__.py +105 -0
  5. phiward/adapters/claude.py +429 -0
  6. phiward/broker.py +831 -0
  7. phiward/carryover.py +1613 -0
  8. phiward/claude_settings.py +1307 -0
  9. phiward/cli.py +110 -0
  10. phiward/config.py +936 -0
  11. phiward/confine.py +476 -0
  12. phiward/copying.py +533 -0
  13. phiward/core.py +105 -0
  14. phiward/data/agents-section.md +43 -0
  15. phiward/doctor.py +1132 -0
  16. phiward/fasthook.py +70 -0
  17. phiward/fsinfo.py +241 -0
  18. phiward/gateway.py +159 -0
  19. phiward/githooks.py +248 -0
  20. phiward/guards/__init__.py +0 -0
  21. phiward/guards/protected_paths.py +1138 -0
  22. phiward/hidden.py +281 -0
  23. phiward/init.py +155 -0
  24. phiward/install.py +828 -0
  25. phiward/inventory.py +173 -0
  26. phiward/mcp.py +186 -0
  27. phiward/mcp_config.py +1736 -0
  28. phiward/names.py +5 -0
  29. phiward/packs/dates.toml +17 -0
  30. phiward/packs/genomics.toml +51 -0
  31. phiward/packs/msk.toml +32 -0
  32. phiward/pathid.py +403 -0
  33. phiward/pathmatch.py +98 -0
  34. phiward/paths.py +942 -0
  35. phiward/pin.py +214 -0
  36. phiward/procinfo.py +73 -0
  37. phiward/remote.py +448 -0
  38. phiward/run.py +788 -0
  39. phiward/safeio.py +84 -0
  40. phiward/sanitize.py +691 -0
  41. phiward/selfcheck.py +1506 -0
  42. phiward/service.py +321 -0
  43. phiward/session_note.py +86 -0
  44. phiward/setup_cmd.py +886 -0
  45. phiward/shell.py +504 -0
  46. phiward/wholetool.py +1463 -0
  47. phiward-0.2.0b1.dist-info/METADATA +147 -0
  48. phiward-0.2.0b1.dist-info/RECORD +53 -0
  49. phiward-0.2.0b1.dist-info/WHEEL +5 -0
  50. phiward-0.2.0b1.dist-info/entry_points.txt +3 -0
  51. phiward-0.2.0b1.dist-info/licenses/LICENSE +662 -0
  52. phiward-0.2.0b1.dist-info/licenses/NOTICE +16 -0
  53. phiward-0.2.0b1.dist-info/top_level.txt +1 -0
phiward/sanitize.py ADDED
@@ -0,0 +1,691 @@
1
+ """`phiward sanitize`: stdin to stdout, every line masked. The one trusted printer.
2
+
3
+ The gateway (`phiward run`, `phiward copy`) sends every line its command prints through
4
+ this masking, in process. The `sanitize` command applies the same masking to stdin, for the git
5
+ hooks and for manual use; the hook does not allow a `| phiward sanitize` pipe as a way to
6
+ touch protected data. The filter fails closed and never echoes a line that was not masked.
7
+
8
+ A line is masked in this order (a placeholder is written in angle brackets):
9
+ 1. protected paths of the config (roots, patterns, and the symlink-resolved spellings): <PROTECTED>;
10
+ then, when the inventory is on, the name of every entry under the roots, wherever it
11
+ appears in the line (inventory.py): <ENTRY>;
12
+ (before step 1, escape sequences and control characters are removed and the line is put in
13
+ Unicode NFKC form, so a name cannot hide behind color codes or look-alike letters);
14
+ 2. the `paths` rules of the packs, then the `identifiers` rules of the packs, then the rules of
15
+ `[sanitize] patterns` in the config;
16
+ 3. name tokens from the `names_from` files and `--names-from` files: <NAME>;
17
+ 4. text mode only: the `positions` rules of the packs.
18
+ Table mode skips step 4 (a table's numeric cells are program-made numbers).
19
+
20
+ Packs: `[sanitize] packs` in the config (default `["genomics"]`). A name without "/" is a built-in
21
+ file `packs/<name>.toml`; a name with "/" is the path of a pack file. The pack format is described
22
+ at the top of `packs/genomics.toml`. A `keep_files` regex keeps whole file names that a <BAMFILE>
23
+ rule would mask.
24
+
25
+ Fail closed: a bad config, names file, pack or pattern stops the filter (exit 3) before it reads
26
+ a line. A line that cannot be processed, or whose masked form a second pass would change again, is
27
+ replaced by a fixed notice.
28
+
29
+ usage:
30
+ producer 2>&1 | phiward sanitize [--names-from FILE ...] [--table]
31
+ phiward sanitize --selftest
32
+ phiward sanitize [--names-from FILE ...] [--table] --check FILE ...
33
+ phiward sanitize [--names-from FILE ...] --check-table FILE ...
34
+ """
35
+
36
+ import os
37
+ import re
38
+ import sys
39
+ from typing import (Any, BinaryIO, Dict, Iterable, List, Optional, Pattern, Sequence,
40
+ TextIO, Tuple)
41
+
42
+ from phiward import _toml, config, inventory, paths, pathmatch
43
+
44
+ WITHHELD = "[line withheld by the sanitizer]"
45
+ KEEP_PLACEHOLDER = "<BAMFILE>" # the placeholder whose matches `keep_files` can spare
46
+ DEFAULT_PACKS = ["genomics"]
47
+ PROBLEM_MESSAGE = ("sanitizer: the config, a names file or a pack could not be used; "
48
+ "stopping before any line is read (run 'phiward doctor')")
49
+ USAGE = ("Mask protected paths and identifiers in text: stdin in, the masked text out. It does "
50
+ "not\nmask entry names under the roots; `phiward run` does. --check reports what a file "
51
+ "would leak.\n"
52
+ "usage: phiward sanitize [--table] [--names-from FILE]...\n"
53
+ " phiward sanitize --selftest\n"
54
+ " phiward sanitize [--names-from FILE]... --check FILE...\n"
55
+ " phiward sanitize [--names-from FILE]... --check-table FILE...\n")
56
+
57
+ HELP = USAGE + """
58
+ Options:
59
+ --names-from FILE also mask the names in FILE, an index file (tab-separated fields, as
60
+ `[sanitize] names_from` in docs/configuration.md); may be given again
61
+ --table mask the text as a table: numeric cells are kept. Only when the operator
62
+ turned it on ([sanitize] table = true in the user config)
63
+ --check FILE... read the files and report what would leak (exit 1 when something would);
64
+ prints no protected text
65
+ --check-table FILE... the same, for files read as tables
66
+ --selftest run the sanitizer's own made-up cases (exit 1 when one fails)
67
+ -h, --help show this help
68
+
69
+ The protected paths, names and identifier packs come from the config in force (`phiward
70
+ doctor` shows it). The first line says how many name tokens and rules are in force; the last
71
+ line counts the lines read and changed.
72
+ """
73
+
74
+ PLACEHOLDER_WORDS = {"protected", "entry", "name", "sampleid", "cmoid", "bamid", "read", "locus", "num",
75
+ "donor", "patientpath", "clinicalpath", "bamfile"}
76
+ # Words that carry no identity and appear in many paths: never a name token.
77
+ STOP_WORDS = {"data1", "share", "analysis", "star", "aligned", "sortedbycoord", "bam", "bams",
78
+ "tumour", "normal", "inputs", "patient", "runs", "test", "scratch", "home",
79
+ "none", "true", "false"} | PLACEHOLDER_WORDS
80
+
81
+ _PACK_KEYS = ("paths", "identifiers", "positions", "keep_files")
82
+ _PLACEHOLDER = re.compile(r"<[A-Z][A-Z0-9_]*>")
83
+ _PACK_NAME = re.compile(r"[A-Za-z0-9_-]+")
84
+
85
+ Rule = Tuple["Pattern[str]", str]
86
+
87
+
88
+ class SanitizerError(Exception):
89
+ """The sanitizer cannot be built (fail closed)."""
90
+
91
+
92
+ def fmt_n(n: int) -> str:
93
+ """A count for a log line without a run of five digits: 1234, 12.3k, 1.2M."""
94
+ if n < 10000:
95
+ return str(n)
96
+ if n < 1_000_000:
97
+ return f"{n / 1000:.1f}k"
98
+ return f"{n / 1_000_000:.1f}M"
99
+
100
+
101
+ # ---- rules and packs ---------------------------------------------------------------------------
102
+
103
+ def parse_rule(text: str, where: str) -> Rule:
104
+ """'REGEX => <PLACEHOLDER>' as a compiled regex and its placeholder."""
105
+ regex, sep, repl = text.rpartition(" => ")
106
+ repl = repl.strip()
107
+ if not sep or not regex or not _PLACEHOLDER.fullmatch(repl):
108
+ raise SanitizerError(f"{where}: a rule must read 'REGEX => <PLACEHOLDER>'")
109
+ rx = _compile(regex, where)
110
+ if rx.search("") is not None:
111
+ raise SanitizerError(f"{where}: a rule must not match the empty string")
112
+ return rx, repl
113
+
114
+
115
+ def _compile(regex: str, where: str) -> "Pattern[str]":
116
+ try:
117
+ return re.compile(regex)
118
+ except (re.error, RecursionError, OverflowError) as exc:
119
+ raise SanitizerError(f"{where}: a regex does not compile ({type(exc).__name__})")
120
+
121
+
122
+ class Pack:
123
+ def __init__(self, name: str):
124
+ self.name = name
125
+ self.paths: List[Rule] = []
126
+ self.identifiers: List[Rule] = []
127
+ self.positions: List[Rule] = []
128
+ self.keep_files: List["Pattern[str]"] = []
129
+
130
+
131
+ def pack_path(name: str) -> str:
132
+ if "/" in name:
133
+ return os.path.expanduser(name)
134
+ if not _PACK_NAME.fullmatch(name):
135
+ raise SanitizerError(f"pack '{name}': not a valid built-in pack name")
136
+ return os.path.join(os.path.dirname(os.path.abspath(__file__)), "packs", name + ".toml")
137
+
138
+
139
+ def load_pack(name: str) -> Pack:
140
+ where = f"pack '{name}'"
141
+ path = pack_path(name)
142
+ try:
143
+ data, _ = _toml.load_file(path, follow=False) # never through a link
144
+ except (OSError, UnicodeDecodeError) as exc:
145
+ raise SanitizerError(f"{where}: cannot be read ({type(exc).__name__})")
146
+ except _toml.TomlError as exc:
147
+ raise SanitizerError(f"{where}: does not parse: {exc}")
148
+ if set(data) != {"pack"} or not isinstance(data["pack"], dict):
149
+ raise SanitizerError(f"{where}: must hold one [pack] table")
150
+ body: Dict[str, Any] = data["pack"]
151
+ pack = Pack(name)
152
+ for key, value in body.items():
153
+ if key == "name":
154
+ if not isinstance(value, str):
155
+ raise SanitizerError(f"{where}: 'name' must be a string")
156
+ elif key in _PACK_KEYS:
157
+ if not isinstance(value, list) or not all(isinstance(v, str) and v for v in value):
158
+ raise SanitizerError(f"{where}: '{key}' must be a list of non-empty strings")
159
+ else:
160
+ raise SanitizerError(f"{where}: unknown key '{key}'")
161
+ for key in ("paths", "identifiers", "positions"):
162
+ getattr(pack, key).extend(parse_rule(v, f"{where} {key}") for v in body.get(key, []))
163
+ pack.keep_files = [_compile(v, f"{where} keep_files") for v in body.get("keep_files", [])]
164
+ return pack
165
+
166
+
167
+ # ---- name tokens -------------------------------------------------------------------------------
168
+
169
+ def name_tokens(lines: Iterable[str], stop: Optional[Iterable[str]] = None) -> "set[str]":
170
+ """The name tokens of index-TAB-path lines: each path part, its sample part (before the first
171
+ dot), and the dash/underscore pieces of the sample part, of 4 or more characters, never a
172
+ stop word and never a bare number. Case does not matter."""
173
+ stops = {s.lower() for s in (STOP_WORDS if stop is None else stop)}
174
+ toks: "set[str]" = set()
175
+
176
+ def add(t: str) -> None:
177
+ if len(t) >= 4 and not t.isdigit() and t.lower() not in stops and t.isascii():
178
+ toks.add(t)
179
+
180
+ for line in lines:
181
+ for field in line.rstrip("\r\n").split("\t"):
182
+ if "/" not in field:
183
+ continue
184
+ for part in (p for p in field.split("/") if p):
185
+ for cand in {part, part.split(".")[0]}:
186
+ add(cand)
187
+ for piece in (p for p in re.split(r"[-_.]", cand) if p):
188
+ add(piece)
189
+ if cand.count("-") >= 1:
190
+ add("-".join(cand.split("-")[:2]))
191
+ return toks
192
+
193
+
194
+ def _read_names_file(path: str, stop: Iterable[str]) -> "set[str]":
195
+ from phiward import safeio
196
+ try:
197
+ # never through a link, never blocking: a `--names-from` file comes from the agent
198
+ text = safeio.read_text(os.path.expanduser(path))
199
+ got = name_tokens(text.splitlines(), stop)
200
+ except (OSError, UnicodeDecodeError) as exc:
201
+ raise SanitizerError(f"a names file cannot be read ({type(exc).__name__})")
202
+ if not got:
203
+ raise SanitizerError("a names file yields no token")
204
+ return got
205
+
206
+
207
+ # ---- the sanitizer -----------------------------------------------------------------------------
208
+
209
+ class Sanitizer:
210
+ def __init__(self, protected: Optional["paths.ProtectedSet"] = None,
211
+ path_rules: Sequence[Rule] = (), id_rules: Sequence[Rule] = (),
212
+ pos_rules: Sequence[Rule] = (), keep_files: Sequence["Pattern[str]"] = (),
213
+ tokens: Iterable[str] = (), entries: Iterable[str] = (),
214
+ known: Iterable[str] = (), roots: Iterable[str] = ()):
215
+ # the protected roots and patterns: their matches in a line (`text_spans`)
216
+ self.protected = protected if protected else None
217
+ # the folders a protected pattern matches now, as `_key`s, and their lengths as written
218
+ # (folding can change a length: `ß`), longest first: a protected match that starts with
219
+ # one is marked to its end (`_marked`)
220
+ known = list(known)
221
+ self.known = {_key(k) for k in known}
222
+ self.known_lengths = sorted({len(k) for k in known}, reverse=True)
223
+ self.root_keys = {_key(r) for r in roots} # the roots' own spellings
224
+ # rules applied before every other step: the remote roots of `run --remote`, so a local
225
+ # entry name inside a remote path cannot break its match
226
+ self.first_rules: List[Rule] = []
227
+ self.entries = inventory.matcher(entries)
228
+ self.path_rules = list(path_rules)
229
+ self.id_rules = list(id_rules)
230
+ self.pos_rules = list(pos_rules)
231
+ self.keep_files = list(keep_files)
232
+ self.tokens = sorted(set(tokens), key=len, reverse=True)
233
+ self.re_names: Optional["Pattern[str]"] = None
234
+ if self.tokens:
235
+ alt = "|".join(re.escape(t) for t in self.tokens)
236
+ self.re_names = re.compile(rf"(?<![A-Za-z0-9])(?:{alt})(?![A-Za-z0-9])", re.I)
237
+
238
+ @classmethod
239
+ def from_config(cls, cfg: "config.Config", extra_names_from: Sequence[str] = (),
240
+ entries: bool = False, names_lines: Sequence[str] = ()) -> "Sanitizer":
241
+ """Build from a loaded config (and extra names files). With `entries`, it also lists the
242
+ protected roots and masks every entry name (the caller must be able to read the roots).
243
+ Raises SanitizerError."""
244
+ return cls._build(cfg, list(extra_names_from), list(names_lines), entries)
245
+
246
+ @classmethod
247
+ def _build(cls, cfg: "config.Config", names_files: Sequence[str],
248
+ name_lines: Iterable[str], entries: bool = False) -> "Sanitizer":
249
+ try:
250
+ if cfg.problem:
251
+ raise SanitizerError("the config has a problem")
252
+ names: List[str] = []
253
+ for pack_name in (cfg.packs or DEFAULT_PACKS):
254
+ if pack_name not in names:
255
+ names.append(pack_name)
256
+ packs = [load_pack(n) for n in names]
257
+ path_rules = [r for p in packs for r in p.paths]
258
+ id_rules = [r for p in packs for r in p.identifiers]
259
+ id_rules += [parse_rule(v, "[sanitize] patterns") for v in cfg.sanitize_patterns]
260
+ pos_rules = [r for p in packs for r in p.positions]
261
+ keep = [k for p in packs for k in p.keep_files]
262
+ # Stop words come from the built-in list only: a rule's placeholder must never
263
+ # exempt an entry name or a name token from masking.
264
+ stop = set(STOP_WORDS)
265
+ tokens = name_tokens(name_lines, stop) if name_lines else set()
266
+ for f in list(cfg.names_from) + list(names_files):
267
+ tokens |= _read_names_file(f, stop)
268
+ # with the inventory (a gateway run): every folder a pattern matches now, looked up
269
+ # in a set (a regex of tens of thousands of paths takes minutes to build)
270
+ known = {f for p in cfg.patterns for m in pathmatch.matches(p)
271
+ for r in (m, os.path.realpath(m)) for f in (r, normalize(r))
272
+ } if entries else set()
273
+ pset = paths.ProtectedSet(cfg.all_roots, cfg.patterns)
274
+ roots = {f for r in pset.roots for f in (r, normalize(r))}
275
+ ents: "set[str]" = set()
276
+ if entries:
277
+ try:
278
+ ents = inventory.build(cfg.roots, cfg.patterns, cfg.inventory_limit,
279
+ {w.casefold() for w in stop})
280
+ except inventory.InventoryError as exc:
281
+ raise SanitizerError(str(exc))
282
+ return cls(pset, path_rules, id_rules, pos_rules, keep, tokens, ents, known, roots)
283
+ except SanitizerError:
284
+ raise
285
+ except Exception as exc: # noqa: BLE001 - fail closed on anything
286
+ raise SanitizerError(f"cannot build the sanitizer ({type(exc).__name__})")
287
+
288
+ def _apply(self, rules: Sequence[Rule], line: str) -> str:
289
+ for rx, repl in rules:
290
+ if repl == KEEP_PLACEHOLDER and self.keep_files:
291
+ line = rx.sub(lambda m, r=repl: m.group(0) if self._kept(m.group(0)) else r, line)
292
+ else:
293
+ line = rx.sub(lambda _m, r=repl: r, line)
294
+ return line
295
+
296
+ def _kept(self, token: str) -> bool:
297
+ base = token.rsplit("/", 1)[-1]
298
+ return any(k.fullmatch(base) for k in self.keep_files)
299
+
300
+ def _mask(self, line: str, table: bool) -> str:
301
+ line = self._apply(self.first_rules, line)
302
+ if self.protected is not None:
303
+ # another spelling of a path (`<root>\/Zorro` from JSON, `<root>//x`, `<root>/./x`):
304
+ # the line is read plain where that makes a protected path, so the name after it
305
+ # is masked too
306
+ plain = paths.plain_spelling(line)
307
+ spans = self.protected.text_spans(plain) if plain != line else []
308
+ if spans:
309
+ # plain from the first protected path on; the text before it stays as written
310
+ plain, pos = paths.plain_spelling_map(line)
311
+ cut = spans[0][0]
312
+ line = line[:pos[cut]] + plain[cut:]
313
+ if self.protected is not None:
314
+ # the root (or pattern) part first, as a mark; then the entry names, which may hold
315
+ # spaces; then the mark with the rest of the path (names, now placeholders, and
316
+ # path characters) becomes one placeholder: no part of `Doe Jane/x` is left
317
+ line = self._marked(line)
318
+ line = inventory.mask(line, self.entries)
319
+ if self.protected is not None:
320
+ # a mark and the rest of its path; one whose path goes on past the root or the
321
+ # pattern (`<root>/New Person 7`) is followed by `_GOES_ON` for `_tail`
322
+ line = _AFTER_MARK.sub(
323
+ lambda m: "<PROTECTED>" + (_GOES_ON if len(m.group(0)) > 1 else ""), line)
324
+ line = _tail(line)
325
+ line = self._apply(self.path_rules, line)
326
+ line = self._apply(self.id_rules, line)
327
+ if self.re_names is not None:
328
+ line = self.re_names.sub("<NAME>", line)
329
+ if not table:
330
+ line = self._apply(self.pos_rules, line)
331
+ return line
332
+
333
+ def _marked(self, line: str) -> str:
334
+ """Every protected root or pattern match as the mark. A pattern's wildcard may hold
335
+ spaces, so its match runs to the end of the name, or past it. When the longest folder
336
+ the pattern matches now starts at the match, that folder is marked whole (also when it
337
+ holds characters the wildcard stops at: `phi_O'Brien Pat`), and the text after it is
338
+ kept only when no name can go on there (`/`, a tab, the line's end, a count as in
339
+ `: 12 reads`); otherwise the mark runs to the end of the line, so a longer name that is
340
+ not on disk never leaks. An open match (its last wildcard met a quote or a shell
341
+ character: `phi_New;Quux`) that no such folder covers runs to the end of the line too.
342
+ Every match counts, also one inside or across another: overlapping matches are joined."""
343
+ if self.protected is None:
344
+ return line
345
+ marks: List[Tuple[int, int]] = []
346
+ for start, end, open_ in self.protected.text_spans(line):
347
+ for size in self.known_lengths:
348
+ stop = start + size
349
+ if stop <= len(line) and _key(line[start:stop]) in self.known and \
350
+ (stop == len(line) or not _NAME_CHAR.match(line[stop])):
351
+ end = stop if _KEEP_AFTER.match(line, stop) else _line_end(line, stop)
352
+ break
353
+ else:
354
+ if open_ and _key(line[start:end]) not in self.root_keys:
355
+ end = _line_end(line, end)
356
+ end = max(end, start + 1)
357
+ if marks and start <= marks[-1][1]:
358
+ marks[-1] = (marks[-1][0], max(marks[-1][1], end))
359
+ else:
360
+ marks.append((start, end))
361
+ out: List[str] = []
362
+ pos = 0
363
+ for start, end in marks:
364
+ out += [line[pos:start], _MARK]
365
+ pos = end
366
+ out.append(line[pos:])
367
+ return "".join(out)
368
+
369
+ def clean(self, line: str, table: bool = False) -> str:
370
+ """The line with its identifiers masked (table mode: identifiers only). Never raises:
371
+ a line it cannot process comes back as the fixed notice."""
372
+ try:
373
+ line = normalize(line)
374
+ out = self._mask(line, table)
375
+ # the masks must not leave what they mask: a second pass changes nothing
376
+ if self._mask(out, table) != out:
377
+ return WITHHELD
378
+ # nor a name written with escapes, as ordinary tools print one (git's octal quoting,
379
+ # JSON's \u, Python's \x, a URL's %XX): the masked line, decoded, holds nothing
380
+ # more to mask (the line is not rewritten: its escapes may be what its reader needs)
381
+ if "\\" in out or "%" in out:
382
+ plain = normalize(unescape(out))
383
+ if plain != out and self._mask(plain, table) != plain:
384
+ return WITHHELD
385
+ return out
386
+ except Exception: # noqa: BLE001 - fail closed on anything
387
+ return WITHHELD
388
+
389
+
390
+ _MARK = "\x00" # never in a line: normalize removes control characters
391
+ _NAME_CHAR = re.compile(r"[\w.\-]") # a character that would go on with a name
392
+ # after a folder on disk: text that cannot be more of a name (a path goes on, a tab, the end,
393
+ # or after `: ` counts, each with at most one plain unit, to a tab or the end: `: 12 reads, 3
394
+ # errors`). Numbers after a space are not kept: `phi_Ann 4471` may be a longer name
395
+ _COUNT = r"\d[\d.,]*(?: (?:" + "|".join([
396
+ "reads?", "lines?", "files?", "folders?", "bytes?", "matches?", "errors?", "warnings?",
397
+ "entries", "entry", "records?", "rows?", "items?", "samples?", "variants?", "calls?",
398
+ "[KMGT]i?B", "kB", "%"]) + "))?"
399
+ _KEEP_AFTER = re.compile(r"$|/|\t|:$|[^\S\t]+(?=/)|: " + _COUNT + r"(?:, " + _COUNT +
400
+ r")*(?=\t|$)")
401
+ _GOES_ON = "\x01" # after a mask whose path went on inside a root or a match (never in a line)
402
+ # where a name inside a protected path has clearly ended: the line's end, a tab, or spaces and
403
+ # another path (`cp <root>/a /tmp/b`). Anything else may be more of a name that is not on disk
404
+ # (`<root>/New Person 7`, `<root>/O'Brien`, `'<root>/O' Brien'`, `<root>/Doe; Jane`), so the
405
+ # mask runs to the next tab or the line's end: P5 holds by construction, and ordinary text
406
+ # after such a path (`cat '<root>/x' done`) is lost
407
+ _ENDED = re.compile(r"[^\S\t]*(?:$|(?=\t)|(?<=[^\S\t])(?=/)|(?=<PROTECTED>))")
408
+
409
+
410
+ def _tail(line: str) -> str:
411
+ """The line with the text after each mask whose name has not clearly ended (`_ENDED`)
412
+ masked to the next tab or the line's end."""
413
+ out: List[str] = []
414
+ pos = 0
415
+ while True:
416
+ i = line.find(_GOES_ON, pos)
417
+ if i < 0:
418
+ out.append(line[pos:])
419
+ return "".join(out)
420
+ out.append(line[pos:i])
421
+ pos = i + 1 if _ENDED.match(line, i + 1) else _line_end(line, i + 1)
422
+
423
+
424
+ def _line_end(line: str, start: int) -> int:
425
+ """Where the text from `start` ends: the next tab or the line's end."""
426
+ tab = line.find("\t", start)
427
+ return len(line) if tab < 0 else tab
428
+
429
+
430
+ def _key(path: str) -> str:
431
+ """A path as the known-folder set holds it: case folded where the volume ignores case."""
432
+ return path.casefold() if paths.FOLD_CASE else path
433
+ _AFTER_MARK = re.compile("\x00" + rf"(?:/(?:<ENTRY>|{paths._PATH_CHAR})*)*")
434
+ normalize = inventory.normalize # the form masking sees (escapes removed, NFKC)
435
+
436
+ # the escapes ordinary tools write a name with: a run of bytes (git's `\303\274`, Python's
437
+ # `\xc3\xbc`, a URL's `%C3%BC`), read as UTF-8 (else one character each: Python's `'\xfc'`),
438
+ # and JSON's or Python's `\u00fc` / `\U0001F600` (a surrogate pair joined)
439
+ _BYTES = re.compile(r"(?:\\[0-3][0-7]{2}|\\x[0-9A-Fa-f]{2}|%[0-9A-Fa-f]{2})+")
440
+ _UNI = re.compile(r"\\u[0-9A-Fa-f]{4}|\\U[0-9A-Fa-f]{8}")
441
+
442
+
443
+ def _bytes_of(run: str) -> bytes:
444
+ out = bytearray()
445
+ i = 0
446
+ while i < len(run):
447
+ if run[i] == "%":
448
+ out.append(int(run[i + 1:i + 3], 16))
449
+ i += 3
450
+ elif run[i + 1] == "x":
451
+ out.append(int(run[i + 2:i + 4], 16))
452
+ i += 4
453
+ else:
454
+ out.append(int(run[i + 1:i + 4], 8))
455
+ i += 4
456
+ return bytes(out)
457
+
458
+
459
+ def _decode_run(m: "re.Match[str]") -> str:
460
+ raw = _bytes_of(m.group(0))
461
+ try:
462
+ return raw.decode("utf-8")
463
+ except UnicodeDecodeError:
464
+ return raw.decode("latin-1")
465
+
466
+
467
+ def unescape(text: str) -> str:
468
+ """`text` with the escapes above decoded, for a second look only (never printed)."""
469
+ text = _BYTES.sub(_decode_run, text)
470
+ text = _UNI.sub(lambda m: chr(int(m.group(0)[2:], 16)) if int(m.group(0)[2:], 16) <= 0x10FFFF
471
+ else m.group(0), text)
472
+ # a surrogate pair from JSON (`\ud83d\ude00`) is one character
473
+ return text.encode("utf-16", "surrogatepass").decode("utf-16", "replace")
474
+
475
+
476
+ def filter_stream(san: Sanitizer, fin: BinaryIO, fout: TextIO, table: bool = False) -> Tuple[int, int]:
477
+ """Mask every line of `fin` onto `fout`. Returns (lines, changed)."""
478
+ lines = changed = 0
479
+ for raw in fin:
480
+ text = raw.decode("utf-8", "replace") if isinstance(raw, bytes) else raw
481
+ text = text.replace("\r\n", "\n").replace("\r", "\n")
482
+ parts = text.split("\n")
483
+ if text.endswith("\n"):
484
+ parts.pop()
485
+ for part in parts:
486
+ out = san.clean(part, table=table)
487
+ lines += 1
488
+ changed += out != part
489
+ fout.write(out + "\n")
490
+ fout.flush()
491
+ return lines, changed
492
+
493
+
494
+ def check_files(san: Sanitizer, files: Sequence[str], table: bool, fout: TextIO) -> bool:
495
+ """Count the lines of each file that the sanitizer changes. False if a file cannot be read."""
496
+ ok = True
497
+ for k, p in enumerate(files, 1):
498
+ n = ch = 0
499
+ try:
500
+ with open(p, encoding="utf-8", errors="replace") as fh:
501
+ for line in fh:
502
+ line = line.rstrip("\r\n")
503
+ n += 1
504
+ ch += san.clean(line, table=table) != line
505
+ except OSError:
506
+ fout.write(f"check file {k}: cannot be read\n")
507
+ ok = False
508
+ continue
509
+ fout.write(f"check file {k}: lines={fmt_n(n)} changed={fmt_n(ch)}\n")
510
+ return ok
511
+
512
+
513
+ # ---- the self-test: made-up lines and the masked lines they must become ------------------------
514
+
515
+ SELFTEST_ROOT = "/tmp/phiward-selftest/protected"
516
+ MADE_UP_RUNS = [ # index TAB path, all made up
517
+ "1\t/tmp/ZedStudy9/analysis/ZedRun_77/STAR/P-9990001-T01-IM6.Aligned.sortedByCoord.out.bam",
518
+ "2\t/tmp/Quux_Case_B/STAR/QQSAMP-ALPHA-12.Aligned.sortedByCoord.out.bam",
519
+ ]
520
+ PROTECTED_CASES = [ # with SELFTEST_ROOT protected; any pack
521
+ (f"see {SELFTEST_ROOT}/a/FAKE-PT-0001.txt", "see <PROTECTED>"),
522
+ (f"see {SELFTEST_ROOT}/a/New Person 7", "see <PROTECTED>"),
523
+ (f"at {SELFTEST_ROOT}, then", "at <PROTECTED>, then"),
524
+ ("elsewhere /tmp/phiward-selftest/other/x stays", "elsewhere /tmp/phiward-selftest/other/x stays"),
525
+ ]
526
+ GENOMICS_CASES = [ # pack "genomics" alone, no names: (made-up input, masked output in text mode)
527
+ ("file Sample7.sorted.bam and Sample7.sorted.bam.bai", "file <BAMFILE> and <BAMFILE>"),
528
+ ("path /tmp/x/ZedRun_77/y.Aligned.bam ok", "path <BAMFILE> ok"),
529
+ ("neutral slice.bam and b01.bam and rows.vcf stay", "neutral slice.bam and b01.bam and rows.vcf stay"),
530
+ ("reads fq lane_1.fastq.gz here", "reads fq <BAMFILE> here"),
531
+ ("calls CALLS.VCF.GZ and idx x.csi", "calls <BAMFILE> and idx <BAMFILE>"),
532
+ ("read A00123:45:HABCDEFXX:1:1101:12345:6789 and the mate", "read <READ> and the mate"),
533
+ ("read A00123:45:HABCDEFXX:1:1101:12345:6789/1 x", "read <READ> x"),
534
+ ("umi A00123:45:HABCDEFXX:2:2201:100:200_ACGTACGT.", "umi <READ>."),
535
+ ("old HWUSI-EAS100R:6:73:941:1973#0/1 x", "old <READ> x"),
536
+ ("pacbio m54012_190101_000000/12345/ccs z", "pacbio <READ> z"),
537
+ ("pos chr1:1234567 and 12:5678901 and chrX:123", "pos <LOCUS> and <LOCUS> and <LOCUS>"),
538
+ ("range 7:100-200 and chr2:1,234,567-1,300,000", "range <LOCUS> and <LOCUS>"),
539
+ ("1\t1234567\t.\tA\tC", "<LOCUS>\t.\tA\tC"),
540
+ ("row x\t12\t7654321\t.\tG", "row x\t<LOCUS>\t.\tG"),
541
+ ("space chrX 12345 end", "space <LOCUS> end"),
542
+ ("id 1_1234_A_C and chr7-55249071-C-T", "id <LOCUS> and <LOCUS>"),
543
+ ("bare 1234567 and 98765. and (4567890)", "bare <NUM> and <NUM>. and (<NUM>)"),
544
+ ("grouped 12,345,678 reads", "grouped <NUM> reads"),
545
+ ("tail position:48999 here", "tail position:<NUM> here"),
546
+ # genomics alone does not know institution IDs; only the digit run goes
547
+ ("sample P-9990001-T01-IM6 failed", "sample P-<NUM>-T01-IM6 failed"),
548
+ # lines that must pass unchanged
549
+ ("task 12 q20 state=complete runs=18", "task 12 q20 state=complete runs=18"),
550
+ ("rows=1234 ad=56 rdad=9876 total=12.3k", "rows=1234 ad=56 rdad=9876 total=12.3k"),
551
+ ("mean=12345.678 and v6.7.0 and 0.0123 and 1e-05", "mean=12345.678 and v6.7.0 and 0.0123 and 1e-05"),
552
+ ]
553
+ GENOMICS_TABLE_CASES = [ # pack "genomics" alone: identifiers masked, numbers kept
554
+ ("q20\tpenalty\ton\tclean\t0-4\t12345\t6\t1234567", "q20\tpenalty\ton\tclean\t0-4\t12345\t6\t1234567"),
555
+ ("x\tlane_1.fastq.gz\t77777", "x\t<BAMFILE>\t77777"),
556
+ ("1\t1234567\tA00123:45:HABCDEFXX:1:1101:12345:6789", "1\t1234567\t<READ>"),
557
+ ]
558
+ MSK_CASES = [ # packs "genomics" + "msk" with the names of MADE_UP_RUNS
559
+ ("sample P-9990001-T01-IM6 failed", "sample <SAMPLEID> failed"),
560
+ ("normal P-9990001-N01-IM6_extra and P-9990002", "normal <SAMPLEID> and <SAMPLEID>"),
561
+ ("cmo C-ABC123-L001-d seen; also s_C_XYZ789_P001_d", "cmo <CMOID> seen; also <CMOID>"),
562
+ ("stem AB123456-T opened", "stem <BAMID> opened"),
563
+ ("donor DonorFake-7 and DONOR_X1", "donor <DONOR> and <DONOR>"),
564
+ ("bam /clinical/ZedStudy9/analysis/ZedRun_77/STAR/x.Aligned.bam ok", "bam <CLINICALPATH> ok"),
565
+ ("impact /clinical/fakeshare/study_00/A/BC/FAKE0001-T.bam end", "impact <CLINICALPATH> end"),
566
+ ("slice S:/tmp/work/inputs/patient/edge/12/slice.bam, done", "slice <PATIENTPATH>, done"),
567
+ ("file Sample7.sorted.bam and neutral slice.bam", "file <BAMFILE> and neutral slice.bam"),
568
+ ("pos chr1:1234567 and bare 7654321", "pos <LOCUS> and bare <NUM>"),
569
+ ("sample tokens ZedRun_77 and zedstudy9 and QQSAMP-ALPHA-12 and Quux_Case_B",
570
+ "sample tokens <NAME> and <NAME> and <NAME> and <NAME>"),
571
+ ("pieces QQSAMP and alpha mask; the number 12 and out and bam stay",
572
+ "pieces <NAME> and <NAME> mask; the number 12 and out and bam stay"),
573
+ ("task 12 q20 state=complete runs=18", "task 12 q20 state=complete runs=18"),
574
+ ("rows=1234 ad=56 rdad=9876 total=12.3k", "rows=1234 ad=56 rdad=9876 total=12.3k"),
575
+ ]
576
+ MSK_TABLE_CASES = [ # packs "genomics" + "msk" with the names of MADE_UP_RUNS
577
+ ("q20\tpenalty\tgtf\ton\tclean\t0-4\t12345\t6\t1234567", "q20\tpenalty\tgtf\ton\tclean\t0-4\t12345\t6\t1234567"),
578
+ ("q20\tP-9990001-T01-IM6\t3\t1234567", "q20\t<SAMPLEID>\t3\t1234567"),
579
+ ("12\t3456\tZedRun_77\t7", "12\t3456\t<NAME>\t7"),
580
+ ]
581
+
582
+
583
+ def selftest(fout: Optional[TextIO] = None) -> bool:
584
+ """Run the built-in made-up cases for the built-in packs. True if all pass."""
585
+ out = fout or sys.stdout
586
+ bad = total_text = total_table = 0
587
+ groups = [
588
+ (["genomics"], False, GENOMICS_CASES, GENOMICS_TABLE_CASES),
589
+ (["genomics", "msk"], True, MSK_CASES, MSK_TABLE_CASES),
590
+ ]
591
+ ntokens = 0
592
+ try:
593
+ for packs, with_names, cases, table_cases in groups:
594
+ cfg = config.Config(roots=[SELFTEST_ROOT], packs=packs)
595
+ san = Sanitizer._build(cfg, (), MADE_UP_RUNS if with_names else ())
596
+ ntokens = max(ntokens, len(san.tokens))
597
+ for src, want in cases + PROTECTED_CASES:
598
+ total_text += 1
599
+ got = san.clean(src)
600
+ if got != want or san.clean(got) != got:
601
+ bad += 1
602
+ out.write("SELFTEST FAIL: a made-up case gave another output\n")
603
+ for src, want in table_cases:
604
+ total_table += 1
605
+ if san.clean(src, table=True) != want:
606
+ bad += 1
607
+ out.write("SELFTEST FAIL: a made-up table case gave another output\n")
608
+ except SanitizerError:
609
+ out.write("SELFTEST FAIL: the built-in packs could not be loaded\n")
610
+ return False
611
+ out.write(f"selftest: {total_text} text cases and {total_table} table cases, {bad} failed; "
612
+ f"up to {ntokens} name tokens from {len(MADE_UP_RUNS)} made-up run lines\n")
613
+ return bad == 0
614
+
615
+
616
+ # ---- the command -------------------------------------------------------------------------------
617
+
618
+ def _parse_args(argv: Sequence[str]) -> Optional[Dict[str, Any]]:
619
+ opts: Dict[str, Any] = {"names": [], "files": [], "table": False, "mode": "filter"}
620
+ it = iter(argv)
621
+ for a in it:
622
+ if a == "--names-from":
623
+ value = next(it, "")
624
+ if not value:
625
+ return None
626
+ opts["names"].append(value)
627
+ elif a == "--table":
628
+ opts["table"] = True
629
+ elif a == "--selftest":
630
+ opts["mode"] = "selftest"
631
+ elif a in ("--check", "--check-table"):
632
+ opts["mode"] = "check"
633
+ opts["table"] = a == "--check-table"
634
+ elif opts["mode"] == "check" and not a.startswith("-"):
635
+ opts["files"].append(a)
636
+ else:
637
+ return None
638
+ if opts["mode"] == "check" and not opts["files"]:
639
+ return None
640
+ if opts["mode"] == "selftest" and (opts["names"] or opts["table"]):
641
+ return None
642
+ return opts
643
+
644
+
645
+ def main(argv: List[str]) -> int:
646
+ if any(a in ("-h", "--help") for a in argv):
647
+ print(HELP, end="")
648
+ return 0
649
+ opts = _parse_args(argv)
650
+ if opts is None:
651
+ print(USAGE, end="", file=sys.stderr)
652
+ return 2
653
+ try:
654
+ sys.stdout.reconfigure(encoding="utf-8", errors="replace") # type: ignore[attr-defined]
655
+ except (AttributeError, ValueError):
656
+ pass
657
+ if opts["mode"] == "selftest":
658
+ return 0 if selftest() else 1
659
+ try:
660
+ cfg = config.load(os.getcwd())
661
+ if cfg.problem:
662
+ raise SanitizerError("the config has a problem")
663
+ if opts["table"] and not cfg.sanitize_table:
664
+ from phiward.run import TABLE_OFF
665
+ print(TABLE_OFF, file=sys.stderr)
666
+ return 2
667
+ san = Sanitizer.from_config(cfg, [os.path.expanduser(n) for n in opts["names"]])
668
+ except Exception: # noqa: BLE001 - fail closed on anything
669
+ print(PROBLEM_MESSAGE)
670
+ return 3
671
+ if opts["mode"] == "check":
672
+ return 0 if check_files(san, opts["files"], opts["table"], sys.stdout) else 1
673
+ mode = "table" if opts["table"] else "text"
674
+ try:
675
+ sys.stdout.write(f"sanitizer on: name tokens={fmt_n(len(san.tokens))} "
676
+ f"rules={fmt_n(len(san.path_rules) + len(san.id_rules) + len(san.pos_rules))} "
677
+ f"mode={mode}\n")
678
+ sys.stdout.flush()
679
+ lines, changed = filter_stream(san, sys.stdin.buffer, sys.stdout, table=opts["table"])
680
+ sys.stdout.write(f"sanitizer done: lines={fmt_n(lines)} changed={fmt_n(changed)}\n")
681
+ sys.stdout.flush()
682
+ except BrokenPipeError:
683
+ try:
684
+ sys.stdout = open(os.devnull, "w") # no error message from the flush at exit
685
+ except OSError:
686
+ pass
687
+ return 0
688
+ except Exception: # noqa: BLE001 - never leave a half-masked state; stop
689
+ print("sanitizer: stopped on an error")
690
+ return 1
691
+ return 0