spanguard 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- spanguard/__init__.py +21 -0
- spanguard/_keys.py +29 -0
- spanguard/_overlap.py +53 -0
- spanguard/_transform.py +54 -0
- spanguard/cli.py +145 -0
- spanguard/policy.py +35 -0
- spanguard/py.typed +0 -0
- spanguard/recognizers/__init__.py +8 -0
- spanguard/recognizers/base.py +28 -0
- spanguard/recognizers/luhn.py +19 -0
- spanguard/recognizers/ner.py +57 -0
- spanguard/recognizers/regex_recognizers.py +156 -0
- spanguard/redactor.py +94 -0
- spanguard/result.py +49 -0
- spanguard/streaming.py +149 -0
- spanguard/tokenize.py +71 -0
- spanguard-0.1.0.dist-info/METADATA +235 -0
- spanguard-0.1.0.dist-info/RECORD +21 -0
- spanguard-0.1.0.dist-info/WHEEL +4 -0
- spanguard-0.1.0.dist-info/entry_points.txt +2 -0
- spanguard-0.1.0.dist-info/licenses/LICENSE +21 -0
spanguard/__init__.py
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"""spanguard: detect and redact PII/PHI with leak-free audit records."""
|
|
2
|
+
|
|
3
|
+
from spanguard.policy import Policy
|
|
4
|
+
from spanguard.recognizers.base import Recognizer
|
|
5
|
+
from spanguard.redactor import Redactor
|
|
6
|
+
from spanguard.result import AuditRecord, RedactionResult, Span
|
|
7
|
+
from spanguard.streaming import StreamBufferFullError, StreamRedactor
|
|
8
|
+
from spanguard.tokenize import Keyring
|
|
9
|
+
|
|
10
|
+
__version__ = "0.1.0"
|
|
11
|
+
__all__ = [
|
|
12
|
+
"AuditRecord",
|
|
13
|
+
"Keyring",
|
|
14
|
+
"Policy",
|
|
15
|
+
"Recognizer",
|
|
16
|
+
"RedactionResult",
|
|
17
|
+
"Redactor",
|
|
18
|
+
"Span",
|
|
19
|
+
"StreamBufferFullError",
|
|
20
|
+
"StreamRedactor",
|
|
21
|
+
]
|
spanguard/_keys.py
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"""Resolve secret keys from explicit args, env, or a fresh random fallback."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import secrets
|
|
7
|
+
import warnings
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def resolve_key(explicit: bytes | None, env_var: str) -> bytes:
|
|
11
|
+
"""Return the explicit key, else the env var's bytes, else a random key.
|
|
12
|
+
|
|
13
|
+
The key is never logged. A random fallback is ephemeral (per process),
|
|
14
|
+
which makes tokenize/hash reversible/comparable only within that process
|
|
15
|
+
unless a stable key is supplied.
|
|
16
|
+
"""
|
|
17
|
+
if explicit is not None:
|
|
18
|
+
return explicit
|
|
19
|
+
env = os.environ.get(env_var)
|
|
20
|
+
if env:
|
|
21
|
+
return env.encode()
|
|
22
|
+
warnings.warn(
|
|
23
|
+
f"No key provided and {env_var} is unset; using an ephemeral random key. "
|
|
24
|
+
"Hashes/tokens will not be reproducible across processes and reveal() will "
|
|
25
|
+
"only work within this process. Set the key explicitly or via the env var "
|
|
26
|
+
"for stable output.",
|
|
27
|
+
stacklevel=2,
|
|
28
|
+
)
|
|
29
|
+
return secrets.token_bytes(32)
|
spanguard/_overlap.py
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""Deterministic overlap resolution: longest-wins, priority tiebreak."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from bisect import bisect_left
|
|
6
|
+
|
|
7
|
+
from spanguard.result import Span
|
|
8
|
+
|
|
9
|
+
PRIORITY: dict[str, int] = {
|
|
10
|
+
"SSN": 0,
|
|
11
|
+
"CREDIT_CARD": 0,
|
|
12
|
+
"API_KEY": 0,
|
|
13
|
+
"EMAIL": 1,
|
|
14
|
+
"PHONE": 1,
|
|
15
|
+
"MRN": 1,
|
|
16
|
+
"DOB": 1,
|
|
17
|
+
"PERSON": 2,
|
|
18
|
+
"LOCATION": 2,
|
|
19
|
+
}
|
|
20
|
+
_DEFAULT_PRIORITY = 3
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _rank(span: Span) -> tuple[int, int, int]:
|
|
24
|
+
"""Sort key for 'best' span: longer first, then higher priority, then earlier."""
|
|
25
|
+
return (-span.length, PRIORITY.get(span.type, _DEFAULT_PRIORITY), span.start)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def resolve_overlaps(spans: list[Span]) -> list[Span]:
|
|
29
|
+
"""Greedily keep the best span, rejecting any that overlap a chosen one.
|
|
30
|
+
|
|
31
|
+
Best = longest; ties broken by priority (lower rank wins), then start.
|
|
32
|
+
Overlap is direct/pairwise (half-open [start, end) intervals), NOT
|
|
33
|
+
transitive: a span that overlaps only an already-rejected span is still
|
|
34
|
+
kept. Returns the surviving spans sorted by start.
|
|
35
|
+
"""
|
|
36
|
+
# Index the chosen non-empty spans as parallel start/end lists sorted by
|
|
37
|
+
# start. They are pairwise disjoint, so their ends are sorted too, and a
|
|
38
|
+
# candidate [s, e) overlaps one of them iff the last one starting before e
|
|
39
|
+
# ends after s: one bisect per span, O(n log n) instead of O(n^2).
|
|
40
|
+
# Empty spans rank last (shortest) and can only be followed by other empty
|
|
41
|
+
# spans, which they never overlap, so they are never indexed.
|
|
42
|
+
starts: list[int] = []
|
|
43
|
+
ends: list[int] = []
|
|
44
|
+
chosen: list[Span] = []
|
|
45
|
+
for span in sorted(spans, key=_rank):
|
|
46
|
+
i = bisect_left(starts, span.end)
|
|
47
|
+
if i and ends[i - 1] > span.start:
|
|
48
|
+
continue # overlaps an already-chosen (better) span
|
|
49
|
+
chosen.append(span)
|
|
50
|
+
if span.end > span.start:
|
|
51
|
+
starts.insert(i, span.start)
|
|
52
|
+
ends.insert(i, span.end)
|
|
53
|
+
return sorted(chosen, key=lambda s: s.start)
|
spanguard/_transform.py
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"""Apply resolved spans to text according to the redaction mode."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Callable
|
|
6
|
+
|
|
7
|
+
from spanguard.result import Span
|
|
8
|
+
|
|
9
|
+
# Modes that require the raw matched text to compute their replacement.
|
|
10
|
+
_RAW_MODES = frozenset({"hash", "tokenize"})
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _replacement(
|
|
14
|
+
span: Span,
|
|
15
|
+
raw: str | None,
|
|
16
|
+
mode: str,
|
|
17
|
+
mask_char: str,
|
|
18
|
+
replace_raw: Callable[[Span, str], str] | None,
|
|
19
|
+
) -> str:
|
|
20
|
+
if mode == "mask":
|
|
21
|
+
return mask_char * span.length
|
|
22
|
+
if mode == "placeholder":
|
|
23
|
+
return f"[{span.type}]"
|
|
24
|
+
if mode in _RAW_MODES:
|
|
25
|
+
if replace_raw is None or raw is None: # pragma: no cover - guarded by caller
|
|
26
|
+
raise ValueError(f"mode {mode!r} requires a replace_raw callback")
|
|
27
|
+
return replace_raw(span, raw)
|
|
28
|
+
raise NotImplementedError(f"unknown mode {mode!r}")
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def apply_spans(
|
|
32
|
+
text: str,
|
|
33
|
+
spans: list[Span],
|
|
34
|
+
mode: str,
|
|
35
|
+
mask_char: str,
|
|
36
|
+
replace_raw: Callable[[Span, str], str] | None = None,
|
|
37
|
+
) -> str:
|
|
38
|
+
"""Rebuild `text` with each span replaced per `mode`. Spans must not overlap.
|
|
39
|
+
|
|
40
|
+
The raw matched slice is passed to `replace_raw` ONLY for raw-requiring
|
|
41
|
+
modes (hash/tokenize); mask/placeholder never receive raw text.
|
|
42
|
+
"""
|
|
43
|
+
if not spans:
|
|
44
|
+
return text
|
|
45
|
+
ordered = sorted(spans, key=lambda s: s.start)
|
|
46
|
+
out: list[str] = []
|
|
47
|
+
cursor = 0
|
|
48
|
+
for span in ordered:
|
|
49
|
+
out.append(text[cursor : span.start])
|
|
50
|
+
raw = text[span.start : span.end] if mode in _RAW_MODES else None
|
|
51
|
+
out.append(_replacement(span, raw, mode, mask_char, replace_raw))
|
|
52
|
+
cursor = span.end
|
|
53
|
+
out.append(text[cursor:])
|
|
54
|
+
return "".join(out)
|
spanguard/cli.py
ADDED
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
"""Command-line interface: `spanguard scan` and `spanguard redact`."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import json
|
|
7
|
+
import sys
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
from spanguard.policy import DEFAULT_ENTITIES, Policy
|
|
11
|
+
from spanguard.redactor import Redactor
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class _UsageError(Exception):
|
|
15
|
+
"""A user-facing error: printed as ``error: <msg>`` with exit code 2.
|
|
16
|
+
|
|
17
|
+
Messages name paths and entity labels only, never input content.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _entities(value: str | None) -> tuple[str, ...]:
|
|
22
|
+
if not value:
|
|
23
|
+
return DEFAULT_ENTITIES
|
|
24
|
+
entities = tuple(e.strip().upper() for e in value.split(",") if e.strip())
|
|
25
|
+
unknown = [e for e in entities if e not in DEFAULT_ENTITIES]
|
|
26
|
+
if unknown or not entities:
|
|
27
|
+
# Silently skipping a typo would leave that type unredacted.
|
|
28
|
+
raise _UsageError(
|
|
29
|
+
f"unknown entity type(s): {', '.join(unknown) or '(none given)'}; "
|
|
30
|
+
f"choose from {', '.join(DEFAULT_ENTITIES)}"
|
|
31
|
+
)
|
|
32
|
+
return entities
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
36
|
+
parser = argparse.ArgumentParser(
|
|
37
|
+
prog="spanguard",
|
|
38
|
+
description="Detect and redact PII/PHI in text files.",
|
|
39
|
+
)
|
|
40
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
41
|
+
|
|
42
|
+
scan = sub.add_parser("scan", help="Report detected entity counts (no raw values).")
|
|
43
|
+
scan.add_argument("file", help="Path to a UTF-8 text file to scan ('-' for stdin).")
|
|
44
|
+
scan.add_argument("--entities", help="Comma-separated entity types (default: all).")
|
|
45
|
+
scan.add_argument("--json", action="store_true", help="Emit the audit record as JSON.")
|
|
46
|
+
|
|
47
|
+
red = sub.add_parser("redact", help="Write redacted text.")
|
|
48
|
+
red.add_argument("file", help="Path to a UTF-8 text file to redact ('-' for stdin).")
|
|
49
|
+
red.add_argument(
|
|
50
|
+
"--mode",
|
|
51
|
+
choices=["mask", "placeholder", "hash", "tokenize"],
|
|
52
|
+
default="mask",
|
|
53
|
+
)
|
|
54
|
+
red.add_argument("--entities", help="Comma-separated entity types (default: all).")
|
|
55
|
+
red.add_argument("--out", help="Write redacted text here (default: stdout).")
|
|
56
|
+
red.add_argument(
|
|
57
|
+
"--audit-json", action="store_true", help="Emit the audit record as JSON to stderr."
|
|
58
|
+
)
|
|
59
|
+
return parser
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _read_input(path: str) -> str:
|
|
63
|
+
"""Read `path` (or stdin for ``-``) as UTF-8, mapping failures to usage errors."""
|
|
64
|
+
label = "<stdin>" if path == "-" else path
|
|
65
|
+
try:
|
|
66
|
+
if path == "-":
|
|
67
|
+
# None under `<&-` or pythonw; a closed stream would raise ValueError.
|
|
68
|
+
if sys.stdin is None or sys.stdin.closed:
|
|
69
|
+
raise _UsageError("cannot read <stdin>: stdin is not available")
|
|
70
|
+
# Decode the raw bytes ourselves: the console/pipe default (e.g.
|
|
71
|
+
# cp1252 on Windows) would silently mis-decode UTF-8 input.
|
|
72
|
+
buffer = getattr(sys.stdin, "buffer", None)
|
|
73
|
+
data = buffer.read() if buffer is not None else sys.stdin.read().encode("utf-8")
|
|
74
|
+
return bytes(data).decode("utf-8")
|
|
75
|
+
return Path(path).read_text(encoding="utf-8")
|
|
76
|
+
except FileNotFoundError:
|
|
77
|
+
raise _UsageError(f"file not found: {label}") from None
|
|
78
|
+
except UnicodeDecodeError:
|
|
79
|
+
raise _UsageError(f"cannot decode {label} as UTF-8") from None
|
|
80
|
+
except OSError as exc:
|
|
81
|
+
raise _UsageError(f"cannot read {label}: {exc.strerror or type(exc).__name__}") from None
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _cmd_scan(args: argparse.Namespace) -> int:
|
|
85
|
+
policy = Policy(mode="placeholder", entities=_entities(args.entities))
|
|
86
|
+
text = _read_input(args.file)
|
|
87
|
+
result = Redactor(policy).redact(text)
|
|
88
|
+
if args.json:
|
|
89
|
+
print(json.dumps(
|
|
90
|
+
{"counts": result.audit.counts,
|
|
91
|
+
"policy_hash": result.audit.policy_hash,
|
|
92
|
+
"ts": result.audit.ts}
|
|
93
|
+
))
|
|
94
|
+
else:
|
|
95
|
+
if result.audit.counts:
|
|
96
|
+
for etype, count in sorted(result.audit.counts.items()):
|
|
97
|
+
print(f"{etype}: {count}")
|
|
98
|
+
else:
|
|
99
|
+
print("No entities detected.")
|
|
100
|
+
return 0
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _cmd_redact(args: argparse.Namespace) -> int:
|
|
104
|
+
policy = Policy(mode=args.mode, entities=_entities(args.entities))
|
|
105
|
+
text = _read_input(args.file)
|
|
106
|
+
result = Redactor(policy).redact(text)
|
|
107
|
+
if args.out:
|
|
108
|
+
try:
|
|
109
|
+
Path(args.out).write_text(result.text, encoding="utf-8")
|
|
110
|
+
except OSError as exc:
|
|
111
|
+
raise _UsageError(
|
|
112
|
+
f"cannot write {args.out}: {exc.strerror or type(exc).__name__}"
|
|
113
|
+
) from None
|
|
114
|
+
else:
|
|
115
|
+
sys.stdout.write(result.text)
|
|
116
|
+
if not result.text.endswith("\n"):
|
|
117
|
+
sys.stdout.write("\n")
|
|
118
|
+
if args.audit_json:
|
|
119
|
+
sys.stderr.write(json.dumps(
|
|
120
|
+
{"counts": result.audit.counts,
|
|
121
|
+
"policy_hash": result.audit.policy_hash,
|
|
122
|
+
"ts": result.audit.ts}
|
|
123
|
+
) + "\n")
|
|
124
|
+
return 0
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def main(argv: list[str] | None = None) -> int:
|
|
128
|
+
for stream in (sys.stdout, sys.stderr):
|
|
129
|
+
reconfigure = getattr(stream, "reconfigure", None)
|
|
130
|
+
if reconfigure is not None:
|
|
131
|
+
reconfigure(encoding="utf-8", errors="backslashreplace")
|
|
132
|
+
|
|
133
|
+
parser = build_parser()
|
|
134
|
+
args = parser.parse_args(argv)
|
|
135
|
+
try:
|
|
136
|
+
if args.command == "scan":
|
|
137
|
+
return _cmd_scan(args)
|
|
138
|
+
return _cmd_redact(args)
|
|
139
|
+
except _UsageError as exc:
|
|
140
|
+
print(f"error: {exc}", file=sys.stderr)
|
|
141
|
+
return 2
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
if __name__ == "__main__": # pragma: no cover
|
|
145
|
+
raise SystemExit(main())
|
spanguard/policy.py
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
"""Redaction policy: what to detect and how to transform it."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
from typing import Literal
|
|
7
|
+
|
|
8
|
+
from pydantic import BaseModel, ConfigDict
|
|
9
|
+
|
|
10
|
+
RedactionMode = Literal["mask", "placeholder", "hash", "tokenize"]
|
|
11
|
+
|
|
12
|
+
DEFAULT_ENTITIES: tuple[str, ...] = (
|
|
13
|
+
"EMAIL",
|
|
14
|
+
"PHONE",
|
|
15
|
+
"SSN",
|
|
16
|
+
"MRN",
|
|
17
|
+
"DOB",
|
|
18
|
+
"CREDIT_CARD",
|
|
19
|
+
"API_KEY",
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class Policy(BaseModel):
|
|
24
|
+
"""Immutable, serializable redaction policy."""
|
|
25
|
+
|
|
26
|
+
model_config = ConfigDict(frozen=True)
|
|
27
|
+
|
|
28
|
+
mode: RedactionMode = "mask"
|
|
29
|
+
entities: tuple[str, ...] = DEFAULT_ENTITIES
|
|
30
|
+
mask_char: str = "█"
|
|
31
|
+
|
|
32
|
+
def policy_hash(self) -> str:
|
|
33
|
+
"""Stable content-addressed hash of the policy for the audit record."""
|
|
34
|
+
canonical = self.model_dump_json()
|
|
35
|
+
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()[:16]
|
spanguard/py.typed
ADDED
|
File without changes
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
"""Recognizers: pluggable detectors for PII/PHI entity types."""
|
|
2
|
+
|
|
3
|
+
from .base import Recognizer
|
|
4
|
+
from .luhn import luhn_valid
|
|
5
|
+
from .ner import SpacyRecognizer
|
|
6
|
+
from .regex_recognizers import DEFAULT_RECOGNIZERS
|
|
7
|
+
|
|
8
|
+
__all__ = ["DEFAULT_RECOGNIZERS", "Recognizer", "SpacyRecognizer", "luhn_valid"]
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"""The Recognizer protocol: pluggable detectors."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Protocol, runtime_checkable
|
|
6
|
+
|
|
7
|
+
from spanguard.result import Span
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@runtime_checkable
|
|
11
|
+
class Recognizer(Protocol):
|
|
12
|
+
"""A detector for one entity type. Users implement this to add detectors.
|
|
13
|
+
|
|
14
|
+
Contract: `entity_type` (and the `type` field of every `Span` returned)
|
|
15
|
+
MUST be a fixed, non-sensitive entity *label* (e.g. ``"EMAIL"``,
|
|
16
|
+
``"PHONE"``), never a value derived from the matched text. The label
|
|
17
|
+
appears verbatim in placeholder output (``[EMAIL]``) and as a key in the
|
|
18
|
+
audit record's ``counts``; putting matched content in it would leak raw
|
|
19
|
+
data into redacted output and audit records.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
entity_type: str
|
|
23
|
+
"""Fixed entity label for this recognizer. See class contract above:
|
|
24
|
+
never derive this from matched text."""
|
|
25
|
+
|
|
26
|
+
def recognize(self, text: str) -> list[Span]:
|
|
27
|
+
"""Return non-overlapping spans of `entity_type` found in `text`."""
|
|
28
|
+
...
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""Luhn checksum for candidate credit-card numbers."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def luhn_valid(digits: str) -> bool:
|
|
7
|
+
"""True iff `digits` (bare digits, no separators) passes the Luhn check."""
|
|
8
|
+
if not digits or not digits.isdigit():
|
|
9
|
+
return False
|
|
10
|
+
total = 0
|
|
11
|
+
reverse = digits[::-1]
|
|
12
|
+
for i, ch in enumerate(reverse):
|
|
13
|
+
d = ord(ch) - 48
|
|
14
|
+
if i % 2 == 1:
|
|
15
|
+
d *= 2
|
|
16
|
+
if d > 9:
|
|
17
|
+
d -= 9
|
|
18
|
+
total += d
|
|
19
|
+
return total % 10 == 0
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""Optional spaCy NER recognizer (PERSON/LOCATION). Requires the [ner] extra."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from spanguard.result import Span
|
|
8
|
+
|
|
9
|
+
_DEFAULT_LABEL_MAP = {"PERSON": "PERSON", "GPE": "LOCATION", "LOC": "LOCATION"}
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class SpacyRecognizer:
|
|
13
|
+
"""Detect PERSON/LOCATION entities via spaCy NER.
|
|
14
|
+
|
|
15
|
+
spaCy is imported lazily so the base install stays light and `import
|
|
16
|
+
spanguard` works without the [ner] extra. Pass `_nlp` to inject a loaded
|
|
17
|
+
pipeline (used by tests) and skip the model load entirely.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
# This recognizer emits multiple mapped types (PERSON/LOCATION); entity_type
|
|
21
|
+
# is a generic label satisfying the Recognizer protocol, not a per-span type.
|
|
22
|
+
entity_type = "NER"
|
|
23
|
+
|
|
24
|
+
def __init__(
|
|
25
|
+
self,
|
|
26
|
+
model: str = "en_core_web_sm",
|
|
27
|
+
label_map: dict[str, str] | None = None,
|
|
28
|
+
_nlp: Any | None = None,
|
|
29
|
+
) -> None:
|
|
30
|
+
self.model = model
|
|
31
|
+
self.label_map = dict(label_map) if label_map is not None else dict(_DEFAULT_LABEL_MAP)
|
|
32
|
+
self._nlp = _nlp
|
|
33
|
+
|
|
34
|
+
def _load(self) -> Any:
|
|
35
|
+
if self._nlp is None:
|
|
36
|
+
try:
|
|
37
|
+
import spacy
|
|
38
|
+
except ModuleNotFoundError as exc:
|
|
39
|
+
if exc.name != "spacy":
|
|
40
|
+
raise
|
|
41
|
+
raise ImportError(
|
|
42
|
+
"SpacyRecognizer requires the optional 'ner' extra: "
|
|
43
|
+
"pip install spanguard[ner], then download a model "
|
|
44
|
+
"(python -m spacy download en_core_web_sm)."
|
|
45
|
+
) from exc
|
|
46
|
+
self._nlp = spacy.load(self.model) # pragma: no cover - env-dependent
|
|
47
|
+
return self._nlp
|
|
48
|
+
|
|
49
|
+
def recognize(self, text: str) -> list[Span]:
|
|
50
|
+
nlp = self._load()
|
|
51
|
+
doc = nlp(text)
|
|
52
|
+
spans: list[Span] = []
|
|
53
|
+
for ent in doc.ents:
|
|
54
|
+
mapped = self.label_map.get(ent.label_)
|
|
55
|
+
if mapped is not None:
|
|
56
|
+
spans.append(Span(type=mapped, start=ent.start_char, end=ent.end_char))
|
|
57
|
+
return spans
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
"""Regex- and checksum-based recognizers for common PII/PHI entity types."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
from typing import ClassVar
|
|
7
|
+
|
|
8
|
+
from spanguard.result import Span
|
|
9
|
+
|
|
10
|
+
from .base import Recognizer
|
|
11
|
+
from .luhn import luhn_valid
|
|
12
|
+
|
|
13
|
+
# Email: Unicode local part and domain labels; a single @ and a dotted TLD of
|
|
14
|
+
# >= 2 Unicode LETTERS (`[^\W\d_]`). Bracket/quote/comma/semicolon/colon chars
|
|
15
|
+
# are excluded from the local+domain classes so the match neither gloms onto
|
|
16
|
+
# surrounding punctuation nor is dropped when glued to a preceding `)`/`]`.
|
|
17
|
+
#
|
|
18
|
+
# Matching is kept linear on long whitespace-free runs (e.g. base64 blobs):
|
|
19
|
+
# the local part is possessive (`++`, safe because its class excludes `@`),
|
|
20
|
+
# and a search may only start at the beginning of a run of local-part chars.
|
|
21
|
+
# Any match starting mid-run is dominated by one starting earlier in the same
|
|
22
|
+
# run, EXCEPT right where a previous match ended mid-run -- that position is
|
|
23
|
+
# tried with the anchored variant (see EmailRecognizer.recognize). Together
|
|
24
|
+
# these reproduce `finditer` over the plain pattern exactly.
|
|
25
|
+
_EMAIL_LOCAL = r"[^\s@()<>\[\]{}\",;:]"
|
|
26
|
+
_EMAIL_ANCHORED = re.compile(
|
|
27
|
+
rf"{_EMAIL_LOCAL}++@{_EMAIL_LOCAL}+\.[^\W\d_]{{2,}}",
|
|
28
|
+
re.UNICODE,
|
|
29
|
+
)
|
|
30
|
+
_EMAIL = re.compile(rf"(?<!{_EMAIL_LOCAL}){_EMAIL_ANCHORED.pattern}", re.UNICODE)
|
|
31
|
+
# NANP-style: optional +country, 3-3-4 with -, ., space, or parens on area code.
|
|
32
|
+
_PHONE = re.compile(
|
|
33
|
+
r"(?<!\d)(?:\+\d{1,2}[ .\-]?)?(?:\(\d{3}\)|\d{3})[ .\-]\d{3}[ .\-]\d{4}(?!\d)"
|
|
34
|
+
)
|
|
35
|
+
_SSN = re.compile(r"(?<!\d)\d{3}-\d{2}-\d{4}(?!\d)")
|
|
36
|
+
_MRN = re.compile(r"\bMRN\s*:?\s*(\d{6,})\b", re.IGNORECASE)
|
|
37
|
+
# DOB: mm/dd/yyyy allowing 1- or 2-digit month/day (validated by range),
|
|
38
|
+
# or ISO yyyy-mm-dd.
|
|
39
|
+
_DOB = re.compile(
|
|
40
|
+
r"(?<!\d)(?:(0?[1-9]|1[0-2])/(0?[1-9]|[12]\d|3[01])/(\d{4})"
|
|
41
|
+
r"|(\d{4})-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01]))(?!\d)"
|
|
42
|
+
)
|
|
43
|
+
# A credit card: a contiguous 13-19 digit run, OR standard printed groupings
|
|
44
|
+
# (4-4-4-4 / 4-4-4-3 for Visa/MC/Discover, 4-6-5 for Amex) separated by a single
|
|
45
|
+
# space or dash. Grouping-awareness prevents merging adjacent numbers (e.g. a
|
|
46
|
+
# phone followed by a card) into one bogus run. Each candidate is Luhn-checked.
|
|
47
|
+
_CARD = re.compile(
|
|
48
|
+
r"""
|
|
49
|
+
(?<![\d-])
|
|
50
|
+
(?:
|
|
51
|
+
\d{13,19}
|
|
52
|
+
| \d{4}(?:[ -]\d{4}){2}[ -]\d{1,4}
|
|
53
|
+
| \d{4}[ -]\d{6}[ -]\d{5}
|
|
54
|
+
)
|
|
55
|
+
(?![\d-])
|
|
56
|
+
""",
|
|
57
|
+
re.VERBOSE,
|
|
58
|
+
)
|
|
59
|
+
# API keys: known prefixes, or a long high-entropy alnum run.
|
|
60
|
+
_API_KEY = re.compile(
|
|
61
|
+
r"\b(?:sk-[A-Za-z0-9]{20,}"
|
|
62
|
+
r"|AKIA[0-9A-Z]{16}"
|
|
63
|
+
r"|ghp_[A-Za-z0-9]{36}"
|
|
64
|
+
r"|[A-Za-z0-9]{32,})\b"
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
class _RegexRecognizer:
|
|
69
|
+
"""Shared base: emit one Span per full-match of `pattern`."""
|
|
70
|
+
|
|
71
|
+
entity_type: str
|
|
72
|
+
pattern: ClassVar[re.Pattern[str]]
|
|
73
|
+
|
|
74
|
+
def recognize(self, text: str) -> list[Span]:
|
|
75
|
+
return [
|
|
76
|
+
Span(type=self.entity_type, start=m.start(), end=m.end())
|
|
77
|
+
for m in self.pattern.finditer(text)
|
|
78
|
+
]
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
class EmailRecognizer:
|
|
82
|
+
"""Emails, matched in linear time (see the `_EMAIL` comment)."""
|
|
83
|
+
|
|
84
|
+
entity_type = "EMAIL"
|
|
85
|
+
|
|
86
|
+
def recognize(self, text: str) -> list[Span]:
|
|
87
|
+
spans: list[Span] = []
|
|
88
|
+
pos = 0
|
|
89
|
+
while True:
|
|
90
|
+
m = None
|
|
91
|
+
# A previous match ended inside a run: an email may start right here.
|
|
92
|
+
if spans:
|
|
93
|
+
m = _EMAIL_ANCHORED.match(text, pos)
|
|
94
|
+
if m is None:
|
|
95
|
+
m = _EMAIL.search(text, pos)
|
|
96
|
+
if m is None:
|
|
97
|
+
return spans
|
|
98
|
+
spans.append(Span(type="EMAIL", start=m.start(), end=m.end()))
|
|
99
|
+
pos = m.end()
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
class PhoneRecognizer(_RegexRecognizer):
|
|
103
|
+
entity_type = "PHONE"
|
|
104
|
+
pattern = _PHONE
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
class SSNRecognizer(_RegexRecognizer):
|
|
108
|
+
entity_type = "SSN"
|
|
109
|
+
pattern = _SSN
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
class DOBRecognizer(_RegexRecognizer):
|
|
113
|
+
entity_type = "DOB"
|
|
114
|
+
pattern = _DOB
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
class MRNRecognizer:
|
|
118
|
+
"""MRN needs the 'MRN' context word; redact only the digit run."""
|
|
119
|
+
|
|
120
|
+
entity_type = "MRN"
|
|
121
|
+
|
|
122
|
+
def recognize(self, text: str) -> list[Span]:
|
|
123
|
+
spans: list[Span] = []
|
|
124
|
+
for m in _MRN.finditer(text):
|
|
125
|
+
spans.append(Span(type="MRN", start=m.start(1), end=m.end(1)))
|
|
126
|
+
return spans
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
class APIKeyRecognizer(_RegexRecognizer):
|
|
130
|
+
entity_type = "API_KEY"
|
|
131
|
+
pattern = _API_KEY
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
class CreditCardRecognizer:
|
|
135
|
+
"""Credit cards: contiguous or standard-grouped digit sequences, Luhn-valid."""
|
|
136
|
+
|
|
137
|
+
entity_type = "CREDIT_CARD"
|
|
138
|
+
|
|
139
|
+
def recognize(self, text: str) -> list[Span]:
|
|
140
|
+
spans: list[Span] = []
|
|
141
|
+
for m in _CARD.finditer(text):
|
|
142
|
+
digits = re.sub(r"[ -]", "", m.group())
|
|
143
|
+
if 13 <= len(digits) <= 19 and luhn_valid(digits):
|
|
144
|
+
spans.append(Span(type="CREDIT_CARD", start=m.start(), end=m.end()))
|
|
145
|
+
return spans
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
DEFAULT_RECOGNIZERS: dict[str, Recognizer] = {
|
|
149
|
+
"EMAIL": EmailRecognizer(),
|
|
150
|
+
"PHONE": PhoneRecognizer(),
|
|
151
|
+
"SSN": SSNRecognizer(),
|
|
152
|
+
"MRN": MRNRecognizer(),
|
|
153
|
+
"DOB": DOBRecognizer(),
|
|
154
|
+
"CREDIT_CARD": CreditCardRecognizer(),
|
|
155
|
+
"API_KEY": APIKeyRecognizer(),
|
|
156
|
+
}
|
spanguard/redactor.py
ADDED
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
"""The Redactor: run recognizers, resolve overlaps, transform, audit."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import warnings
|
|
6
|
+
|
|
7
|
+
from spanguard._keys import resolve_key
|
|
8
|
+
from spanguard._overlap import resolve_overlaps
|
|
9
|
+
from spanguard._transform import apply_spans
|
|
10
|
+
from spanguard.policy import Policy
|
|
11
|
+
from spanguard.recognizers.base import Recognizer
|
|
12
|
+
from spanguard.recognizers.regex_recognizers import DEFAULT_RECOGNIZERS
|
|
13
|
+
from spanguard.result import AuditRecord, RedactionResult, Span
|
|
14
|
+
from spanguard.tokenize import Keyring, hmac_digest
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class Redactor:
|
|
18
|
+
"""Detect and redact PII/PHI in text according to a `Policy`."""
|
|
19
|
+
|
|
20
|
+
def __init__(
|
|
21
|
+
self,
|
|
22
|
+
policy: Policy,
|
|
23
|
+
recognizers: list[Recognizer] | None = None,
|
|
24
|
+
*,
|
|
25
|
+
hmac_key: bytes | None = None,
|
|
26
|
+
tokenize_key: bytes | None = None,
|
|
27
|
+
) -> None:
|
|
28
|
+
self.policy = policy
|
|
29
|
+
if recognizers is None:
|
|
30
|
+
missing = [e for e in policy.entities if e not in DEFAULT_RECOGNIZERS]
|
|
31
|
+
if missing:
|
|
32
|
+
# Labels only (never text), so this cannot leak raw values.
|
|
33
|
+
warnings.warn(
|
|
34
|
+
f"Policy entities with no built-in recognizer are not detected: "
|
|
35
|
+
f"{', '.join(missing)}. Pass recognizers=[...] (e.g. SpacyRecognizer "
|
|
36
|
+
f"for PERSON/LOCATION) to detect them.",
|
|
37
|
+
stacklevel=2,
|
|
38
|
+
)
|
|
39
|
+
recognizers = [
|
|
40
|
+
DEFAULT_RECOGNIZERS[e]
|
|
41
|
+
for e in policy.entities
|
|
42
|
+
if e in DEFAULT_RECOGNIZERS
|
|
43
|
+
]
|
|
44
|
+
self.recognizers = recognizers
|
|
45
|
+
self._hmac_key_arg = hmac_key
|
|
46
|
+
self._tokenize_key_arg = tokenize_key
|
|
47
|
+
self._hmac_key_cached: bytes | None = None
|
|
48
|
+
self._keyring_cached: Keyring | None = None
|
|
49
|
+
|
|
50
|
+
@property
|
|
51
|
+
def _hmac_key(self) -> bytes:
|
|
52
|
+
if self._hmac_key_cached is None:
|
|
53
|
+
self._hmac_key_cached = resolve_key(self._hmac_key_arg, "SPANGUARD_HMAC_KEY")
|
|
54
|
+
return self._hmac_key_cached
|
|
55
|
+
|
|
56
|
+
@property
|
|
57
|
+
def keyring(self) -> Keyring:
|
|
58
|
+
if self._keyring_cached is None:
|
|
59
|
+
self._keyring_cached = Keyring(
|
|
60
|
+
resolve_key(self._tokenize_key_arg, "SPANGUARD_TOKENIZE_KEY")
|
|
61
|
+
)
|
|
62
|
+
return self._keyring_cached
|
|
63
|
+
|
|
64
|
+
def _detect(self, text: str) -> list[Span]:
|
|
65
|
+
spans: list[Span] = []
|
|
66
|
+
for rec in self.recognizers:
|
|
67
|
+
spans.extend(rec.recognize(text))
|
|
68
|
+
return resolve_overlaps(spans)
|
|
69
|
+
|
|
70
|
+
def _replace_raw(self, span: Span, raw: str) -> str:
|
|
71
|
+
if self.policy.mode == "hash":
|
|
72
|
+
return f"[{span.type}:{hmac_digest(self._hmac_key, span.type, raw)}]"
|
|
73
|
+
if self.policy.mode == "tokenize":
|
|
74
|
+
return self.keyring.tokenize(span.type, raw)
|
|
75
|
+
raise ValueError(f"no raw replacement for mode {self.policy.mode!r}")
|
|
76
|
+
|
|
77
|
+
def redact(self, text: str) -> RedactionResult:
|
|
78
|
+
return self._redact_with_spans(text, self._detect(text))
|
|
79
|
+
|
|
80
|
+
def _redact_with_spans(self, text: str, spans: list[Span]) -> RedactionResult:
|
|
81
|
+
"""Redact `text` using already-resolved `spans` (from `_detect(text)`)."""
|
|
82
|
+
redacted = apply_spans(
|
|
83
|
+
text,
|
|
84
|
+
spans,
|
|
85
|
+
mode=self.policy.mode,
|
|
86
|
+
mask_char=self.policy.mask_char,
|
|
87
|
+
replace_raw=self._replace_raw,
|
|
88
|
+
)
|
|
89
|
+
audit = AuditRecord.from_spans(spans, policy_hash=self.policy.policy_hash())
|
|
90
|
+
return RedactionResult(text=redacted, audit=audit, spans=spans)
|
|
91
|
+
|
|
92
|
+
def reveal(self, text: str, keyring: Keyring | None = None) -> str:
|
|
93
|
+
"""Restore tokenized values using `keyring` (defaults to this Redactor's)."""
|
|
94
|
+
return (keyring or self.keyring).reveal(text)
|
spanguard/result.py
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"""Value types for redaction results and leak-free audit records."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections import Counter
|
|
6
|
+
from dataclasses import dataclass, field
|
|
7
|
+
from datetime import UTC, datetime
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@dataclass(frozen=True, slots=True)
|
|
11
|
+
class Span:
|
|
12
|
+
"""A detected sensitive region: entity `type` over half-open [start, end).
|
|
13
|
+
|
|
14
|
+
`type` is a fixed entity label (e.g. ``"EMAIL"``), never the matched
|
|
15
|
+
value — it appears verbatim in the audit record's ``counts`` and in
|
|
16
|
+
placeholder output.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
type: str
|
|
20
|
+
start: int
|
|
21
|
+
end: int
|
|
22
|
+
|
|
23
|
+
@property
|
|
24
|
+
def length(self) -> int:
|
|
25
|
+
return self.end - self.start
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass(frozen=True, slots=True)
|
|
29
|
+
class AuditRecord:
|
|
30
|
+
"""Per-operation audit trail. Contains counts only — never raw values."""
|
|
31
|
+
|
|
32
|
+
counts: dict[str, int]
|
|
33
|
+
policy_hash: str
|
|
34
|
+
ts: str
|
|
35
|
+
|
|
36
|
+
@classmethod
|
|
37
|
+
def from_spans(cls, spans: list[Span], policy_hash: str) -> AuditRecord:
|
|
38
|
+
counts = dict(Counter(s.type for s in spans))
|
|
39
|
+
ts = datetime.now(UTC).isoformat()
|
|
40
|
+
return cls(counts=counts, policy_hash=policy_hash, ts=ts)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@dataclass(frozen=True, slots=True)
|
|
44
|
+
class RedactionResult:
|
|
45
|
+
"""Result of a redaction: transformed `text`, its `audit`, and the `spans`."""
|
|
46
|
+
|
|
47
|
+
text: str
|
|
48
|
+
audit: AuditRecord
|
|
49
|
+
spans: list[Span] = field(default_factory=list)
|
spanguard/streaming.py
ADDED
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
"""Chunked streaming redaction that preserves entities across chunk boundaries."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
|
|
7
|
+
from spanguard.redactor import Redactor
|
|
8
|
+
from spanguard.result import Span
|
|
9
|
+
|
|
10
|
+
_DEFAULT_TAIL = 128
|
|
11
|
+
_MIN_TAIL = 64
|
|
12
|
+
_WHITESPACE = re.compile(r"\s")
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class StreamBufferFullError(Exception):
|
|
16
|
+
"""Raised by ``StreamRedactor.feed`` when the held-back buffer exceeds
|
|
17
|
+
``max_buffer`` and no point to emit up to is safe yet.
|
|
18
|
+
|
|
19
|
+
The chunk passed to that ``feed()`` has already been accepted, so do not
|
|
20
|
+
feed it again. The buffer is kept intact: call ``flush()`` to redact and
|
|
21
|
+
emit everything held so far (an entity still arriving may then be split),
|
|
22
|
+
or discard the redactor.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class StreamRedactor:
|
|
27
|
+
"""Redact a text arriving in chunks, holding back a tail so an entity split
|
|
28
|
+
across a boundary is still redacted as one.
|
|
29
|
+
|
|
30
|
+
Guarantee: for any chunking, the concatenation of all ``feed()`` outputs
|
|
31
|
+
plus ``flush()`` equals ``redactor.redact(full_text).text`` -- for every
|
|
32
|
+
mode (mask, placeholder, hash, tokenize) with the same keys.
|
|
33
|
+
|
|
34
|
+
Memory: text is emitted only up to a whitespace character that no entity
|
|
35
|
+
straddles, so while the stream keeps supplying such separators the buffer
|
|
36
|
+
stays near ``tail`` plus one chunk. A stream without one (a long base64
|
|
37
|
+
blob, or whitespace only inside a phone number) is held until one arrives
|
|
38
|
+
or ``flush()`` is called. Pass ``max_buffer`` to cap that: a ``feed()``
|
|
39
|
+
that leaves more than ``max_buffer`` characters held and can emit nothing
|
|
40
|
+
raises ``StreamBufferFullError``. (A feed that does emit may still leave
|
|
41
|
+
the buffer over the cap; the next feed that cannot emit raises.)
|
|
42
|
+
|
|
43
|
+
``tail`` must be >= ``_MIN_TAIL`` (64),
|
|
44
|
+
which is enforced. An entity longer than ``tail`` with no internal
|
|
45
|
+
separators (e.g. an email or an API key) degrades gracefully to a
|
|
46
|
+
word-boundary split. The floor exists specifically to prevent a
|
|
47
|
+
sub-threshold leak of entities that DO contain internal separators (e.g.
|
|
48
|
+
a spaced credit card number): below the floor, the buffer never holds
|
|
49
|
+
enough of the entity for detection to fire, so every whitespace cut looks
|
|
50
|
+
safe and the raw value streams out unmasked in pieces.
|
|
51
|
+
"""
|
|
52
|
+
|
|
53
|
+
def __init__(
|
|
54
|
+
self,
|
|
55
|
+
redactor: Redactor,
|
|
56
|
+
tail: int = _DEFAULT_TAIL,
|
|
57
|
+
max_buffer: int | None = None,
|
|
58
|
+
) -> None:
|
|
59
|
+
if tail < _MIN_TAIL:
|
|
60
|
+
raise ValueError(
|
|
61
|
+
f"tail must be >= {_MIN_TAIL} to guarantee no raw value leaks; "
|
|
62
|
+
f"a smaller tail can let a multi-token entity (e.g. a credit card) "
|
|
63
|
+
f"stream out in sub-threshold pieces that never match. Got tail={tail}."
|
|
64
|
+
)
|
|
65
|
+
if max_buffer is not None and max_buffer <= tail:
|
|
66
|
+
raise ValueError(
|
|
67
|
+
f"max_buffer must be greater than tail ({tail}); got max_buffer={max_buffer}."
|
|
68
|
+
)
|
|
69
|
+
self._redactor = redactor
|
|
70
|
+
self._tail = tail
|
|
71
|
+
self._max_buffer = max_buffer
|
|
72
|
+
self._buffer = ""
|
|
73
|
+
# Cut candidates in buffer[:_tried_upto] were already tried and none
|
|
74
|
+
# was safe; only whitespace past it warrants another detection pass.
|
|
75
|
+
self._tried_upto = 0
|
|
76
|
+
|
|
77
|
+
@property
|
|
78
|
+
def pending(self) -> int:
|
|
79
|
+
return len(self._buffer)
|
|
80
|
+
|
|
81
|
+
def _safe_cut(self) -> tuple[int, list[Span]]:
|
|
82
|
+
"""Largest separator index that provably reproduces the one-shot result.
|
|
83
|
+
|
|
84
|
+
A cut at ``c`` (just after a whitespace char, ``c <= len - tail``) is
|
|
85
|
+
safe iff detecting the prefix ``buffer[:c]`` yields exactly the
|
|
86
|
+
whole-buffer spans ending at or before ``c``, AND detecting the suffix
|
|
87
|
+
``buffer[c:]`` yields exactly the whole-buffer spans starting at or
|
|
88
|
+
after ``c`` (offsets shifted by ``-c``). This behavioral check -- not a
|
|
89
|
+
positional guess -- is what makes "split == whole" hold even for
|
|
90
|
+
entities that contain internal separators or need left context.
|
|
91
|
+
|
|
92
|
+
Returns ``(cut, prefix_spans)`` -- the verified detection result for
|
|
93
|
+
``buffer[:cut]``, so the caller can emit without re-detecting -- or
|
|
94
|
+
``(0, [])`` when no cut is safe yet (nothing may be emitted).
|
|
95
|
+
"""
|
|
96
|
+
buffer = self._buffer
|
|
97
|
+
limit = len(buffer) - self._tail
|
|
98
|
+
# Cut points sit just after a whitespace char (`\s` is exactly
|
|
99
|
+
# str.isspace()), latest first. Candidates already tried are retried
|
|
100
|
+
# only once new whitespace arrives past them, so a stream with no
|
|
101
|
+
# usable separator costs O(n) total rather than a detection pass per
|
|
102
|
+
# feed. Output is unchanged either way; only emission timing can lag.
|
|
103
|
+
limit = max(limit, 0)
|
|
104
|
+
start, self._tried_upto = self._tried_upto, limit
|
|
105
|
+
if _WHITESPACE.search(buffer, start, limit) is None:
|
|
106
|
+
return 0, [] # no new cut candidate: skip detection entirely
|
|
107
|
+
candidates = [m.end() for m in _WHITESPACE.finditer(buffer, 0, limit)][::-1]
|
|
108
|
+
full = self._redactor._detect(buffer)
|
|
109
|
+
# Worst case re-detects prefix and suffix per candidate; in practice the
|
|
110
|
+
# first candidate no span straddles is accepted (~3 detection passes).
|
|
111
|
+
for c in candidates:
|
|
112
|
+
if any(s.start < c < s.end for s in full):
|
|
113
|
+
continue # a span straddles this cut
|
|
114
|
+
pre_expected = [s for s in full if s.end <= c]
|
|
115
|
+
suf_expected = [Span(s.type, s.start - c, s.end - c) for s in full if s.start >= c]
|
|
116
|
+
if self._redactor._detect(buffer[:c]) != pre_expected:
|
|
117
|
+
continue
|
|
118
|
+
if self._redactor._detect(buffer[c:]) != suf_expected:
|
|
119
|
+
continue
|
|
120
|
+
return c, pre_expected
|
|
121
|
+
return 0, []
|
|
122
|
+
|
|
123
|
+
def feed(self, chunk: str) -> str:
|
|
124
|
+
self._buffer += chunk
|
|
125
|
+
cut, spans = self._safe_cut()
|
|
126
|
+
if cut == 0 and self._max_buffer is not None and len(self._buffer) > self._max_buffer:
|
|
127
|
+
# A skipped (already-tried) cut may have become safe since; retry
|
|
128
|
+
# every candidate before reporting that none exists.
|
|
129
|
+
self._tried_upto = 0
|
|
130
|
+
cut, spans = self._safe_cut()
|
|
131
|
+
if cut == 0:
|
|
132
|
+
raise StreamBufferFullError(
|
|
133
|
+
f"stream buffer holds {len(self._buffer)} characters, over "
|
|
134
|
+
f"max_buffer={self._max_buffer}, with no safe point to emit up to"
|
|
135
|
+
)
|
|
136
|
+
if cut == 0:
|
|
137
|
+
return ""
|
|
138
|
+
emit_src = self._buffer[:cut]
|
|
139
|
+
self._buffer = self._buffer[cut:]
|
|
140
|
+
self._tried_upto = 0
|
|
141
|
+
return self._redactor._redact_with_spans(emit_src, spans).text
|
|
142
|
+
|
|
143
|
+
def flush(self) -> str:
|
|
144
|
+
if not self._buffer:
|
|
145
|
+
return ""
|
|
146
|
+
out = self._redactor.redact(self._buffer).text
|
|
147
|
+
self._buffer = ""
|
|
148
|
+
self._tried_upto = 0
|
|
149
|
+
return out
|
spanguard/tokenize.py
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
"""Deterministic HMAC hashing and reversible tokenization with a keyring."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hmac
|
|
6
|
+
import re
|
|
7
|
+
from hashlib import sha256
|
|
8
|
+
|
|
9
|
+
_DIGEST_LEN = 16
|
|
10
|
+
_TOKEN_SHAPE = re.compile("⟦[^⟦⟧]*⟧")
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def hmac_digest(key: bytes, entity_type: str, raw: str) -> str:
|
|
14
|
+
"""Deterministic HMAC-SHA256 digest (first 16 hex chars) over type:raw."""
|
|
15
|
+
msg = f"{entity_type}:{raw}".encode()
|
|
16
|
+
return hmac.new(key, msg, sha256).hexdigest()[:_DIGEST_LEN]
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def make_token(entity_type: str, digest: str) -> str:
|
|
20
|
+
"""Stable token wrapper for a digest."""
|
|
21
|
+
return f"⟦{entity_type}_{digest}⟧"
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class Keyring:
|
|
25
|
+
"""Holds the tokenize key and an in-memory token->raw map for reveal().
|
|
26
|
+
|
|
27
|
+
The key and the map are never exposed via repr/str, so a Keyring can be
|
|
28
|
+
logged without leaking secrets or raw values.
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
def __init__(self, tokenize_key: bytes) -> None:
|
|
32
|
+
self._key = tokenize_key
|
|
33
|
+
self._map: dict[str, str] = {}
|
|
34
|
+
# Set when a label contains a token bracket, which defeats the
|
|
35
|
+
# bracket-delimited fast path in reveal().
|
|
36
|
+
self._bracketed_label = False
|
|
37
|
+
|
|
38
|
+
def tokenize(self, entity_type: str, raw: str) -> str:
|
|
39
|
+
digest = hmac_digest(self._key, entity_type, raw)
|
|
40
|
+
token = make_token(entity_type, digest)
|
|
41
|
+
if "⟦" in entity_type or "⟧" in entity_type:
|
|
42
|
+
self._bracketed_label = True
|
|
43
|
+
self._map[token] = raw
|
|
44
|
+
return token
|
|
45
|
+
|
|
46
|
+
def reveal_token(self, token: str) -> str | None:
|
|
47
|
+
return self._map.get(token)
|
|
48
|
+
|
|
49
|
+
def reveal(self, text: str) -> str:
|
|
50
|
+
"""Replace every token this keyring issued with its original value.
|
|
51
|
+
|
|
52
|
+
Substitution is a single left-to-right pass (one re.sub over an
|
|
53
|
+
alternation of all issued tokens), so a raw value that happens to
|
|
54
|
+
contain another issued token's string is never re-expanded.
|
|
55
|
+
"""
|
|
56
|
+
if not self._map:
|
|
57
|
+
return text
|
|
58
|
+
if not self._bracketed_label:
|
|
59
|
+
# Every issued token is ⟦...⟧ with no inner brackets, so scanning
|
|
60
|
+
# bracket-delimited candidates and looking each up is equivalent
|
|
61
|
+
# to the alternation below, in O(len(text)) regardless of map size.
|
|
62
|
+
return _TOKEN_SHAPE.sub(lambda m: self._map.get(m[0], m[0]), text)
|
|
63
|
+
pattern = re.compile("|".join(re.escape(tok) for tok in self._map))
|
|
64
|
+
return pattern.sub(lambda m: self._map[m[0]], text)
|
|
65
|
+
|
|
66
|
+
@property
|
|
67
|
+
def tokens(self) -> frozenset[str]:
|
|
68
|
+
return frozenset(self._map)
|
|
69
|
+
|
|
70
|
+
def __repr__(self) -> str:
|
|
71
|
+
return f"Keyring(tokens={len(self._map)})"
|
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: spanguard
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Detect and redact PII/PHI in text and streaming transcripts, with leak-free audit records.
|
|
5
|
+
Project-URL: Homepage, https://github.com/Clufy/spanguard
|
|
6
|
+
Project-URL: Documentation, https://clufy.github.io/spanguard
|
|
7
|
+
Project-URL: Issues, https://github.com/Clufy/spanguard/issues
|
|
8
|
+
Project-URL: Changelog, https://github.com/Clufy/spanguard/blob/master/CHANGELOG.md
|
|
9
|
+
Author: spanguard authors
|
|
10
|
+
License-Expression: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: compliance,phi,pii,privacy,redaction
|
|
13
|
+
Classifier: Development Status :: 3 - Alpha
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Operating System :: OS Independent
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Topic :: Security
|
|
19
|
+
Classifier: Topic :: Text Processing
|
|
20
|
+
Classifier: Typing :: Typed
|
|
21
|
+
Requires-Python: >=3.12
|
|
22
|
+
Requires-Dist: pydantic<3,>=2.6
|
|
23
|
+
Provides-Extra: docs
|
|
24
|
+
Requires-Dist: mkdocs-material>=9; extra == 'docs'
|
|
25
|
+
Requires-Dist: mkdocstrings[python]>=0.24; extra == 'docs'
|
|
26
|
+
Provides-Extra: ner
|
|
27
|
+
Requires-Dist: spacy<4,>=3.7; extra == 'ner'
|
|
28
|
+
Description-Content-Type: text/markdown
|
|
29
|
+
|
|
30
|
+
# spanguard
|
|
31
|
+
|
|
32
|
+
**Redact PII/PHI from text and streams — with an audit trail that can never leak the data it's auditing.**
|
|
33
|
+
|
|
34
|
+
[](https://github.com/Clufy/spanguard/actions/workflows/ci.yml)
|
|
35
|
+
[](https://pypi.org/project/spanguard/)
|
|
36
|
+
[](https://github.com/Clufy/spanguard/actions/workflows/ci.yml)
|
|
37
|
+
[](https://clufy.github.io/spanguard)
|
|
38
|
+
[](https://www.python.org/downloads/)
|
|
39
|
+
[](https://mypy-lang.org/)
|
|
40
|
+
[](https://github.com/Clufy/spanguard/blob/master/LICENSE)
|
|
41
|
+
|
|
42
|
+
spanguard detects and redacts PII/PHI (personally identifiable information and
|
|
43
|
+
protected health information) in text and streaming transcripts. Every
|
|
44
|
+
`Redactor.redact()` call returns a `RedactionResult` whose `audit` field holds
|
|
45
|
+
only entity-type counts, a policy hash, and a timestamp — **never the raw
|
|
46
|
+
matched values** — so the audit trail itself can never become a second place
|
|
47
|
+
where sensitive data leaks out. That guarantee is enforced by the test suite,
|
|
48
|
+
not just documented.
|
|
49
|
+
|
|
50
|
+
## Contents
|
|
51
|
+
|
|
52
|
+
- [Why spanguard](#why-spanguard)
|
|
53
|
+
- [Install](#install)
|
|
54
|
+
- [60-second quickstart](#60-second-quickstart)
|
|
55
|
+
- [Streaming](#streaming)
|
|
56
|
+
- [CLI](#cli)
|
|
57
|
+
- [Built-in entity table](#built-in-entity-table)
|
|
58
|
+
- [Compliance / audit guarantee](#compliance--audit-guarantee)
|
|
59
|
+
- [Docs](#docs)
|
|
60
|
+
- [Contributing](#contributing)
|
|
61
|
+
- [Security](#security)
|
|
62
|
+
- [License](#license)
|
|
63
|
+
|
|
64
|
+
## Why spanguard
|
|
65
|
+
|
|
66
|
+
- **Leak-free audit record** — `result.audit` carries entity-type counts, a
|
|
67
|
+
policy hash, and a timestamp only; raw matches never enter it, and tests
|
|
68
|
+
verify this holds across every mode.
|
|
69
|
+
- **Four redaction modes** — `mask`, `placeholder`, `hash` (deterministic
|
|
70
|
+
HMAC), and `tokenize` (reversible).
|
|
71
|
+
- **Reversible tokenization** — turn tokens back into the original text with
|
|
72
|
+
`Redactor.reveal()` and an in-memory `Keyring`, scoped to a single process.
|
|
73
|
+
- **Built-in detectors** — email (Unicode-aware), phone, SSN, MRN, DOB, and
|
|
74
|
+
Luhn-validated credit card numbers, plus API key patterns.
|
|
75
|
+
- **Optional NER** — spaCy-backed `PERSON`/`LOCATION` detection behind the
|
|
76
|
+
`[ner]` extra, off by default so the core install stays light.
|
|
77
|
+
- **Streaming-safe** — `StreamRedactor` redacts chunked input (e.g. a live
|
|
78
|
+
transcript) so the output matches a one-shot redaction, even when an entity
|
|
79
|
+
straddles a chunk boundary.
|
|
80
|
+
- **Pluggable recognizers** — register a custom `Recognizer` for domain-specific
|
|
81
|
+
entities without forking the library.
|
|
82
|
+
- **Typed and tested** — mypy `--strict` and ruff clean, 128 tests, 97.9%
|
|
83
|
+
coverage.
|
|
84
|
+
- **CLI included** — `spanguard scan` / `spanguard redact` for one-off files
|
|
85
|
+
and shell pipelines.
|
|
86
|
+
|
|
87
|
+
## Install
|
|
88
|
+
|
|
89
|
+
```bash
|
|
90
|
+
pip install spanguard
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
For the optional NER (`PERSON`/`LOCATION`) recognizer:
|
|
94
|
+
|
|
95
|
+
```bash
|
|
96
|
+
pip install spanguard[ner]
|
|
97
|
+
python -m spacy download en_core_web_sm
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
NER is opt-in per `Redactor`: pass the recognizer alongside the built-ins.
|
|
101
|
+
|
|
102
|
+
```python
|
|
103
|
+
from spanguard import Policy, Redactor
|
|
104
|
+
from spanguard.recognizers import DEFAULT_RECOGNIZERS, SpacyRecognizer
|
|
105
|
+
|
|
106
|
+
redactor = Redactor(Policy(), recognizers=[*DEFAULT_RECOGNIZERS.values(), SpacyRecognizer()])
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
## 60-second quickstart
|
|
110
|
+
|
|
111
|
+
```python
|
|
112
|
+
from spanguard import Redactor, Policy
|
|
113
|
+
|
|
114
|
+
text = "Contact: jane@example.com or 555-123-4567."
|
|
115
|
+
|
|
116
|
+
# --- mask: replace each character with mask_char, preserving length ---
|
|
117
|
+
mask_policy = Policy(mode="mask", entities=("EMAIL", "PHONE"))
|
|
118
|
+
mask_result = Redactor(mask_policy).redact(text)
|
|
119
|
+
print(mask_result.text)
|
|
120
|
+
# Contact: ████████████████ or ████████████.
|
|
121
|
+
print(mask_result.audit.counts)
|
|
122
|
+
# {'EMAIL': 1, 'PHONE': 1}
|
|
123
|
+
|
|
124
|
+
# --- placeholder: replace with a [TYPE] tag ---
|
|
125
|
+
placeholder_policy = Policy(mode="placeholder", entities=("EMAIL", "PHONE"))
|
|
126
|
+
placeholder_result = Redactor(placeholder_policy).redact(text)
|
|
127
|
+
print(placeholder_result.text)
|
|
128
|
+
# Contact: [EMAIL] or [PHONE].
|
|
129
|
+
|
|
130
|
+
# --- hash: deterministic HMAC digest, reproducible with the same key ---
|
|
131
|
+
hash_policy = Policy(mode="hash", entities=("EMAIL", "PHONE"))
|
|
132
|
+
hash_redactor = Redactor(hash_policy, hmac_key=b"docs-example-key")
|
|
133
|
+
hash_result = hash_redactor.redact(text)
|
|
134
|
+
print(hash_result.text)
|
|
135
|
+
# Contact: [EMAIL:11d041e6c58a0530] or [PHONE:cbe886551f4e27ed].
|
|
136
|
+
|
|
137
|
+
# --- tokenize + reveal: reversible via an in-memory keyring ---
|
|
138
|
+
tokenize_policy = Policy(mode="tokenize", entities=("EMAIL", "PHONE"))
|
|
139
|
+
tokenize_redactor = Redactor(tokenize_policy, tokenize_key=b"docs-example-key")
|
|
140
|
+
tokenize_result = tokenize_redactor.redact(text)
|
|
141
|
+
print(tokenize_result.text)
|
|
142
|
+
# Contact: ⟦EMAIL_11d041e6c58a0530⟧ or ⟦PHONE_cbe886551f4e27ed⟧.
|
|
143
|
+
|
|
144
|
+
revealed = tokenize_redactor.reveal(tokenize_result.text)
|
|
145
|
+
print(revealed == text)
|
|
146
|
+
# True
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
See the [Quickstart docs](https://clufy.github.io/spanguard/quickstart/)
|
|
150
|
+
for notes on keys, the audit record, and `result.spans`.
|
|
151
|
+
|
|
152
|
+
## Streaming
|
|
153
|
+
|
|
154
|
+
```python
|
|
155
|
+
from spanguard import Redactor, Policy, StreamRedactor
|
|
156
|
+
|
|
157
|
+
redactor = Redactor(Policy(mode="mask", entities=("EMAIL", "PHONE")))
|
|
158
|
+
stream = StreamRedactor(redactor)
|
|
159
|
+
|
|
160
|
+
output = stream.feed("Contact: jane@example.com ") + stream.feed("or 555-123-4567.")
|
|
161
|
+
output += stream.flush()
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
`StreamRedactor` holds back a tail of the buffer (minimum 64 characters) so an
|
|
165
|
+
entity split across a chunk boundary is still detected as one; the
|
|
166
|
+
concatenation of every `feed()` plus the final `flush()` is guaranteed to
|
|
167
|
+
equal a one-shot `redactor.redact(full_text).text`.
|
|
168
|
+
|
|
169
|
+
Text is emitted only up to a safe split point (whitespace outside any
|
|
170
|
+
entity), so a stream with none, such as a long base64 blob, is held in memory.
|
|
171
|
+
Pass `StreamRedactor(redactor, max_buffer=N)` to get a `StreamBufferFullError`
|
|
172
|
+
instead of unbounded growth. See
|
|
173
|
+
[Streaming](https://clufy.github.io/spanguard/streaming/).
|
|
174
|
+
|
|
175
|
+
## CLI
|
|
176
|
+
|
|
177
|
+
```bash
|
|
178
|
+
spanguard scan file.txt
|
|
179
|
+
spanguard redact file.txt --mode mask
|
|
180
|
+
spanguard redact file.txt --mode tokenize --out redacted.txt
|
|
181
|
+
cat file.txt | spanguard redact - --mode placeholder # '-' reads stdin
|
|
182
|
+
```
|
|
183
|
+
|
|
184
|
+
`scan` reports entity-type counts only (no raw values); `redact` writes the
|
|
185
|
+
redacted text and, with `--audit-json`, the audit record to stderr. Bad input
|
|
186
|
+
(an unknown `--entities` name, a missing or non-UTF-8 file) exits with code 2
|
|
187
|
+
and a one-line `error:` message that never echoes input content. See
|
|
188
|
+
[CLI](https://clufy.github.io/spanguard/cli/).
|
|
189
|
+
|
|
190
|
+
## Built-in entity table
|
|
191
|
+
|
|
192
|
+
| Entity type | Detector | Notes |
|
|
193
|
+
|---------------|------------------------------------|--------------------------------------------------------------------|
|
|
194
|
+
| `EMAIL` | regex | Unicode local part and domain; requires a dotted TLD of >= 2 letters. |
|
|
195
|
+
| `PHONE` | regex | NANP-style 3-3-4, optional `+country`, `-`/`.`/space/parens separators. |
|
|
196
|
+
| `SSN` | regex | `###-##-####`. |
|
|
197
|
+
| `MRN` | regex (context-anchored) | Requires an `MRN` prefix; only the digit run is redacted. |
|
|
198
|
+
| `DOB` | regex | `mm/dd/yyyy` (1- or 2-digit month/day) or ISO `yyyy-mm-dd`. |
|
|
199
|
+
| `CREDIT_CARD` | regex + Luhn checksum | Contiguous 13-19 digits, or standard printed groupings; Luhn-validated. |
|
|
200
|
+
| `API_KEY` | regex | Known prefixes (`sk-`, `AKIA`, `ghp_`) or a long high-entropy alnum run. |
|
|
201
|
+
| `PERSON`, `LOCATION` | spaCy NER (optional `[ner]` extra) | Requires the `[ner]` extra, a spaCy model, and passing `SpacyRecognizer()` (see [Install](#install)). |
|
|
202
|
+
|
|
203
|
+
Four redaction modes are available on every `Policy`: `mask`, `placeholder`,
|
|
204
|
+
`hash`, and `tokenize` (reversible via `Redactor.reveal()`). See
|
|
205
|
+
[Recognizers](https://clufy.github.io/spanguard/recognizers/) for the
|
|
206
|
+
full detail and how to plug in a custom `Recognizer`.
|
|
207
|
+
|
|
208
|
+
## Compliance / audit guarantee
|
|
209
|
+
|
|
210
|
+
`result.audit` (an `AuditRecord`) is designed to be safe to log, ship to a
|
|
211
|
+
SIEM, or hand to an auditor without a second review: it contains entity-type
|
|
212
|
+
**counts**, a **policy hash** (so you can prove which redaction policy ran),
|
|
213
|
+
and a **timestamp** — and structurally nothing else. No raw matched value, no
|
|
214
|
+
span offsets, no context window. The test suite asserts this property directly
|
|
215
|
+
so a future change that widens the audit record would fail CI, not just review.
|
|
216
|
+
|
|
217
|
+
## Docs
|
|
218
|
+
|
|
219
|
+
Full documentation, including the API reference, streaming redaction, and the
|
|
220
|
+
CLI, is at [clufy.github.io/spanguard](https://clufy.github.io/spanguard).
|
|
221
|
+
|
|
222
|
+
## Contributing
|
|
223
|
+
|
|
224
|
+
Bug reports and pull requests are welcome — see
|
|
225
|
+
[CONTRIBUTING.md](https://github.com/Clufy/spanguard/blob/master/CONTRIBUTING.md) for how to set up a dev environment and
|
|
226
|
+
run the test suite.
|
|
227
|
+
|
|
228
|
+
## Security
|
|
229
|
+
|
|
230
|
+
To report a vulnerability, please see [SECURITY.md](https://github.com/Clufy/spanguard/blob/master/SECURITY.md) rather than
|
|
231
|
+
opening a public issue.
|
|
232
|
+
|
|
233
|
+
## License
|
|
234
|
+
|
|
235
|
+
MIT — see [LICENSE](https://github.com/Clufy/spanguard/blob/master/LICENSE).
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
spanguard/__init__.py,sha256=cDdM_iTmnwDsvJdl48MUvsdrU0qVNTpDrgwtmhJnOlU,578
|
|
2
|
+
spanguard/_keys.py,sha256=ALRpk3x3DPhRr2lC8nashz7HBgi7MeVhOF7AZ3N9gwY,989
|
|
3
|
+
spanguard/_overlap.py,sha256=oerQogEcZee-CaPdIFXZrtAb_cMU1ffIO31-gwXIsCk,1927
|
|
4
|
+
spanguard/_transform.py,sha256=s59lWT0HooZZPDR0OQLtyprXV1U1sXmDcY3wwM_pBeU,1683
|
|
5
|
+
spanguard/cli.py,sha256=yWxe2DwmPRWThIMYN_4i_iMyzULXrbQtfxp8uzdyiGc,5379
|
|
6
|
+
spanguard/policy.py,sha256=PqT9QAzB-9Jif6B9L0zfBVrREsXLb4kygQik3f3KGdE,854
|
|
7
|
+
spanguard/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
8
|
+
spanguard/redactor.py,sha256=UPFCYGtlDI3O_Y2stnekrIlx-82f_k7q4m2pamgT_28,3692
|
|
9
|
+
spanguard/result.py,sha256=itBgD-66yCBOO7xf_Nw1aONFjtKTAfj6hSCQK-Nwirg,1337
|
|
10
|
+
spanguard/streaming.py,sha256=Ius82Ijc3DdAC5F6jcQzCL0CXn30HNceYuA2fZqXSGc,6815
|
|
11
|
+
spanguard/tokenize.py,sha256=8E_oTFT9vfDMTht_-estSajjYeDx0Q6Tf13A5VBD_oE,2603
|
|
12
|
+
spanguard/recognizers/__init__.py,sha256=vW8bvqpI0NRZY0tIEhKgSqzYPMvJaBSaLH0dk2wUgz0,290
|
|
13
|
+
spanguard/recognizers/base.py,sha256=U7CGyV_B813rwZ5W33eYcrYb8fnUpFiF7thD61GtA2A,1006
|
|
14
|
+
spanguard/recognizers/luhn.py,sha256=l4vKX9UGp3Px90W4xfd_9I7oC2KPBEuRtj9FXfHqZ8g,510
|
|
15
|
+
spanguard/recognizers/ner.py,sha256=b6SfP5Jx0ccIesfE5pyRMaTNd7jSUxmdkisdkXHv-4g,2021
|
|
16
|
+
spanguard/recognizers/regex_recognizers.py,sha256=t8BZwZhD0-hsdUC3i4lmyOnYhxQhj75QoXE0pvMdzgk,5004
|
|
17
|
+
spanguard-0.1.0.dist-info/METADATA,sha256=R5idIrFfWqF846KEQAfx0QGxqo2vXRSXoSKgReLoFCQ,10516
|
|
18
|
+
spanguard-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
19
|
+
spanguard-0.1.0.dist-info/entry_points.txt,sha256=DcGxztV3YXOE4N9fdUTC8JHfPMkVuiwvr4qJzYekX58,49
|
|
20
|
+
spanguard-0.1.0.dist-info/licenses/LICENSE,sha256=h7YUOU_WJFAP4W9WbZv2IbHwquJMioZBYqX1HlTPVfU,1074
|
|
21
|
+
spanguard-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 spanguard authors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|