gheim 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- gheim/__init__.py +41 -0
- gheim/core/__init__.py +12 -0
- gheim/core/sentinels.py +105 -0
- gheim/core/session.py +223 -0
- gheim/core/stream.py +118 -0
- gheim/detectors/__init__.py +14 -0
- gheim/detectors/base.py +48 -0
- gheim/detectors/composite.py +234 -0
- gheim/detectors/local.py +243 -0
- gheim/detectors/remote.py +98 -0
- gheim/google.py +11 -0
- gheim/normalizers.py +174 -0
- gheim/openai/__init__.py +42 -0
- gheim/openai/_anonymize_only.py +110 -0
- gheim/openai/_audio.py +174 -0
- gheim/openai/_base.py +170 -0
- gheim/openai/_chat.py +145 -0
- gheim/openai/_client.py +219 -0
- gheim/openai/_completions.py +95 -0
- gheim/openai/_responses.py +171 -0
- gheim/openai/_strict.py +137 -0
- gheim/plain.py +147 -0
- gheim-0.1.0.dist-info/METADATA +224 -0
- gheim-0.1.0.dist-info/RECORD +26 -0
- gheim-0.1.0.dist-info/WHEEL +4 -0
- gheim-0.1.0.dist-info/licenses/LICENSE +190 -0
gheim/__init__.py
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"""gheim — PII round-trip for LLM APIs.
|
|
2
|
+
|
|
3
|
+
Public API:
|
|
4
|
+
|
|
5
|
+
- ``Session`` — per-request state (sentinel mapping)
|
|
6
|
+
- ``LocalDetector`` / ``RemoteDetector`` — pluggable PII detection backends
|
|
7
|
+
- ``anonymize`` / ``deanonymize`` — message-level helpers
|
|
8
|
+
- ``anonymize_text`` / ``deanonymize_text`` — string-level helpers
|
|
9
|
+
- ``deanonymize_stream`` — wrap a chunk iterator
|
|
10
|
+
- ``gheim.openai.OpenAI`` — drop-in OpenAI client (subpackage import)
|
|
11
|
+
"""
|
|
12
|
+
from .core.session import Session
|
|
13
|
+
from .core.stream import StreamDeanonymizer
|
|
14
|
+
from .detectors import Detector, LocalDetector, RemoteDetector, Span, default_detector
|
|
15
|
+
from .plain import (
|
|
16
|
+
anonymize,
|
|
17
|
+
anonymize_messages,
|
|
18
|
+
anonymize_text,
|
|
19
|
+
deanonymize,
|
|
20
|
+
deanonymize_stream,
|
|
21
|
+
deanonymize_text,
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
__version__ = "0.1.0"
|
|
25
|
+
|
|
26
|
+
__all__ = [
|
|
27
|
+
"Detector",
|
|
28
|
+
"LocalDetector",
|
|
29
|
+
"RemoteDetector",
|
|
30
|
+
"Session",
|
|
31
|
+
"Span",
|
|
32
|
+
"StreamDeanonymizer",
|
|
33
|
+
"__version__",
|
|
34
|
+
"anonymize",
|
|
35
|
+
"anonymize_messages",
|
|
36
|
+
"anonymize_text",
|
|
37
|
+
"deanonymize",
|
|
38
|
+
"deanonymize_stream",
|
|
39
|
+
"deanonymize_text",
|
|
40
|
+
"default_detector",
|
|
41
|
+
]
|
gheim/core/__init__.py
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
from .sentinels import LABEL_TO_TAG, SENTINEL_RE, Sentinel, label_tag
|
|
2
|
+
from .session import Session
|
|
3
|
+
from .stream import StreamDeanonymizer
|
|
4
|
+
|
|
5
|
+
__all__ = [
|
|
6
|
+
"LABEL_TO_TAG",
|
|
7
|
+
"SENTINEL_RE",
|
|
8
|
+
"Sentinel",
|
|
9
|
+
"Session",
|
|
10
|
+
"StreamDeanonymizer",
|
|
11
|
+
"label_tag",
|
|
12
|
+
]
|
gheim/core/sentinels.py
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
"""Sentinel format, allocation, and regex matching.
|
|
2
|
+
|
|
3
|
+
A sentinel is a stable, uniquely-numbered placeholder like ``<PERSON_1>`` or
|
|
4
|
+
``<EMAIL_2>``. Each PII label maps to a tag (``private_person`` → ``PERSON``),
|
|
5
|
+
and each session allocates indices starting at 1 per tag.
|
|
6
|
+
|
|
7
|
+
The format is ASCII-only angle brackets so that downstream LLM tokenizers are
|
|
8
|
+
most likely to preserve it as-is. Brackets + uppercase letters + digits + one
|
|
9
|
+
underscore is a narrow enough shape that most producers won't mangle it.
|
|
10
|
+
"""
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import re
|
|
14
|
+
from collections.abc import Iterator
|
|
15
|
+
from dataclasses import dataclass
|
|
16
|
+
from typing import Annotated
|
|
17
|
+
|
|
18
|
+
# Canonical label → short tag used in the sentinel.
|
|
19
|
+
LABEL_TO_TAG: dict[str, str] = {
|
|
20
|
+
"private_person": "PERSON",
|
|
21
|
+
"private_email": "EMAIL",
|
|
22
|
+
"private_phone": "PHONE",
|
|
23
|
+
"private_address": "ADDRESS",
|
|
24
|
+
"private_url": "URL",
|
|
25
|
+
"private_date": "DATE",
|
|
26
|
+
"account_number": "ACCOUNT",
|
|
27
|
+
"secret": "SECRET",
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
# Reverse map for completeness (not strictly needed for round-trip).
|
|
31
|
+
TAG_TO_LABEL: dict[str, str] = {v: k for k, v in LABEL_TO_TAG.items()}
|
|
32
|
+
|
|
33
|
+
_TAG_PATTERN = r"[A-Z][A-Z0-9_]*"
|
|
34
|
+
_INDEX_PATTERN = r"\d+"
|
|
35
|
+
|
|
36
|
+
# A fully-formed sentinel anywhere in a string.
|
|
37
|
+
SENTINEL_RE = re.compile(rf"<(?P<tag>{_TAG_PATTERN})_(?P<index>{_INDEX_PATTERN})>")
|
|
38
|
+
|
|
39
|
+
# A *possibly-partial* sentinel anchored at the start of a buffer. Used by the
|
|
40
|
+
# streaming deanonymizer to decide "should I hold back these bytes or emit them?"
|
|
41
|
+
#
|
|
42
|
+
# Matches any strict prefix of a sentinel:
|
|
43
|
+
# "<", "<P", "<PERSON", "<PERSON_", "<PERSON_1", "<PERSON_12>"
|
|
44
|
+
# Does NOT match if the buffer contains disqualifying characters (e.g. "<x", "<PERSON ").
|
|
45
|
+
_PARTIAL_SENTINEL_RE = re.compile(
|
|
46
|
+
rf"^<({_TAG_PATTERN})?(_{_INDEX_PATTERN}?)?>?"
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def label_tag(
|
|
51
|
+
label: Annotated[str, "Detector label, e.g. 'private_person' or 'account_number'."],
|
|
52
|
+
) -> str:
|
|
53
|
+
"""Map a detector label to its sentinel tag. Unknown labels fall back to the upper-cased label."""
|
|
54
|
+
return LABEL_TO_TAG.get(label, label.upper())
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
@dataclass(frozen=True, slots=True)
|
|
58
|
+
class Sentinel:
|
|
59
|
+
"""A specific placeholder issued inside a session."""
|
|
60
|
+
|
|
61
|
+
tag: Annotated[str, "Short uppercase tag (e.g. 'PERSON')."]
|
|
62
|
+
index: Annotated[int, "1-based per-tag counter."]
|
|
63
|
+
|
|
64
|
+
def render(self) -> str:
|
|
65
|
+
return f"<{self.tag}_{self.index}>"
|
|
66
|
+
|
|
67
|
+
@classmethod
|
|
68
|
+
def parse(
|
|
69
|
+
cls,
|
|
70
|
+
s: Annotated[str, "Candidate sentinel string to parse, e.g. '<PERSON_7>'."],
|
|
71
|
+
) -> Sentinel | None:
|
|
72
|
+
m = SENTINEL_RE.fullmatch(s)
|
|
73
|
+
if not m:
|
|
74
|
+
return None
|
|
75
|
+
return cls(tag=m["tag"], index=int(m["index"]))
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def find_sentinels(
|
|
79
|
+
text: Annotated[str, "Text to scan for fully-formed sentinels."],
|
|
80
|
+
) -> Iterator[tuple[int, int, Sentinel]]:
|
|
81
|
+
"""Yield (start, end, sentinel) for each sentinel occurrence in ``text``."""
|
|
82
|
+
for m in SENTINEL_RE.finditer(text):
|
|
83
|
+
yield m.start(), m.end(), Sentinel(tag=m["tag"], index=int(m["index"]))
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def is_possible_sentinel_prefix(
|
|
87
|
+
buffer: Annotated[str, "Buffer accumulated since the last '<' was seen by the streaming buffer."],
|
|
88
|
+
) -> bool:
|
|
89
|
+
"""Could ``buffer`` still become a sentinel if more characters arrive?
|
|
90
|
+
|
|
91
|
+
Returns True if the buffer is a non-empty strict prefix of some valid sentinel
|
|
92
|
+
(i.e. consumable) AND does not already contain disqualifying characters.
|
|
93
|
+
A fully-formed sentinel (ending in ``>``) is NOT a prefix — caller should have
|
|
94
|
+
already matched and consumed it.
|
|
95
|
+
"""
|
|
96
|
+
if not buffer or buffer[0] != "<":
|
|
97
|
+
return False
|
|
98
|
+
m = _PARTIAL_SENTINEL_RE.match(buffer)
|
|
99
|
+
if m is None:
|
|
100
|
+
return False
|
|
101
|
+
# Must consume the whole buffer — otherwise there's trailing garbage.
|
|
102
|
+
if m.end() != len(buffer):
|
|
103
|
+
return False
|
|
104
|
+
# If it ends with ">" it's already complete — not a prefix.
|
|
105
|
+
return not buffer.endswith(">")
|
gheim/core/session.py
ADDED
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
"""Session state: tracks which sentinels map to which originals for one round-trip."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import json
|
|
5
|
+
import unicodedata
|
|
6
|
+
from collections.abc import Mapping
|
|
7
|
+
from dataclasses import dataclass, field
|
|
8
|
+
from typing import TYPE_CHECKING, Annotated, Any
|
|
9
|
+
|
|
10
|
+
from .sentinels import SENTINEL_RE, Sentinel, label_tag
|
|
11
|
+
|
|
12
|
+
if TYPE_CHECKING:
|
|
13
|
+
from ..detectors.base import Detector, Span
|
|
14
|
+
from ..normalizers import Normalizer
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
# Maximum number of whitespace characters allowed between two spans for them to
|
|
18
|
+
# count as "adjacent" and be merged. Covers the common BPE artifact where the
|
|
19
|
+
# tokenizer splits an email into two adjacent same-label spans.
|
|
20
|
+
_MERGE_GAP = 1
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _normalize_surface(s: str) -> str:
|
|
24
|
+
"""Normalize a surface form for sentinel-coalescing key purposes.
|
|
25
|
+
|
|
26
|
+
NFKC unicode normalization (so "fi" ligature == "fi"), Unicode-aware
|
|
27
|
+
casefolding (so "Joel" == "JOEL" == "joel" and "Müller" == "MÜLLER";
|
|
28
|
+
casefold also expands German "ß" → "ss"), then collapse all
|
|
29
|
+
whitespace runs to a single space and strip. The mapping still
|
|
30
|
+
stores the original surface so restoration preserves the first-seen
|
|
31
|
+
casing; only the lookup key is normalized.
|
|
32
|
+
"""
|
|
33
|
+
return " ".join(unicodedata.normalize("NFKC", s).casefold().split())
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _merge_adjacent(text: str, spans: list[Span]) -> list[tuple[int, int, str]]:
|
|
37
|
+
"""Merge runs of same-label spans that touch (or are separated by ≤ ``_MERGE_GAP``
|
|
38
|
+
whitespace characters)."""
|
|
39
|
+
out: list[tuple[int, int, str]] = []
|
|
40
|
+
for span in spans:
|
|
41
|
+
if span.start < 0 or span.end > len(text) or span.start >= span.end:
|
|
42
|
+
continue
|
|
43
|
+
if not out:
|
|
44
|
+
out.append((span.start, span.end, span.label))
|
|
45
|
+
continue
|
|
46
|
+
prev_start, prev_end, prev_label = out[-1]
|
|
47
|
+
if span.start < prev_end:
|
|
48
|
+
# Overlap — drop the later span (greedy first-wins).
|
|
49
|
+
continue
|
|
50
|
+
gap = text[prev_end : span.start]
|
|
51
|
+
if (
|
|
52
|
+
span.label == prev_label
|
|
53
|
+
and len(gap) <= _MERGE_GAP
|
|
54
|
+
and gap.strip() == ""
|
|
55
|
+
):
|
|
56
|
+
out[-1] = (prev_start, span.end, prev_label)
|
|
57
|
+
else:
|
|
58
|
+
out.append((span.start, span.end, span.label))
|
|
59
|
+
return out
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
@dataclass
|
|
63
|
+
class Session:
|
|
64
|
+
"""Mutable per-request state for anonymizing outbound text and restoring inbound text.
|
|
65
|
+
|
|
66
|
+
- ``mapping`` goes from rendered sentinel (``"<PERSON_1>"``) to the original surface form.
|
|
67
|
+
- Same (tag, surface) reuses the same sentinel so the LLM can refer back coherently
|
|
68
|
+
("Joel said Joel's email is ...").
|
|
69
|
+
|
|
70
|
+
Sessions are JSON-serializable so callers can persist them across requests if they want
|
|
71
|
+
long-lived conversations (same `Joel` keeps mapping to `<PERSON_1>` across turns).
|
|
72
|
+
"""
|
|
73
|
+
|
|
74
|
+
mapping: Annotated[
|
|
75
|
+
dict[str, str], "Rendered sentinel ('<PERSON_1>') → original surface form."
|
|
76
|
+
] = field(default_factory=dict)
|
|
77
|
+
_coalesce: dict[tuple[str, str], str] = field(default_factory=dict)
|
|
78
|
+
_counters: dict[str, int] = field(default_factory=dict)
|
|
79
|
+
detector: Annotated[
|
|
80
|
+
Detector | None, "Optional default detector reused across calls in this session."
|
|
81
|
+
] = None
|
|
82
|
+
normalizers: Annotated[
|
|
83
|
+
Mapping[str, "str | Normalizer"] | None,
|
|
84
|
+
"Optional per-label canonical-form normalizers, e.g. "
|
|
85
|
+
"``{'private_phone': 'e164', 'private_date': 'iso_date'}`` or "
|
|
86
|
+
"``{'private_phone': e164(region='CH')}``. A label not listed "
|
|
87
|
+
"here uses the default NFKC+casefold key. See "
|
|
88
|
+
"``gheim.normalizers``.",
|
|
89
|
+
] = None
|
|
90
|
+
_resolved_normalizers: dict[str, "Normalizer"] = field(
|
|
91
|
+
default_factory=dict, init=False, repr=False,
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
def __post_init__(self) -> None:
|
|
95
|
+
if self.normalizers:
|
|
96
|
+
from ..normalizers import resolve_normalizer
|
|
97
|
+
self._resolved_normalizers = {
|
|
98
|
+
label: resolve_normalizer(spec)
|
|
99
|
+
for label, spec in self.normalizers.items()
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
def allocate(
|
|
103
|
+
self,
|
|
104
|
+
label: Annotated[str, "Detector label (e.g. 'private_person')."],
|
|
105
|
+
surface: Annotated[str, "Original surface form to be hidden behind a sentinel."],
|
|
106
|
+
) -> str:
|
|
107
|
+
"""Return (or allocate) the sentinel for this (label, surface) pair.
|
|
108
|
+
|
|
109
|
+
The coalesce key uses a normalized form of ``surface`` (NFKC +
|
|
110
|
+
casefold + collapsed whitespace) so trivial variants of the same
|
|
111
|
+
identity collapse to one sentinel: ``Joel`` / ``JOEL`` / ``joel``,
|
|
112
|
+
``Müller`` / ``MÜLLER``, ``"alice@example.com"`` / ``"ALICE@EXAMPLE.COM"``.
|
|
113
|
+
When a per-label normalizer is configured (via ``normalizers=``)
|
|
114
|
+
it overrides the default key for that label — used to collapse
|
|
115
|
+
e.g. phone-format variants under E.164. If the normalizer
|
|
116
|
+
returns ``None`` (un-parseable), the default NFKC+casefold key
|
|
117
|
+
is used as a safe fallback so the surface still gets a sentinel.
|
|
118
|
+
|
|
119
|
+
The mapping stores the original (first-seen) surface so
|
|
120
|
+
restoration preserves casing.
|
|
121
|
+
"""
|
|
122
|
+
tag = label_tag(label)
|
|
123
|
+
norm_fn = self._resolved_normalizers.get(label)
|
|
124
|
+
normalized: str | None = None
|
|
125
|
+
if norm_fn is not None:
|
|
126
|
+
normalized = norm_fn(surface)
|
|
127
|
+
if normalized is None:
|
|
128
|
+
normalized = _normalize_surface(surface)
|
|
129
|
+
key = (tag, normalized)
|
|
130
|
+
existing = self._coalesce.get(key)
|
|
131
|
+
if existing is not None:
|
|
132
|
+
return existing
|
|
133
|
+
self._counters[tag] = self._counters.get(tag, 0) + 1
|
|
134
|
+
sentinel = Sentinel(tag=tag, index=self._counters[tag]).render()
|
|
135
|
+
self._coalesce[key] = sentinel
|
|
136
|
+
self.mapping[sentinel] = surface
|
|
137
|
+
return sentinel
|
|
138
|
+
|
|
139
|
+
def apply_spans(
|
|
140
|
+
self,
|
|
141
|
+
text: Annotated[str, "Original text the spans were detected in."],
|
|
142
|
+
spans: Annotated[list[Span], "Detected PII spans, possibly unsorted."],
|
|
143
|
+
) -> str:
|
|
144
|
+
"""Replace each span in ``text`` with an allocated sentinel. Left-to-right, non-overlapping.
|
|
145
|
+
|
|
146
|
+
Overlapping spans are resolved by keeping the earliest-starting, longest span.
|
|
147
|
+
Adjacent same-label spans (``alice@example`` + ``.com``) are merged before
|
|
148
|
+
allocation so a split email doesn't end up as two separate sentinels.
|
|
149
|
+
"""
|
|
150
|
+
if not spans:
|
|
151
|
+
return text
|
|
152
|
+
sorted_spans = sorted(spans, key=lambda s: (s.start, -(s.end - s.start)))
|
|
153
|
+
merged = _merge_adjacent(text, sorted_spans)
|
|
154
|
+
out: list[str] = []
|
|
155
|
+
cursor = 0
|
|
156
|
+
for span_start, span_end, label in merged:
|
|
157
|
+
if span_start < cursor:
|
|
158
|
+
continue
|
|
159
|
+
if span_start < 0 or span_end > len(text) or span_start >= span_end:
|
|
160
|
+
continue
|
|
161
|
+
surface = text[span_start:span_end]
|
|
162
|
+
out.append(text[cursor:span_start])
|
|
163
|
+
out.append(self.allocate(label, surface))
|
|
164
|
+
cursor = span_end
|
|
165
|
+
out.append(text[cursor:])
|
|
166
|
+
return "".join(out)
|
|
167
|
+
|
|
168
|
+
def restore(
|
|
169
|
+
self,
|
|
170
|
+
text: Annotated[str, "Text containing sentinels (typically an LLM response)."],
|
|
171
|
+
) -> str:
|
|
172
|
+
"""Replace every known sentinel in ``text`` with its original surface form.
|
|
173
|
+
|
|
174
|
+
Unknown sentinels (ones not in ``mapping``) are left as-is so callers can see leaks.
|
|
175
|
+
"""
|
|
176
|
+
if not self.mapping:
|
|
177
|
+
return text
|
|
178
|
+
|
|
179
|
+
def _sub(m: Any) -> str:
|
|
180
|
+
s = m.group(0)
|
|
181
|
+
return self.mapping.get(s, s)
|
|
182
|
+
|
|
183
|
+
return SENTINEL_RE.sub(_sub, text)
|
|
184
|
+
|
|
185
|
+
def to_json(self) -> str:
|
|
186
|
+
return json.dumps({"mapping": self.mapping})
|
|
187
|
+
|
|
188
|
+
@classmethod
|
|
189
|
+
def from_json(
|
|
190
|
+
cls,
|
|
191
|
+
raw: Annotated[str, "JSON string previously produced by Session.to_json()."],
|
|
192
|
+
*,
|
|
193
|
+
normalizers: Annotated[
|
|
194
|
+
Mapping[str, "str | Normalizer"] | None,
|
|
195
|
+
"Re-apply the same normalizers the session was created with. "
|
|
196
|
+
"Required if the session originally used custom normalizers; "
|
|
197
|
+
"otherwise dedup of new variants will fall back to NFKC+casefold "
|
|
198
|
+
"and miss e.g. phone-format collapses.",
|
|
199
|
+
] = None,
|
|
200
|
+
) -> Session:
|
|
201
|
+
data = json.loads(raw)
|
|
202
|
+
mapping: dict[str, str] = data.get("mapping", {})
|
|
203
|
+
s = cls(mapping=dict(mapping), normalizers=normalizers)
|
|
204
|
+
# The persisted form is just the {sentinel: surface} mapping. To
|
|
205
|
+
# rebuild the coalesce map we re-normalize each surface, using
|
|
206
|
+
# any per-label normalizer that was passed back in.
|
|
207
|
+
for sentinel, surface in mapping.items():
|
|
208
|
+
parsed = Sentinel.parse(sentinel)
|
|
209
|
+
if parsed is None:
|
|
210
|
+
continue
|
|
211
|
+
# Reverse-lookup the label name from the tag — we don't store
|
|
212
|
+
# the original label, so try every label that maps to this tag.
|
|
213
|
+
normalized: str | None = None
|
|
214
|
+
for label, fn in s._resolved_normalizers.items():
|
|
215
|
+
if label_tag(label) == parsed.tag:
|
|
216
|
+
normalized = fn(surface)
|
|
217
|
+
if normalized is not None:
|
|
218
|
+
break
|
|
219
|
+
if normalized is None:
|
|
220
|
+
normalized = _normalize_surface(surface)
|
|
221
|
+
s._coalesce[(parsed.tag, normalized)] = sentinel
|
|
222
|
+
s._counters[parsed.tag] = max(s._counters.get(parsed.tag, 0), parsed.index)
|
|
223
|
+
return s
|
gheim/core/stream.py
ADDED
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
"""Streaming de-anonymizer.
|
|
2
|
+
|
|
3
|
+
Problem: the LLM produces tokens one at a time. A sentinel like ``<PERSON_1>`` may
|
|
4
|
+
split across chunks (``<PERS``, ``ON_1>``), or even across single-character tokens.
|
|
5
|
+
We cannot emit ``<`` to the end user until we know whether what follows it is a
|
|
6
|
+
sentinel (to be substituted) or incidental text like ``<html>`` or ``x < 3``.
|
|
7
|
+
|
|
8
|
+
Strategy: character-level state machine with a hold-back buffer.
|
|
9
|
+
|
|
10
|
+
- While scanning, flush everything up to but not including the first ``<``.
|
|
11
|
+
- Once we see ``<``, enter HOLDING. Keep appending.
|
|
12
|
+
- While HOLDING, after every append, ask three questions:
|
|
13
|
+
1. Does the buffer end with a complete sentinel? → substitute, emit, drop buffer.
|
|
14
|
+
2. Is the buffer still a possible sentinel prefix? → keep holding, wait for more.
|
|
15
|
+
3. Otherwise: the ``<`` turned out to be incidental. Emit the ``<`` literally,
|
|
16
|
+
rewind past it, and restart scanning at the next character.
|
|
17
|
+
|
|
18
|
+
The buffer only grows until we decide, so worst-case hold-back is bounded by the
|
|
19
|
+
longest plausible sentinel (a few dozen chars).
|
|
20
|
+
"""
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
from collections.abc import AsyncIterable, AsyncIterator, Iterable, Iterator
|
|
24
|
+
from dataclasses import dataclass, field
|
|
25
|
+
from typing import Annotated
|
|
26
|
+
|
|
27
|
+
from .sentinels import SENTINEL_RE, is_possible_sentinel_prefix
|
|
28
|
+
|
|
29
|
+
# Hard upper bound. A sentinel has shape ``<TAG_N>`` with a short TAG and small N.
|
|
30
|
+
# If a hold-back ever exceeds this, it's definitely not a sentinel — emit verbatim.
|
|
31
|
+
MAX_SENTINEL_LEN = 64
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@dataclass
|
|
35
|
+
class StreamDeanonymizer:
|
|
36
|
+
"""Incremental chunk-based deanonymizer.
|
|
37
|
+
|
|
38
|
+
``mapping`` is a dict of rendered sentinel (``"<PERSON_1>"``) → original surface.
|
|
39
|
+
Unknown sentinels are passed through untouched.
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
mapping: Annotated[
|
|
43
|
+
dict[str, str], "Sentinel string ('<PERSON_1>') → original surface form."
|
|
44
|
+
]
|
|
45
|
+
_hold: str = field(default="", init=False, repr=False)
|
|
46
|
+
|
|
47
|
+
def feed(
|
|
48
|
+
self,
|
|
49
|
+
chunk: Annotated[str, "Next text chunk from the upstream stream."],
|
|
50
|
+
) -> str:
|
|
51
|
+
"""Consume a chunk, return what can be safely emitted right now."""
|
|
52
|
+
if not chunk:
|
|
53
|
+
return ""
|
|
54
|
+
|
|
55
|
+
buffer = self._hold + chunk
|
|
56
|
+
out: list[str] = []
|
|
57
|
+
i = 0
|
|
58
|
+
n = len(buffer)
|
|
59
|
+
|
|
60
|
+
while i < n:
|
|
61
|
+
if buffer[i] != "<":
|
|
62
|
+
lt = buffer.find("<", i)
|
|
63
|
+
if lt == -1:
|
|
64
|
+
out.append(buffer[i:])
|
|
65
|
+
i = n
|
|
66
|
+
break
|
|
67
|
+
out.append(buffer[i:lt])
|
|
68
|
+
i = lt
|
|
69
|
+
|
|
70
|
+
m = SENTINEL_RE.match(buffer, i)
|
|
71
|
+
if m is not None:
|
|
72
|
+
sentinel = m.group(0)
|
|
73
|
+
out.append(self.mapping.get(sentinel, sentinel))
|
|
74
|
+
i = m.end()
|
|
75
|
+
continue
|
|
76
|
+
|
|
77
|
+
tail = buffer[i:]
|
|
78
|
+
if len(tail) < MAX_SENTINEL_LEN and is_possible_sentinel_prefix(tail):
|
|
79
|
+
self._hold = tail
|
|
80
|
+
return "".join(out)
|
|
81
|
+
|
|
82
|
+
out.append("<")
|
|
83
|
+
i += 1
|
|
84
|
+
|
|
85
|
+
self._hold = ""
|
|
86
|
+
return "".join(out)
|
|
87
|
+
|
|
88
|
+
def flush(self) -> str:
|
|
89
|
+
"""Emit anything still held back. Call after the upstream stream ends."""
|
|
90
|
+
held = self._hold
|
|
91
|
+
self._hold = ""
|
|
92
|
+
return held
|
|
93
|
+
|
|
94
|
+
def transform(
|
|
95
|
+
self,
|
|
96
|
+
chunks: Annotated[Iterable[str], "Synchronous chunk iterable."],
|
|
97
|
+
) -> Iterator[str]:
|
|
98
|
+
"""Apply ``feed`` to each chunk and emit non-empty outputs; flush at end."""
|
|
99
|
+
for chunk in chunks:
|
|
100
|
+
out = self.feed(chunk)
|
|
101
|
+
if out:
|
|
102
|
+
yield out
|
|
103
|
+
tail = self.flush()
|
|
104
|
+
if tail:
|
|
105
|
+
yield tail
|
|
106
|
+
|
|
107
|
+
async def transform_async(
|
|
108
|
+
self,
|
|
109
|
+
chunks: Annotated[AsyncIterable[str], "Async chunk iterable (e.g. async SSE)."],
|
|
110
|
+
) -> AsyncIterator[str]:
|
|
111
|
+
"""Async counterpart of ``transform``."""
|
|
112
|
+
async for chunk in chunks:
|
|
113
|
+
out = self.feed(chunk)
|
|
114
|
+
if out:
|
|
115
|
+
yield out
|
|
116
|
+
tail = self.flush()
|
|
117
|
+
if tail:
|
|
118
|
+
yield tail
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
from .base import Detector, Span
|
|
2
|
+
from .local import LocalDetector
|
|
3
|
+
from .remote import RemoteDetector
|
|
4
|
+
|
|
5
|
+
__all__ = ["Detector", "LocalDetector", "RemoteDetector", "Span"]
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def default_detector() -> Detector:
|
|
9
|
+
"""Pick a reasonable default: remote if a key is configured, else local."""
|
|
10
|
+
import os
|
|
11
|
+
|
|
12
|
+
if os.getenv("GHEIM_API_KEY"):
|
|
13
|
+
return RemoteDetector()
|
|
14
|
+
return LocalDetector()
|
gheim/detectors/base.py
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""Detector protocol — anything that maps text to PII spans."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
from typing import Annotated, Protocol
|
|
6
|
+
|
|
7
|
+
# Characters stripped when `trim_whitespace=True` is applied to a raw span.
|
|
8
|
+
# The upstream `openai/privacy-filter` model tends to include leading
|
|
9
|
+
# whitespace in spans (BPE tokenization artifact), which makes restored text
|
|
10
|
+
# look wrong (" Alice" instead of "Alice"). Trimming corrects this.
|
|
11
|
+
_TRIM_CHARS = " \t\n"
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def trim_span(
|
|
15
|
+
text: str, start: int, end: int
|
|
16
|
+
) -> tuple[int, int] | None:
|
|
17
|
+
"""Return new (start, end) with leading/trailing whitespace stripped.
|
|
18
|
+
Returns None if the trimmed span is empty."""
|
|
19
|
+
while start < end and text[start] in _TRIM_CHARS:
|
|
20
|
+
start += 1
|
|
21
|
+
while end > start and text[end - 1] in _TRIM_CHARS:
|
|
22
|
+
end -= 1
|
|
23
|
+
if start >= end:
|
|
24
|
+
return None
|
|
25
|
+
return start, end
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass(frozen=True, slots=True)
|
|
29
|
+
class Span:
|
|
30
|
+
"""A detected PII span with character offsets into the source text."""
|
|
31
|
+
|
|
32
|
+
label: Annotated[str, "Detector label, e.g. 'private_person', 'private_email'."]
|
|
33
|
+
start: Annotated[int, "Inclusive character offset into the original string."]
|
|
34
|
+
end: Annotated[int, "Exclusive character offset into the original string."]
|
|
35
|
+
text: Annotated[str, "Original surface form (text[start:end])."]
|
|
36
|
+
score: Annotated[float, "Detector confidence in [0, 1]."] = 1.0
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class Detector(Protocol):
|
|
40
|
+
"""Anything that takes a string and returns a list of PII spans."""
|
|
41
|
+
|
|
42
|
+
def detect(
|
|
43
|
+
self,
|
|
44
|
+
text: Annotated[str, "Source text to scan for PII."],
|
|
45
|
+
*,
|
|
46
|
+
model: Annotated[str | None, "Optional per-call model id override."] = None,
|
|
47
|
+
) -> list[Span]: # pragma: no cover - protocol
|
|
48
|
+
...
|