gheim 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
gheim/__init__.py ADDED
@@ -0,0 +1,41 @@
1
+ """gheim — PII round-trip for LLM APIs.
2
+
3
+ Public API:
4
+
5
+ - ``Session`` — per-request state (sentinel mapping)
6
+ - ``LocalDetector`` / ``RemoteDetector`` — pluggable PII detection backends
7
+ - ``anonymize`` / ``deanonymize`` — message-level helpers
8
+ - ``anonymize_text`` / ``deanonymize_text`` — string-level helpers
9
+ - ``deanonymize_stream`` — wrap a chunk iterator
10
+ - ``gheim.openai.OpenAI`` — drop-in OpenAI client (subpackage import)
11
+ """
12
+ from .core.session import Session
13
+ from .core.stream import StreamDeanonymizer
14
+ from .detectors import Detector, LocalDetector, RemoteDetector, Span, default_detector
15
+ from .plain import (
16
+ anonymize,
17
+ anonymize_messages,
18
+ anonymize_text,
19
+ deanonymize,
20
+ deanonymize_stream,
21
+ deanonymize_text,
22
+ )
23
+
24
+ __version__ = "0.1.0"
25
+
26
+ __all__ = [
27
+ "Detector",
28
+ "LocalDetector",
29
+ "RemoteDetector",
30
+ "Session",
31
+ "Span",
32
+ "StreamDeanonymizer",
33
+ "__version__",
34
+ "anonymize",
35
+ "anonymize_messages",
36
+ "anonymize_text",
37
+ "deanonymize",
38
+ "deanonymize_stream",
39
+ "deanonymize_text",
40
+ "default_detector",
41
+ ]
gheim/core/__init__.py ADDED
@@ -0,0 +1,12 @@
1
+ from .sentinels import LABEL_TO_TAG, SENTINEL_RE, Sentinel, label_tag
2
+ from .session import Session
3
+ from .stream import StreamDeanonymizer
4
+
5
+ __all__ = [
6
+ "LABEL_TO_TAG",
7
+ "SENTINEL_RE",
8
+ "Sentinel",
9
+ "Session",
10
+ "StreamDeanonymizer",
11
+ "label_tag",
12
+ ]
@@ -0,0 +1,105 @@
1
+ """Sentinel format, allocation, and regex matching.
2
+
3
+ A sentinel is a stable, uniquely-numbered placeholder like ``<PERSON_1>`` or
4
+ ``<EMAIL_2>``. Each PII label maps to a tag (``private_person`` → ``PERSON``),
5
+ and each session allocates indices starting at 1 per tag.
6
+
7
+ The format is ASCII-only angle brackets so that downstream LLM tokenizers are
8
+ most likely to preserve it as-is. Brackets + uppercase letters + digits + one
9
+ underscore is a narrow enough shape that most producers won't mangle it.
10
+ """
11
+ from __future__ import annotations
12
+
13
+ import re
14
+ from collections.abc import Iterator
15
+ from dataclasses import dataclass
16
+ from typing import Annotated
17
+
18
+ # Canonical label → short tag used in the sentinel.
19
+ LABEL_TO_TAG: dict[str, str] = {
20
+ "private_person": "PERSON",
21
+ "private_email": "EMAIL",
22
+ "private_phone": "PHONE",
23
+ "private_address": "ADDRESS",
24
+ "private_url": "URL",
25
+ "private_date": "DATE",
26
+ "account_number": "ACCOUNT",
27
+ "secret": "SECRET",
28
+ }
29
+
30
+ # Reverse map for completeness (not strictly needed for round-trip).
31
+ TAG_TO_LABEL: dict[str, str] = {v: k for k, v in LABEL_TO_TAG.items()}
32
+
33
+ _TAG_PATTERN = r"[A-Z][A-Z0-9_]*"
34
+ _INDEX_PATTERN = r"\d+"
35
+
36
+ # A fully-formed sentinel anywhere in a string.
37
+ SENTINEL_RE = re.compile(rf"<(?P<tag>{_TAG_PATTERN})_(?P<index>{_INDEX_PATTERN})>")
38
+
39
+ # A *possibly-partial* sentinel anchored at the start of a buffer. Used by the
40
+ # streaming deanonymizer to decide "should I hold back these bytes or emit them?"
41
+ #
42
+ # Matches any strict prefix of a sentinel:
43
+ # "<", "<P", "<PERSON", "<PERSON_", "<PERSON_1", "<PERSON_12>"
44
+ # Does NOT match if the buffer contains disqualifying characters (e.g. "<x", "<PERSON ").
45
+ _PARTIAL_SENTINEL_RE = re.compile(
46
+ rf"^<({_TAG_PATTERN})?(_{_INDEX_PATTERN}?)?>?"
47
+ )
48
+
49
+
50
+ def label_tag(
51
+ label: Annotated[str, "Detector label, e.g. 'private_person' or 'account_number'."],
52
+ ) -> str:
53
+ """Map a detector label to its sentinel tag. Unknown labels fall back to the upper-cased label."""
54
+ return LABEL_TO_TAG.get(label, label.upper())
55
+
56
+
57
+ @dataclass(frozen=True, slots=True)
58
+ class Sentinel:
59
+ """A specific placeholder issued inside a session."""
60
+
61
+ tag: Annotated[str, "Short uppercase tag (e.g. 'PERSON')."]
62
+ index: Annotated[int, "1-based per-tag counter."]
63
+
64
+ def render(self) -> str:
65
+ return f"<{self.tag}_{self.index}>"
66
+
67
+ @classmethod
68
+ def parse(
69
+ cls,
70
+ s: Annotated[str, "Candidate sentinel string to parse, e.g. '<PERSON_7>'."],
71
+ ) -> Sentinel | None:
72
+ m = SENTINEL_RE.fullmatch(s)
73
+ if not m:
74
+ return None
75
+ return cls(tag=m["tag"], index=int(m["index"]))
76
+
77
+
78
+ def find_sentinels(
79
+ text: Annotated[str, "Text to scan for fully-formed sentinels."],
80
+ ) -> Iterator[tuple[int, int, Sentinel]]:
81
+ """Yield (start, end, sentinel) for each sentinel occurrence in ``text``."""
82
+ for m in SENTINEL_RE.finditer(text):
83
+ yield m.start(), m.end(), Sentinel(tag=m["tag"], index=int(m["index"]))
84
+
85
+
86
+ def is_possible_sentinel_prefix(
87
+ buffer: Annotated[str, "Buffer accumulated since the last '<' was seen by the streaming buffer."],
88
+ ) -> bool:
89
+ """Could ``buffer`` still become a sentinel if more characters arrive?
90
+
91
+ Returns True if the buffer is a non-empty strict prefix of some valid sentinel
92
+ (i.e. consumable) AND does not already contain disqualifying characters.
93
+ A fully-formed sentinel (ending in ``>``) is NOT a prefix — caller should have
94
+ already matched and consumed it.
95
+ """
96
+ if not buffer or buffer[0] != "<":
97
+ return False
98
+ m = _PARTIAL_SENTINEL_RE.match(buffer)
99
+ if m is None:
100
+ return False
101
+ # Must consume the whole buffer — otherwise there's trailing garbage.
102
+ if m.end() != len(buffer):
103
+ return False
104
+ # If it ends with ">" it's already complete — not a prefix.
105
+ return not buffer.endswith(">")
gheim/core/session.py ADDED
@@ -0,0 +1,223 @@
1
+ """Session state: tracks which sentinels map to which originals for one round-trip."""
2
+ from __future__ import annotations
3
+
4
+ import json
5
+ import unicodedata
6
+ from collections.abc import Mapping
7
+ from dataclasses import dataclass, field
8
+ from typing import TYPE_CHECKING, Annotated, Any
9
+
10
+ from .sentinels import SENTINEL_RE, Sentinel, label_tag
11
+
12
+ if TYPE_CHECKING:
13
+ from ..detectors.base import Detector, Span
14
+ from ..normalizers import Normalizer
15
+
16
+
17
+ # Maximum number of whitespace characters allowed between two spans for them to
18
+ # count as "adjacent" and be merged. Covers the common BPE artifact where the
19
+ # tokenizer splits an email into two adjacent same-label spans.
20
+ _MERGE_GAP = 1
21
+
22
+
23
+ def _normalize_surface(s: str) -> str:
24
+ """Normalize a surface form for sentinel-coalescing key purposes.
25
+
26
+ NFKC unicode normalization (so "fi" ligature == "fi"), Unicode-aware
27
+ casefolding (so "Joel" == "JOEL" == "joel" and "Müller" == "MÜLLER";
28
+ casefold also expands German "ß" → "ss"), then collapse all
29
+ whitespace runs to a single space and strip. The mapping still
30
+ stores the original surface so restoration preserves the first-seen
31
+ casing; only the lookup key is normalized.
32
+ """
33
+ return " ".join(unicodedata.normalize("NFKC", s).casefold().split())
34
+
35
+
36
+ def _merge_adjacent(text: str, spans: list[Span]) -> list[tuple[int, int, str]]:
37
+ """Merge runs of same-label spans that touch (or are separated by ≤ ``_MERGE_GAP``
38
+ whitespace characters)."""
39
+ out: list[tuple[int, int, str]] = []
40
+ for span in spans:
41
+ if span.start < 0 or span.end > len(text) or span.start >= span.end:
42
+ continue
43
+ if not out:
44
+ out.append((span.start, span.end, span.label))
45
+ continue
46
+ prev_start, prev_end, prev_label = out[-1]
47
+ if span.start < prev_end:
48
+ # Overlap — drop the later span (greedy first-wins).
49
+ continue
50
+ gap = text[prev_end : span.start]
51
+ if (
52
+ span.label == prev_label
53
+ and len(gap) <= _MERGE_GAP
54
+ and gap.strip() == ""
55
+ ):
56
+ out[-1] = (prev_start, span.end, prev_label)
57
+ else:
58
+ out.append((span.start, span.end, span.label))
59
+ return out
60
+
61
+
62
+ @dataclass
63
+ class Session:
64
+ """Mutable per-request state for anonymizing outbound text and restoring inbound text.
65
+
66
+ - ``mapping`` goes from rendered sentinel (``"<PERSON_1>"``) to the original surface form.
67
+ - Same (tag, surface) reuses the same sentinel so the LLM can refer back coherently
68
+ ("Joel said Joel's email is ...").
69
+
70
+ Sessions are JSON-serializable so callers can persist them across requests if they want
71
+ long-lived conversations (same `Joel` keeps mapping to `<PERSON_1>` across turns).
72
+ """
73
+
74
+ mapping: Annotated[
75
+ dict[str, str], "Rendered sentinel ('<PERSON_1>') → original surface form."
76
+ ] = field(default_factory=dict)
77
+ _coalesce: dict[tuple[str, str], str] = field(default_factory=dict)
78
+ _counters: dict[str, int] = field(default_factory=dict)
79
+ detector: Annotated[
80
+ Detector | None, "Optional default detector reused across calls in this session."
81
+ ] = None
82
+ normalizers: Annotated[
83
+ Mapping[str, "str | Normalizer"] | None,
84
+ "Optional per-label canonical-form normalizers, e.g. "
85
+ "``{'private_phone': 'e164', 'private_date': 'iso_date'}`` or "
86
+ "``{'private_phone': e164(region='CH')}``. A label not listed "
87
+ "here uses the default NFKC+casefold key. See "
88
+ "``gheim.normalizers``.",
89
+ ] = None
90
+ _resolved_normalizers: dict[str, "Normalizer"] = field(
91
+ default_factory=dict, init=False, repr=False,
92
+ )
93
+
94
+ def __post_init__(self) -> None:
95
+ if self.normalizers:
96
+ from ..normalizers import resolve_normalizer
97
+ self._resolved_normalizers = {
98
+ label: resolve_normalizer(spec)
99
+ for label, spec in self.normalizers.items()
100
+ }
101
+
102
+ def allocate(
103
+ self,
104
+ label: Annotated[str, "Detector label (e.g. 'private_person')."],
105
+ surface: Annotated[str, "Original surface form to be hidden behind a sentinel."],
106
+ ) -> str:
107
+ """Return (or allocate) the sentinel for this (label, surface) pair.
108
+
109
+ The coalesce key uses a normalized form of ``surface`` (NFKC +
110
+ casefold + collapsed whitespace) so trivial variants of the same
111
+ identity collapse to one sentinel: ``Joel`` / ``JOEL`` / ``joel``,
112
+ ``Müller`` / ``MÜLLER``, ``"alice@example.com"`` / ``"ALICE@EXAMPLE.COM"``.
113
+ When a per-label normalizer is configured (via ``normalizers=``)
114
+ it overrides the default key for that label — used to collapse
115
+ e.g. phone-format variants under E.164. If the normalizer
116
+ returns ``None`` (un-parseable), the default NFKC+casefold key
117
+ is used as a safe fallback so the surface still gets a sentinel.
118
+
119
+ The mapping stores the original (first-seen) surface so
120
+ restoration preserves casing.
121
+ """
122
+ tag = label_tag(label)
123
+ norm_fn = self._resolved_normalizers.get(label)
124
+ normalized: str | None = None
125
+ if norm_fn is not None:
126
+ normalized = norm_fn(surface)
127
+ if normalized is None:
128
+ normalized = _normalize_surface(surface)
129
+ key = (tag, normalized)
130
+ existing = self._coalesce.get(key)
131
+ if existing is not None:
132
+ return existing
133
+ self._counters[tag] = self._counters.get(tag, 0) + 1
134
+ sentinel = Sentinel(tag=tag, index=self._counters[tag]).render()
135
+ self._coalesce[key] = sentinel
136
+ self.mapping[sentinel] = surface
137
+ return sentinel
138
+
139
+ def apply_spans(
140
+ self,
141
+ text: Annotated[str, "Original text the spans were detected in."],
142
+ spans: Annotated[list[Span], "Detected PII spans, possibly unsorted."],
143
+ ) -> str:
144
+ """Replace each span in ``text`` with an allocated sentinel. Left-to-right, non-overlapping.
145
+
146
+ Overlapping spans are resolved by keeping the earliest-starting, longest span.
147
+ Adjacent same-label spans (``alice@example`` + ``.com``) are merged before
148
+ allocation so a split email doesn't end up as two separate sentinels.
149
+ """
150
+ if not spans:
151
+ return text
152
+ sorted_spans = sorted(spans, key=lambda s: (s.start, -(s.end - s.start)))
153
+ merged = _merge_adjacent(text, sorted_spans)
154
+ out: list[str] = []
155
+ cursor = 0
156
+ for span_start, span_end, label in merged:
157
+ if span_start < cursor:
158
+ continue
159
+ if span_start < 0 or span_end > len(text) or span_start >= span_end:
160
+ continue
161
+ surface = text[span_start:span_end]
162
+ out.append(text[cursor:span_start])
163
+ out.append(self.allocate(label, surface))
164
+ cursor = span_end
165
+ out.append(text[cursor:])
166
+ return "".join(out)
167
+
168
+ def restore(
169
+ self,
170
+ text: Annotated[str, "Text containing sentinels (typically an LLM response)."],
171
+ ) -> str:
172
+ """Replace every known sentinel in ``text`` with its original surface form.
173
+
174
+ Unknown sentinels (ones not in ``mapping``) are left as-is so callers can see leaks.
175
+ """
176
+ if not self.mapping:
177
+ return text
178
+
179
+ def _sub(m: Any) -> str:
180
+ s = m.group(0)
181
+ return self.mapping.get(s, s)
182
+
183
+ return SENTINEL_RE.sub(_sub, text)
184
+
185
+ def to_json(self) -> str:
186
+ return json.dumps({"mapping": self.mapping})
187
+
188
+ @classmethod
189
+ def from_json(
190
+ cls,
191
+ raw: Annotated[str, "JSON string previously produced by Session.to_json()."],
192
+ *,
193
+ normalizers: Annotated[
194
+ Mapping[str, "str | Normalizer"] | None,
195
+ "Re-apply the same normalizers the session was created with. "
196
+ "Required if the session originally used custom normalizers; "
197
+ "otherwise dedup of new variants will fall back to NFKC+casefold "
198
+ "and miss e.g. phone-format collapses.",
199
+ ] = None,
200
+ ) -> Session:
201
+ data = json.loads(raw)
202
+ mapping: dict[str, str] = data.get("mapping", {})
203
+ s = cls(mapping=dict(mapping), normalizers=normalizers)
204
+ # The persisted form is just the {sentinel: surface} mapping. To
205
+ # rebuild the coalesce map we re-normalize each surface, using
206
+ # any per-label normalizer that was passed back in.
207
+ for sentinel, surface in mapping.items():
208
+ parsed = Sentinel.parse(sentinel)
209
+ if parsed is None:
210
+ continue
211
+ # Reverse-lookup the label name from the tag — we don't store
212
+ # the original label, so try every label that maps to this tag.
213
+ normalized: str | None = None
214
+ for label, fn in s._resolved_normalizers.items():
215
+ if label_tag(label) == parsed.tag:
216
+ normalized = fn(surface)
217
+ if normalized is not None:
218
+ break
219
+ if normalized is None:
220
+ normalized = _normalize_surface(surface)
221
+ s._coalesce[(parsed.tag, normalized)] = sentinel
222
+ s._counters[parsed.tag] = max(s._counters.get(parsed.tag, 0), parsed.index)
223
+ return s
gheim/core/stream.py ADDED
@@ -0,0 +1,118 @@
1
+ """Streaming de-anonymizer.
2
+
3
+ Problem: the LLM produces tokens one at a time. A sentinel like ``<PERSON_1>`` may
4
+ split across chunks (``<PERS``, ``ON_1>``), or even across single-character tokens.
5
+ We cannot emit ``<`` to the end user until we know whether what follows it is a
6
+ sentinel (to be substituted) or incidental text like ``<html>`` or ``x < 3``.
7
+
8
+ Strategy: character-level state machine with a hold-back buffer.
9
+
10
+ - While scanning, flush everything up to but not including the first ``<``.
11
+ - Once we see ``<``, enter HOLDING. Keep appending.
12
+ - While HOLDING, after every append, ask three questions:
13
+ 1. Does the buffer end with a complete sentinel? → substitute, emit, drop buffer.
14
+ 2. Is the buffer still a possible sentinel prefix? → keep holding, wait for more.
15
+ 3. Otherwise: the ``<`` turned out to be incidental. Emit the ``<`` literally,
16
+ rewind past it, and restart scanning at the next character.
17
+
18
+ The buffer only grows until we decide, so worst-case hold-back is bounded by the
19
+ longest plausible sentinel (a few dozen chars).
20
+ """
21
+ from __future__ import annotations
22
+
23
+ from collections.abc import AsyncIterable, AsyncIterator, Iterable, Iterator
24
+ from dataclasses import dataclass, field
25
+ from typing import Annotated
26
+
27
+ from .sentinels import SENTINEL_RE, is_possible_sentinel_prefix
28
+
29
+ # Hard upper bound. A sentinel has shape ``<TAG_N>`` with a short TAG and small N.
30
+ # If a hold-back ever exceeds this, it's definitely not a sentinel — emit verbatim.
31
+ MAX_SENTINEL_LEN = 64
32
+
33
+
34
+ @dataclass
35
+ class StreamDeanonymizer:
36
+ """Incremental chunk-based deanonymizer.
37
+
38
+ ``mapping`` is a dict of rendered sentinel (``"<PERSON_1>"``) → original surface.
39
+ Unknown sentinels are passed through untouched.
40
+ """
41
+
42
+ mapping: Annotated[
43
+ dict[str, str], "Sentinel string ('<PERSON_1>') → original surface form."
44
+ ]
45
+ _hold: str = field(default="", init=False, repr=False)
46
+
47
+ def feed(
48
+ self,
49
+ chunk: Annotated[str, "Next text chunk from the upstream stream."],
50
+ ) -> str:
51
+ """Consume a chunk, return what can be safely emitted right now."""
52
+ if not chunk:
53
+ return ""
54
+
55
+ buffer = self._hold + chunk
56
+ out: list[str] = []
57
+ i = 0
58
+ n = len(buffer)
59
+
60
+ while i < n:
61
+ if buffer[i] != "<":
62
+ lt = buffer.find("<", i)
63
+ if lt == -1:
64
+ out.append(buffer[i:])
65
+ i = n
66
+ break
67
+ out.append(buffer[i:lt])
68
+ i = lt
69
+
70
+ m = SENTINEL_RE.match(buffer, i)
71
+ if m is not None:
72
+ sentinel = m.group(0)
73
+ out.append(self.mapping.get(sentinel, sentinel))
74
+ i = m.end()
75
+ continue
76
+
77
+ tail = buffer[i:]
78
+ if len(tail) < MAX_SENTINEL_LEN and is_possible_sentinel_prefix(tail):
79
+ self._hold = tail
80
+ return "".join(out)
81
+
82
+ out.append("<")
83
+ i += 1
84
+
85
+ self._hold = ""
86
+ return "".join(out)
87
+
88
+ def flush(self) -> str:
89
+ """Emit anything still held back. Call after the upstream stream ends."""
90
+ held = self._hold
91
+ self._hold = ""
92
+ return held
93
+
94
+ def transform(
95
+ self,
96
+ chunks: Annotated[Iterable[str], "Synchronous chunk iterable."],
97
+ ) -> Iterator[str]:
98
+ """Apply ``feed`` to each chunk and emit non-empty outputs; flush at end."""
99
+ for chunk in chunks:
100
+ out = self.feed(chunk)
101
+ if out:
102
+ yield out
103
+ tail = self.flush()
104
+ if tail:
105
+ yield tail
106
+
107
+ async def transform_async(
108
+ self,
109
+ chunks: Annotated[AsyncIterable[str], "Async chunk iterable (e.g. async SSE)."],
110
+ ) -> AsyncIterator[str]:
111
+ """Async counterpart of ``transform``."""
112
+ async for chunk in chunks:
113
+ out = self.feed(chunk)
114
+ if out:
115
+ yield out
116
+ tail = self.flush()
117
+ if tail:
118
+ yield tail
@@ -0,0 +1,14 @@
1
+ from .base import Detector, Span
2
+ from .local import LocalDetector
3
+ from .remote import RemoteDetector
4
+
5
+ __all__ = ["Detector", "LocalDetector", "RemoteDetector", "Span"]
6
+
7
+
8
+ def default_detector() -> Detector:
9
+ """Pick a reasonable default: remote if a key is configured, else local."""
10
+ import os
11
+
12
+ if os.getenv("GHEIM_API_KEY"):
13
+ return RemoteDetector()
14
+ return LocalDetector()
@@ -0,0 +1,48 @@
1
+ """Detector protocol — anything that maps text to PII spans."""
2
+ from __future__ import annotations
3
+
4
+ from dataclasses import dataclass
5
+ from typing import Annotated, Protocol
6
+
7
+ # Characters stripped when `trim_whitespace=True` is applied to a raw span.
8
+ # The upstream `openai/privacy-filter` model tends to include leading
9
+ # whitespace in spans (BPE tokenization artifact), which makes restored text
10
+ # look wrong (" Alice" instead of "Alice"). Trimming corrects this.
11
+ _TRIM_CHARS = " \t\n"
12
+
13
+
14
+ def trim_span(
15
+ text: str, start: int, end: int
16
+ ) -> tuple[int, int] | None:
17
+ """Return new (start, end) with leading/trailing whitespace stripped.
18
+ Returns None if the trimmed span is empty."""
19
+ while start < end and text[start] in _TRIM_CHARS:
20
+ start += 1
21
+ while end > start and text[end - 1] in _TRIM_CHARS:
22
+ end -= 1
23
+ if start >= end:
24
+ return None
25
+ return start, end
26
+
27
+
28
+ @dataclass(frozen=True, slots=True)
29
+ class Span:
30
+ """A detected PII span with character offsets into the source text."""
31
+
32
+ label: Annotated[str, "Detector label, e.g. 'private_person', 'private_email'."]
33
+ start: Annotated[int, "Inclusive character offset into the original string."]
34
+ end: Annotated[int, "Exclusive character offset into the original string."]
35
+ text: Annotated[str, "Original surface form (text[start:end])."]
36
+ score: Annotated[float, "Detector confidence in [0, 1]."] = 1.0
37
+
38
+
39
+ class Detector(Protocol):
40
+ """Anything that takes a string and returns a list of PII spans."""
41
+
42
+ def detect(
43
+ self,
44
+ text: Annotated[str, "Source text to scan for PII."],
45
+ *,
46
+ model: Annotated[str | None, "Optional per-call model id override."] = None,
47
+ ) -> list[Span]: # pragma: no cover - protocol
48
+ ...