aliasgate 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,195 @@
1
+ """ML detector (architecture §3.2, priority 40): a local GLiNER model, from the optional extra.
2
+
3
+ The model is injected through `EntityModel`, so this module never imports `gliner` (or torch)
4
+ unless `load_gliner` is called.
5
+ """
6
+
7
+ import importlib
8
+ import json
9
+ import math
10
+ import threading
11
+ from collections.abc import Callable, Iterator, Mapping
12
+ from typing import Final, Protocol, TypeGuard
13
+
14
+ from aliasgate.detectors.chunking import Window, windows
15
+ from aliasgate.types import PRIORITY_ML, Span, validate_entity_type
16
+
17
+ _SOURCE: Final = "gliner"
18
+ _MIN_SPAN_LENGTH: Final = 2
19
+ _INSTALL_HINT: Final = (
20
+ "the ML detector needs the optional 'ml' extra: install it with "
21
+ "pip install 'aliasgate[ml]' (or uv sync --extra ml in a checkout)"
22
+ )
23
+
24
+
25
+ class EntityModel(Protocol):
26
+ """A named-entity model: character offsets into `text`, labels from `labels`."""
27
+
28
+ def predict_entities(
29
+ self, text: str, labels: list[str], threshold: float
30
+ ) -> list[dict[str, object]]: ...
31
+
32
+
33
+ class GlinerDetector:
34
+ """Finds people, organisations, addresses and other configured labels with a local model.
35
+
36
+ Long texts are cut into overlapping windows of the model's own words. Results are mapped
37
+ from model labels to entity types, trimmed of surrounding whitespace, deduplicated across
38
+ windows (the highest score wins), and returned sorted by position. A malformed model result
39
+ raises; errors name positions and types, never text. One inference runs at a time.
40
+ """
41
+
42
+ name = _SOURCE
43
+
44
+ def __init__(
45
+ self,
46
+ model: EntityModel,
47
+ *,
48
+ model_id: str,
49
+ revision: str | None = None,
50
+ labels: Mapping[str, str],
51
+ threshold: float,
52
+ window_words: int = 250,
53
+ overlap_words: int = 50,
54
+ ) -> None:
55
+ if not labels or not all(labels):
56
+ raise ValueError("labels must map at least one non-empty model label")
57
+ for entity_type in labels.values():
58
+ validate_entity_type(entity_type)
59
+ if not 0.0 <= threshold <= 1.0: # also rejects NaN
60
+ raise ValueError("threshold must be between 0 and 1")
61
+ windows("", window_words=window_words, overlap_words=overlap_words) # validates both
62
+ self._model = model
63
+ self._labels = dict(labels)
64
+ self._label_names = list(labels)
65
+ self._threshold = threshold
66
+ self._window_words = window_words
67
+ self._overlap_words = overlap_words
68
+ self._lock = threading.Lock()
69
+ self._fingerprint = json.dumps(
70
+ {
71
+ "detector": _SOURCE,
72
+ "model": model_id,
73
+ "revision": revision,
74
+ "threshold": threshold,
75
+ "labels": sorted(self._labels.items()),
76
+ "window_words": window_words,
77
+ "overlap_words": overlap_words,
78
+ },
79
+ sort_keys=True,
80
+ separators=(",", ":"),
81
+ )
82
+
83
+ @property
84
+ def fingerprint(self) -> str:
85
+ """Identifies the model and every setting that changes the output; used as cache key."""
86
+ return self._fingerprint
87
+
88
+ def detect(self, text: str) -> list[Span]:
89
+ """Spans of priority 40 and source "gliner"; overlapping spans are left to the resolver."""
90
+ best: dict[tuple[int, int, str], Span] = {}
91
+ for window in windows(
92
+ text, window_words=self._window_words, overlap_words=self._overlap_words
93
+ ):
94
+ chunk = text[window.start : window.end]
95
+ # Inference is not guaranteed to be thread-safe: one call at a time.
96
+ with self._lock:
97
+ results = self._model.predict_entities(
98
+ chunk, list(self._label_names), threshold=self._threshold
99
+ )
100
+ for position, result in enumerate(results, start=1):
101
+ span = self._span(result, position, window, text)
102
+ if span is None:
103
+ continue
104
+ key = (span.start, span.end, span.entity_type)
105
+ current = best.get(key)
106
+ if current is None or span.score > current.score:
107
+ best[key] = span
108
+ return sorted(best.values(), key=lambda span: (span.start, span.end, span.entity_type))
109
+
110
+ def _span(self, result: object, position: int, window: Window, text: str) -> Span | None:
111
+ where = f"model result {position} of a window"
112
+ if not isinstance(result, Mapping):
113
+ raise ValueError(f"{where} is not a mapping")
114
+ label = result.get("label")
115
+ if not isinstance(label, str) or label not in self._labels:
116
+ return None
117
+ start, end, score = result.get("start"), result.get("end"), result.get("score")
118
+ length = window.end - window.start
119
+ if not (_is_int(start) and _is_int(end) and 0 <= start <= end <= length):
120
+ raise ValueError(f"{where} has offsets outside the window (length {length})")
121
+ if not isinstance(score, int | float) or isinstance(score, bool) or math.isnan(score):
122
+ raise ValueError(f"{where} has a score that is not a number")
123
+ start += window.start
124
+ end += window.start
125
+ while start < end and text[start].isspace():
126
+ start += 1
127
+ while end > start and text[end - 1].isspace():
128
+ end -= 1
129
+ if end - start < _MIN_SPAN_LENGTH:
130
+ return None
131
+ clamped = min(max(float(score), 0.0), 1.0)
132
+ return Span(start, end, self._labels[label], clamped, _SOURCE, PRIORITY_ML)
133
+
134
+
135
+ def _is_int(value: object) -> TypeGuard[int]:
136
+ return isinstance(value, int) and not isinstance(value, bool)
137
+
138
+
139
+ class _GlinerLike(Protocol):
140
+ def predict_entities(
141
+ self, text: str, labels: list[str], *, threshold: float
142
+ ) -> list[dict[str, object]]: ...
143
+
144
+
145
+ class _LoadedGliner:
146
+ """A loaded GLiNER model that refuses any text it would truncate.
147
+
148
+ GLiNER silently keeps only the first `max_len` of its words (with a warning). The words
149
+ after that would never be scored, so a longer text is an error instead.
150
+ """
151
+
152
+ def __init__(
153
+ self,
154
+ model: _GlinerLike,
155
+ splitter: Callable[[str], Iterator[object]],
156
+ max_len: int,
157
+ ) -> None:
158
+ self._model = model
159
+ self._splitter = splitter
160
+ self._max_len = max_len
161
+
162
+ def predict_entities(
163
+ self, text: str, labels: list[str], threshold: float
164
+ ) -> list[dict[str, object]]:
165
+ words = sum(1 for _ in self._splitter(text))
166
+ if words > self._max_len:
167
+ raise ValueError(
168
+ f"a window of {words} model words is over the model's limit of {self._max_len};"
169
+ " lower detection.ml.window_words"
170
+ )
171
+ # By keyword: GLiNER's third positional parameter is flat_ner.
172
+ return self._model.predict_entities(text, labels, threshold=threshold)
173
+
174
+
175
+ def load_gliner(model_id: str, revision: str | None) -> EntityModel:
176
+ """Import gliner lazily and load the model. Raise a clear error if the extra is missing.
177
+
178
+ The returned model refuses any text longer than the model's input limit instead of
179
+ letting GLiNER truncate it.
180
+ """
181
+ try:
182
+ gliner = importlib.import_module("gliner")
183
+ except ImportError as exc:
184
+ raise RuntimeError(_INSTALL_HINT) from exc
185
+ model = gliner.GLiNER.from_pretrained(model_id, revision=revision)
186
+ try:
187
+ max_len = model.config.max_len
188
+ splitter = model.data_processor.words_splitter
189
+ except AttributeError:
190
+ max_len = splitter = None
191
+ if not _is_int(max_len) or max_len < 1 or not callable(splitter):
192
+ raise RuntimeError(
193
+ "cannot read the model's input limit; this gliner version is not supported"
194
+ )
195
+ return _LoadedGliner(model, splitter, max_len)
@@ -0,0 +1,17 @@
1
+ """Detector for input text that already looks like a placeholder (architecture §4.4)."""
2
+
3
+ from aliasgate.placeholders import TOLERANT
4
+ from aliasgate.types import LITERAL, PRIORITY_LITERAL, Span
5
+
6
+
7
+ class LiteralDetector:
8
+ """Reports placeholder-shaped input so that restoration can never mistake it for a value."""
9
+
10
+ name = "literal"
11
+
12
+ def detect(self, text: str) -> list[Span]:
13
+ """One span of type LITERAL, priority 100, score 1.0, per match of TOLERANT."""
14
+ return [
15
+ Span(match.start(), match.end(), LITERAL, 1.0, self.name, PRIORITY_LITERAL)
16
+ for match in TOLERANT.finditer(text)
17
+ ]
@@ -0,0 +1,318 @@
1
+ """Rule-based detection of structured identifiers, validated by checksums or libraries."""
2
+
3
+ import ipaddress
4
+ import re
5
+ import string
6
+ import sys
7
+ from collections.abc import Callable, Collection, Iterator, Sequence
8
+ from dataclasses import dataclass
9
+ from typing import Final
10
+
11
+ import phonenumbers
12
+
13
+ from aliasgate.detectors.checksums import (
14
+ es_cif_valid,
15
+ es_dni_valid,
16
+ es_nie_valid,
17
+ iban_valid,
18
+ luhn_valid,
19
+ us_ssn_plausible,
20
+ )
21
+ from aliasgate.types import (
22
+ CREDIT_CARD,
23
+ EMAIL,
24
+ IBAN,
25
+ IP_ADDRESS,
26
+ NATIONAL_ID,
27
+ PHONE,
28
+ PRIORITY_RULE,
29
+ SECRET,
30
+ TAX_ID,
31
+ URL,
32
+ Span,
33
+ )
34
+
35
+ DEFAULT_RULE_TYPES: Final[frozenset[str]] = frozenset(
36
+ {EMAIL, PHONE, IBAN, CREDIT_CARD, NATIONAL_ID, TAX_ID, IP_ADDRESS, SECRET}
37
+ ) # "URL" exists but is off by default
38
+
39
+ _RULE_TYPES: Final = DEFAULT_RULE_TYPES | {URL}
40
+ _SOURCE: Final = "rules"
41
+ _PHONE_SCORE: Final = 0.9
42
+ _ISBN_LENGTH: Final = 13
43
+ _ISBN_PREFIXES: Final = ("978", "979")
44
+ _INTERNATIONAL_SOURCES: Final = frozenset(
45
+ {
46
+ phonenumbers.CountryCodeSource.FROM_NUMBER_WITH_PLUS_SIGN,
47
+ phonenumbers.CountryCodeSource.FROM_NUMBER_WITH_IDD,
48
+ }
49
+ )
50
+ _SECRET_SCORE: Final = 1.0
51
+
52
+
53
+ def _ipv4_valid(candidate: str) -> bool:
54
+ try:
55
+ ipaddress.IPv4Address(candidate)
56
+ except ValueError:
57
+ return False
58
+ return True
59
+
60
+
61
+ def _ipv6_valid(candidate: str) -> bool:
62
+ # "::" alone parses, but it identifies nobody and is common in code ("x :: Int").
63
+ if not any(char in string.hexdigits for char in candidate):
64
+ return False
65
+ try:
66
+ ipaddress.IPv6Address(candidate)
67
+ except ValueError:
68
+ return False
69
+ return True
70
+
71
+
72
+ def _card_valid(candidate: str) -> bool:
73
+ """Luhn-valid, and not 13 digits starting with 978 or 979: that is an ISBN-13 (the EAN
74
+ "Bookland" prefix), and no card network issues 13-digit numbers with that prefix."""
75
+ digits = candidate.replace(" ", "").replace("-", "")
76
+ if len(digits) == _ISBN_LENGTH and digits.startswith(_ISBN_PREFIXES):
77
+ return False
78
+ return luhn_valid(candidate)
79
+
80
+
81
+ def _drop_last_group(separators: str, min_length: int) -> Callable[[str], str | None]:
82
+ """Shortener that drops the last separator-delimited group, while at least `min_length`
83
+ non-separator characters would remain."""
84
+
85
+ def shorten(candidate: str) -> str | None:
86
+ cut = max(candidate.rfind(separator) for separator in separators)
87
+ if cut <= 0:
88
+ return None
89
+ shorter = candidate[:cut]
90
+ remaining = len(shorter) - sum(shorter.count(separator) for separator in separators)
91
+ return shorter if remaining >= min_length else None
92
+
93
+ return shorten
94
+
95
+
96
+ @dataclass(frozen=True, slots=True)
97
+ class _Rule:
98
+ """A candidate pattern, an optional validator, and how to shorten a rejected candidate."""
99
+
100
+ entity_type: str
101
+ pattern: re.Pattern[str]
102
+ score: float
103
+ validate: Callable[[str], bool] | None = None
104
+ shorten: Callable[[str], str | None] | None = None
105
+
106
+
107
+ _RULES: Final[tuple[_Rule, ...]] = (
108
+ _Rule(
109
+ EMAIL,
110
+ re.compile(r"(?<![\w.%+-])[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)+(?![\w-])"),
111
+ 1.0,
112
+ ),
113
+ _Rule(
114
+ IBAN,
115
+ re.compile(
116
+ # Any letter case: people write IBANs in lower case too, and the checksum still
117
+ # filters out words that happen to have the shape.
118
+ r"(?<![A-Za-z0-9])[A-Za-z]{2}\d{2}(?:[ ]?[A-Za-z0-9]{4}){2,7}"
119
+ r"(?:[ ]?[A-Za-z0-9]{1,3})?(?![A-Za-z0-9])"
120
+ ),
121
+ 1.0,
122
+ iban_valid,
123
+ _drop_last_group(" ", min_length=15),
124
+ ),
125
+ _Rule(
126
+ CREDIT_CARD,
127
+ re.compile(r"(?<![\d-])\d(?:[ -]?\d){12,18}(?![\d-])"),
128
+ 1.0,
129
+ _card_valid,
130
+ _drop_last_group(" -", min_length=13),
131
+ ),
132
+ _Rule(
133
+ NATIONAL_ID,
134
+ re.compile(r"(?<![A-Za-z0-9])\d{8}[- ]?[A-Za-z](?![A-Za-z0-9])"),
135
+ 1.0,
136
+ es_dni_valid,
137
+ ),
138
+ _Rule(
139
+ NATIONAL_ID,
140
+ re.compile(r"(?<![A-Za-z0-9])[XYZxyz][- ]?\d{7}[- ]?[A-Za-z](?![A-Za-z0-9])"),
141
+ 1.0,
142
+ es_nie_valid,
143
+ ),
144
+ _Rule(
145
+ NATIONAL_ID,
146
+ re.compile(r"(?<![\d-])\d{3}-\d{2}-\d{4}(?![\d-])"),
147
+ 0.85,
148
+ us_ssn_plausible,
149
+ ),
150
+ _Rule(
151
+ TAX_ID,
152
+ re.compile(r"(?<![A-Za-z0-9])[A-Za-z]\d{7}[0-9A-Ja-j](?![A-Za-z0-9])"),
153
+ 1.0,
154
+ es_cif_valid,
155
+ ),
156
+ _Rule(
157
+ IP_ADDRESS,
158
+ # A sentence may end right after the address, so a dot is a boundary unless a digit
159
+ # follows it.
160
+ re.compile(r"(?<!\d)(?<!\d\.)(?:\d{1,3}\.){3}\d{1,3}(?!\d)(?!\.\d)"),
161
+ 0.9,
162
+ _ipv4_valid,
163
+ ),
164
+ _Rule(
165
+ IP_ADDRESS,
166
+ re.compile(
167
+ r"(?<![\w:])(?:[0-9A-Fa-f]{0,4}:){2,8}"
168
+ r"(?:(?:\d{1,3}\.){3}\d{1,3}|[0-9A-Fa-f]{1,4})?(?![\w:])(?!\.\d)"
169
+ ),
170
+ 0.9,
171
+ _ipv6_valid,
172
+ ),
173
+ _Rule(
174
+ URL,
175
+ # The last character excludes .,;:!? so that sentence punctuation is not part of it.
176
+ re.compile(r"https?://[^\s<>\"')\]]*[^\s<>\"')\].,;:!?]"),
177
+ 0.9,
178
+ ),
179
+ _Rule(SECRET, re.compile(r"(?<![A-Za-z0-9])sk-[A-Za-z0-9_-]{20,}"), _SECRET_SCORE),
180
+ _Rule(SECRET, re.compile(r"(?<![A-Za-z0-9])gh[pousr]_[A-Za-z0-9]{36,}"), _SECRET_SCORE),
181
+ _Rule(SECRET, re.compile(r"(?<![A-Za-z0-9])github_pat_[A-Za-z0-9_]{50,}"), _SECRET_SCORE),
182
+ _Rule(SECRET, re.compile(r"(?<![A-Z0-9])(?:AKIA|ASIA)[0-9A-Z]{16}(?![A-Z0-9])"), _SECRET_SCORE),
183
+ _Rule(SECRET, re.compile(r"(?<![A-Za-z0-9])xox[baprs]-[A-Za-z0-9-]{10,}"), _SECRET_SCORE),
184
+ _Rule(
185
+ SECRET,
186
+ re.compile(
187
+ r"(?<![A-Za-z0-9])eyJ[A-Za-z0-9_-]{5,}\.eyJ[A-Za-z0-9_-]{5,}\.[A-Za-z0-9_-]{5,}"
188
+ ),
189
+ _SECRET_SCORE,
190
+ ),
191
+ )
192
+
193
+ _PEM_BEGIN: Final = re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----")
194
+ _PEM_END: Final = re.compile(r"-----END [A-Z ]*PRIVATE KEY-----")
195
+
196
+
197
+ def _scan(rule: _Rule, text: str) -> Iterator[tuple[int, int]]:
198
+ """Yield (start, end) of every accepted candidate, leftmost first.
199
+
200
+ A rejected candidate is shortened while that is possible; if no version validates, the
201
+ search resumes one character after its start, so a valid value that begins inside a
202
+ rejected candidate is still found.
203
+ """
204
+ position = 0
205
+ while (match := rule.pattern.search(text, position)) is not None:
206
+ start = match.start()
207
+ length = _accepted_length(rule, match.group())
208
+ if length is None:
209
+ position = start + 1
210
+ else:
211
+ yield start, start + length
212
+ position = start + length
213
+
214
+
215
+ def _accepted_length(rule: _Rule, candidate: str) -> int | None:
216
+ if rule.validate is None:
217
+ return len(candidate)
218
+ current: str | None = candidate
219
+ while current is not None:
220
+ if rule.validate(current):
221
+ return len(current)
222
+ current = rule.shorten(current) if rule.shorten is not None else None
223
+ return None
224
+
225
+
226
+ def _scan_pem(text: str) -> Iterator[tuple[int, int]]:
227
+ """Yield each private key block from its BEGIN line to the first END line after it.
228
+
229
+ Same result as a non-greedy DOTALL pattern, in linear time: once a BEGIN line has no END
230
+ line after it, no later BEGIN line has one either.
231
+ """
232
+ position = 0
233
+ while (begin := _PEM_BEGIN.search(text, position)) is not None:
234
+ end = _PEM_END.search(text, begin.end())
235
+ if end is None:
236
+ return
237
+ yield begin.start(), end.end()
238
+ position = end.end()
239
+
240
+
241
+ class RuleDetector:
242
+ """Finds structured identifiers with patterns plus checksum or library validation.
243
+
244
+ Deterministic: the same text always yields the same spans, sorted by (start, end). Overlaps
245
+ between rules are kept; resolving them is the resolver's job.
246
+ """
247
+
248
+ name = "rules"
249
+
250
+ def __init__(
251
+ self,
252
+ *,
253
+ enabled_types: Collection[str] = DEFAULT_RULE_TYPES,
254
+ phone_regions: Sequence[str] = ("ES", "US"),
255
+ ) -> None:
256
+ enabled = frozenset(enabled_types)
257
+ unknown = enabled - _RULE_TYPES
258
+ if unknown:
259
+ raise ValueError(
260
+ f"{len(unknown)} unknown rule type(s) in enabled_types; "
261
+ f"known types are {', '.join(sorted(_RULE_TYPES))}"
262
+ )
263
+ regions = tuple(dict.fromkeys(phone_regions))
264
+ if any(region not in phonenumbers.SUPPORTED_REGIONS for region in regions):
265
+ raise ValueError("phone_regions must contain only supported upper-case region codes")
266
+ if PHONE in enabled and not regions:
267
+ raise ValueError("phone_regions must not be empty when PHONE is enabled")
268
+ self._rules = tuple(rule for rule in _RULES if rule.entity_type in enabled)
269
+ self._secrets = SECRET in enabled
270
+ self._phone_regions = regions if PHONE in enabled else ()
271
+
272
+ def detect(self, text: str) -> list[Span]:
273
+ """Spans of every enabled rule, sorted by (start, end)."""
274
+ spans = [
275
+ Span(start, end, rule.entity_type, rule.score, _SOURCE, PRIORITY_RULE)
276
+ for rule in self._rules
277
+ for start, end in _scan(rule, text)
278
+ ]
279
+ if self._secrets:
280
+ spans.extend(
281
+ Span(start, end, SECRET, _SECRET_SCORE, _SOURCE, PRIORITY_RULE)
282
+ for start, end in _scan_pem(text)
283
+ )
284
+ spans.extend(
285
+ Span(start, end, PHONE, _PHONE_SCORE, _SOURCE, PRIORITY_RULE)
286
+ for start, end in self._phone_offsets(text)
287
+ )
288
+ spans.sort(key=lambda span: (span.start, span.end))
289
+ return spans
290
+
291
+ def _phone_offsets(self, text: str) -> list[tuple[int, int]]:
292
+ """Numbers written with an international prefix (+ or the region's IDD) need only be
293
+ valid; national numbers must also be grouped as their country writes them, so that
294
+ digit groups such as "2013-015180" in an order code are not taken for a phone."""
295
+ offsets: dict[tuple[int, int], None] = {}
296
+ for region in self._phone_regions:
297
+ for match in _phone_matches(text, region, phonenumbers.Leniency.STRICT_GROUPING):
298
+ offsets[(match.start, match.end)] = None
299
+ for match in _phone_matches(text, region, phonenumbers.Leniency.VALID):
300
+ if _has_international_prefix(match.raw_string, region):
301
+ offsets[(match.start, match.end)] = None
302
+ return sorted(offsets)
303
+
304
+
305
+ def _phone_matches(
306
+ text: str, region: str, leniency: int
307
+ ) -> Iterator[phonenumbers.PhoneNumberMatch]:
308
+ # The library gives up silently after max_tries invalid candidates (65535 by default),
309
+ # which would leave later numbers in a long text undetected.
310
+ return iter(
311
+ phonenumbers.PhoneNumberMatcher(text, region, leniency=leniency, max_tries=sys.maxsize)
312
+ )
313
+
314
+
315
+ def _has_international_prefix(raw: str, region: str) -> bool:
316
+ # The matcher clears the country code source of the numbers it returns, so parse again.
317
+ number = phonenumbers.parse(raw, region, keep_raw_input=True)
318
+ return number.country_code_source in _INTERNATIONAL_SOURCES
@@ -0,0 +1 @@
1
+ """Dialects: the JSON shapes of provider APIs, translated into slots and back."""