aliasgate 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- aliasgate/__init__.py +5 -0
- aliasgate/__main__.py +8 -0
- aliasgate/audit.py +84 -0
- aliasgate/cli.py +321 -0
- aliasgate/config.py +367 -0
- aliasgate/detectors/__init__.py +1 -0
- aliasgate/detectors/cache.py +57 -0
- aliasgate/detectors/checksums.py +88 -0
- aliasgate/detectors/chunking.py +44 -0
- aliasgate/detectors/dictionary.py +96 -0
- aliasgate/detectors/gliner.py +195 -0
- aliasgate/detectors/literal.py +17 -0
- aliasgate/detectors/rules.py +318 -0
- aliasgate/dialects/__init__.py +1 -0
- aliasgate/dialects/anthropic_messages.py +329 -0
- aliasgate/dialects/base.py +193 -0
- aliasgate/dialects/openai_chat.py +359 -0
- aliasgate/dialects/sse.py +108 -0
- aliasgate/engine.py +237 -0
- aliasgate/eval/__init__.py +4 -0
- aliasgate/eval/__main__.py +139 -0
- aliasgate/eval/corpus.py +105 -0
- aliasgate/eval/metrics.py +197 -0
- aliasgate/eval/report.py +103 -0
- aliasgate/factory.py +67 -0
- aliasgate/mapping.py +103 -0
- aliasgate/metrics.py +131 -0
- aliasgate/placeholders.py +41 -0
- aliasgate/proxy/__init__.py +1 -0
- aliasgate/proxy/app.py +602 -0
- aliasgate/proxy/errors.py +106 -0
- aliasgate/proxy/headers.py +50 -0
- aliasgate/py.typed +0 -0
- aliasgate/resolve.py +32 -0
- aliasgate/stream.py +50 -0
- aliasgate/terms.py +18 -0
- aliasgate/types.py +70 -0
- aliasgate-0.1.0.dist-info/METADATA +151 -0
- aliasgate-0.1.0.dist-info/RECORD +42 -0
- aliasgate-0.1.0.dist-info/WHEEL +4 -0
- aliasgate-0.1.0.dist-info/entry_points.txt +2 -0
- aliasgate-0.1.0.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
"""ML detector (architecture §3.2, priority 40): a local GLiNER model, from the optional extra.
|
|
2
|
+
|
|
3
|
+
The model is injected through `EntityModel`, so this module never imports `gliner` (or torch)
|
|
4
|
+
unless `load_gliner` is called.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import importlib
|
|
8
|
+
import json
|
|
9
|
+
import math
|
|
10
|
+
import threading
|
|
11
|
+
from collections.abc import Callable, Iterator, Mapping
|
|
12
|
+
from typing import Final, Protocol, TypeGuard
|
|
13
|
+
|
|
14
|
+
from aliasgate.detectors.chunking import Window, windows
|
|
15
|
+
from aliasgate.types import PRIORITY_ML, Span, validate_entity_type
|
|
16
|
+
|
|
17
|
+
_SOURCE: Final = "gliner"
|
|
18
|
+
_MIN_SPAN_LENGTH: Final = 2
|
|
19
|
+
_INSTALL_HINT: Final = (
|
|
20
|
+
"the ML detector needs the optional 'ml' extra: install it with "
|
|
21
|
+
"pip install 'aliasgate[ml]' (or uv sync --extra ml in a checkout)"
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class EntityModel(Protocol):
|
|
26
|
+
"""A named-entity model: character offsets into `text`, labels from `labels`."""
|
|
27
|
+
|
|
28
|
+
def predict_entities(
|
|
29
|
+
self, text: str, labels: list[str], threshold: float
|
|
30
|
+
) -> list[dict[str, object]]: ...
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class GlinerDetector:
|
|
34
|
+
"""Finds people, organisations, addresses and other configured labels with a local model.
|
|
35
|
+
|
|
36
|
+
Long texts are cut into overlapping windows of the model's own words. Results are mapped
|
|
37
|
+
from model labels to entity types, trimmed of surrounding whitespace, deduplicated across
|
|
38
|
+
windows (the highest score wins), and returned sorted by position. A malformed model result
|
|
39
|
+
raises; errors name positions and types, never text. One inference runs at a time.
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
name = _SOURCE
|
|
43
|
+
|
|
44
|
+
def __init__(
|
|
45
|
+
self,
|
|
46
|
+
model: EntityModel,
|
|
47
|
+
*,
|
|
48
|
+
model_id: str,
|
|
49
|
+
revision: str | None = None,
|
|
50
|
+
labels: Mapping[str, str],
|
|
51
|
+
threshold: float,
|
|
52
|
+
window_words: int = 250,
|
|
53
|
+
overlap_words: int = 50,
|
|
54
|
+
) -> None:
|
|
55
|
+
if not labels or not all(labels):
|
|
56
|
+
raise ValueError("labels must map at least one non-empty model label")
|
|
57
|
+
for entity_type in labels.values():
|
|
58
|
+
validate_entity_type(entity_type)
|
|
59
|
+
if not 0.0 <= threshold <= 1.0: # also rejects NaN
|
|
60
|
+
raise ValueError("threshold must be between 0 and 1")
|
|
61
|
+
windows("", window_words=window_words, overlap_words=overlap_words) # validates both
|
|
62
|
+
self._model = model
|
|
63
|
+
self._labels = dict(labels)
|
|
64
|
+
self._label_names = list(labels)
|
|
65
|
+
self._threshold = threshold
|
|
66
|
+
self._window_words = window_words
|
|
67
|
+
self._overlap_words = overlap_words
|
|
68
|
+
self._lock = threading.Lock()
|
|
69
|
+
self._fingerprint = json.dumps(
|
|
70
|
+
{
|
|
71
|
+
"detector": _SOURCE,
|
|
72
|
+
"model": model_id,
|
|
73
|
+
"revision": revision,
|
|
74
|
+
"threshold": threshold,
|
|
75
|
+
"labels": sorted(self._labels.items()),
|
|
76
|
+
"window_words": window_words,
|
|
77
|
+
"overlap_words": overlap_words,
|
|
78
|
+
},
|
|
79
|
+
sort_keys=True,
|
|
80
|
+
separators=(",", ":"),
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
@property
|
|
84
|
+
def fingerprint(self) -> str:
|
|
85
|
+
"""Identifies the model and every setting that changes the output; used as cache key."""
|
|
86
|
+
return self._fingerprint
|
|
87
|
+
|
|
88
|
+
def detect(self, text: str) -> list[Span]:
|
|
89
|
+
"""Spans of priority 40 and source "gliner"; overlapping spans are left to the resolver."""
|
|
90
|
+
best: dict[tuple[int, int, str], Span] = {}
|
|
91
|
+
for window in windows(
|
|
92
|
+
text, window_words=self._window_words, overlap_words=self._overlap_words
|
|
93
|
+
):
|
|
94
|
+
chunk = text[window.start : window.end]
|
|
95
|
+
# Inference is not guaranteed to be thread-safe: one call at a time.
|
|
96
|
+
with self._lock:
|
|
97
|
+
results = self._model.predict_entities(
|
|
98
|
+
chunk, list(self._label_names), threshold=self._threshold
|
|
99
|
+
)
|
|
100
|
+
for position, result in enumerate(results, start=1):
|
|
101
|
+
span = self._span(result, position, window, text)
|
|
102
|
+
if span is None:
|
|
103
|
+
continue
|
|
104
|
+
key = (span.start, span.end, span.entity_type)
|
|
105
|
+
current = best.get(key)
|
|
106
|
+
if current is None or span.score > current.score:
|
|
107
|
+
best[key] = span
|
|
108
|
+
return sorted(best.values(), key=lambda span: (span.start, span.end, span.entity_type))
|
|
109
|
+
|
|
110
|
+
def _span(self, result: object, position: int, window: Window, text: str) -> Span | None:
|
|
111
|
+
where = f"model result {position} of a window"
|
|
112
|
+
if not isinstance(result, Mapping):
|
|
113
|
+
raise ValueError(f"{where} is not a mapping")
|
|
114
|
+
label = result.get("label")
|
|
115
|
+
if not isinstance(label, str) or label not in self._labels:
|
|
116
|
+
return None
|
|
117
|
+
start, end, score = result.get("start"), result.get("end"), result.get("score")
|
|
118
|
+
length = window.end - window.start
|
|
119
|
+
if not (_is_int(start) and _is_int(end) and 0 <= start <= end <= length):
|
|
120
|
+
raise ValueError(f"{where} has offsets outside the window (length {length})")
|
|
121
|
+
if not isinstance(score, int | float) or isinstance(score, bool) or math.isnan(score):
|
|
122
|
+
raise ValueError(f"{where} has a score that is not a number")
|
|
123
|
+
start += window.start
|
|
124
|
+
end += window.start
|
|
125
|
+
while start < end and text[start].isspace():
|
|
126
|
+
start += 1
|
|
127
|
+
while end > start and text[end - 1].isspace():
|
|
128
|
+
end -= 1
|
|
129
|
+
if end - start < _MIN_SPAN_LENGTH:
|
|
130
|
+
return None
|
|
131
|
+
clamped = min(max(float(score), 0.0), 1.0)
|
|
132
|
+
return Span(start, end, self._labels[label], clamped, _SOURCE, PRIORITY_ML)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _is_int(value: object) -> TypeGuard[int]:
|
|
136
|
+
return isinstance(value, int) and not isinstance(value, bool)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
class _GlinerLike(Protocol):
|
|
140
|
+
def predict_entities(
|
|
141
|
+
self, text: str, labels: list[str], *, threshold: float
|
|
142
|
+
) -> list[dict[str, object]]: ...
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
class _LoadedGliner:
|
|
146
|
+
"""A loaded GLiNER model that refuses any text it would truncate.
|
|
147
|
+
|
|
148
|
+
GLiNER silently keeps only the first `max_len` of its words (with a warning). The words
|
|
149
|
+
after that would never be scored, so a longer text is an error instead.
|
|
150
|
+
"""
|
|
151
|
+
|
|
152
|
+
def __init__(
|
|
153
|
+
self,
|
|
154
|
+
model: _GlinerLike,
|
|
155
|
+
splitter: Callable[[str], Iterator[object]],
|
|
156
|
+
max_len: int,
|
|
157
|
+
) -> None:
|
|
158
|
+
self._model = model
|
|
159
|
+
self._splitter = splitter
|
|
160
|
+
self._max_len = max_len
|
|
161
|
+
|
|
162
|
+
def predict_entities(
|
|
163
|
+
self, text: str, labels: list[str], threshold: float
|
|
164
|
+
) -> list[dict[str, object]]:
|
|
165
|
+
words = sum(1 for _ in self._splitter(text))
|
|
166
|
+
if words > self._max_len:
|
|
167
|
+
raise ValueError(
|
|
168
|
+
f"a window of {words} model words is over the model's limit of {self._max_len};"
|
|
169
|
+
" lower detection.ml.window_words"
|
|
170
|
+
)
|
|
171
|
+
# By keyword: GLiNER's third positional parameter is flat_ner.
|
|
172
|
+
return self._model.predict_entities(text, labels, threshold=threshold)
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def load_gliner(model_id: str, revision: str | None) -> EntityModel:
|
|
176
|
+
"""Import gliner lazily and load the model. Raise a clear error if the extra is missing.
|
|
177
|
+
|
|
178
|
+
The returned model refuses any text longer than the model's input limit instead of
|
|
179
|
+
letting GLiNER truncate it.
|
|
180
|
+
"""
|
|
181
|
+
try:
|
|
182
|
+
gliner = importlib.import_module("gliner")
|
|
183
|
+
except ImportError as exc:
|
|
184
|
+
raise RuntimeError(_INSTALL_HINT) from exc
|
|
185
|
+
model = gliner.GLiNER.from_pretrained(model_id, revision=revision)
|
|
186
|
+
try:
|
|
187
|
+
max_len = model.config.max_len
|
|
188
|
+
splitter = model.data_processor.words_splitter
|
|
189
|
+
except AttributeError:
|
|
190
|
+
max_len = splitter = None
|
|
191
|
+
if not _is_int(max_len) or max_len < 1 or not callable(splitter):
|
|
192
|
+
raise RuntimeError(
|
|
193
|
+
"cannot read the model's input limit; this gliner version is not supported"
|
|
194
|
+
)
|
|
195
|
+
return _LoadedGliner(model, splitter, max_len)
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"""Detector for input text that already looks like a placeholder (architecture §4.4)."""
|
|
2
|
+
|
|
3
|
+
from aliasgate.placeholders import TOLERANT
|
|
4
|
+
from aliasgate.types import LITERAL, PRIORITY_LITERAL, Span
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class LiteralDetector:
|
|
8
|
+
"""Reports placeholder-shaped input so that restoration can never mistake it for a value."""
|
|
9
|
+
|
|
10
|
+
name = "literal"
|
|
11
|
+
|
|
12
|
+
def detect(self, text: str) -> list[Span]:
|
|
13
|
+
"""One span of type LITERAL, priority 100, score 1.0, per match of TOLERANT."""
|
|
14
|
+
return [
|
|
15
|
+
Span(match.start(), match.end(), LITERAL, 1.0, self.name, PRIORITY_LITERAL)
|
|
16
|
+
for match in TOLERANT.finditer(text)
|
|
17
|
+
]
|
|
@@ -0,0 +1,318 @@
|
|
|
1
|
+
"""Rule-based detection of structured identifiers, validated by checksums or libraries."""
|
|
2
|
+
|
|
3
|
+
import ipaddress
|
|
4
|
+
import re
|
|
5
|
+
import string
|
|
6
|
+
import sys
|
|
7
|
+
from collections.abc import Callable, Collection, Iterator, Sequence
|
|
8
|
+
from dataclasses import dataclass
|
|
9
|
+
from typing import Final
|
|
10
|
+
|
|
11
|
+
import phonenumbers
|
|
12
|
+
|
|
13
|
+
from aliasgate.detectors.checksums import (
|
|
14
|
+
es_cif_valid,
|
|
15
|
+
es_dni_valid,
|
|
16
|
+
es_nie_valid,
|
|
17
|
+
iban_valid,
|
|
18
|
+
luhn_valid,
|
|
19
|
+
us_ssn_plausible,
|
|
20
|
+
)
|
|
21
|
+
from aliasgate.types import (
|
|
22
|
+
CREDIT_CARD,
|
|
23
|
+
EMAIL,
|
|
24
|
+
IBAN,
|
|
25
|
+
IP_ADDRESS,
|
|
26
|
+
NATIONAL_ID,
|
|
27
|
+
PHONE,
|
|
28
|
+
PRIORITY_RULE,
|
|
29
|
+
SECRET,
|
|
30
|
+
TAX_ID,
|
|
31
|
+
URL,
|
|
32
|
+
Span,
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
DEFAULT_RULE_TYPES: Final[frozenset[str]] = frozenset(
|
|
36
|
+
{EMAIL, PHONE, IBAN, CREDIT_CARD, NATIONAL_ID, TAX_ID, IP_ADDRESS, SECRET}
|
|
37
|
+
) # "URL" exists but is off by default
|
|
38
|
+
|
|
39
|
+
_RULE_TYPES: Final = DEFAULT_RULE_TYPES | {URL}
|
|
40
|
+
_SOURCE: Final = "rules"
|
|
41
|
+
_PHONE_SCORE: Final = 0.9
|
|
42
|
+
_ISBN_LENGTH: Final = 13
|
|
43
|
+
_ISBN_PREFIXES: Final = ("978", "979")
|
|
44
|
+
_INTERNATIONAL_SOURCES: Final = frozenset(
|
|
45
|
+
{
|
|
46
|
+
phonenumbers.CountryCodeSource.FROM_NUMBER_WITH_PLUS_SIGN,
|
|
47
|
+
phonenumbers.CountryCodeSource.FROM_NUMBER_WITH_IDD,
|
|
48
|
+
}
|
|
49
|
+
)
|
|
50
|
+
_SECRET_SCORE: Final = 1.0
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _ipv4_valid(candidate: str) -> bool:
|
|
54
|
+
try:
|
|
55
|
+
ipaddress.IPv4Address(candidate)
|
|
56
|
+
except ValueError:
|
|
57
|
+
return False
|
|
58
|
+
return True
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _ipv6_valid(candidate: str) -> bool:
|
|
62
|
+
# "::" alone parses, but it identifies nobody and is common in code ("x :: Int").
|
|
63
|
+
if not any(char in string.hexdigits for char in candidate):
|
|
64
|
+
return False
|
|
65
|
+
try:
|
|
66
|
+
ipaddress.IPv6Address(candidate)
|
|
67
|
+
except ValueError:
|
|
68
|
+
return False
|
|
69
|
+
return True
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _card_valid(candidate: str) -> bool:
|
|
73
|
+
"""Luhn-valid, and not 13 digits starting with 978 or 979: that is an ISBN-13 (the EAN
|
|
74
|
+
"Bookland" prefix), and no card network issues 13-digit numbers with that prefix."""
|
|
75
|
+
digits = candidate.replace(" ", "").replace("-", "")
|
|
76
|
+
if len(digits) == _ISBN_LENGTH and digits.startswith(_ISBN_PREFIXES):
|
|
77
|
+
return False
|
|
78
|
+
return luhn_valid(candidate)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _drop_last_group(separators: str, min_length: int) -> Callable[[str], str | None]:
|
|
82
|
+
"""Shortener that drops the last separator-delimited group, while at least `min_length`
|
|
83
|
+
non-separator characters would remain."""
|
|
84
|
+
|
|
85
|
+
def shorten(candidate: str) -> str | None:
|
|
86
|
+
cut = max(candidate.rfind(separator) for separator in separators)
|
|
87
|
+
if cut <= 0:
|
|
88
|
+
return None
|
|
89
|
+
shorter = candidate[:cut]
|
|
90
|
+
remaining = len(shorter) - sum(shorter.count(separator) for separator in separators)
|
|
91
|
+
return shorter if remaining >= min_length else None
|
|
92
|
+
|
|
93
|
+
return shorten
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
@dataclass(frozen=True, slots=True)
|
|
97
|
+
class _Rule:
|
|
98
|
+
"""A candidate pattern, an optional validator, and how to shorten a rejected candidate."""
|
|
99
|
+
|
|
100
|
+
entity_type: str
|
|
101
|
+
pattern: re.Pattern[str]
|
|
102
|
+
score: float
|
|
103
|
+
validate: Callable[[str], bool] | None = None
|
|
104
|
+
shorten: Callable[[str], str | None] | None = None
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
_RULES: Final[tuple[_Rule, ...]] = (
|
|
108
|
+
_Rule(
|
|
109
|
+
EMAIL,
|
|
110
|
+
re.compile(r"(?<![\w.%+-])[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)+(?![\w-])"),
|
|
111
|
+
1.0,
|
|
112
|
+
),
|
|
113
|
+
_Rule(
|
|
114
|
+
IBAN,
|
|
115
|
+
re.compile(
|
|
116
|
+
# Any letter case: people write IBANs in lower case too, and the checksum still
|
|
117
|
+
# filters out words that happen to have the shape.
|
|
118
|
+
r"(?<![A-Za-z0-9])[A-Za-z]{2}\d{2}(?:[ ]?[A-Za-z0-9]{4}){2,7}"
|
|
119
|
+
r"(?:[ ]?[A-Za-z0-9]{1,3})?(?![A-Za-z0-9])"
|
|
120
|
+
),
|
|
121
|
+
1.0,
|
|
122
|
+
iban_valid,
|
|
123
|
+
_drop_last_group(" ", min_length=15),
|
|
124
|
+
),
|
|
125
|
+
_Rule(
|
|
126
|
+
CREDIT_CARD,
|
|
127
|
+
re.compile(r"(?<![\d-])\d(?:[ -]?\d){12,18}(?![\d-])"),
|
|
128
|
+
1.0,
|
|
129
|
+
_card_valid,
|
|
130
|
+
_drop_last_group(" -", min_length=13),
|
|
131
|
+
),
|
|
132
|
+
_Rule(
|
|
133
|
+
NATIONAL_ID,
|
|
134
|
+
re.compile(r"(?<![A-Za-z0-9])\d{8}[- ]?[A-Za-z](?![A-Za-z0-9])"),
|
|
135
|
+
1.0,
|
|
136
|
+
es_dni_valid,
|
|
137
|
+
),
|
|
138
|
+
_Rule(
|
|
139
|
+
NATIONAL_ID,
|
|
140
|
+
re.compile(r"(?<![A-Za-z0-9])[XYZxyz][- ]?\d{7}[- ]?[A-Za-z](?![A-Za-z0-9])"),
|
|
141
|
+
1.0,
|
|
142
|
+
es_nie_valid,
|
|
143
|
+
),
|
|
144
|
+
_Rule(
|
|
145
|
+
NATIONAL_ID,
|
|
146
|
+
re.compile(r"(?<![\d-])\d{3}-\d{2}-\d{4}(?![\d-])"),
|
|
147
|
+
0.85,
|
|
148
|
+
us_ssn_plausible,
|
|
149
|
+
),
|
|
150
|
+
_Rule(
|
|
151
|
+
TAX_ID,
|
|
152
|
+
re.compile(r"(?<![A-Za-z0-9])[A-Za-z]\d{7}[0-9A-Ja-j](?![A-Za-z0-9])"),
|
|
153
|
+
1.0,
|
|
154
|
+
es_cif_valid,
|
|
155
|
+
),
|
|
156
|
+
_Rule(
|
|
157
|
+
IP_ADDRESS,
|
|
158
|
+
# A sentence may end right after the address, so a dot is a boundary unless a digit
|
|
159
|
+
# follows it.
|
|
160
|
+
re.compile(r"(?<!\d)(?<!\d\.)(?:\d{1,3}\.){3}\d{1,3}(?!\d)(?!\.\d)"),
|
|
161
|
+
0.9,
|
|
162
|
+
_ipv4_valid,
|
|
163
|
+
),
|
|
164
|
+
_Rule(
|
|
165
|
+
IP_ADDRESS,
|
|
166
|
+
re.compile(
|
|
167
|
+
r"(?<![\w:])(?:[0-9A-Fa-f]{0,4}:){2,8}"
|
|
168
|
+
r"(?:(?:\d{1,3}\.){3}\d{1,3}|[0-9A-Fa-f]{1,4})?(?![\w:])(?!\.\d)"
|
|
169
|
+
),
|
|
170
|
+
0.9,
|
|
171
|
+
_ipv6_valid,
|
|
172
|
+
),
|
|
173
|
+
_Rule(
|
|
174
|
+
URL,
|
|
175
|
+
# The last character excludes .,;:!? so that sentence punctuation is not part of it.
|
|
176
|
+
re.compile(r"https?://[^\s<>\"')\]]*[^\s<>\"')\].,;:!?]"),
|
|
177
|
+
0.9,
|
|
178
|
+
),
|
|
179
|
+
_Rule(SECRET, re.compile(r"(?<![A-Za-z0-9])sk-[A-Za-z0-9_-]{20,}"), _SECRET_SCORE),
|
|
180
|
+
_Rule(SECRET, re.compile(r"(?<![A-Za-z0-9])gh[pousr]_[A-Za-z0-9]{36,}"), _SECRET_SCORE),
|
|
181
|
+
_Rule(SECRET, re.compile(r"(?<![A-Za-z0-9])github_pat_[A-Za-z0-9_]{50,}"), _SECRET_SCORE),
|
|
182
|
+
_Rule(SECRET, re.compile(r"(?<![A-Z0-9])(?:AKIA|ASIA)[0-9A-Z]{16}(?![A-Z0-9])"), _SECRET_SCORE),
|
|
183
|
+
_Rule(SECRET, re.compile(r"(?<![A-Za-z0-9])xox[baprs]-[A-Za-z0-9-]{10,}"), _SECRET_SCORE),
|
|
184
|
+
_Rule(
|
|
185
|
+
SECRET,
|
|
186
|
+
re.compile(
|
|
187
|
+
r"(?<![A-Za-z0-9])eyJ[A-Za-z0-9_-]{5,}\.eyJ[A-Za-z0-9_-]{5,}\.[A-Za-z0-9_-]{5,}"
|
|
188
|
+
),
|
|
189
|
+
_SECRET_SCORE,
|
|
190
|
+
),
|
|
191
|
+
)
|
|
192
|
+
|
|
193
|
+
_PEM_BEGIN: Final = re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----")
|
|
194
|
+
_PEM_END: Final = re.compile(r"-----END [A-Z ]*PRIVATE KEY-----")
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def _scan(rule: _Rule, text: str) -> Iterator[tuple[int, int]]:
|
|
198
|
+
"""Yield (start, end) of every accepted candidate, leftmost first.
|
|
199
|
+
|
|
200
|
+
A rejected candidate is shortened while that is possible; if no version validates, the
|
|
201
|
+
search resumes one character after its start, so a valid value that begins inside a
|
|
202
|
+
rejected candidate is still found.
|
|
203
|
+
"""
|
|
204
|
+
position = 0
|
|
205
|
+
while (match := rule.pattern.search(text, position)) is not None:
|
|
206
|
+
start = match.start()
|
|
207
|
+
length = _accepted_length(rule, match.group())
|
|
208
|
+
if length is None:
|
|
209
|
+
position = start + 1
|
|
210
|
+
else:
|
|
211
|
+
yield start, start + length
|
|
212
|
+
position = start + length
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def _accepted_length(rule: _Rule, candidate: str) -> int | None:
|
|
216
|
+
if rule.validate is None:
|
|
217
|
+
return len(candidate)
|
|
218
|
+
current: str | None = candidate
|
|
219
|
+
while current is not None:
|
|
220
|
+
if rule.validate(current):
|
|
221
|
+
return len(current)
|
|
222
|
+
current = rule.shorten(current) if rule.shorten is not None else None
|
|
223
|
+
return None
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def _scan_pem(text: str) -> Iterator[tuple[int, int]]:
|
|
227
|
+
"""Yield each private key block from its BEGIN line to the first END line after it.
|
|
228
|
+
|
|
229
|
+
Same result as a non-greedy DOTALL pattern, in linear time: once a BEGIN line has no END
|
|
230
|
+
line after it, no later BEGIN line has one either.
|
|
231
|
+
"""
|
|
232
|
+
position = 0
|
|
233
|
+
while (begin := _PEM_BEGIN.search(text, position)) is not None:
|
|
234
|
+
end = _PEM_END.search(text, begin.end())
|
|
235
|
+
if end is None:
|
|
236
|
+
return
|
|
237
|
+
yield begin.start(), end.end()
|
|
238
|
+
position = end.end()
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
class RuleDetector:
|
|
242
|
+
"""Finds structured identifiers with patterns plus checksum or library validation.
|
|
243
|
+
|
|
244
|
+
Deterministic: the same text always yields the same spans, sorted by (start, end). Overlaps
|
|
245
|
+
between rules are kept; resolving them is the resolver's job.
|
|
246
|
+
"""
|
|
247
|
+
|
|
248
|
+
name = "rules"
|
|
249
|
+
|
|
250
|
+
def __init__(
|
|
251
|
+
self,
|
|
252
|
+
*,
|
|
253
|
+
enabled_types: Collection[str] = DEFAULT_RULE_TYPES,
|
|
254
|
+
phone_regions: Sequence[str] = ("ES", "US"),
|
|
255
|
+
) -> None:
|
|
256
|
+
enabled = frozenset(enabled_types)
|
|
257
|
+
unknown = enabled - _RULE_TYPES
|
|
258
|
+
if unknown:
|
|
259
|
+
raise ValueError(
|
|
260
|
+
f"{len(unknown)} unknown rule type(s) in enabled_types; "
|
|
261
|
+
f"known types are {', '.join(sorted(_RULE_TYPES))}"
|
|
262
|
+
)
|
|
263
|
+
regions = tuple(dict.fromkeys(phone_regions))
|
|
264
|
+
if any(region not in phonenumbers.SUPPORTED_REGIONS for region in regions):
|
|
265
|
+
raise ValueError("phone_regions must contain only supported upper-case region codes")
|
|
266
|
+
if PHONE in enabled and not regions:
|
|
267
|
+
raise ValueError("phone_regions must not be empty when PHONE is enabled")
|
|
268
|
+
self._rules = tuple(rule for rule in _RULES if rule.entity_type in enabled)
|
|
269
|
+
self._secrets = SECRET in enabled
|
|
270
|
+
self._phone_regions = regions if PHONE in enabled else ()
|
|
271
|
+
|
|
272
|
+
def detect(self, text: str) -> list[Span]:
|
|
273
|
+
"""Spans of every enabled rule, sorted by (start, end)."""
|
|
274
|
+
spans = [
|
|
275
|
+
Span(start, end, rule.entity_type, rule.score, _SOURCE, PRIORITY_RULE)
|
|
276
|
+
for rule in self._rules
|
|
277
|
+
for start, end in _scan(rule, text)
|
|
278
|
+
]
|
|
279
|
+
if self._secrets:
|
|
280
|
+
spans.extend(
|
|
281
|
+
Span(start, end, SECRET, _SECRET_SCORE, _SOURCE, PRIORITY_RULE)
|
|
282
|
+
for start, end in _scan_pem(text)
|
|
283
|
+
)
|
|
284
|
+
spans.extend(
|
|
285
|
+
Span(start, end, PHONE, _PHONE_SCORE, _SOURCE, PRIORITY_RULE)
|
|
286
|
+
for start, end in self._phone_offsets(text)
|
|
287
|
+
)
|
|
288
|
+
spans.sort(key=lambda span: (span.start, span.end))
|
|
289
|
+
return spans
|
|
290
|
+
|
|
291
|
+
def _phone_offsets(self, text: str) -> list[tuple[int, int]]:
|
|
292
|
+
"""Numbers written with an international prefix (+ or the region's IDD) need only be
|
|
293
|
+
valid; national numbers must also be grouped as their country writes them, so that
|
|
294
|
+
digit groups such as "2013-015180" in an order code are not taken for a phone."""
|
|
295
|
+
offsets: dict[tuple[int, int], None] = {}
|
|
296
|
+
for region in self._phone_regions:
|
|
297
|
+
for match in _phone_matches(text, region, phonenumbers.Leniency.STRICT_GROUPING):
|
|
298
|
+
offsets[(match.start, match.end)] = None
|
|
299
|
+
for match in _phone_matches(text, region, phonenumbers.Leniency.VALID):
|
|
300
|
+
if _has_international_prefix(match.raw_string, region):
|
|
301
|
+
offsets[(match.start, match.end)] = None
|
|
302
|
+
return sorted(offsets)
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
def _phone_matches(
|
|
306
|
+
text: str, region: str, leniency: int
|
|
307
|
+
) -> Iterator[phonenumbers.PhoneNumberMatch]:
|
|
308
|
+
# The library gives up silently after max_tries invalid candidates (65535 by default),
|
|
309
|
+
# which would leave later numbers in a long text undetected.
|
|
310
|
+
return iter(
|
|
311
|
+
phonenumbers.PhoneNumberMatcher(text, region, leniency=leniency, max_tries=sys.maxsize)
|
|
312
|
+
)
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def _has_international_prefix(raw: str, region: str) -> bool:
|
|
316
|
+
# The matcher clears the country code source of the numbers it returns, so parse again.
|
|
317
|
+
number = phonenumbers.parse(raw, region, keep_raw_input=True)
|
|
318
|
+
return number.country_code_source in _INTERNATIONAL_SOURCES
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Dialects: the JSON shapes of provider APIs, translated into slots and back."""
|