aliasgate 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- aliasgate/__init__.py +5 -0
- aliasgate/__main__.py +8 -0
- aliasgate/audit.py +84 -0
- aliasgate/cli.py +321 -0
- aliasgate/config.py +367 -0
- aliasgate/detectors/__init__.py +1 -0
- aliasgate/detectors/cache.py +57 -0
- aliasgate/detectors/checksums.py +88 -0
- aliasgate/detectors/chunking.py +44 -0
- aliasgate/detectors/dictionary.py +96 -0
- aliasgate/detectors/gliner.py +195 -0
- aliasgate/detectors/literal.py +17 -0
- aliasgate/detectors/rules.py +318 -0
- aliasgate/dialects/__init__.py +1 -0
- aliasgate/dialects/anthropic_messages.py +329 -0
- aliasgate/dialects/base.py +193 -0
- aliasgate/dialects/openai_chat.py +359 -0
- aliasgate/dialects/sse.py +108 -0
- aliasgate/engine.py +237 -0
- aliasgate/eval/__init__.py +4 -0
- aliasgate/eval/__main__.py +139 -0
- aliasgate/eval/corpus.py +105 -0
- aliasgate/eval/metrics.py +197 -0
- aliasgate/eval/report.py +103 -0
- aliasgate/factory.py +67 -0
- aliasgate/mapping.py +103 -0
- aliasgate/metrics.py +131 -0
- aliasgate/placeholders.py +41 -0
- aliasgate/proxy/__init__.py +1 -0
- aliasgate/proxy/app.py +602 -0
- aliasgate/proxy/errors.py +106 -0
- aliasgate/proxy/headers.py +50 -0
- aliasgate/py.typed +0 -0
- aliasgate/resolve.py +32 -0
- aliasgate/stream.py +50 -0
- aliasgate/terms.py +18 -0
- aliasgate/types.py +70 -0
- aliasgate-0.1.0.dist-info/METADATA +151 -0
- aliasgate-0.1.0.dist-info/RECORD +42 -0
- aliasgate-0.1.0.dist-info/WHEEL +4 -0
- aliasgate-0.1.0.dist-info/entry_points.txt +2 -0
- aliasgate-0.1.0.dist-info/licenses/LICENSE +202 -0
aliasgate/config.py
ADDED
|
@@ -0,0 +1,367 @@
|
|
|
1
|
+
"""Gateway configuration (architecture §9) and dictionary files.
|
|
2
|
+
|
|
3
|
+
`load_settings` reads one YAML file, overlays `ALIASGATE_*` environment variables and returns
|
|
4
|
+
an immutable, validated `Settings`. Every section forbids unknown keys, and validation errors
|
|
5
|
+
never repeat input values. `Settings` holds no secrets: the gateway forwards client
|
|
6
|
+
credentials and stores none.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import os
|
|
10
|
+
import re
|
|
11
|
+
from collections.abc import Mapping
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from typing import Annotated, Any, Final, Literal, Self
|
|
14
|
+
from urllib.parse import unquote, urlsplit
|
|
15
|
+
|
|
16
|
+
import yaml
|
|
17
|
+
from pydantic import (
|
|
18
|
+
AfterValidator,
|
|
19
|
+
BaseModel,
|
|
20
|
+
ConfigDict,
|
|
21
|
+
Field,
|
|
22
|
+
StringConstraints,
|
|
23
|
+
ValidationInfo,
|
|
24
|
+
create_model,
|
|
25
|
+
field_validator,
|
|
26
|
+
model_validator,
|
|
27
|
+
)
|
|
28
|
+
from pydantic_settings import BaseSettings, EnvSettingsSource
|
|
29
|
+
from pydantic_settings.sources.utils import parse_env_vars
|
|
30
|
+
|
|
31
|
+
from aliasgate.detectors.dictionary import Term
|
|
32
|
+
from aliasgate.mapping import normalize_key
|
|
33
|
+
from aliasgate.types import validate_entity_type
|
|
34
|
+
|
|
35
|
+
_ENV_PREFIX: Final = "ALIASGATE_"
|
|
36
|
+
_ENV_NESTED_DELIMITER: Final = "__"
|
|
37
|
+
_BASE_DIR: Final = "base_dir"
|
|
38
|
+
# One path segment as RFC 3986 §3.3 writes it: unreserved characters, sub-delimiters, ":" and
|
|
39
|
+
# "@", and percent-encoded octets. httpx sends such a segment exactly as written.
|
|
40
|
+
_PATH_SEGMENT: Final = re.compile(r"(?:[A-Za-z0-9\-._~!$&'()*+,;=:@]|%[0-9A-Fa-f]{2})+")
|
|
41
|
+
_DOT_SEGMENTS: Final = frozenset({".", ".."})
|
|
42
|
+
# A passthrough path segment: the characters of _PATH_SEGMENT that never need escaping.
|
|
43
|
+
_PLAIN_SEGMENT: Final = re.compile(r"[A-Za-z0-9\-._~!$&'()*+,;=:@]+")
|
|
44
|
+
_PASSTHROUGH_UPSTREAMS: Final = frozenset({"openai", "anthropic"}) # fields of UpstreamsSettings
|
|
45
|
+
_PASSTHROUGH_METHODS: Final = frozenset({"GET", "POST", "DELETE"})
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _base_url(value: str) -> str:
|
|
49
|
+
"""An http(s) URL with an optional base path, returned without trailing slashes, such that
|
|
50
|
+
appending a request path to it gives exactly the URL that is sent (architecture §7)."""
|
|
51
|
+
# urlsplit drops tabs and newlines, and an empty "?" or "#" leaves no query or fragment, so
|
|
52
|
+
# these are checked on the raw value: after concatenation they would move the request path.
|
|
53
|
+
if any(char in "?#" or char.isspace() or not char.isprintable() for char in value):
|
|
54
|
+
raise ValueError("must not contain '?', '#', whitespace or control characters")
|
|
55
|
+
parts = urlsplit(value)
|
|
56
|
+
if parts.scheme not in ("http", "https") or not parts.hostname:
|
|
57
|
+
raise ValueError("must be an http or https URL with a host")
|
|
58
|
+
if parts.username is not None or parts.password is not None:
|
|
59
|
+
raise ValueError("must not contain credentials")
|
|
60
|
+
_ = parts.port # raises ValueError for a malformed port
|
|
61
|
+
for segment in parts.path.rstrip("/").split("/")[1:]:
|
|
62
|
+
if not segment:
|
|
63
|
+
raise ValueError("the path must not have empty segments")
|
|
64
|
+
if not _PATH_SEGMENT.fullmatch(segment):
|
|
65
|
+
raise ValueError(
|
|
66
|
+
"the path may only contain letters, digits, -._~!$&'()*+,;=:@ and %XX escapes"
|
|
67
|
+
)
|
|
68
|
+
if unquote(segment) in _DOT_SEGMENTS:
|
|
69
|
+
raise ValueError("the path must not have '.' or '..' segments")
|
|
70
|
+
return value.rstrip("/")
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _passthrough_entry(value: str) -> str:
|
|
74
|
+
"""`"<upstream> <METHOD> <path>"` with a path of non-empty plain segments. Requests match on
|
|
75
|
+
their decoded path and are forwarded with the entry's path, so a path that needs no escaping
|
|
76
|
+
is the only kind for which the two are the same text."""
|
|
77
|
+
parts = value.split(" ")
|
|
78
|
+
if (
|
|
79
|
+
len(parts) != 3
|
|
80
|
+
or parts[0] not in _PASSTHROUGH_UPSTREAMS
|
|
81
|
+
or parts[1] not in _PASSTHROUGH_METHODS
|
|
82
|
+
or not parts[2].startswith("/")
|
|
83
|
+
):
|
|
84
|
+
raise ValueError(
|
|
85
|
+
"must be '<upstream> <METHOD> <path>' with upstream openai or anthropic, method GET,"
|
|
86
|
+
" POST or DELETE, and a path starting with '/'"
|
|
87
|
+
)
|
|
88
|
+
for segment in parts[2].split("/")[1:]:
|
|
89
|
+
if not segment:
|
|
90
|
+
raise ValueError("the path must not have empty segments")
|
|
91
|
+
if not _PLAIN_SEGMENT.fullmatch(segment):
|
|
92
|
+
raise ValueError("the path may only contain letters, digits and -._~!$&'()*+,;=:@")
|
|
93
|
+
if segment in _DOT_SEGMENTS:
|
|
94
|
+
raise ValueError("the path must not have '.' or '..' segments")
|
|
95
|
+
return value
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
BaseUrl = Annotated[str, AfterValidator(_base_url)]
|
|
99
|
+
EntityType = Annotated[str, AfterValidator(validate_entity_type)]
|
|
100
|
+
PhoneRegion = Annotated[str, StringConstraints(pattern=r"^[A-Z]{2}$")]
|
|
101
|
+
PassthroughPath = Annotated[str, AfterValidator(_passthrough_entry)]
|
|
102
|
+
ModelLabel = Annotated[str, StringConstraints(min_length=1)]
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
class _Section(BaseModel):
|
|
106
|
+
model_config = ConfigDict(extra="forbid", frozen=True, hide_input_in_errors=True)
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
class ListenSettings(_Section):
|
|
110
|
+
"""Address the gateway serves on; loopback unless chosen otherwise."""
|
|
111
|
+
|
|
112
|
+
host: str = "127.0.0.1"
|
|
113
|
+
port: int = Field(default=8080, ge=1, le=65535)
|
|
114
|
+
max_request_bytes: int = Field(default=10_485_760, ge=1)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
class UpstreamSettings(_Section):
|
|
118
|
+
"""One provider; `base_url` may end in a base path and has no trailing slash, so
|
|
119
|
+
`base_url + request path` is the upstream URL."""
|
|
120
|
+
|
|
121
|
+
base_url: BaseUrl
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
class UpstreamsSettings(_Section):
|
|
125
|
+
"""Providers the gateway forwards to, and the extra paths it may forward unchanged.
|
|
126
|
+
|
|
127
|
+
A passthrough entry is `"<upstream> <METHOD> <path>"`, for example
|
|
128
|
+
`"anthropic GET /v1/models"`; each method and path appears in at most one entry, so a
|
|
129
|
+
request names exactly one upstream.
|
|
130
|
+
"""
|
|
131
|
+
|
|
132
|
+
openai: UpstreamSettings = UpstreamSettings(base_url="https://api.openai.com")
|
|
133
|
+
anthropic: UpstreamSettings = UpstreamSettings(base_url="https://api.anthropic.com")
|
|
134
|
+
timeout_seconds: float = Field(default=600, gt=0)
|
|
135
|
+
passthrough_paths: tuple[PassthroughPath, ...] = ()
|
|
136
|
+
|
|
137
|
+
@field_validator("passthrough_paths")
|
|
138
|
+
@classmethod
|
|
139
|
+
def _one_upstream_per_path(cls, entries: tuple[str, ...]) -> tuple[str, ...]:
|
|
140
|
+
routes = [entry.split(" ", 1)[1] for entry in entries]
|
|
141
|
+
if len(set(routes)) != len(routes):
|
|
142
|
+
raise ValueError("each method and path may appear in only one entry")
|
|
143
|
+
return entries
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
class RulesSettings(_Section):
|
|
147
|
+
"""Rule detector: which rule types run, and the regions for national phone formats."""
|
|
148
|
+
|
|
149
|
+
enabled_types: tuple[EntityType, ...] = (
|
|
150
|
+
"EMAIL",
|
|
151
|
+
"PHONE",
|
|
152
|
+
"IBAN",
|
|
153
|
+
"CREDIT_CARD",
|
|
154
|
+
"NATIONAL_ID",
|
|
155
|
+
"TAX_ID",
|
|
156
|
+
"IP_ADDRESS",
|
|
157
|
+
"SECRET",
|
|
158
|
+
)
|
|
159
|
+
phone_regions: tuple[PhoneRegion, ...] = ("ES", "US")
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
class DictionarySettings(_Section):
|
|
163
|
+
"""Dictionary files; relative paths are resolved against the configuration file."""
|
|
164
|
+
|
|
165
|
+
files: tuple[Path, ...] = ()
|
|
166
|
+
|
|
167
|
+
@field_validator("files")
|
|
168
|
+
@classmethod
|
|
169
|
+
def _resolve_relative_paths(
|
|
170
|
+
cls, files: tuple[Path, ...], info: ValidationInfo
|
|
171
|
+
) -> tuple[Path, ...]:
|
|
172
|
+
context = info.context if isinstance(info.context, dict) else {}
|
|
173
|
+
base = context.get(_BASE_DIR, Path.cwd())
|
|
174
|
+
return tuple(file if file.is_absolute() else base / file for file in files)
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
class MLSettings(_Section):
|
|
178
|
+
"""ML detector: a GLiNER model from the optional `ml` extra, off by default.
|
|
179
|
+
|
|
180
|
+
The default model is `urchade/gliner_multi_pii-v1` (Apache-2.0; en, fr, de, es, pt, it),
|
|
181
|
+
pinned to a revision. `labels` maps model labels to entity types. Texts are cut into windows
|
|
182
|
+
of `window_words` model words overlapping by `overlap_words`; results are cached for
|
|
183
|
+
`cache_entries` texts (0 disables the cache).
|
|
184
|
+
"""
|
|
185
|
+
|
|
186
|
+
enabled: bool = False
|
|
187
|
+
model: Annotated[str, StringConstraints(min_length=1)] = "urchade/gliner_multi_pii-v1"
|
|
188
|
+
revision: Annotated[str, StringConstraints(min_length=1)] | None = (
|
|
189
|
+
"1fcf13e85f4eef5394e1fcd406cf2ca9ea82351d"
|
|
190
|
+
)
|
|
191
|
+
threshold: float = Field(default=0.4, ge=0.0, le=1.0)
|
|
192
|
+
labels: dict[ModelLabel, EntityType] = Field(
|
|
193
|
+
default_factory=lambda: {"person": "PERSON", "organization": "ORG", "address": "ADDRESS"},
|
|
194
|
+
min_length=1,
|
|
195
|
+
)
|
|
196
|
+
window_words: int = Field(default=250, ge=1)
|
|
197
|
+
overlap_words: int = Field(default=50, ge=0)
|
|
198
|
+
cache_entries: int = Field(default=4096, ge=0)
|
|
199
|
+
|
|
200
|
+
@model_validator(mode="after")
|
|
201
|
+
def _overlap_is_smaller_than_the_window(self) -> Self:
|
|
202
|
+
if self.overlap_words >= self.window_words:
|
|
203
|
+
raise ValueError("overlap_words must be smaller than window_words")
|
|
204
|
+
return self
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
class DetectionSettings(_Section):
|
|
208
|
+
"""Detection layers, allow list and propagation."""
|
|
209
|
+
|
|
210
|
+
rules: RulesSettings = RulesSettings()
|
|
211
|
+
dictionary: DictionarySettings = DictionarySettings()
|
|
212
|
+
allow_list: tuple[str, ...] = ()
|
|
213
|
+
ml: MLSettings = MLSettings()
|
|
214
|
+
propagate: bool = True
|
|
215
|
+
min_propagation_length: int = Field(default=3, ge=1)
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
class PolicySettings(_Section):
|
|
219
|
+
"""What happens to content the gateway cannot pseudonymise or should not forward.
|
|
220
|
+
|
|
221
|
+
With `leak_audit`, a request whose outbound body still holds a detected value is rejected
|
|
222
|
+
instead of forwarded (architecture §8).
|
|
223
|
+
"""
|
|
224
|
+
|
|
225
|
+
unsupported_content: Literal["block", "pass"] = "block"
|
|
226
|
+
strip_user_identifiers: bool = True
|
|
227
|
+
tool_definitions: bool = True
|
|
228
|
+
leak_audit: bool = True
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
class LoggingSettings(_Section):
|
|
232
|
+
"""Log level. Logs never contain text (architecture §7.1)."""
|
|
233
|
+
|
|
234
|
+
level: Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] = "INFO"
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
class MetricsSettings(_Section):
|
|
238
|
+
"""Prometheus metrics at `GET /metrics`, on the gateway's own port. Labels never hold text."""
|
|
239
|
+
|
|
240
|
+
enabled: bool = True
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
class Settings(_Section):
|
|
244
|
+
"""Complete gateway configuration: immutable, validated, free of secrets."""
|
|
245
|
+
|
|
246
|
+
listen: ListenSettings = ListenSettings()
|
|
247
|
+
upstreams: UpstreamsSettings = UpstreamsSettings()
|
|
248
|
+
detection: DetectionSettings = DetectionSettings()
|
|
249
|
+
policy: PolicySettings = PolicySettings()
|
|
250
|
+
logging: LoggingSettings = LoggingSettings()
|
|
251
|
+
metrics: MetricsSettings = MetricsSettings()
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
# The fields of Settings on a BaseSettings class, which is what pydantic-settings needs to parse
|
|
255
|
+
# environment variables. Settings itself stays a plain model that never reads the environment.
|
|
256
|
+
_SETTINGS_FIELDS: Final[dict[str, Any]] = {
|
|
257
|
+
name: (field.annotation, field) for name, field in Settings.model_fields.items()
|
|
258
|
+
}
|
|
259
|
+
_EnvironmentShape = create_model("_EnvironmentShape", __base__=BaseSettings, **_SETTINGS_FIELDS)
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
class _EnvironmentSource(EnvSettingsSource):
|
|
263
|
+
"""Parses `ALIASGATE_*` variables from a given mapping instead of `os.environ`."""
|
|
264
|
+
|
|
265
|
+
def __init__(self, env: Mapping[str, str]) -> None:
|
|
266
|
+
self._given_env = env
|
|
267
|
+
super().__init__(
|
|
268
|
+
_EnvironmentShape,
|
|
269
|
+
case_sensitive=False,
|
|
270
|
+
env_prefix=_ENV_PREFIX,
|
|
271
|
+
env_nested_delimiter=_ENV_NESTED_DELIMITER,
|
|
272
|
+
)
|
|
273
|
+
|
|
274
|
+
def _load_env_vars(self) -> Mapping[str, str | None]:
|
|
275
|
+
return parse_env_vars(
|
|
276
|
+
self._given_env, self.case_sensitive, self.env_ignore_empty, self.env_parse_none_str
|
|
277
|
+
)
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
def _environment_data(env: Mapping[str, str]) -> dict[str, Any]:
|
|
281
|
+
data = dict(_EnvironmentSource(env)())
|
|
282
|
+
for name, value in env.items():
|
|
283
|
+
if not name.upper().startswith(_ENV_PREFIX):
|
|
284
|
+
continue
|
|
285
|
+
section = name[len(_ENV_PREFIX) :].split(_ENV_NESTED_DELIMITER, 1)[0].lower()
|
|
286
|
+
if section not in Settings.model_fields:
|
|
287
|
+
# pydantic-settings skips variables that match no field; keep them so that
|
|
288
|
+
# extra="forbid" turns a misspelled variable into a start-up error.
|
|
289
|
+
data[name] = value
|
|
290
|
+
return data
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
def _deep_merge(
|
|
294
|
+
base: Mapping[str, Any], override: Mapping[str, Any], section: type[BaseModel]
|
|
295
|
+
) -> dict[str, Any]:
|
|
296
|
+
"""Sections merge key by key; any other value, mappings such as `ml.labels` included, is
|
|
297
|
+
replaced, so the environment can also remove entries the file sets."""
|
|
298
|
+
merged = dict(base)
|
|
299
|
+
for key, value in override.items():
|
|
300
|
+
current = merged.get(key)
|
|
301
|
+
field = section.model_fields.get(key)
|
|
302
|
+
subsection = field.annotation if field is not None else None
|
|
303
|
+
if (
|
|
304
|
+
isinstance(subsection, type)
|
|
305
|
+
and issubclass(subsection, BaseModel)
|
|
306
|
+
and isinstance(value, Mapping)
|
|
307
|
+
and isinstance(current, Mapping)
|
|
308
|
+
):
|
|
309
|
+
merged[key] = _deep_merge(current, value, subsection)
|
|
310
|
+
else:
|
|
311
|
+
merged[key] = value
|
|
312
|
+
return merged
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def load_settings(path: Path | None, env: Mapping[str, str] = os.environ) -> Settings:
|
|
316
|
+
"""Load YAML (if a path is given), overlay ALIASGATE_* environment variables, validate."""
|
|
317
|
+
file_data: Any = {}
|
|
318
|
+
if path is not None:
|
|
319
|
+
with path.open(encoding="utf-8") as stream:
|
|
320
|
+
file_data = yaml.safe_load(stream)
|
|
321
|
+
if file_data is None:
|
|
322
|
+
file_data = {}
|
|
323
|
+
env_data = _environment_data(env)
|
|
324
|
+
data = (
|
|
325
|
+
_deep_merge(file_data, env_data, Settings) if isinstance(file_data, Mapping) else file_data
|
|
326
|
+
)
|
|
327
|
+
base_dir = path.absolute().parent if path is not None else Path.cwd()
|
|
328
|
+
return Settings.model_validate(data, context={_BASE_DIR: base_dir})
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
def load_terms(path: Path) -> list[Term]:
|
|
332
|
+
"""Parse and validate a dictionary file; errors name entry positions, never term content."""
|
|
333
|
+
with path.open(encoding="utf-8") as stream:
|
|
334
|
+
try:
|
|
335
|
+
data = yaml.safe_load(stream)
|
|
336
|
+
except yaml.YAMLError as exc:
|
|
337
|
+
# The exception text quotes the offending line, which may contain a term.
|
|
338
|
+
mark = getattr(exc, "problem_mark", None)
|
|
339
|
+
where = f" at line {mark.line + 1}, column {mark.column + 1}" if mark else ""
|
|
340
|
+
raise ValueError(f"dictionary file {path} is not valid YAML{where}") from None
|
|
341
|
+
if not isinstance(data, dict) or set(data) != {"terms"}:
|
|
342
|
+
raise ValueError(f"dictionary file {path} must be a mapping with a single 'terms' key")
|
|
343
|
+
entries = data["terms"] if data["terms"] is not None else []
|
|
344
|
+
if not isinstance(entries, list):
|
|
345
|
+
raise ValueError(f"dictionary file {path}: 'terms' must be a list")
|
|
346
|
+
return [_term(path, position, entry) for position, entry in enumerate(entries, start=1)]
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
def _term(path: Path, position: int, entry: object) -> Term:
|
|
350
|
+
where = f"dictionary file {path}, entry {position}"
|
|
351
|
+
if not isinstance(entry, dict):
|
|
352
|
+
raise ValueError(f"{where}: must be a mapping with 'term' and 'type'")
|
|
353
|
+
if set(entry) - {"term", "type"}:
|
|
354
|
+
raise ValueError(f"{where}: only 'term' and 'type' are allowed")
|
|
355
|
+
for key in ("term", "type"):
|
|
356
|
+
if key not in entry:
|
|
357
|
+
raise ValueError(f"{where}: missing '{key}'")
|
|
358
|
+
if not isinstance(entry[key], str):
|
|
359
|
+
raise ValueError(f"{where}: '{key}' must be a string")
|
|
360
|
+
term, entity_type = entry["term"], entry["type"]
|
|
361
|
+
try:
|
|
362
|
+
validate_entity_type(entity_type)
|
|
363
|
+
except ValueError as exc:
|
|
364
|
+
raise ValueError(f"{where}: {exc}") from None
|
|
365
|
+
if not normalize_key(term):
|
|
366
|
+
raise ValueError(f"{where}: 'term' is empty")
|
|
367
|
+
return Term(term, entity_type)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Detectors: components that find spans of entities in a text."""
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""Detection cache (architecture §4.5): conversations resend their history on every turn."""
|
|
2
|
+
|
|
3
|
+
import hashlib
|
|
4
|
+
import threading
|
|
5
|
+
from collections import OrderedDict
|
|
6
|
+
|
|
7
|
+
from aliasgate.types import Detector, Span
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class CachedDetector:
|
|
11
|
+
"""Wraps a detector with a bounded, thread-safe LRU cache of its results.
|
|
12
|
+
|
|
13
|
+
Entries are keyed by `sha256(fingerprint + "\\x00" + text)` and hold spans only, never
|
|
14
|
+
text, in process memory only. Failures of the inner detector are not cached, and every
|
|
15
|
+
call returns a new list. `max_entries == 0` disables caching.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
def __init__(self, inner: Detector, *, fingerprint: str, max_entries: int) -> None:
|
|
19
|
+
if max_entries < 0:
|
|
20
|
+
raise ValueError("max_entries must not be negative")
|
|
21
|
+
self.name: str = inner.name
|
|
22
|
+
self._inner = inner
|
|
23
|
+
self._fingerprint = fingerprint
|
|
24
|
+
self._max_entries = max_entries
|
|
25
|
+
self._entries: OrderedDict[bytes, tuple[Span, ...]] = OrderedDict()
|
|
26
|
+
self._lock = threading.Lock()
|
|
27
|
+
self._hits = 0
|
|
28
|
+
self._misses = 0
|
|
29
|
+
|
|
30
|
+
def detect(self, text: str) -> list[Span]:
|
|
31
|
+
"""The inner detector's spans for `text`, from the cache when it has seen `text`."""
|
|
32
|
+
# "surrogatepass" keeps the encoding injective for texts with unpaired surrogates,
|
|
33
|
+
# which JSON allows; plain UTF-8 would refuse them.
|
|
34
|
+
key = hashlib.sha256(
|
|
35
|
+
(self._fingerprint + "\x00" + text).encode("utf-8", "surrogatepass")
|
|
36
|
+
).digest()
|
|
37
|
+
with self._lock:
|
|
38
|
+
cached = self._entries.get(key)
|
|
39
|
+
if cached is not None:
|
|
40
|
+
self._entries.move_to_end(key)
|
|
41
|
+
self._hits += 1
|
|
42
|
+
return list(cached)
|
|
43
|
+
self._misses += 1
|
|
44
|
+
# Outside the lock, so that a slow detection does not hold up cache hits.
|
|
45
|
+
spans = tuple(self._inner.detect(text))
|
|
46
|
+
if self._max_entries:
|
|
47
|
+
with self._lock:
|
|
48
|
+
self._entries[key] = spans
|
|
49
|
+
self._entries.move_to_end(key)
|
|
50
|
+
while len(self._entries) > self._max_entries:
|
|
51
|
+
self._entries.popitem(last=False)
|
|
52
|
+
return list(spans)
|
|
53
|
+
|
|
54
|
+
def stats(self) -> tuple[int, int]:
|
|
55
|
+
"""(hits, misses) since construction; a failed detection counts as a miss."""
|
|
56
|
+
with self._lock:
|
|
57
|
+
return self._hits, self._misses
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
"""Checksum and plausibility checks for structured identifiers.
|
|
2
|
+
|
|
3
|
+
Every function ignores spaces and hyphens, accepts letters in either case, only accepts ASCII
|
|
4
|
+
digits and letters, returns False for malformed input and never raises.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import re
|
|
8
|
+
from typing import Final
|
|
9
|
+
|
|
10
|
+
_DNI_LETTERS: Final = "TRWAGMYFPDXBNJZSQVHLCKE"
|
|
11
|
+
_NIE_PREFIX_DIGIT: Final = {"X": "0", "Y": "1", "Z": "2"}
|
|
12
|
+
_CIF_CONTROL_LETTERS: Final = "JABCDEFGHI"
|
|
13
|
+
|
|
14
|
+
_CARD: Final = re.compile(r"[0-9]{13,19}")
|
|
15
|
+
_IBAN: Final = re.compile(r"[A-Z]{2}[0-9]{2}[A-Z0-9]{11,30}")
|
|
16
|
+
_DNI: Final = re.compile(r"([0-9]{8})([A-Z])")
|
|
17
|
+
_NIE: Final = re.compile(r"([XYZ])([0-9]{7})([A-Z])")
|
|
18
|
+
_CIF: Final = re.compile(r"[ABCDEFGHJNPQRSUVW]([0-9]{7})([0-9A-J])")
|
|
19
|
+
_SSN: Final = re.compile(r"([0-9]{3})([0-9]{2})([0-9]{4})")
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _compact(value: str) -> str:
|
|
23
|
+
"""Remove spaces and hyphens and upper-case; non-ASCII input becomes "" (always invalid)."""
|
|
24
|
+
compact = value.replace(" ", "").replace("-", "")
|
|
25
|
+
# Checked before upper-casing: "\u017f".upper() == "S" and "ß".upper() == "SS".
|
|
26
|
+
return compact.upper() if compact.isascii() else ""
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def luhn_valid(number: str) -> bool:
|
|
30
|
+
"""True if `number` has 13 to 19 digits and passes the Luhn mod 10 check."""
|
|
31
|
+
digits = _compact(number)
|
|
32
|
+
if _CARD.fullmatch(digits) is None:
|
|
33
|
+
return False
|
|
34
|
+
total = 0
|
|
35
|
+
for position, char in enumerate(reversed(digits)):
|
|
36
|
+
value = int(char)
|
|
37
|
+
if position % 2 == 1:
|
|
38
|
+
value = value * 2 - 9 if value > 4 else value * 2
|
|
39
|
+
total += value
|
|
40
|
+
return total % 10 == 0
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def iban_valid(iban: str) -> bool:
|
|
44
|
+
"""True if `iban` has 15 to 34 characters and its ISO 13616 mod-97 remainder is 1."""
|
|
45
|
+
compact = _compact(iban)
|
|
46
|
+
if _IBAN.fullmatch(compact) is None:
|
|
47
|
+
return False
|
|
48
|
+
rearranged = compact[4:] + compact[:4]
|
|
49
|
+
return int("".join(str(int(char, 36)) for char in rearranged)) % 97 == 1
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _dni_letter_matches(digits: str, letter: str) -> bool:
|
|
53
|
+
return _DNI_LETTERS[int(digits) % 23] == letter
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def es_dni_valid(value: str) -> bool:
|
|
57
|
+
"""True if `value` is 8 digits followed by the matching Spanish DNI control letter."""
|
|
58
|
+
match = _DNI.fullmatch(_compact(value))
|
|
59
|
+
return match is not None and _dni_letter_matches(match[1], match[2])
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def es_nie_valid(value: str) -> bool:
|
|
63
|
+
"""True if `value` is X, Y or Z, 7 digits and the matching Spanish NIE control letter."""
|
|
64
|
+
match = _NIE.fullmatch(_compact(value))
|
|
65
|
+
return match is not None and _dni_letter_matches(
|
|
66
|
+
_NIE_PREFIX_DIGIT[match[1]] + match[2], match[3]
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def es_cif_valid(value: str) -> bool:
|
|
71
|
+
"""True if `value` is a Spanish CIF whose control character is the matching digit or letter."""
|
|
72
|
+
match = _CIF.fullmatch(_compact(value))
|
|
73
|
+
if match is None:
|
|
74
|
+
return False
|
|
75
|
+
digits = [int(char) for char in match[1]]
|
|
76
|
+
even = digits[1] + digits[3] + digits[5]
|
|
77
|
+
odd = sum(sum(divmod(digit * 2, 10)) for digit in digits[0::2])
|
|
78
|
+
control = (10 - (even + odd) % 10) % 10
|
|
79
|
+
return match[2] in (str(control), _CIF_CONTROL_LETTERS[control])
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def us_ssn_plausible(value: str) -> bool:
|
|
83
|
+
"""True if `value` is a 9-digit US SSN outside the never-issued areas, groups and serials."""
|
|
84
|
+
match = _SSN.fullmatch(_compact(value))
|
|
85
|
+
if match is None:
|
|
86
|
+
return False
|
|
87
|
+
area, group, serial = match.groups()
|
|
88
|
+
return area not in ("000", "666") and area[0] != "9" and group != "00" and serial != "0000"
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""Overlapping word windows over long texts, for models with a limited input length."""
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
from typing import Final
|
|
6
|
+
|
|
7
|
+
# A word as GLiNER counts it (gliner 0.2.30, WhitespaceTokenSplitter): a run of word characters,
|
|
8
|
+
# possibly joined by "-" or "_", or any other single non-space character. Counting the model's
|
|
9
|
+
# own unit makes `window_words` a bound on what the model sees, so it never truncates a window.
|
|
10
|
+
_WORD: Final = re.compile(r"\w+(?:[-_]\w+)*|\S")
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@dataclass(frozen=True, slots=True)
|
|
14
|
+
class Window:
|
|
15
|
+
"""A range of the original text, as Python slice offsets."""
|
|
16
|
+
|
|
17
|
+
start: int
|
|
18
|
+
end: int
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def windows(text: str, *, window_words: int, overlap_words: int) -> list[Window]:
|
|
22
|
+
"""Cover the text with windows of at most `window_words` words, each overlapping the
|
|
23
|
+
previous one by exactly `overlap_words` words. Windows start and end on word boundaries,
|
|
24
|
+
every non-whitespace character is inside at least one window, and a text with at most
|
|
25
|
+
`window_words` words yields one window covering all of them.
|
|
26
|
+
|
|
27
|
+
A word is a run of word characters joined by "-" or "_", or any other single non-space
|
|
28
|
+
character, which is the unit GLiNER counts. Text without words yields no window.
|
|
29
|
+
"""
|
|
30
|
+
if not 0 <= overlap_words < window_words:
|
|
31
|
+
raise ValueError(
|
|
32
|
+
"window_words and overlap_words must satisfy 0 <= overlap_words < window_words"
|
|
33
|
+
)
|
|
34
|
+
words = [match.span() for match in _WORD.finditer(text)]
|
|
35
|
+
step = window_words - overlap_words
|
|
36
|
+
result: list[Window] = []
|
|
37
|
+
first = 0
|
|
38
|
+
while first < len(words):
|
|
39
|
+
last = min(first + window_words, len(words)) - 1
|
|
40
|
+
result.append(Window(words[first][0], words[last][1]))
|
|
41
|
+
if last == len(words) - 1:
|
|
42
|
+
break
|
|
43
|
+
first += step
|
|
44
|
+
return result
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
"""Company dictionary detector: user-provided terms, matched as whole terms (architecture §3.2)."""
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
from collections.abc import Sequence
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
|
|
7
|
+
from aliasgate.mapping import normalize_key
|
|
8
|
+
from aliasgate.types import PRIORITY_DICTIONARY, Span, validate_entity_type
|
|
9
|
+
|
|
10
|
+
_SOURCE = "dictionary"
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@dataclass(frozen=True, slots=True)
|
|
14
|
+
class Term:
|
|
15
|
+
"""One dictionary entry; its repr hides the term, which is confidential."""
|
|
16
|
+
|
|
17
|
+
term: str = field(repr=False)
|
|
18
|
+
entity_type: str
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _body(term: str) -> str:
|
|
22
|
+
return r"\s+".join(re.escape(token) for token in term.split())
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _shape(text: str) -> tuple[int, ...]:
|
|
26
|
+
return tuple(len(token) for token in text.split())
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class DictionaryDetector:
|
|
30
|
+
"""Finds dictionary terms as whole terms, case-insensitively, with flexible whitespace.
|
|
31
|
+
|
|
32
|
+
The same semantics as `build_term_pattern`; where several terms match at one position, the
|
|
33
|
+
longest wins. Error messages name term positions (1-based), never term content.
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
name = "dictionary"
|
|
37
|
+
|
|
38
|
+
def __init__(self, terms: Sequence[Term]) -> None:
|
|
39
|
+
unique: dict[str, tuple[int, Term]] = {}
|
|
40
|
+
for position, term in enumerate(terms, start=1):
|
|
41
|
+
key = normalize_key(term.term)
|
|
42
|
+
if not key:
|
|
43
|
+
raise ValueError(f"term {position}: the term is empty")
|
|
44
|
+
try:
|
|
45
|
+
validate_entity_type(term.entity_type)
|
|
46
|
+
except ValueError as exc:
|
|
47
|
+
raise ValueError(f"term {position}: {exc}") from None
|
|
48
|
+
first = unique.setdefault(key, (position, term))
|
|
49
|
+
if first[1].entity_type != term.entity_type:
|
|
50
|
+
raise ValueError(
|
|
51
|
+
f"terms {first[0]} and {position} are the same term with different entity types"
|
|
52
|
+
)
|
|
53
|
+
ordered = sorted(
|
|
54
|
+
(term for _, term in unique.values()),
|
|
55
|
+
key=lambda term: (-len(normalize_key(term.term)), normalize_key(term.term)),
|
|
56
|
+
)
|
|
57
|
+
self._types = {normalize_key(term.term): term.entity_type for term in ordered}
|
|
58
|
+
self._by_shape: dict[tuple[int, ...], list[Term]] = {}
|
|
59
|
+
for term in ordered:
|
|
60
|
+
self._by_shape.setdefault(_shape(term.term), []).append(term)
|
|
61
|
+
self._pattern = (
|
|
62
|
+
re.compile(
|
|
63
|
+
rf"(?<!\w)(?:{'|'.join(_body(term.term) for term in ordered)})(?!\w)",
|
|
64
|
+
re.IGNORECASE,
|
|
65
|
+
)
|
|
66
|
+
if ordered
|
|
67
|
+
else None
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
def detect(self, text: str) -> list[Span]:
|
|
71
|
+
"""One span per match, typed as its term, with priority 80 and score 1.0."""
|
|
72
|
+
if self._pattern is None:
|
|
73
|
+
return []
|
|
74
|
+
return [
|
|
75
|
+
Span(
|
|
76
|
+
match.start(),
|
|
77
|
+
match.end(),
|
|
78
|
+
self._type_of(match.group()),
|
|
79
|
+
1.0,
|
|
80
|
+
_SOURCE,
|
|
81
|
+
PRIORITY_DICTIONARY,
|
|
82
|
+
)
|
|
83
|
+
for match in self._pattern.finditer(text)
|
|
84
|
+
]
|
|
85
|
+
|
|
86
|
+
def _type_of(self, surface: str) -> str:
|
|
87
|
+
entity_type = self._types.get(normalize_key(surface))
|
|
88
|
+
if entity_type is not None:
|
|
89
|
+
return entity_type
|
|
90
|
+
# re.IGNORECASE equates a few letters that casefold keeps apart (I with U+0130 and
|
|
91
|
+
# U+0131), so the key lookup can miss. Case-insensitive matching is per character, so
|
|
92
|
+
# the matching term has the same token lengths; take the first in alternation order.
|
|
93
|
+
for term in self._by_shape.get(_shape(surface), ()):
|
|
94
|
+
if re.fullmatch(_body(term.term), surface, re.IGNORECASE):
|
|
95
|
+
return term.entity_type
|
|
96
|
+
raise RuntimeError("a dictionary match corresponds to no term") # pragma: no cover
|