aliasgate 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
aliasgate/config.py ADDED
@@ -0,0 +1,367 @@
1
+ """Gateway configuration (architecture §9) and dictionary files.
2
+
3
+ `load_settings` reads one YAML file, overlays `ALIASGATE_*` environment variables and returns
4
+ an immutable, validated `Settings`. Every section forbids unknown keys, and validation errors
5
+ never repeat input values. `Settings` holds no secrets: the gateway forwards client
6
+ credentials and stores none.
7
+ """
8
+
9
+ import os
10
+ import re
11
+ from collections.abc import Mapping
12
+ from pathlib import Path
13
+ from typing import Annotated, Any, Final, Literal, Self
14
+ from urllib.parse import unquote, urlsplit
15
+
16
+ import yaml
17
+ from pydantic import (
18
+ AfterValidator,
19
+ BaseModel,
20
+ ConfigDict,
21
+ Field,
22
+ StringConstraints,
23
+ ValidationInfo,
24
+ create_model,
25
+ field_validator,
26
+ model_validator,
27
+ )
28
+ from pydantic_settings import BaseSettings, EnvSettingsSource
29
+ from pydantic_settings.sources.utils import parse_env_vars
30
+
31
+ from aliasgate.detectors.dictionary import Term
32
+ from aliasgate.mapping import normalize_key
33
+ from aliasgate.types import validate_entity_type
34
+
35
+ _ENV_PREFIX: Final = "ALIASGATE_"
36
+ _ENV_NESTED_DELIMITER: Final = "__"
37
+ _BASE_DIR: Final = "base_dir"
38
+ # One path segment as RFC 3986 §3.3 writes it: unreserved characters, sub-delimiters, ":" and
39
+ # "@", and percent-encoded octets. httpx sends such a segment exactly as written.
40
+ _PATH_SEGMENT: Final = re.compile(r"(?:[A-Za-z0-9\-._~!$&'()*+,;=:@]|%[0-9A-Fa-f]{2})+")
41
+ _DOT_SEGMENTS: Final = frozenset({".", ".."})
42
+ # A passthrough path segment: the characters of _PATH_SEGMENT that never need escaping.
43
+ _PLAIN_SEGMENT: Final = re.compile(r"[A-Za-z0-9\-._~!$&'()*+,;=:@]+")
44
+ _PASSTHROUGH_UPSTREAMS: Final = frozenset({"openai", "anthropic"}) # fields of UpstreamsSettings
45
+ _PASSTHROUGH_METHODS: Final = frozenset({"GET", "POST", "DELETE"})
46
+
47
+
48
+ def _base_url(value: str) -> str:
49
+ """An http(s) URL with an optional base path, returned without trailing slashes, such that
50
+ appending a request path to it gives exactly the URL that is sent (architecture §7)."""
51
+ # urlsplit drops tabs and newlines, and an empty "?" or "#" leaves no query or fragment, so
52
+ # these are checked on the raw value: after concatenation they would move the request path.
53
+ if any(char in "?#" or char.isspace() or not char.isprintable() for char in value):
54
+ raise ValueError("must not contain '?', '#', whitespace or control characters")
55
+ parts = urlsplit(value)
56
+ if parts.scheme not in ("http", "https") or not parts.hostname:
57
+ raise ValueError("must be an http or https URL with a host")
58
+ if parts.username is not None or parts.password is not None:
59
+ raise ValueError("must not contain credentials")
60
+ _ = parts.port # raises ValueError for a malformed port
61
+ for segment in parts.path.rstrip("/").split("/")[1:]:
62
+ if not segment:
63
+ raise ValueError("the path must not have empty segments")
64
+ if not _PATH_SEGMENT.fullmatch(segment):
65
+ raise ValueError(
66
+ "the path may only contain letters, digits, -._~!$&'()*+,;=:@ and %XX escapes"
67
+ )
68
+ if unquote(segment) in _DOT_SEGMENTS:
69
+ raise ValueError("the path must not have '.' or '..' segments")
70
+ return value.rstrip("/")
71
+
72
+
73
+ def _passthrough_entry(value: str) -> str:
74
+ """`"<upstream> <METHOD> <path>"` with a path of non-empty plain segments. Requests match on
75
+ their decoded path and are forwarded with the entry's path, so a path that needs no escaping
76
+ is the only kind for which the two are the same text."""
77
+ parts = value.split(" ")
78
+ if (
79
+ len(parts) != 3
80
+ or parts[0] not in _PASSTHROUGH_UPSTREAMS
81
+ or parts[1] not in _PASSTHROUGH_METHODS
82
+ or not parts[2].startswith("/")
83
+ ):
84
+ raise ValueError(
85
+ "must be '<upstream> <METHOD> <path>' with upstream openai or anthropic, method GET,"
86
+ " POST or DELETE, and a path starting with '/'"
87
+ )
88
+ for segment in parts[2].split("/")[1:]:
89
+ if not segment:
90
+ raise ValueError("the path must not have empty segments")
91
+ if not _PLAIN_SEGMENT.fullmatch(segment):
92
+ raise ValueError("the path may only contain letters, digits and -._~!$&'()*+,;=:@")
93
+ if segment in _DOT_SEGMENTS:
94
+ raise ValueError("the path must not have '.' or '..' segments")
95
+ return value
96
+
97
+
98
+ BaseUrl = Annotated[str, AfterValidator(_base_url)]
99
+ EntityType = Annotated[str, AfterValidator(validate_entity_type)]
100
+ PhoneRegion = Annotated[str, StringConstraints(pattern=r"^[A-Z]{2}$")]
101
+ PassthroughPath = Annotated[str, AfterValidator(_passthrough_entry)]
102
+ ModelLabel = Annotated[str, StringConstraints(min_length=1)]
103
+
104
+
105
+ class _Section(BaseModel):
106
+ model_config = ConfigDict(extra="forbid", frozen=True, hide_input_in_errors=True)
107
+
108
+
109
+ class ListenSettings(_Section):
110
+ """Address the gateway serves on; loopback unless chosen otherwise."""
111
+
112
+ host: str = "127.0.0.1"
113
+ port: int = Field(default=8080, ge=1, le=65535)
114
+ max_request_bytes: int = Field(default=10_485_760, ge=1)
115
+
116
+
117
+ class UpstreamSettings(_Section):
118
+ """One provider; `base_url` may end in a base path and has no trailing slash, so
119
+ `base_url + request path` is the upstream URL."""
120
+
121
+ base_url: BaseUrl
122
+
123
+
124
+ class UpstreamsSettings(_Section):
125
+ """Providers the gateway forwards to, and the extra paths it may forward unchanged.
126
+
127
+ A passthrough entry is `"<upstream> <METHOD> <path>"`, for example
128
+ `"anthropic GET /v1/models"`; each method and path appears in at most one entry, so a
129
+ request names exactly one upstream.
130
+ """
131
+
132
+ openai: UpstreamSettings = UpstreamSettings(base_url="https://api.openai.com")
133
+ anthropic: UpstreamSettings = UpstreamSettings(base_url="https://api.anthropic.com")
134
+ timeout_seconds: float = Field(default=600, gt=0)
135
+ passthrough_paths: tuple[PassthroughPath, ...] = ()
136
+
137
+ @field_validator("passthrough_paths")
138
+ @classmethod
139
+ def _one_upstream_per_path(cls, entries: tuple[str, ...]) -> tuple[str, ...]:
140
+ routes = [entry.split(" ", 1)[1] for entry in entries]
141
+ if len(set(routes)) != len(routes):
142
+ raise ValueError("each method and path may appear in only one entry")
143
+ return entries
144
+
145
+
146
+ class RulesSettings(_Section):
147
+ """Rule detector: which rule types run, and the regions for national phone formats."""
148
+
149
+ enabled_types: tuple[EntityType, ...] = (
150
+ "EMAIL",
151
+ "PHONE",
152
+ "IBAN",
153
+ "CREDIT_CARD",
154
+ "NATIONAL_ID",
155
+ "TAX_ID",
156
+ "IP_ADDRESS",
157
+ "SECRET",
158
+ )
159
+ phone_regions: tuple[PhoneRegion, ...] = ("ES", "US")
160
+
161
+
162
+ class DictionarySettings(_Section):
163
+ """Dictionary files; relative paths are resolved against the configuration file."""
164
+
165
+ files: tuple[Path, ...] = ()
166
+
167
+ @field_validator("files")
168
+ @classmethod
169
+ def _resolve_relative_paths(
170
+ cls, files: tuple[Path, ...], info: ValidationInfo
171
+ ) -> tuple[Path, ...]:
172
+ context = info.context if isinstance(info.context, dict) else {}
173
+ base = context.get(_BASE_DIR, Path.cwd())
174
+ return tuple(file if file.is_absolute() else base / file for file in files)
175
+
176
+
177
+ class MLSettings(_Section):
178
+ """ML detector: a GLiNER model from the optional `ml` extra, off by default.
179
+
180
+ The default model is `urchade/gliner_multi_pii-v1` (Apache-2.0; en, fr, de, es, pt, it),
181
+ pinned to a revision. `labels` maps model labels to entity types. Texts are cut into windows
182
+ of `window_words` model words overlapping by `overlap_words`; results are cached for
183
+ `cache_entries` texts (0 disables the cache).
184
+ """
185
+
186
+ enabled: bool = False
187
+ model: Annotated[str, StringConstraints(min_length=1)] = "urchade/gliner_multi_pii-v1"
188
+ revision: Annotated[str, StringConstraints(min_length=1)] | None = (
189
+ "1fcf13e85f4eef5394e1fcd406cf2ca9ea82351d"
190
+ )
191
+ threshold: float = Field(default=0.4, ge=0.0, le=1.0)
192
+ labels: dict[ModelLabel, EntityType] = Field(
193
+ default_factory=lambda: {"person": "PERSON", "organization": "ORG", "address": "ADDRESS"},
194
+ min_length=1,
195
+ )
196
+ window_words: int = Field(default=250, ge=1)
197
+ overlap_words: int = Field(default=50, ge=0)
198
+ cache_entries: int = Field(default=4096, ge=0)
199
+
200
+ @model_validator(mode="after")
201
+ def _overlap_is_smaller_than_the_window(self) -> Self:
202
+ if self.overlap_words >= self.window_words:
203
+ raise ValueError("overlap_words must be smaller than window_words")
204
+ return self
205
+
206
+
207
+ class DetectionSettings(_Section):
208
+ """Detection layers, allow list and propagation."""
209
+
210
+ rules: RulesSettings = RulesSettings()
211
+ dictionary: DictionarySettings = DictionarySettings()
212
+ allow_list: tuple[str, ...] = ()
213
+ ml: MLSettings = MLSettings()
214
+ propagate: bool = True
215
+ min_propagation_length: int = Field(default=3, ge=1)
216
+
217
+
218
+ class PolicySettings(_Section):
219
+ """What happens to content the gateway cannot pseudonymise or should not forward.
220
+
221
+ With `leak_audit`, a request whose outbound body still holds a detected value is rejected
222
+ instead of forwarded (architecture §8).
223
+ """
224
+
225
+ unsupported_content: Literal["block", "pass"] = "block"
226
+ strip_user_identifiers: bool = True
227
+ tool_definitions: bool = True
228
+ leak_audit: bool = True
229
+
230
+
231
+ class LoggingSettings(_Section):
232
+ """Log level. Logs never contain text (architecture §7.1)."""
233
+
234
+ level: Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] = "INFO"
235
+
236
+
237
+ class MetricsSettings(_Section):
238
+ """Prometheus metrics at `GET /metrics`, on the gateway's own port. Labels never hold text."""
239
+
240
+ enabled: bool = True
241
+
242
+
243
+ class Settings(_Section):
244
+ """Complete gateway configuration: immutable, validated, free of secrets."""
245
+
246
+ listen: ListenSettings = ListenSettings()
247
+ upstreams: UpstreamsSettings = UpstreamsSettings()
248
+ detection: DetectionSettings = DetectionSettings()
249
+ policy: PolicySettings = PolicySettings()
250
+ logging: LoggingSettings = LoggingSettings()
251
+ metrics: MetricsSettings = MetricsSettings()
252
+
253
+
254
+ # The fields of Settings on a BaseSettings class, which is what pydantic-settings needs to parse
255
+ # environment variables. Settings itself stays a plain model that never reads the environment.
256
+ _SETTINGS_FIELDS: Final[dict[str, Any]] = {
257
+ name: (field.annotation, field) for name, field in Settings.model_fields.items()
258
+ }
259
+ _EnvironmentShape = create_model("_EnvironmentShape", __base__=BaseSettings, **_SETTINGS_FIELDS)
260
+
261
+
262
+ class _EnvironmentSource(EnvSettingsSource):
263
+ """Parses `ALIASGATE_*` variables from a given mapping instead of `os.environ`."""
264
+
265
+ def __init__(self, env: Mapping[str, str]) -> None:
266
+ self._given_env = env
267
+ super().__init__(
268
+ _EnvironmentShape,
269
+ case_sensitive=False,
270
+ env_prefix=_ENV_PREFIX,
271
+ env_nested_delimiter=_ENV_NESTED_DELIMITER,
272
+ )
273
+
274
+ def _load_env_vars(self) -> Mapping[str, str | None]:
275
+ return parse_env_vars(
276
+ self._given_env, self.case_sensitive, self.env_ignore_empty, self.env_parse_none_str
277
+ )
278
+
279
+
280
+ def _environment_data(env: Mapping[str, str]) -> dict[str, Any]:
281
+ data = dict(_EnvironmentSource(env)())
282
+ for name, value in env.items():
283
+ if not name.upper().startswith(_ENV_PREFIX):
284
+ continue
285
+ section = name[len(_ENV_PREFIX) :].split(_ENV_NESTED_DELIMITER, 1)[0].lower()
286
+ if section not in Settings.model_fields:
287
+ # pydantic-settings skips variables that match no field; keep them so that
288
+ # extra="forbid" turns a misspelled variable into a start-up error.
289
+ data[name] = value
290
+ return data
291
+
292
+
293
+ def _deep_merge(
294
+ base: Mapping[str, Any], override: Mapping[str, Any], section: type[BaseModel]
295
+ ) -> dict[str, Any]:
296
+ """Sections merge key by key; any other value, mappings such as `ml.labels` included, is
297
+ replaced, so the environment can also remove entries the file sets."""
298
+ merged = dict(base)
299
+ for key, value in override.items():
300
+ current = merged.get(key)
301
+ field = section.model_fields.get(key)
302
+ subsection = field.annotation if field is not None else None
303
+ if (
304
+ isinstance(subsection, type)
305
+ and issubclass(subsection, BaseModel)
306
+ and isinstance(value, Mapping)
307
+ and isinstance(current, Mapping)
308
+ ):
309
+ merged[key] = _deep_merge(current, value, subsection)
310
+ else:
311
+ merged[key] = value
312
+ return merged
313
+
314
+
315
+ def load_settings(path: Path | None, env: Mapping[str, str] = os.environ) -> Settings:
316
+ """Load YAML (if a path is given), overlay ALIASGATE_* environment variables, validate."""
317
+ file_data: Any = {}
318
+ if path is not None:
319
+ with path.open(encoding="utf-8") as stream:
320
+ file_data = yaml.safe_load(stream)
321
+ if file_data is None:
322
+ file_data = {}
323
+ env_data = _environment_data(env)
324
+ data = (
325
+ _deep_merge(file_data, env_data, Settings) if isinstance(file_data, Mapping) else file_data
326
+ )
327
+ base_dir = path.absolute().parent if path is not None else Path.cwd()
328
+ return Settings.model_validate(data, context={_BASE_DIR: base_dir})
329
+
330
+
331
+ def load_terms(path: Path) -> list[Term]:
332
+ """Parse and validate a dictionary file; errors name entry positions, never term content."""
333
+ with path.open(encoding="utf-8") as stream:
334
+ try:
335
+ data = yaml.safe_load(stream)
336
+ except yaml.YAMLError as exc:
337
+ # The exception text quotes the offending line, which may contain a term.
338
+ mark = getattr(exc, "problem_mark", None)
339
+ where = f" at line {mark.line + 1}, column {mark.column + 1}" if mark else ""
340
+ raise ValueError(f"dictionary file {path} is not valid YAML{where}") from None
341
+ if not isinstance(data, dict) or set(data) != {"terms"}:
342
+ raise ValueError(f"dictionary file {path} must be a mapping with a single 'terms' key")
343
+ entries = data["terms"] if data["terms"] is not None else []
344
+ if not isinstance(entries, list):
345
+ raise ValueError(f"dictionary file {path}: 'terms' must be a list")
346
+ return [_term(path, position, entry) for position, entry in enumerate(entries, start=1)]
347
+
348
+
349
+ def _term(path: Path, position: int, entry: object) -> Term:
350
+ where = f"dictionary file {path}, entry {position}"
351
+ if not isinstance(entry, dict):
352
+ raise ValueError(f"{where}: must be a mapping with 'term' and 'type'")
353
+ if set(entry) - {"term", "type"}:
354
+ raise ValueError(f"{where}: only 'term' and 'type' are allowed")
355
+ for key in ("term", "type"):
356
+ if key not in entry:
357
+ raise ValueError(f"{where}: missing '{key}'")
358
+ if not isinstance(entry[key], str):
359
+ raise ValueError(f"{where}: '{key}' must be a string")
360
+ term, entity_type = entry["term"], entry["type"]
361
+ try:
362
+ validate_entity_type(entity_type)
363
+ except ValueError as exc:
364
+ raise ValueError(f"{where}: {exc}") from None
365
+ if not normalize_key(term):
366
+ raise ValueError(f"{where}: 'term' is empty")
367
+ return Term(term, entity_type)
@@ -0,0 +1 @@
1
+ """Detectors: components that find spans of entities in a text."""
@@ -0,0 +1,57 @@
1
+ """Detection cache (architecture §4.5): conversations resend their history on every turn."""
2
+
3
+ import hashlib
4
+ import threading
5
+ from collections import OrderedDict
6
+
7
+ from aliasgate.types import Detector, Span
8
+
9
+
10
+ class CachedDetector:
11
+ """Wraps a detector with a bounded, thread-safe LRU cache of its results.
12
+
13
+ Entries are keyed by `sha256(fingerprint + "\\x00" + text)` and hold spans only, never
14
+ text, in process memory only. Failures of the inner detector are not cached, and every
15
+ call returns a new list. `max_entries == 0` disables caching.
16
+ """
17
+
18
+ def __init__(self, inner: Detector, *, fingerprint: str, max_entries: int) -> None:
19
+ if max_entries < 0:
20
+ raise ValueError("max_entries must not be negative")
21
+ self.name: str = inner.name
22
+ self._inner = inner
23
+ self._fingerprint = fingerprint
24
+ self._max_entries = max_entries
25
+ self._entries: OrderedDict[bytes, tuple[Span, ...]] = OrderedDict()
26
+ self._lock = threading.Lock()
27
+ self._hits = 0
28
+ self._misses = 0
29
+
30
+ def detect(self, text: str) -> list[Span]:
31
+ """The inner detector's spans for `text`, from the cache when it has seen `text`."""
32
+ # "surrogatepass" keeps the encoding injective for texts with unpaired surrogates,
33
+ # which JSON allows; plain UTF-8 would refuse them.
34
+ key = hashlib.sha256(
35
+ (self._fingerprint + "\x00" + text).encode("utf-8", "surrogatepass")
36
+ ).digest()
37
+ with self._lock:
38
+ cached = self._entries.get(key)
39
+ if cached is not None:
40
+ self._entries.move_to_end(key)
41
+ self._hits += 1
42
+ return list(cached)
43
+ self._misses += 1
44
+ # Outside the lock, so that a slow detection does not hold up cache hits.
45
+ spans = tuple(self._inner.detect(text))
46
+ if self._max_entries:
47
+ with self._lock:
48
+ self._entries[key] = spans
49
+ self._entries.move_to_end(key)
50
+ while len(self._entries) > self._max_entries:
51
+ self._entries.popitem(last=False)
52
+ return list(spans)
53
+
54
+ def stats(self) -> tuple[int, int]:
55
+ """(hits, misses) since construction; a failed detection counts as a miss."""
56
+ with self._lock:
57
+ return self._hits, self._misses
@@ -0,0 +1,88 @@
1
+ """Checksum and plausibility checks for structured identifiers.
2
+
3
+ Every function ignores spaces and hyphens, accepts letters in either case, only accepts ASCII
4
+ digits and letters, returns False for malformed input and never raises.
5
+ """
6
+
7
+ import re
8
+ from typing import Final
9
+
10
+ _DNI_LETTERS: Final = "TRWAGMYFPDXBNJZSQVHLCKE"
11
+ _NIE_PREFIX_DIGIT: Final = {"X": "0", "Y": "1", "Z": "2"}
12
+ _CIF_CONTROL_LETTERS: Final = "JABCDEFGHI"
13
+
14
+ _CARD: Final = re.compile(r"[0-9]{13,19}")
15
+ _IBAN: Final = re.compile(r"[A-Z]{2}[0-9]{2}[A-Z0-9]{11,30}")
16
+ _DNI: Final = re.compile(r"([0-9]{8})([A-Z])")
17
+ _NIE: Final = re.compile(r"([XYZ])([0-9]{7})([A-Z])")
18
+ _CIF: Final = re.compile(r"[ABCDEFGHJNPQRSUVW]([0-9]{7})([0-9A-J])")
19
+ _SSN: Final = re.compile(r"([0-9]{3})([0-9]{2})([0-9]{4})")
20
+
21
+
22
+ def _compact(value: str) -> str:
23
+ """Remove spaces and hyphens and upper-case; non-ASCII input becomes "" (always invalid)."""
24
+ compact = value.replace(" ", "").replace("-", "")
25
+ # Checked before upper-casing: "\u017f".upper() == "S" and "ß".upper() == "SS".
26
+ return compact.upper() if compact.isascii() else ""
27
+
28
+
29
+ def luhn_valid(number: str) -> bool:
30
+ """True if `number` has 13 to 19 digits and passes the Luhn mod 10 check."""
31
+ digits = _compact(number)
32
+ if _CARD.fullmatch(digits) is None:
33
+ return False
34
+ total = 0
35
+ for position, char in enumerate(reversed(digits)):
36
+ value = int(char)
37
+ if position % 2 == 1:
38
+ value = value * 2 - 9 if value > 4 else value * 2
39
+ total += value
40
+ return total % 10 == 0
41
+
42
+
43
+ def iban_valid(iban: str) -> bool:
44
+ """True if `iban` has 15 to 34 characters and its ISO 13616 mod-97 remainder is 1."""
45
+ compact = _compact(iban)
46
+ if _IBAN.fullmatch(compact) is None:
47
+ return False
48
+ rearranged = compact[4:] + compact[:4]
49
+ return int("".join(str(int(char, 36)) for char in rearranged)) % 97 == 1
50
+
51
+
52
+ def _dni_letter_matches(digits: str, letter: str) -> bool:
53
+ return _DNI_LETTERS[int(digits) % 23] == letter
54
+
55
+
56
+ def es_dni_valid(value: str) -> bool:
57
+ """True if `value` is 8 digits followed by the matching Spanish DNI control letter."""
58
+ match = _DNI.fullmatch(_compact(value))
59
+ return match is not None and _dni_letter_matches(match[1], match[2])
60
+
61
+
62
+ def es_nie_valid(value: str) -> bool:
63
+ """True if `value` is X, Y or Z, 7 digits and the matching Spanish NIE control letter."""
64
+ match = _NIE.fullmatch(_compact(value))
65
+ return match is not None and _dni_letter_matches(
66
+ _NIE_PREFIX_DIGIT[match[1]] + match[2], match[3]
67
+ )
68
+
69
+
70
+ def es_cif_valid(value: str) -> bool:
71
+ """True if `value` is a Spanish CIF whose control character is the matching digit or letter."""
72
+ match = _CIF.fullmatch(_compact(value))
73
+ if match is None:
74
+ return False
75
+ digits = [int(char) for char in match[1]]
76
+ even = digits[1] + digits[3] + digits[5]
77
+ odd = sum(sum(divmod(digit * 2, 10)) for digit in digits[0::2])
78
+ control = (10 - (even + odd) % 10) % 10
79
+ return match[2] in (str(control), _CIF_CONTROL_LETTERS[control])
80
+
81
+
82
+ def us_ssn_plausible(value: str) -> bool:
83
+ """True if `value` is a 9-digit US SSN outside the never-issued areas, groups and serials."""
84
+ match = _SSN.fullmatch(_compact(value))
85
+ if match is None:
86
+ return False
87
+ area, group, serial = match.groups()
88
+ return area not in ("000", "666") and area[0] != "9" and group != "00" and serial != "0000"
@@ -0,0 +1,44 @@
1
+ """Overlapping word windows over long texts, for models with a limited input length."""
2
+
3
+ import re
4
+ from dataclasses import dataclass
5
+ from typing import Final
6
+
7
+ # A word as GLiNER counts it (gliner 0.2.30, WhitespaceTokenSplitter): a run of word characters,
8
+ # possibly joined by "-" or "_", or any other single non-space character. Counting the model's
9
+ # own unit makes `window_words` a bound on what the model sees, so it never truncates a window.
10
+ _WORD: Final = re.compile(r"\w+(?:[-_]\w+)*|\S")
11
+
12
+
13
+ @dataclass(frozen=True, slots=True)
14
+ class Window:
15
+ """A range of the original text, as Python slice offsets."""
16
+
17
+ start: int
18
+ end: int
19
+
20
+
21
+ def windows(text: str, *, window_words: int, overlap_words: int) -> list[Window]:
22
+ """Cover the text with windows of at most `window_words` words, each overlapping the
23
+ previous one by exactly `overlap_words` words. Windows start and end on word boundaries,
24
+ every non-whitespace character is inside at least one window, and a text with at most
25
+ `window_words` words yields one window covering all of them.
26
+
27
+ A word is a run of word characters joined by "-" or "_", or any other single non-space
28
+ character, which is the unit GLiNER counts. Text without words yields no window.
29
+ """
30
+ if not 0 <= overlap_words < window_words:
31
+ raise ValueError(
32
+ "window_words and overlap_words must satisfy 0 <= overlap_words < window_words"
33
+ )
34
+ words = [match.span() for match in _WORD.finditer(text)]
35
+ step = window_words - overlap_words
36
+ result: list[Window] = []
37
+ first = 0
38
+ while first < len(words):
39
+ last = min(first + window_words, len(words)) - 1
40
+ result.append(Window(words[first][0], words[last][1]))
41
+ if last == len(words) - 1:
42
+ break
43
+ first += step
44
+ return result
@@ -0,0 +1,96 @@
1
+ """Company dictionary detector: user-provided terms, matched as whole terms (architecture §3.2)."""
2
+
3
+ import re
4
+ from collections.abc import Sequence
5
+ from dataclasses import dataclass, field
6
+
7
+ from aliasgate.mapping import normalize_key
8
+ from aliasgate.types import PRIORITY_DICTIONARY, Span, validate_entity_type
9
+
10
+ _SOURCE = "dictionary"
11
+
12
+
13
+ @dataclass(frozen=True, slots=True)
14
+ class Term:
15
+ """One dictionary entry; its repr hides the term, which is confidential."""
16
+
17
+ term: str = field(repr=False)
18
+ entity_type: str
19
+
20
+
21
+ def _body(term: str) -> str:
22
+ return r"\s+".join(re.escape(token) for token in term.split())
23
+
24
+
25
+ def _shape(text: str) -> tuple[int, ...]:
26
+ return tuple(len(token) for token in text.split())
27
+
28
+
29
+ class DictionaryDetector:
30
+ """Finds dictionary terms as whole terms, case-insensitively, with flexible whitespace.
31
+
32
+ The same semantics as `build_term_pattern`; where several terms match at one position, the
33
+ longest wins. Error messages name term positions (1-based), never term content.
34
+ """
35
+
36
+ name = "dictionary"
37
+
38
+ def __init__(self, terms: Sequence[Term]) -> None:
39
+ unique: dict[str, tuple[int, Term]] = {}
40
+ for position, term in enumerate(terms, start=1):
41
+ key = normalize_key(term.term)
42
+ if not key:
43
+ raise ValueError(f"term {position}: the term is empty")
44
+ try:
45
+ validate_entity_type(term.entity_type)
46
+ except ValueError as exc:
47
+ raise ValueError(f"term {position}: {exc}") from None
48
+ first = unique.setdefault(key, (position, term))
49
+ if first[1].entity_type != term.entity_type:
50
+ raise ValueError(
51
+ f"terms {first[0]} and {position} are the same term with different entity types"
52
+ )
53
+ ordered = sorted(
54
+ (term for _, term in unique.values()),
55
+ key=lambda term: (-len(normalize_key(term.term)), normalize_key(term.term)),
56
+ )
57
+ self._types = {normalize_key(term.term): term.entity_type for term in ordered}
58
+ self._by_shape: dict[tuple[int, ...], list[Term]] = {}
59
+ for term in ordered:
60
+ self._by_shape.setdefault(_shape(term.term), []).append(term)
61
+ self._pattern = (
62
+ re.compile(
63
+ rf"(?<!\w)(?:{'|'.join(_body(term.term) for term in ordered)})(?!\w)",
64
+ re.IGNORECASE,
65
+ )
66
+ if ordered
67
+ else None
68
+ )
69
+
70
+ def detect(self, text: str) -> list[Span]:
71
+ """One span per match, typed as its term, with priority 80 and score 1.0."""
72
+ if self._pattern is None:
73
+ return []
74
+ return [
75
+ Span(
76
+ match.start(),
77
+ match.end(),
78
+ self._type_of(match.group()),
79
+ 1.0,
80
+ _SOURCE,
81
+ PRIORITY_DICTIONARY,
82
+ )
83
+ for match in self._pattern.finditer(text)
84
+ ]
85
+
86
+ def _type_of(self, surface: str) -> str:
87
+ entity_type = self._types.get(normalize_key(surface))
88
+ if entity_type is not None:
89
+ return entity_type
90
+ # re.IGNORECASE equates a few letters that casefold keeps apart (I with U+0130 and
91
+ # U+0131), so the key lookup can miss. Case-insensitive matching is per character, so
92
+ # the matching term has the same token lengths; take the first in alternation order.
93
+ for term in self._by_shape.get(_shape(surface), ()):
94
+ if re.fullmatch(_body(term.term), surface, re.IGNORECASE):
95
+ return term.entity_type
96
+ raise RuntimeError("a dictionary match corresponds to no term") # pragma: no cover