aliasgate 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,103 @@
1
+ """Renders evaluation reports as a table or JSON: numbers, entity types and example ids only."""
2
+
3
+ import json
4
+ from collections.abc import Mapping
5
+ from typing import Final
6
+
7
+ from aliasgate.eval.metrics import Miss, Report
8
+
9
+ _COLUMNS: Final = (
10
+ ("type", 14),
11
+ ("gold", 6),
12
+ ("predicted", 10),
13
+ ("exact", 6),
14
+ ("overlap", 8),
15
+ ("recall_exact", 13),
16
+ ("recall_overlap", 15),
17
+ ("precision_exact", 16),
18
+ )
19
+
20
+
21
+ def format_table(lang: str, report: Report, *, misses: int = 0) -> str:
22
+ """A plain-text summary of one language, listing up to `misses` missed spans."""
23
+ lines = [
24
+ f"{lang}: {report.examples} examples, leak rate {report.leak_rate:.4f} "
25
+ f"({report.leaked_chars} of {report.gold_chars} characters), "
26
+ f"round-trip failures {report.round_trip_failures}",
27
+ f"examples without entities: {report.entity_free}, with a predicted span: "
28
+ f"{report.entity_free_flagged} ({report.entity_free_flag_rate:.1%})",
29
+ "".join(
30
+ name.ljust(width) if index == 0 else name.rjust(width)
31
+ for index, (name, width) in enumerate(_COLUMNS)
32
+ ),
33
+ ]
34
+ for score in report.by_type:
35
+ values = (
36
+ score.entity_type,
37
+ str(score.gold),
38
+ str(score.predicted),
39
+ str(score.exact_tp),
40
+ str(score.overlap_tp),
41
+ f"{score.recall_exact:.3f}",
42
+ f"{score.recall_overlap:.3f}",
43
+ f"{score.precision_exact:.3f}",
44
+ )
45
+ lines.append(
46
+ "".join(
47
+ value.ljust(width) if index == 0 else value.rjust(width)
48
+ for index, (value, (_, width)) in enumerate(zip(values, _COLUMNS, strict=True))
49
+ )
50
+ )
51
+ if misses > 0 and report.misses:
52
+ shown = report.misses[:misses]
53
+ lines.append(f"missed spans ({len(shown)} of {len(report.misses)}):")
54
+ lines.extend(
55
+ f" {miss.example_id} {miss.start}..{miss.end} {miss.entity_type}, "
56
+ f"{miss.leaked_chars} characters not covered"
57
+ for miss in shown
58
+ )
59
+ return "\n".join(lines)
60
+
61
+
62
+ def format_json(reports: Mapping[str, Report], *, misses: int = 0) -> str:
63
+ """All languages as one JSON object keyed by language, with up to `misses` missed spans."""
64
+ return json.dumps(
65
+ {lang: _as_dict(report, misses) for lang, report in reports.items()}, indent=2
66
+ )
67
+
68
+
69
+ def _as_dict(report: Report, misses: int) -> dict[str, object]:
70
+ return {
71
+ "examples": report.examples,
72
+ "leak_rate": report.leak_rate,
73
+ "leaked_chars": report.leaked_chars,
74
+ "gold_chars": report.gold_chars,
75
+ "round_trip_failures": report.round_trip_failures,
76
+ "examples_without_entities": report.entity_free,
77
+ "flagged_without_entities": report.entity_free_flagged,
78
+ "by_type": [
79
+ {
80
+ "entity_type": score.entity_type,
81
+ "gold": score.gold,
82
+ "predicted": score.predicted,
83
+ "exact_tp": score.exact_tp,
84
+ "overlap_tp": score.overlap_tp,
85
+ "recall_exact": score.recall_exact,
86
+ "recall_overlap": score.recall_overlap,
87
+ "precision_exact": score.precision_exact,
88
+ }
89
+ for score in report.by_type
90
+ ],
91
+ "missed_spans": len(report.misses),
92
+ "misses": [_miss(miss) for miss in report.misses[:misses]],
93
+ }
94
+
95
+
96
+ def _miss(miss: Miss) -> dict[str, object]:
97
+ return {
98
+ "example_id": miss.example_id,
99
+ "start": miss.start,
100
+ "end": miss.end,
101
+ "entity_type": miss.entity_type,
102
+ "leaked_chars": miss.leaked_chars,
103
+ }
aliasgate/factory.py ADDED
@@ -0,0 +1,67 @@
1
+ """Builds a ready Pseudonymizer from the configuration."""
2
+
3
+ from itertools import cycle, islice
4
+
5
+ from aliasgate.config import MLSettings, Settings, load_terms
6
+ from aliasgate.detectors.cache import CachedDetector
7
+ from aliasgate.detectors.dictionary import DictionaryDetector
8
+ from aliasgate.detectors.gliner import GlinerDetector, load_gliner
9
+ from aliasgate.detectors.rules import RuleDetector
10
+ from aliasgate.engine import Pseudonymizer
11
+ from aliasgate.types import Detector
12
+
13
+ # Synthetic words, each one model word, repeated to fill one whole window at warm-up.
14
+ _WARM_UP_WORDS = (
15
+ "Alice",
16
+ "Johnson",
17
+ "from",
18
+ "ACME",
19
+ "Corporation",
20
+ "met",
21
+ "Juan",
22
+ "Perez",
23
+ "in",
24
+ "Springfield",
25
+ )
26
+
27
+
28
+ def build_pseudonymizer(settings: Settings) -> Pseudonymizer:
29
+ """Wire detectors in this order: dictionary (if any terms), rules, ML (if enabled).
30
+
31
+ The ML model is loaded here, at start-up, and run once on a synthetic text of one full
32
+ window, so that a missing extra, a broken model or a window the model would truncate stops
33
+ the gateway before it serves a request.
34
+ """
35
+ detection = settings.detection
36
+ terms = [term for path in detection.dictionary.files for term in load_terms(path)]
37
+ detectors: list[Detector] = []
38
+ if terms:
39
+ detectors.append(DictionaryDetector(terms))
40
+ detectors.append(
41
+ RuleDetector(
42
+ enabled_types=detection.rules.enabled_types,
43
+ phone_regions=detection.rules.phone_regions,
44
+ )
45
+ )
46
+ if detection.ml.enabled:
47
+ detectors.append(_ml_detector(detection.ml))
48
+ return Pseudonymizer(
49
+ detectors,
50
+ allow_list=detection.allow_list,
51
+ propagate=detection.propagate,
52
+ min_propagation_length=detection.min_propagation_length,
53
+ )
54
+
55
+
56
+ def _ml_detector(ml: MLSettings) -> CachedDetector:
57
+ detector = GlinerDetector(
58
+ load_gliner(ml.model, ml.revision),
59
+ model_id=ml.model,
60
+ revision=ml.revision,
61
+ labels=ml.labels,
62
+ threshold=ml.threshold,
63
+ window_words=ml.window_words,
64
+ overlap_words=ml.overlap_words,
65
+ )
66
+ detector.detect(" ".join(islice(cycle(_WARM_UP_WORDS), ml.window_words)))
67
+ return CachedDetector(detector, fingerprint=detector.fingerprint, max_entries=ml.cache_entries)
aliasgate/mapping.py ADDED
@@ -0,0 +1,103 @@
1
+ """Per-request mapping between placeholders and canonical surfaces, held in memory only."""
2
+
3
+ import unicodedata
4
+ from dataclasses import dataclass, field
5
+ from typing import Final
6
+
7
+ from aliasgate.placeholders import format_placeholder
8
+ from aliasgate.types import validate_entity_type
9
+
10
+
11
+ def normalize_key(surface: str) -> str:
12
+ """NFKC, casefold, collapse whitespace runs to one space, strip."""
13
+ return " ".join(unicodedata.normalize("NFKC", surface).casefold().split())
14
+
15
+
16
+ _NOT_SERIALISABLE: Final = "the mapping and its entities are never serialised or copied"
17
+
18
+
19
+ @dataclass(frozen=True, slots=True)
20
+ class Entity:
21
+ """One entity of a mapping; its repr shows only the type and the placeholder.
22
+
23
+ It cannot be pickled or copied, and its state cannot be read, because it holds a surface.
24
+ """
25
+
26
+ key: str = field(repr=False)
27
+ entity_type: str
28
+ surface: str = field(repr=False) # canonical surface: the first one seen
29
+ placeholder: str
30
+
31
+ def __getstate__(self) -> object:
32
+ raise TypeError(_NOT_SERIALISABLE)
33
+
34
+ def __reduce__(self) -> str | tuple[object, ...]:
35
+ raise TypeError(_NOT_SERIALISABLE)
36
+
37
+
38
+ class Mapping:
39
+ """Placeholder-to-surface table of one request; numbers start at 1 per entity type.
40
+
41
+ Not thread-safe: use one instance per request. Its repr reveals only the entity count, and
42
+ it cannot be pickled or copied.
43
+ """
44
+
45
+ __slots__ = ("_by_key", "_by_placeholder", "_counters")
46
+
47
+ def __init__(self) -> None:
48
+ self._by_key: dict[str, Entity] = {}
49
+ self._by_placeholder: dict[tuple[str, int], Entity] = {}
50
+ self._counters: dict[str, int] = {}
51
+
52
+ def assign(self, entity_type: str, surface: str, *, key: str | None = None) -> Entity:
53
+ """Return the entity for the key (default: normalize_key(surface)), creating it with the
54
+ next number for `entity_type` if the key is new. For an existing key the stored entity
55
+ is returned unchanged, even if `entity_type` or the surface's case differ.
56
+
57
+ Raises ValueError for an invalid entity type, an empty key or an empty surface."""
58
+ validate_entity_type(entity_type)
59
+ if key is None:
60
+ key = normalize_key(surface)
61
+ if not key:
62
+ raise ValueError("entity key must not be empty")
63
+ if not surface:
64
+ raise ValueError("entity surface must not be empty")
65
+ existing = self._by_key.get(key)
66
+ if existing is not None:
67
+ return existing
68
+ number = self._counters.get(entity_type, 0) + 1
69
+ entity = Entity(
70
+ key=key,
71
+ entity_type=entity_type,
72
+ surface=surface,
73
+ placeholder=format_placeholder(entity_type, number),
74
+ )
75
+ self._counters[entity_type] = number
76
+ self._by_key[key] = entity
77
+ self._by_placeholder[(entity_type, number)] = entity
78
+ return entity
79
+
80
+ def get(self, key: str) -> Entity | None:
81
+ """Return the entity recorded for `key`, or None."""
82
+ return self._by_key.get(key)
83
+
84
+ def lookup(self, entity_type: str, number: int) -> str | None:
85
+ """Return the canonical surface for a placeholder, or None."""
86
+ entity = self._by_placeholder.get((entity_type, number))
87
+ return None if entity is None else entity.surface
88
+
89
+ def entities(self) -> tuple[Entity, ...]:
90
+ """All entities in order of assignment."""
91
+ return tuple(self._by_key.values())
92
+
93
+ def __len__(self) -> int:
94
+ return len(self._by_key)
95
+
96
+ def __repr__(self) -> str:
97
+ return f"Mapping(entities={len(self._by_key)})"
98
+
99
+ def __getstate__(self) -> object:
100
+ raise TypeError(_NOT_SERIALISABLE)
101
+
102
+ def __reduce__(self) -> str | tuple[object, ...]:
103
+ raise TypeError(_NOT_SERIALISABLE)
aliasgate/metrics.py ADDED
@@ -0,0 +1,131 @@
1
+ """Prometheus metrics of one gateway, in a registry of its own.
2
+
3
+ Every label value is chosen by the gateway: routes, status classes, outcomes, error codes,
4
+ detector names and entity types from the configuration. None is ever taken from a request.
5
+ """
6
+
7
+ from collections.abc import Callable, Iterable, Sequence
8
+ from typing import Final, Literal
9
+
10
+ from prometheus_client import (
11
+ CONTENT_TYPE_LATEST,
12
+ CollectorRegistry,
13
+ Counter,
14
+ Histogram,
15
+ generate_latest,
16
+ )
17
+ from prometheus_client.metrics_core import CounterMetricFamily, Metric
18
+ from prometheus_client.registry import Collector
19
+
20
+ from aliasgate.engine import BatchStats
21
+
22
+ type Outcome = Literal["ok", "rejected", "upstream_error", "gateway_error"]
23
+ type Phase = Literal["pseudonymize", "restore"]
24
+
25
+ CONTENT_TYPE: Final = CONTENT_TYPE_LATEST
26
+ # Rules take well under a millisecond, the ML detector up to seconds.
27
+ _BUCKETS: Final = (0.0005, 0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10)
28
+
29
+
30
+ class _CacheCollector(Collector):
31
+ """Reads the detection caches' cumulative (hits, misses) at scrape time."""
32
+
33
+ def __init__(self, sources: Sequence[Callable[[], tuple[int, int]]]) -> None:
34
+ self._sources = tuple(sources)
35
+
36
+ def collect(self) -> Iterable[Metric]:
37
+ counts = [source() for source in self._sources]
38
+ family = CounterMetricFamily(
39
+ "aliasgate_detection_cache", "Detection cache lookups, by result.", labels=["result"]
40
+ )
41
+ family.add_metric(["hit"], sum(hits for hits, _ in counts))
42
+ family.add_metric(["miss"], sum(misses for _, misses in counts))
43
+ yield family
44
+
45
+
46
+ class GatewayMetrics:
47
+ """The gateway's counters and histograms. `cache_stats` are the `stats` methods of the
48
+ detection caches; without any, there is no cache metric."""
49
+
50
+ def __init__(self, cache_stats: Sequence[Callable[[], tuple[int, int]]] = ()) -> None:
51
+ self.registry = CollectorRegistry()
52
+ registry = self.registry
53
+ self._requests = Counter(
54
+ "aliasgate_requests",
55
+ "Requests answered, by route, status class and outcome.",
56
+ ["route", "status_class", "outcome"],
57
+ registry=registry,
58
+ )
59
+ self._entities = Counter(
60
+ "aliasgate_entities",
61
+ "Occurrences replaced by a placeholder, by entity type and the detector that found "
62
+ "them.",
63
+ ["entity_type", "source"],
64
+ registry=registry,
65
+ )
66
+ self._unresolved = Counter(
67
+ "aliasgate_unresolved_placeholders",
68
+ "Placeholders in responses that are not in the request's mapping.",
69
+ ["route"],
70
+ registry=registry,
71
+ )
72
+ self._leaks = Counter(
73
+ "aliasgate_leak_audit_failures",
74
+ "Requests rejected by the leak audit, by route and leaked entity type.",
75
+ ["route", "entity_type"],
76
+ registry=registry,
77
+ )
78
+ self._rejections = Counter(
79
+ "aliasgate_rejections",
80
+ "Error responses produced by the gateway itself, by route and error code.",
81
+ ["route", "code"],
82
+ registry=registry,
83
+ )
84
+ self._detection = Histogram(
85
+ "aliasgate_detection_seconds",
86
+ "Time each detector spent on one request.",
87
+ ["detector"],
88
+ registry=registry,
89
+ buckets=_BUCKETS,
90
+ )
91
+ self._overhead = Histogram(
92
+ "aliasgate_request_overhead_seconds",
93
+ "Time the gateway added to one request, by phase.",
94
+ ["route", "phase"],
95
+ registry=registry,
96
+ buckets=_BUCKETS,
97
+ )
98
+ if cache_stats:
99
+ registry.register(_CacheCollector(cache_stats))
100
+
101
+ def request(self, route: str, status: int, outcome: Outcome) -> None:
102
+ """One answered request."""
103
+ self._requests.labels(route, f"{status // 100}xx", outcome).inc()
104
+
105
+ def rejection(self, route: str, code: str) -> None:
106
+ """One error response the gateway produced, with its fixed error code."""
107
+ self._rejections.labels(route, code).inc()
108
+
109
+ def batch(self, stats: BatchStats) -> None:
110
+ """The replaced occurrences and detector timings of one pseudonymised request."""
111
+ for (entity_type, source), count in stats.entities_by_type_and_source.items():
112
+ self._entities.labels(entity_type, source).inc(count)
113
+ for detector, seconds in stats.detector_seconds.items():
114
+ self._detection.labels(detector).observe(seconds)
115
+
116
+ def unresolved(self, route: str, count: int) -> None:
117
+ """Placeholders left unresolved in one response."""
118
+ self._unresolved.labels(route).inc(count)
119
+
120
+ def leak(self, route: str, entity_types: Iterable[str]) -> None:
121
+ """One request rejected by the leak audit, once per leaked entity type."""
122
+ for entity_type in entity_types:
123
+ self._leaks.labels(route, entity_type).inc()
124
+
125
+ def overhead(self, route: str, phase: Phase, seconds: float) -> None:
126
+ """Time the gateway spent in one phase of one request."""
127
+ self._overhead.labels(route, phase).observe(seconds)
128
+
129
+ def render(self) -> bytes:
130
+ """The registry in Prometheus text format, served with CONTENT_TYPE."""
131
+ return generate_latest(self.registry)
@@ -0,0 +1,41 @@
1
+ """Placeholder format `[TYPE_N]` and the tolerant pattern used to find placeholders again."""
2
+
3
+ import re
4
+ from typing import Final
5
+
6
+ from aliasgate.types import validate_entity_type
7
+
8
+ MAX_PLACEHOLDER_LENGTH: Final = 64 # canonical placeholders
9
+ MAX_TOLERANT_LENGTH: Final = 128 # any match of TOLERANT
10
+
11
+ # Architecture §5.2, copied verbatim. Every quantifier is bounded so that no match can exceed
12
+ # MAX_TOLERANT_LENGTH; the stream restorer depends on that bound. Do not simplify this pattern
13
+ # without re-running the property tests.
14
+ _BODY = r"[ \t]{0,2}([A-Za-z](?:\\?_|[A-Za-z]){0,39}?)(?:\\?_| )(\d{1,9})[ \t]{0,2}"
15
+ TOLERANT: Final[re.Pattern[str]] = re.compile(rf"\\\[{_BODY}\\\]|\[{_BODY}\]")
16
+
17
+
18
+ def format_placeholder(entity_type: str, number: int) -> str:
19
+ """Return the canonical placeholder, e.g. format_placeholder("PERSON", 1) == "[PERSON_1]"."""
20
+ validate_entity_type(entity_type)
21
+ if number < 1:
22
+ raise ValueError("placeholder number must be at least 1")
23
+ placeholder = f"[{entity_type}_{number:d}]"
24
+ if len(placeholder) > MAX_PLACEHOLDER_LENGTH:
25
+ raise ValueError(
26
+ f"placeholder length {len(placeholder)} exceeds the maximum of {MAX_PLACEHOLDER_LENGTH}"
27
+ )
28
+ return placeholder
29
+
30
+
31
+ def parse_match(match: re.Match[str]) -> tuple[str, int]:
32
+ """Return (entity type, number) for a match of TOLERANT: take groups 1-2 or 3-4, whichever
33
+ matched, remove backslashes from the type and upper-case it."""
34
+ if match.re.groups != TOLERANT.groups:
35
+ raise ValueError("match is not a match of TOLERANT")
36
+ type_text, number_text = match.group(1, 2)
37
+ if type_text is None:
38
+ type_text, number_text = match.group(3, 4)
39
+ if type_text is None or number_text is None:
40
+ raise ValueError("match is not a match of TOLERANT")
41
+ return type_text.replace("\\", "").upper(), int(number_text)
@@ -0,0 +1 @@
1
+ """HTTP proxy: the gateway application, header filtering and error responses."""