aliasgate 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- aliasgate/__init__.py +5 -0
- aliasgate/__main__.py +8 -0
- aliasgate/audit.py +84 -0
- aliasgate/cli.py +321 -0
- aliasgate/config.py +367 -0
- aliasgate/detectors/__init__.py +1 -0
- aliasgate/detectors/cache.py +57 -0
- aliasgate/detectors/checksums.py +88 -0
- aliasgate/detectors/chunking.py +44 -0
- aliasgate/detectors/dictionary.py +96 -0
- aliasgate/detectors/gliner.py +195 -0
- aliasgate/detectors/literal.py +17 -0
- aliasgate/detectors/rules.py +318 -0
- aliasgate/dialects/__init__.py +1 -0
- aliasgate/dialects/anthropic_messages.py +329 -0
- aliasgate/dialects/base.py +193 -0
- aliasgate/dialects/openai_chat.py +359 -0
- aliasgate/dialects/sse.py +108 -0
- aliasgate/engine.py +237 -0
- aliasgate/eval/__init__.py +4 -0
- aliasgate/eval/__main__.py +139 -0
- aliasgate/eval/corpus.py +105 -0
- aliasgate/eval/metrics.py +197 -0
- aliasgate/eval/report.py +103 -0
- aliasgate/factory.py +67 -0
- aliasgate/mapping.py +103 -0
- aliasgate/metrics.py +131 -0
- aliasgate/placeholders.py +41 -0
- aliasgate/proxy/__init__.py +1 -0
- aliasgate/proxy/app.py +602 -0
- aliasgate/proxy/errors.py +106 -0
- aliasgate/proxy/headers.py +50 -0
- aliasgate/py.typed +0 -0
- aliasgate/resolve.py +32 -0
- aliasgate/stream.py +50 -0
- aliasgate/terms.py +18 -0
- aliasgate/types.py +70 -0
- aliasgate-0.1.0.dist-info/METADATA +151 -0
- aliasgate-0.1.0.dist-info/RECORD +42 -0
- aliasgate-0.1.0.dist-info/WHEEL +4 -0
- aliasgate-0.1.0.dist-info/entry_points.txt +2 -0
- aliasgate-0.1.0.dist-info/licenses/LICENSE +202 -0
aliasgate/engine.py
ADDED
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
"""Pseudonymisation engine (architecture §4) and placeholder restoration (§5.3)."""
|
|
2
|
+
|
|
3
|
+
import dataclasses
|
|
4
|
+
import json
|
|
5
|
+
import re
|
|
6
|
+
import time
|
|
7
|
+
from bisect import bisect_left
|
|
8
|
+
from collections import Counter, abc
|
|
9
|
+
from collections.abc import Iterable, Sequence
|
|
10
|
+
from dataclasses import dataclass
|
|
11
|
+
|
|
12
|
+
from aliasgate.detectors.literal import LiteralDetector
|
|
13
|
+
from aliasgate.mapping import Entity, Mapping, normalize_key
|
|
14
|
+
from aliasgate.placeholders import TOLERANT, parse_match
|
|
15
|
+
from aliasgate.resolve import resolve_overlaps
|
|
16
|
+
from aliasgate.terms import build_term_pattern
|
|
17
|
+
from aliasgate.types import LITERAL, PRIORITY_PROPAGATION, Detector, Span
|
|
18
|
+
|
|
19
|
+
# Literal keys are the exact surface behind a prefix, so that "[PERSON_1]" and "[person_1]"
|
|
20
|
+
# in the input stay two entities and both restore byte for byte (architecture §4.4).
|
|
21
|
+
_LITERAL_KEY_PREFIX = "\x00literal:"
|
|
22
|
+
|
|
23
|
+
_PROPAGATION_SOURCE = "propagation"
|
|
24
|
+
|
|
25
|
+
# A replaced range and its entity. The span carries the entity's type and the detector that
|
|
26
|
+
# found it ("propagation" for propagated ones).
|
|
27
|
+
type _Occurrence = tuple[Span, Entity]
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass(frozen=True, slots=True)
|
|
31
|
+
class BatchStats:
|
|
32
|
+
"""Counts and timings of one batch, for metrics: no text, no surface.
|
|
33
|
+
|
|
34
|
+
`entities_by_type_and_source` counts replaced occurrences by the entity's type and the
|
|
35
|
+
detector name that found them ("propagation" for propagated ones). `detector_seconds` is
|
|
36
|
+
the time each detector spent on the batch, by detector name.
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
entities_by_type_and_source: abc.Mapping[tuple[str, str], int]
|
|
40
|
+
detector_seconds: abc.Mapping[str, float]
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@dataclass(frozen=True, slots=True)
|
|
44
|
+
class PseudonymizedBatch:
|
|
45
|
+
"""Pseudonymised texts in input order, the mapping that restores them, and their stats."""
|
|
46
|
+
|
|
47
|
+
texts: tuple[str, ...]
|
|
48
|
+
mapping: Mapping
|
|
49
|
+
stats: BatchStats
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class DetectionError(Exception):
|
|
53
|
+
"""A detector failed. Carries the detector name; never carries text."""
|
|
54
|
+
|
|
55
|
+
def __init__(self, detector_name: str, reason: str = "raised an exception") -> None:
|
|
56
|
+
super().__init__(f"detector {detector_name!r} {reason}")
|
|
57
|
+
self.detector_name = detector_name
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class _Occupied:
|
|
61
|
+
"""Disjoint [start, end) ranges of one text."""
|
|
62
|
+
|
|
63
|
+
def __init__(self, ranges: Iterable[tuple[int, int]]) -> None:
|
|
64
|
+
self._ranges = sorted(ranges)
|
|
65
|
+
self._starts = [start for start, _ in self._ranges]
|
|
66
|
+
|
|
67
|
+
def claim(self, start: int, end: int) -> bool:
|
|
68
|
+
"""Record the range and return True, or return False if it overlaps a recorded one."""
|
|
69
|
+
index = bisect_left(self._starts, end)
|
|
70
|
+
if index > 0 and self._ranges[index - 1][1] > start:
|
|
71
|
+
return False
|
|
72
|
+
self._ranges.insert(index, (start, end))
|
|
73
|
+
self._starts.insert(index, start)
|
|
74
|
+
return True
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
class Pseudonymizer:
|
|
78
|
+
"""Replaces detected values with numbered placeholders, consistently across one batch.
|
|
79
|
+
|
|
80
|
+
Deterministic: numbering depends only on the content and order of the texts. Fails
|
|
81
|
+
closed: if any detector fails, `pseudonymize` raises DetectionError and returns nothing.
|
|
82
|
+
"""
|
|
83
|
+
|
|
84
|
+
def __init__(
|
|
85
|
+
self,
|
|
86
|
+
detectors: Sequence[Detector],
|
|
87
|
+
*,
|
|
88
|
+
allow_list: Iterable[str] = (),
|
|
89
|
+
propagate: bool = True,
|
|
90
|
+
min_propagation_length: int = 3,
|
|
91
|
+
) -> None:
|
|
92
|
+
if min_propagation_length < 1:
|
|
93
|
+
raise ValueError("min_propagation_length must be at least 1")
|
|
94
|
+
self._detectors: tuple[Detector, ...] = (LiteralDetector(), *detectors)
|
|
95
|
+
self._allow_keys = frozenset(normalize_key(term) for term in allow_list) - {""}
|
|
96
|
+
self._propagate = propagate
|
|
97
|
+
self._min_propagation_length = min_propagation_length
|
|
98
|
+
|
|
99
|
+
@property
|
|
100
|
+
def detectors(self) -> tuple[Detector, ...]:
|
|
101
|
+
"""The detectors in the order they run; the literal detector comes first."""
|
|
102
|
+
return self._detectors
|
|
103
|
+
|
|
104
|
+
def analyze(self, texts: Sequence[str]) -> list[list[Span]]:
|
|
105
|
+
"""The spans `pseudonymize` replaces in each text, sorted by start, without replacing.
|
|
106
|
+
|
|
107
|
+
Detected spans keep their detector's score, source and priority but carry the type of
|
|
108
|
+
the entity they are replaced as (that of its first occurrence); propagated spans have
|
|
109
|
+
source "propagation". Fails closed like `pseudonymize`.
|
|
110
|
+
"""
|
|
111
|
+
return [[span for span, _ in found] for found in self._analyze(tuple(texts))[1]]
|
|
112
|
+
|
|
113
|
+
def pseudonymize(self, texts: Sequence[str]) -> PseudonymizedBatch:
|
|
114
|
+
"""Pseudonymise all texts of one request as a batch (architecture §4.3)."""
|
|
115
|
+
batch = tuple(texts)
|
|
116
|
+
mapping, occurrences, seconds = self._analyze(batch)
|
|
117
|
+
counts = Counter(
|
|
118
|
+
(span.entity_type, span.source) for found in occurrences for span, _ in found
|
|
119
|
+
)
|
|
120
|
+
return PseudonymizedBatch(
|
|
121
|
+
tuple(_replace(text, found) for text, found in zip(batch, occurrences, strict=True)),
|
|
122
|
+
mapping,
|
|
123
|
+
BatchStats(dict(counts), seconds),
|
|
124
|
+
)
|
|
125
|
+
|
|
126
|
+
def _analyze(
|
|
127
|
+
self, batch: tuple[str, ...]
|
|
128
|
+
) -> tuple[Mapping, list[list[_Occurrence]], dict[str, float]]:
|
|
129
|
+
"""Detect, assign placeholders and propagate: everything but the replacement.
|
|
130
|
+
|
|
131
|
+
Returns the mapping, each text's occurrences sorted by start, and detector timings.
|
|
132
|
+
`analyze` and `pseudonymize` are both views of this one result, so they cannot drift.
|
|
133
|
+
"""
|
|
134
|
+
seconds = dict.fromkeys((detector.name for detector in self._detectors), 0.0)
|
|
135
|
+
resolved = [self._detect(text, seconds) for text in batch]
|
|
136
|
+
mapping = Mapping()
|
|
137
|
+
occurrences: list[list[_Occurrence]] = []
|
|
138
|
+
for text, spans in zip(batch, resolved, strict=True):
|
|
139
|
+
found = []
|
|
140
|
+
for span in spans:
|
|
141
|
+
entity = _assign(mapping, span, text[span.start : span.end])
|
|
142
|
+
found.append((dataclasses.replace(span, entity_type=entity.entity_type), entity))
|
|
143
|
+
occurrences.append(found)
|
|
144
|
+
if self._propagate:
|
|
145
|
+
self._propagate_entities(mapping, batch, occurrences)
|
|
146
|
+
for found in occurrences:
|
|
147
|
+
found.sort(key=lambda occurrence: occurrence[0].start)
|
|
148
|
+
return mapping, occurrences, seconds
|
|
149
|
+
|
|
150
|
+
def _detect(self, text: str, seconds: dict[str, float]) -> list[Span]:
|
|
151
|
+
spans: list[Span] = []
|
|
152
|
+
for detector in self._detectors:
|
|
153
|
+
started = time.perf_counter()
|
|
154
|
+
try:
|
|
155
|
+
found = list(detector.detect(text))
|
|
156
|
+
except Exception as exc:
|
|
157
|
+
raise DetectionError(detector.name) from exc
|
|
158
|
+
finally:
|
|
159
|
+
seconds[detector.name] += time.perf_counter() - started
|
|
160
|
+
if not all(isinstance(span, Span) and span.end <= len(text) for span in found):
|
|
161
|
+
raise DetectionError(detector.name, "returned a span outside the text")
|
|
162
|
+
spans.extend(found)
|
|
163
|
+
return resolve_overlaps(span for span in spans if self._keep(span, text))
|
|
164
|
+
|
|
165
|
+
def _keep(self, span: Span, text: str) -> bool:
|
|
166
|
+
if span.entity_type == LITERAL:
|
|
167
|
+
return True
|
|
168
|
+
key = normalize_key(text[span.start : span.end])
|
|
169
|
+
return bool(key) and key not in self._allow_keys
|
|
170
|
+
|
|
171
|
+
def _propagate_entities(
|
|
172
|
+
self, mapping: Mapping, texts: tuple[str, ...], occurrences: list[list[_Occurrence]]
|
|
173
|
+
) -> None:
|
|
174
|
+
entities = sorted(
|
|
175
|
+
(
|
|
176
|
+
entity
|
|
177
|
+
for entity in mapping.entities()
|
|
178
|
+
if entity.entity_type != LITERAL and len(entity.key) >= self._min_propagation_length
|
|
179
|
+
),
|
|
180
|
+
key=lambda entity: -len(entity.surface),
|
|
181
|
+
)
|
|
182
|
+
occupied = [_Occupied((span.start, span.end) for span, _ in found) for found in occurrences]
|
|
183
|
+
for entity in entities:
|
|
184
|
+
pattern = build_term_pattern(entity.surface)
|
|
185
|
+
for text, taken, found in zip(texts, occupied, occurrences, strict=True):
|
|
186
|
+
for match in pattern.finditer(text):
|
|
187
|
+
if taken.claim(match.start(), match.end()):
|
|
188
|
+
span = Span(
|
|
189
|
+
match.start(),
|
|
190
|
+
match.end(),
|
|
191
|
+
entity.entity_type,
|
|
192
|
+
1.0,
|
|
193
|
+
_PROPAGATION_SOURCE,
|
|
194
|
+
PRIORITY_PROPAGATION,
|
|
195
|
+
)
|
|
196
|
+
found.append((span, entity))
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def _assign(mapping: Mapping, span: Span, surface: str) -> Entity:
|
|
200
|
+
if span.entity_type == LITERAL:
|
|
201
|
+
return mapping.assign(LITERAL, surface, key=_LITERAL_KEY_PREFIX + surface)
|
|
202
|
+
return mapping.assign(span.entity_type, surface)
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def _replace(text: str, occurrences: list[_Occurrence]) -> str:
|
|
206
|
+
pieces: list[str] = []
|
|
207
|
+
position = 0
|
|
208
|
+
for span, entity in occurrences: # sorted by start
|
|
209
|
+
pieces.append(text[position : span.start])
|
|
210
|
+
pieces.append(entity.placeholder)
|
|
211
|
+
position = span.end
|
|
212
|
+
pieces.append(text[position:])
|
|
213
|
+
return "".join(pieces)
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def restore(text: str, mapping: Mapping, *, json_escape: bool = False) -> str:
|
|
217
|
+
"""Replace every placeholder of `mapping` in `text` by its canonical surface.
|
|
218
|
+
|
|
219
|
+
One left-to-right pass: restored values are not scanned again, and placeholders that are
|
|
220
|
+
not in the mapping are left exactly as they are. With `json_escape`, surfaces are inserted
|
|
221
|
+
escaped for the inside of a JSON string.
|
|
222
|
+
"""
|
|
223
|
+
|
|
224
|
+
def replacement(match: re.Match[str]) -> str:
|
|
225
|
+
surface = mapping.lookup(*parse_match(match))
|
|
226
|
+
if surface is None:
|
|
227
|
+
return match.group()
|
|
228
|
+
return json.dumps(surface, ensure_ascii=False)[1:-1] if json_escape else surface
|
|
229
|
+
|
|
230
|
+
return TOLERANT.sub(replacement, text)
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def count_unresolved(text: str, mapping: Mapping) -> int:
|
|
234
|
+
"""Number of TOLERANT matches in `text` that are not in the mapping."""
|
|
235
|
+
return sum(
|
|
236
|
+
1 for match in TOLERANT.finditer(text) if mapping.lookup(*parse_match(match)) is None
|
|
237
|
+
)
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
"""`python -m aliasgate.eval CORPUS...`: measure detection on labelled corpora, per language.
|
|
2
|
+
|
|
3
|
+
The engine is built from the configuration exactly as the gateway builds it. The output holds
|
|
4
|
+
numbers, entity types and example ids, never example text. Exit codes: 0, 1 when a leak rate is
|
|
5
|
+
above `--max-leak-rate`, 2 for invalid input or a failure.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import argparse
|
|
9
|
+
import math
|
|
10
|
+
import os
|
|
11
|
+
import sys
|
|
12
|
+
from collections.abc import Mapping, Sequence
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
import yaml
|
|
16
|
+
|
|
17
|
+
from aliasgate.config import load_settings
|
|
18
|
+
from aliasgate.engine import DetectionError
|
|
19
|
+
from aliasgate.eval.corpus import Example, load_corpus
|
|
20
|
+
from aliasgate.eval.metrics import Report, evaluate
|
|
21
|
+
from aliasgate.eval.report import format_json, format_table
|
|
22
|
+
from aliasgate.factory import build_pseudonymizer
|
|
23
|
+
from aliasgate.types import validate_entity_type
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _rate(value: str) -> float:
|
|
27
|
+
rate = float(value)
|
|
28
|
+
if math.isnan(rate) or not 0.0 <= rate <= 1.0:
|
|
29
|
+
raise argparse.ArgumentTypeError("must be a number between 0 and 1")
|
|
30
|
+
return rate
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _types(value: str) -> frozenset[str]:
|
|
34
|
+
names = [name.strip() for name in value.split(",")]
|
|
35
|
+
try:
|
|
36
|
+
return frozenset(validate_entity_type(name) for name in names)
|
|
37
|
+
except ValueError:
|
|
38
|
+
raise argparse.ArgumentTypeError(
|
|
39
|
+
"must be comma-separated entity types such as EMAIL,IBAN"
|
|
40
|
+
) from None
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _count(value: str) -> int:
|
|
44
|
+
count = int(value)
|
|
45
|
+
if count < 0:
|
|
46
|
+
raise argparse.ArgumentTypeError("must not be negative")
|
|
47
|
+
return count
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _parser() -> argparse.ArgumentParser:
|
|
51
|
+
parser = argparse.ArgumentParser(
|
|
52
|
+
prog="python -m aliasgate.eval",
|
|
53
|
+
description="Measure detection on labelled JSON Lines corpora, per language.",
|
|
54
|
+
)
|
|
55
|
+
parser.add_argument("corpus", nargs="+", type=Path, help="JSON Lines corpus files")
|
|
56
|
+
parser.add_argument(
|
|
57
|
+
"--config",
|
|
58
|
+
type=Path,
|
|
59
|
+
default=None,
|
|
60
|
+
metavar="PATH",
|
|
61
|
+
help="gateway configuration; ALIASGATE_* environment variables override it",
|
|
62
|
+
)
|
|
63
|
+
parser.add_argument("--format", choices=("table", "json"), default="table")
|
|
64
|
+
parser.add_argument(
|
|
65
|
+
"--max-leak-rate",
|
|
66
|
+
type=_rate,
|
|
67
|
+
default=None,
|
|
68
|
+
metavar="RATE",
|
|
69
|
+
help="exit with code 1 if a language's leak rate is above RATE",
|
|
70
|
+
)
|
|
71
|
+
parser.add_argument(
|
|
72
|
+
"--types", type=_types, default=None, metavar="TYPES", help="e.g. EMAIL,IBAN"
|
|
73
|
+
)
|
|
74
|
+
parser.add_argument(
|
|
75
|
+
"--show-misses",
|
|
76
|
+
type=_count,
|
|
77
|
+
default=0,
|
|
78
|
+
metavar="N",
|
|
79
|
+
help="list up to N missed spans by example id and offsets",
|
|
80
|
+
)
|
|
81
|
+
return parser
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def main(argv: Sequence[str] | None = None, *, env: Mapping[str, str] = os.environ) -> int:
|
|
85
|
+
"""Evaluate the corpora, print the report and return the exit code."""
|
|
86
|
+
args = _parser().parse_args(argv)
|
|
87
|
+
try:
|
|
88
|
+
settings = load_settings(args.config, env)
|
|
89
|
+
by_lang = _by_language([example for path in args.corpus for example in load_corpus(path)])
|
|
90
|
+
pseudonymizer = build_pseudonymizer(settings)
|
|
91
|
+
except yaml.YAMLError:
|
|
92
|
+
# The parser's message quotes the offending line.
|
|
93
|
+
return _fail("the configuration file is not valid YAML")
|
|
94
|
+
except (ValueError, RuntimeError, OSError) as exc:
|
|
95
|
+
# Configuration, corpus and dictionary errors never repeat their input.
|
|
96
|
+
return _fail(f"invalid configuration or corpus: {exc}")
|
|
97
|
+
try:
|
|
98
|
+
reports = {
|
|
99
|
+
lang: evaluate(pseudonymizer, examples, types=args.types)
|
|
100
|
+
for lang, examples in sorted(by_lang.items())
|
|
101
|
+
}
|
|
102
|
+
except DetectionError as exc:
|
|
103
|
+
return _fail(str(exc))
|
|
104
|
+
except Exception as exc: # anything else may carry text in its message
|
|
105
|
+
return _fail(f"evaluation failed ({type(exc).__name__})")
|
|
106
|
+
if args.format == "json":
|
|
107
|
+
print(format_json(reports, misses=args.show_misses))
|
|
108
|
+
else:
|
|
109
|
+
print(
|
|
110
|
+
"\n\n".join(
|
|
111
|
+
format_table(lang, report, misses=args.show_misses)
|
|
112
|
+
for lang, report in reports.items()
|
|
113
|
+
)
|
|
114
|
+
)
|
|
115
|
+
return 1 if _above(reports, args.max_leak_rate) else 0
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _by_language(examples: list[Example]) -> dict[str, list[Example]]:
|
|
119
|
+
by_lang: dict[str, list[Example]] = {}
|
|
120
|
+
seen: set[str] = set()
|
|
121
|
+
for example in examples:
|
|
122
|
+
if example.id in seen:
|
|
123
|
+
raise ValueError(f"example id {example.id!r} appears in more than one place")
|
|
124
|
+
seen.add(example.id)
|
|
125
|
+
by_lang.setdefault(example.lang, []).append(example)
|
|
126
|
+
return by_lang
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def _above(reports: Mapping[str, Report], limit: float | None) -> bool:
|
|
130
|
+
return limit is not None and any(report.leak_rate > limit for report in reports.values())
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def _fail(message: str) -> int:
|
|
134
|
+
print(f"error: {message}", file=sys.stderr)
|
|
135
|
+
return 2
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
if __name__ == "__main__":
|
|
139
|
+
sys.exit(main())
|
aliasgate/eval/corpus.py
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
"""Labelled corpora in JSON Lines: one example per line, gold spans as Python string offsets.
|
|
2
|
+
|
|
3
|
+
{"id": "en-0001", "lang": "en", "text": "...",
|
|
4
|
+
"spans": [{"start": 10, "end": 20, "type": "PERSON"}]}
|
|
5
|
+
|
|
6
|
+
Each example is one line of the file. Offsets count code points, as Python slices do. Errors
|
|
7
|
+
name the file, the line and the example id, never the text.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from typing import Final
|
|
14
|
+
|
|
15
|
+
from aliasgate.types import validate_entity_type
|
|
16
|
+
|
|
17
|
+
_EXAMPLE_KEYS: Final = frozenset({"id", "lang", "text", "spans"})
|
|
18
|
+
_SPAN_KEYS: Final = frozenset({"start", "end", "type"})
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass(frozen=True, slots=True)
|
|
22
|
+
class GoldSpan:
|
|
23
|
+
"""A labelled range of one example: Python string offsets and an entity type."""
|
|
24
|
+
|
|
25
|
+
start: int
|
|
26
|
+
end: int
|
|
27
|
+
entity_type: str
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass(frozen=True, slots=True)
|
|
31
|
+
class Example:
|
|
32
|
+
"""One labelled text; its spans are in range, sorted, non-overlapping and non-empty."""
|
|
33
|
+
|
|
34
|
+
id: str
|
|
35
|
+
lang: str
|
|
36
|
+
text: str
|
|
37
|
+
spans: tuple[GoldSpan, ...]
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class CorpusError(ValueError):
|
|
41
|
+
"""A corpus file is malformed. The message gives a position, never text."""
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def load_corpus(path: Path) -> list[Example]:
|
|
45
|
+
"""Validate offsets (in range, non-overlapping, sorted) and entity types; raise on error.
|
|
46
|
+
|
|
47
|
+
Blank lines are skipped. Example ids must be unique within the file.
|
|
48
|
+
"""
|
|
49
|
+
examples: list[Example] = []
|
|
50
|
+
seen: set[str] = set()
|
|
51
|
+
with path.open(encoding="utf-8") as stream:
|
|
52
|
+
for number, line in enumerate(stream, start=1):
|
|
53
|
+
if not line.strip():
|
|
54
|
+
continue
|
|
55
|
+
example = _example(f"{path}, line {number}", line)
|
|
56
|
+
if example.id in seen:
|
|
57
|
+
raise CorpusError(f"{path}, line {number}, example {example.id!r}: duplicate id")
|
|
58
|
+
seen.add(example.id)
|
|
59
|
+
examples.append(example)
|
|
60
|
+
return examples
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _example(where: str, line: str) -> Example:
|
|
64
|
+
try:
|
|
65
|
+
data = json.loads(line)
|
|
66
|
+
except json.JSONDecodeError:
|
|
67
|
+
data = None
|
|
68
|
+
if not isinstance(data, dict):
|
|
69
|
+
raise CorpusError(f"{where}: not a JSON object")
|
|
70
|
+
if set(data) != _EXAMPLE_KEYS:
|
|
71
|
+
raise CorpusError(f"{where}: an example has exactly the keys id, lang, text and spans")
|
|
72
|
+
example_id, lang, text, spans = data["id"], data["lang"], data["text"], data["spans"]
|
|
73
|
+
if not isinstance(example_id, str) or not example_id:
|
|
74
|
+
raise CorpusError(f"{where}: 'id' must be a non-empty string")
|
|
75
|
+
where = f"{where}, example {example_id!r}"
|
|
76
|
+
if not isinstance(lang, str) or not lang:
|
|
77
|
+
raise CorpusError(f"{where}: 'lang' must be a non-empty string")
|
|
78
|
+
if not isinstance(text, str):
|
|
79
|
+
raise CorpusError(f"{where}: 'text' must be a string")
|
|
80
|
+
if not isinstance(spans, list):
|
|
81
|
+
raise CorpusError(f"{where}: 'spans' must be a list")
|
|
82
|
+
gold: list[GoldSpan] = []
|
|
83
|
+
for index, span in enumerate(spans, start=1):
|
|
84
|
+
gold.append(_span(f"{where}: span {index}", span, len(text)))
|
|
85
|
+
if index > 1 and gold[-1].start < gold[-2].end:
|
|
86
|
+
raise CorpusError(f"{where}: span {index} overlaps or precedes span {index - 1}")
|
|
87
|
+
return Example(example_id, lang, text, tuple(gold))
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _span(where: str, span: object, length: int) -> GoldSpan:
|
|
91
|
+
if not isinstance(span, dict) or set(span) != _SPAN_KEYS:
|
|
92
|
+
raise CorpusError(f"{where} must be an object with start, end and type")
|
|
93
|
+
start, end, entity_type = span["start"], span["end"], span["type"]
|
|
94
|
+
# bool is an int subclass; true and false are not offsets.
|
|
95
|
+
if any(not isinstance(value, int) or isinstance(value, bool) for value in (start, end)):
|
|
96
|
+
raise CorpusError(f"{where} offsets must be integers")
|
|
97
|
+
if not 0 <= start < end <= length:
|
|
98
|
+
raise CorpusError(f"{where} is out of range (text length {length})")
|
|
99
|
+
try:
|
|
100
|
+
if not isinstance(entity_type, str):
|
|
101
|
+
raise ValueError
|
|
102
|
+
validate_entity_type(entity_type)
|
|
103
|
+
except ValueError:
|
|
104
|
+
raise CorpusError(f"{where} has an invalid entity type") from None
|
|
105
|
+
return GoldSpan(start, end, entity_type)
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
"""Detection quality of the whole engine on labelled examples: counts and ratios, never text.
|
|
2
|
+
|
|
3
|
+
The headline number is the leak rate: the share of gold characters that no predicted span
|
|
4
|
+
covers, whatever its type. Recall by overlap ignores the type on purpose, because a person
|
|
5
|
+
detected as an organisation is still protected.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from collections import Counter
|
|
9
|
+
from collections.abc import Collection, Sequence
|
|
10
|
+
from dataclasses import dataclass
|
|
11
|
+
|
|
12
|
+
from aliasgate.engine import Pseudonymizer, restore
|
|
13
|
+
from aliasgate.eval.corpus import Example, GoldSpan
|
|
14
|
+
from aliasgate.mapping import Entity, Mapping, normalize_key
|
|
15
|
+
from aliasgate.terms import build_term_pattern
|
|
16
|
+
from aliasgate.types import LITERAL, Span
|
|
17
|
+
|
|
18
|
+
_PROPAGATION_SOURCE = "propagation"
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _ratio(numerator: int, denominator: int, *, empty: float) -> float:
|
|
22
|
+
return numerator / denominator if denominator else empty
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass(frozen=True, slots=True)
|
|
26
|
+
class TypeScore:
|
|
27
|
+
"""Counts for one entity type. Recall is 1.0 when there is no gold span of the type, and
|
|
28
|
+
precision is 1.0 when nothing of the type is predicted."""
|
|
29
|
+
|
|
30
|
+
entity_type: str
|
|
31
|
+
gold: int
|
|
32
|
+
predicted: int
|
|
33
|
+
exact_tp: int # gold spans predicted with the same start, end and type
|
|
34
|
+
overlap_tp: int # gold spans that any predicted span overlaps, of any type
|
|
35
|
+
|
|
36
|
+
@property
|
|
37
|
+
def recall_exact(self) -> float:
|
|
38
|
+
"""exact_tp / gold, or 1.0 without gold spans."""
|
|
39
|
+
return _ratio(self.exact_tp, self.gold, empty=1.0)
|
|
40
|
+
|
|
41
|
+
@property
|
|
42
|
+
def recall_overlap(self) -> float:
|
|
43
|
+
"""overlap_tp / gold, or 1.0 without gold spans."""
|
|
44
|
+
return _ratio(self.overlap_tp, self.gold, empty=1.0)
|
|
45
|
+
|
|
46
|
+
@property
|
|
47
|
+
def precision_exact(self) -> float:
|
|
48
|
+
"""exact_tp / predicted, or 1.0 when nothing is predicted."""
|
|
49
|
+
return _ratio(self.exact_tp, self.predicted, empty=1.0)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@dataclass(frozen=True, slots=True)
|
|
53
|
+
class Miss:
|
|
54
|
+
"""A gold span with characters no predicted span covers: its position, never its text."""
|
|
55
|
+
|
|
56
|
+
example_id: str
|
|
57
|
+
start: int
|
|
58
|
+
end: int
|
|
59
|
+
entity_type: str
|
|
60
|
+
leaked_chars: int
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
@dataclass(frozen=True, slots=True)
|
|
64
|
+
class Report:
|
|
65
|
+
"""Scores of one evaluation, by entity type, plus leak, round-trip and false-alarm counts.
|
|
66
|
+
|
|
67
|
+
`entity_free` counts the examples without gold spans and `entity_free_flagged` those of
|
|
68
|
+
them with at least one predicted span. With a type restriction, both count only spans of
|
|
69
|
+
the selected types.
|
|
70
|
+
"""
|
|
71
|
+
|
|
72
|
+
by_type: tuple[TypeScore, ...]
|
|
73
|
+
leaked_chars: int # characters of gold spans still present after pseudonymisation
|
|
74
|
+
gold_chars: int
|
|
75
|
+
round_trip_failures: int
|
|
76
|
+
examples: int
|
|
77
|
+
entity_free: int
|
|
78
|
+
entity_free_flagged: int
|
|
79
|
+
misses: tuple[Miss, ...]
|
|
80
|
+
|
|
81
|
+
@property
|
|
82
|
+
def leak_rate(self) -> float:
|
|
83
|
+
"""leaked_chars / gold_chars, or 0.0 without gold spans."""
|
|
84
|
+
return _ratio(self.leaked_chars, self.gold_chars, empty=0.0)
|
|
85
|
+
|
|
86
|
+
@property
|
|
87
|
+
def entity_free_flag_rate(self) -> float:
|
|
88
|
+
"""entity_free_flagged / entity_free, or 0.0 without such examples."""
|
|
89
|
+
return _ratio(self.entity_free_flagged, self.entity_free, empty=0.0)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def evaluate(
|
|
93
|
+
pseudonymizer: Pseudonymizer,
|
|
94
|
+
examples: Sequence[Example],
|
|
95
|
+
*,
|
|
96
|
+
types: Collection[str] | None = None,
|
|
97
|
+
) -> Report:
|
|
98
|
+
"""Run the whole engine on each example as its own batch and score its final spans.
|
|
99
|
+
|
|
100
|
+
With `types`, only gold and predicted spans of those types are scored, but a predicted
|
|
101
|
+
span of any type still covers gold characters.
|
|
102
|
+
"""
|
|
103
|
+
gold_counts: Counter[str] = Counter()
|
|
104
|
+
predicted_counts: Counter[str] = Counter()
|
|
105
|
+
exact: Counter[str] = Counter()
|
|
106
|
+
overlap: Counter[str] = Counter()
|
|
107
|
+
leaked = gold_chars = failures = entity_free = flagged = 0
|
|
108
|
+
misses: list[Miss] = []
|
|
109
|
+
for example in examples:
|
|
110
|
+
spans = pseudonymizer.analyze([example.text])[0]
|
|
111
|
+
batch = pseudonymizer.pseudonymize([example.text])
|
|
112
|
+
expected = expected_restoration(example.text, spans, batch.mapping)
|
|
113
|
+
if restore(batch.texts[0], batch.mapping) != expected:
|
|
114
|
+
failures += 1
|
|
115
|
+
gold = [span for span in example.spans if types is None or span.entity_type in types]
|
|
116
|
+
predicted = [span for span in spans if types is None or span.entity_type in types]
|
|
117
|
+
predicted_counts.update(span.entity_type for span in predicted)
|
|
118
|
+
if not gold:
|
|
119
|
+
entity_free += 1
|
|
120
|
+
flagged += bool(predicted)
|
|
121
|
+
for span in gold:
|
|
122
|
+
covered = _covered(span, spans)
|
|
123
|
+
gold_counts[span.entity_type] += 1
|
|
124
|
+
gold_chars += span.end - span.start
|
|
125
|
+
leaked += span.end - span.start - covered
|
|
126
|
+
overlap[span.entity_type] += covered > 0
|
|
127
|
+
exact[span.entity_type] += any(
|
|
128
|
+
(found.start, found.end, found.entity_type)
|
|
129
|
+
== (span.start, span.end, span.entity_type)
|
|
130
|
+
for found in spans
|
|
131
|
+
)
|
|
132
|
+
if covered < span.end - span.start:
|
|
133
|
+
misses.append(
|
|
134
|
+
Miss(
|
|
135
|
+
example.id,
|
|
136
|
+
span.start,
|
|
137
|
+
span.end,
|
|
138
|
+
span.entity_type,
|
|
139
|
+
span.end - span.start - covered,
|
|
140
|
+
)
|
|
141
|
+
)
|
|
142
|
+
by_type = tuple(
|
|
143
|
+
TypeScore(name, gold_counts[name], predicted_counts[name], exact[name], overlap[name])
|
|
144
|
+
for name in sorted(gold_counts.keys() | predicted_counts.keys())
|
|
145
|
+
)
|
|
146
|
+
return Report(
|
|
147
|
+
by_type, leaked, gold_chars, failures, len(examples), entity_free, flagged, tuple(misses)
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _covered(gold: GoldSpan, spans: Sequence[Span]) -> int:
|
|
152
|
+
"""Characters of `gold` inside predicted spans; these never overlap one another."""
|
|
153
|
+
return sum(max(0, min(gold.end, span.end) - max(gold.start, span.start)) for span in spans)
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def expected_restoration(text: str, spans: Sequence[Span], mapping: Mapping) -> str | None:
|
|
157
|
+
"""What `restore(pseudonymize(text))` must give: the text with every span `analyze`
|
|
158
|
+
reports rendered as its entity's canonical surface, a placeholder literal as itself.
|
|
159
|
+
|
|
160
|
+
A detected span's entity is the one with its key; a propagated span's is the longest
|
|
161
|
+
entity of its type whose term pattern matches it, as propagation chose it (keys and
|
|
162
|
+
patterns differ for a few characters, such as a dotted capital I). None if a span has no
|
|
163
|
+
entity in the mapping.
|
|
164
|
+
"""
|
|
165
|
+
pieces: list[str] = []
|
|
166
|
+
position = 0
|
|
167
|
+
for span in spans:
|
|
168
|
+
surface = text[span.start : span.end]
|
|
169
|
+
if span.entity_type == LITERAL:
|
|
170
|
+
canonical: str | None = surface
|
|
171
|
+
else:
|
|
172
|
+
entity = _entity_of(span, surface, mapping)
|
|
173
|
+
canonical = entity.surface if entity is not None else None
|
|
174
|
+
if canonical is None:
|
|
175
|
+
return None
|
|
176
|
+
pieces.append(text[position : span.start])
|
|
177
|
+
pieces.append(canonical)
|
|
178
|
+
position = span.end
|
|
179
|
+
pieces.append(text[position:])
|
|
180
|
+
return "".join(pieces)
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def _entity_of(span: Span, surface: str, mapping: Mapping) -> Entity | None:
|
|
184
|
+
if span.source != _PROPAGATION_SOURCE:
|
|
185
|
+
return mapping.get(normalize_key(surface))
|
|
186
|
+
candidates = sorted(
|
|
187
|
+
(entity for entity in mapping.entities() if entity.entity_type == span.entity_type),
|
|
188
|
+
key=lambda entity: -len(entity.surface),
|
|
189
|
+
)
|
|
190
|
+
return next(
|
|
191
|
+
(
|
|
192
|
+
entity
|
|
193
|
+
for entity in candidates
|
|
194
|
+
if build_term_pattern(entity.surface).fullmatch(surface) is not None
|
|
195
|
+
),
|
|
196
|
+
None,
|
|
197
|
+
)
|