aliasgate 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
aliasgate/engine.py ADDED
@@ -0,0 +1,237 @@
1
+ """Pseudonymisation engine (architecture §4) and placeholder restoration (§5.3)."""
2
+
3
+ import dataclasses
4
+ import json
5
+ import re
6
+ import time
7
+ from bisect import bisect_left
8
+ from collections import Counter, abc
9
+ from collections.abc import Iterable, Sequence
10
+ from dataclasses import dataclass
11
+
12
+ from aliasgate.detectors.literal import LiteralDetector
13
+ from aliasgate.mapping import Entity, Mapping, normalize_key
14
+ from aliasgate.placeholders import TOLERANT, parse_match
15
+ from aliasgate.resolve import resolve_overlaps
16
+ from aliasgate.terms import build_term_pattern
17
+ from aliasgate.types import LITERAL, PRIORITY_PROPAGATION, Detector, Span
18
+
19
+ # Literal keys are the exact surface behind a prefix, so that "[PERSON_1]" and "[person_1]"
20
+ # in the input stay two entities and both restore byte for byte (architecture §4.4).
21
+ _LITERAL_KEY_PREFIX = "\x00literal:"
22
+
23
+ _PROPAGATION_SOURCE = "propagation"
24
+
25
+ # A replaced range and its entity. The span carries the entity's type and the detector that
26
+ # found it ("propagation" for propagated ones).
27
+ type _Occurrence = tuple[Span, Entity]
28
+
29
+
30
+ @dataclass(frozen=True, slots=True)
31
+ class BatchStats:
32
+ """Counts and timings of one batch, for metrics: no text, no surface.
33
+
34
+ `entities_by_type_and_source` counts replaced occurrences by the entity's type and the
35
+ detector name that found them ("propagation" for propagated ones). `detector_seconds` is
36
+ the time each detector spent on the batch, by detector name.
37
+ """
38
+
39
+ entities_by_type_and_source: abc.Mapping[tuple[str, str], int]
40
+ detector_seconds: abc.Mapping[str, float]
41
+
42
+
43
+ @dataclass(frozen=True, slots=True)
44
+ class PseudonymizedBatch:
45
+ """Pseudonymised texts in input order, the mapping that restores them, and their stats."""
46
+
47
+ texts: tuple[str, ...]
48
+ mapping: Mapping
49
+ stats: BatchStats
50
+
51
+
52
+ class DetectionError(Exception):
53
+ """A detector failed. Carries the detector name; never carries text."""
54
+
55
+ def __init__(self, detector_name: str, reason: str = "raised an exception") -> None:
56
+ super().__init__(f"detector {detector_name!r} {reason}")
57
+ self.detector_name = detector_name
58
+
59
+
60
+ class _Occupied:
61
+ """Disjoint [start, end) ranges of one text."""
62
+
63
+ def __init__(self, ranges: Iterable[tuple[int, int]]) -> None:
64
+ self._ranges = sorted(ranges)
65
+ self._starts = [start for start, _ in self._ranges]
66
+
67
+ def claim(self, start: int, end: int) -> bool:
68
+ """Record the range and return True, or return False if it overlaps a recorded one."""
69
+ index = bisect_left(self._starts, end)
70
+ if index > 0 and self._ranges[index - 1][1] > start:
71
+ return False
72
+ self._ranges.insert(index, (start, end))
73
+ self._starts.insert(index, start)
74
+ return True
75
+
76
+
77
+ class Pseudonymizer:
78
+ """Replaces detected values with numbered placeholders, consistently across one batch.
79
+
80
+ Deterministic: numbering depends only on the content and order of the texts. Fails
81
+ closed: if any detector fails, `pseudonymize` raises DetectionError and returns nothing.
82
+ """
83
+
84
+ def __init__(
85
+ self,
86
+ detectors: Sequence[Detector],
87
+ *,
88
+ allow_list: Iterable[str] = (),
89
+ propagate: bool = True,
90
+ min_propagation_length: int = 3,
91
+ ) -> None:
92
+ if min_propagation_length < 1:
93
+ raise ValueError("min_propagation_length must be at least 1")
94
+ self._detectors: tuple[Detector, ...] = (LiteralDetector(), *detectors)
95
+ self._allow_keys = frozenset(normalize_key(term) for term in allow_list) - {""}
96
+ self._propagate = propagate
97
+ self._min_propagation_length = min_propagation_length
98
+
99
+ @property
100
+ def detectors(self) -> tuple[Detector, ...]:
101
+ """The detectors in the order they run; the literal detector comes first."""
102
+ return self._detectors
103
+
104
+ def analyze(self, texts: Sequence[str]) -> list[list[Span]]:
105
+ """The spans `pseudonymize` replaces in each text, sorted by start, without replacing.
106
+
107
+ Detected spans keep their detector's score, source and priority but carry the type of
108
+ the entity they are replaced as (that of its first occurrence); propagated spans have
109
+ source "propagation". Fails closed like `pseudonymize`.
110
+ """
111
+ return [[span for span, _ in found] for found in self._analyze(tuple(texts))[1]]
112
+
113
+ def pseudonymize(self, texts: Sequence[str]) -> PseudonymizedBatch:
114
+ """Pseudonymise all texts of one request as a batch (architecture §4.3)."""
115
+ batch = tuple(texts)
116
+ mapping, occurrences, seconds = self._analyze(batch)
117
+ counts = Counter(
118
+ (span.entity_type, span.source) for found in occurrences for span, _ in found
119
+ )
120
+ return PseudonymizedBatch(
121
+ tuple(_replace(text, found) for text, found in zip(batch, occurrences, strict=True)),
122
+ mapping,
123
+ BatchStats(dict(counts), seconds),
124
+ )
125
+
126
+ def _analyze(
127
+ self, batch: tuple[str, ...]
128
+ ) -> tuple[Mapping, list[list[_Occurrence]], dict[str, float]]:
129
+ """Detect, assign placeholders and propagate: everything but the replacement.
130
+
131
+ Returns the mapping, each text's occurrences sorted by start, and detector timings.
132
+ `analyze` and `pseudonymize` are both views of this one result, so they cannot drift.
133
+ """
134
+ seconds = dict.fromkeys((detector.name for detector in self._detectors), 0.0)
135
+ resolved = [self._detect(text, seconds) for text in batch]
136
+ mapping = Mapping()
137
+ occurrences: list[list[_Occurrence]] = []
138
+ for text, spans in zip(batch, resolved, strict=True):
139
+ found = []
140
+ for span in spans:
141
+ entity = _assign(mapping, span, text[span.start : span.end])
142
+ found.append((dataclasses.replace(span, entity_type=entity.entity_type), entity))
143
+ occurrences.append(found)
144
+ if self._propagate:
145
+ self._propagate_entities(mapping, batch, occurrences)
146
+ for found in occurrences:
147
+ found.sort(key=lambda occurrence: occurrence[0].start)
148
+ return mapping, occurrences, seconds
149
+
150
+ def _detect(self, text: str, seconds: dict[str, float]) -> list[Span]:
151
+ spans: list[Span] = []
152
+ for detector in self._detectors:
153
+ started = time.perf_counter()
154
+ try:
155
+ found = list(detector.detect(text))
156
+ except Exception as exc:
157
+ raise DetectionError(detector.name) from exc
158
+ finally:
159
+ seconds[detector.name] += time.perf_counter() - started
160
+ if not all(isinstance(span, Span) and span.end <= len(text) for span in found):
161
+ raise DetectionError(detector.name, "returned a span outside the text")
162
+ spans.extend(found)
163
+ return resolve_overlaps(span for span in spans if self._keep(span, text))
164
+
165
+ def _keep(self, span: Span, text: str) -> bool:
166
+ if span.entity_type == LITERAL:
167
+ return True
168
+ key = normalize_key(text[span.start : span.end])
169
+ return bool(key) and key not in self._allow_keys
170
+
171
+ def _propagate_entities(
172
+ self, mapping: Mapping, texts: tuple[str, ...], occurrences: list[list[_Occurrence]]
173
+ ) -> None:
174
+ entities = sorted(
175
+ (
176
+ entity
177
+ for entity in mapping.entities()
178
+ if entity.entity_type != LITERAL and len(entity.key) >= self._min_propagation_length
179
+ ),
180
+ key=lambda entity: -len(entity.surface),
181
+ )
182
+ occupied = [_Occupied((span.start, span.end) for span, _ in found) for found in occurrences]
183
+ for entity in entities:
184
+ pattern = build_term_pattern(entity.surface)
185
+ for text, taken, found in zip(texts, occupied, occurrences, strict=True):
186
+ for match in pattern.finditer(text):
187
+ if taken.claim(match.start(), match.end()):
188
+ span = Span(
189
+ match.start(),
190
+ match.end(),
191
+ entity.entity_type,
192
+ 1.0,
193
+ _PROPAGATION_SOURCE,
194
+ PRIORITY_PROPAGATION,
195
+ )
196
+ found.append((span, entity))
197
+
198
+
199
+ def _assign(mapping: Mapping, span: Span, surface: str) -> Entity:
200
+ if span.entity_type == LITERAL:
201
+ return mapping.assign(LITERAL, surface, key=_LITERAL_KEY_PREFIX + surface)
202
+ return mapping.assign(span.entity_type, surface)
203
+
204
+
205
+ def _replace(text: str, occurrences: list[_Occurrence]) -> str:
206
+ pieces: list[str] = []
207
+ position = 0
208
+ for span, entity in occurrences: # sorted by start
209
+ pieces.append(text[position : span.start])
210
+ pieces.append(entity.placeholder)
211
+ position = span.end
212
+ pieces.append(text[position:])
213
+ return "".join(pieces)
214
+
215
+
216
+ def restore(text: str, mapping: Mapping, *, json_escape: bool = False) -> str:
217
+ """Replace every placeholder of `mapping` in `text` by its canonical surface.
218
+
219
+ One left-to-right pass: restored values are not scanned again, and placeholders that are
220
+ not in the mapping are left exactly as they are. With `json_escape`, surfaces are inserted
221
+ escaped for the inside of a JSON string.
222
+ """
223
+
224
+ def replacement(match: re.Match[str]) -> str:
225
+ surface = mapping.lookup(*parse_match(match))
226
+ if surface is None:
227
+ return match.group()
228
+ return json.dumps(surface, ensure_ascii=False)[1:-1] if json_escape else surface
229
+
230
+ return TOLERANT.sub(replacement, text)
231
+
232
+
233
+ def count_unresolved(text: str, mapping: Mapping) -> int:
234
+ """Number of TOLERANT matches in `text` that are not in the mapping."""
235
+ return sum(
236
+ 1 for match in TOLERANT.finditer(text) if mapping.lookup(*parse_match(match)) is None
237
+ )
@@ -0,0 +1,4 @@
1
+ """Evaluation harness: labelled corpora, detection metrics and reports (numbers, never text).
2
+
3
+ Run it with `python -m aliasgate.eval`. Nothing here imports the corpus generator's Faker.
4
+ """
@@ -0,0 +1,139 @@
1
+ """`python -m aliasgate.eval CORPUS...`: measure detection on labelled corpora, per language.
2
+
3
+ The engine is built from the configuration exactly as the gateway builds it. The output holds
4
+ numbers, entity types and example ids, never example text. Exit codes: 0, 1 when a leak rate is
5
+ above `--max-leak-rate`, 2 for invalid input or a failure.
6
+ """
7
+
8
+ import argparse
9
+ import math
10
+ import os
11
+ import sys
12
+ from collections.abc import Mapping, Sequence
13
+ from pathlib import Path
14
+
15
+ import yaml
16
+
17
+ from aliasgate.config import load_settings
18
+ from aliasgate.engine import DetectionError
19
+ from aliasgate.eval.corpus import Example, load_corpus
20
+ from aliasgate.eval.metrics import Report, evaluate
21
+ from aliasgate.eval.report import format_json, format_table
22
+ from aliasgate.factory import build_pseudonymizer
23
+ from aliasgate.types import validate_entity_type
24
+
25
+
26
+ def _rate(value: str) -> float:
27
+ rate = float(value)
28
+ if math.isnan(rate) or not 0.0 <= rate <= 1.0:
29
+ raise argparse.ArgumentTypeError("must be a number between 0 and 1")
30
+ return rate
31
+
32
+
33
+ def _types(value: str) -> frozenset[str]:
34
+ names = [name.strip() for name in value.split(",")]
35
+ try:
36
+ return frozenset(validate_entity_type(name) for name in names)
37
+ except ValueError:
38
+ raise argparse.ArgumentTypeError(
39
+ "must be comma-separated entity types such as EMAIL,IBAN"
40
+ ) from None
41
+
42
+
43
+ def _count(value: str) -> int:
44
+ count = int(value)
45
+ if count < 0:
46
+ raise argparse.ArgumentTypeError("must not be negative")
47
+ return count
48
+
49
+
50
+ def _parser() -> argparse.ArgumentParser:
51
+ parser = argparse.ArgumentParser(
52
+ prog="python -m aliasgate.eval",
53
+ description="Measure detection on labelled JSON Lines corpora, per language.",
54
+ )
55
+ parser.add_argument("corpus", nargs="+", type=Path, help="JSON Lines corpus files")
56
+ parser.add_argument(
57
+ "--config",
58
+ type=Path,
59
+ default=None,
60
+ metavar="PATH",
61
+ help="gateway configuration; ALIASGATE_* environment variables override it",
62
+ )
63
+ parser.add_argument("--format", choices=("table", "json"), default="table")
64
+ parser.add_argument(
65
+ "--max-leak-rate",
66
+ type=_rate,
67
+ default=None,
68
+ metavar="RATE",
69
+ help="exit with code 1 if a language's leak rate is above RATE",
70
+ )
71
+ parser.add_argument(
72
+ "--types", type=_types, default=None, metavar="TYPES", help="e.g. EMAIL,IBAN"
73
+ )
74
+ parser.add_argument(
75
+ "--show-misses",
76
+ type=_count,
77
+ default=0,
78
+ metavar="N",
79
+ help="list up to N missed spans by example id and offsets",
80
+ )
81
+ return parser
82
+
83
+
84
+ def main(argv: Sequence[str] | None = None, *, env: Mapping[str, str] = os.environ) -> int:
85
+ """Evaluate the corpora, print the report and return the exit code."""
86
+ args = _parser().parse_args(argv)
87
+ try:
88
+ settings = load_settings(args.config, env)
89
+ by_lang = _by_language([example for path in args.corpus for example in load_corpus(path)])
90
+ pseudonymizer = build_pseudonymizer(settings)
91
+ except yaml.YAMLError:
92
+ # The parser's message quotes the offending line.
93
+ return _fail("the configuration file is not valid YAML")
94
+ except (ValueError, RuntimeError, OSError) as exc:
95
+ # Configuration, corpus and dictionary errors never repeat their input.
96
+ return _fail(f"invalid configuration or corpus: {exc}")
97
+ try:
98
+ reports = {
99
+ lang: evaluate(pseudonymizer, examples, types=args.types)
100
+ for lang, examples in sorted(by_lang.items())
101
+ }
102
+ except DetectionError as exc:
103
+ return _fail(str(exc))
104
+ except Exception as exc: # anything else may carry text in its message
105
+ return _fail(f"evaluation failed ({type(exc).__name__})")
106
+ if args.format == "json":
107
+ print(format_json(reports, misses=args.show_misses))
108
+ else:
109
+ print(
110
+ "\n\n".join(
111
+ format_table(lang, report, misses=args.show_misses)
112
+ for lang, report in reports.items()
113
+ )
114
+ )
115
+ return 1 if _above(reports, args.max_leak_rate) else 0
116
+
117
+
118
+ def _by_language(examples: list[Example]) -> dict[str, list[Example]]:
119
+ by_lang: dict[str, list[Example]] = {}
120
+ seen: set[str] = set()
121
+ for example in examples:
122
+ if example.id in seen:
123
+ raise ValueError(f"example id {example.id!r} appears in more than one place")
124
+ seen.add(example.id)
125
+ by_lang.setdefault(example.lang, []).append(example)
126
+ return by_lang
127
+
128
+
129
+ def _above(reports: Mapping[str, Report], limit: float | None) -> bool:
130
+ return limit is not None and any(report.leak_rate > limit for report in reports.values())
131
+
132
+
133
+ def _fail(message: str) -> int:
134
+ print(f"error: {message}", file=sys.stderr)
135
+ return 2
136
+
137
+
138
+ if __name__ == "__main__":
139
+ sys.exit(main())
@@ -0,0 +1,105 @@
1
+ """Labelled corpora in JSON Lines: one example per line, gold spans as Python string offsets.
2
+
3
+ {"id": "en-0001", "lang": "en", "text": "...",
4
+ "spans": [{"start": 10, "end": 20, "type": "PERSON"}]}
5
+
6
+ Each example is one line of the file. Offsets count code points, as Python slices do. Errors
7
+ name the file, the line and the example id, never the text.
8
+ """
9
+
10
+ import json
11
+ from dataclasses import dataclass
12
+ from pathlib import Path
13
+ from typing import Final
14
+
15
+ from aliasgate.types import validate_entity_type
16
+
17
+ _EXAMPLE_KEYS: Final = frozenset({"id", "lang", "text", "spans"})
18
+ _SPAN_KEYS: Final = frozenset({"start", "end", "type"})
19
+
20
+
21
+ @dataclass(frozen=True, slots=True)
22
+ class GoldSpan:
23
+ """A labelled range of one example: Python string offsets and an entity type."""
24
+
25
+ start: int
26
+ end: int
27
+ entity_type: str
28
+
29
+
30
+ @dataclass(frozen=True, slots=True)
31
+ class Example:
32
+ """One labelled text; its spans are in range, sorted, non-overlapping and non-empty."""
33
+
34
+ id: str
35
+ lang: str
36
+ text: str
37
+ spans: tuple[GoldSpan, ...]
38
+
39
+
40
+ class CorpusError(ValueError):
41
+ """A corpus file is malformed. The message gives a position, never text."""
42
+
43
+
44
+ def load_corpus(path: Path) -> list[Example]:
45
+ """Validate offsets (in range, non-overlapping, sorted) and entity types; raise on error.
46
+
47
+ Blank lines are skipped. Example ids must be unique within the file.
48
+ """
49
+ examples: list[Example] = []
50
+ seen: set[str] = set()
51
+ with path.open(encoding="utf-8") as stream:
52
+ for number, line in enumerate(stream, start=1):
53
+ if not line.strip():
54
+ continue
55
+ example = _example(f"{path}, line {number}", line)
56
+ if example.id in seen:
57
+ raise CorpusError(f"{path}, line {number}, example {example.id!r}: duplicate id")
58
+ seen.add(example.id)
59
+ examples.append(example)
60
+ return examples
61
+
62
+
63
+ def _example(where: str, line: str) -> Example:
64
+ try:
65
+ data = json.loads(line)
66
+ except json.JSONDecodeError:
67
+ data = None
68
+ if not isinstance(data, dict):
69
+ raise CorpusError(f"{where}: not a JSON object")
70
+ if set(data) != _EXAMPLE_KEYS:
71
+ raise CorpusError(f"{where}: an example has exactly the keys id, lang, text and spans")
72
+ example_id, lang, text, spans = data["id"], data["lang"], data["text"], data["spans"]
73
+ if not isinstance(example_id, str) or not example_id:
74
+ raise CorpusError(f"{where}: 'id' must be a non-empty string")
75
+ where = f"{where}, example {example_id!r}"
76
+ if not isinstance(lang, str) or not lang:
77
+ raise CorpusError(f"{where}: 'lang' must be a non-empty string")
78
+ if not isinstance(text, str):
79
+ raise CorpusError(f"{where}: 'text' must be a string")
80
+ if not isinstance(spans, list):
81
+ raise CorpusError(f"{where}: 'spans' must be a list")
82
+ gold: list[GoldSpan] = []
83
+ for index, span in enumerate(spans, start=1):
84
+ gold.append(_span(f"{where}: span {index}", span, len(text)))
85
+ if index > 1 and gold[-1].start < gold[-2].end:
86
+ raise CorpusError(f"{where}: span {index} overlaps or precedes span {index - 1}")
87
+ return Example(example_id, lang, text, tuple(gold))
88
+
89
+
90
+ def _span(where: str, span: object, length: int) -> GoldSpan:
91
+ if not isinstance(span, dict) or set(span) != _SPAN_KEYS:
92
+ raise CorpusError(f"{where} must be an object with start, end and type")
93
+ start, end, entity_type = span["start"], span["end"], span["type"]
94
+ # bool is an int subclass; true and false are not offsets.
95
+ if any(not isinstance(value, int) or isinstance(value, bool) for value in (start, end)):
96
+ raise CorpusError(f"{where} offsets must be integers")
97
+ if not 0 <= start < end <= length:
98
+ raise CorpusError(f"{where} is out of range (text length {length})")
99
+ try:
100
+ if not isinstance(entity_type, str):
101
+ raise ValueError
102
+ validate_entity_type(entity_type)
103
+ except ValueError:
104
+ raise CorpusError(f"{where} has an invalid entity type") from None
105
+ return GoldSpan(start, end, entity_type)
@@ -0,0 +1,197 @@
1
+ """Detection quality of the whole engine on labelled examples: counts and ratios, never text.
2
+
3
+ The headline number is the leak rate: the share of gold characters that no predicted span
4
+ covers, whatever its type. Recall by overlap ignores the type on purpose, because a person
5
+ detected as an organisation is still protected.
6
+ """
7
+
8
+ from collections import Counter
9
+ from collections.abc import Collection, Sequence
10
+ from dataclasses import dataclass
11
+
12
+ from aliasgate.engine import Pseudonymizer, restore
13
+ from aliasgate.eval.corpus import Example, GoldSpan
14
+ from aliasgate.mapping import Entity, Mapping, normalize_key
15
+ from aliasgate.terms import build_term_pattern
16
+ from aliasgate.types import LITERAL, Span
17
+
18
+ _PROPAGATION_SOURCE = "propagation"
19
+
20
+
21
+ def _ratio(numerator: int, denominator: int, *, empty: float) -> float:
22
+ return numerator / denominator if denominator else empty
23
+
24
+
25
+ @dataclass(frozen=True, slots=True)
26
+ class TypeScore:
27
+ """Counts for one entity type. Recall is 1.0 when there is no gold span of the type, and
28
+ precision is 1.0 when nothing of the type is predicted."""
29
+
30
+ entity_type: str
31
+ gold: int
32
+ predicted: int
33
+ exact_tp: int # gold spans predicted with the same start, end and type
34
+ overlap_tp: int # gold spans that any predicted span overlaps, of any type
35
+
36
+ @property
37
+ def recall_exact(self) -> float:
38
+ """exact_tp / gold, or 1.0 without gold spans."""
39
+ return _ratio(self.exact_tp, self.gold, empty=1.0)
40
+
41
+ @property
42
+ def recall_overlap(self) -> float:
43
+ """overlap_tp / gold, or 1.0 without gold spans."""
44
+ return _ratio(self.overlap_tp, self.gold, empty=1.0)
45
+
46
+ @property
47
+ def precision_exact(self) -> float:
48
+ """exact_tp / predicted, or 1.0 when nothing is predicted."""
49
+ return _ratio(self.exact_tp, self.predicted, empty=1.0)
50
+
51
+
52
+ @dataclass(frozen=True, slots=True)
53
+ class Miss:
54
+ """A gold span with characters no predicted span covers: its position, never its text."""
55
+
56
+ example_id: str
57
+ start: int
58
+ end: int
59
+ entity_type: str
60
+ leaked_chars: int
61
+
62
+
63
+ @dataclass(frozen=True, slots=True)
64
+ class Report:
65
+ """Scores of one evaluation, by entity type, plus leak, round-trip and false-alarm counts.
66
+
67
+ `entity_free` counts the examples without gold spans and `entity_free_flagged` those of
68
+ them with at least one predicted span. With a type restriction, both count only spans of
69
+ the selected types.
70
+ """
71
+
72
+ by_type: tuple[TypeScore, ...]
73
+ leaked_chars: int # characters of gold spans still present after pseudonymisation
74
+ gold_chars: int
75
+ round_trip_failures: int
76
+ examples: int
77
+ entity_free: int
78
+ entity_free_flagged: int
79
+ misses: tuple[Miss, ...]
80
+
81
+ @property
82
+ def leak_rate(self) -> float:
83
+ """leaked_chars / gold_chars, or 0.0 without gold spans."""
84
+ return _ratio(self.leaked_chars, self.gold_chars, empty=0.0)
85
+
86
+ @property
87
+ def entity_free_flag_rate(self) -> float:
88
+ """entity_free_flagged / entity_free, or 0.0 without such examples."""
89
+ return _ratio(self.entity_free_flagged, self.entity_free, empty=0.0)
90
+
91
+
92
+ def evaluate(
93
+ pseudonymizer: Pseudonymizer,
94
+ examples: Sequence[Example],
95
+ *,
96
+ types: Collection[str] | None = None,
97
+ ) -> Report:
98
+ """Run the whole engine on each example as its own batch and score its final spans.
99
+
100
+ With `types`, only gold and predicted spans of those types are scored, but a predicted
101
+ span of any type still covers gold characters.
102
+ """
103
+ gold_counts: Counter[str] = Counter()
104
+ predicted_counts: Counter[str] = Counter()
105
+ exact: Counter[str] = Counter()
106
+ overlap: Counter[str] = Counter()
107
+ leaked = gold_chars = failures = entity_free = flagged = 0
108
+ misses: list[Miss] = []
109
+ for example in examples:
110
+ spans = pseudonymizer.analyze([example.text])[0]
111
+ batch = pseudonymizer.pseudonymize([example.text])
112
+ expected = expected_restoration(example.text, spans, batch.mapping)
113
+ if restore(batch.texts[0], batch.mapping) != expected:
114
+ failures += 1
115
+ gold = [span for span in example.spans if types is None or span.entity_type in types]
116
+ predicted = [span for span in spans if types is None or span.entity_type in types]
117
+ predicted_counts.update(span.entity_type for span in predicted)
118
+ if not gold:
119
+ entity_free += 1
120
+ flagged += bool(predicted)
121
+ for span in gold:
122
+ covered = _covered(span, spans)
123
+ gold_counts[span.entity_type] += 1
124
+ gold_chars += span.end - span.start
125
+ leaked += span.end - span.start - covered
126
+ overlap[span.entity_type] += covered > 0
127
+ exact[span.entity_type] += any(
128
+ (found.start, found.end, found.entity_type)
129
+ == (span.start, span.end, span.entity_type)
130
+ for found in spans
131
+ )
132
+ if covered < span.end - span.start:
133
+ misses.append(
134
+ Miss(
135
+ example.id,
136
+ span.start,
137
+ span.end,
138
+ span.entity_type,
139
+ span.end - span.start - covered,
140
+ )
141
+ )
142
+ by_type = tuple(
143
+ TypeScore(name, gold_counts[name], predicted_counts[name], exact[name], overlap[name])
144
+ for name in sorted(gold_counts.keys() | predicted_counts.keys())
145
+ )
146
+ return Report(
147
+ by_type, leaked, gold_chars, failures, len(examples), entity_free, flagged, tuple(misses)
148
+ )
149
+
150
+
151
+ def _covered(gold: GoldSpan, spans: Sequence[Span]) -> int:
152
+ """Characters of `gold` inside predicted spans; these never overlap one another."""
153
+ return sum(max(0, min(gold.end, span.end) - max(gold.start, span.start)) for span in spans)
154
+
155
+
156
+ def expected_restoration(text: str, spans: Sequence[Span], mapping: Mapping) -> str | None:
157
+ """What `restore(pseudonymize(text))` must give: the text with every span `analyze`
158
+ reports rendered as its entity's canonical surface, a placeholder literal as itself.
159
+
160
+ A detected span's entity is the one with its key; a propagated span's is the longest
161
+ entity of its type whose term pattern matches it, as propagation chose it (keys and
162
+ patterns differ for a few characters, such as a dotted capital I). None if a span has no
163
+ entity in the mapping.
164
+ """
165
+ pieces: list[str] = []
166
+ position = 0
167
+ for span in spans:
168
+ surface = text[span.start : span.end]
169
+ if span.entity_type == LITERAL:
170
+ canonical: str | None = surface
171
+ else:
172
+ entity = _entity_of(span, surface, mapping)
173
+ canonical = entity.surface if entity is not None else None
174
+ if canonical is None:
175
+ return None
176
+ pieces.append(text[position : span.start])
177
+ pieces.append(canonical)
178
+ position = span.end
179
+ pieces.append(text[position:])
180
+ return "".join(pieces)
181
+
182
+
183
+ def _entity_of(span: Span, surface: str, mapping: Mapping) -> Entity | None:
184
+ if span.source != _PROPAGATION_SOURCE:
185
+ return mapping.get(normalize_key(surface))
186
+ candidates = sorted(
187
+ (entity for entity in mapping.entities() if entity.entity_type == span.entity_type),
188
+ key=lambda entity: -len(entity.surface),
189
+ )
190
+ return next(
191
+ (
192
+ entity
193
+ for entity in candidates
194
+ if build_term_pattern(entity.surface).fullmatch(surface) is not None
195
+ ),
196
+ None,
197
+ )