scribe-eval 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- scribe/__init__.py +148 -0
- scribe/align.py +224 -0
- scribe/analysis.py +322 -0
- scribe/charts.py +281 -0
- scribe/config/__init__.py +0 -0
- scribe/config/legal_terms.txt +369 -0
- scribe/config/medical_terms.txt +29 -0
- scribe/config/technical_terms.txt +34 -0
- scribe/constants.py +137 -0
- scribe/domain_config.py +277 -0
- scribe/measure.py +346 -0
- scribe/measure_batch.py +430 -0
- scribe/normalize.py +181 -0
- scribe/reporting.py +558 -0
- scribe/tokenize.py +146 -0
- scribe/visualizer/__init__.py +0 -0
- scribe/visualizer/__main__.py +27 -0
- scribe/visualizer/app.py +870 -0
- scribe_eval-0.1.1.dist-info/METADATA +285 -0
- scribe_eval-0.1.1.dist-info/RECORD +24 -0
- scribe_eval-0.1.1.dist-info/WHEEL +5 -0
- scribe_eval-0.1.1.dist-info/entry_points.txt +2 -0
- scribe_eval-0.1.1.dist-info/licenses/LICENSE +201 -0
- scribe_eval-0.1.1.dist-info/top_level.txt +1 -0
scribe/__init__.py
ADDED
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
"""scribe: diagnostic evaluation framework for ASR rich transcription.
|
|
2
|
+
|
|
3
|
+
Provides categorical error decomposition (lexical, numeral, punctuation,
|
|
4
|
+
and domain-specific rates) with sandhi-tolerant alignment. Language-agnostic
|
|
5
|
+
at the core; bundled domain configs target Indic and legal/medical domains.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
9
|
+
|
|
10
|
+
try:
|
|
11
|
+
__version__ = version("scribe-eval")
|
|
12
|
+
except PackageNotFoundError: # pragma: no cover - editable install edge case
|
|
13
|
+
__version__ = "0.0.0+unknown"
|
|
14
|
+
|
|
15
|
+
# --- Categories & Constants ---
|
|
16
|
+
# --- Alignment Logic ---
|
|
17
|
+
from .align import DEFAULT_WEIGHTS as DEFAULT_WEIGHTS
|
|
18
|
+
from .align import align_arrays as align_arrays
|
|
19
|
+
|
|
20
|
+
# --- Analysis & Insights ---
|
|
21
|
+
from .analysis import (
|
|
22
|
+
compute_category_contributions as compute_category_contributions,
|
|
23
|
+
)
|
|
24
|
+
from .analysis import (
|
|
25
|
+
compute_error_summary as compute_error_summary,
|
|
26
|
+
)
|
|
27
|
+
from .analysis import (
|
|
28
|
+
compute_error_type_distribution as compute_error_type_distribution,
|
|
29
|
+
)
|
|
30
|
+
from .analysis import (
|
|
31
|
+
compute_frequent_deletions as compute_frequent_deletions,
|
|
32
|
+
)
|
|
33
|
+
from .analysis import (
|
|
34
|
+
compute_frequent_insertions as compute_frequent_insertions,
|
|
35
|
+
)
|
|
36
|
+
from .analysis import (
|
|
37
|
+
compute_frequent_sandhi_merges as compute_frequent_sandhi_merges,
|
|
38
|
+
)
|
|
39
|
+
from .analysis import (
|
|
40
|
+
compute_frequent_sandhi_splits as compute_frequent_sandhi_splits,
|
|
41
|
+
)
|
|
42
|
+
from .analysis import (
|
|
43
|
+
compute_frequent_substitutions as compute_frequent_substitutions,
|
|
44
|
+
)
|
|
45
|
+
from .analysis import (
|
|
46
|
+
compute_wer_scribe as compute_wer_scribe,
|
|
47
|
+
)
|
|
48
|
+
from .constants import (
|
|
49
|
+
CAT_LEXICAL as CAT_LEXICAL,
|
|
50
|
+
)
|
|
51
|
+
from .constants import (
|
|
52
|
+
CAT_NUMERAL as CAT_NUMERAL,
|
|
53
|
+
)
|
|
54
|
+
from .constants import (
|
|
55
|
+
CAT_PUNCT as CAT_PUNCT,
|
|
56
|
+
)
|
|
57
|
+
from .constants import (
|
|
58
|
+
CATEGORIES as CATEGORIES,
|
|
59
|
+
)
|
|
60
|
+
from .constants import (
|
|
61
|
+
get_categories as get_categories,
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
# --- Domain Configuration ---
|
|
65
|
+
from .domain_config import DomainConfig as DomainConfig
|
|
66
|
+
|
|
67
|
+
# --- Measurement & Error Rates ---
|
|
68
|
+
from .measure import (
|
|
69
|
+
compute_cer_scribe as compute_cer_scribe,
|
|
70
|
+
)
|
|
71
|
+
from .measure import (
|
|
72
|
+
text_error_details as text_error_details,
|
|
73
|
+
)
|
|
74
|
+
from .measure import (
|
|
75
|
+
text_error_rates as text_error_rates,
|
|
76
|
+
)
|
|
77
|
+
from .measure import (
|
|
78
|
+
token_error_details as token_error_details,
|
|
79
|
+
)
|
|
80
|
+
from .measure import (
|
|
81
|
+
token_error_rates as token_error_rates,
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
# --- Batch Processing & Reporting ---
|
|
85
|
+
from .measure_batch import (
|
|
86
|
+
aggregate_error_details as aggregate_error_details,
|
|
87
|
+
)
|
|
88
|
+
from .measure_batch import (
|
|
89
|
+
compute_aggregate_metrics as compute_aggregate_metrics,
|
|
90
|
+
)
|
|
91
|
+
from .measure_batch import (
|
|
92
|
+
compute_sample_errors as compute_sample_errors,
|
|
93
|
+
)
|
|
94
|
+
from .measure_batch import (
|
|
95
|
+
evaluate_records as evaluate_records,
|
|
96
|
+
)
|
|
97
|
+
from .measure_batch import (
|
|
98
|
+
print_evaluation_summary as print_evaluation_summary,
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
# --- Normalization ---
|
|
102
|
+
from .normalize import (
|
|
103
|
+
normalize_currency as normalize_currency,
|
|
104
|
+
)
|
|
105
|
+
from .normalize import (
|
|
106
|
+
normalize_date as normalize_date,
|
|
107
|
+
)
|
|
108
|
+
from .normalize import (
|
|
109
|
+
normalize_numeral as normalize_numeral,
|
|
110
|
+
)
|
|
111
|
+
from .normalize import (
|
|
112
|
+
normalize_token as normalize_token,
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
# --- Report Formatting ---
|
|
116
|
+
from .reporting import (
|
|
117
|
+
extract_error_rates as extract_error_rates,
|
|
118
|
+
)
|
|
119
|
+
from .reporting import (
|
|
120
|
+
format_alignment_dict as format_alignment_dict,
|
|
121
|
+
)
|
|
122
|
+
from .reporting import (
|
|
123
|
+
format_alignment_table as format_alignment_table,
|
|
124
|
+
)
|
|
125
|
+
from .reporting import (
|
|
126
|
+
format_category_chips as format_category_chips,
|
|
127
|
+
)
|
|
128
|
+
from .reporting import (
|
|
129
|
+
format_contribution_table as format_contribution_table,
|
|
130
|
+
)
|
|
131
|
+
from .reporting import (
|
|
132
|
+
format_dataset_table as format_dataset_table,
|
|
133
|
+
)
|
|
134
|
+
from .reporting import (
|
|
135
|
+
format_error_counts_table as format_error_counts_table,
|
|
136
|
+
)
|
|
137
|
+
from .reporting import (
|
|
138
|
+
format_frequent_errors_table as format_frequent_errors_table,
|
|
139
|
+
)
|
|
140
|
+
from .reporting import (
|
|
141
|
+
format_metrics_dict as format_metrics_dict,
|
|
142
|
+
)
|
|
143
|
+
from .reporting import (
|
|
144
|
+
write_summary_to_file as write_summary_to_file,
|
|
145
|
+
)
|
|
146
|
+
|
|
147
|
+
# --- Tokenization ---
|
|
148
|
+
from .tokenize import domain_aware_tokenizer as domain_aware_tokenizer
|
scribe/align.py
ADDED
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
import Levenshtein as levenshtein
|
|
2
|
+
|
|
3
|
+
from .constants import CAT_NUMERAL, CAT_PUNCT
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def is_sandhi_eligible(tag) -> bool:
|
|
7
|
+
"""Check if a tag is eligible for Sandhi split/merge operations.
|
|
8
|
+
|
|
9
|
+
Sandhi operations apply to all categories except PUNCT and NUMERAL.
|
|
10
|
+
This includes LEXICAL and all domain-specific categories (LEGAL, MEDICAL, etc.).
|
|
11
|
+
"""
|
|
12
|
+
return tag not in (CAT_PUNCT, CAT_NUMERAL)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
# --- SCORING CONFIGURATION ---
|
|
16
|
+
DEFAULT_WEIGHTS = {
|
|
17
|
+
"gap_penalty": -2.5, # Gap penalty for words, legal terms, numerals
|
|
18
|
+
"gap_penalty_punct": -1.2, # Gap penalty for punctuation (lighter penalty)
|
|
19
|
+
"match_reward": 3.0,
|
|
20
|
+
"mismatch_default_penalty": -1.5,
|
|
21
|
+
"mismatch_cross_punct_penalty": -3.0, # High penalty for cross-category align (e.g. Law/Punct)
|
|
22
|
+
"split_merge_penalty": -0.5, # Small penalty for Sandhi logic
|
|
23
|
+
"sandhi_char_tolerance": 2, # Max character diff for Sandhi
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def levenshtein_distance(s1, s2) -> int:
|
|
28
|
+
return levenshtein.distance(s1, s2)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def get_gap_penalty(tag, weights=DEFAULT_WEIGHTS) -> float:
|
|
32
|
+
"""Returns the appropriate gap penalty based on token category."""
|
|
33
|
+
if tag == CAT_PUNCT:
|
|
34
|
+
return weights.get("gap_penalty_punct", weights["gap_penalty"])
|
|
35
|
+
return weights["gap_penalty"]
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def get_match_score(w1, w2, weights=DEFAULT_WEIGHTS) -> float:
|
|
39
|
+
# Normalized reward: Match base + character similarity
|
|
40
|
+
dist = levenshtein_distance(w1, w2)
|
|
41
|
+
length = max(len(w1), len(w2), 1)
|
|
42
|
+
return weights["match_reward"] + (1.0 - dist * 2 / length)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def get_mismatch_penalty(w1, t1, w2, t2, weights=DEFAULT_WEIGHTS) -> float:
|
|
46
|
+
# If categories are different, apply a heavy penalty
|
|
47
|
+
if t1 != t2:
|
|
48
|
+
if t1 == CAT_PUNCT or t2 == CAT_PUNCT:
|
|
49
|
+
return weights["mismatch_cross_punct_penalty"]
|
|
50
|
+
return weights["mismatch_cross_punct_penalty"]
|
|
51
|
+
|
|
52
|
+
# For mismatch between other categories, penalty based on string distance
|
|
53
|
+
dist = levenshtein_distance(w1, w2)
|
|
54
|
+
return weights["mismatch_default_penalty"] - (dist * 0.2)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def check_sandhi_match(combined_words, single_text, weights) -> float:
|
|
58
|
+
"""Checks if two words (split) equal one word (merge) with Sandhi rules."""
|
|
59
|
+
if len(combined_words) != 2:
|
|
60
|
+
return -float("inf")
|
|
61
|
+
|
|
62
|
+
w1, w2 = combined_words[0], combined_words[1]
|
|
63
|
+
if len(w1) < 2 or len(w2) < 2:
|
|
64
|
+
return -float("inf")
|
|
65
|
+
|
|
66
|
+
# Boundary analysis
|
|
67
|
+
s1, s2 = w1[:-1], w1[-1:]
|
|
68
|
+
s3, s4 = w2[:1], w2[1:]
|
|
69
|
+
|
|
70
|
+
if not single_text.startswith(s1) or not single_text.endswith(s4):
|
|
71
|
+
return -float("inf")
|
|
72
|
+
|
|
73
|
+
# If single_text is identical to one of the pair, the other word
|
|
74
|
+
# contributed nothing — it was dropped/inserted, not merged/split.
|
|
75
|
+
# Without this guard the boundary region collapses to empty and the
|
|
76
|
+
# distance (== len(split_boundary) == 2) lands within tolerance.
|
|
77
|
+
if single_text == w1 or single_text == w2:
|
|
78
|
+
return -float("inf")
|
|
79
|
+
|
|
80
|
+
# The prefix and suffix must not claim the same characters: overlap
|
|
81
|
+
# makes the boundary slice silently return "" instead of failing.
|
|
82
|
+
if len(s1) + len(s4) > len(single_text):
|
|
83
|
+
return -float("inf")
|
|
84
|
+
|
|
85
|
+
boundary_region = single_text[len(s1) : len(single_text) - len(s4)]
|
|
86
|
+
split_boundary = s2 + s3
|
|
87
|
+
|
|
88
|
+
dist = levenshtein_distance(split_boundary, boundary_region)
|
|
89
|
+
if dist <= weights.get("sandhi_char_tolerance", 2):
|
|
90
|
+
score = weights["match_reward"] + weights["split_merge_penalty"]
|
|
91
|
+
return score - (dist / len(single_text))
|
|
92
|
+
|
|
93
|
+
return -float("inf")
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def align_arrays(
|
|
97
|
+
arr1, tags1, arr2, tags2, weights=None, use_sandhi: bool = True
|
|
98
|
+
) -> tuple[list[tuple[str, str]], list[tuple[str, str]], float]:
|
|
99
|
+
if weights is None:
|
|
100
|
+
weights = DEFAULT_WEIGHTS
|
|
101
|
+
|
|
102
|
+
m, n = len(arr1), len(arr2)
|
|
103
|
+
|
|
104
|
+
# Per-position facts are fixed for the whole alignment: a token's
|
|
105
|
+
# gap penalty and sandhi eligibility depend only on its tag, and
|
|
106
|
+
# two equal tokens always score match_reward + 1.0 (their character
|
|
107
|
+
# distance is zero). Precompute them once here instead of
|
|
108
|
+
# re-deriving them inside the O(m*n) fill loop.
|
|
109
|
+
gap1 = [get_gap_penalty(tag, weights) for tag in tags1]
|
|
110
|
+
gap2 = [get_gap_penalty(tag, weights) for tag in tags2]
|
|
111
|
+
sandhi_ok1 = [is_sandhi_eligible(tag) for tag in tags1]
|
|
112
|
+
sandhi_ok2 = [is_sandhi_eligible(tag) for tag in tags2]
|
|
113
|
+
exact_match_score = weights["match_reward"] + 1.0
|
|
114
|
+
neg_inf = -float("inf")
|
|
115
|
+
|
|
116
|
+
dp = [[neg_inf for _ in range(n + 1)] for _ in range(m + 1)]
|
|
117
|
+
dp[0][0] = 0
|
|
118
|
+
|
|
119
|
+
# Initialize gaps
|
|
120
|
+
for i in range(1, m + 1):
|
|
121
|
+
dp[i][0] = dp[i - 1][0] + gap1[i - 1]
|
|
122
|
+
for j in range(1, n + 1):
|
|
123
|
+
dp[0][j] = dp[0][j - 1] + gap2[j - 1]
|
|
124
|
+
|
|
125
|
+
# Fill DP
|
|
126
|
+
for i in range(1, m + 1):
|
|
127
|
+
w1 = arr1[i - 1]
|
|
128
|
+
t1 = tags1[i - 1]
|
|
129
|
+
row = dp[i]
|
|
130
|
+
prev_row = dp[i - 1]
|
|
131
|
+
for j in range(1, n + 1):
|
|
132
|
+
w2 = arr2[j - 1]
|
|
133
|
+
|
|
134
|
+
# 1. Standard Match/Mismatch (Category Aware)
|
|
135
|
+
if w1 == w2:
|
|
136
|
+
score = exact_match_score
|
|
137
|
+
else:
|
|
138
|
+
score = get_mismatch_penalty(w1, t1, w2, tags2[j - 1], weights)
|
|
139
|
+
match_val = prev_row[j - 1] + score
|
|
140
|
+
|
|
141
|
+
# 2. Standard Indel (Category-aware gap penalties)
|
|
142
|
+
del_val = prev_row[j] + gap1[i - 1]
|
|
143
|
+
ins_val = row[j - 1] + gap2[j - 1]
|
|
144
|
+
|
|
145
|
+
# 3. Sandhi Split/Merge (for all categories except PUNCT and NUMERAL)
|
|
146
|
+
split_val = merge_val = neg_inf
|
|
147
|
+
|
|
148
|
+
if use_sandhi:
|
|
149
|
+
# Split: 1 Ref matches 2 Hyp. Only probe pairs that can
|
|
150
|
+
# pass check_sandhi_match's own guards: both hyp words
|
|
151
|
+
# at least 2 chars, and the single word sharing the
|
|
152
|
+
# first word's first char (startswith w[: -1]) and the
|
|
153
|
+
# second word's last char (endswith w[1:]).
|
|
154
|
+
if j >= 2 and sandhi_ok1[i - 1] and sandhi_ok2[j - 2] and sandhi_ok2[j - 1]:
|
|
155
|
+
a, b = arr2[j - 2], arr2[j - 1]
|
|
156
|
+
if len(a) >= 2 and len(b) >= 2 and w1 and w1[0] == a[0] and w1[-1] == b[-1]:
|
|
157
|
+
split_val = dp[i - 1][j - 2] + check_sandhi_match([a, b], w1, weights)
|
|
158
|
+
|
|
159
|
+
# Merge: 2 Ref match 1 Hyp
|
|
160
|
+
if i >= 2 and sandhi_ok1[i - 2] and sandhi_ok1[i - 1] and sandhi_ok2[j - 1]:
|
|
161
|
+
a, b = arr1[i - 2], arr1[i - 1]
|
|
162
|
+
if len(a) >= 2 and len(b) >= 2 and w2 and w2[0] == a[0] and w2[-1] == b[-1]:
|
|
163
|
+
merge_val = dp[i - 2][j - 1] + check_sandhi_match([a, b], w2, weights)
|
|
164
|
+
|
|
165
|
+
row[j] = max(match_val, del_val, ins_val, split_val, merge_val)
|
|
166
|
+
|
|
167
|
+
# Traceback
|
|
168
|
+
aligned_ref = []
|
|
169
|
+
aligned_hyp = []
|
|
170
|
+
i, j = m, n
|
|
171
|
+
|
|
172
|
+
while i > 0 or j > 0:
|
|
173
|
+
curr = dp[i][j]
|
|
174
|
+
|
|
175
|
+
def is_close(v):
|
|
176
|
+
return abs(curr - v) < 1e-7
|
|
177
|
+
|
|
178
|
+
# Trace Sandhi Split
|
|
179
|
+
if use_sandhi and j >= 2 and i > 0 and is_sandhi_eligible(tags1[i - 1]):
|
|
180
|
+
score = check_sandhi_match([arr2[j - 2], arr2[j - 1]], arr1[i - 1], weights)
|
|
181
|
+
if is_close(dp[i - 1][j - 2] + score):
|
|
182
|
+
aligned_ref.append((arr1[i - 1], tags1[i - 1]))
|
|
183
|
+
aligned_hyp.append((f"SPLIT:{arr2[j - 2]} {arr2[j - 1]}", tags1[i - 1]))
|
|
184
|
+
i -= 1
|
|
185
|
+
j -= 2
|
|
186
|
+
continue
|
|
187
|
+
|
|
188
|
+
# Trace Sandhi Merge
|
|
189
|
+
if use_sandhi and i >= 2 and j > 0 and is_sandhi_eligible(tags2[j - 1]):
|
|
190
|
+
score = check_sandhi_match([arr1[i - 2], arr1[i - 1]], arr2[j - 1], weights)
|
|
191
|
+
if is_close(dp[i - 2][j - 1] + score):
|
|
192
|
+
aligned_ref.append((f"MERGE:{arr1[i - 2]} {arr1[i - 1]}", tags2[j - 1]))
|
|
193
|
+
aligned_hyp.append((arr2[j - 1], tags2[j - 1]))
|
|
194
|
+
i -= 2
|
|
195
|
+
j -= 1
|
|
196
|
+
continue
|
|
197
|
+
|
|
198
|
+
# Trace Standard Match/Mismatch
|
|
199
|
+
if i > 0 and j > 0:
|
|
200
|
+
if arr1[i - 1] == arr2[j - 1]:
|
|
201
|
+
score = get_match_score(arr1[i - 1], arr2[j - 1], weights)
|
|
202
|
+
else:
|
|
203
|
+
score = get_mismatch_penalty(
|
|
204
|
+
arr1[i - 1], tags1[i - 1], arr2[j - 1], tags2[j - 1], weights
|
|
205
|
+
)
|
|
206
|
+
|
|
207
|
+
if is_close(dp[i - 1][j - 1] + score):
|
|
208
|
+
aligned_ref.append((arr1[i - 1], tags1[i - 1]))
|
|
209
|
+
aligned_hyp.append((arr2[j - 1], tags2[j - 1]))
|
|
210
|
+
i -= 1
|
|
211
|
+
j -= 1
|
|
212
|
+
continue
|
|
213
|
+
|
|
214
|
+
# Trace Gaps (Category-aware)
|
|
215
|
+
if i > 0 and is_close(dp[i - 1][j] + get_gap_penalty(tags1[i - 1], weights)):
|
|
216
|
+
aligned_ref.append((arr1[i - 1], tags1[i - 1]))
|
|
217
|
+
aligned_hyp.append(("**", "GAP"))
|
|
218
|
+
i -= 1
|
|
219
|
+
else:
|
|
220
|
+
aligned_ref.append(("**", "GAP"))
|
|
221
|
+
aligned_hyp.append((arr2[j - 1], tags2[j - 1]))
|
|
222
|
+
j -= 1
|
|
223
|
+
|
|
224
|
+
return aligned_ref[::-1], aligned_hyp[::-1], dp[m][n]
|
scribe/analysis.py
ADDED
|
@@ -0,0 +1,322 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Analytical computations for ASR error insights.
|
|
3
|
+
|
|
4
|
+
This module transforms measured error data into actionable insights:
|
|
5
|
+
the composite WER_SCRIBE, category contributions, error type
|
|
6
|
+
distributions, and frequency analysis of specific token errors.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from collections import Counter
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def compute_wer_scribe(metrics: dict[str, dict]) -> float:
|
|
13
|
+
"""
|
|
14
|
+
Compute the composite WER_SCRIBE from category metrics.
|
|
15
|
+
|
|
16
|
+
WER_SCRIBE is the sum of all category error rates. Since each
|
|
17
|
+
category's error_rate uses combined_total as denominator, the sum
|
|
18
|
+
naturally equals total_errors / combined_total.
|
|
19
|
+
|
|
20
|
+
Args:
|
|
21
|
+
metrics: Output of token_error_rates() or aggregate metrics —
|
|
22
|
+
maps category name -> dict with 'error_rate' key
|
|
23
|
+
|
|
24
|
+
Returns:
|
|
25
|
+
WER_SCRIBE as a float (0.0 to 1.0+; can exceed 1.0 when
|
|
26
|
+
insertions outnumber matched reference tokens).
|
|
27
|
+
"""
|
|
28
|
+
return sum(cat_data["error_rate"] for cat_data in metrics.values())
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def compute_category_contributions(
|
|
32
|
+
metrics: dict[str, dict],
|
|
33
|
+
) -> dict[str, dict]:
|
|
34
|
+
"""
|
|
35
|
+
Compute full breakdown per category: correct/sub/del/ins counts and rates.
|
|
36
|
+
|
|
37
|
+
Args:
|
|
38
|
+
metrics: Output of token_error_rates() or aggregate metrics
|
|
39
|
+
|
|
40
|
+
Returns:
|
|
41
|
+
Dict mapping category -> {
|
|
42
|
+
"correct": int,
|
|
43
|
+
"substitutions": int,
|
|
44
|
+
"deletions": int,
|
|
45
|
+
"insertions": int,
|
|
46
|
+
"ref_tokens": int, # correct + sub + del (reference side)
|
|
47
|
+
"correct_pct": float, # correct / ref_tokens * 100
|
|
48
|
+
"error_count": int, # sub + ins + del
|
|
49
|
+
"error_rate": float, # from metrics (uses combined denominator)
|
|
50
|
+
"contribution_pct": float, # this category's share of total errors
|
|
51
|
+
}
|
|
52
|
+
"""
|
|
53
|
+
contributions = {}
|
|
54
|
+
total_errors = 0
|
|
55
|
+
|
|
56
|
+
for cat, data in metrics.items():
|
|
57
|
+
correct = data["correct"]
|
|
58
|
+
subs = data["substitutions"]
|
|
59
|
+
ins = data["insertions"]
|
|
60
|
+
dels = data["deletions"]
|
|
61
|
+
ref_tokens = data.get("total_ref", data.get("total", 0))
|
|
62
|
+
error_count = subs + ins + dels
|
|
63
|
+
total_errors += error_count
|
|
64
|
+
|
|
65
|
+
contributions[cat] = {
|
|
66
|
+
"correct": correct,
|
|
67
|
+
"substitutions": subs,
|
|
68
|
+
"deletions": dels,
|
|
69
|
+
"insertions": ins,
|
|
70
|
+
"ref_tokens": ref_tokens,
|
|
71
|
+
"correct_pct": (correct / ref_tokens * 100) if ref_tokens > 0 else 0.0,
|
|
72
|
+
"error_count": error_count,
|
|
73
|
+
"error_rate": data["error_rate"],
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
for cat in contributions:
|
|
77
|
+
if total_errors > 0:
|
|
78
|
+
contributions[cat]["contribution_pct"] = (
|
|
79
|
+
contributions[cat]["error_count"] / total_errors * 100
|
|
80
|
+
)
|
|
81
|
+
else:
|
|
82
|
+
contributions[cat]["contribution_pct"] = 0.0
|
|
83
|
+
|
|
84
|
+
return contributions
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def compute_error_type_distribution(
|
|
88
|
+
metrics: dict[str, dict],
|
|
89
|
+
) -> dict[str, dict[str, float]]:
|
|
90
|
+
"""
|
|
91
|
+
For each category, compute the fraction of errors that are subs/ins/del.
|
|
92
|
+
|
|
93
|
+
Args:
|
|
94
|
+
metrics: Output of token_error_rates() or aggregate metrics
|
|
95
|
+
|
|
96
|
+
Returns:
|
|
97
|
+
Dict mapping category -> {
|
|
98
|
+
"substitution_pct": float (0-100),
|
|
99
|
+
"insertion_pct": float (0-100),
|
|
100
|
+
"deletion_pct": float (0-100),
|
|
101
|
+
}
|
|
102
|
+
"""
|
|
103
|
+
result = {}
|
|
104
|
+
for cat, data in metrics.items():
|
|
105
|
+
total = data["substitutions"] + data["insertions"] + data["deletions"]
|
|
106
|
+
if total > 0:
|
|
107
|
+
result[cat] = {
|
|
108
|
+
"substitution_pct": data["substitutions"] / total * 100,
|
|
109
|
+
"insertion_pct": data["insertions"] / total * 100,
|
|
110
|
+
"deletion_pct": data["deletions"] / total * 100,
|
|
111
|
+
}
|
|
112
|
+
else:
|
|
113
|
+
result[cat] = {
|
|
114
|
+
"substitution_pct": 0.0,
|
|
115
|
+
"insertion_pct": 0.0,
|
|
116
|
+
"deletion_pct": 0.0,
|
|
117
|
+
}
|
|
118
|
+
return result
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def compute_frequent_substitutions(
|
|
122
|
+
error_details: list[dict],
|
|
123
|
+
top_n: int = 20,
|
|
124
|
+
) -> dict[str, list[tuple[str, str, int]]]:
|
|
125
|
+
"""
|
|
126
|
+
Find the most frequently substituted token pairs.
|
|
127
|
+
|
|
128
|
+
Args:
|
|
129
|
+
error_details: List of error record dicts from token_error_details()
|
|
130
|
+
top_n: Number of top substitutions to return per category
|
|
131
|
+
|
|
132
|
+
Returns:
|
|
133
|
+
Dict mapping category -> list of (ref_token, hyp_token, count)
|
|
134
|
+
sorted by count descending. Key "_all" has the overall list.
|
|
135
|
+
"""
|
|
136
|
+
subs = [e for e in error_details if e["error_type"] == "substitution"]
|
|
137
|
+
|
|
138
|
+
# Overall
|
|
139
|
+
overall_counter = Counter((e["ref_token"], e["hyp_token"]) for e in subs)
|
|
140
|
+
result = {
|
|
141
|
+
"_all": [(ref, hyp, count) for (ref, hyp), count in overall_counter.most_common(top_n)]
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
# By category
|
|
145
|
+
by_cat: dict[str, list[dict]] = {}
|
|
146
|
+
for e in subs:
|
|
147
|
+
by_cat.setdefault(e["category"], []).append(e)
|
|
148
|
+
|
|
149
|
+
for cat, cat_errors in by_cat.items():
|
|
150
|
+
counter = Counter((e["ref_token"], e["hyp_token"]) for e in cat_errors)
|
|
151
|
+
result[cat] = [(ref, hyp, count) for (ref, hyp), count in counter.most_common(top_n)]
|
|
152
|
+
|
|
153
|
+
return result
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def compute_frequent_deletions(
|
|
157
|
+
error_details: list[dict],
|
|
158
|
+
top_n: int = 20,
|
|
159
|
+
) -> dict[str, list[tuple[str, int]]]:
|
|
160
|
+
"""
|
|
161
|
+
Find the most frequently deleted tokens.
|
|
162
|
+
|
|
163
|
+
Args:
|
|
164
|
+
error_details: from token_error_details()
|
|
165
|
+
top_n: number of top deletions to return
|
|
166
|
+
|
|
167
|
+
Returns:
|
|
168
|
+
Dict mapping category -> list of (ref_token, count) sorted descending.
|
|
169
|
+
Key "_all" for the overall list.
|
|
170
|
+
"""
|
|
171
|
+
dels = [e for e in error_details if e["error_type"] == "deletion"]
|
|
172
|
+
|
|
173
|
+
overall_counter = Counter(e["ref_token"] for e in dels)
|
|
174
|
+
result = {"_all": [(token, count) for token, count in overall_counter.most_common(top_n)]}
|
|
175
|
+
|
|
176
|
+
by_cat: dict[str, list[dict]] = {}
|
|
177
|
+
for e in dels:
|
|
178
|
+
by_cat.setdefault(e["category"], []).append(e)
|
|
179
|
+
|
|
180
|
+
for cat, cat_errors in by_cat.items():
|
|
181
|
+
counter = Counter(e["ref_token"] for e in cat_errors)
|
|
182
|
+
result[cat] = [(token, count) for token, count in counter.most_common(top_n)]
|
|
183
|
+
|
|
184
|
+
return result
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def compute_frequent_insertions(
|
|
188
|
+
error_details: list[dict],
|
|
189
|
+
top_n: int = 20,
|
|
190
|
+
) -> dict[str, list[tuple[str, int]]]:
|
|
191
|
+
"""
|
|
192
|
+
Find the most frequently inserted tokens.
|
|
193
|
+
|
|
194
|
+
Args:
|
|
195
|
+
error_details: from token_error_details()
|
|
196
|
+
top_n: number of top insertions to return
|
|
197
|
+
|
|
198
|
+
Returns:
|
|
199
|
+
Dict mapping category -> list of (hyp_token, count) sorted descending.
|
|
200
|
+
Key "_all" for the overall list.
|
|
201
|
+
"""
|
|
202
|
+
ins = [e for e in error_details if e["error_type"] == "insertion"]
|
|
203
|
+
|
|
204
|
+
overall_counter = Counter(e["hyp_token"] for e in ins)
|
|
205
|
+
result = {"_all": [(token, count) for token, count in overall_counter.most_common(top_n)]}
|
|
206
|
+
|
|
207
|
+
by_cat: dict[str, list[dict]] = {}
|
|
208
|
+
for e in ins:
|
|
209
|
+
by_cat.setdefault(e["category"], []).append(e)
|
|
210
|
+
|
|
211
|
+
for cat, cat_errors in by_cat.items():
|
|
212
|
+
counter = Counter(e["hyp_token"] for e in cat_errors)
|
|
213
|
+
result[cat] = [(token, count) for token, count in counter.most_common(top_n)]
|
|
214
|
+
|
|
215
|
+
return result
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def _compute_frequent_sandhi(
|
|
219
|
+
error_details: list[dict],
|
|
220
|
+
error_type: str,
|
|
221
|
+
top_n: int,
|
|
222
|
+
) -> dict[str, list[tuple[str, str, int]]]:
|
|
223
|
+
"""Shared helper for frequent sandhi merges/splits."""
|
|
224
|
+
records = [e for e in error_details if e["error_type"] == error_type]
|
|
225
|
+
|
|
226
|
+
overall_counter = Counter((e["ref_token"], e["hyp_token"]) for e in records)
|
|
227
|
+
result = {
|
|
228
|
+
"_all": [(ref, hyp, count) for (ref, hyp), count in overall_counter.most_common(top_n)]
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
by_cat: dict[str, list[dict]] = {}
|
|
232
|
+
for e in records:
|
|
233
|
+
by_cat.setdefault(e["category"], []).append(e)
|
|
234
|
+
|
|
235
|
+
for cat, cat_records in by_cat.items():
|
|
236
|
+
counter = Counter((e["ref_token"], e["hyp_token"]) for e in cat_records)
|
|
237
|
+
result[cat] = [(ref, hyp, count) for (ref, hyp), count in counter.most_common(top_n)]
|
|
238
|
+
|
|
239
|
+
return result
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def compute_frequent_sandhi_merges(
|
|
243
|
+
error_details: list[dict],
|
|
244
|
+
top_n: int = 20,
|
|
245
|
+
) -> dict[str, list[tuple[str, str, int]]]:
|
|
246
|
+
"""
|
|
247
|
+
Find the most frequent sandhi merges (two ref words → one hyp word).
|
|
248
|
+
|
|
249
|
+
Args:
|
|
250
|
+
error_details: from token_error_details()
|
|
251
|
+
top_n: number of top merges per category
|
|
252
|
+
|
|
253
|
+
Returns:
|
|
254
|
+
Dict mapping category -> list of (ref_token, hyp_token, count) where
|
|
255
|
+
ref_token is the two-word source ("word1 word2") and hyp_token is the
|
|
256
|
+
merged form. Key "_all" for the overall list.
|
|
257
|
+
"""
|
|
258
|
+
return _compute_frequent_sandhi(error_details, "sandhi_merge", top_n)
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
def compute_frequent_sandhi_splits(
|
|
262
|
+
error_details: list[dict],
|
|
263
|
+
top_n: int = 20,
|
|
264
|
+
) -> dict[str, list[tuple[str, str, int]]]:
|
|
265
|
+
"""
|
|
266
|
+
Find the most frequent sandhi splits (one ref word → two hyp words).
|
|
267
|
+
|
|
268
|
+
Args:
|
|
269
|
+
error_details: from token_error_details()
|
|
270
|
+
top_n: number of top splits per category
|
|
271
|
+
|
|
272
|
+
Returns:
|
|
273
|
+
Dict mapping category -> list of (ref_token, hyp_token, count) where
|
|
274
|
+
ref_token is the single source word and hyp_token is the split form
|
|
275
|
+
("word1 word2"). Key "_all" for the overall list.
|
|
276
|
+
"""
|
|
277
|
+
return _compute_frequent_sandhi(error_details, "sandhi_split", top_n)
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
def compute_error_summary(
|
|
281
|
+
metrics: dict[str, dict],
|
|
282
|
+
error_details: list[dict],
|
|
283
|
+
top_n: int = 10,
|
|
284
|
+
) -> dict:
|
|
285
|
+
"""
|
|
286
|
+
Compute all analysis in one call.
|
|
287
|
+
|
|
288
|
+
Args:
|
|
289
|
+
metrics: Output of token_error_rates() or aggregate metrics
|
|
290
|
+
error_details: Flat list of error records from token_error_details()
|
|
291
|
+
top_n: Number of top frequent errors to include
|
|
292
|
+
|
|
293
|
+
Returns:
|
|
294
|
+
{
|
|
295
|
+
"wer_scribe": float,
|
|
296
|
+
"contributions": dict from compute_category_contributions,
|
|
297
|
+
"error_type_distribution": dict from compute_error_type_distribution,
|
|
298
|
+
"frequent_substitutions": dict from compute_frequent_substitutions,
|
|
299
|
+
"frequent_deletions": dict from compute_frequent_deletions,
|
|
300
|
+
"frequent_insertions": dict from compute_frequent_insertions,
|
|
301
|
+
"frequent_sandhi_merges": dict from compute_frequent_sandhi_merges,
|
|
302
|
+
"frequent_sandhi_splits": dict from compute_frequent_sandhi_splits,
|
|
303
|
+
}
|
|
304
|
+
"""
|
|
305
|
+
contributions = compute_category_contributions(metrics)
|
|
306
|
+
|
|
307
|
+
# Overall correct rate across all categories
|
|
308
|
+
total_correct = sum(c["correct"] for c in contributions.values())
|
|
309
|
+
total_ref = sum(c["ref_tokens"] for c in contributions.values())
|
|
310
|
+
total_correct_pct = (total_correct / total_ref * 100) if total_ref > 0 else 0.0
|
|
311
|
+
|
|
312
|
+
return {
|
|
313
|
+
"wer_scribe": compute_wer_scribe(metrics),
|
|
314
|
+
"total_correct_pct": total_correct_pct,
|
|
315
|
+
"contributions": contributions,
|
|
316
|
+
"error_type_distribution": compute_error_type_distribution(metrics),
|
|
317
|
+
"frequent_substitutions": compute_frequent_substitutions(error_details, top_n),
|
|
318
|
+
"frequent_deletions": compute_frequent_deletions(error_details, top_n),
|
|
319
|
+
"frequent_insertions": compute_frequent_insertions(error_details, top_n),
|
|
320
|
+
"frequent_sandhi_merges": compute_frequent_sandhi_merges(error_details, top_n),
|
|
321
|
+
"frequent_sandhi_splits": compute_frequent_sandhi_splits(error_details, top_n),
|
|
322
|
+
}
|