sable-sphynx 0.7.0.dev0__py3-none-any.whl

sphynx/_rag_ranking.py

@@ -0,0 +1,149 @@
+"""Deterministic ranking and evidence-group metric formulas."""
+
+from __future__ import annotations
+
+import math
+from collections.abc import Mapping, Sequence
+
+
+def dcg(grades: Sequence[int], depth: int) -> float:
+    """Graded discounted cumulative gain using exponential relevance gain."""
+
+    return float(
+        sum(
+            (2**grade - 1) / math.log2(rank + 1)
+            for rank, grade in enumerate(grades[:depth], start=1)
+        )
+    )
+
+
+def ndcg(grades: Sequence[int], depth: int) -> float | None:
+    """Normalized DCG, or null when the pool has no relevant item."""
+
+    ideal = dcg(sorted(grades, reverse=True), depth)
+    return None if ideal == 0 else dcg(grades, depth) / ideal
+
+
+def reciprocal_rank(grades: Sequence[int], threshold: int) -> float:
+    for rank, grade in enumerate(grades, start=1):
+        if grade >= threshold:
+            return 1 / rank
+    return 0.0
+
+
+def hit_rate(grades: Sequence[int], depth: int, threshold: int) -> float:
+    return float(any(grade >= threshold for grade in grades[:depth]))
+
+
+def precision_at(grades: Sequence[int], depth: int, threshold: int) -> float:
+    return sum(grade >= threshold for grade in grades[:depth]) / depth
+
+
+def recall_at(grades: Sequence[int], depth: int, threshold: int) -> float | None:
+    relevant = sum(grade >= threshold for grade in grades)
+    return None if relevant == 0 else sum(grade >= threshold for grade in grades[:depth]) / relevant
+
+
+def average_precision(grades: Sequence[int], threshold: int) -> float | None:
+    relevant = sum(grade >= threshold for grade in grades)
+    if relevant == 0:
+        return None
+    found = 0
+    total = 0.0
+    for rank, grade in enumerate(grades, start=1):
+        if grade >= threshold:
+            found += 1
+            total += found / rank
+    return total / relevant
+
+
+def pairwise_accuracy(
+    ordered_candidate_ids: Sequence[str], qrels: Mapping[str, int]
+) -> float | None:
+    rank = {candidate_id: index for index, candidate_id in enumerate(ordered_candidate_ids)}
+    comparable = 0
+    correct = 0
+    candidate_ids = tuple(ordered_candidate_ids)
+    for left_index, left_id in enumerate(candidate_ids):
+        for right_id in candidate_ids[left_index + 1 :]:
+            if qrels[left_id] == qrels[right_id]:
+                continue
+            comparable += 1
+            preferred = left_id if qrels[left_id] > qrels[right_id] else right_id
+            other = right_id if preferred == left_id else left_id
+            correct += int(rank[preferred] < rank[other])
+    return None if comparable == 0 else correct / comparable
+
+
+def weighted_evidence_recall(
+    ordered_candidate_ids: Sequence[str],
+    candidate_groups: Mapping[str, frozenset[str]],
+    group_weights: Mapping[str, float],
+    depth: int,
+) -> float | None:
+    total = sum(group_weights.values())
+    if total == 0:
+        return None
+    covered = set().union(
+        *(
+            candidate_groups.get(candidate_id, frozenset())
+            for candidate_id in ordered_candidate_ids[:depth]
+        )
+    )
+    return sum(weight for group, weight in group_weights.items() if group in covered) / total
+
+
+def alpha_dcg(
+    ordered_candidate_ids: Sequence[str],
+    candidate_groups: Mapping[str, frozenset[str]],
+    group_weights: Mapping[str, float],
+    depth: int,
+    *,
+    alpha: float = 0.5,
+) -> float:
+    seen: dict[str, int] = {}
+    total = 0.0
+    for rank, candidate_id in enumerate(ordered_candidate_ids[:depth], start=1):
+        gain = 0.0
+        for group in candidate_groups.get(candidate_id, frozenset()):
+            gain += group_weights.get(group, 0.0) * (1 - alpha) ** seen.get(group, 0)
+            seen[group] = seen.get(group, 0) + 1
+        total += gain / math.log2(rank + 1)
+    return total
+
+
+def alpha_ndcg(
+    ordered_candidate_ids: Sequence[str],
+    candidate_groups: Mapping[str, frozenset[str]],
+    group_weights: Mapping[str, float],
+    depth: int,
+    *,
+    alpha: float = 0.5,
+) -> float | None:
+    """Redundancy-aware nDCG with a deterministic greedy ideal ordering."""
+
+    remaining = set(ordered_candidate_ids)
+    ideal: list[str] = []
+    seen: dict[str, int] = {}
+    for _ in range(min(depth, len(remaining))):
+        best = max(
+            remaining,
+            key=lambda candidate_id: (
+                sum(
+                    group_weights.get(group, 0.0) * (1 - alpha) ** seen.get(group, 0)
+                    for group in candidate_groups.get(candidate_id, frozenset())
+                ),
+                candidate_id,
+            ),
+        )
+        ideal.append(best)
+        remaining.remove(best)
+        for group in candidate_groups.get(best, frozenset()):
+            seen[group] = seen.get(group, 0) + 1
+    ideal_score = alpha_dcg(ideal, candidate_groups, group_weights, depth, alpha=alpha)
+    if ideal_score == 0:
+        return None
+    return (
+        alpha_dcg(ordered_candidate_ids, candidate_groups, group_weights, depth, alpha=alpha)
+        / ideal_score
+    )

sphynx/_speech_benchmark_scoring.py

@@ -0,0 +1,140 @@
+"""Descriptive speech benchmark calculations; no model or listener inference."""
+from __future__ import annotations
+from collections import Counter,defaultdict
+import statistics
+
+from ._speech_kido import number,require
+from ._speech_scoring import timing_summary
+from ._speech_benchmark_sources import DIMENSIONS
+
+
+def _groups(rows,extra=()):
+    groups=defaultdict(list)
+    for row in rows:groups[tuple(row.get(k) for k in ('profile',*extra))].append(row)
+    return sorted(groups.items(),key=lambda p:str(p[0]))
+
+
+def _rating(values,low,high):
+    require(all(v is None or type(v) in (int,float) and low<=v<=high for v in values),'Invalid rating')
+    present=[v for v in values if v is not None]
+    return dict(expected=len(values),rated=len(present),pending=len(values)-len(present),
+        mean=statistics.mean(present) if present and len(present)==len(values) else None,
+        observed_subset_mean=statistics.mean(present) if present else None,
+        distribution={str(v):present.count(v) for v in range(low,high+1)})
+
+
+def evaluate_pt_br_listening(rows):
+    return [dict(profile=key[0],samples=len(group),dimensions={d:_rating([r['ratings'][d] for r in group],1,5) for d in DIMENSIONS},
+        historical_rater_count=1,confidence_interval=None,machine_intelligibility=None,
+        note='Four rating dimensions are separate; no composite or population inference.') for key,group in _groups(rows)]
+
+
+def evaluate_pronunciation(rows):
+    result=[]
+    for key,group in _groups(rows):
+        direct=[r for r in group if r['review_scope']=='individually_listened' and r['manifest_bound'] and r['provider_gate'] and not r['customization_observed']]
+        assigned=[r for r in group if r['review_scope']=='user_assigned']
+        result.append(dict(profile=key[0],lines=len(group),supplied_ratings=_rating([r['score'] for r in group],0,5),
+            individually_listened_lines=len(direct),individually_listened_mean=_rating([r['score'] for r in direct],0,5)['mean'] if len(direct)==len(group) else None,
+            assigned_lines=len(assigned),assigned_observed_mean=_rating([r['score'] for r in assigned],0,5)['observed_subset_mean'],
+            assigned_review_units=1 if assigned else 0,unbound_review_lines=sum(not r['manifest_bound'] for r in group),
+            invalid_provider_lines=sum(not r['provider_gate'] for r in group),customized_lines=sum(r['customization_observed'] for r in group),
+            historical_rater_count=1,confidence_interval=None,
+            categories={category:_rating([r['score'] for r in group if r['category']==category],0,5) for category in sorted({r['category'] for r in group})}))
+    return result
+
+
+def evaluate_language_switching(rows):
+    result=[]
+    for key,group in _groups(rows,('model','runtime_sha')):
+        attempts=[a for r in group for a in r['attempts']]
+        valid=[r for r in group if r['valid']]
+        values=[a['fresh_ttfa_ms'] for r in valid for a in r['attempts'] if a['fresh_ttfa_ms'] is not None]
+        result.append(dict(profile=key[0],model=key[1],runtime_sha=key[2],sessions=len({r['session_id'] for r in group}),
+            expected_steps=len(group),valid_steps=len(valid),missing_steps=sum(not r['attempts'] for r in group),
+            ambiguous_steps=sum(r.get('alignment')=='ambiguous' for r in group),
+            captured_turns=len({(r['session_id'],t) for r in group for t in r['selected_turn_ids']}),
+            invalid_or_missing_steps=len(group)-len(valid),attempts=len(attempts),provider_mismatch_attempts=sum(not a['provider_matches'] for a in attempts),
+            failed_attempts=sum(not a['success'] for a in attempts),cache_attempts=sum(a['cache_hit'] for a in attempts),
+            fresh_valid_ttfa_ms=timing_summary(values,convention='kido.live-median-rounded-index.v1'),
+            correct_language_spoken=None,voice_identity_stability=None,perceived_latency=None,confidence_interval=None,
+            timing_scope='retained server synthesis start to first bytes; provider validity does not prove spoken language'))
+    return result
+
+
+def evaluate_stt_turn_lifecycle(rows):
+    """Check request/turn state transitions without treating snapshots as deltas."""
+    results=[]
+    for row in rows:
+        states={};counts=Counter();errors=[];last=-1;connected=False;request=None;transcripts={}
+        require(type(row['events']) is list,'STT events must be a list')
+        for event in row['events']:
+            require(type(event) is dict,'Invalid STT event')
+            now=number(event.get('t'),'STT event clock');kind=event.get('type');counts[kind]+=1
+            if not isinstance(event.get('request_id'),str) or not event['request_id']:errors.append('missing_request_identity')
+            if now<last:errors.append('reversed_clock')
+            last=now
+            if kind=='connected':
+                if connected:errors.append('duplicate_connection')
+                connected=True;request=event.get('request_id');continue
+            if not connected:errors.append('event_before_connection')
+            if request is not None and event.get('request_id')!=request:errors.append('request_identity_changed')
+            turn=event.get('turn_id');state=states.get(turn)
+            if not isinstance(turn,str) or not turn:errors.append('missing_turn_identity');continue
+            if kind=='turn.start':
+                if state is not None or any(v!='ended' for v in states.values()):errors.append('overlapping_turn_start')
+                states[turn]='active'
+            elif kind=='turn.update':
+                if state not in ('active','eager'):errors.append('update_outside_turn')
+            elif kind=='turn.eager_end':
+                if state!='active':errors.append('eager_end_outside_active_turn')
+                states[turn]='eager'
+            elif kind=='turn.resume':
+                if state!='eager':errors.append('resume_without_eager_end')
+                states[turn]='active'
+            elif kind=='turn.end':
+                if state not in ('active','eager'):errors.append('end_outside_turn')
+                states[turn]='ended'
+                text=event.get('transcript');require(isinstance(text,str),'Invalid final transcript snapshot')
+                transcripts[turn]=text
+            else:errors.append('unknown_turn_event')
+        if not connected:errors.append('connection_missing')
+        if not states or any(state!='ended' for state in states.values()):errors.append('final_turn_missing')
+        results.append(dict(id=row['id'],profile=row['profile'],clip_id=row['clip_id'],
+            source_status=row['status'],protocol_valid=not errors and not row['error_present'] and row['status'] in ('ok','success'),
+            protocol_errors=errors,event_counts=dict(counts),completed_turns=len(transcripts),
+            final_snapshots=transcripts,wer=None,endpoint_delay_ms=None,
+            note='Retained lifecycle behavior only; audio identity and speech-end ground truth are unavailable.'))
+    return results
+
+
+def evaluate_voice_continuity(rows):
+    return [dict(profile=r['profile'],score=r['score'],review_units=1,
+        review_scope=r['review_scope'],independent_turn_ratings=0,confidence_interval=None) for r in rows]
+
+
+def evaluate_code_switching(rows):
+    return [dict(profile=key[0],model=key[1],attempts=len(group),successful=sum(r['success'] for r in group),
+        failed=sum(not r['success'] for r in group),unique_texts=len({r['case_id'] for r in group}),
+        quality_inferred_from_bytes=False,perceived_latency=None,
+        observed_success_ttfa_ms=timing_summary([r['ttfa_ms'] for r in group if r['success'] and r['ttfa_ms'] is not None],convention='kido.live-median-rounded-index.v1'),
+        timing_scope='retained direct API probe; separate from live-agent switching') for key,group in _groups(rows,('model',))]
+
+
+def evaluate_playout_interruptions(sessions):
+    return [dict(session_id=s['session_id'],profile=s['profile'],
+        terminal_playout_deliveries=s['terminal_playout_deliveries'],interrupted_deliveries=s['interrupted_deliveries'],
+        controlled_user_barge_in=None,barge_in_success_rate=None,rapid_turn_stress_result=None,
+        note='Terminal playout flags are observations, not proof of a controlled microphone interruption.') for s in sessions]
+
+
+def grade_records(evidence):
+    rows=evidence['observations'];by_track={name:[r for r in rows if r['track']==name] for name in {r['track'] for r in rows}}
+    evaluators={'pt_br_listening':evaluate_pt_br_listening,'pronunciation':evaluate_pronunciation,
+        'language_switching':evaluate_language_switching,'stt_turn_lifecycle':evaluate_stt_turn_lifecycle,
+        'voice_continuity':evaluate_voice_continuity,'code_switching':evaluate_code_switching}
+    require(set(by_track)==set(evaluators),'Benchmark track census changed')
+    result={name:fn(by_track[name]) for name,fn in evaluators.items()}
+    sessions=evidence['audit']['pronunciation']['sessions']+evidence['audit']['switching_sessions']
+    result['playout_interruptions']=evaluate_playout_interruptions(sessions)
+    return result

sphynx/_speech_benchmark_sources.py

@@ -0,0 +1,275 @@
+"""Native Kido benchmark intake. Source files are inert, private evidence."""
+from __future__ import annotations
+
+from collections import Counter
+from pathlib import Path
+import re
+
+from ._speech_kido import csv_rows, digest, encoded, json_value, number, require
+from .speech_evidence import _plain, _read
+
+SOURCE_COMMIT = '71ec7e89805360479ea19c5bb43fae7707229ce9'
+SOURCE_REPOSITORY = 'https://github.com/sable-inc/sable-tts-eval'
+# Reviewed against committed Git blobs; no session inventory ships in the wheel.
+SOURCE_LOCK_SHA256 = 'e043aa19f68be82974120c894b2b0276bf7ba538db2002084d65a33ecf14bfab'
+PT = 'results/isolated/multilingual-extension-v1/pt-br-listening-pack/'
+PRON = 'results/live-agent/pronunciation-v1/'
+SWITCH = 'results/live-agent/language-switching-v1/'
+TURN = 'results/exploratory/cartesia-ink-2-stt/'
+DIMENSIONS = ('overall_quality_1_5', 'pronunciation_1_5',
+              'native_sounding_1_5', 'voice_consistency_1_5')
+
+
+def source_inventory(root):
+    root = Path(root).absolute(); _plain(root)
+    paths = {'harness/corpus/pt-br-listening-pack-v1.json',
+             PT+'mapping.json', PT+'scores-completed.csv', PT+'manifest.json',
+             PRON+'listening-review/manifest.csv', PRON+'listening-review/human-review-scored.csv',
+             SWITCH+'track-b-voice-continuity/README.md',
+             SWITCH+'track-c-code-switching/README.md',
+             SWITCH+'track-c-code-switching/term_list.json',
+             SWITCH+'track-c-code-switching/code_switch_results.json'}
+    for folder in ('turn-detection-v1', 'turn-detection-ink2-v1'):
+        paths.update(TURN+folder+'/'+p for p in ('turn_events_raw.json', 'pause_clips_manifest.json', 'run_turn_test.py'))
+    for prefix in (PRON, SWITCH):
+        base = root/prefix/'sessions'; _plain(base)
+        for directory in sorted(base.glob('*/RM_*')):
+            _plain(directory)
+            if not directory.is_dir(): continue
+            paths.update(p.relative_to(root).as_posix() for p in directory.glob('events/chunk_*.jsonl'))
+            paths.update((directory/p).relative_to(root).as_posix() for p in ('session.json', 'metadata.json', 'agent_config.json'))
+            if prefix == SWITCH:
+                paths.add((directory.parent/'summary.md').relative_to(root).as_posix())
+                paths.update(p.relative_to(root).as_posix() for p in directory.glob('record/transcript/chunk_*.jsonl'))
+    require(0 < len(paths) <= 1000, 'Invalid benchmark source census')
+    data = {}; total = 0
+    for p in sorted(paths):
+        raw = _read(root, p); total += len(raw)
+        require(total <= 64*1024*1024, 'Benchmark source exceeds byte limit')
+        data[p] = raw
+    inventory = dict(schema_version='sphynx.speech.kido-source.v1', repository=SOURCE_REPOSITORY,
+                     source_commit=SOURCE_COMMIT, files={p:dict(sha256=digest(b),size_bytes=len(b)) for p,b in data.items()})
+    return inventory, data
+
+
+def source_bytes(root):
+    inventory, data = source_inventory(root)
+    require(digest(encoded(inventory)) == SOURCE_LOCK_SHA256, 'Benchmark source differs from reviewed bytes')
+    return inventory, data
+
+
+def _unique(rows, key, label):
+    identities = [key(r) for r in rows]
+    require(len(set(identities)) == len(identities), 'Duplicate '+label)
+    return dict(zip(identities, rows))
+
+
+def _score(value, low, high):
+    if value is None or str(value).strip() == '': return None
+    require(isinstance(value, str) and re.fullmatch(r'[0-9]+', value), 'Rating must be an integer')
+    result = int(value); require(low <= result <= high, 'Rating out of range')
+    return result
+
+
+def _case(cases, benchmark, case_id, text, language):
+    require(isinstance(text, str) and 0 < len(text) <= 10000, 'Invalid benchmark input text')
+    identity = benchmark+'/'+case_id
+    value = dict(id=identity, text=text, language=language)
+    require(identity not in cases or cases[identity] == value, 'Conflicting benchmark input identity')
+    cases[identity] = value
+    return identity
+
+
+def _profile(provider, model):
+    return {'eleven_v4':'eleven-v4', 'eleven_v4_turbo':'eleven-v4-turbo'}.get(model, provider)
+
+
+def synthesis_record(event, provider):
+    d = event['data']
+    require(type(d) is dict and all(type(d.get(k)) is bool for k in ('complete','cancelled','cache_hit')), 'Invalid synthesis flags')
+    audio = d.get('produced_audio_bytes'); require(type(audio) is int and audio >= 0, 'Invalid synthesis audio count')
+    error = d.get('error') or ''; require(isinstance(error,str), 'Invalid synthesis error')
+    observed = d.get('provider') or None
+    success = d['complete'] and not d['cancelled'] and not error and audio > 0
+    clocks = [number(d.get(k), k, nullable=True) for k in ('started_at','first_audio_at','ended_at')]
+    ordered = all(v is not None for v in clocks) and clocks[0] <= clocks[1] <= clocks[2]
+    return dict(turn_id=event.get('turn_id'), response_id=d.get('response_id'),
+        success=success, provider_matches=observed == provider, observed_provider=observed,
+        observed_model=d.get('model') or None, cache_hit=d['cache_hit'], cancelled=d['cancelled'],
+        error_present=bool(error), audio_bytes=audio,
+        fresh_ttfa_ms=(clocks[1]-clocks[0])*1000 if success and observed == provider and not d['cache_hit'] and ordered else None)
+
+
+def session_records(files, prefix):
+    result = []
+    for path in sorted(p for p in files if p.startswith(prefix+'sessions/') and p.endswith('/session.json')):
+        base = path.removesuffix('session.json'); metadata = json_value(files[base+'metadata.json'])
+        config = json_value(files[base+'agent_config.json']); tts = config.get('tts', {})
+        provider, model = tts.get('provider'), tts.get('model')
+        require(isinstance(provider,str) and isinstance(model,str), 'Missing configured speech identity')
+        chunks = sorted(p for p in files if p.startswith(base+'events/chunk_') and p.endswith('.jsonl'))
+        nums = [int(re.search(r'chunk_([0-9]+)\.jsonl$', p)[1]) for p in chunks]
+        require(nums and nums == list(range(nums[0], nums[0]+len(nums))), 'Missing or unordered benchmark event chunks')
+        events = [json_value(line) for p in chunks for line in files[p].splitlines() if line.strip()]
+        _unique(events, lambda e:e['id'], 'session event ID')
+        synthesis = [synthesis_record(e, provider) for e in events if e.get('type') == 'synthesis_complete']
+        valid = bool(synthesis) and all(s['success'] and s['provider_matches'] for s in synthesis)
+        # Last transport playout span per delivery is the terminal observation.
+        playout = {}
+        for e in events:
+            d = e.get('data', {})
+            if e.get('type') == 'turn_trace' and d.get('name') == 'playout' and d.get('measurement') == 'transport_buffer_estimate':
+                require(isinstance(d.get('delivery_id'),str) and type(d.get('interrupted')) is bool, 'Invalid playout event')
+                playout[d['delivery_id']] = dict(interrupted=d['interrupted'], complete=d.get('complete'))
+        result.append(dict(session_id=base.rstrip('/').split('/')[-1], base=base, provider=provider, model=model,
+            profile=_profile(provider,model), runtime_sha=metadata.get('git_sha'), provider_gate=valid,
+            attempts=synthesis, interrupted_deliveries=sum(x['interrupted'] for x in playout.values()),
+            terminal_playout_deliveries=len(playout), record=json_value(files[path]).get('record', {}),
+            config_sha256=digest(files[base+'agent_config.json']),
+            transcripts=[json_value(line) for p in sorted(files) if p.startswith(base+'record/transcript/chunk_') for line in files[p].splitlines() if line.strip()],
+            pronunciation_customization=bool(tts.get('pronunciation_dictionary_id') or tts.get('pronunciation_rules') or tts.get('pronunciation_terms'))))
+    return result
+
+
+def _markdown_scores(raw, heading):
+    text = raw.decode('utf-8'); lines = text.splitlines(); result = []
+    for index, line in enumerate(lines):
+        if not line.startswith('|') or heading not in line: continue
+        for row in lines[index+2:]:
+            if not row.startswith('|'): break
+            columns = [v.strip() for v in row.strip('|').split('|')]
+            require(len(columns)==2 and re.fullmatch(r'[1-5](?:\.0)?',columns[1]) is not None, 'Invalid qualitative score table')
+            result.append((columns[0],float(columns[1])))
+    require(result, 'Missing qualitative review table')
+    return result
+
+
+def exact_script_turns(transcripts,text):
+    """Match whole scripts or adjacent agent fragments without fuzzy matching.
+
+    User turns, repeated snapshots, unrelated text and ambiguous matches do
+    not become an alignment. Each matched synthesis delivery stays separate.
+    """
+    norm=lambda value:' '.join(value.split())
+    target=norm(text);matches=set()
+    direct={r['turn_id'] for r in transcripts if r.get('speaker')=='agent' and r.get('turn_id') and norm(r.get('text',''))==target}
+    if direct:return (sorted(direct),'exact-single-delivery' if len(direct)==1 else 'ambiguous')
+    for start in range(len(transcripts)):
+        segments=[];turns=[]
+        for row in transcripts[start:start+4]:
+            if row.get('speaker')!='agent' or not row.get('turn_id') or row['turn_id'] in turns:break
+            segments.append(row.get('text',''));turns.append(row['turn_id']);combined=norm(' '.join(segments))
+            if combined==target:matches.add(tuple(turns));break
+            if not target.startswith(combined):break
+    if len(matches)==1:return (list(next(iter(matches))),'exact-adjacent-agent-fragments')
+    return ([], 'ambiguous' if matches else 'unavailable')
+
+
+def normalize_sources(files):
+    observations=[]; cases={}; audit={}
+    corpus=json_value(files['harness/corpus/pt-br-listening-pack-v1.json'])
+    lines=_unique(corpus['lines'],lambda r:r['id'],'PT-BR corpus line')
+    require(corpus['line_count']==len(lines) and corpus['language']=='pt-BR','PT-BR corpus identity mismatch')
+    mapping=json_value(files[PT+'mapping.json'])['samples']
+    scores=_unique(csv_rows(files[PT+'scores-completed.csv']),lambda r:r['sample_id'],'PT-BR rating')
+    require(set(scores)<=set(mapping),'PT-BR rating has no mapped sample')
+    manifest=json_value(files[PT+'manifest.json'])
+    identities=_unique(manifest['providers_synthesized'],lambda r:_profile(r['provider'],r['model']),'PT-BR model profile')
+    for sample, row in sorted(mapping.items()):
+        require(row['line_id'] in lines and row['provider'] in identities,'Unbound PT-BR sample')
+        line=lines[row['line_id']]; require(row['text']==line['text'] and row['category']==line['category'],'PT-BR mapping disagrees with corpus')
+        case_id=_case(cases,'pt-br-listening',row['line_id'],line['text'],'pt-BR')
+        observations.append(dict(track='pt_br_listening',id='pt-br/'+sample,profile=row['provider'],
+            model=identities[row['provider']]['model'],case_id=case_id,category=line['category'],
+            ratings={dim:_score(scores.get(sample,{}).get(dim),1,5) for dim in DIMENSIONS},
+            review_scope='historical-single-rater',audio_available=False))
+    audit['pt_br']=dict(mapped_samples=len(mapping),completed_rows=len(scores),
+        pending_samples=len(mapping)-len(scores),machine_intelligibility=None,
+        synthesis_manifest=manifest,latency_status='supplied-summary-only-not-recomputed-from-trials')
+
+    pron_sessions=session_records(files,PRON); switch_sessions=session_records(files,SWITCH)
+    manifest=_unique(csv_rows(files[PRON+'listening-review/manifest.csv']),lambda r:(r['provider'],r['line']),'pronunciation manifest row')
+    ratings=_unique(csv_rows(files[PRON+'listening-review/human-review-scored.csv']),lambda r:(r['provider'],r['line_id']),'pronunciation rating')
+    profiles={}
+    for session in pron_sessions: profiles.setdefault(session['profile'],[]).append(session)
+    for (profile,line_id),row in sorted(ratings.items()):
+        require(profile in profiles,'Pronunciation review has no captured profile')
+        m=manifest.get((profile,line_id)); scope=row['review_source']; score=_score(row['human_score_0_5'],0,5)
+        require(scope in ('individually_listened','user_assigned','pending') and (score is None)==(scope=='pending'),'Pronunciation review status mismatch')
+        if m:
+            require(m['category']==row['category'] and m['expected_entity']==row['expected_entity'],'Pronunciation review mapping mismatch')
+            case_id=_case(cases,'pronunciation','line-'+line_id+'-'+digest(m['corpus_text'].encode())[:12],m['corpus_text'],'en')
+        else: case_id=None
+        sessions=profiles[profile]
+        observations.append(dict(track='pronunciation',id='pronunciation/'+profile+'/'+line_id,profile=profile,
+            model=sessions[0]['model'],category=row['category'],case_id=case_id,score=score,review_scope=scope,
+            manifest_bound=m is not None,provider_gate=all(s['provider_gate'] for s in sessions),
+            customization_observed=any(s['pronunciation_customization'] for s in sessions),audio_available=False))
+    audit['pronunciation']=dict(mapped_rows=len(manifest),rating_rows=len(ratings),
+        unbound_review_rows=sum(k not in manifest for k in ratings),
+        sessions=[{k:v for k,v in s.items() if k not in ('base','transcripts')} for s in pron_sessions])
+
+    for s in switch_sessions:
+        summary=files[str(Path(s['base']).parent/'summary.md').replace('\\','/')]
+        selected=[]
+        for line in summary.decode('utf-8').splitlines():
+            cols=[x.strip() for x in line.strip('|').split('|')]
+            if line.startswith('|') and re.fullmatch(r'turn_[0-9]+',cols[0]) and not cols[1].startswith('*(unscripted'):
+                selected.append(([cols[0]],cols[1]))
+        basis='native-session-summary-turn-IDs'
+        if s['profile']=='google' and not selected:
+            # Google summaries lack turn IDs. Join exact scripted text to retained
+            # transcript records, never guess the last eight synthesis events.
+            basis='exact-script-to-retained-transcript-join';in_google=False
+            for line in files[SWITCH+'track-b-voice-continuity/README.md'].decode('utf-8').splitlines():
+                if line.startswith('## '):in_google=line.startswith('## Google')
+                cols=[x.strip() for x in line.strip('|').split('|')]
+                if in_google and line.startswith('|') and re.fullmatch(r'[0-9]{2}:[0-9]{2}',cols[0]) and cols[1]!='English (first)':
+                    text=cols[2].strip('"“”')
+                    turns,alignment=exact_script_turns(s['transcripts'],text)
+                    selected.append((turns,cols[1],alignment))
+        selected=[(*row,'native-turn-ID') if len(row)==2 else row for row in selected]
+        require(len(selected)==8 and len({r[1] for r in selected})==8,'Switching selection is not the declared eight-step protocol')
+        require(len({t for turns,_,_ in selected for t in turns})==sum(len(turns) for turns,_,_ in selected),'Switching turn mapped to multiple steps')
+        for index,(turns,language,alignment) in enumerate(selected):
+            attempts=[a for a in s['attempts'] if a['turn_id'] in turns]
+            observations.append(dict(track='language_switching',id='switch/'+s['session_id']+'/step-'+str(index+1),
+                profile=s['profile'],model=s['model'],session_id=s['session_id'],runtime_sha=s['runtime_sha'],
+                language=language,selected_turn_ids=turns,selection_basis=basis,alignment=alignment,attempts=attempts,
+                valid=alignment!='ambiguous' and bool(attempts) and all(a['success'] and a['provider_matches'] for a in attempts)))
+    audit['switching_sessions']=[{k:v for k,v in s.items() if k not in ('base','transcripts')} for s in switch_sessions]
+
+    for folder,model in [('turn-detection-v1','ink-preview'),('turn-detection-ink2-v1','ink-2')]:
+        prefix=TURN+folder+'/'
+        # AST parsing reads constants only, never imports the original provider script.
+        import ast
+        tree=ast.parse(files[prefix+'run_turn_test.py'].decode('utf-8'))
+        constants={n.targets[0].id:n.value.value for n in tree.body if isinstance(n,ast.Assign) and len(n.targets)==1 and isinstance(n.targets[0],ast.Name) and isinstance(n.value,ast.Constant)}
+        require(constants.get('MODEL')==model and str(constants.get('WS_URL_BASE','')).endswith('/stt/turns/websocket'),'STT endpoint or requested model mismatch')
+        clips=_unique(json_value(files[prefix+'pause_clips_manifest.json']),lambda r:r['name'],'pause clip')
+        trials=_unique(json_value(files[prefix+'turn_events_raw.json']),lambda r:r['name'],'turn lifecycle attempt')
+        require(set(clips)==set(trials),'Missing STT lifecycle attempts')
+        for name,row in trials.items():
+            observations.append(dict(track='stt_turn_lifecycle',id='turn/'+model+'/'+name,profile=model,
+                clip_id=name,status=row['status'],error_present=bool(row.get('error')),events=row['events'],
+                audio_duration_s=number(clips[name]['duration_s'],'clip duration'),audio_available=False))
+
+    review_aliases={'ElevenLabs':'elevenlabs','Cartesia':'cartesia','MiniMax':'minimax','Google':'google'}
+    for provider,score in _markdown_scores(files[SWITCH+'track-b-voice-continuity/README.md'],'Voice continuity'):
+        require(provider in review_aliases,'Unknown continuity review profile')
+        profile=review_aliases[provider]; require(any(s['profile']==profile for s in switch_sessions),'Unbound continuity review')
+        observations.append(dict(track='voice_continuity',id='continuity/'+profile,profile=profile,score=score,
+            review_scope='historical-provider-level-qualitative-single-rater',independent_turn_ratings=0,audio_available=False))
+    quality=_markdown_scores(files[SWITCH+'track-c-code-switching/README.md'],'Qualitative score')
+    audit['code_switching_quality']=[dict(language_pair=pair,score=score,review_scope='historical-language-pair-level-single-rater') for pair,score in quality]
+    trials=json_value(files[SWITCH+'track-c-code-switching/code_switch_results.json'])
+    _unique(trials,lambda r:(r['id'],r['rep']),'code-switching attempt')
+    for row in trials:
+        case_id=_case(cases,'code-switching',row['id'],row['text'],'mul')
+        observations.append(dict(track='code_switching',id='code-switch/'+row['id']+'/'+str(row['rep']),
+            profile=row['provider'],model=row['model'],case_id=case_id,
+            success=row['status']=='success' and not row.get('error') and number(row['audio_bytes'],'code-switch audio')>0,
+            ttfa_ms=number(row['ttfa_ms'],'code-switch TTFA',nullable=True),audio_available=False))
+    require(len({r['id'] for r in observations})==len(observations),'Duplicate benchmark observation')
+    audit['source_scope']='selected retained benchmark files; no audio, source execution, new listening or provider calls'
+    return dict(observations=observations,cases=sorted(cases.values(),key=lambda r:r['id']),audit=audit)