sable-sphynx 0.7.0.dev0__py3-none-any.whl

sphynx/speech_campaigns.py

@@ -0,0 +1,378 @@
+"""Frozen speech protocol plans, explicit capture ingestion and offline fixtures.
+
+No account access, microphone, provider or live session is launched here. Use
+the separate TTS runner for TTS execution, or a reviewed collector for live
+campaigns. Missing captures remain in every denominator.
+"""
+from collections import Counter
+from pathlib import Path
+import argparse
+import io
+import json
+import math
+import uuid
+import wave
+
+from ._speech_kido import digest,encoded,json_value,require,number
+from ._speech_scoring import score_transcript
+from .speech_contracts import identifier
+from .speech_evidence import _new_directory,_read,_write_files,_sha
+
+SUITE_ID='speech.kido-campaigns.v1'
+SCHEMA='sphynx.speech.kido-campaigns'
+PROTOCOLS=('session-reliability','controlled-barge-in','rapid-turn-stress','machine-intelligibility')
+
+
+def _candidate(value):
+    require(type(value) is dict and set(value)=={'id','provider','model','voice_id','source_commit','source_pr'},'Candidate identity fields differ')
+    for k in ('id','provider','model'):identifier(value[k])
+    require(isinstance(value['source_commit'],str) and len(value['source_commit'])==40
+        and all(c in '0123456789abcdef' for c in value['source_commit']),'Pin full candidate commit')
+    import re
+    require(isinstance(value['source_pr'],str) and re.fullmatch(r'https://github.com/[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+/pull/[1-9][0-9]*',value['source_pr']),'Pin candidate PR')
+    require(value['voice_id'] is None or isinstance(value['voice_id'],str) and 0<len(value['voice_id'])<=128,'Invalid voice identity')
+    return dict(value)
+
+
+def _audio(path):
+    path=Path(path).absolute();raw=_read(path.parent,path.name)
+    try:
+        with wave.open(io.BytesIO(raw),'rb') as reader:
+            require(reader.getnchannels()==1 and reader.getsampwidth()==2 and reader.getcomptype()=='NONE'
+                and reader.getframerate() in (8000,16000,24000,44100,48000),'Use bounded mono PCM16 WAV audio')
+            count=reader.getnframes();rate=reader.getframerate();pcm=reader.readframes(count)
+            require(count>0 and count<=rate*120 and len(pcm)==count*2,'Empty, truncated or oversized PCM audio')
+    except (wave.Error,EOFError):raise ValueError('Invalid campaign WAV') from None
+    return raw,dict(sha256=digest(raw),size_bytes=len(raw),sample_rate=rate,frames=count,duration_ms=count/rate*1000)
+
+
+def freeze_speech_campaign(bundle,*,protocol,candidates,output_dir,mode='capture',sessions_per_candidate=20,
+        rapid_turns=8,max_idle_ms=200,interrupt_after_ms=500,interrupt_tolerance_ms=250,
+        microphone_audio=None,audio_cases=None,asr=None,max_units=200,collector_source_sha256=None):
+    """Freeze an exact design. Audio references stay out of collector inputs."""
+    from .speech_complete import _verified
+    evidence=_verified(bundle)
+    require(protocol in PROTOCOLS and mode in ('capture','fixture'),'Unknown campaign protocol or mode')
+    if collector_source_sha256 is not None:_sha(collector_source_sha256)
+    require(type(candidates) is list and 1<=len(candidates)<=16,'Invalid candidate census')
+    candidates=[_candidate(c) for c in candidates]
+    require(len({c['id'] for c in candidates})==len(candidates),'Duplicate candidate')
+    require(type(sessions_per_candidate) is int and 1<=sessions_per_candidate<=100,'Invalid sessions per candidate')
+    require(type(max_units) is int and 1<=max_units<=200,'Invalid campaign unit cap')
+    for v in (max_idle_ms,interrupt_after_ms,interrupt_tolerance_ms):number(v,'campaign timing')
+    require(0<interrupt_tolerance_ms<=1000 and 0<interrupt_after_ms<=10000 and max_idle_ms<=2000,'Campaign timing exceeds limits')
+    require(type(rapid_turns) is int and 8<=rapid_turns<=10,'Rapid-turn design requires eight to ten turns')
+    payload={};references={};script=evidence['official_script']
+    if protocol=='session-reliability':steps=script[:4]
+    elif protocol=='controlled-barge-in':steps=script[5:7]
+    elif protocol=='rapid-turn-stress':steps=[script[6+i%4] for i in range(rapid_turns)]
+    else:steps=[]
+    steps=[dict(id='step:'+str(i+1),case_id=s['id'],text=s['text'],language=s['language']) for i,s in enumerate(steps)]
+    microphone=None
+    if protocol=='controlled-barge-in':
+        if mode=='capture':
+            require(microphone_audio is not None,'Freeze actual interruption audio before a microphone campaign')
+            raw,microphone=_audio(microphone_audio);payload['audio/microphone.wav']=raw
+        else:microphone=dict(sha256=digest(b'fixture microphone'),duration_ms=100,fixture=True)
+    if protocol=='machine-intelligibility':
+        require(type(asr) is dict and set(asr)=={'provider','model','revision','config'} and all(isinstance(asr[k],str) and asr[k] for k in ('provider','model','revision'))
+            and asr['revision'].lower() not in ('latest','default','auto') and type(asr['config']) is dict,'Freeze ASR revision and configuration')
+        require(type(audio_cases) is list and 0<len(audio_cases)<=100,'Supply approved audio cases')
+        inputs=[]
+        for row in audio_cases:
+            require(type(row) is dict and set(row)=={'id','path','reference','language','candidate_id'},'Audio case fields differ')
+            identifier(row['id']);require(isinstance(row['reference'],str) and len(row['reference'])<=10000,'Invalid reference')
+            import re
+            require(isinstance(row['language'],str) and re.fullmatch('[a-z]{2}(?:-[A-Za-z]{2,4})?',row['language']),'Invalid audio language')
+            require(row['candidate_id'] in {c['id'] for c in candidates},'Unknown audio candidate')
+            raw,meta=_audio(row['path']);name='audio/'+row['id'].replace(':','_')+'.wav'
+            require(name not in payload,'Duplicate audio identity');payload[name]=raw
+            inputs.append(dict(id=row['id'],candidate_id=row['candidate_id'],language=row['language'],audio=meta,audio_file=name))
+            references[row['id']]=row['reference']
+        require(len({r['id'] for r in inputs})==len(inputs),'Duplicate audio case ID')
+        units=[dict(id='unit:'+str(i+1),candidate_id=r['candidate_id'],audio_case=r,steps=[]) for i,r in enumerate(inputs)]
+    else:
+        require(asr is None and audio_cases is None,'ASR inputs belong only to machine intelligibility')
+        units=[dict(id='unit:'+str(i+1),candidate_id=c['id'],steps=steps) for i,(c,_) in enumerate(
+            (c,n) for c in candidates for n in range(sessions_per_candidate))]
+    require(len(units)<=max_units,'Campaign exceeds explicit unit cap')
+    plan=dict(schema_version=SCHEMA+'.plan.v1',suite_id=SUITE_ID,id='speech-campaign-'+uuid.uuid4().hex,
+        protocol=protocol,mode=mode,bundle_manifest_sha256=bundle.manifest_sha256,candidates=candidates,units=units,
+        source_commit='71ec7e89805360479ea19c5bb43fae7707229ce9',microphone=microphone,asr=asr,
+        design=dict(sessions_per_candidate=sessions_per_candidate,rapid_turns=rapid_turns,max_idle_ms=max_idle_ms,
+            interrupt_after_ms=interrupt_after_ms,interrupt_tolerance_ms=interrupt_tolerance_ms,max_units=max_units),
+        planned_units=len(units),planned_steps=sum(len(u['steps']) for u in units),provider_calls_made=False,
+        provider_cost_usd=None,monetary_cap_enforced=False,collector_source_verified=False,collector_source_sha256=collector_source_sha256,
+        implementation_sha256={p:digest(Path(__file__).with_name(p).read_bytes()) for p in ('speech_campaigns.py','_speech_scoring.py','speech_contracts.py')})
+    files=_write_files(_new_directory(output_dir),payload|{'references.json':encoded(references)})
+    dest=Path(output_dir).absolute();plan['files']=files;raw=encoded(plan);(dest/'plan.json').write_bytes(raw)
+    return dict(path=str(dest),plan_sha256=digest(raw),suite_id=SUITE_ID,protocol=protocol,planned_units=len(units),planned_steps=plan['planned_steps'])
+
+
+def _load_plan(path,pin):
+    root=Path(path).absolute();raw=_read(root,'plan.json');require(digest(raw)==_sha(pin),'Campaign plan changed')
+    plan=json_value(raw)
+    require(plan.get('schema_version')==SCHEMA+'.plan.v1' and plan.get('suite_id')==SUITE_ID and
+        plan.get('protocol') in PROTOCOLS and plan.get('mode') in ('fixture','capture') and plan.get('provider_calls_made') is False,'Invalid campaign plan')
+    require(type(plan.get('units')) is list and len(plan['units'])==plan['planned_units'] and
+        len({u['id'] for u in plan['units']})==len(plan['units']),'Campaign unit census changed')
+    for c in plan['candidates']:_candidate(c)
+    for p,h in plan.get('files',{}).items():
+        b=_read(root,p);require(h==dict(sha256=digest(b),size_bytes=len(b)),'Campaign input artifact changed')
+    require('references.json' in plan['files'],'Campaign references missing')
+    return root,plan,json_value(_read(root,'references.json'))
+
+
+def speech_campaign_requests(path,*,expected_plan_sha256):
+    """Collector view excludes reference text and unrelated repository data."""
+    root,plan,_=_load_plan(path,expected_plan_sha256)
+    return dict(plan_sha256=expected_plan_sha256,protocol=plan['protocol'],mode=plan['mode'],candidates=plan['candidates'],
+        design=plan['design'],asr=plan['asr'],microphone=plan['microphone'],units=plan['units'],
+        input_files={p:h for p,h in plan['files'].items() if p!='references.json'},
+        collector='reviewed external collector required for real sessions; this call launches nothing')
+
+
+def _captures(plan,rows):
+    require(plan.get('schema_version')==SCHEMA+'.plan.v1' and plan.get('suite_id')==SUITE_ID and
+        plan.get('protocol') in PROTOCOLS and plan.get('mode') in ('fixture','capture') and
+        type(plan.get('units')) is list and 0<len(plan['units'])==plan.get('planned_units')<=200 and
+        len({u['id'] for u in plan['units']})==len(plan['units']),'Invalid frozen campaign census or scope')
+    require(type(plan.get('candidates')) is list and 0<len(plan['candidates'])<=16 and
+        len({c['id'] for c in plan['candidates']})==len(plan['candidates']),'Invalid frozen candidate census')
+    for c in plan['candidates']:_candidate(c)
+    require(type(rows) is list and len(rows)<=plan['planned_units'],'Invalid campaign capture census')
+    expected={u['id']:u for u in plan['units']};seen=set();sessions=set();result={}
+    candidates={c['id']:c for c in plan['candidates']}
+    for row in rows:
+        require(type(row) is dict and row.get('unit_id') in expected and row['unit_id'] not in seen,'Unknown or duplicate captured unit')
+        unit=expected[row['unit_id']];candidate=candidates[unit['candidate_id']];seen.add(row['unit_id'])
+        require(row.get('candidate_id')==candidate['id'] and row.get('candidate_sha256')==digest(encoded(candidate)),'Captured candidate identity differs')
+        require(row.get('source_kind')==('fixture' if plan['mode']=='fixture' else 'operator-capture'),'Capture provenance differs from plan')
+        require(row.get('status') in ('completed','connection_failed','disconnected','error','collector_error'),'Unknown capture status')
+        session=row.get('session_id')
+        if plan['protocol']!='machine-intelligibility' and row['status'] not in ('connection_failed','collector_error'):
+            identifier(session);require(session not in sessions,'Repeated session ID violates fresh-session design');sessions.add(session)
+        require(type(row.get('steps')) is list and len(row['steps'])<=len(unit['steps']),'Invalid captured step count')
+        require(len({s['step_id'] for s in row['steps']})==len(row['steps']),'Duplicate captured step')
+        wanted={s['id']:s for s in unit['steps']}
+        for s in row['steps']:
+            require(s['step_id'] in wanted and s.get('case_id')==wanted[s['step_id']]['case_id'],'Unknown captured step identity')
+            require(all(type(s.get(k)) is bool for k in ('complete','cancelled','error_present','cache_hit','interrupted')),'Invalid captured step flags')
+            require(type(s.get('audio_bytes')) is int and s['audio_bytes']>=0,'Invalid captured audio count')
+            for k in ('tts_start_ms','first_audio_ms','playout_start_ms','playout_end_ms'):number(s.get(k),k,nullable=True)
+            require(s.get('provider') is None or isinstance(s['provider'],str),'Invalid provider observation')
+            require(s.get('model') is None or isinstance(s['model'],str),'Invalid model observation')
+        result[row['unit_id']]=row
+    return result
+
+
+def _successful(step,candidate):
+    return step['complete'] and not step['cancelled'] and not step['error_present'] and step['audio_bytes']>0 and step.get('provider')==candidate['provider'] and step.get('model') in (None,candidate['model'])
+
+
+def evaluate_session_reliability(plan,captures):
+    bound=_captures(plan,captures);result=[]
+    for c in plan['candidates']:
+        units=[u for u in plan['units'] if u['candidate_id']==c['id']];rows=[bound[u['id']] for u in units if u['id'] in bound]
+        steps=[s for r in rows for s in r['steps']]
+        complete=sum(r['status']=='completed' and len(r['steps'])==len(next(u for u in units if u['id']==r['unit_id'])['steps']) and all(_successful(s,c) for s in r['steps']) for r in rows)
+        result.append(dict(candidate_id=c['id'],sessions_planned=len(units),sessions_attempted=len(rows),captures_missing=len(units)-len(rows),
+            connections_failed=sum(r['status']=='connection_failed' for r in rows),sessions_with_synthesis=sum(bool(r['steps']) for r in rows),
+            collector_errors=sum(r['status']=='collector_error' for r in rows),
+            all_scripted_steps_completed=complete,complete_session_rate=complete/len(units) if len(rows)==len(units) else None,
+            observed_completed_fraction=complete/len(units),synthesis_attempts=len(steps),synthesis_errors=sum(s['error_present'] for s in steps),
+            zero_audio_events=sum(s['complete'] and s['audio_bytes']==0 for s in steps),fallback_or_unconfirmed_attempts=sum(s.get('provider')!=c['provider'] for s in steps),
+            model_mismatch_attempts=sum(s.get('model') is not None and s['model']!=c['model'] for s in steps),model_identity_independently_verified=False,
+            missing_synthesis_telemetry=sum(r['status'] not in ('connection_failed','collector_error') and not r['steps'] for r in rows),
+            disconnected_sessions=sum(r['status']=='disconnected' for r in rows),disconnect_attributed_to_tts=None,
+            scope='fixture protocol check' if plan['mode']=='fixture' else 'controlled benchmark sessions; not production reliability',confidence_interval=None))
+    return result
+
+
+def evaluate_controlled_barge_in(plan,captures):
+    bound=_captures(plan,captures);candidates={c['id']:c for c in plan['candidates']};result=[]
+    for unit in plan['units']:
+        row=bound.get(unit['id']);steps={s['step_id']:s for s in row['steps']} if row else {};c=candidates[unit['candidate_id']]
+        long=steps.get('step:1');recovery=steps.get('step:2');trigger=row.get('trigger_ms') if row else None
+        number(trigger,'actual microphone trigger',nullable=True)
+        start=long.get('playout_start_ms') if long else None;end=long.get('playout_end_ms') if long else None
+        timing=(all(x is not None for x in (start,end,trigger)) and start<trigger<=end and
+            abs(trigger-start-plan['design']['interrupt_after_ms'])<=plan['design']['interrupt_tolerance_ms'] and
+            0<=end-trigger<=plan['design']['interrupt_tolerance_ms'])
+        mic=row.get('injected_audio_sha256')==plan['microphone']['sha256'] if row else False
+        recovered=bool(recovery and _successful(recovery,c) and trigger is not None and recovery['tts_start_ms'] is not None and recovery['tts_start_ms']>=trigger)
+        observed=bool(long and long['interrupted'] and long['audio_bytes']>0 and long.get('provider')==c['provider']
+            and long.get('model') in (None,c['model']) and timing and mic)
+        result.append(dict(unit_id=unit['id'],capture_available=row is not None,microphone_identity_matches=mic,
+            interruption_timing_valid=bool(timing),terminal_interruption_observed=bool(long and long['interrupted']),
+            recovery_completed=recovered,protocol_success=observed and recovered and row['status']=='completed',
+            interruption_to_playout_end_ms=end-trigger if timing else None,
+            subjective_latency=None,scope='fixture' if plan['mode']=='fixture' else 'frozen microphone interruption design'))
+    return result
+
+
+def evaluate_rapid_turn_stress(plan,captures):
+    bound=_captures(plan,captures);candidates={c['id']:c for c in plan['candidates']};result=[]
+    for unit in plan['units']:
+        row=bound.get(unit['id']);steps={s['step_id']:s for s in row['steps']} if row else {};c=candidates[unit['candidate_id']]
+        sent=row.get('prompt_times_ms',{}) if row else {};values=[];idle=[];valid=bool(row and row['status']=='completed');previous=None
+        require(type(sent) is dict,'Invalid prompt timing observations')
+        for expected in unit['steps']:
+            s=steps.get(expected['id']);when=sent.get(expected['id']);number(when,'prompt injection time',nullable=True)
+            if s is None or not _successful(s,c) or when is None:valid=False
+            if previous is not None:
+                gap=when-previous if when is not None else None;idle.append(gap)
+                if gap is None or not 0<=gap<=plan['design']['max_idle_ms']:valid=False
+            if s:
+                a,b=s['tts_start_ms'],s['first_audio_ms']
+                clocks=(when,a,b,s['playout_start_ms'],s['playout_end_ms'])
+                if any(v is None for v in clocks) or list(clocks)!=sorted(clocks):valid=False
+                values.append(b-a if a is not None and b is not None and b>=a and not s['cache_hit'] and _successful(s,c) else None)
+                previous=s['playout_end_ms']
+                if previous is None:valid=False
+            else:values.append(None);previous=None
+        result.append(dict(unit_id=unit['id'],expected_turns=len(unit['steps']),captured_turns=len(steps),
+            timing_control_valid=valid and len(steps)==len(unit['steps']),idle_ms=idle,fresh_ttfa_ms_in_turn_order=values,
+            fitted_degradation_slope=None,confidence_interval=None,scope='fixture' if plan['mode']=='fixture' else 'timed rapid-turn campaign'))
+    return result
+
+
+def evaluate_machine_intelligibility(plan,captures,references):
+    bound=_captures(plan,captures);result=[]
+    for c in plan['candidates']:
+        units=[u for u in plan['units'] if u['candidate_id']==c['id']];scores=[]
+        for unit in units:
+            row=bound.get(unit['id']);audio=unit['audio_case'];valid=bool(row and row['status']=='completed' and row.get('audio_sha256')==audio['audio']['sha256']
+                and row.get('asr_sha256')==digest(encoded(plan['asr'])) and isinstance(row.get('hypothesis'),str))
+            score=score_transcript(references[audio['id']],row['hypothesis']) if valid else None
+            if score is not None:scores.append((audio['language'],score))
+        languages=sorted({u['audio_case']['language'] for u in units});by_language={}
+        for language in languages:
+            group=[s for lg,s in scores if lg==language];expected=sum(u['audio_case']['language']==language for u in units)
+            words=sum(s['reference_words'] for s in group);chars=sum(s['reference_characters'] for s in group)
+            by_language[language]=dict(expected=expected,scored=len(group),missing_or_invalid=expected-len(group),
+                wer=sum(s['word_errors'] for s in group)/words if words and len(group)==expected and language!='ja' else None,
+                cer=sum(s['character_errors'] for s in group)/chars if chars and len(group)==expected else None,
+                silence_inserted_words=sum(s['silence_inserted_words'] for s in group),
+                structured_tokens={k:dict(matched=sum(s['structured_tokens'][k]['matched'] for s in group),reference_tokens=sum(s['structured_tokens'][k]['reference_tokens'] for s in group)) for k in (group[0]['structured_tokens'] if group else {})})
+        result.append(dict(candidate_id=c['id'],by_language=by_language,normalization='kido.nfkc-lower-punctuation.v1',
+            structured_token_scorer='kido.structured-substring-presence.v1',human_quality=None,confidence_interval=None,
+            asr_sha256=digest(encoded(plan['asr'])),scope='fixture' if plan['mode']=='fixture' else 'fixed-ASR machine intelligibility proxy'))
+    return result
+
+
+def _grade(plan,captures,references):
+    funcs={'session-reliability':evaluate_session_reliability,'controlled-barge-in':evaluate_controlled_barge_in,'rapid-turn-stress':evaluate_rapid_turn_stress}
+    return evaluate_machine_intelligibility(plan,captures,references) if plan['protocol']=='machine-intelligibility' else funcs[plan['protocol']](plan,captures)
+
+
+def grade_speech_campaign(path,*,expected_plan_sha256,captures,output_dir):
+    root,plan,refs=_load_plan(path,expected_plan_sha256);result=_grade(plan,captures,refs)
+    invocation='speech-campaign-invocation-'+uuid.uuid4().hex
+    report=dict(schema_version=SCHEMA+'.report.v1',suite_id=SUITE_ID,invocation_id=invocation,protocol=plan['protocol'],
+        mode=plan['mode'],plan_sha256=expected_plan_sha256,results=result,real_provider_calls_made_by_grader=0,
+        captures_supplied=len(captures),captures_planned=plan['planned_units'],subjective_perceived_latency=None)
+    dest=_new_directory(output_dir);files=_write_files(dest,{'plan.json':encoded(plan),'references.json':encoded(refs),'captures.json':encoded(captures),'report.json':encoded(report)})
+    receipt=dict(schema_version=SCHEMA+'.receipt.v1',suite_id=SUITE_ID,invocation_id=invocation,files=files)
+    raw=encoded(receipt);(dest/'receipt.json').write_bytes(raw)
+    return dict(path=str(dest),suite_id=SUITE_ID,invocation_id=invocation,receipt_sha256=digest(raw),mode=plan['mode'])
+
+
+def load_speech_campaign_report(path,*,expected_receipt_sha256):
+    root=Path(path).absolute();raw=_read(root,'receipt.json');require(digest(raw)==_sha(expected_receipt_sha256),'Campaign receipt changed')
+    receipt=json_value(raw);require(receipt.get('schema_version')==SCHEMA+'.receipt.v1' and receipt.get('suite_id')==SUITE_ID and
+        set(receipt.get('files',{}))=={'plan.json','references.json','captures.json','report.json'},'Invalid campaign report census')
+    values={}
+    for p,h in receipt['files'].items():
+        b=_read(root,p);require(h==dict(sha256=digest(b),size_bytes=len(b)),'Campaign artifact changed');values[p]=json_value(b)
+    plan=values['plan.json'];report=values['report.json'];refs=values['references.json']
+    require(plan.get('schema_version')==SCHEMA+'.plan.v1' and plan.get('suite_id')==SUITE_ID and
+        report.get('schema_version')==SCHEMA+'.report.v1' and report.get('suite_id')==SUITE_ID and
+        report.get('invocation_id')==receipt.get('invocation_id') and report.get('plan_sha256')==digest(encoded(plan)) and
+        plan['files']['references.json']==dict(sha256=digest(encoded(refs)),size_bytes=len(encoded(refs))) and
+        report.get('mode')==plan['mode'] and report.get('protocol')==plan['protocol'] and
+        report.get('real_provider_calls_made_by_grader')==0 and report.get('subjective_perceived_latency') is None and
+        report.get('captures_supplied')==len(values['captures.json']) and report.get('captures_planned')==plan['planned_units'] and
+        report.get('results')==_grade(plan,values['captures.json'],refs),'Campaign report calculations or scope changed')
+    return report
+
+
+def run_fixture_speech_campaign(path,*,expected_plan_sha256,output_dir):
+    _,plan,refs=_load_plan(path,expected_plan_sha256);require(plan['mode']=='fixture','Fixtures cannot populate a real capture campaign')
+    candidates={c['id']:c for c in plan['candidates']};rows=[]
+    for unit in plan['units']:
+        c=candidates[unit['candidate_id']];steps=[];sent={};clock=0
+        for i,expected in enumerate(unit['steps']):
+            sent[expected['id']]=clock;start=clock+10
+            end=start+100 if plan['protocol']!='controlled-barge-in' or i else start+plan['design']['interrupt_after_ms']+10
+            steps.append(dict(step_id=expected['id'],case_id=expected['case_id'],provider=c['provider'],model=c['model'],audio_bytes=32,
+                complete=True,cancelled=False,error_present=False,cache_hit=False,interrupted=plan['protocol']=='controlled-barge-in' and i==0,
+                tts_start_ms=clock,first_audio_ms=start,playout_start_ms=start,playout_end_ms=end))
+            clock=end+min(10,plan['design']['max_idle_ms'])
+        row=dict(unit_id=unit['id'],candidate_id=c['id'],candidate_sha256=digest(encoded(c)),source_kind='fixture',
+            status='completed',session_id='fixture:'+unit['id'],steps=steps,prompt_times_ms=sent)
+        if plan['protocol']=='controlled-barge-in':row.update(trigger_ms=steps[0]['playout_start_ms']+plan['design']['interrupt_after_ms'],injected_audio_sha256=plan['microphone']['sha256'])
+        if plan['protocol']=='machine-intelligibility':
+            a=unit['audio_case'];row.update(audio_sha256=a['audio']['sha256'],asr_sha256=digest(encoded(plan['asr'])),hypothesis='fixture transcript')
+        rows.append(row)
+    return grade_speech_campaign(path,expected_plan_sha256=expected_plan_sha256,captures=rows,output_dir=output_dir)
+
+
+def run_speech_campaign(path,*,expected_plan_sha256,collector,output_dir,allow_external_calls=False):
+    """Run a deliberately supplied, source-hashed trusted-local collector.
+
+    The collector owns provider/session integration, dependency review and
+    request deadlines. This is not an OS sandbox or a provider implementation.
+    No collector is imported or discovered by the SDK. Input views omit gold.
+    """
+    import inspect
+    root,plan,_=_load_plan(path,expected_plan_sha256)
+    require(plan['mode']=='capture' and type(allow_external_calls) is bool and allow_external_calls,
+        'Explicit external campaign launch is required')
+    require(callable(collector) and plan.get('collector_source_sha256') is not None,'Freeze the reviewed collector source hash')
+    file=inspect.getsourcefile(collector);require(file is not None,'Collector source must be inspectable')
+    file=Path(file).absolute();require(digest(_read(file.parent,file.name))==plan['collector_source_sha256'],'Collector source differs from reviewed plan')
+    requests=speech_campaign_requests(root,expected_plan_sha256=expected_plan_sha256)
+    dest=_new_directory(output_dir);candidates={c['id']:c for c in plan['candidates']};captures=[]
+    for i,unit in enumerate(plan['units']):
+        assets=dest/'collector-inputs'/('unit-'+str(i+1)+'-assets');assets.mkdir(parents=True)
+        names=[unit['audio_case']['audio_file']] if 'audio_case' in unit else ['audio/microphone.wav'] if plan['microphone'] else []
+        files={p:requests['input_files'][p] for p in names}
+        _write_files(assets,{p:_read(root,p) for p in files})
+        request=dict(protocol=plan['protocol'],plan_sha256=expected_plan_sha256,unit=unit,
+            candidate=candidates[unit['candidate_id']],design=plan['design'],asr=plan['asr'],
+            microphone=plan['microphone'],input_files=files,input_root=str(assets))
+        _write_files(dest,{'collector-inputs/unit-'+str(i+1)+'.json':encoded(request)})
+        try:row=collector(request)
+        except Exception:
+            row=dict(unit_id=unit['id'],candidate_id=unit['candidate_id'],candidate_sha256=digest(encoded(candidates[unit['candidate_id']])),
+                source_kind='operator-capture',status='collector_error',session_id=None,steps=[])
+        _captures(plan,[row]);captures.append(row)
+        _write_files(dest,{'collector-captures/unit-'+str(i+1)+'.json':encoded(row)})
+    return grade_speech_campaign(root,expected_plan_sha256=expected_plan_sha256,captures=captures,output_dir=dest/'evaluation')
+
+
+def main(argv=None):
+    p=argparse.ArgumentParser(description=__doc__);commands=p.add_subparsers(dest='command',required=True)
+    commands.add_parser('describe')
+    c=commands.add_parser('freeze');c.add_argument('--bundle',required=True);c.add_argument('--expected-manifest-sha256',required=True);c.add_argument('--design-json',required=True);c.add_argument('--output',required=True)
+    for name in ('requests','fixture','grade'):
+        c=commands.add_parser(name);c.add_argument('--plan',required=True);c.add_argument('--expected-plan-sha256',required=True)
+        if name!='requests':c.add_argument('--output',required=True)
+        if name=='grade':c.add_argument('--captures-json',required=True)
+    args=p.parse_args(argv)
+    try:
+        if args.command=='describe':result=dict(suite_id=SUITE_ID,protocols=PROTOCOLS,provider_calls=False,live_collector='separately reviewed external capture')
+        elif args.command=='freeze':
+            from .speech_complete import load_kido_complete
+            b=load_kido_complete(args.bundle,expected_manifest_sha256=args.expected_manifest_sha256)
+            result=freeze_speech_campaign(b,output_dir=args.output,**json_value(_read(Path(args.design_json).absolute().parent,Path(args.design_json).name)))
+        elif args.command=='requests':result=speech_campaign_requests(args.plan,expected_plan_sha256=args.expected_plan_sha256)
+        elif args.command=='fixture':result=run_fixture_speech_campaign(args.plan,expected_plan_sha256=args.expected_plan_sha256,output_dir=args.output)
+        else:result=grade_speech_campaign(args.plan,expected_plan_sha256=args.expected_plan_sha256,output_dir=args.output,
+            captures=json_value(_read(Path(args.captures_json).absolute().parent,Path(args.captures_json).name)))
+    except (ValueError,OSError,KeyError,TypeError):p.exit(2,'Speech campaign failed; check frozen design, capture identities and output.\n')
+    print(json.dumps(result,indent=2,ensure_ascii=True))
+
+
+if __name__=='__main__':main()

sphynx/speech_complete.py

@@ -0,0 +1,236 @@
+"""Exhaustive Kido benchmark catalog and retained-evidence evaluation.
+
+Post-0.5.0 source only. Separate invocation; no provider or imported-code runs.
+"""
+from dataclasses import dataclass
+from pathlib import Path
+import argparse
+import uuid
+
+from ._speech_kido import digest,encoded,json_value,require,csv_rows
+from . import _speech_complete_source as source
+from ._speech_complete_scoring import (evaluate_google_direct_api,evaluate_pronunciation_proxy,
+    evaluate_isolated_diagnostics,evaluate_listening_diagnostics,evaluate_pt_br_categories,evaluate_official_turns,evaluate_stt_diagnostics)
+from ._speech_benchmark_scoring import grade_records
+from .speech_evidence import _new_directory,_read,_write_files,_sha,_grade_rows
+
+SUITE_ID='speech.kido-complete.v1'
+SCHEMA='sphynx.speech.kido-complete'
+MODULES=('_speech_complete_source.py','_speech_complete_scoring.py','speech_complete.py',
+    '_speech_kido.py','_speech_scoring.py','speech_evidence.py','_speech_benchmark_sources.py','_speech_benchmark_scoring.py','speech_contracts.py')
+LIMITS=dict(historical_only=True,real_provider_calls=0,hosted_admitted=False,
+    population_inference=False,new_waveform_quality=False,subjective_perceived_latency=False)
+SOURCE_FAMILIES={
+    'results/human-eval/':'blind_quality',
+    'results/isolated/official-v1/':'isolated_synthesis',
+    'results/isolated/multilingual-extension-v1/':'multilingual_isolated',
+    'results/live-agent/official-v1/':'live_agent_synthesis',
+    'results/live-agent/reliability-v1/':'session_reliability',
+    'results/live-agent/pronunciation-v1/':'pronunciation',
+    'results/live-agent/language-switching-v1/':'language_switching',
+    'results/exploratory/cartesia-ink-2-stt/':'stt_recognition',
+    'results/exploratory/google-gemini-tts/':'google_direct_api',
+    'results/exploratory/pronunciation-v1-cartesia-only/':'pronunciation_proxy',
+}
+# Every methodology document has an explicit home. A plan is not an executed study.
+CATALOG=(
+ ('isolated_synthesis','saved-evidence','isolated-benchmark.md'),
+ ('blind_quality','saved-evidence','human-eval.md'),
+ ('pt_br_listening','saved-evidence','pt-br-listening-pack.md'),
+ ('live_agent_synthesis','saved-evidence','live-agent.md'),
+ ('stt_recognition','saved-evidence','cartesia-ink2-exploratory.md'),
+ ('stt_diagnostics','saved-evidence','cartesia-ink2-exploratory.md'),
+ ('stt_turn_lifecycle','saved-evidence','live-telemetry-schema.md'),
+ ('pronunciation','saved-evidence','pronunciation.md'),
+ ('language_switching','saved-evidence','language-switching.md'),
+ ('voice_continuity','saved-evidence','language-switching.md'),
+ ('code_switching','saved-evidence','multilingual.md'),
+ ('playout_interruptions','saved-evidence','barge-in-and-rapid-turns.md'),
+ ('google_direct_api','saved-evidence','google-direct-api-extension.md'),
+ ('pronunciation_proxy','saved-evidence','pronunciation.md'),
+ ('isolated_diagnostics','saved-evidence','percentile-conventions.md'),
+ ('listening_diagnostics','saved-evidence','corpus.md'),
+ ('pt_br_categories','saved-evidence','pt-br-listening-pack.md'),
+ ('official_turn_diagnostics','saved-evidence','barge-in-and-rapid-turns.md'),
+ ('multilingual_isolated','campaign','multilingual.md'),
+ ('session_reliability','campaign','reliability.md'),
+ ('machine_intelligibility','campaign','intelligibility.md'),
+ ('controlled_barge_in','campaign','barge-in-and-rapid-turns.md'),
+ ('rapid_turn_stress','campaign','barge-in-and-rapid-turns.md'),
+ ('third_party_stt_research','reference-only','cartesia-ink2-exploratory.md'),
+ ('pronunciation_customization','capability-only','pronunciation.md'),
+ ('protocol_overview','reference-only','protocol.md'),
+)
+
+
+def list_kido_evaluations():
+    return dict(suite_id=SUITE_ID,source_commit=source.SOURCE_COMMIT,source_lock_sha256=source.SOURCE_PIN,
+        invocation_schema=SCHEMA+'.invocation.v1',catalog=[dict(id=k,surface=s,methodology='methodology/'+p) for k,s,p in CATALOG],
+        source_families=SOURCE_FAMILIES,
+        execution='retained evidence only; campaign execution has a separate invocation',
+        implementation_sha256={name:digest(Path(__file__).with_name(name).read_bytes()) for name in MODULES},limits=LIMITS)
+
+
+def export_kido_complete_source(repository,output_dir):
+    lock,data=source.git_bytes(repository);dest=_new_directory(output_dir);_write_files(dest,data)
+    return dict(path=str(dest),source_commit=source.SOURCE_COMMIT,source_lock_sha256=digest(encoded(lock)),files=len(data),source_executed=False)
+
+
+def import_kido_complete(source_path,output_dir):
+    lock,data=source.source_bytes(source_path);evidence=source.normalize(data)
+    evidence['isolated_trials']=csv_rows(data['results/isolated/official-v1/combined/trials.csv'])
+    methodologies={p for p in data if p.startswith('methodology/') and p.endswith('.md')}
+    require(methodologies<={'methodology/'+p for _,_,p in CATALOG},'Uncatalogued methodology document')
+    require(all(any(p.startswith(prefix) for prefix in SOURCE_FAMILIES) or p in
+        ('results/live-agent/README.md','results/README.md') for p in data if p.startswith('results/')),
+        'Uncatalogued result family')
+    dest=_new_directory(output_dir);files=_write_files(dest,{'source/'+p:b for p,b in data.items()}|{'evidence.json':encoded(evidence)})
+    manifest=dict(schema_version=SCHEMA+'.bundle.v1',suite_id=SUITE_ID,source_lock_sha256=digest(encoded(lock)),
+        source_commit=source.SOURCE_COMMIT,files=files,provider_called=False,candidate_executed=False,hosted_admitted=False)
+    raw=encoded(manifest);(dest/'manifest.json').write_bytes(raw)
+    return dict(path=str(dest),manifest_sha256=digest(raw),suite_id=SUITE_ID,source_files=len(data))
+
+
+@dataclass(frozen=True)
+class KidoComplete:
+    path:Path
+    manifest_sha256:str
+
+
+def _verified(bundle):
+    raw=_read(bundle.path,'manifest.json');require(digest(raw)==bundle.manifest_sha256,'Complete manifest changed')
+    manifest=json_value(raw)
+    require(manifest.get('schema_version')==SCHEMA+'.bundle.v1' and manifest.get('suite_id')==SUITE_ID and
+        manifest.get('source_lock_sha256')==source.SOURCE_PIN and manifest.get('source_commit')==source.SOURCE_COMMIT,
+        'Not the pinned complete benchmark source')
+    _,data=source.source_bytes(bundle.path/'source')
+    require(set(manifest.get('files',{}))=={'source/'+p for p in data}|{'evidence.json'},'Complete file census changed')
+    for p,h in manifest['files'].items():
+        b=_read(bundle.path,p);require(h==dict(sha256=digest(b),size_bytes=len(b)),'Complete artifact changed')
+    evidence=source.normalize(data);evidence['isolated_trials']=csv_rows(data['results/isolated/official-v1/combined/trials.csv'])
+    require(_read(bundle.path,'evidence.json')==encoded(evidence),'Complete evidence differs from source')
+    require(all(manifest.get(k) is False for k in ('provider_called','candidate_executed','hosted_admitted')),'Complete execution boundary changed')
+    return evidence
+
+
+def load_kido_complete(path,*,expected_manifest_sha256):
+    bundle=KidoComplete(Path(path).absolute(),_sha(expected_manifest_sha256));_verified(bundle);return bundle
+
+
+def _results(evidence):
+    saved=evidence['saved'];summary,cases=_grade_rows(saved['observations'],saved['references'])
+    pt=[r for r in evidence['benchmarks']['observations'] if r['track']=='pt_br_listening']
+    return dict(saved_evidence=dict(summaries=summary,per_case=cases,audit=saved['audit']),
+        benchmarks=grade_records(evidence['benchmarks']),google_direct_api=evaluate_google_direct_api(evidence['google']),
+        pronunciation_proxy=evaluate_pronunciation_proxy(evidence['proxy']),
+        isolated_diagnostics=evaluate_isolated_diagnostics(evidence['isolated_trials']),
+        listening_diagnostics=evaluate_listening_diagnostics(saved['observations'],saved['references'],evidence['isolated_trials']),
+        pt_br_categories=evaluate_pt_br_categories(pt),official_turn_diagnostics=evaluate_official_turns(evidence['official_sessions']),
+        stt_diagnostics=evaluate_stt_diagnostics(evidence['stt_analyses']),
+        unexecuted_campaigns=[k for k,s,_ in CATALOG if s=='campaign'],
+        source_documents=evidence['source_documents'],
+        nonexperimental_references=[k for k,s,_ in CATALOG if s in ('reference-only','capability-only')])
+
+
+def grade_kido_complete(bundle,output_dir):
+    evidence=_verified(bundle);definition=list_kido_evaluations()
+    invocation=dict(schema_version=SCHEMA+'.invocation.v1',id='speech-complete-invocation-'+uuid.uuid4().hex,
+        suite_id=SUITE_ID,bundle_manifest_sha256=bundle.manifest_sha256,definition=definition,provider_called=False,candidate_executed=False)
+    report=dict(schema_version=SCHEMA+'.report.v1',suite_id=SUITE_ID,invocation_id=invocation['id'],
+        invocation_sha256=digest(encoded(invocation)),results=_results(evidence),limits=LIMITS)
+    dest=_new_directory(output_dir);files=_write_files(dest,{'evidence.json':encoded(evidence),'invocation.json':encoded(invocation),'report.json':encoded(report)})
+    receipt=dict(schema_version=SCHEMA+'.receipt.v1',suite_id=SUITE_ID,invocation_id=invocation['id'],files=files)
+    raw=encoded(receipt);(dest/'receipt.json').write_bytes(raw)
+    return dict(path=str(dest),receipt_sha256=digest(raw),invocation_id=invocation['id'],suite_id=SUITE_ID)
+
+
+def load_kido_complete_report(path,*,expected_receipt_sha256):
+    root=Path(path).absolute();raw=_read(root,'receipt.json')
+    require(digest(raw)==_sha(expected_receipt_sha256),'Complete report receipt changed');receipt=json_value(raw)
+    require(receipt.get('schema_version')==SCHEMA+'.receipt.v1' and receipt.get('suite_id')==SUITE_ID and
+        set(receipt.get('files',{}))=={'evidence.json','invocation.json','report.json'},'Not a complete benchmark report')
+    payload={}
+    for p,h in receipt['files'].items():
+        b=_read(root,p);require(h==dict(sha256=digest(b),size_bytes=len(b)),'Complete report artifact changed');payload[p]=json_value(b)
+    invocation=payload['invocation.json'];report=payload['report.json']
+    require(invocation.get('schema_version')==SCHEMA+'.invocation.v1' and report.get('schema_version')==SCHEMA+'.report.v1' and
+        invocation.get('suite_id')==report.get('suite_id')==SUITE_ID and invocation.get('id')==report.get('invocation_id')==receipt.get('invocation_id') and
+        report.get('invocation_sha256')==digest(encoded(invocation)) and invocation.get('provider_called') is False and invocation.get('candidate_executed') is False,
+        'Complete invocation identity or execution boundary changed')
+    definition=invocation.get('definition',{})
+    require(definition.get('source_lock_sha256')==source.SOURCE_PIN and definition.get('source_commit')==source.SOURCE_COMMIT
+        and definition.get('catalog')==list_kido_evaluations()['catalog'] and definition.get('source_families')==SOURCE_FAMILIES
+        and definition.get('limits')==LIMITS,'Complete source or catalog changed')
+    require(set(definition.get('implementation_sha256',{}))==set(MODULES),'Missing complete implementation identities')
+    for v in definition['implementation_sha256'].values():_sha(v)
+    require(report.get('limits')==LIMITS and report.get('results')==_results(payload['evidence.json']),'Complete calculations or limits changed')
+    return report
+
+
+def complete_tts_cases(bundle,*,benchmark,case_ids=None,profile=None,language_hint=None):
+    """Input-only cases for existing reviewed candidate execution or campaign plans."""
+    evidence=_verified(bundle);cases=[]
+    if benchmark=='isolated-official':
+        unique={}
+        for r in evidence['saved']['observations']:
+            if r['track']!='tts_isolated':continue
+            value=dict(id='isolated:'+r['case_id'],text=r['input_text'],language=r['language'])
+            require(value['id'] not in unique or unique[value['id']]==value,'Conflicting original corpus identity')
+            unique[value['id']]=value
+        cases=sorted(unique.values(),key=lambda r:r['id'])
+    elif benchmark=='official-live-script':
+        cases=[{k:r[k] for k in ('id','text','language')} for r in evidence['official_script']]
+    elif benchmark=='multilingual-isolated':
+        rows=evidence['multilingual']['lines']
+        support={'elevenlabs':{'en','pt-BR','es','fr','ja'},'cartesia':{'en','pt-BR','es','fr','ja'},
+            'minimax':{'en','pt-BR','es','fr','ja'},'rime':{'en','es','fr'},'deepgram':{'en'}}
+        require(profile is None or profile in support,'No frozen language support matrix for this profile')
+        cases=[dict(id='multilingual:'+r['id'],text=r['text'],language=r['language']) for r in rows if profile is None or r['language'] in support[profile]]
+    elif benchmark=='google-direct-api':
+        cases=[dict(id='google-probe:'+k,text=v,language='pt-BR' if k in ('short_pt','long_pt','numbers_pt','mixed_pt_en') else 'en') for k,v in evidence['google_texts'].items()]
+    elif benchmark=='pronunciation-proxy':
+        cases=[dict(id='pronunciation-proxy:'+r['id'],text=r['reference'],language='en') for r in evidence['proxy']]
+    else:
+        require(benchmark in ('pt-br-listening','pronunciation','code-switching'),'Unknown complete TTS corpus')
+        cases=[dict(r,id=r['id'].replace('/',':',1)) for r in evidence['benchmarks']['cases'] if r['id'].startswith(benchmark+'/')]
+    require(profile is None or benchmark=='multilingual-isolated','Profile filtering applies only to frozen multilingual support')
+    if benchmark=='code-switching':
+        import re
+        require(isinstance(language_hint,str) and re.fullmatch('[a-z]{2}',language_hint),'Select an explicit mixed-language hint')
+        for row in cases:row['language']=language_hint
+    else:require(language_hint is None,'Native corpus language cannot be overridden')
+    from .speech_contracts import identifier
+    for row in cases:identifier(row['id'])
+    if case_ids is not None:
+        require(type(case_ids) is list and len(set(case_ids))==len(case_ids) and set(case_ids)<={r['id'] for r in cases},'Invalid complete case selection')
+        cases=[r for r in cases if r['id'] in case_ids]
+    require(cases,'Empty complete case selection')
+    return dict(cases=cases,cohort_identity=dict(kind=SUITE_ID+':'+benchmark,sha256=digest(encoded(dict(bundle=bundle.manifest_sha256,cases=cases)))),
+        suite_id=SUITE_ID,new_waveform_ratings_available=False,adaptation='TTS contract PCM16 16k; not historical transport parity')
+
+
+def main(argv=None):
+    import json
+    parser=argparse.ArgumentParser(description=__doc__);commands=parser.add_subparsers(dest='command',required=True)
+    commands.add_parser('describe')
+    c=commands.add_parser('export-source');c.add_argument('--repository',required=True);c.add_argument('--output',required=True)
+    c=commands.add_parser('import-kido');c.add_argument('--source',required=True);c.add_argument('--output',required=True)
+    for name in ('grade','tts-cases'):
+        c=commands.add_parser(name);c.add_argument('--bundle',required=True);c.add_argument('--expected-manifest-sha256',required=True)
+        if name=='grade':c.add_argument('--output',required=True)
+        else:
+            c.add_argument('--benchmark',required=True);c.add_argument('--case-id',action='append');c.add_argument('--profile');c.add_argument('--language-hint')
+    args=parser.parse_args(argv)
+    try:
+        if args.command=='describe':result=list_kido_evaluations()
+        elif args.command=='export-source':result=export_kido_complete_source(args.repository,args.output)
+        elif args.command=='import-kido':result=import_kido_complete(args.source,args.output)
+        else:
+            bundle=load_kido_complete(args.bundle,expected_manifest_sha256=args.expected_manifest_sha256)
+            result=grade_kido_complete(bundle,args.output) if args.command=='grade' else complete_tts_cases(bundle,
+                benchmark=args.benchmark,case_ids=args.case_id,profile=args.profile,language_hint=args.language_hint)
+    except (ValueError,OSError,KeyError,TypeError):parser.exit(2,'Complete speech operation failed; check source pin, selection and output.\n')
+    print(json.dumps(result,indent=2,ensure_ascii=True))
+
+
+if __name__=='__main__':main()