sable-sphynx 0.7.0.dev0__py3-none-any.whl

sphynx/speech_listening.py

@@ -0,0 +1,115 @@
+"""Blind packs bound to new TTS waveforms; explicit human reviews only."""
+from collections import defaultdict
+import csv
+import io
+from pathlib import Path
+import random
+import statistics
+import uuid
+
+from ._speech_kido import digest,encoded,json_value,csv_rows,require
+from ._speech_benchmark_sources import DIMENSIONS
+from .speech_evidence import _new_directory,_read,_write_files,_sha
+from .speech_tts import load_tts_run,_load_plan
+
+SUITE_ID='speech.listening.v1'
+
+
+def prepare_speech_listening_pack(run_path,*,expected_manifest_sha256,output_dir,protocol='blind-quality',seed=20260921,repetition=1,case_ids=None):
+    require(protocol in ('blind-quality','pt-br-listening') and type(seed) is int and type(repetition) is int and repetition>=1,'Invalid listening design')
+    root=Path(run_path).absolute();load_tts_run(root,expected_manifest_sha256=expected_manifest_sha256)
+    manifest=json_value(_read(root,'manifest.json'));_,plan=_load_plan(root/'frozen-plan',manifest['plan_sha256'])
+    cases={r['id']:r for r in plan['cases']}
+    require(case_ids is None or type(case_ids) is list and len(set(case_ids))==len(case_ids) and set(case_ids)<=set(cases),'Invalid listening case selection')
+    selected=set(cases) if case_ids is None else set(case_ids);require(selected,'Empty listening pack')
+    if protocol=='pt-br-listening':require(all(cases[i]['language']=='pt-BR' for i in selected),'PT-BR pack contains another language')
+    captures=[(p,json_value(_read(root,p))) for p in manifest['files'] if p.startswith('captures/') and p.endswith('/capture.json')]
+    captures=[(p,r) for p,r in captures if r['repetition']==repetition and r['case_id'] in selected]
+    require(len(captures)==2*len(selected) and len({(r['case_id'],r['method_id']) for _,r in captures})==len(captures)
+        and all(r['success'] for _,r in captures),'Listening pack requires both successful arms for every selected text')
+    random.Random(seed).shuffle(captures);pack_id='listening-pack-'+uuid.uuid4().hex;mapping=[];payload={};rows=[]
+    dimensions=list(DIMENSIONS) if protocol=='pt-br-listening' else ['overall_quality_1_5']
+    for index,(path,capture) in enumerate(captures):
+        sample='sample-'+str(index+1).zfill(3);audio=_read(root,path.removesuffix('capture.json')+'audio.wav');sha=digest(audio)
+        payload['blind/audio/'+sample+'.wav']=audio
+        case=cases[capture['case_id']];method=plan['methods'][capture['method_id']]
+        mapping.append(dict(sample_id=sample,case_id=case['id'],method_id=capture['method_id'],provider=method['provider'],model=method['model'],
+            audio_sha256=sha,request_id=capture['request_id'],text=case['text'],language=case['language']))
+        rows.append(dict(pack_id=pack_id,sample_id=sample,audio_sha256=sha,text=case['text'],language=case['language'],**{d:'' for d in dimensions},notes=''))
+    out=io.StringIO(newline='');writer=csv.DictWriter(out,fieldnames=list(rows[0]));writer.writeheader();writer.writerows(rows)
+    payload['blind/scores.csv']=out.getvalue().encode('utf-8');payload['mapping.json']=encoded(mapping)
+    dest=_new_directory(output_dir);files=_write_files(dest,payload)
+    pack=dict(schema_version='sphynx.speech.listening.pack.v1',suite_id=SUITE_ID,pack_id=pack_id,protocol=protocol,dimensions=dimensions,
+        source_run_sha256=expected_manifest_sha256,source_mode=plan['mode'],seed=seed,repetition=repetition,files=files,
+        samples=len(rows),quality_ratings_available=False,historical_ratings_reused=False)
+    raw=encoded(pack);(dest/'manifest.json').write_bytes(raw)
+    return dict(path=str(dest),pack_sha256=digest(raw),suite_id=SUITE_ID,pack_id=pack_id,samples=len(rows),distribute='blind directory only; keep mapping private')
+
+
+def _pack(path,pin):
+    root=Path(path).absolute();raw=_read(root,'manifest.json');require(digest(raw)==_sha(pin),'Listening pack changed');pack=json_value(raw)
+    require(pack.get('schema_version')=='sphynx.speech.listening.pack.v1' and pack.get('suite_id')==SUITE_ID and
+        pack.get('historical_ratings_reused') is False and pack.get('quality_ratings_available') is False,'Invalid new-waveform listening pack')
+    for p,h in pack['files'].items():
+        b=_read(root,p);require(h==dict(sha256=digest(b),size_bytes=len(b)),'Listening audio, mapping or template changed')
+    mapping=json_value(_read(root,'mapping.json'))
+    require(len(mapping)==pack['samples'] and len({r['sample_id'] for r in mapping})==len(mapping),'Listening mapping census differs')
+    for row in mapping:require(digest(_read(root,'blind/audio/'+row['sample_id']+'.wav'))==row['audio_sha256'],'Listening waveform binding differs')
+    return root,pack,mapping
+
+
+def _grade(pack,mapping,reviews):
+    require(pack.get('schema_version')=='sphynx.speech.listening.pack.v1' and pack.get('suite_id')==SUITE_ID and
+        pack.get('protocol') in ('blind-quality','pt-br-listening') and pack.get('source_mode') in ('fixture','provider') and
+        pack.get('historical_ratings_reused') is False and pack.get('quality_ratings_available') is False and
+        pack.get('dimensions')==(list(DIMENSIONS) if pack['protocol']=='pt-br-listening' else ['overall_quality_1_5']),
+        'Listening definition or evidence boundary differs')
+    require(type(reviews) is list and reviews and len({r['rater_id'] for r in reviews})==len(reviews),'Declare unique review identities')
+    by_id={r['sample_id']:r for r in mapping};groups=defaultdict(list)
+    for review in reviews:
+        require(isinstance(review['rater_id'],str) and review['rater_id'] and type(review['rows']) is list,'Invalid rater review')
+        rows=review['rows'];require(len({r['sample_id'] for r in rows})==len(rows) and {r['sample_id'] for r in rows}<=set(by_id),'Unknown or duplicated blind review sample')
+        supplied={r['sample_id']:r for r in rows}
+        for sample,identity in by_id.items():
+            row=supplied.get(sample)
+            if row:require(row.get('pack_id')==pack['pack_id'] and row.get('audio_sha256')==identity['audio_sha256'] and
+                row.get('text')==identity['text'] and row.get('language')==identity['language'],'Review belongs to another pack, waveform or text')
+            for dimension in pack['dimensions']:
+                value=row.get(dimension,'') if row else ''
+                require(value in ('','1','2','3','4','5'),'Human rating must be blank or an integer from one to five')
+                groups[(identity['method_id'],dimension)].append(int(value) if value else None)
+    results=[]
+    for (method,dimension),values in sorted(groups.items()):
+        present=[v for v in values if v is not None]
+        results.append(dict(method_id=method,dimension=dimension,expected_reviews=len(values),rated=len(present),pending=len(values)-len(present),
+            mean=statistics.mean(present) if len(present)==len(values) else None,observed_subset_mean=statistics.mean(present) if present else None,
+            declared_raters=len(reviews),independent_raters_verified=False,confidence_interval=None))
+    return results
+
+
+def grade_speech_listening_pack(path,*,expected_pack_sha256,reviews,output_dir):
+    """reviews is [{rater_id, rows}], with rows read from a completed CSV."""
+    _,pack,mapping=_pack(path,expected_pack_sha256);results=_grade(pack,mapping,reviews)
+    report=dict(schema_version='sphynx.speech.listening.report.v1',suite_id=SUITE_ID,
+        invocation_id='listening-invocation-'+uuid.uuid4().hex,pack_sha256=expected_pack_sha256,results=results,
+        source_mode=pack['source_mode'],historical_ratings_reused=False,subjective_latency_inferred=False)
+    dest=_new_directory(output_dir);files=_write_files(dest,{'pack.json':encoded(pack),'mapping.json':encoded(mapping),'reviews.json':encoded(reviews),'report.json':encoded(report)})
+    receipt=dict(schema_version='sphynx.speech.listening.receipt.v1',suite_id=SUITE_ID,files=files,invocation_id=report['invocation_id'])
+    raw=encoded(receipt);(dest/'receipt.json').write_bytes(raw)
+    return dict(path=str(dest),receipt_sha256=digest(raw),suite_id=SUITE_ID,invocation_id=report['invocation_id'])
+
+
+def load_speech_listening_report(path,*,expected_receipt_sha256):
+    root=Path(path).absolute();raw=_read(root,'receipt.json');require(digest(raw)==_sha(expected_receipt_sha256),'Listening receipt changed');receipt=json_value(raw)
+    require(receipt.get('schema_version')=='sphynx.speech.listening.receipt.v1' and receipt.get('suite_id')==SUITE_ID and
+        set(receipt.get('files',{}))=={'pack.json','mapping.json','reviews.json','report.json'},'Invalid listening report')
+    values={}
+    for p,h in receipt['files'].items():
+        b=_read(root,p);require(h==dict(sha256=digest(b),size_bytes=len(b)),'Listening report artifact changed');values[p]=json_value(b)
+    pack=values['pack.json'];report=values['report.json']
+    require(report.get('schema_version')=='sphynx.speech.listening.report.v1' and report.get('suite_id')==SUITE_ID and
+        report.get('invocation_id')==receipt.get('invocation_id') and report.get('pack_sha256')==digest(encoded(pack)) and
+        pack['files']['mapping.json']==dict(sha256=digest(encoded(values['mapping.json'])),size_bytes=len(encoded(values['mapping.json']))) and
+        report.get('historical_ratings_reused') is False and report.get('subjective_latency_inferred') is False and report.get('source_mode')==pack['source_mode'] and
+        report.get('results')==_grade(pack,values['mapping.json'],values['reviews.json']),'Listening arithmetic, identity or scope changed')
+    return report

sphynx/speech_multilingual.py

@@ -0,0 +1,71 @@
+"""Separate frozen multilingual TTS campaigns using the existing TTS runner."""
+from pathlib import Path
+import uuid
+from ._speech_kido import encoded,digest,json_value,require
+from .speech_evidence import _new_directory,_read,_write_files,_sha
+from .speech_complete import complete_tts_cases
+from .speech_tts import freeze_tts_plan,run_tts_plan,load_tts_run,_load_plan
+
+SUITE_ID='speech.multilingual.execution.v1'
+
+
+def freeze_multilingual_tts_campaign(bundle,repository,*,source_commit,source_pr,baseline,candidate,
+        output_dir,case_ids=None,repetitions=3,max_requests=20,baseline_profile=None,candidate_profile=None):
+    """Default to the frozen source's three repetitions, within one total cap.
+
+    Both arms use only the shared, historically supported language subset. The
+    profile filter is an explicit operator declaration, not live capability proof.
+    """
+    views=[complete_tts_cases(bundle,benchmark='multilingual-isolated',case_ids=None,profile=p) for p in (baseline_profile,candidate_profile)]
+    shared={r['id'] for r in views[0]['cases']}&{r['id'] for r in views[1]['cases']}
+    require(case_ids is None or type(case_ids) is list and len(set(case_ids))==len(case_ids) and set(case_ids)<=shared,'Invalid or unsupported multilingual selection')
+    cases=[r for r in views[0]['cases'] if r['id'] in shared and (case_ids is None or r['id'] in case_ids)]
+    require(cases and type(repetitions) is int and 1<=repetitions<=3,'Invalid multilingual repetition design')
+    require(type(max_requests) is int and 1<=max_requests<=1000 and len(cases)*repetitions*2<=max_requests,'Multilingual campaign exceeds total request cap')
+    per_batch=20//(repetitions*2);dest=_new_directory(output_dir);plans=[]
+    for i in range(0,len(cases),per_batch):
+        chunk=cases[i:i+per_batch]
+        plans.append(freeze_tts_plan(repository,source_commit=source_commit,source_pr=source_pr,cases=chunk,
+            cohort_identity=dict(kind=SUITE_ID,sha256=digest(encoded(dict(bundle=bundle.manifest_sha256,cases=chunk)))),
+            baseline=baseline,candidate=candidate,output_dir=dest/('batch-'+str(len(plans)+1)),repetitions=repetitions))
+    manifest=dict(schema_version='sphynx.speech.multilingual.plan.v1',suite_id=SUITE_ID,
+        bundle_manifest_sha256=bundle.manifest_sha256,plans=plans,total_requests=len(cases)*repetitions*2,
+        max_requests=max_requests,baseline_profile=baseline_profile,candidate_profile=candidate_profile,
+        historical_support_filter_only=True,native_speaker_validation_available=False,provider_calls_made=False)
+    raw=encoded(manifest);(dest/'plan.json').write_bytes(raw)
+    return dict(path=str(dest),plan_sha256=digest(raw),total_requests=manifest['total_requests'],batches=len(plans),suite_id=SUITE_ID)
+
+
+def run_multilingual_tts_campaign(path,*,expected_plan_sha256,output_dir,allow_provider_calls=False):
+    root=Path(path).absolute();raw=_read(root,'plan.json');require(digest(raw)==_sha(expected_plan_sha256),'Multilingual plan changed')
+    plan=json_value(raw);require(plan.get('schema_version')=='sphynx.speech.multilingual.plan.v1' and plan.get('suite_id')==SUITE_ID and
+        0<plan['total_requests']<=plan['max_requests'] and plan.get('provider_calls_made') is False,'Invalid multilingual plan')
+    verified=[]
+    # Verify every subplan and the total budget before starting the first arm.
+    for i,sub in enumerate(plan['plans']):
+        batch=root/('batch-'+str(i+1));_,p=_load_plan(batch,sub['plan_sha256']);verified.append((batch,p,sub))
+    require(sum(p['budget']['planned_requests'] for _,p,_ in verified)==plan['total_requests'],'Multilingual request budget changed')
+    require(allow_provider_calls or all(p['mode']=='fixture' for _,p,_ in verified),'Explicit provider launch required for the entire campaign')
+    dest=_new_directory(output_dir);runs=[]
+    for i,(batch,p,sub) in enumerate(verified):
+        r=run_tts_plan(batch,expected_plan_sha256=sub['plan_sha256'],output_dir=dest/('batch-'+str(i+1)),allow_provider_calls=allow_provider_calls)
+        load_tts_run(r['path'],expected_manifest_sha256=r['manifest_sha256']);runs.append(r)
+    receipt=dict(schema_version='sphynx.speech.multilingual.run.v1',suite_id=SUITE_ID,
+        invocation_id='multilingual-invocation-'+uuid.uuid4().hex,plan_sha256=expected_plan_sha256,
+        status='completed' if all(r['status']=='completed' for r in runs) else 'completed_with_failures',runs=runs,
+        planned_requests=plan['total_requests'],quality=None,subjective_perceived_latency=None)
+    raw=encoded(receipt);(dest/'receipt.json').write_bytes(raw)
+    return dict(path=str(dest),receipt_sha256=digest(raw),status=receipt['status'],suite_id=SUITE_ID,invocation_id=receipt['invocation_id'])
+
+
+def load_multilingual_tts_campaign(path,*,expected_receipt_sha256):
+    root=Path(path).absolute();raw=_read(root,'receipt.json');require(digest(raw)==_sha(expected_receipt_sha256),'Multilingual run receipt changed')
+    receipt=json_value(raw);require(receipt.get('schema_version')=='sphynx.speech.multilingual.run.v1' and receipt.get('suite_id')==SUITE_ID,'Invalid multilingual run')
+    reports=[];planned=0
+    for i,r in enumerate(receipt['runs']):
+        batch=root/('batch-'+str(i+1));reports.append(load_tts_run(batch,expected_manifest_sha256=r['manifest_sha256']))
+        manifest=json_value(_read(batch,'manifest.json'));planned+=manifest['planned_requests']
+        require(r.get('status')==manifest['status'] and r.get('invocation_id')==manifest['invocation_id'],'Multilingual batch status or identity changed')
+    require(receipt['status']==('completed' if all(r['status']=='completed' for r in receipt['runs']) else 'completed_with_failures') and
+        receipt.get('quality') is None and receipt.get('subjective_perceived_latency') is None and receipt.get('planned_requests')==planned,'Multilingual scope changed')
+    return dict(receipt=receipt,reports=reports)