sable-sphynx 0.7.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sable_sphynx-0.7.0.dev0.dist-info/METADATA +152 -0
- sable_sphynx-0.7.0.dev0.dist-info/RECORD +174 -0
- sable_sphynx-0.7.0.dev0.dist-info/WHEEL +5 -0
- sable_sphynx-0.7.0.dev0.dist-info/top_level.txt +1 -0
- sphynx/__init__.py +469 -0
- sphynx/__init__.pyi +376 -0
- sphynx/_analytical_client.py +77 -0
- sphynx/_analytical_values.py +252 -0
- sphynx/_audit_client.py +173 -0
- sphynx/_comparison_adapters.py +113 -0
- sphynx/_componentized_full_rag.py +34 -0
- sphynx/_composed_pipeline.py +109 -0
- sphynx/_full_rag_grading.py +218 -0
- sphynx/_full_rag_trace.py +221 -0
- sphynx/_local_eval.py +268 -0
- sphynx/_pipeline_core.py +243 -0
- sphynx/_pr_client.py +237 -0
- sphynx/_rag_diagnostics_calibration.py +285 -0
- sphynx/_rag_diagnostics_catalog.py +39 -0
- sphynx/_rag_diagnostics_common.py +26 -0
- sphynx/_rag_diagnostics_contracts.py +186 -0
- sphynx/_rag_diagnostics_engine.py +243 -0
- sphynx/_rag_diagnostics_importers.py +242 -0
- sphynx/_rag_diagnostics_ranking.py +81 -0
- sphynx/_rag_diagnostics_semantic.py +325 -0
- sphynx/_rag_ranking.py +149 -0
- sphynx/_speech_benchmark_scoring.py +140 -0
- sphynx/_speech_benchmark_sources.py +275 -0
- sphynx/_speech_complete_scoring.py +218 -0
- sphynx/_speech_complete_source.py +118 -0
- sphynx/_speech_kido.py +294 -0
- sphynx/_speech_scoring.py +77 -0
- sphynx/_track_capture.py +213 -0
- sphynx/analytical_views.py +515 -0
- sphynx/assessment_contracts.py +378 -0
- sphynx/assessment_selection.py +57 -0
- sphynx/audit_access.py +245 -0
- sphynx/audit_contracts.py +204 -0
- sphynx/audit_requests.py +180 -0
- sphynx/authoring.py +188 -0
- sphynx/benchmarks.py +568 -0
- sphynx/candidates.py +62 -0
- sphynx/catalog.py +272 -0
- sphynx/chunking.py +372 -0
- sphynx/clickhouse_reader.py +169 -0
- sphynx/client.py +486 -0
- sphynx/comparison_core.py +149 -0
- sphynx/comparison_views.py +223 -0
- sphynx/components.py +1661 -0
- sphynx/contracts.py +514 -0
- sphynx/customer_rerank.py +299 -0
- sphynx/customer_retrieval.py +146 -0
- sphynx/data/benchmark-releases.v1.json +164 -0
- sphynx/data/public-benchmark-release.v1.json +354 -0
- sphynx/data/rag-diagnostics-synthetic.v1.json +6982 -0
- sphynx/data/rag-paper-references.v1.json +176 -0
- sphynx/data/rag-paper-schemas.v1.json +1069 -0
- sphynx/data/rag-rerank-v1.json +507 -0
- sphynx/dataset_catalog_data.py +1214 -0
- sphynx/dataset_readiness.py +138 -0
- sphynx/dataset_v3.py +999 -0
- sphynx/demo.py +130 -0
- sphynx/demo_candidates/__init__.py +1 -0
- sphynx/demo_candidates/dense.py +11 -0
- sphynx/demo_candidates/lexical.py +21 -0
- sphynx/diagnostics.py +207 -0
- sphynx/documentation.py +737 -0
- sphynx/documentation_candidate.py +939 -0
- sphynx/evals.py +538 -0
- sphynx/evaluation_protocols.py +124 -0
- sphynx/evaluator_bindings.py +370 -0
- sphynx/evidence.py +352 -0
- sphynx/execution_capabilities.py +109 -0
- sphynx/fast_search.py +285 -0
- sphynx/full_rag.py +341 -0
- sphynx/hosted/__init__.py +18 -0
- sphynx/hosted/benchmark_intake.py +218 -0
- sphynx/hosted/contracts.py +361 -0
- sphynx/hosted/model_admission.py +130 -0
- sphynx/hosted/model_artifacts.py +298 -0
- sphynx/hosted/model_contracts.py +63 -0
- sphynx/hosted/model_gcs.py +181 -0
- sphynx/hosted/model_history.py +84 -0
- sphynx/hosted/model_session.py +233 -0
- sphynx/hosted/public_suite.py +74 -0
- sphynx/hosted/registry.py +97 -0
- sphynx/hosted/revisions.py +126 -0
- sphynx/hosted/sandbox.py +105 -0
- sphynx/hosted/submission.py +287 -0
- sphynx/hosted_workflow.py +451 -0
- sphynx/http_transport.py +86 -0
- sphynx/integrated_rag.py +352 -0
- sphynx/integrated_workflow.py +552 -0
- sphynx/integration_contracts.py +474 -0
- sphynx/invocation.py +791 -0
- sphynx/invocation_candidates.py +392 -0
- sphynx/invocation_catalog_data.py +40 -0
- sphynx/learned_catalog_data.py +30 -0
- sphynx/learned_rerank.py +267 -0
- sphynx/legacy_full_rag.py +67 -0
- sphynx/lifecycle.py +885 -0
- sphynx/lifecycle_workflow.py +490 -0
- sphynx/metric_views.py +474 -0
- sphynx/metrics.py +282 -0
- sphynx/model_adapters.py +438 -0
- sphynx/model_catalog_data.py +30 -0
- sphynx/official_benchmarks.py +385 -0
- sphynx/packs.py +258 -0
- sphynx/pipeline.py +313 -0
- sphynx/pipeline_workflow.py +551 -0
- sphynx/pr_evaluations.py +560 -0
- sphynx/progress.py +89 -0
- sphynx/public_bundle.py +173 -0
- sphynx/public_eval.py +62 -0
- sphynx/public_rerank.py +355 -0
- sphynx/py.typed +0 -0
- sphynx/pytest_plugin.py +62 -0
- sphynx/question_robustness.py +437 -0
- sphynx/rag.py +329 -0
- sphynx/rag_diagnostics.py +309 -0
- sphynx/rag_papers/__init__.py +29 -0
- sphynx/rag_papers/__main__.py +45 -0
- sphynx/rag_papers/candidates.py +19 -0
- sphynx/rag_papers/contracts.py +282 -0
- sphynx/rag_papers/datasets.py +291 -0
- sphynx/rag_papers/experiments.py +116 -0
- sphynx/rag_papers/judges.py +308 -0
- sphynx/rag_papers/registry.py +33 -0
- sphynx/rag_papers/runner.py +176 -0
- sphynx/rag_papers/sable.py +135 -0
- sphynx/rag_papers/schemas.py +12 -0
- sphynx/rag_papers/scoring.py +249 -0
- sphynx/rag_papers/semantic.py +288 -0
- sphynx/rag_papers/trec.py +80 -0
- sphynx/recorded_scores.py +352 -0
- sphynx/records.py +124 -0
- sphynx/reflex_traces.py +193 -0
- sphynx/report_rendering.py +71 -0
- sphynx/rerank_family_contracts.py +150 -0
- sphynx/rerank_workflow.py +424 -0
- sphynx/research_candidate.py +190 -0
- sphynx/results.py +852 -0
- sphynx/retrieval.py +750 -0
- sphynx/retrieval_candidates.py +295 -0
- sphynx/retrieval_catalog_data.py +19 -0
- sphynx/retrieval_recordings.py +125 -0
- sphynx/reviewer_audit.py +266 -0
- sphynx/routing_policies.py +375 -0
- sphynx/runner.py +544 -0
- sphynx/sable_rerank.py +489 -0
- sphynx/score_reliability.py +393 -0
- sphynx/speech_benchmarks.py +204 -0
- sphynx/speech_campaigns.py +378 -0
- sphynx/speech_complete.py +236 -0
- sphynx/speech_contracts.py +225 -0
- sphynx/speech_evidence.py +429 -0
- sphynx/speech_listening.py +115 -0
- sphynx/speech_multilingual.py +71 -0
- sphynx/speech_tts.py +479 -0
- sphynx/sql/001_sync_v1.sql +30 -0
- sphynx/stage_contracts.py +199 -0
- sphynx/statistical_catalog_data.py +53 -0
- sphynx/statistical_rerank.py +232 -0
- sphynx/sync.py +797 -0
- sphynx/workbench.py +378 -0
- sphynx/workbench_catalog_data.py +13 -0
- sphynx/workbench_tracks.py +78 -0
- sphynx/workflow_async.py +197 -0
- sphynx/workflow_contracts.py +594 -0
- sphynx/workflow_decisions.py +211 -0
- sphynx/workflow_defaults.py +16 -0
- sphynx/workflow_extensions.py +212 -0
- sphynx/workflow_registry.py +513 -0
- sphynx/workflows.py +792 -0
sphynx/speech_tts.py
ADDED
|
@@ -0,0 +1,479 @@
|
|
|
1
|
+
"""Independent, trusted-local TTS candidate execution and saved grading.
|
|
2
|
+
|
|
3
|
+
No hosted admission or automatic provider launch. Freeze source and inputs,
|
|
4
|
+
inspect the plan, then execute fixtures or explicitly authorize provider calls.
|
|
5
|
+
"""
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import argparse
|
|
9
|
+
import base64
|
|
10
|
+
from collections import Counter
|
|
11
|
+
import hashlib
|
|
12
|
+
import io
|
|
13
|
+
import json
|
|
14
|
+
import math
|
|
15
|
+
import os
|
|
16
|
+
from pathlib import Path, PurePosixPath
|
|
17
|
+
import platform
|
|
18
|
+
import queue
|
|
19
|
+
import re
|
|
20
|
+
import subprocess
|
|
21
|
+
import sys
|
|
22
|
+
import threading
|
|
23
|
+
import time
|
|
24
|
+
import uuid
|
|
25
|
+
import wave
|
|
26
|
+
|
|
27
|
+
from .speech_contracts import (EVENT_SCHEMA, FORMAT, TTS_REQUEST, SpeechContractError,
|
|
28
|
+
StreamValidator, checksum, encode, fields, identifier, parse, require, sha, validate_method, validate_request)
|
|
29
|
+
from .speech_evidence import _new_directory as _saved_new_directory, _read as _saved_read, _write_files
|
|
30
|
+
|
|
31
|
+
SUITE_ID = "speech.tts.execution.v1"
|
|
32
|
+
PLAN_SCHEMA = "sphynx.speech.tts.plan.v1"
|
|
33
|
+
INVOCATION_SCHEMA = "sphynx.speech.tts.execution-invocation.v1"
|
|
34
|
+
FAMILY_SCHEMA = "sphynx.speech.tts.family.v1"
|
|
35
|
+
RUN_SCHEMA = "sphynx.speech.tts.run.v1"
|
|
36
|
+
GRADER = "speech.tts.integrity-grader.v1"
|
|
37
|
+
FAMILY_PATH = "sphynx_candidates/tts/family.json"
|
|
38
|
+
_BOOTSTRAP = {"PATH", "PATHEXT", "SYSTEMROOT", "WINDIR", "COMSPEC", "TEMP", "TMP", "TMPDIR", "HOME", "USERPROFILE", "LANG"}
|
|
39
|
+
_CREDENTIAL = {"elevenlabs": "ELEVEN_API_KEY", "cartesia": "CARTESIA_API_KEY"}
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _new_directory(path):
|
|
43
|
+
try:
|
|
44
|
+
return _saved_new_directory(path)
|
|
45
|
+
except ValueError as exc:
|
|
46
|
+
raise SpeechContractError(str(exc)) from None
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _read(root, path):
|
|
50
|
+
try:
|
|
51
|
+
return _saved_read(root, path)
|
|
52
|
+
except ValueError as exc:
|
|
53
|
+
raise SpeechContractError(str(exc)) from None
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def describe_tts_suite():
|
|
57
|
+
return {"suite_id": SUITE_ID, "plan_schema": PLAN_SCHEMA, "invocation_schema": INVOCATION_SCHEMA,
|
|
58
|
+
"request_schema": TTS_REQUEST, "event_schema": EVENT_SCHEMA, "grader": GRADER,
|
|
59
|
+
"execution": "trusted_local_subprocess", "hosted_admitted": False,
|
|
60
|
+
"provider_authorization": "explicit_launch_flag_required", "os_isolation": False,
|
|
61
|
+
"perceived_latency_measured": False, "stt_execution": "contract_only"}
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _relative(path):
|
|
65
|
+
require(isinstance(path, str) and re.fullmatch(r"[A-Za-z0-9_./-]{1,240}", path)
|
|
66
|
+
and all(p not in ("", ".", "..") and not p.startswith(".") for p in path.split("/")),
|
|
67
|
+
"invalid speech source path")
|
|
68
|
+
return path
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _family(raw):
|
|
72
|
+
value = parse(raw)
|
|
73
|
+
fields(value, ("schema_version", "entry_point", "source_files", "methods"))
|
|
74
|
+
require(value["schema_version"] == FAMILY_SCHEMA, "unsupported speech candidate family")
|
|
75
|
+
_relative(value["entry_point"])
|
|
76
|
+
require(isinstance(value["source_files"], list) and 1 <= len(value["source_files"]) <= 16,
|
|
77
|
+
"invalid speech source closure")
|
|
78
|
+
for path in value["source_files"]:
|
|
79
|
+
_relative(path)
|
|
80
|
+
require(len(set(value["source_files"])) == len(value["source_files"]) and FAMILY_PATH in value["source_files"]
|
|
81
|
+
and value["entry_point"] in value["source_files"], "incomplete candidate source closure")
|
|
82
|
+
require(isinstance(value["methods"], dict) and 1 <= len(value["methods"]) <= 8, "invalid speech method census")
|
|
83
|
+
for key, method in value["methods"].items():
|
|
84
|
+
identifier(key); validate_method(method)
|
|
85
|
+
return value
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _git(root, *args):
|
|
89
|
+
result = subprocess.run(["git", "-C", str(root), *args], capture_output=True, timeout=15, check=False)
|
|
90
|
+
require(result.returncode == 0 and len(result.stdout) <= 1_000_000, "could not verify local candidate Git source")
|
|
91
|
+
return result.stdout
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def freeze_tts_plan(repository, *, source_commit, source_pr, cases, cohort_identity,
|
|
95
|
+
baseline, candidate, output_dir, repetitions=1, timeout_seconds=30,
|
|
96
|
+
max_audio_bytes=2_000_000, max_requests=20):
|
|
97
|
+
"""Freeze two named methods from one exact local Git commit, without execution."""
|
|
98
|
+
require(isinstance(source_commit, str) and re.fullmatch(r"[0-9a-f]{40}", source_commit), "pin the complete candidate commit")
|
|
99
|
+
match = re.fullmatch(r"https://github.com/([A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+)/pull/([1-9][0-9]*)", source_pr)
|
|
100
|
+
require(match is not None, "pin the candidate source PR")
|
|
101
|
+
root = Path(repository).absolute()
|
|
102
|
+
origin = _git(root, "config", "--get", "remote.origin.url").decode().strip()
|
|
103
|
+
repo_name = match.group(1)
|
|
104
|
+
require(origin in (f"https://github.com/{repo_name}.git", f"https://github.com/{repo_name}", f"git@github.com:{repo_name}.git"),
|
|
105
|
+
"candidate PR and local repository origin differ")
|
|
106
|
+
require(_git(root, "rev-parse", "HEAD").decode().strip() == source_commit, "candidate checkout is not the requested commit")
|
|
107
|
+
family_raw = _git(root, "cat-file", "blob", source_commit + ":" + FAMILY_PATH)
|
|
108
|
+
family = _family(family_raw)
|
|
109
|
+
require(baseline != candidate and baseline in family["methods"] and candidate in family["methods"], "select two distinct named methods")
|
|
110
|
+
require(family["methods"][baseline]["mode"] == family["methods"][candidate]["mode"], "fixture and provider arms cannot be mixed")
|
|
111
|
+
require(type(repetitions) is int and 1 <= repetitions <= 3 and type(timeout_seconds) is int and 1 <= timeout_seconds <= 120,
|
|
112
|
+
"invalid TTS trial limits")
|
|
113
|
+
require(type(max_audio_bytes) is int and 2 <= max_audio_bytes <= 8_000_000 and max_audio_bytes % 2 == 0,
|
|
114
|
+
"invalid TTS audio limit")
|
|
115
|
+
require(type(max_requests) is int and 2 <= max_requests <= 20, "pilot request budget must be between 2 and 20")
|
|
116
|
+
require(isinstance(cases, list) and 0 < len(cases) <= 10, "select one to ten pilot cases")
|
|
117
|
+
for case in cases:
|
|
118
|
+
fields(case, ("id", "text", "language")); identifier(case["id"])
|
|
119
|
+
require(isinstance(case["text"], str) and 0 < len(case["text"]) <= 10000, "invalid TTS pilot text")
|
|
120
|
+
require(isinstance(case["language"], str) and re.fullmatch(r"[a-z]{2}(?:-[A-Za-z]{2,4})?", case["language"]), "invalid TTS pilot language")
|
|
121
|
+
require(len({c["id"] for c in cases}) == len(cases), "duplicate pilot cases")
|
|
122
|
+
fields(cohort_identity, ("kind", "sha256")); identifier(cohort_identity["kind"]); checksum(cohort_identity["sha256"])
|
|
123
|
+
planned_calls = len(cases) * 2 * repetitions
|
|
124
|
+
require(planned_calls <= max_requests, "pilot exceeds request budget")
|
|
125
|
+
require(sum(len(c["text"]) for c in cases) * 2 * repetitions <= 20000, "pilot exceeds input-character budget")
|
|
126
|
+
sources = {}
|
|
127
|
+
for path in family["source_files"]:
|
|
128
|
+
blob = _git(root, "cat-file", "blob", source_commit + ":" + path)
|
|
129
|
+
require(_read(root, path) == blob, "candidate working bytes differ from pinned Git source")
|
|
130
|
+
sources["source/" + path] = blob
|
|
131
|
+
output = _new_directory(output_dir)
|
|
132
|
+
files = _write_files(output, sources)
|
|
133
|
+
plan = {"schema_version": PLAN_SCHEMA, "suite_id": SUITE_ID, "id": "tts-plan-" + uuid.uuid4().hex,
|
|
134
|
+
"source_pr": source_pr, "source_commit": source_commit, "family_path": FAMILY_PATH,
|
|
135
|
+
"family_sha256": sha(family_raw), "entry_point": family["entry_point"], "files": files,
|
|
136
|
+
"baseline": baseline, "candidate": candidate, "methods": {k: family["methods"][k] for k in (baseline, candidate)},
|
|
137
|
+
"cases": cases, "cases_sha256": sha(encode(cases)), "cohort_identity": cohort_identity,
|
|
138
|
+
"repetitions": repetitions, "limits": {"timeout_seconds": timeout_seconds, "max_audio_bytes": max_audio_bytes},
|
|
139
|
+
"budget": {"max_requests": max_requests, "planned_requests": planned_calls,
|
|
140
|
+
"planned_input_characters": sum(len(c["text"]) for c in cases) * 2 * repetitions,
|
|
141
|
+
"retries": 0, "cost_usd": None, "monetary_cap_enforced": False},
|
|
142
|
+
"mode": family["methods"][baseline]["mode"], "review_status": "prepared_for_review",
|
|
143
|
+
"model_voice_availability_verified": False, "hosted_admitted": False}
|
|
144
|
+
raw = encode(plan); (output / "plan.json").write_bytes(raw)
|
|
145
|
+
return {"path": str(output), "plan_sha256": sha(raw), "suite_id": SUITE_ID, "budget": plan["budget"],
|
|
146
|
+
"mode": plan["mode"], "provider_calls_made": False}
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def _load_plan(path, expected_sha256):
|
|
150
|
+
root = Path(path).absolute()
|
|
151
|
+
raw = _read(root, "plan.json")
|
|
152
|
+
require(sha(raw) == checksum(expected_sha256), "TTS plan digest differs")
|
|
153
|
+
plan = parse(raw)
|
|
154
|
+
require(plan.get("schema_version") == PLAN_SCHEMA and plan.get("suite_id") == SUITE_ID,
|
|
155
|
+
"not an executable TTS plan")
|
|
156
|
+
require(isinstance(plan.get("files"), dict) and len(plan["files"]) <= 16, "invalid TTS source census")
|
|
157
|
+
for path_, receipt in plan["files"].items():
|
|
158
|
+
require(path_.startswith("source/"), "invalid TTS source archive path")
|
|
159
|
+
_relative(path_[7:])
|
|
160
|
+
data = _read(root, path_)
|
|
161
|
+
require(len(data) == receipt["size_bytes"] and sha(data) == receipt["sha256"], "frozen TTS source changed")
|
|
162
|
+
family = _family(_read(root, "source/" + FAMILY_PATH))
|
|
163
|
+
require(plan["family_sha256"] == sha(_read(root, "source/" + FAMILY_PATH)) and
|
|
164
|
+
set(plan["files"]) == {"source/" + p for p in family["source_files"]} and
|
|
165
|
+
plan["entry_point"] == family["entry_point"], "TTS source closure differs")
|
|
166
|
+
require(plan["baseline"] != plan["candidate"] and set(plan["methods"]) == {plan["baseline"], plan["candidate"]}, "TTS arm census differs")
|
|
167
|
+
require(all(plan["methods"][k] == family["methods"].get(k) for k in plan["methods"]), "TTS selected method changed")
|
|
168
|
+
require(all(m["mode"] == plan["mode"] for m in plan["methods"].values()), "TTS execution mode differs")
|
|
169
|
+
require(sha(encode(plan["cases"])) == plan["cases_sha256"] and len({c["id"] for c in plan["cases"]}) == len(plan["cases"]),
|
|
170
|
+
"TTS cases changed")
|
|
171
|
+
require(type(plan["repetitions"]) is int and 1 <= plan["repetitions"] <= 3 and 1 <= len(plan["cases"]) <= 10,
|
|
172
|
+
"TTS trial population is invalid")
|
|
173
|
+
require(plan["budget"]["planned_requests"] == len(plan["cases"]) * 2 * plan["repetitions"] <= plan["budget"]["max_requests"] <= 20
|
|
174
|
+
and plan["budget"]["retries"] == 0, "TTS request budget differs")
|
|
175
|
+
require(plan["budget"]["planned_input_characters"] == sum(len(c["text"]) for c in plan["cases"]) * 2 * plan["repetitions"] <= 20000,
|
|
176
|
+
"TTS character budget differs")
|
|
177
|
+
for case in plan["cases"]:
|
|
178
|
+
fields(case, ("id", "text", "language"))
|
|
179
|
+
for method in plan["methods"]:
|
|
180
|
+
validate_request(_request(plan, case, method, "preflight"))
|
|
181
|
+
return root, plan
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def _request(plan, case, method, request_id):
|
|
185
|
+
config = plan["methods"][method]
|
|
186
|
+
return {"schema_version": TTS_REQUEST, "request_id": request_id, "case_id": case["id"], "method_id": method,
|
|
187
|
+
"config": config, "config_sha256": sha(encode(config)), "input": {"text": case["text"], "language": case["language"]},
|
|
188
|
+
"format": FORMAT, "limits": plan["limits"]}
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def _execute(root, plan, request, authorize):
|
|
192
|
+
validator = StreamValidator(request)
|
|
193
|
+
env = {k: v for k, v in os.environ.items() if k.upper() in _BOOTSTRAP}
|
|
194
|
+
env.update(PYTHONPATH=str(Path(__file__).resolve().parents[1]), PYTHONUTF8="1", PYTHONDONTWRITEBYTECODE="1")
|
|
195
|
+
if request["config"]["mode"] == "provider":
|
|
196
|
+
require(authorize, "provider calls require explicit launch authorization")
|
|
197
|
+
key = _CREDENTIAL[request["config"]["provider"]]
|
|
198
|
+
require(bool(os.environ.get(key)), "selected provider credential is unavailable")
|
|
199
|
+
env[key] = os.environ[key]
|
|
200
|
+
env["SPHYNX_TTS_ALLOW_PROVIDER"] = "1"
|
|
201
|
+
started = time.monotonic_ns()
|
|
202
|
+
process = subprocess.Popen([sys.executable, str(root / "source" / plan["entry_point"])],
|
|
203
|
+
cwd=root / "source", env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
204
|
+
events_queue, stop = queue.Queue(maxsize=64), threading.Event()
|
|
205
|
+
stderr_count = [0]
|
|
206
|
+
def enqueue(item):
|
|
207
|
+
while not stop.is_set():
|
|
208
|
+
try:
|
|
209
|
+
events_queue.put(item, timeout=.05); return
|
|
210
|
+
except queue.Full:
|
|
211
|
+
pass
|
|
212
|
+
def read_stdout():
|
|
213
|
+
try:
|
|
214
|
+
while not stop.is_set():
|
|
215
|
+
line = process.stdout.readline(16385)
|
|
216
|
+
if not line:
|
|
217
|
+
enqueue(("eof", None, time.monotonic_ns())); return
|
|
218
|
+
if len(line) > 16384:
|
|
219
|
+
enqueue(("overflow", None, time.monotonic_ns())); return
|
|
220
|
+
enqueue(("line", line, time.monotonic_ns()))
|
|
221
|
+
except OSError:
|
|
222
|
+
enqueue(("reader_error", None, time.monotonic_ns()))
|
|
223
|
+
def discard_stderr():
|
|
224
|
+
try:
|
|
225
|
+
while True:
|
|
226
|
+
chunk = process.stderr.read(8192)
|
|
227
|
+
if not chunk:
|
|
228
|
+
return
|
|
229
|
+
stderr_count[0] += len(chunk)
|
|
230
|
+
except OSError:
|
|
231
|
+
pass
|
|
232
|
+
def write_input():
|
|
233
|
+
try:
|
|
234
|
+
process.stdin.write(encode(request)); process.stdin.flush()
|
|
235
|
+
except (OSError, ValueError):
|
|
236
|
+
pass
|
|
237
|
+
finally:
|
|
238
|
+
try:
|
|
239
|
+
process.stdin.close()
|
|
240
|
+
except (OSError, ValueError):
|
|
241
|
+
pass
|
|
242
|
+
readers = [threading.Thread(target=read_stdout, daemon=True), threading.Thread(target=discard_stderr, daemon=True),
|
|
243
|
+
threading.Thread(target=write_input, daemon=True)]
|
|
244
|
+
for reader in readers:
|
|
245
|
+
reader.start()
|
|
246
|
+
audio, events, first, failure, stream_bytes = bytearray(), [], None, None, 0
|
|
247
|
+
deadline = time.monotonic() + request["limits"]["timeout_seconds"]
|
|
248
|
+
try:
|
|
249
|
+
while True:
|
|
250
|
+
left = deadline - time.monotonic()
|
|
251
|
+
if left <= 0:
|
|
252
|
+
failure = "timeout"; break
|
|
253
|
+
try:
|
|
254
|
+
kind, line, arrived = events_queue.get(timeout=left)
|
|
255
|
+
except queue.Empty:
|
|
256
|
+
failure = "timeout"; break
|
|
257
|
+
if kind == "eof":
|
|
258
|
+
break
|
|
259
|
+
if kind != "line":
|
|
260
|
+
failure = "stream_limit_or_read_error"; break
|
|
261
|
+
if len(events) >= 4096:
|
|
262
|
+
failure = "event_limit"; break
|
|
263
|
+
stream_bytes += len(line)
|
|
264
|
+
if stream_bytes > request["limits"]["max_audio_bytes"] * 2 + 65536:
|
|
265
|
+
failure = "stream_limit"; break
|
|
266
|
+
event = parse(line)
|
|
267
|
+
chunk = validator.accept(event)
|
|
268
|
+
if chunk:
|
|
269
|
+
if first is None:
|
|
270
|
+
first = arrived
|
|
271
|
+
audio.extend(chunk)
|
|
272
|
+
events.append({"event": event, "parent_received_ns": arrived - started})
|
|
273
|
+
if failure is None:
|
|
274
|
+
process.wait(timeout=max(.01, deadline - time.monotonic()))
|
|
275
|
+
if process.returncode != 0 or validator.terminal is None:
|
|
276
|
+
failure = "process_error_or_missing_terminal"
|
|
277
|
+
elif validator.terminal["status"] != "success":
|
|
278
|
+
failure = validator.terminal["error_code"]
|
|
279
|
+
except (SpeechContractError, KeyError, TypeError, ValueError):
|
|
280
|
+
failure = "invalid_stream"
|
|
281
|
+
except subprocess.TimeoutExpired:
|
|
282
|
+
failure = "timeout"
|
|
283
|
+
finally:
|
|
284
|
+
stop.set()
|
|
285
|
+
if process.poll() is None:
|
|
286
|
+
process.kill(); process.wait(timeout=5)
|
|
287
|
+
for reader in readers:
|
|
288
|
+
reader.join(timeout=1)
|
|
289
|
+
process.stdout.close(); process.stderr.close()
|
|
290
|
+
if not process.stdin.closed:
|
|
291
|
+
try:
|
|
292
|
+
process.stdin.close()
|
|
293
|
+
except OSError:
|
|
294
|
+
pass
|
|
295
|
+
return {"request_id": request["request_id"], "case_id": request["case_id"], "method_id": request["method_id"],
|
|
296
|
+
"config_sha256": request["config_sha256"], "success": failure is None, "failure": failure,
|
|
297
|
+
"audio_bytes": len(audio), "audio_sha256": sha(bytes(audio)),
|
|
298
|
+
"pcm_complete": failure is None and len(audio) > 0 and len(audio) % 2 == 0,
|
|
299
|
+
"parent_first_audio_ms": (first - started) / 1_000_000 if first is not None else None,
|
|
300
|
+
"parent_completion_ms": (time.monotonic_ns() - started) / 1_000_000,
|
|
301
|
+
"timing_scope": "parent-process-launch-to-validated-audio-event-includes-startup-and-IPC",
|
|
302
|
+
"candidate_reported_first_audio_ns": next((e["event"]["t_ns"] for e in events if e["event"]["type"] == "audio"), None),
|
|
303
|
+
"identity_evidence": validator.terminal["identity_evidence"] if validator.terminal else None,
|
|
304
|
+
"stderr_bytes_discarded": stderr_count[0], "exit_code": process.returncode,
|
|
305
|
+
"events": events}, bytes(audio)
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
def run_tts_plan(path, *, expected_plan_sha256, output_dir, allow_provider_calls=False):
|
|
309
|
+
root, plan = _load_plan(path, expected_plan_sha256)
|
|
310
|
+
require(type(allow_provider_calls) is bool, "provider launch authorization must be explicit")
|
|
311
|
+
require(plan["mode"] == "fixture" or allow_provider_calls, "provider calls require explicit launch authorization")
|
|
312
|
+
# Validate both selected credentials before creating a run or starting either arm.
|
|
313
|
+
if plan["mode"] == "provider":
|
|
314
|
+
require(all(os.environ.get(_CREDENTIAL[m["provider"]]) for m in plan["methods"].values()), "selected provider credential is unavailable")
|
|
315
|
+
output = _new_directory(output_dir)
|
|
316
|
+
frozen = {"frozen-plan/plan.json": _read(root, "plan.json")}
|
|
317
|
+
frozen.update({"frozen-plan/" + name: _read(root, name) for name in plan["files"]})
|
|
318
|
+
artifacts = _write_files(output, frozen)
|
|
319
|
+
root, plan = _load_plan(output / "frozen-plan", expected_plan_sha256)
|
|
320
|
+
invocation_id = "tts-invocation-" + uuid.uuid4().hex
|
|
321
|
+
invocation = {"schema_version": INVOCATION_SCHEMA, "suite_id": SUITE_ID, "id": invocation_id,
|
|
322
|
+
"plan_sha256": expected_plan_sha256, "source_pr": plan["source_pr"], "source_commit": plan["source_commit"],
|
|
323
|
+
"mode": plan["mode"], "provider_calls_authorized": allow_provider_calls and plan["mode"] == "provider",
|
|
324
|
+
"provider_calls_observed": None if plan["mode"] == "provider" else 0,
|
|
325
|
+
"runtime": {"python": sys.version, "platform": platform.platform(), "machine": platform.machine()},
|
|
326
|
+
"implementation_sha256": {name: sha(Path(__file__).with_name(name).read_bytes()) for name in ("speech_tts.py", "speech_contracts.py")}}
|
|
327
|
+
(output / "invocation.json").write_bytes(encode(invocation))
|
|
328
|
+
captures = []
|
|
329
|
+
artifacts["invocation.json"] = {"sha256": sha(encode(invocation)), "size_bytes": len(encode(invocation))}
|
|
330
|
+
for rep in range(plan["repetitions"]):
|
|
331
|
+
for case_index, case in enumerate(plan["cases"]):
|
|
332
|
+
arms = (plan["baseline"], plan["candidate"])
|
|
333
|
+
if (rep + case_index) % 2:
|
|
334
|
+
arms = tuple(reversed(arms))
|
|
335
|
+
for method in arms:
|
|
336
|
+
request_id = "tts-request-" + uuid.uuid4().hex
|
|
337
|
+
request = _request(plan, case, method, request_id)
|
|
338
|
+
try:
|
|
339
|
+
capture, audio = _execute(root, plan, request, allow_provider_calls)
|
|
340
|
+
except OSError:
|
|
341
|
+
capture, audio = {"request_id": request_id, "case_id": case["id"], "method_id": method,
|
|
342
|
+
"success": False, "failure": "process_start_error", "audio_bytes": 0, "audio_sha256": sha(b""),
|
|
343
|
+
"pcm_complete": False, "parent_first_audio_ms": None, "parent_completion_ms": None,
|
|
344
|
+
"events": []}, b""
|
|
345
|
+
capture["repetition"] = rep + 1
|
|
346
|
+
capture["order_index"] = len(captures)
|
|
347
|
+
capture["request_sha256"] = sha(encode(request))
|
|
348
|
+
relative = "captures/" + request_id + "/"
|
|
349
|
+
payloads = {relative + "request.json": encode(request), relative + "capture.json": encode(capture)}
|
|
350
|
+
if audio:
|
|
351
|
+
wav = io.BytesIO()
|
|
352
|
+
with wave.open(wav, "wb") as stream:
|
|
353
|
+
stream.setnchannels(1); stream.setsampwidth(2); stream.setframerate(16000); stream.writeframes(audio)
|
|
354
|
+
payloads[relative + "audio.wav"] = wav.getvalue()
|
|
355
|
+
artifacts.update(_write_files(output, payloads))
|
|
356
|
+
captures.append(capture)
|
|
357
|
+
report = grade_tts_captures(plan, captures)
|
|
358
|
+
report.update(invocation_id=invocation_id, invocation_sha256=sha(encode(invocation)), plan_sha256=expected_plan_sha256)
|
|
359
|
+
artifacts.update(_write_files(output, {"report.json": encode(report)}))
|
|
360
|
+
manifest = {"schema_version": RUN_SCHEMA, "suite_id": SUITE_ID, "invocation_id": invocation_id,
|
|
361
|
+
"plan_sha256": expected_plan_sha256, "files": artifacts,
|
|
362
|
+
"completed_requests": len(captures), "planned_requests": plan["budget"]["planned_requests"],
|
|
363
|
+
"status": "completed" if all(c["success"] for c in captures) else "completed_with_failures"}
|
|
364
|
+
raw = encode(manifest); (output / "manifest.json").write_bytes(raw)
|
|
365
|
+
return {"path": str(output), "suite_id": SUITE_ID, "invocation_id": invocation_id,
|
|
366
|
+
"manifest_sha256": sha(raw), "status": manifest["status"], "mode": plan["mode"]}
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
def grade_tts_captures(plan, captures):
|
|
370
|
+
for capture in captures:
|
|
371
|
+
require(type(capture.get("success")) is bool, "TTS outcome must be a boolean")
|
|
372
|
+
value = capture.get("parent_first_audio_ms")
|
|
373
|
+
require(value is None or (type(value) in (int, float) and math.isfinite(value) and value >= 0),
|
|
374
|
+
"TTS timing must be finite and nonnegative")
|
|
375
|
+
expected = {(case["id"], method, rep + 1) for case in plan["cases"] for method in plan["methods"] for rep in range(plan["repetitions"])}
|
|
376
|
+
actual = [(c["case_id"], c["method_id"], c["repetition"]) for c in captures]
|
|
377
|
+
require(len(actual) == len(set(actual)) and set(actual) == expected, "TTS capture census is incomplete or duplicated")
|
|
378
|
+
arms = {}
|
|
379
|
+
for method in plan["methods"]:
|
|
380
|
+
rows = [c for c in captures if c["method_id"] == method]
|
|
381
|
+
times = [c["parent_first_audio_ms"] for c in rows if c["success"] and c["parent_first_audio_ms"] is not None]
|
|
382
|
+
arms[method] = {"attempts": len(rows), "successful": sum(c["success"] for c in rows),
|
|
383
|
+
"failure_counts": dict(Counter(c["failure"] for c in rows if c["failure"])),
|
|
384
|
+
"observed_success_first_audio_ms_mean": sum(times) / len(times) if times else None,
|
|
385
|
+
"timing_count": len(times), "provider_latency_ms": None, "perceived_latency_ms": None}
|
|
386
|
+
pairs = []
|
|
387
|
+
for case in plan["cases"]:
|
|
388
|
+
for rep in range(1, plan["repetitions"] + 1):
|
|
389
|
+
pair = {c["method_id"]: c for c in captures if c["case_id"] == case["id"] and c["repetition"] == rep}
|
|
390
|
+
left, right = pair[plan["baseline"]], pair[plan["candidate"]]
|
|
391
|
+
eligible = left["success"] and right["success"] and left["parent_first_audio_ms"] is not None and right["parent_first_audio_ms"] is not None
|
|
392
|
+
pairs.append({"case_id": case["id"], "repetition": rep, "paired_success": eligible,
|
|
393
|
+
"parent_first_audio_delta_ms": right["parent_first_audio_ms"] - left["parent_first_audio_ms"] if eligible else None,
|
|
394
|
+
"identical_pcm": left["audio_sha256"] == right["audio_sha256"] if eligible else None})
|
|
395
|
+
observed = [p["parent_first_audio_delta_ms"] for p in pairs if p["parent_first_audio_delta_ms"] is not None]
|
|
396
|
+
return {"schema_version": "sphynx.speech.tts.integrity-report.v1", "suite_id": SUITE_ID, "grader": GRADER,
|
|
397
|
+
"mode": plan["mode"], "arms": arms, "pairs": pairs, "paired_count": len(observed), "missing_pairs": len(pairs) - len(observed),
|
|
398
|
+
"primary_parent_first_audio_delta_ms": sum(observed) / len(observed) if observed and len(observed) == len(pairs) else None,
|
|
399
|
+
"confidence_interval": None, "quality": None, "cost_usd": None,
|
|
400
|
+
"timing_scope": "local process startup plus adapter plus IPC; not pure provider or audible latency",
|
|
401
|
+
"claim_scope": "fixture plumbing only" if plan["mode"] == "fixture" else "trusted local pilot; requested model/voice not independently attested"}
|
|
402
|
+
|
|
403
|
+
|
|
404
|
+
def load_tts_run(path, *, expected_manifest_sha256):
|
|
405
|
+
root = Path(path).absolute(); raw = _read(root, "manifest.json")
|
|
406
|
+
require(sha(raw) == checksum(expected_manifest_sha256), "TTS run manifest differs")
|
|
407
|
+
manifest = parse(raw)
|
|
408
|
+
require(manifest.get("schema_version") == RUN_SCHEMA and manifest.get("suite_id") == SUITE_ID, "not a TTS execution run")
|
|
409
|
+
require(isinstance(manifest.get("files"), dict) and {"report.json", "invocation.json", "frozen-plan/plan.json"}.issubset(manifest["files"]),
|
|
410
|
+
"incomplete TTS run file census")
|
|
411
|
+
for name, receipt in manifest["files"].items():
|
|
412
|
+
data = _read(root, name)
|
|
413
|
+
require(sha(data) == receipt["sha256"] and len(data) == receipt["size_bytes"], "saved TTS artifact changed")
|
|
414
|
+
invocation, report = parse(_read(root, "invocation.json")), parse(_read(root, "report.json"))
|
|
415
|
+
require(report["invocation_sha256"] == sha(encode(invocation)) and
|
|
416
|
+
report["invocation_id"] == invocation["id"] == manifest["invocation_id"] and
|
|
417
|
+
report["plan_sha256"] == invocation["plan_sha256"] == manifest["plan_sha256"], "TTS run identity binding differs")
|
|
418
|
+
require(invocation.get("schema_version") == INVOCATION_SCHEMA and invocation.get("suite_id") == report.get("suite_id") == SUITE_ID,
|
|
419
|
+
"TTS invocation namespace differs")
|
|
420
|
+
_, plan = _load_plan(root / "frozen-plan", manifest["plan_sha256"])
|
|
421
|
+
captures = [parse(_read(root, name)) for name in manifest["files"] if name.startswith("captures/") and name.endswith("/capture.json")]
|
|
422
|
+
require(manifest["completed_requests"] == len(captures) == manifest["planned_requests"] == plan["budget"]["planned_requests"],
|
|
423
|
+
"saved TTS request census differs")
|
|
424
|
+
recomputed = grade_tts_captures(plan, captures)
|
|
425
|
+
cases = {c["id"]: c for c in plan["cases"]}
|
|
426
|
+
for capture in captures:
|
|
427
|
+
relative = "captures/" + identifier(capture["request_id"]) + "/"
|
|
428
|
+
expected_request = _request(plan, cases[capture["case_id"]], capture["method_id"], capture["request_id"])
|
|
429
|
+
require(_read(root, relative + "request.json") == encode(expected_request) and
|
|
430
|
+
capture["request_sha256"] == sha(encode(expected_request)), "saved TTS request differs from frozen inputs")
|
|
431
|
+
validator = StreamValidator(expected_request)
|
|
432
|
+
chunks = []
|
|
433
|
+
for item in capture["events"]:
|
|
434
|
+
chunks.append(validator.accept(item["event"]))
|
|
435
|
+
pcm = b"".join(chunks)
|
|
436
|
+
require(sha(pcm) == capture["audio_sha256"] and len(pcm) == capture["audio_bytes"], "saved TTS audio differs from events")
|
|
437
|
+
require(capture["success"] == (capture["failure"] is None), "TTS success and failure status differ")
|
|
438
|
+
if capture["success"]:
|
|
439
|
+
require(validator.terminal is not None and validator.terminal["status"] == "success", "successful saved TTS stream lacks terminal")
|
|
440
|
+
if pcm:
|
|
441
|
+
require(relative + "audio.wav" in manifest["files"], "saved TTS audio artifact is missing")
|
|
442
|
+
with wave.open(io.BytesIO(_read(root, relative + "audio.wav")), "rb") as stream:
|
|
443
|
+
require((stream.getnchannels(), stream.getsampwidth(), stream.getframerate()) == (1, 2, 16000)
|
|
444
|
+
and stream.readframes(stream.getnframes()) == pcm, "saved WAV differs from speech stream")
|
|
445
|
+
require(all(report.get(k) == v for k, v in recomputed.items()), "saved TTS report differs from capture grading")
|
|
446
|
+
return report
|
|
447
|
+
|
|
448
|
+
|
|
449
|
+
def main(argv=None):
|
|
450
|
+
parser = argparse.ArgumentParser(description="Independent trusted-local speech.tts.execution.v1 suite")
|
|
451
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
452
|
+
sub.add_parser("describe")
|
|
453
|
+
freeze = sub.add_parser("freeze")
|
|
454
|
+
for arg in ("repository", "source-commit", "source-pr", "cases-json", "baseline", "candidate", "output"):
|
|
455
|
+
freeze.add_argument("--" + arg, required=True)
|
|
456
|
+
freeze.add_argument("--repetitions", type=int, default=1)
|
|
457
|
+
launch = sub.add_parser("run")
|
|
458
|
+
for arg in ("plan", "expected-plan-sha256", "output"):
|
|
459
|
+
launch.add_argument("--" + arg, required=True)
|
|
460
|
+
launch.add_argument("--allow-provider-calls", action="store_true")
|
|
461
|
+
args = parser.parse_args(argv)
|
|
462
|
+
try:
|
|
463
|
+
if args.command == "describe":
|
|
464
|
+
result = describe_tts_suite()
|
|
465
|
+
elif args.command == "freeze":
|
|
466
|
+
cases = parse(Path(args.cases_json).read_bytes())
|
|
467
|
+
result = freeze_tts_plan(args.repository, source_commit=args.source_commit, source_pr=args.source_pr,
|
|
468
|
+
cases=cases, cohort_identity={"kind": "owner-supplied-cases", "sha256": sha(encode(cases))},
|
|
469
|
+
baseline=args.baseline, candidate=args.candidate, repetitions=args.repetitions, output_dir=args.output)
|
|
470
|
+
else:
|
|
471
|
+
result = run_tts_plan(args.plan, expected_plan_sha256=args.expected_plan_sha256,
|
|
472
|
+
output_dir=args.output, allow_provider_calls=args.allow_provider_calls)
|
|
473
|
+
except (SpeechContractError, ValueError, OSError, KeyError, TypeError, subprocess.SubprocessError):
|
|
474
|
+
parser.exit(2, "TTS operation failed; check frozen identities, limits and launch authorization\n")
|
|
475
|
+
print(json.dumps(result, indent=2))
|
|
476
|
+
|
|
477
|
+
|
|
478
|
+
if __name__ == "__main__":
|
|
479
|
+
main()
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
-- Sphynx-owned projection schema. Provision {database} and grant the sync
|
|
2
|
+
-- principal INSERT/SELECT on only these tables before invoking sync().
|
|
3
|
+
-- No Litmus database, tables, buckets, or credentials are used.
|
|
4
|
+
CREATE TABLE IF NOT EXISTS {database}.sphynx_facts_v1
|
|
5
|
+
(
|
|
6
|
+
projection_id String,
|
|
7
|
+
row_id String,
|
|
8
|
+
row_hash String,
|
|
9
|
+
fact_type LowCardinality(String),
|
|
10
|
+
run_id String,
|
|
11
|
+
case_id String,
|
|
12
|
+
capture_id String,
|
|
13
|
+
evaluation_id String,
|
|
14
|
+
metric_id String,
|
|
15
|
+
capability_id String,
|
|
16
|
+
status LowCardinality(String),
|
|
17
|
+
numeric_value Nullable(Float64),
|
|
18
|
+
payload_json String
|
|
19
|
+
)
|
|
20
|
+
ENGINE = MergeTree
|
|
21
|
+
ORDER BY (projection_id, row_id);
|
|
22
|
+
|
|
23
|
+
CREATE TABLE IF NOT EXISTS {database}.sphynx_projection_commits_v1
|
|
24
|
+
(
|
|
25
|
+
projection_id String,
|
|
26
|
+
commit_hash String,
|
|
27
|
+
payload_json String
|
|
28
|
+
)
|
|
29
|
+
ENGINE = MergeTree
|
|
30
|
+
ORDER BY projection_id;
|