sable-sphynx 0.7.0.dev0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (174) hide show
  1. sable_sphynx-0.7.0.dev0.dist-info/METADATA +152 -0
  2. sable_sphynx-0.7.0.dev0.dist-info/RECORD +174 -0
  3. sable_sphynx-0.7.0.dev0.dist-info/WHEEL +5 -0
  4. sable_sphynx-0.7.0.dev0.dist-info/top_level.txt +1 -0
  5. sphynx/__init__.py +469 -0
  6. sphynx/__init__.pyi +376 -0
  7. sphynx/_analytical_client.py +77 -0
  8. sphynx/_analytical_values.py +252 -0
  9. sphynx/_audit_client.py +173 -0
  10. sphynx/_comparison_adapters.py +113 -0
  11. sphynx/_componentized_full_rag.py +34 -0
  12. sphynx/_composed_pipeline.py +109 -0
  13. sphynx/_full_rag_grading.py +218 -0
  14. sphynx/_full_rag_trace.py +221 -0
  15. sphynx/_local_eval.py +268 -0
  16. sphynx/_pipeline_core.py +243 -0
  17. sphynx/_pr_client.py +237 -0
  18. sphynx/_rag_diagnostics_calibration.py +285 -0
  19. sphynx/_rag_diagnostics_catalog.py +39 -0
  20. sphynx/_rag_diagnostics_common.py +26 -0
  21. sphynx/_rag_diagnostics_contracts.py +186 -0
  22. sphynx/_rag_diagnostics_engine.py +243 -0
  23. sphynx/_rag_diagnostics_importers.py +242 -0
  24. sphynx/_rag_diagnostics_ranking.py +81 -0
  25. sphynx/_rag_diagnostics_semantic.py +325 -0
  26. sphynx/_rag_ranking.py +149 -0
  27. sphynx/_speech_benchmark_scoring.py +140 -0
  28. sphynx/_speech_benchmark_sources.py +275 -0
  29. sphynx/_speech_complete_scoring.py +218 -0
  30. sphynx/_speech_complete_source.py +118 -0
  31. sphynx/_speech_kido.py +294 -0
  32. sphynx/_speech_scoring.py +77 -0
  33. sphynx/_track_capture.py +213 -0
  34. sphynx/analytical_views.py +515 -0
  35. sphynx/assessment_contracts.py +378 -0
  36. sphynx/assessment_selection.py +57 -0
  37. sphynx/audit_access.py +245 -0
  38. sphynx/audit_contracts.py +204 -0
  39. sphynx/audit_requests.py +180 -0
  40. sphynx/authoring.py +188 -0
  41. sphynx/benchmarks.py +568 -0
  42. sphynx/candidates.py +62 -0
  43. sphynx/catalog.py +272 -0
  44. sphynx/chunking.py +372 -0
  45. sphynx/clickhouse_reader.py +169 -0
  46. sphynx/client.py +486 -0
  47. sphynx/comparison_core.py +149 -0
  48. sphynx/comparison_views.py +223 -0
  49. sphynx/components.py +1661 -0
  50. sphynx/contracts.py +514 -0
  51. sphynx/customer_rerank.py +299 -0
  52. sphynx/customer_retrieval.py +146 -0
  53. sphynx/data/benchmark-releases.v1.json +164 -0
  54. sphynx/data/public-benchmark-release.v1.json +354 -0
  55. sphynx/data/rag-diagnostics-synthetic.v1.json +6982 -0
  56. sphynx/data/rag-paper-references.v1.json +176 -0
  57. sphynx/data/rag-paper-schemas.v1.json +1069 -0
  58. sphynx/data/rag-rerank-v1.json +507 -0
  59. sphynx/dataset_catalog_data.py +1214 -0
  60. sphynx/dataset_readiness.py +138 -0
  61. sphynx/dataset_v3.py +999 -0
  62. sphynx/demo.py +130 -0
  63. sphynx/demo_candidates/__init__.py +1 -0
  64. sphynx/demo_candidates/dense.py +11 -0
  65. sphynx/demo_candidates/lexical.py +21 -0
  66. sphynx/diagnostics.py +207 -0
  67. sphynx/documentation.py +737 -0
  68. sphynx/documentation_candidate.py +939 -0
  69. sphynx/evals.py +538 -0
  70. sphynx/evaluation_protocols.py +124 -0
  71. sphynx/evaluator_bindings.py +370 -0
  72. sphynx/evidence.py +352 -0
  73. sphynx/execution_capabilities.py +109 -0
  74. sphynx/fast_search.py +285 -0
  75. sphynx/full_rag.py +341 -0
  76. sphynx/hosted/__init__.py +18 -0
  77. sphynx/hosted/benchmark_intake.py +218 -0
  78. sphynx/hosted/contracts.py +361 -0
  79. sphynx/hosted/model_admission.py +130 -0
  80. sphynx/hosted/model_artifacts.py +298 -0
  81. sphynx/hosted/model_contracts.py +63 -0
  82. sphynx/hosted/model_gcs.py +181 -0
  83. sphynx/hosted/model_history.py +84 -0
  84. sphynx/hosted/model_session.py +233 -0
  85. sphynx/hosted/public_suite.py +74 -0
  86. sphynx/hosted/registry.py +97 -0
  87. sphynx/hosted/revisions.py +126 -0
  88. sphynx/hosted/sandbox.py +105 -0
  89. sphynx/hosted/submission.py +287 -0
  90. sphynx/hosted_workflow.py +451 -0
  91. sphynx/http_transport.py +86 -0
  92. sphynx/integrated_rag.py +352 -0
  93. sphynx/integrated_workflow.py +552 -0
  94. sphynx/integration_contracts.py +474 -0
  95. sphynx/invocation.py +791 -0
  96. sphynx/invocation_candidates.py +392 -0
  97. sphynx/invocation_catalog_data.py +40 -0
  98. sphynx/learned_catalog_data.py +30 -0
  99. sphynx/learned_rerank.py +267 -0
  100. sphynx/legacy_full_rag.py +67 -0
  101. sphynx/lifecycle.py +885 -0
  102. sphynx/lifecycle_workflow.py +490 -0
  103. sphynx/metric_views.py +474 -0
  104. sphynx/metrics.py +282 -0
  105. sphynx/model_adapters.py +438 -0
  106. sphynx/model_catalog_data.py +30 -0
  107. sphynx/official_benchmarks.py +385 -0
  108. sphynx/packs.py +258 -0
  109. sphynx/pipeline.py +313 -0
  110. sphynx/pipeline_workflow.py +551 -0
  111. sphynx/pr_evaluations.py +560 -0
  112. sphynx/progress.py +89 -0
  113. sphynx/public_bundle.py +173 -0
  114. sphynx/public_eval.py +62 -0
  115. sphynx/public_rerank.py +355 -0
  116. sphynx/py.typed +0 -0
  117. sphynx/pytest_plugin.py +62 -0
  118. sphynx/question_robustness.py +437 -0
  119. sphynx/rag.py +329 -0
  120. sphynx/rag_diagnostics.py +309 -0
  121. sphynx/rag_papers/__init__.py +29 -0
  122. sphynx/rag_papers/__main__.py +45 -0
  123. sphynx/rag_papers/candidates.py +19 -0
  124. sphynx/rag_papers/contracts.py +282 -0
  125. sphynx/rag_papers/datasets.py +291 -0
  126. sphynx/rag_papers/experiments.py +116 -0
  127. sphynx/rag_papers/judges.py +308 -0
  128. sphynx/rag_papers/registry.py +33 -0
  129. sphynx/rag_papers/runner.py +176 -0
  130. sphynx/rag_papers/sable.py +135 -0
  131. sphynx/rag_papers/schemas.py +12 -0
  132. sphynx/rag_papers/scoring.py +249 -0
  133. sphynx/rag_papers/semantic.py +288 -0
  134. sphynx/rag_papers/trec.py +80 -0
  135. sphynx/recorded_scores.py +352 -0
  136. sphynx/records.py +124 -0
  137. sphynx/reflex_traces.py +193 -0
  138. sphynx/report_rendering.py +71 -0
  139. sphynx/rerank_family_contracts.py +150 -0
  140. sphynx/rerank_workflow.py +424 -0
  141. sphynx/research_candidate.py +190 -0
  142. sphynx/results.py +852 -0
  143. sphynx/retrieval.py +750 -0
  144. sphynx/retrieval_candidates.py +295 -0
  145. sphynx/retrieval_catalog_data.py +19 -0
  146. sphynx/retrieval_recordings.py +125 -0
  147. sphynx/reviewer_audit.py +266 -0
  148. sphynx/routing_policies.py +375 -0
  149. sphynx/runner.py +544 -0
  150. sphynx/sable_rerank.py +489 -0
  151. sphynx/score_reliability.py +393 -0
  152. sphynx/speech_benchmarks.py +204 -0
  153. sphynx/speech_campaigns.py +378 -0
  154. sphynx/speech_complete.py +236 -0
  155. sphynx/speech_contracts.py +225 -0
  156. sphynx/speech_evidence.py +429 -0
  157. sphynx/speech_listening.py +115 -0
  158. sphynx/speech_multilingual.py +71 -0
  159. sphynx/speech_tts.py +479 -0
  160. sphynx/sql/001_sync_v1.sql +30 -0
  161. sphynx/stage_contracts.py +199 -0
  162. sphynx/statistical_catalog_data.py +53 -0
  163. sphynx/statistical_rerank.py +232 -0
  164. sphynx/sync.py +797 -0
  165. sphynx/workbench.py +378 -0
  166. sphynx/workbench_catalog_data.py +13 -0
  167. sphynx/workbench_tracks.py +78 -0
  168. sphynx/workflow_async.py +197 -0
  169. sphynx/workflow_contracts.py +594 -0
  170. sphynx/workflow_decisions.py +211 -0
  171. sphynx/workflow_defaults.py +16 -0
  172. sphynx/workflow_extensions.py +212 -0
  173. sphynx/workflow_registry.py +513 -0
  174. sphynx/workflows.py +792 -0
sphynx/speech_tts.py ADDED
@@ -0,0 +1,479 @@
1
+ """Independent, trusted-local TTS candidate execution and saved grading.
2
+
3
+ No hosted admission or automatic provider launch. Freeze source and inputs,
4
+ inspect the plan, then execute fixtures or explicitly authorize provider calls.
5
+ """
6
+ from __future__ import annotations
7
+
8
+ import argparse
9
+ import base64
10
+ from collections import Counter
11
+ import hashlib
12
+ import io
13
+ import json
14
+ import math
15
+ import os
16
+ from pathlib import Path, PurePosixPath
17
+ import platform
18
+ import queue
19
+ import re
20
+ import subprocess
21
+ import sys
22
+ import threading
23
+ import time
24
+ import uuid
25
+ import wave
26
+
27
+ from .speech_contracts import (EVENT_SCHEMA, FORMAT, TTS_REQUEST, SpeechContractError,
28
+ StreamValidator, checksum, encode, fields, identifier, parse, require, sha, validate_method, validate_request)
29
+ from .speech_evidence import _new_directory as _saved_new_directory, _read as _saved_read, _write_files
30
+
31
+ SUITE_ID = "speech.tts.execution.v1"
32
+ PLAN_SCHEMA = "sphynx.speech.tts.plan.v1"
33
+ INVOCATION_SCHEMA = "sphynx.speech.tts.execution-invocation.v1"
34
+ FAMILY_SCHEMA = "sphynx.speech.tts.family.v1"
35
+ RUN_SCHEMA = "sphynx.speech.tts.run.v1"
36
+ GRADER = "speech.tts.integrity-grader.v1"
37
+ FAMILY_PATH = "sphynx_candidates/tts/family.json"
38
+ _BOOTSTRAP = {"PATH", "PATHEXT", "SYSTEMROOT", "WINDIR", "COMSPEC", "TEMP", "TMP", "TMPDIR", "HOME", "USERPROFILE", "LANG"}
39
+ _CREDENTIAL = {"elevenlabs": "ELEVEN_API_KEY", "cartesia": "CARTESIA_API_KEY"}
40
+
41
+
42
+ def _new_directory(path):
43
+ try:
44
+ return _saved_new_directory(path)
45
+ except ValueError as exc:
46
+ raise SpeechContractError(str(exc)) from None
47
+
48
+
49
+ def _read(root, path):
50
+ try:
51
+ return _saved_read(root, path)
52
+ except ValueError as exc:
53
+ raise SpeechContractError(str(exc)) from None
54
+
55
+
56
+ def describe_tts_suite():
57
+ return {"suite_id": SUITE_ID, "plan_schema": PLAN_SCHEMA, "invocation_schema": INVOCATION_SCHEMA,
58
+ "request_schema": TTS_REQUEST, "event_schema": EVENT_SCHEMA, "grader": GRADER,
59
+ "execution": "trusted_local_subprocess", "hosted_admitted": False,
60
+ "provider_authorization": "explicit_launch_flag_required", "os_isolation": False,
61
+ "perceived_latency_measured": False, "stt_execution": "contract_only"}
62
+
63
+
64
+ def _relative(path):
65
+ require(isinstance(path, str) and re.fullmatch(r"[A-Za-z0-9_./-]{1,240}", path)
66
+ and all(p not in ("", ".", "..") and not p.startswith(".") for p in path.split("/")),
67
+ "invalid speech source path")
68
+ return path
69
+
70
+
71
+ def _family(raw):
72
+ value = parse(raw)
73
+ fields(value, ("schema_version", "entry_point", "source_files", "methods"))
74
+ require(value["schema_version"] == FAMILY_SCHEMA, "unsupported speech candidate family")
75
+ _relative(value["entry_point"])
76
+ require(isinstance(value["source_files"], list) and 1 <= len(value["source_files"]) <= 16,
77
+ "invalid speech source closure")
78
+ for path in value["source_files"]:
79
+ _relative(path)
80
+ require(len(set(value["source_files"])) == len(value["source_files"]) and FAMILY_PATH in value["source_files"]
81
+ and value["entry_point"] in value["source_files"], "incomplete candidate source closure")
82
+ require(isinstance(value["methods"], dict) and 1 <= len(value["methods"]) <= 8, "invalid speech method census")
83
+ for key, method in value["methods"].items():
84
+ identifier(key); validate_method(method)
85
+ return value
86
+
87
+
88
+ def _git(root, *args):
89
+ result = subprocess.run(["git", "-C", str(root), *args], capture_output=True, timeout=15, check=False)
90
+ require(result.returncode == 0 and len(result.stdout) <= 1_000_000, "could not verify local candidate Git source")
91
+ return result.stdout
92
+
93
+
94
+ def freeze_tts_plan(repository, *, source_commit, source_pr, cases, cohort_identity,
95
+ baseline, candidate, output_dir, repetitions=1, timeout_seconds=30,
96
+ max_audio_bytes=2_000_000, max_requests=20):
97
+ """Freeze two named methods from one exact local Git commit, without execution."""
98
+ require(isinstance(source_commit, str) and re.fullmatch(r"[0-9a-f]{40}", source_commit), "pin the complete candidate commit")
99
+ match = re.fullmatch(r"https://github.com/([A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+)/pull/([1-9][0-9]*)", source_pr)
100
+ require(match is not None, "pin the candidate source PR")
101
+ root = Path(repository).absolute()
102
+ origin = _git(root, "config", "--get", "remote.origin.url").decode().strip()
103
+ repo_name = match.group(1)
104
+ require(origin in (f"https://github.com/{repo_name}.git", f"https://github.com/{repo_name}", f"git@github.com:{repo_name}.git"),
105
+ "candidate PR and local repository origin differ")
106
+ require(_git(root, "rev-parse", "HEAD").decode().strip() == source_commit, "candidate checkout is not the requested commit")
107
+ family_raw = _git(root, "cat-file", "blob", source_commit + ":" + FAMILY_PATH)
108
+ family = _family(family_raw)
109
+ require(baseline != candidate and baseline in family["methods"] and candidate in family["methods"], "select two distinct named methods")
110
+ require(family["methods"][baseline]["mode"] == family["methods"][candidate]["mode"], "fixture and provider arms cannot be mixed")
111
+ require(type(repetitions) is int and 1 <= repetitions <= 3 and type(timeout_seconds) is int and 1 <= timeout_seconds <= 120,
112
+ "invalid TTS trial limits")
113
+ require(type(max_audio_bytes) is int and 2 <= max_audio_bytes <= 8_000_000 and max_audio_bytes % 2 == 0,
114
+ "invalid TTS audio limit")
115
+ require(type(max_requests) is int and 2 <= max_requests <= 20, "pilot request budget must be between 2 and 20")
116
+ require(isinstance(cases, list) and 0 < len(cases) <= 10, "select one to ten pilot cases")
117
+ for case in cases:
118
+ fields(case, ("id", "text", "language")); identifier(case["id"])
119
+ require(isinstance(case["text"], str) and 0 < len(case["text"]) <= 10000, "invalid TTS pilot text")
120
+ require(isinstance(case["language"], str) and re.fullmatch(r"[a-z]{2}(?:-[A-Za-z]{2,4})?", case["language"]), "invalid TTS pilot language")
121
+ require(len({c["id"] for c in cases}) == len(cases), "duplicate pilot cases")
122
+ fields(cohort_identity, ("kind", "sha256")); identifier(cohort_identity["kind"]); checksum(cohort_identity["sha256"])
123
+ planned_calls = len(cases) * 2 * repetitions
124
+ require(planned_calls <= max_requests, "pilot exceeds request budget")
125
+ require(sum(len(c["text"]) for c in cases) * 2 * repetitions <= 20000, "pilot exceeds input-character budget")
126
+ sources = {}
127
+ for path in family["source_files"]:
128
+ blob = _git(root, "cat-file", "blob", source_commit + ":" + path)
129
+ require(_read(root, path) == blob, "candidate working bytes differ from pinned Git source")
130
+ sources["source/" + path] = blob
131
+ output = _new_directory(output_dir)
132
+ files = _write_files(output, sources)
133
+ plan = {"schema_version": PLAN_SCHEMA, "suite_id": SUITE_ID, "id": "tts-plan-" + uuid.uuid4().hex,
134
+ "source_pr": source_pr, "source_commit": source_commit, "family_path": FAMILY_PATH,
135
+ "family_sha256": sha(family_raw), "entry_point": family["entry_point"], "files": files,
136
+ "baseline": baseline, "candidate": candidate, "methods": {k: family["methods"][k] for k in (baseline, candidate)},
137
+ "cases": cases, "cases_sha256": sha(encode(cases)), "cohort_identity": cohort_identity,
138
+ "repetitions": repetitions, "limits": {"timeout_seconds": timeout_seconds, "max_audio_bytes": max_audio_bytes},
139
+ "budget": {"max_requests": max_requests, "planned_requests": planned_calls,
140
+ "planned_input_characters": sum(len(c["text"]) for c in cases) * 2 * repetitions,
141
+ "retries": 0, "cost_usd": None, "monetary_cap_enforced": False},
142
+ "mode": family["methods"][baseline]["mode"], "review_status": "prepared_for_review",
143
+ "model_voice_availability_verified": False, "hosted_admitted": False}
144
+ raw = encode(plan); (output / "plan.json").write_bytes(raw)
145
+ return {"path": str(output), "plan_sha256": sha(raw), "suite_id": SUITE_ID, "budget": plan["budget"],
146
+ "mode": plan["mode"], "provider_calls_made": False}
147
+
148
+
149
+ def _load_plan(path, expected_sha256):
150
+ root = Path(path).absolute()
151
+ raw = _read(root, "plan.json")
152
+ require(sha(raw) == checksum(expected_sha256), "TTS plan digest differs")
153
+ plan = parse(raw)
154
+ require(plan.get("schema_version") == PLAN_SCHEMA and plan.get("suite_id") == SUITE_ID,
155
+ "not an executable TTS plan")
156
+ require(isinstance(plan.get("files"), dict) and len(plan["files"]) <= 16, "invalid TTS source census")
157
+ for path_, receipt in plan["files"].items():
158
+ require(path_.startswith("source/"), "invalid TTS source archive path")
159
+ _relative(path_[7:])
160
+ data = _read(root, path_)
161
+ require(len(data) == receipt["size_bytes"] and sha(data) == receipt["sha256"], "frozen TTS source changed")
162
+ family = _family(_read(root, "source/" + FAMILY_PATH))
163
+ require(plan["family_sha256"] == sha(_read(root, "source/" + FAMILY_PATH)) and
164
+ set(plan["files"]) == {"source/" + p for p in family["source_files"]} and
165
+ plan["entry_point"] == family["entry_point"], "TTS source closure differs")
166
+ require(plan["baseline"] != plan["candidate"] and set(plan["methods"]) == {plan["baseline"], plan["candidate"]}, "TTS arm census differs")
167
+ require(all(plan["methods"][k] == family["methods"].get(k) for k in plan["methods"]), "TTS selected method changed")
168
+ require(all(m["mode"] == plan["mode"] for m in plan["methods"].values()), "TTS execution mode differs")
169
+ require(sha(encode(plan["cases"])) == plan["cases_sha256"] and len({c["id"] for c in plan["cases"]}) == len(plan["cases"]),
170
+ "TTS cases changed")
171
+ require(type(plan["repetitions"]) is int and 1 <= plan["repetitions"] <= 3 and 1 <= len(plan["cases"]) <= 10,
172
+ "TTS trial population is invalid")
173
+ require(plan["budget"]["planned_requests"] == len(plan["cases"]) * 2 * plan["repetitions"] <= plan["budget"]["max_requests"] <= 20
174
+ and plan["budget"]["retries"] == 0, "TTS request budget differs")
175
+ require(plan["budget"]["planned_input_characters"] == sum(len(c["text"]) for c in plan["cases"]) * 2 * plan["repetitions"] <= 20000,
176
+ "TTS character budget differs")
177
+ for case in plan["cases"]:
178
+ fields(case, ("id", "text", "language"))
179
+ for method in plan["methods"]:
180
+ validate_request(_request(plan, case, method, "preflight"))
181
+ return root, plan
182
+
183
+
184
+ def _request(plan, case, method, request_id):
185
+ config = plan["methods"][method]
186
+ return {"schema_version": TTS_REQUEST, "request_id": request_id, "case_id": case["id"], "method_id": method,
187
+ "config": config, "config_sha256": sha(encode(config)), "input": {"text": case["text"], "language": case["language"]},
188
+ "format": FORMAT, "limits": plan["limits"]}
189
+
190
+
191
+ def _execute(root, plan, request, authorize):
192
+ validator = StreamValidator(request)
193
+ env = {k: v for k, v in os.environ.items() if k.upper() in _BOOTSTRAP}
194
+ env.update(PYTHONPATH=str(Path(__file__).resolve().parents[1]), PYTHONUTF8="1", PYTHONDONTWRITEBYTECODE="1")
195
+ if request["config"]["mode"] == "provider":
196
+ require(authorize, "provider calls require explicit launch authorization")
197
+ key = _CREDENTIAL[request["config"]["provider"]]
198
+ require(bool(os.environ.get(key)), "selected provider credential is unavailable")
199
+ env[key] = os.environ[key]
200
+ env["SPHYNX_TTS_ALLOW_PROVIDER"] = "1"
201
+ started = time.monotonic_ns()
202
+ process = subprocess.Popen([sys.executable, str(root / "source" / plan["entry_point"])],
203
+ cwd=root / "source", env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
204
+ events_queue, stop = queue.Queue(maxsize=64), threading.Event()
205
+ stderr_count = [0]
206
+ def enqueue(item):
207
+ while not stop.is_set():
208
+ try:
209
+ events_queue.put(item, timeout=.05); return
210
+ except queue.Full:
211
+ pass
212
+ def read_stdout():
213
+ try:
214
+ while not stop.is_set():
215
+ line = process.stdout.readline(16385)
216
+ if not line:
217
+ enqueue(("eof", None, time.monotonic_ns())); return
218
+ if len(line) > 16384:
219
+ enqueue(("overflow", None, time.monotonic_ns())); return
220
+ enqueue(("line", line, time.monotonic_ns()))
221
+ except OSError:
222
+ enqueue(("reader_error", None, time.monotonic_ns()))
223
+ def discard_stderr():
224
+ try:
225
+ while True:
226
+ chunk = process.stderr.read(8192)
227
+ if not chunk:
228
+ return
229
+ stderr_count[0] += len(chunk)
230
+ except OSError:
231
+ pass
232
+ def write_input():
233
+ try:
234
+ process.stdin.write(encode(request)); process.stdin.flush()
235
+ except (OSError, ValueError):
236
+ pass
237
+ finally:
238
+ try:
239
+ process.stdin.close()
240
+ except (OSError, ValueError):
241
+ pass
242
+ readers = [threading.Thread(target=read_stdout, daemon=True), threading.Thread(target=discard_stderr, daemon=True),
243
+ threading.Thread(target=write_input, daemon=True)]
244
+ for reader in readers:
245
+ reader.start()
246
+ audio, events, first, failure, stream_bytes = bytearray(), [], None, None, 0
247
+ deadline = time.monotonic() + request["limits"]["timeout_seconds"]
248
+ try:
249
+ while True:
250
+ left = deadline - time.monotonic()
251
+ if left <= 0:
252
+ failure = "timeout"; break
253
+ try:
254
+ kind, line, arrived = events_queue.get(timeout=left)
255
+ except queue.Empty:
256
+ failure = "timeout"; break
257
+ if kind == "eof":
258
+ break
259
+ if kind != "line":
260
+ failure = "stream_limit_or_read_error"; break
261
+ if len(events) >= 4096:
262
+ failure = "event_limit"; break
263
+ stream_bytes += len(line)
264
+ if stream_bytes > request["limits"]["max_audio_bytes"] * 2 + 65536:
265
+ failure = "stream_limit"; break
266
+ event = parse(line)
267
+ chunk = validator.accept(event)
268
+ if chunk:
269
+ if first is None:
270
+ first = arrived
271
+ audio.extend(chunk)
272
+ events.append({"event": event, "parent_received_ns": arrived - started})
273
+ if failure is None:
274
+ process.wait(timeout=max(.01, deadline - time.monotonic()))
275
+ if process.returncode != 0 or validator.terminal is None:
276
+ failure = "process_error_or_missing_terminal"
277
+ elif validator.terminal["status"] != "success":
278
+ failure = validator.terminal["error_code"]
279
+ except (SpeechContractError, KeyError, TypeError, ValueError):
280
+ failure = "invalid_stream"
281
+ except subprocess.TimeoutExpired:
282
+ failure = "timeout"
283
+ finally:
284
+ stop.set()
285
+ if process.poll() is None:
286
+ process.kill(); process.wait(timeout=5)
287
+ for reader in readers:
288
+ reader.join(timeout=1)
289
+ process.stdout.close(); process.stderr.close()
290
+ if not process.stdin.closed:
291
+ try:
292
+ process.stdin.close()
293
+ except OSError:
294
+ pass
295
+ return {"request_id": request["request_id"], "case_id": request["case_id"], "method_id": request["method_id"],
296
+ "config_sha256": request["config_sha256"], "success": failure is None, "failure": failure,
297
+ "audio_bytes": len(audio), "audio_sha256": sha(bytes(audio)),
298
+ "pcm_complete": failure is None and len(audio) > 0 and len(audio) % 2 == 0,
299
+ "parent_first_audio_ms": (first - started) / 1_000_000 if first is not None else None,
300
+ "parent_completion_ms": (time.monotonic_ns() - started) / 1_000_000,
301
+ "timing_scope": "parent-process-launch-to-validated-audio-event-includes-startup-and-IPC",
302
+ "candidate_reported_first_audio_ns": next((e["event"]["t_ns"] for e in events if e["event"]["type"] == "audio"), None),
303
+ "identity_evidence": validator.terminal["identity_evidence"] if validator.terminal else None,
304
+ "stderr_bytes_discarded": stderr_count[0], "exit_code": process.returncode,
305
+ "events": events}, bytes(audio)
306
+
307
+
308
+ def run_tts_plan(path, *, expected_plan_sha256, output_dir, allow_provider_calls=False):
309
+ root, plan = _load_plan(path, expected_plan_sha256)
310
+ require(type(allow_provider_calls) is bool, "provider launch authorization must be explicit")
311
+ require(plan["mode"] == "fixture" or allow_provider_calls, "provider calls require explicit launch authorization")
312
+ # Validate both selected credentials before creating a run or starting either arm.
313
+ if plan["mode"] == "provider":
314
+ require(all(os.environ.get(_CREDENTIAL[m["provider"]]) for m in plan["methods"].values()), "selected provider credential is unavailable")
315
+ output = _new_directory(output_dir)
316
+ frozen = {"frozen-plan/plan.json": _read(root, "plan.json")}
317
+ frozen.update({"frozen-plan/" + name: _read(root, name) for name in plan["files"]})
318
+ artifacts = _write_files(output, frozen)
319
+ root, plan = _load_plan(output / "frozen-plan", expected_plan_sha256)
320
+ invocation_id = "tts-invocation-" + uuid.uuid4().hex
321
+ invocation = {"schema_version": INVOCATION_SCHEMA, "suite_id": SUITE_ID, "id": invocation_id,
322
+ "plan_sha256": expected_plan_sha256, "source_pr": plan["source_pr"], "source_commit": plan["source_commit"],
323
+ "mode": plan["mode"], "provider_calls_authorized": allow_provider_calls and plan["mode"] == "provider",
324
+ "provider_calls_observed": None if plan["mode"] == "provider" else 0,
325
+ "runtime": {"python": sys.version, "platform": platform.platform(), "machine": platform.machine()},
326
+ "implementation_sha256": {name: sha(Path(__file__).with_name(name).read_bytes()) for name in ("speech_tts.py", "speech_contracts.py")}}
327
+ (output / "invocation.json").write_bytes(encode(invocation))
328
+ captures = []
329
+ artifacts["invocation.json"] = {"sha256": sha(encode(invocation)), "size_bytes": len(encode(invocation))}
330
+ for rep in range(plan["repetitions"]):
331
+ for case_index, case in enumerate(plan["cases"]):
332
+ arms = (plan["baseline"], plan["candidate"])
333
+ if (rep + case_index) % 2:
334
+ arms = tuple(reversed(arms))
335
+ for method in arms:
336
+ request_id = "tts-request-" + uuid.uuid4().hex
337
+ request = _request(plan, case, method, request_id)
338
+ try:
339
+ capture, audio = _execute(root, plan, request, allow_provider_calls)
340
+ except OSError:
341
+ capture, audio = {"request_id": request_id, "case_id": case["id"], "method_id": method,
342
+ "success": False, "failure": "process_start_error", "audio_bytes": 0, "audio_sha256": sha(b""),
343
+ "pcm_complete": False, "parent_first_audio_ms": None, "parent_completion_ms": None,
344
+ "events": []}, b""
345
+ capture["repetition"] = rep + 1
346
+ capture["order_index"] = len(captures)
347
+ capture["request_sha256"] = sha(encode(request))
348
+ relative = "captures/" + request_id + "/"
349
+ payloads = {relative + "request.json": encode(request), relative + "capture.json": encode(capture)}
350
+ if audio:
351
+ wav = io.BytesIO()
352
+ with wave.open(wav, "wb") as stream:
353
+ stream.setnchannels(1); stream.setsampwidth(2); stream.setframerate(16000); stream.writeframes(audio)
354
+ payloads[relative + "audio.wav"] = wav.getvalue()
355
+ artifacts.update(_write_files(output, payloads))
356
+ captures.append(capture)
357
+ report = grade_tts_captures(plan, captures)
358
+ report.update(invocation_id=invocation_id, invocation_sha256=sha(encode(invocation)), plan_sha256=expected_plan_sha256)
359
+ artifacts.update(_write_files(output, {"report.json": encode(report)}))
360
+ manifest = {"schema_version": RUN_SCHEMA, "suite_id": SUITE_ID, "invocation_id": invocation_id,
361
+ "plan_sha256": expected_plan_sha256, "files": artifacts,
362
+ "completed_requests": len(captures), "planned_requests": plan["budget"]["planned_requests"],
363
+ "status": "completed" if all(c["success"] for c in captures) else "completed_with_failures"}
364
+ raw = encode(manifest); (output / "manifest.json").write_bytes(raw)
365
+ return {"path": str(output), "suite_id": SUITE_ID, "invocation_id": invocation_id,
366
+ "manifest_sha256": sha(raw), "status": manifest["status"], "mode": plan["mode"]}
367
+
368
+
369
+ def grade_tts_captures(plan, captures):
370
+ for capture in captures:
371
+ require(type(capture.get("success")) is bool, "TTS outcome must be a boolean")
372
+ value = capture.get("parent_first_audio_ms")
373
+ require(value is None or (type(value) in (int, float) and math.isfinite(value) and value >= 0),
374
+ "TTS timing must be finite and nonnegative")
375
+ expected = {(case["id"], method, rep + 1) for case in plan["cases"] for method in plan["methods"] for rep in range(plan["repetitions"])}
376
+ actual = [(c["case_id"], c["method_id"], c["repetition"]) for c in captures]
377
+ require(len(actual) == len(set(actual)) and set(actual) == expected, "TTS capture census is incomplete or duplicated")
378
+ arms = {}
379
+ for method in plan["methods"]:
380
+ rows = [c for c in captures if c["method_id"] == method]
381
+ times = [c["parent_first_audio_ms"] for c in rows if c["success"] and c["parent_first_audio_ms"] is not None]
382
+ arms[method] = {"attempts": len(rows), "successful": sum(c["success"] for c in rows),
383
+ "failure_counts": dict(Counter(c["failure"] for c in rows if c["failure"])),
384
+ "observed_success_first_audio_ms_mean": sum(times) / len(times) if times else None,
385
+ "timing_count": len(times), "provider_latency_ms": None, "perceived_latency_ms": None}
386
+ pairs = []
387
+ for case in plan["cases"]:
388
+ for rep in range(1, plan["repetitions"] + 1):
389
+ pair = {c["method_id"]: c for c in captures if c["case_id"] == case["id"] and c["repetition"] == rep}
390
+ left, right = pair[plan["baseline"]], pair[plan["candidate"]]
391
+ eligible = left["success"] and right["success"] and left["parent_first_audio_ms"] is not None and right["parent_first_audio_ms"] is not None
392
+ pairs.append({"case_id": case["id"], "repetition": rep, "paired_success": eligible,
393
+ "parent_first_audio_delta_ms": right["parent_first_audio_ms"] - left["parent_first_audio_ms"] if eligible else None,
394
+ "identical_pcm": left["audio_sha256"] == right["audio_sha256"] if eligible else None})
395
+ observed = [p["parent_first_audio_delta_ms"] for p in pairs if p["parent_first_audio_delta_ms"] is not None]
396
+ return {"schema_version": "sphynx.speech.tts.integrity-report.v1", "suite_id": SUITE_ID, "grader": GRADER,
397
+ "mode": plan["mode"], "arms": arms, "pairs": pairs, "paired_count": len(observed), "missing_pairs": len(pairs) - len(observed),
398
+ "primary_parent_first_audio_delta_ms": sum(observed) / len(observed) if observed and len(observed) == len(pairs) else None,
399
+ "confidence_interval": None, "quality": None, "cost_usd": None,
400
+ "timing_scope": "local process startup plus adapter plus IPC; not pure provider or audible latency",
401
+ "claim_scope": "fixture plumbing only" if plan["mode"] == "fixture" else "trusted local pilot; requested model/voice not independently attested"}
402
+
403
+
404
+ def load_tts_run(path, *, expected_manifest_sha256):
405
+ root = Path(path).absolute(); raw = _read(root, "manifest.json")
406
+ require(sha(raw) == checksum(expected_manifest_sha256), "TTS run manifest differs")
407
+ manifest = parse(raw)
408
+ require(manifest.get("schema_version") == RUN_SCHEMA and manifest.get("suite_id") == SUITE_ID, "not a TTS execution run")
409
+ require(isinstance(manifest.get("files"), dict) and {"report.json", "invocation.json", "frozen-plan/plan.json"}.issubset(manifest["files"]),
410
+ "incomplete TTS run file census")
411
+ for name, receipt in manifest["files"].items():
412
+ data = _read(root, name)
413
+ require(sha(data) == receipt["sha256"] and len(data) == receipt["size_bytes"], "saved TTS artifact changed")
414
+ invocation, report = parse(_read(root, "invocation.json")), parse(_read(root, "report.json"))
415
+ require(report["invocation_sha256"] == sha(encode(invocation)) and
416
+ report["invocation_id"] == invocation["id"] == manifest["invocation_id"] and
417
+ report["plan_sha256"] == invocation["plan_sha256"] == manifest["plan_sha256"], "TTS run identity binding differs")
418
+ require(invocation.get("schema_version") == INVOCATION_SCHEMA and invocation.get("suite_id") == report.get("suite_id") == SUITE_ID,
419
+ "TTS invocation namespace differs")
420
+ _, plan = _load_plan(root / "frozen-plan", manifest["plan_sha256"])
421
+ captures = [parse(_read(root, name)) for name in manifest["files"] if name.startswith("captures/") and name.endswith("/capture.json")]
422
+ require(manifest["completed_requests"] == len(captures) == manifest["planned_requests"] == plan["budget"]["planned_requests"],
423
+ "saved TTS request census differs")
424
+ recomputed = grade_tts_captures(plan, captures)
425
+ cases = {c["id"]: c for c in plan["cases"]}
426
+ for capture in captures:
427
+ relative = "captures/" + identifier(capture["request_id"]) + "/"
428
+ expected_request = _request(plan, cases[capture["case_id"]], capture["method_id"], capture["request_id"])
429
+ require(_read(root, relative + "request.json") == encode(expected_request) and
430
+ capture["request_sha256"] == sha(encode(expected_request)), "saved TTS request differs from frozen inputs")
431
+ validator = StreamValidator(expected_request)
432
+ chunks = []
433
+ for item in capture["events"]:
434
+ chunks.append(validator.accept(item["event"]))
435
+ pcm = b"".join(chunks)
436
+ require(sha(pcm) == capture["audio_sha256"] and len(pcm) == capture["audio_bytes"], "saved TTS audio differs from events")
437
+ require(capture["success"] == (capture["failure"] is None), "TTS success and failure status differ")
438
+ if capture["success"]:
439
+ require(validator.terminal is not None and validator.terminal["status"] == "success", "successful saved TTS stream lacks terminal")
440
+ if pcm:
441
+ require(relative + "audio.wav" in manifest["files"], "saved TTS audio artifact is missing")
442
+ with wave.open(io.BytesIO(_read(root, relative + "audio.wav")), "rb") as stream:
443
+ require((stream.getnchannels(), stream.getsampwidth(), stream.getframerate()) == (1, 2, 16000)
444
+ and stream.readframes(stream.getnframes()) == pcm, "saved WAV differs from speech stream")
445
+ require(all(report.get(k) == v for k, v in recomputed.items()), "saved TTS report differs from capture grading")
446
+ return report
447
+
448
+
449
+ def main(argv=None):
450
+ parser = argparse.ArgumentParser(description="Independent trusted-local speech.tts.execution.v1 suite")
451
+ sub = parser.add_subparsers(dest="command", required=True)
452
+ sub.add_parser("describe")
453
+ freeze = sub.add_parser("freeze")
454
+ for arg in ("repository", "source-commit", "source-pr", "cases-json", "baseline", "candidate", "output"):
455
+ freeze.add_argument("--" + arg, required=True)
456
+ freeze.add_argument("--repetitions", type=int, default=1)
457
+ launch = sub.add_parser("run")
458
+ for arg in ("plan", "expected-plan-sha256", "output"):
459
+ launch.add_argument("--" + arg, required=True)
460
+ launch.add_argument("--allow-provider-calls", action="store_true")
461
+ args = parser.parse_args(argv)
462
+ try:
463
+ if args.command == "describe":
464
+ result = describe_tts_suite()
465
+ elif args.command == "freeze":
466
+ cases = parse(Path(args.cases_json).read_bytes())
467
+ result = freeze_tts_plan(args.repository, source_commit=args.source_commit, source_pr=args.source_pr,
468
+ cases=cases, cohort_identity={"kind": "owner-supplied-cases", "sha256": sha(encode(cases))},
469
+ baseline=args.baseline, candidate=args.candidate, repetitions=args.repetitions, output_dir=args.output)
470
+ else:
471
+ result = run_tts_plan(args.plan, expected_plan_sha256=args.expected_plan_sha256,
472
+ output_dir=args.output, allow_provider_calls=args.allow_provider_calls)
473
+ except (SpeechContractError, ValueError, OSError, KeyError, TypeError, subprocess.SubprocessError):
474
+ parser.exit(2, "TTS operation failed; check frozen identities, limits and launch authorization\n")
475
+ print(json.dumps(result, indent=2))
476
+
477
+
478
+ if __name__ == "__main__":
479
+ main()
@@ -0,0 +1,30 @@
1
+ -- Sphynx-owned projection schema. Provision {database} and grant the sync
2
+ -- principal INSERT/SELECT on only these tables before invoking sync().
3
+ -- No Litmus database, tables, buckets, or credentials are used.
4
+ CREATE TABLE IF NOT EXISTS {database}.sphynx_facts_v1
5
+ (
6
+ projection_id String,
7
+ row_id String,
8
+ row_hash String,
9
+ fact_type LowCardinality(String),
10
+ run_id String,
11
+ case_id String,
12
+ capture_id String,
13
+ evaluation_id String,
14
+ metric_id String,
15
+ capability_id String,
16
+ status LowCardinality(String),
17
+ numeric_value Nullable(Float64),
18
+ payload_json String
19
+ )
20
+ ENGINE = MergeTree
21
+ ORDER BY (projection_id, row_id);
22
+
23
+ CREATE TABLE IF NOT EXISTS {database}.sphynx_projection_commits_v1
24
+ (
25
+ projection_id String,
26
+ commit_hash String,
27
+ payload_json String
28
+ )
29
+ ENGINE = MergeTree
30
+ ORDER BY projection_id;