sable-sphynx 0.7.0.dev0__py3-none-any.whl

sphynx/_speech_scoring.py

@@ -0,0 +1,77 @@
+"""Pure, versioned speech scoring; no providers or RAG dependencies.
+
+The normalization and structured-token compatibility score reproduce
+sable-tts-eval/scripts/wer_cer.py at 71ec7e8. Its token matcher deliberately
+uses substring presence; this is not a stricter entity-boundary metric.
+"""
+from __future__ import annotations
+
+import re
+import unicodedata
+
+NORMALIZATION = "kido.nfkc-lower-punctuation.v1"
+TOKEN_SCORER = "kido.structured-substring-presence.v1"
+PATTERNS = {
+    "identifier": r"\b[A-Z]{2,}-\d{3,}-?[A-Z]?\b",
+    "currency": r"\$\s?\d[\d,]*(?:\.\d+)?",
+    "number": r"\b\d[\d,]*(?:\.\d+)?\b",
+    "email": r"[\w.+-]+@[\w-]+\.[\w.-]+",
+    "url": r"\b(?:[\w-]+\.)+[a-z]{2,}(?:/[\w./-]*)?\b",
+    "acronym": r"\b[A-Z]{2,6}(?:\s?\d)?\b",
+}
+
+
+def normalize(text: str, *, strip_punctuation: bool = True) -> str:
+    text = unicodedata.normalize("NFKC", text).lower()
+    if strip_punctuation:
+        text = re.sub(r"['’]", "", text)
+        text = re.sub(r"[^\w\s@./\-]", " ", text)
+    return re.sub(r"\s+", " ", text).strip()
+
+
+def edit_distance(a, b) -> int:
+    previous = list(range(len(b) + 1))
+    for i, value in enumerate(a, 1):
+        current = [i]
+        for j, other in enumerate(b, 1):
+            current.append(min(previous[j] + 1, current[-1] + 1,
+                               previous[j - 1] + (value != other)))
+        previous = current
+    return previous[-1]
+
+
+def score_transcript(reference: str, hypothesis: str) -> dict:
+    ref, hyp = normalize(reference), normalize(hypothesis)
+    words, chars = ref.split(), list(ref)
+    word_errors, char_errors = edit_distance(words, hyp.split()), edit_distance(chars, list(hyp))
+    tokens = {}
+    for name, pattern in PATTERNS.items():
+        expected = re.findall(pattern, reference)
+        observed = normalize(hypothesis, strip_punctuation=False)
+        matched = sum(normalize(token, strip_punctuation=False) in observed for token in expected)
+        tokens[name] = {"matched": matched, "reference_tokens": len(expected),
+                        "accuracy": matched / len(expected) if expected else None}
+    return {"reference_words": len(words), "word_errors": word_errors,
+            "wer": word_errors / len(words) if words else None,
+            "reference_characters": len(chars), "character_errors": char_errors,
+            "cer": char_errors / len(chars) if chars else None,
+            "structured_tokens": tokens,
+            "silence_inserted_words": len(hyp.split()) if not words else 0}
+
+
+def timing_summary(values: list[float], *, convention: str) -> dict:
+    """Retain two historical quantile conventions explicitly, never pool them."""
+    import statistics
+    values = sorted(values)
+    if not values:
+        return {"n": 0, "mean": None, "p50": None, "p95": None, "convention": convention}
+    n = len(values)
+    if convention == "kido.isolated-rounded-index.v1":
+        quantile = lambda p: values[max(0, min(n - 1, round(n * p) - 1))]
+        p50, p95 = quantile(.5), quantile(.95)
+    elif convention == "kido.live-median-rounded-index.v1":
+        p50, p95 = statistics.median(values), values[min(n - 1, round(.95 * (n - 1)))]
+    else:
+        raise ValueError("unknown speech percentile convention")
+    return {"n": n, "mean": sum(values) / n, "p50": p50, "p95": p95,
+            "convention": convention}

sphynx/_track_capture.py

@@ -0,0 +1,213 @@
+"""Durable local captures shared by chunking, pipeline and full-RAG tracks."""
+from __future__ import annotations
+
+import hashlib
+import json
+import math
+from collections.abc import Mapping
+from pathlib import Path
+from uuid import uuid4
+from .contracts import ContractError, canonical_json, _id
+from .integration_contracts import (ArtifactBinding, TrackCaptureManifest, TrackRunRef,
+    TrackEvaluationManifest, TrackEvaluationRef, require_digest)
+
+
+def byte_digest(data):
+    return 'sha256:' + hashlib.sha256(data).hexdigest()
+
+
+def strict_json(data):
+    def pairs(items):
+        result = {}
+        for key, value in items:
+            if key in result:
+                raise ContractError('duplicate JSON object key')
+            result[key] = value
+        return result
+    def constant(value):
+        raise ContractError('nonfinite JSON constant')
+    try:
+        result = json.loads(data, object_pairs_hook=pairs, parse_constant=constant)
+        # Reject overflow such as 1e999 as well as NaN literals.
+        canonical_json(result)
+        def check(value):
+            if isinstance(value, float) and not math.isfinite(value):
+                raise ContractError('nonfinite JSON number')
+            if isinstance(value, dict):
+                for item in value.values(): check(item)
+            if isinstance(value, list):
+                for item in value: check(item)
+        check(result)
+        return result
+    except (ValueError, TypeError, UnicodeError) as error:
+        raise ContractError('invalid strict JSON artifact') from error
+
+
+def _request_bytes(value):
+    """Validate JSON types before encoding; JSON must never coerce object keys."""
+    def validate(item):
+        if item is None or type(item) in (str, bool, int):
+            return item
+        if type(item) is float and math.isfinite(item):
+            return item
+        if isinstance(item, Mapping):
+            if any(type(key) is not str for key in item):
+                raise ContractError('request JSON keys must be strings')
+            return {key: validate(child) for key, child in item.items()}
+        if isinstance(item, (list, tuple)):
+            return [validate(child) for child in item]
+        raise ContractError('request must contain finite JSON values')
+    encoded = canonical_json(validate(value)).encode('utf-8')
+    strict_json(encoded)
+    return encoded
+
+
+def safe_path(root, relative):
+    # Reuse the frozen wire path validator before touching the filesystem.
+    ArtifactBinding(relative, 'sha256:' + '0' * 64, 0)
+    root = Path(root).absolute()
+    if root.is_symlink():
+        raise ContractError('capture root cannot be a symlink')
+    resolved_root = root.resolve()
+    current = root
+    for part in relative.split('/'):
+        current = current / part
+        if current.is_symlink():
+            raise ContractError('capture artifact cannot traverse a symlink')
+    if not current.resolve().is_relative_to(resolved_root):
+        raise ContractError('artifact escapes capture root')
+    return current
+
+
+def _read_file(path, expected_digest):
+    require_digest(expected_digest, 'expected digest')
+    path = Path(path)
+    if path.is_symlink() or not path.is_file():
+        raise ContractError('expected regular local artifact')
+    data = path.read_bytes()
+    if byte_digest(data) != expected_digest:
+        raise ContractError('artifact digest mismatch')
+    return strict_json(data)
+
+
+def read_artifact(run_dir, binding):
+    path = safe_path(run_dir, binding.path)
+    if not path.is_file() or path.stat().st_size != binding.size_bytes:
+        raise ContractError('artifact missing or byte size mismatch')
+    return _read_file(path, binding.sha256)
+
+
+def write_artifact(run_dir, relative_path, value):
+    data = canonical_json(value).encode('utf-8')
+    strict_json(data)
+    path = safe_path(run_dir, relative_path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open('xb') as stream:
+        stream.write(data)
+    return ArtifactBinding(relative_path, byte_digest(data), len(data))
+
+
+def create_run(output_dir, identity, candidate, case_ids, *, run_id=None):
+    run_id = run_id or 'run-' + uuid4().hex
+    _id(run_id, 'run_id')
+    # Validate identity/configuration before creating files.
+    identity.to_dict(); candidate.to_dict()
+    if not case_ids or len(case_ids) != len(set(case_ids)):
+        raise ContractError('run requires distinct ordered cases')
+    path = safe_path(output_dir, 'runs/' + run_id)
+    path.mkdir(parents=True, exist_ok=False)
+    return path
+
+
+def finish_run(run_dir, identity, candidate, case_ids, captures):
+    path = Path(run_dir)
+    manifest = TrackCaptureManifest(path.name, identity, candidate, tuple(case_ids), tuple(captures))
+    for capture in manifest.captures:
+        for binding in (capture.request, capture.response, capture.observed):
+            if binding is not None: read_artifact(path, binding)
+    binding = write_artifact(path, 'manifest.json', manifest.to_dict())
+    ref = TrackRunRef(path, binding.sha256)
+    verify_track_run(ref)
+    return ref
+
+
+def open_track_run(path, *, expected_manifest_sha256):
+    ref = TrackRunRef(Path(path), expected_manifest_sha256)
+    verify_track_run(ref)
+    return ref
+
+
+def verify_track_run(ref, *, expected_case_ids=None, expected_requests=None, trials=None, attempts=None):
+    manifest = TrackCaptureManifest.from_dict(_read_file(safe_path(ref.path, 'manifest.json'), ref.manifest_sha256))
+    if manifest.run_id != ref.run_id:
+        raise ContractError('run identity mismatch')
+    if expected_case_ids is not None and tuple(expected_case_ids) != manifest.case_ids:
+        raise ContractError('ordered case identity mismatch')
+    if expected_requests is not None and set(expected_requests) != set(manifest.case_ids):
+        raise ContractError('expected request cohort mismatch')
+    expected_bytes = None if expected_requests is None else {
+        case: _request_bytes(value) for case, value in expected_requests.items()}
+    for number, name in ((trials, 'trials'), (attempts, 'attempts')):
+        if number is not None and (type(number) is not int or number < 1):
+            raise ContractError(name + ' must be a positive integer')
+    keys = {(c.case_id, c.trial_index, c.attempt_index) for c in manifest.captures}
+    if trials is not None:
+        for case in manifest.case_ids:
+            actual = {c.trial_index for c in manifest.captures if c.case_id == case}
+            if actual != set(range(1, trials + 1)):
+                raise ContractError('declared trial coverage mismatch')
+    if attempts is not None:
+        for case, trial in {(c.case_id, c.trial_index) for c in manifest.captures}:
+            actual = {a for cid, t, a in keys if cid == case and t == trial}
+            if actual != set(range(1, attempts + 1)):
+                raise ContractError('declared attempt coverage mismatch')
+    for capture in manifest.captures:
+        request = read_artifact(ref.path, capture.request)
+        if expected_requests is not None:
+            if _request_bytes(request) != expected_bytes[capture.case_id]:
+                raise ContractError('captured request identity differs from frozen input')
+        for binding in (capture.response, capture.observed):
+            if binding is not None: read_artifact(ref.path, binding)
+    return manifest
+
+
+def _evaluation_binding(evaluation, run_manifest, run):
+    if evaluation.run_id != run.run_id or evaluation.capture_manifest_digest != run.manifest_sha256:
+        raise ContractError('evaluation original capture mismatch')
+    left, right = evaluation.identity.to_dict(), run_manifest.identity.to_dict()
+    # A new grading policy may regrade the original execution capture.
+    for key in ('metric_release', 'grader_release', 'judgment_policy_digest'):
+        left.pop(key); right.pop(key)
+    if left != right:
+        raise ContractError('evaluation execution identity mismatch')
+
+
+def write_evaluation(run, identity, metrics, *, judgment_artifact_digest=None):
+    manifest = verify_track_run(run)
+    evaluation_id = 'eval-' + uuid4().hex
+    evaluation = TrackEvaluationManifest(evaluation_id, run.run_id, run.manifest_sha256,
+                                         identity, metrics, judgment_artifact_digest)
+    _evaluation_binding(evaluation, manifest, run)
+    relative = 'evaluations/' + evaluation_id + '/scorecard.json'
+    binding = write_artifact(run.path, relative, evaluation.to_dict())
+    return TrackEvaluationRef(run.path / 'evaluations' / evaluation_id, binding.sha256, run)
+
+
+def open_track_evaluation(path, *, expected_scorecard_sha256, run=None):
+    path = Path(path)
+    evaluation = TrackEvaluationManifest.from_dict(_read_file(safe_path(path, 'scorecard.json'), expected_scorecard_sha256))
+    if run is None:
+        run = open_track_run(path.parent.parent, expected_manifest_sha256=evaluation.capture_manifest_digest)
+    ref = TrackEvaluationRef(path, expected_scorecard_sha256, run)
+    verify_track_evaluation(ref)
+    return ref
+
+
+def verify_track_evaluation(ref):
+    manifest = verify_track_run(ref.run)
+    relative = 'evaluations/' + ref.evaluation_id + '/scorecard.json'
+    evaluation = TrackEvaluationManifest.from_dict(_read_file(safe_path(ref.run.path, relative), ref.scorecard_sha256))
+    if evaluation.evaluation_id != ref.evaluation_id:
+        raise ContractError('evaluation identity mismatch')
+    _evaluation_binding(evaluation, manifest, ref.run)
+    return evaluation