sable-sphynx 0.7.0.dev0__py3-none-any.whl

sphynx/dataset_readiness.py

@@ -0,0 +1,138 @@
+"""Small source-readiness summaries with explicit identity and provenance.
+
+These utilities describe supplied rows and pinned bytes. They do not establish
+review authority, independent labels, holdout quality, or benchmark capability.
+Summaries omit raw question text, labels, identifiers, and local source paths.
+"""
+from __future__ import annotations
+
+import hashlib
+from collections.abc import Iterable, Mapping
+from pathlib import Path
+import re
+from typing import Any
+
+from .contracts import ContractError, canonical_json
+
+_SHA256 = re.compile(r"sha256:[0-9a-f]{64}\Z")
+
+
+def _text(value: Any, field: str) -> str:
+    if type(value) is not str or not value.strip() or "\x00" in value:
+        raise ContractError(f"{field} must be nonempty text")
+    try:
+        value.encode("utf-8", errors="strict")
+    except UnicodeEncodeError:
+        raise ContractError(f"{field} must contain valid Unicode") from None
+    return value
+
+
+def _rows(values: Iterable[Any], field: str) -> list[Any]:
+    if isinstance(values, (str, bytes, Mapping)):
+        raise ContractError(f"{field} must be an iterable of rows")
+    try:
+        return list(values)
+    except TypeError:
+        raise ContractError(f"{field} must be an iterable of rows") from None
+
+
+def overlap_summary(left: Iterable[str], right: Iterable[str]) -> dict[str, Any]:
+    """Count exact unique identities; hash their sorted unique intersection.
+
+    Row counts retain duplicates. Overlap, union, and the overlap digest use
+    sets, so duplicate rows and ordering do not change their identity meaning.
+    IDs are compared exactly, without question-text or case normalization.
+    """
+    left_rows = _rows(left, "left identities")
+    right_rows = _rows(right, "right identities")
+    left_ids = {_text(value, "left identity") for value in left_rows}
+    right_ids = {_text(value, "right identity") for value in right_rows}
+    overlap = left_ids & right_ids
+    digest = hashlib.sha256(canonical_json(sorted(overlap)).encode("utf-8")).hexdigest()
+    return {
+        "left_rows": len(left_rows), "right_rows": len(right_rows),
+        "left_unique": len(left_ids), "right_unique": len(right_ids),
+        "union_unique": len(left_ids | right_ids), "overlap_unique": len(overlap),
+        "left_only_unique": len(left_ids - right_ids), "right_only_unique": len(right_ids - left_ids),
+        "overlap_digest": "sha256:" + digest,
+    }
+
+
+def routing_provenance_summary(questions: Iterable[Mapping[str, Any]],
+                               labels: Iterable[Mapping[str, Any]]) -> dict[str, Any]:
+    """Join unique question IDs and summarize supplied routing provenance.
+
+    Repeated questions are extra rows after the first occurrence of a company
+    and user-turn pair. Normalization collapses whitespace and applies casefold;
+    it does not remove punctuation, infer paraphrases, or match text across companies.
+    Family identity is supplied label metadata, not inferred from repeated text.
+    Incomplete or empty family metadata remains unknown.
+    """
+    question_rows = _rows(questions, "questions")
+    label_rows = _rows(labels, "labels")
+    question_by_id: dict[str, Mapping[str, Any]] = {}
+    repeated_keys: set[tuple[str, str]] = set()
+    for row in question_rows:
+        if not isinstance(row, Mapping):
+            raise ContractError("question rows must be objects")
+        question_id = _text(row.get("question_id"), "question ID")
+        if question_id in question_by_id:
+            raise ContractError("question IDs must be unique")
+        company = _text(row.get("company_id"), "company ID")
+        turn = _text(row.get("user_turn"), "user turn")
+        question_by_id[question_id] = row
+        repeated_keys.add((" ".join(company.casefold().split()), " ".join(turn.casefold().split())))
+
+    label_by_id: dict[str, Mapping[str, Any]] = {}
+    for row in label_rows:
+        if not isinstance(row, Mapping):
+            raise ContractError("label rows must be objects")
+        question_id = _text(row.get("question_id"), "label question ID")
+        if question_id in label_by_id:
+            raise ContractError("label question IDs must be unique")
+        if question_id not in question_by_id:
+            raise ContractError("label question IDs do not match question IDs")
+        if "company_id" in row and row["company_id"] != question_by_id[question_id]["company_id"]:
+            raise ContractError("label company identity differs from its question")
+        label_by_id[question_id] = row
+    if set(label_by_id) != set(question_by_id):
+        raise ContractError("labels must match questions one to one")
+
+    missing_families = 0
+    families: set[tuple[str, str]] = set()
+    for question_id, row in label_by_id.items():
+        family = row.get("family_id")
+        if family is None or (type(family) is str and not family.strip()):
+            missing_families += 1
+            continue
+        family = _text(family, "family ID")
+        families.add((question_by_id[question_id]["company_id"], family))
+    known_families = bool(question_rows) and missing_families == 0
+    return {
+        "question_rows": len(question_rows), "label_rows": len(label_rows),
+        "question_unique": len(question_by_id),
+        "repeated_question_rows": len(question_rows) - len(repeated_keys),
+        "repeated_question_status": "known",
+        "family_status": "known" if known_families else "unknown",
+        "family_count": len(families) if known_families else None,
+        "missing_family_rows": missing_families,
+    }
+
+
+def verify_source_file(path: str | Path, expected_sha256: str | None = None) -> dict[str, Any]:
+    """Verify actual file bytes against an explicit lowercase SHA-256 pin.
+
+    Missing or malformed pins fail before reading. The calculated digest is
+    compared with the caller's pin; it is never adopted as a new authority.
+    Return byte metadata only, without exposing the source path or content.
+    """
+    if type(expected_sha256) is not str or not _SHA256.fullmatch(expected_sha256):
+        raise ContractError("source verification requires an explicit lowercase sha256 pin")
+    try:
+        raw = Path(path).read_bytes()
+    except (OSError, TypeError, ValueError):
+        raise ContractError("source file could not be read") from None
+    actual = "sha256:" + hashlib.sha256(raw).hexdigest()
+    if actual != expected_sha256:
+        raise ContractError("source bytes differ from the supplied SHA-256 pin")
+    return {"sha256": actual, "size_bytes": len(raw)}