sable-sphynx 0.7.0.dev0__py3-none-any.whl

sphynx/demo.py

@@ -0,0 +1,130 @@
+"""Run the complete synthetic beginner demo from an ordinary installed SDK."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+from typing import Any
+
+from .contracts import INTERFACE_VERSION, CandidateSpec
+from .metric_views import summarize_metrics
+from .packs import check_candidate, list_packs
+from .results import Scorecard, compare, grade, report
+from .runner import run
+
+_PACK_ID = "rag.rerank.v1"
+_BASELINE_LABEL = "Supplied-score baseline (A)"
+_CANDIDATE_LABEL = "Word-overlap example (B)"
+_COMPARISON_LABEL = (
+    "A = supplied-score baseline (supplied-order-example). "
+    "B = word-overlap example (word-overlap-example).\n"
+    "Deltas are B - A: candidate minus baseline."
+)
+
+
+def _save(path: Path, text: str) -> Path:
+    # Exclusive creation keeps earlier reports intact, including on collisions.
+    with path.open("x", encoding="utf-8") as stream:
+        stream.write(text)
+    return path
+
+
+def _save_scorecard(reports: Path, run_id: str, label: str, scorecard: Scorecard) -> dict[str, Path]:
+    summary = summarize_metrics(scorecard)
+    return {
+        "scorecard_json_path": _save(reports / f"{run_id}-scorecard.json", scorecard.to_json() + "\n"),
+        "scorecard_path": _save(reports / f"{run_id}-scorecard.md", scorecard.to_markdown()),
+        "summary_path": _save(reports / f"{run_id}-summary.md", f"# {label}\n\n" + summary.to_markdown()),
+        "summary_text_path": _save(reports / f"{run_id}-summary.txt", label + "\n\n" + summary.to_text()),
+    }
+
+
+def _print_summary(label: str, scorecard: Scorecard) -> None:
+    counts = scorecard.to_dict()["counts"]
+    print(f"\n{label}")
+    print(
+        f"Cases: {counts['attempted_count']} | "
+        f"Valid: {counts['success_count']} | Failed: {counts['failure_count']}\n"
+    )
+    print(summarize_metrics(scorecard).to_text())
+
+
+def run_demo(output_dir: str | Path | None = None) -> dict[str, Any]:
+    """Evaluate both packaged examples and save durable local reports.
+
+    Each method runs the six installed synthetic cases in fresh subprocesses
+    using ``sys.executable``. No model, provider key, or cloud setup is needed.
+    The default output is ``Path.home() / ".sphynx" / "beginner-quickstart"``.
+    Every invocation keeps earlier runs, evaluations, and reports. Returned
+    references include the original ``run_quickstart`` mapping keys; comparison
+    A is the supplied-score baseline and B is the word-overlap example.
+    """
+    output = (
+        Path(output_dir) if output_dir is not None
+        else Path.home() / ".sphynx" / "beginner-quickstart"
+    ).expanduser().resolve()
+    reports = output / "reports"
+    reports.mkdir(parents=True, exist_ok=True)
+    candidate = CandidateSpec(
+        name="word-overlap-example", version="example-v1",
+        argv=(sys.executable, "-I", "-m", "sphynx.demo_candidates.lexical"),
+        interface_version=INTERFACE_VERSION,
+    )
+    baseline = CandidateSpec(
+        name="supplied-order-example", version="example-v1",
+        argv=(sys.executable, "-I", "-m", "sphynx.demo_candidates.dense"),
+        interface_version=INTERFACE_VERSION,
+    )
+    for method in (candidate, baseline):
+        check = check_candidate(method, _PACK_ID)
+        if not check.compatible or not check.pack_runnable:
+            raise RuntimeError("; ".join(check.reasons))
+
+    print("SPHYNX - LOCAL RERANKING EXAMPLE")
+    print("Installed packs:")
+    for pack in list_packs():
+        print(f"  {pack.pack_id}: {pack.status}")
+    print("\nRunning six synthetic reranking cases for each method...")
+    first_run = run(candidate, _PACK_ID, output)
+    first_evaluation = grade(first_run)
+    if first_evaluation.path is None:
+        raise RuntimeError(first_evaluation.reason)
+    scorecard = report(first_evaluation)
+    _print_summary(_CANDIDATE_LABEL, scorecard)
+    candidate_paths = _save_scorecard(reports, first_run.run_id, _CANDIDATE_LABEL, scorecard)
+
+    baseline_run = run(baseline, _PACK_ID, output)
+    baseline_evaluation = grade(baseline_run)
+    if baseline_evaluation.path is None:
+        raise RuntimeError(baseline_evaluation.reason)
+    baseline_scorecard = report(baseline_evaluation)
+    _print_summary(_BASELINE_LABEL, baseline_scorecard)
+    baseline_paths = _save_scorecard(reports, baseline_run.run_id, _BASELINE_LABEL, baseline_scorecard)
+
+    comparison = compare(baseline_evaluation, first_evaluation)
+    if not comparison.compatible:
+        raise RuntimeError(comparison.reason)
+    comparison_json_path = _save(
+        reports / f"{first_run.run_id}-comparison.json", comparison.to_json() + "\n"
+    )
+    comparison_path = _save(
+        reports / f"{first_run.run_id}-comparison.md",
+        _COMPARISON_LABEL + "\n\n" + comparison.to_markdown(),
+    )
+    print("COMPARISON")
+    print(_COMPARISON_LABEL)
+    print(f"\nReports: {reports}")
+    print(f"Candidate summary: {candidate_paths['summary_path'].name}")
+    print(f"Baseline summary: {baseline_paths['summary_path'].name}")
+    print(f"Comparison: {comparison_path.name}")
+    print("\nThese are synthetic demonstration results, not an official benchmark.")
+    print("No cloud sync was requested.")
+    return {
+        "run": first_run, "evaluation": first_evaluation, "scorecard": scorecard,
+        "baseline_run": baseline_run, "baseline_evaluation": baseline_evaluation,
+        "baseline_scorecard": baseline_scorecard, "comparison": comparison,
+        **candidate_paths,
+        **{f"baseline_{key}": path for key, path in baseline_paths.items()},
+        "comparison_path": comparison_path, "comparison_json_path": comparison_json_path,
+        "reports_dir": reports,
+    }

sphynx/demo_candidates/__init__.py

@@ -0,0 +1 @@
+"""Packaged synthetic demo candidates, executed by the installed interpreter."""

sphynx/demo_candidates/dense.py

@@ -0,0 +1,11 @@
+"""Illustrative baseline: return each frozen passage's supplied dense score."""
+
+from sphynx.candidates import serve_reranker
+
+
+def score(query: str, passage: str, supplied_score: float) -> float:
+    return supplied_score
+
+
+if __name__ == "__main__":
+    serve_reranker(score)

sphynx/demo_candidates/lexical.py

@@ -0,0 +1,21 @@
+"""Illustrative query/passage word overlap; no model or provider required."""
+
+import re
+
+from sphynx.candidates import serve_reranker
+
+_TOKENS = re.compile(r"[a-z0-9]+")
+_STOP = frozenset({"a", "an", "and", "are", "can", "do", "how", "i", "is", "of", "the", "to", "what", "where"})
+
+
+def tokens(text: str) -> set[str]:
+    return set(_TOKENS.findall(text.lower())) - _STOP
+
+
+def score(query: str, passage: str, supplied_score: float) -> float:
+    overlap = len(tokens(query) & tokens(passage))
+    return float(overlap) + float(supplied_score) / 100.0
+
+
+if __name__ == "__main__":
+    serve_reranker(score)

sphynx/diagnostics.py

@@ -0,0 +1,207 @@
+"""Safe guidance over existing exceptions; never retry, refresh or execute work.
+
+The original exception and its code stay unchanged. Default guidance copies no
+exception message, command, token, request payload or private reference content.
+Recovery values are explicit operation identifiers, not authentication grants.
+"""
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+import json
+from types import MappingProxyType
+from typing import Literal, TypeGuard, cast
+
+Retryability = Literal["fix_required", "wait_for_review", "read_original_handle",
+                       "reconcile_before_retry", "unknown"]
+_RETRY = {"fix_required", "wait_for_review", "read_original_handle", "reconcile_before_retry", "unknown"}
+_RECOVERY = {"submission_id", "experiment_id", "run_id", "grading_id", "evaluation_id", "idempotency_key", "record_id"}
+
+
+def _identity_value(value: object) -> TypeGuard[str]:
+    # Existing workflow identities use the workflow text contract. Presentation
+    # must not narrow keys after a driver has accepted the original operation.
+    if type(value) is not str or not value.strip() or '\x00' in value:
+        return False
+    try:
+        value.encode('utf-8')
+    except UnicodeError:
+        return False
+    return True
+
+
+def _safe_text(value: str) -> str:
+    text = ''.join(char if char.isprintable() else ' ' for char in value)
+    return text if len(text) <= 256 else text[:256] + '... (full value in to_dict())'
+
+
+@dataclass(frozen=True)
+class Diagnostic:
+    """A readable hint. It neither grants access nor decides an evaluation."""
+
+    code: str
+    summary: str
+    next_actions: tuple[str, ...]
+    retryability: Retryability = "unknown"
+    recovery: Mapping[str, str] = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        if (not isinstance(self.code, str) or not self.code or not isinstance(self.summary, str)
+                or not self.summary or self.retryability not in _RETRY
+                or not isinstance(self.next_actions, (tuple, list))
+                or any(type(action) is not str or not action for action in self.next_actions)):
+            raise ValueError("diagnostic needs a code, summary and valid guidance")
+        values = dict(self.recovery)
+        if any(key not in _RECOVERY or not _identity_value(value)
+               for key, value in values.items()):
+            raise ValueError("diagnostic recovery contains an invalid operation identity")
+        object.__setattr__(self, "next_actions", tuple(self.next_actions))
+        object.__setattr__(self, "recovery", MappingProxyType(values))
+
+    def to_dict(self) -> dict:
+        return {"schema_version": "sphynx.diagnostic.v1", "code": self.code, "summary": self.summary,
+                "next_actions": list(self.next_actions), "retryability": self.retryability,
+                "recovery": dict(self.recovery)}
+
+    def to_json(self) -> str:
+        return json.dumps(self.to_dict(), sort_keys=True, allow_nan=False)
+
+    def to_text(self) -> str:
+        lines = [f"{_safe_text(self.code)}: {_safe_text(self.summary)}", *["- " + _safe_text(action) for action in self.next_actions],
+                 "Recovery: " + self.retryability]
+        lines.extend(f"{name}: {_safe_text(value)}" for name, value in self.recovery.items())
+        return "\n".join(lines)
+
+    def __str__(self) -> str:
+        return self.to_text()
+
+
+# These statements describe next actions, not suite-specific acceptance rules.
+_CONFIG = ("The configuration needs attention.",
+           "Check the selected suite, schemas, versions and supported options before starting work.", "fix_required")
+_ACCESS = ("The selected record or access grant is unavailable.",
+           "Check the record identity and ask its operator for the required access.", "fix_required")
+_AUTH = ("The hosted client needs valid configuration or sign-in.",
+         "Check the trusted API origin and restore your operator-issued Sphynx credentials.", "fix_required")
+_INTEGRITY = ("The saved identity or response could not be verified.",
+              "Inspect the original saved record and ask the operator to resolve its identity mismatch.", "fix_required")
+_UNCERTAIN = ("The request outcome needs to be reconciled.",
+              "Read the original work status or replay the exact original request and key before a new submission.", "reconcile_before_retry")
+_WAIT = ("Waiting ended; the original work may still be active.",
+         "Refresh or wait on the original handle and keep its operation identity.", "read_original_handle")
+_REVIEW = ("The selected bank still needs review.",
+           "Read its current review status and follow that bank's approved review process.", "wait_for_review")
+_COMPARISON = ("These results cannot provide the requested comparison.",
+               "Check source, bank/cohort, grading versions and the declared comparison conditions.", "fix_required")
+_KNOWN: dict[str, tuple[str, str, str]] = {}
+for _codes, _hint in [
+    (("client_not_configured", "invalid_client_config", "invalid_api_url", "invalid_client_token",
+      "authentication_required", "authentication_failed"), _AUTH),
+    (("submission_not_authorized", "bank_release_unavailable", "reference_access_denied"), _ACCESS),
+    (("invalid_server_response", "bank_review_integrity", "artifact_identity_mismatch", "comparison_identity_mismatch"), _INTEGRITY),
+    (("hosted_transport_failed", "store_unavailable", "service_unavailable", "submission_failed", "grading_request_uncertain"), _UNCERTAIN),
+    (("wait_timeout",), _WAIT), (("bank_review_not_ready",), _REVIEW),
+    (("comparison_unavailable", "comparison_incompatible"), _COMPARISON),
+    (("invalid_bank_review", "invalid_bank_review_page", "bank_review_limit"), _CONFIG),
+]:
+    _KNOWN.update({code: _hint for code in _codes})
+_KNOWN["bank_review_conflict"] = ("The review version, current set or operation key changed.",
+    "Recover an uncertain original request first; for a new judgment, read the current release and target.", "reconcile_before_retry")
+_KNOWN["bank_release_inactive"] = ("The release is held or withdrawn.",
+    "Read its current state; an operator can resume a held release, while withdrawal needs a new release.", "fix_required")
+
+
+def _fields(value: object) -> dict:
+    try:
+        return object.__getattribute__(value, "__dict__")
+    except (AttributeError, TypeError):
+        return {}
+
+
+def _recovery(error: BaseException) -> dict[str, str]:
+    fields = _fields(error)
+    sources = [fields]
+    handle = fields.get("handle")
+    if handle is not None:
+        saved = _fields(handle)
+        sources.append(saved)
+        if type(saved.get("_record")) is dict:
+            sources.append(saved["_record"])
+        if saved.get("snapshot") is not None:
+            snapshot = _fields(saved["snapshot"])
+            sources.append(snapshot)
+            # GradingSnapshot stores its identity in a ReferenceBinding. Read
+            # cached fields only; its convenience properties need no invocation.
+            if snapshot.get("reference") is not None:
+                reference = _fields(snapshot["reference"])
+                sources.append({"grading_id": reference.get("record_id"),
+                                "run_id": reference.get("run_id")})
+        identity = saved.get("id")
+        if _identity_value(identity):
+            key = ("experiment_id" if identity.startswith("exp_") else "evaluation_id"
+                   if identity.startswith("eval_") else "run_id" if identity.startswith(("run_", "run-")) else "record_id")
+            sources.append({key: identity})
+    if fields.get("intent") is not None:
+        sources.append(_fields(fields["intent"]))
+    result: dict[str, str] = {}
+    for source in sources:
+        for key in sorted(_RECOVERY):
+            value = source.get(key)
+            if _identity_value(value):
+                result.setdefault(key, value)
+    return result
+
+
+def explain_error(error: BaseException) -> Diagnostic:
+    """Explain an existing error without reading data or changing remote work.
+
+    Retryability names the suggested recovery action. It never authorizes a
+    candidate retry. Unknown errors keep unknown guidance; inspect the original
+    exception explicitly when more detail is needed.
+    """
+    if not isinstance(error, BaseException):
+        raise TypeError("explain_error needs an exception")
+    fields = _fields(error)
+    raw_code = fields.get("code")
+    code = raw_code if type(raw_code) is str and raw_code in _KNOWN else "unknown_error"
+    hint = _KNOWN.get(code)
+    names = {cls.__name__ for cls in type(error).__mro__}
+    sdk_error = type(error).__module__.startswith("sphynx.")
+    if hint is None and sdk_error:
+        if "WorkflowTimeoutError" in names:
+            code, hint = "wait_timeout", _WAIT
+        elif "GradingRequestError" in names or "SubmissionError" in names:
+            code, hint = "grading_request_uncertain" if "GradingRequestError" in names else "submission_failed", _UNCERTAIN
+        elif "PendingWorkError" in names or "EvaluationExecutionError" in names:
+            code, hint = "execution_incomplete", ("Execution or grading did not complete.",
+                "Inspect the original run and failure evidence before selecting recovery supported by its driver.", "unknown")
+        elif "EvaluationCheckFailed" in names:
+            code, hint = "check_failed", ("The test's explicit report check failed.",
+                "Inspect the saved assessments and the selected check before changing the candidate.", "fix_required")
+        elif type(raw_code) is str:
+            hint = ("The failure has no registered safe guidance.",
+                    "Keep the original error and work handle, then inspect the permitted details.", "unknown")
+        else:
+            # Match only plain SDK message text; never render it or call __str__.
+            args = BaseException.__dict__['args'].__get__(error, BaseException)
+            message = args[0].lower() if args and type(args[0]) is str else ""
+            if any(word in message for word in ("unsupported", "does not support", "capabilit", "does not implement")):
+                code, hint = "unsupported_controls", ("This execution path does not support the requested controls.",
+                    "Check its declared scope/mode/capabilities and select an implementing driver or supported settings.", "fix_required")
+            elif "compar" in message:
+                code, hint = "comparison_unavailable", _COMPARISON
+            else:
+                code, hint = "workflow_configuration", _CONFIG
+    if hint is None:
+        if isinstance(error, PermissionError):
+            code, hint = "access_required", _ACCESS
+        elif isinstance(error, FileNotFoundError):
+            code, hint = "asset_or_path_unavailable", ("A required asset or local path is unavailable.",
+                "Check the approved assets and output directory; use a short writable output root on Windows.", "fix_required")
+        elif isinstance(error, TimeoutError):
+            code, hint = "request_outcome_unknown", _UNCERTAIN
+        else:
+            hint = ("The failure has no registered safe guidance.",
+                    "Keep the original error and work handle, then inspect the permitted details.", "unknown")
+    summary, action, retryability = hint
+    return Diagnostic(code, summary, (action,), cast(Retryability, retryability), _recovery(error))