screamingface 0.1.0__py3-none-any.whl

screamingface/__init__.py

@@ -0,0 +1,109 @@
+"""ScreamingFace — evaluate composable Candidate Recipes on research Benchmarks."""
+
+from screamingface import benchmarks, connections, events, leaderboards, models
+from screamingface._default_client import close, configure, connect, disconnect, evaluate
+from screamingface._ui.connections import ConnectionPanel
+from screamingface.client import AsyncClient, Client
+from screamingface.connections import AsyncOAuthFlow, Connection, OAuthFlow
+from screamingface.discovery import (
+    Benchmark,
+    BenchmarkInfo,
+    ModelCapability,
+    ModelDetails,
+    ModelInfo,
+    ModelParameter,
+    ModelParameterSchema,
+)
+from screamingface.errors import (
+    AuthenticationError,
+    EngineUnavailableError,
+    ExecutionError,
+    LeaderboardError,
+    PlanningError,
+    ProviderConnectionError,
+    ScreamingFaceError,
+)
+from screamingface.events import Event
+from screamingface.fusion import Fusion
+from screamingface.leaderboard import (
+    Leaderboard,
+    LeaderboardBaseline,
+    LeaderboardEntry,
+    LeaderboardInfo,
+    LeaderboardScore,
+)
+from screamingface.model import Model
+from screamingface.operation import OperationInfo
+from screamingface.pipeline import Pipeline
+from screamingface.recipe import Recipe
+from screamingface.report import (
+    CandidateResult,
+    CaseGrade,
+    CaseResult,
+    Check,
+    Evidence,
+    EvidenceProducer,
+    Failure,
+    MemberResult,
+    Report,
+    Usage,
+)
+from screamingface.url4 import Url4
+from screamingface.warnings import CoverageWarning, EvaluationWarning
+
+__all__ = [
+    "AsyncClient",
+    "AuthenticationError",
+    "Benchmark",
+    "BenchmarkInfo",
+    "CaseGrade",
+    "CaseResult",
+    "CandidateResult",
+    "Check",
+    "Client",
+    "Connection",
+    "ConnectionPanel",
+    "AsyncOAuthFlow",
+    "CoverageWarning",
+    "close",
+    "configure",
+    "connect",
+    "connections",
+    "disconnect",
+    "Evidence",
+    "EvidenceProducer",
+    "Event",
+    "EngineUnavailableError",
+    "ExecutionError",
+    "EvaluationWarning",
+    "evaluate",
+    "Failure",
+    "Fusion",
+    "Leaderboard",
+    "LeaderboardBaseline",
+    "LeaderboardEntry",
+    "LeaderboardError",
+    "LeaderboardInfo",
+    "LeaderboardScore",
+    "MemberResult",
+    "Model",
+    "ModelCapability",
+    "ModelDetails",
+    "ModelInfo",
+    "ModelParameter",
+    "ModelParameterSchema",
+    "OperationInfo",
+    "OAuthFlow",
+    "PlanningError",
+    "Pipeline",
+    "ProviderConnectionError",
+    "Recipe",
+    "Report",
+    "ScreamingFaceError",
+    "Usage",
+    "Url4",
+    "benchmarks",
+    "events",
+    "leaderboards",
+    "models",
+]

screamingface/_candidate_policy.py

@@ -0,0 +1,80 @@
+"""Validated immutable Candidate generation overrides."""
+
+from __future__ import annotations
+
+import math
+from collections.abc import Mapping
+from types import MappingProxyType
+
+type GenerationScalar = str | int | float | bool
+type GenerationParams = Mapping[str, GenerationScalar]
+
+_EXECUTION_OWNED_PARAMS = frozenset(
+    {
+        "input",
+        "messages",
+        "model",
+        "plugins",
+        "provider",
+        "q",
+        "stream",
+        "tool_choice",
+        "tools",
+        "web_search",
+        "web_search_exclude",
+        "web_search_excluded_domains",
+        "web_search_policy",
+    }
+)
+"""Transport, routing, tool, and Benchmark protocol fields users cannot override."""
+
+
+def prompt(value: object | None, label: str) -> str | None:
+    if value is None:
+        return None
+    if not isinstance(value, str):
+        raise TypeError(f"{label} must be a string or None")
+    if not value.strip():
+        raise ValueError(f"{label} must not be empty")
+    return value
+
+
+def params(value: object | None, label: str) -> GenerationParams:
+    if value is None:
+        return MappingProxyType({})
+    if not isinstance(value, Mapping):
+        raise TypeError(f"{label} must be a mapping or None")
+    selected: dict[str, GenerationScalar] = {}
+    for name, item in value.items():
+        selected_name = _parameter_name(name, label)
+        selected[selected_name] = _parameter_value(selected_name, item, label)
+    return MappingProxyType(selected)
+
+
+def _parameter_name(value: object, label: str) -> str:
+    if not isinstance(value, str) or not value.strip():
+        raise ValueError(f"{label} names must be non-empty strings")
+    if not value.isascii() or not all(
+        character.isalnum() or character in "._" for character in value
+    ):
+        raise ValueError(f"{label} parameter {value!r} cannot be encoded")
+    if value in _EXECUTION_OWNED_PARAMS:
+        raise ValueError(
+            f"{label} parameter {value!r} is reserved for Benchmark and Engine execution"
+        )
+    return value
+
+
+def _parameter_value(name: str, value: object, label: str) -> GenerationScalar:
+    if not isinstance(value, str | int | float | bool):
+        raise TypeError(f"{label} value {name!r} must be a scalar")
+    if isinstance(value, float) and not math.isfinite(value):
+        raise ValueError(f"{label} value {name!r} must be finite")
+    if isinstance(value, str) and any(
+        character in "';(),&" or ord(character) < 32 or ord(character) == 127 for character in value
+    ):
+        raise ValueError(f"{label} value {name!r} cannot be encoded")
+    return value
+
+
+__all__: list[str] = []

screamingface/_client_connections.py

@@ -0,0 +1,123 @@
+"""Shared Client origin, auth-listener, and provider-connection support."""
+
+from __future__ import annotations
+
+import threading
+from collections.abc import Callable
+from typing import TYPE_CHECKING, Any, Literal
+from urllib.parse import urlsplit, urlunsplit
+
+from screamingface.errors import ProviderConnectionError
+
+if TYPE_CHECKING:
+    from screamingface._engine.connections import AsyncConnections, Connections
+    from screamingface.connections import AsyncOAuthFlow, Connection, OAuthFlow
+
+
+class _AuthListeners:
+    """Thread-safe observer set shared by the sync and async Client facades."""
+
+    def __init__(self) -> None:
+        self._lock = threading.Lock()
+        self._callbacks: set[Callable[[], None]] = set()
+
+    def subscribe(self, callback: Callable[[], None]) -> Callable[[], None]:
+        with self._lock:
+            self._callbacks.add(callback)
+
+        def unsubscribe() -> None:
+            with self._lock:
+                self._callbacks.discard(callback)
+
+        return unsubscribe
+
+    def notify(self) -> None:
+        with self._lock:
+            callbacks = tuple(self._callbacks)
+        for callback in callbacks:
+            callback()
+
+
+def _engine_origin(value: Any) -> str:
+    return _http_origin(value, "engine_url")
+
+
+def _scoreboard_origin(value: Any) -> str:
+    return _http_origin(value, "scoreboard_url")
+
+
+def _http_origin(value: Any, name: str) -> str:
+    if not isinstance(value, str) or not value.strip():
+        raise ValueError(f"{name} must be an HTTP(S) origin")
+    parts = urlsplit(value.strip())
+    if (
+        parts.scheme not in {"http", "https"}
+        or not parts.netloc
+        or parts.path not in {"", "/"}
+        or parts.query
+        or parts.fragment
+        or parts.username is not None
+        or parts.password is not None
+    ):
+        raise ValueError(f"{name} must be an HTTP(S) origin without credentials or a path")
+    return urlunsplit((parts.scheme, parts.netloc, "", "", ""))
+
+
+def _require_secure_connection_origin(engine_url: str) -> None:
+    parts = urlsplit(engine_url)
+    if parts.scheme == "https" or (
+        parts.scheme == "http" and parts.hostname in {"localhost", "127.0.0.1", "::1"}
+    ):
+        return
+    raise ProviderConnectionError(
+        "Provider connection operations require HTTPS outside a loopback SF Engine",
+        code="secure_transport_required",
+        permanent=True,
+    )
+
+
+def _connect_sync(
+    connections: Connections,
+    provider: str,
+    api_key: str | None,
+    method: Literal["api_key", "oauth"] | None,
+) -> Connection | OAuthFlow:
+    _validate_auth_method(method)
+    if api_key is not None:
+        _require_api_key_action(method)
+        return connections.connect(provider, api_key)
+    if method == "api_key":
+        raise ValueError("api_key is required for API-key authentication")
+    if method == "oauth":
+        return connections.start_oauth(provider)
+    raise ValueError("api_key is required unless method='oauth' is selected")
+
+
+async def _connect_async(
+    connections: AsyncConnections,
+    provider: str,
+    api_key: str | None,
+    method: Literal["api_key", "oauth"] | None,
+) -> Connection | AsyncOAuthFlow:
+    _validate_auth_method(method)
+    if api_key is not None:
+        _require_api_key_action(method)
+        return await connections.connect(provider, api_key)
+    if method == "api_key":
+        raise ValueError("api_key is required for API-key authentication")
+    if method == "oauth":
+        return await connections.start_oauth(provider)
+    raise ValueError("api_key is required unless method='oauth' is selected")
+
+
+def _validate_auth_method(method: object) -> None:
+    if method not in {None, "api_key", "oauth"}:
+        raise ValueError("method must be 'api_key', 'oauth', or None")
+
+
+def _require_api_key_action(method: object) -> None:
+    if method == "oauth":
+        raise ValueError("api_key cannot be combined with OAuth")
+
+
+__all__: list[str] = []

screamingface/_core/__init__.py

@@ -0,0 +1,3 @@
+"""Internal domain ports shared by SDK adapters."""
+
+__all__: list[str] = []

screamingface/_core/ports.py

@@ -0,0 +1,59 @@
+"""Client-owned ports for SF Engine integration."""
+
+from __future__ import annotations
+
+from collections.abc import Awaitable, Callable
+from dataclasses import dataclass
+from datetime import datetime
+from typing import TYPE_CHECKING, Protocol
+
+if TYPE_CHECKING:
+    from screamingface._evaluation.model import Candidate
+    from screamingface.events import Event
+    from screamingface.report import Usage
+
+type SyncEventObserver = Callable[[Event], None]
+type AsyncEventObserver = Callable[[Event], None | Awaitable[None]]
+
+
+@dataclass(frozen=True, slots=True)
+class _RunOutcome:
+    """Transport-neutral root result retained for strict Report decoding."""
+
+    run_id: str
+    started_at: datetime
+    completed_at: datetime
+    result_body: str
+    media_type: str | None
+    root_usage: Usage | None
+
+
+class SyncRunTransport(Protocol):
+    """Execute one inspected Candidate through an SF Engine."""
+
+    def run(
+        self,
+        candidate: Candidate,
+        on_event: SyncEventObserver | None,
+    ) -> _RunOutcome: ...
+
+    def cancel_active(self) -> None: ...
+
+    def close(self) -> None: ...
+
+
+class AsyncRunTransport(Protocol):
+    """Asynchronous counterpart of :class:`SyncRunTransport`."""
+
+    async def run(
+        self,
+        candidate: Candidate,
+        on_event: AsyncEventObserver | None,
+    ) -> _RunOutcome: ...
+
+    async def cancel_active(self) -> None: ...
+
+    async def close(self) -> None: ...
+
+
+__all__: list[str] = []

screamingface/_core/wire.py

@@ -0,0 +1,36 @@
+"""Small, policy-free primitives for decoding external wire values."""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Mapping
+from typing import NoReturn, cast
+
+type _Invalid = Callable[[str], NoReturn]
+
+# httpx request-extension key marking a request as safe to re-send after an interactive
+# Cloudflare Access login.
+# INVARIANT: default-deny. Replay safety is a property of the REQUEST, not of the response
+# status, so a call site that forgets the marker pays one extra round trip — never an extra
+# paid Run. "GET /?q=" starts billable work despite being a GET and is never marked.
+# WHY here: this module has no dependencies, so marking a call site never drags the Access
+# stack (and its native crypto) into an import path that must stay cheap.
+_REPLAY_SAFE = "screamingface_replay_safe"
+
+
+def mapping(value: object, label: str, invalid: _Invalid) -> Mapping[str, object]:
+    """Return a JSON-like mapping or delegate the domain-specific failure."""
+
+    if not isinstance(value, Mapping):
+        invalid(f"{label} must be an object")
+    return cast(Mapping[str, object], value)
+
+
+def text(value: object, label: str, invalid: _Invalid) -> str:
+    """Return stripped non-blank wire text or delegate failure policy."""
+
+    if not isinstance(value, str) or not value.strip():
+        invalid(f"{label} must be non-blank text")
+    return cast(str, value).strip()
+
+
+__all__: list[str] = []

screamingface/_default_client.py

@@ -0,0 +1,181 @@
+"""One lazily constructed synchronous module-level Client."""
+
+from __future__ import annotations
+
+import os
+from threading import Lock
+from typing import TYPE_CHECKING, Literal, overload
+
+from screamingface.client import DEFAULT_ENGINE_URL, DEFAULT_SCOREBOARD_URL, Client
+
+if TYPE_CHECKING:
+    from collections.abc import Callable, Sequence
+
+    from screamingface._ui.connections import ConnectionPanel
+    from screamingface.connections import Connection, OAuthFlow
+    from screamingface.events import Event
+    from screamingface.recipe import Recipe
+    from screamingface.report import Report
+
+_client: Client | None = None
+_lock = Lock()
+
+
+def default_client() -> Client:
+    """Return the process-wide Client, constructing it on first use."""
+
+    global _client
+    if _client is not None:
+        return _client
+    with _lock:
+        if _client is None:
+            _client = Client(
+                engine_url=os.environ.get("SCREAMINGFACE_ENGINE_URL", DEFAULT_ENGINE_URL),
+                scoreboard_url=os.environ.get(
+                    "SCREAMINGFACE_SCOREBOARD_URL",
+                    DEFAULT_SCOREBOARD_URL,
+                ),
+            )
+    return _client
+
+
+def configure(
+    *,
+    engine_url: str = DEFAULT_ENGINE_URL,
+    scoreboard_url: str = DEFAULT_SCOREBOARD_URL,
+) -> Client:
+    """Replace the process-wide Client used by module-level convenience functions."""
+
+    global _client
+    replacement = Client(engine_url=engine_url, scoreboard_url=scoreboard_url)
+    with _lock:
+        previous = _client
+        _client = replacement
+    if previous is not None:
+        previous.close()
+    return replacement
+
+
+def close() -> None:
+    """Close and forget the process-wide Client, if it has been created."""
+
+    global _client
+    with _lock:
+        previous = _client
+        _client = None
+    if previous is not None:
+        previous.close()
+
+
+@overload
+def evaluate(
+    candidates: str,
+    *,
+    benchmark: None = None,
+    limit: None = None,
+    on_event: Callable[[Event], None] | None = None,
+    progress: bool | None = None,
+) -> Report: ...
+
+
+@overload
+def evaluate(
+    candidates: Recipe | Sequence[Recipe],
+    *,
+    benchmark: str,
+    limit: int | None = None,
+    on_event: Callable[[Event], None] | None = None,
+    progress: bool | None = None,
+) -> Report: ...
+
+
+def evaluate(
+    candidates: Recipe | Sequence[Recipe] | str,
+    *,
+    benchmark: str | None = None,
+    limit: int | None = None,
+    on_event: Callable[[Event], None] | None = None,
+    progress: bool | None = None,
+) -> Report:
+    """Evaluate Recipes or a complete URL4 through the lazy default Client."""
+
+    client = default_client()
+    if isinstance(candidates, str):
+        if benchmark is not None:
+            raise TypeError("benchmark must not be passed when evaluating a complete URL4")
+        if limit is not None:
+            raise TypeError("limit must not be passed when evaluating a complete URL4")
+        return client.evaluate(candidates, on_event=on_event, progress=progress)
+    if benchmark is None:
+        raise TypeError("benchmark is required when evaluating Recipes")
+    return client.evaluate(
+        candidates,
+        benchmark=benchmark,
+        limit=limit,
+        on_event=on_event,
+        progress=progress,
+    )
+
+
+@overload
+def connect(
+    provider: None = None,
+    *,
+    api_key: None = None,
+    method: None = None,
+) -> ConnectionPanel: ...
+
+
+@overload
+def connect(
+    provider: str,
+    *,
+    api_key: str,
+    method: Literal["api_key"] | None = None,
+) -> Connection: ...
+
+
+@overload
+def connect(
+    provider: str,
+    *,
+    api_key: None = None,
+    method: Literal["oauth"],
+) -> OAuthFlow: ...
+
+
+def connect(
+    provider: str | None = None,
+    *,
+    api_key: str | None = None,
+    method: Literal["api_key", "oauth"] | None = None,
+) -> Connection | ConnectionPanel | OAuthFlow:
+    """Open the provider panel or connect through the lazy default Client."""
+
+    if provider is None:
+        if api_key is not None or method is not None:
+            raise TypeError("provider is required when api_key or method is supplied")
+        from screamingface._ui.connections import ConnectionPanel
+
+        return ConnectionPanel(default_client())
+    client = default_client()
+    if api_key is not None:
+        if method == "oauth":
+            raise ValueError("api_key cannot be combined with OAuth")
+        result = client.connect(provider, api_key=api_key, method=method)
+    elif method == "api_key":
+        raise ValueError("api_key is required for API-key authentication")
+    elif method == "oauth":
+        result = client.connect(provider, method="oauth")
+    else:
+        raise ValueError("api_key is required unless method='oauth' is selected")
+    return result
+
+
+def disconnect(provider: str) -> Connection:
+    """Disconnect a provider through the lazy default Client."""
+
+    return default_client().disconnect(provider)
+
+
+__all__ = ["close", "configure", "connect", "disconnect", "evaluate"]

screamingface/_engine/__init__.py

@@ -0,0 +1,3 @@
+"""Internal Engine adapters and wire contracts."""
+
+__all__: list[str] = []