valkyrie-sdk 0.1.0__py3-none-any.whl

valkyrie/sdk/__init__.py

@@ -0,0 +1,48 @@
+"""Public interface for the async Valkyrie SDK."""
+
+from valkyrie.sdk.models import (
+    AgentContractRequest,
+    BenchmarkStatus,
+    FetchBenchmarkResponse,
+    FetchBenchmarksRequest,
+    FetchBenchmarksResponse,
+    FinalViewResponse,
+    RetryMode,
+    RetryOrResumeBenchmarkResponse,
+    S3UploadResultsResponse,
+    StartBenchmarkResponse,
+    StopBenchmarkResponse,
+)
+
+from valkyrie.sdk.client import ValkyrieClient
+from valkyrie.sdk.config import ValkyrieConfig
+from valkyrie.sdk.errors import (
+    ValkyrieAPIError,
+    ValkyrieConfigError,
+    ValkyrieRunError,
+    ValkyrieSDKError,
+    ValkyrieStreamError,
+    ValkyrieTransportError,
+)
+
+__all__ = [
+    "AgentContractRequest",
+    "BenchmarkStatus",
+    "FetchBenchmarkResponse",
+    "FetchBenchmarksRequest",
+    "FetchBenchmarksResponse",
+    "FinalViewResponse",
+    "RetryMode",
+    "RetryOrResumeBenchmarkResponse",
+    "S3UploadResultsResponse",
+    "StartBenchmarkResponse",
+    "StopBenchmarkResponse",
+    "ValkyrieAPIError",
+    "ValkyrieClient",
+    "ValkyrieConfig",
+    "ValkyrieConfigError",
+    "ValkyrieRunError",
+    "ValkyrieSDKError",
+    "ValkyrieStreamError",
+    "ValkyrieTransportError",
+]

valkyrie/sdk/client.py

@@ -0,0 +1,120 @@
+"""Async client for the Valkyrie API."""
+
+from __future__ import annotations
+
+import os
+from contextlib import AbstractAsyncContextManager
+from pathlib import Path
+from types import TracebackType
+from typing import Any, TypeVar
+
+import httpx
+from pydantic import BaseModel
+
+from valkyrie.sdk.config import DEFAULT_CONFIG_PATH, ValkyrieConfig
+from valkyrie.sdk.errors import ValkyrieAPIError, ValkyrieTransportError
+from valkyrie.sdk.resources.runs import RunsResource
+
+DEFAULT_BASE_URL = "https://benchmark-tracker.vals.ai"
+ResponseModel = TypeVar("ResponseModel", bound=BaseModel)
+
+
+class ValkyrieClient:
+    """Async client for Valkyrie API resources."""
+
+    def __init__(
+        self,
+        config: ValkyrieConfig,
+        *,
+        base_url: str | None = None,
+        timeout: float | httpx.Timeout = 120,
+        transport: httpx.AsyncBaseTransport | None = None,
+    ) -> None:
+        self.config = config
+        resolved_base_url = base_url or os.environ.get("TRACKER_SERVICE_URL", DEFAULT_BASE_URL)
+        self._timeout = timeout if isinstance(timeout, httpx.Timeout) else httpx.Timeout(timeout)
+        self._client = httpx.AsyncClient(
+            base_url=resolved_base_url.rstrip("/"),
+            timeout=self._timeout,
+            headers=config.request_headers(),
+            transport=transport,
+        )
+        self.runs = RunsResource(self)
+
+    @classmethod
+    def from_config(
+        cls,
+        path: str | Path = DEFAULT_CONFIG_PATH,
+        *,
+        base_url: str | None = None,
+        timeout: float | httpx.Timeout = 120,
+        transport: httpx.AsyncBaseTransport | None = None,
+    ) -> ValkyrieClient:
+        """Load a client from a CLI-compatible YAML config."""
+        return cls(
+            ValkyrieConfig.from_yaml(path),
+            base_url=base_url,
+            timeout=timeout,
+            transport=transport,
+        )
+
+    async def __aenter__(self) -> ValkyrieClient:
+        return self
+
+    async def __aexit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_value: BaseException | None,
+        traceback: TracebackType | None,
+    ) -> None:
+        await self.close()
+
+    async def close(self) -> None:
+        """Close the underlying HTTP connection pool."""
+        await self._client.aclose()
+
+    async def request_model(
+        self,
+        method: str,
+        path: str,
+        model: type[ResponseModel],
+        *,
+        params: dict[str, Any] | None = None,
+        json: Any = None,
+    ) -> ResponseModel:
+        """Send a request and validate its response as a Pydantic model."""
+        try:
+            response = await self._client.request(method, path, params=params, json=json)
+        except httpx.HTTPError as exc:
+            raise ValkyrieTransportError(f"Valkyrie request failed: {exc}") from exc
+
+        self.raise_for_status(response)
+        return model.model_validate(response.json())
+
+    @staticmethod
+    def raise_for_status(response: httpx.Response) -> None:
+        """Convert a non-success response into an SDK API error."""
+        if response.is_success:
+            return
+        try:
+            body: Any = response.json()
+        except ValueError:
+            body = response.text
+        detail = body.get("detail", body) if isinstance(body, dict) else body
+        raise ValkyrieAPIError(response.status_code, detail)
+
+    def stream_response(
+        self,
+        method: str,
+        path: str,
+        *,
+        params: dict[str, Any] | None = None,
+    ) -> AbstractAsyncContextManager[httpx.Response]:
+        """Open a streaming response without a read timeout."""
+        timeout = httpx.Timeout(
+            connect=self._timeout.connect,
+            read=None,
+            write=self._timeout.write,
+            pool=self._timeout.pool,
+        )
+        return self._client.stream(method, path, params=params, timeout=timeout)

valkyrie/sdk/config.py

@@ -0,0 +1,121 @@
+"""Typed configuration for the Valkyrie SDK."""
+
+from pathlib import Path
+from typing import TypeVar, cast
+
+import yaml
+from pydantic import BaseModel, ConfigDict, Field, SecretStr, ValidationError, field_validator
+
+from valkyrie.sdk.errors import ValkyrieConfigError
+from valkyrie.sdk.models import AWSCredentials, HarnessConfig
+
+DEFAULT_CONFIG_PATH = Path("~/.config/valkyrie/valkyrie.yaml")
+ConfigT = TypeVar("ConfigT", bound="ValkyrieConfig")
+
+
+class ValkyrieConfig(BaseModel):
+    """Validated SDK configuration using ``valkyrie.yaml`` field aliases."""
+
+    model_config = ConfigDict(populate_by_name=True, extra="forbid")
+
+    api_key: SecretStr | None = Field(default=None, repr=False)
+    aws_access_key_id: SecretStr = Field(alias="AWS_ACCESS_KEY_ID", repr=False)
+    aws_secret_access_key: SecretStr = Field(alias="AWS_SECRET_ACCESS_KEY", repr=False)
+    aws_default_region: str = Field(alias="AWS_DEFAULT_REGION")
+    aws_session_token: SecretStr | None = Field(default=None, alias="AWS_SESSION_TOKEN", repr=False)
+    s3_bucket: str = Field(alias="S3_BUCKET")
+    log_group: str = Field(default="benchmarks", alias="LOG_GROUP")
+    log_retention_policy: int = Field(default=365, alias="LOG_RETENTION_POLICY", gt=0)
+    sandbox_providers: dict[str, str] = Field(min_length=1, repr=False)
+    default_sandbox_provider: str | None = None
+    custom_benchmark_services: dict[str, str] = Field(default_factory=dict)
+    benchmark_auth: dict[str, SecretStr] = Field(default_factory=dict, repr=False)
+    webhook: str | None = Field(default=None, repr=False)
+
+    @field_validator(
+        "aws_default_region",
+        "s3_bucket",
+        "log_group",
+    )
+    @classmethod
+    def reject_blank_required_values(cls, value: str) -> str:
+        """Reject blank required values."""
+        if not value.strip():
+            raise ValueError("must not be blank")
+        return value
+
+    @field_validator("aws_access_key_id", "aws_secret_access_key")
+    @classmethod
+    def reject_blank_required_secrets(cls, value: SecretStr) -> SecretStr:
+        """Reject blank required secret values."""
+        if not value.get_secret_value().strip():
+            raise ValueError("must not be blank")
+        return value
+
+    @field_validator("custom_benchmark_services")
+    @classmethod
+    def normalize_service_urls(cls, services: dict[str, str]) -> dict[str, str]:
+        """Remove trailing slashes from service URLs."""
+        return {name: url.rstrip("/") for name, url in services.items()}
+
+    @classmethod
+    def from_yaml(cls: type[ConfigT], path: str | Path = DEFAULT_CONFIG_PATH) -> ConfigT:
+        """Load and validate a CLI-compatible YAML config."""
+        config_path = Path(path).expanduser()
+        try:
+            with config_path.open(encoding="utf-8") as config_file:
+                raw_config = cast(object, yaml.safe_load(config_file))
+        except OSError as exc:
+            raise ValkyrieConfigError(f"Could not read Valkyrie config at {config_path}: {exc}") from exc
+        except yaml.YAMLError as exc:
+            raise ValkyrieConfigError(f"Invalid YAML in Valkyrie config at {config_path}: {exc}") from exc
+
+        if not isinstance(raw_config, dict):
+            raise ValkyrieConfigError(f"Valkyrie config at {config_path} must contain a YAML mapping")
+
+        try:
+            return cls.model_validate(cast(dict[str, object], raw_config))
+        except ValidationError as exc:
+            raise ValkyrieConfigError(f"Invalid Valkyrie config at {config_path}: {exc}") from exc
+
+    def resolve_sandbox_provider(self, provider: str | None = None) -> tuple[str, str]:
+        """Resolve the selected sandbox provider and secret name."""
+        provider_name = provider or self.default_sandbox_provider or next(iter(self.sandbox_providers))
+        secret_name = self.sandbox_providers.get(provider_name)
+        if secret_name is None:
+            configured = ", ".join(self.sandbox_providers)
+            raise ValkyrieConfigError(f"Unknown sandbox provider '{provider_name}'. Configured providers: {configured}")
+        return provider_name, secret_name
+
+    def harness_config(self, provider_secret_name: str) -> HarnessConfig:
+        """Build the nested harness config expected by the tracker."""
+        return HarnessConfig(
+            aws=AWSCredentials(
+                aws_access_key_id=self.aws_access_key_id.get_secret_value(),
+                aws_secret_access_key=self.aws_secret_access_key.get_secret_value(),
+                aws_default_region=self.aws_default_region,
+                aws_session_token=(self.aws_session_token.get_secret_value() if self.aws_session_token else None),
+            ),
+            s3_bucket=self.s3_bucket,
+            log_group=self.log_group,
+            log_retention_policy=self.log_retention_policy,
+            sandbox_provider_secret_name=provider_secret_name,
+        )
+
+    def request_headers(self) -> dict[str, str]:
+        """Build API-key and harness headers for tracker requests."""
+        values: dict[str, str | None] = {
+            "AWS_ACCESS_KEY_ID": self.aws_access_key_id.get_secret_value(),
+            "AWS_SECRET_ACCESS_KEY": self.aws_secret_access_key.get_secret_value(),
+            "AWS_DEFAULT_REGION": self.aws_default_region,
+            "AWS_SESSION_TOKEN": self.aws_session_token.get_secret_value() if self.aws_session_token else None,
+            "S3_BUCKET": self.s3_bucket,
+            "LOG_GROUP": self.log_group,
+            "LOG_RETENTION_POLICY": str(self.log_retention_policy),
+        }
+        headers = {
+            f"X-Harness-{key.replace('_', '-').title()}": value for key, value in values.items() if value is not None
+        }
+        if self.api_key and (api_key := self.api_key.get_secret_value()):
+            headers["X-Api-Key"] = api_key
+        return headers

valkyrie/sdk/errors.py

@@ -0,0 +1,32 @@
+"""Exceptions raised by the Valkyrie SDK."""
+
+from typing import Any
+
+
+class ValkyrieSDKError(Exception):
+    """Base class for all SDK errors."""
+
+
+class ValkyrieConfigError(ValkyrieSDKError):
+    """The supplied Valkyrie configuration is missing or invalid."""
+
+
+class ValkyrieRunError(ValkyrieSDKError):
+    """A run operation received invalid client-side input."""
+
+
+class ValkyrieTransportError(ValkyrieSDKError):
+    """A request could not reach the Valkyrie service."""
+
+
+class ValkyrieAPIError(ValkyrieSDKError):
+    """The Valkyrie API returned a non-success response."""
+
+    def __init__(self, status_code: int, detail: Any):
+        self.status_code = status_code
+        self.detail = detail
+        super().__init__(f"Valkyrie API request failed ({status_code}): {detail}")
+
+
+class ValkyrieStreamError(ValkyrieSDKError):
+    """A run update stream returned an error or invalid event."""

valkyrie/sdk/models/__init__.py

@@ -0,0 +1,54 @@
+"""Models owned by the standalone Valkyrie SDK."""
+
+from valkyrie.sdk.models.agents import AgentContractRequest, OutputArtifact, OutputArtifactSpec
+from valkyrie.sdk.models.config import AWSCredentials, HarnessConfig
+from valkyrie.sdk.models.runs import (
+    AverageTaskBreakdown,
+    BenchmarkArguments,
+    BenchmarkDetails,
+    BenchmarkStatus,
+    BenchmarkTableRow,
+    DocentReadingStatus,
+    FetchBenchmarkResponse,
+    FetchBenchmarksRequest,
+    FetchBenchmarksResponse,
+    FinalEvaluation,
+    FinalViewResponse,
+    Order,
+    RetrieveResultsResponse,
+    RetryMode,
+    RetryOrResumeBenchmarkResponse,
+    S3UploadResultsResponse,
+    StartBenchmarkRequest,
+    StartBenchmarkResponse,
+    StopBenchmarkResponse,
+    TaskStatus,
+)
+
+__all__ = [
+    "AWSCredentials",
+    "AgentContractRequest",
+    "AverageTaskBreakdown",
+    "BenchmarkArguments",
+    "BenchmarkDetails",
+    "BenchmarkStatus",
+    "BenchmarkTableRow",
+    "DocentReadingStatus",
+    "FetchBenchmarkResponse",
+    "FetchBenchmarksRequest",
+    "FetchBenchmarksResponse",
+    "FinalEvaluation",
+    "FinalViewResponse",
+    "HarnessConfig",
+    "Order",
+    "OutputArtifact",
+    "OutputArtifactSpec",
+    "RetrieveResultsResponse",
+    "RetryMode",
+    "RetryOrResumeBenchmarkResponse",
+    "S3UploadResultsResponse",
+    "StartBenchmarkRequest",
+    "StartBenchmarkResponse",
+    "StopBenchmarkResponse",
+    "TaskStatus",
+]

valkyrie/sdk/models/_base.py

@@ -0,0 +1,20 @@
+"""Shared SDK model behavior."""
+
+from datetime import datetime, timezone
+
+from pydantic import BaseModel, ConfigDict
+
+
+class ResponseModel(BaseModel):
+    """Forward-compatible model for data returned by the Valkyrie API."""
+
+    model_config = ConfigDict(extra="ignore")
+
+
+def serialize_utc(value: datetime | None) -> str | None:
+    """Serialize datetimes with an explicit UTC offset."""
+    if value is None:
+        return None
+    if value.tzinfo is None:
+        value = value.replace(tzinfo=timezone.utc)
+    return value.isoformat()

valkyrie/sdk/models/agents.py

@@ -0,0 +1,77 @@
+"""Agent contract models used by SDK run requests."""
+
+from pathlib import PurePosixPath
+
+from pydantic import BaseModel, Field, field_validator
+
+MAX_OUTPUT_ARTIFACT_COUNT = 10
+
+
+def _source_has_glob(source: str) -> bool:
+    return any(char in source for char in "*?[")
+
+
+def _source_glob_root(source: str) -> str:
+    glob_indices = [source.find(char) for char in "*?[" if source.find(char) != -1]
+    first_glob_index = min(glob_indices)
+    return source[:first_glob_index].rsplit("/", 1)[0] or "/"
+
+
+class OutputArtifact(BaseModel):
+    """One artifact copied from a sandbox after an agent run."""
+
+    path: str
+    source: str | None = None
+
+    @field_validator("source")
+    @classmethod
+    def validate_source(cls, value: str | None) -> str | None:
+        """Validate an optional absolute sandbox source path."""
+        if not value:
+            return None
+        if not value.startswith("/"):
+            raise ValueError("output_artifacts source paths must be absolute sandbox paths")
+
+        path = PurePosixPath(value)
+        if not path.parts or ".." in path.parts or "." in path.parts:
+            raise ValueError("output_artifacts source paths cannot contain empty, '.', or '..' path parts")
+        if _source_has_glob(value) and _source_glob_root(value) == "/":
+            raise ValueError("output_artifacts glob sources must include a non-root directory prefix")
+        return value
+
+
+OutputArtifactSpec = str | OutputArtifact
+
+
+class AgentContractRequest(BaseModel):
+    """Agent definition submitted when starting a run."""
+
+    name: str
+    model: str | None = None
+    install_cmd: str = ""
+    run_cmd: str = ""
+    final_output: str | None = None
+    output_artifacts: list[OutputArtifactSpec] = Field(default_factory=list)
+    egress_allowlist: list[str] = Field(default_factory=list)
+    secrets: dict[str, str] = Field(default_factory=dict)
+    kwargs: dict[str, str] = Field(default_factory=dict)
+
+    @field_validator("output_artifacts")
+    @classmethod
+    def validate_output_artifacts(cls, value: list[OutputArtifactSpec]) -> list[OutputArtifactSpec]:
+        """Validate and normalize artifact destination paths."""
+        if len(value) > MAX_OUTPUT_ARTIFACT_COUNT:
+            raise ValueError(f"output_artifacts cannot contain more than {MAX_OUTPUT_ARTIFACT_COUNT} entries")
+
+        normalized_artifacts: list[OutputArtifactSpec] = []
+        for artifact in value:
+            artifact_path = artifact if isinstance(artifact, str) else artifact.path
+            path = PurePosixPath(artifact_path)
+            if path.is_absolute():
+                raise ValueError("output_artifacts paths must be relative paths")
+            if not path.parts or ".." in path.parts or "." in path.parts:
+                raise ValueError("output_artifacts paths cannot contain empty, '.', or '..' path parts")
+            normalized_artifacts.append(
+                str(path) if isinstance(artifact, str) else artifact.model_copy(update={"path": str(path)})
+            )
+        return normalized_artifacts

valkyrie/sdk/models/config.py

@@ -0,0 +1,22 @@
+"""Nested configuration models sent to the Valkyrie API."""
+
+from pydantic import BaseModel, Field
+
+
+class AWSCredentials(BaseModel, frozen=True):
+    """AWS credentials forwarded in the run harness configuration."""
+
+    aws_access_key_id: str
+    aws_secret_access_key: str = Field(repr=False)
+    aws_default_region: str
+    aws_session_token: str | None = Field(default=None, repr=False)
+
+
+class HarnessConfig(BaseModel):
+    """Harness configuration required to start a run."""
+
+    aws: AWSCredentials
+    s3_bucket: str
+    log_group: str
+    log_retention_policy: int
+    sandbox_provider_secret_name: str