elementary-runtime 0.1.0rc1__py3-none-any.whl

elementary_runtime/__init__.py

@@ -0,0 +1,5 @@
+from importlib.metadata import version
+
+__version__ = version("elementary-runtime")
+
+__all__ = ["__version__"]

elementary_runtime/common/validators.py

@@ -0,0 +1,27 @@
+from collections.abc import Callable, Hashable, Sequence
+from typing import TypeVar
+
+T = TypeVar("T")
+H = TypeVar("H", bound=Hashable)
+
+
+def require_unique(items: Sequence[H]) -> list[H]:
+    seen: set[H] = set()
+    for item in items:
+        if item in seen:
+            raise ValueError("values must be unique")
+        seen.add(item)
+    return list(items)
+
+
+def require_unique_attr(attr: str) -> Callable[[Sequence[T]], list[T]]:
+    def validator(items: Sequence[T]) -> list[T]:
+        seen: set[object] = set()
+        for item in items:
+            value = getattr(item, attr)
+            if value in seen:
+                raise ValueError(f"duplicate {attr} {value!r}")
+            seen.add(value)
+        return list(items)
+
+    return validator

elementary_runtime/config/__init__.py

@@ -0,0 +1,40 @@
+from elementary_runtime.config.base import ConfigModel
+from elementary_runtime.config.connections import (
+    SnowflakeConnectionConfig,
+    WarehouseConnection,
+    WarehouseConnectionConfig,
+    WarehouseType,
+)
+from elementary_runtime.config.errors import (
+    ConfigError,
+    InvalidPlaceholderError,
+    MissingEnvironmentVariableError,
+)
+from elementary_runtime.config.runtime import RuntimeConfig
+from elementary_runtime.config.secrets import FileBacked, FileBackedSecret
+from elementary_runtime.config.sections import (
+    DEFAULT_API_URL,
+    CloudConfig,
+    ConcurrencyConfig,
+    InstanceConfig,
+    LimitsConfig,
+)
+
+__all__ = [
+    "DEFAULT_API_URL",
+    "CloudConfig",
+    "ConcurrencyConfig",
+    "ConfigError",
+    "ConfigModel",
+    "FileBacked",
+    "FileBackedSecret",
+    "InstanceConfig",
+    "InvalidPlaceholderError",
+    "LimitsConfig",
+    "MissingEnvironmentVariableError",
+    "RuntimeConfig",
+    "SnowflakeConnectionConfig",
+    "WarehouseConnection",
+    "WarehouseConnectionConfig",
+    "WarehouseType",
+]

elementary_runtime/config/base.py

@@ -0,0 +1,19 @@
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, model_validator
+
+from elementary_runtime.config.secrets import resolve_file_backed
+
+
+class ConfigModel(BaseModel):
+    model_config = ConfigDict(
+        extra="forbid",
+        frozen=True,
+        allow_inf_nan=False,
+        validate_default=True,
+    )
+
+    @model_validator(mode="before")
+    @classmethod
+    def _resolve_file_backed(cls, data: Any) -> Any:
+        return resolve_file_backed(cls, data)

elementary_runtime/config/connections.py

@@ -0,0 +1,55 @@
+from enum import Enum
+from typing import Annotated, Literal
+
+from pydantic import Field, model_validator
+from typing_extensions import Self
+
+from elementary_runtime.config.base import ConfigModel
+from elementary_runtime.config.secrets import FileBackedSecret
+from elementary_runtime.protocol import IdStr, NonEmptyStr
+
+
+class WarehouseType(str, Enum):
+    SNOWFLAKE = "snowflake"
+
+
+class WarehouseConnectionConfig(ConfigModel):
+    warehouse_connection_id: Annotated[
+        IdStr, Field(description="Elementary Cloud connection id")
+    ]
+    type: WarehouseType
+
+
+class SnowflakeConnectionConfig(WarehouseConnectionConfig):
+    type: Literal[WarehouseType.SNOWFLAKE] = WarehouseType.SNOWFLAKE
+    account: NonEmptyStr
+    user: NonEmptyStr
+    password: FileBackedSecret | None = None
+    private_key: Annotated[
+        FileBackedSecret | None, Field(description="key-pair auth")
+    ] = None
+    private_key_passphrase: FileBackedSecret | None = None
+    role: NonEmptyStr | None = None
+    warehouse: Annotated[
+        NonEmptyStr | None, Field(description="Snowflake compute warehouse")
+    ] = None
+
+    @model_validator(mode="after")
+    def _password_or_private_key(self) -> Self:
+        if self.password is None and self.private_key is None:
+            raise ValueError("password or private_key is required")
+        if self.password is not None and self.private_key is not None:
+            raise ValueError("password and private_key cannot both be set")
+        if self.private_key is not None and self.private_key_passphrase is None:
+            raise ValueError(
+                "private_key_passphrase is required when private_key is set"
+            )
+        if self.password is not None and self.private_key_passphrase is not None:
+            raise ValueError("private_key_passphrase cannot be set with password")
+        return self
+
+
+WarehouseConnection = Annotated[
+    SnowflakeConnectionConfig,
+    Field(discriminator="type"),
+]

elementary_runtime/config/env.py

@@ -0,0 +1,66 @@
+import os
+from typing import Any
+
+from expandvars import (
+    ExpandvarsException,
+    ParameterNullOrNotSet,
+    UnboundVariable,
+    expand,
+)
+
+from elementary_runtime.config.errors import (
+    InvalidPlaceholderError,
+    MissingEnvironmentVariableError,
+)
+
+# expandvars honors this as a process-wide escape hatch: when set, missing
+# required variables are replaced with its value instead of raising. Strip it
+# so a customer environment cannot silently weaken nounset.
+_RECOVER_NULL = "EXPANDVARS_RECOVER_NULL"
+
+
+def expand_env(value: Any, environ: dict[str, str] | None = None) -> Any:
+    env = dict(os.environ if environ is None else environ)
+    env.pop(_RECOVER_NULL, None)
+    return _expand(value, env)
+
+
+def _expand(value: Any, environ: dict[str, str]) -> Any:
+    if isinstance(value, str):
+        return _expand_string(value, environ)
+    if isinstance(value, list):
+        return [_expand(item, environ) for item in value]
+    if isinstance(value, dict):
+        return {key: _expand(item, environ) for key, item in value.items()}
+    return value
+
+
+def _exc_message(exc: BaseException) -> str:
+    if exc.args and isinstance(exc.args[0], str):
+        return exc.args[0]
+    return str(exc)
+
+
+def _placeholder_name(exc: BaseException) -> tuple[str, str | None]:
+    name, sep, detail = _exc_message(exc).partition(": ")
+    if not sep or detail in {"unbound variable", "parameter null or not set"}:
+        return name, None
+    return name, detail
+
+
+def _expand_string(value: str, environ: dict[str, str]) -> str:
+    try:
+        expanded = expand(
+            value,
+            nounset=True,
+            environ=environ,
+            surrounded_vars_only=True,
+        )
+    except (UnboundVariable, ParameterNullOrNotSet) as exc:
+        name, hint = _placeholder_name(exc)
+        raise MissingEnvironmentVariableError(name, hint) from exc
+    except ExpandvarsException as exc:
+        raise InvalidPlaceholderError(value, _exc_message(exc)) from exc
+    if not isinstance(expanded, str):
+        raise InvalidPlaceholderError(value, "expansion must produce a string")
+    return expanded

elementary_runtime/config/errors.py

@@ -0,0 +1,20 @@
+class ConfigError(Exception):
+    def __init__(self, message: str) -> None:
+        super().__init__(message)
+
+
+class MissingEnvironmentVariableError(ConfigError):
+    def __init__(self, name: str, hint: str | None = None) -> None:
+        self.name = name
+        self.hint = hint
+        if hint is None:
+            super().__init__(f"missing environment variable {name}")
+        else:
+            super().__init__(f"missing environment variable {name}: {hint}")
+
+
+class InvalidPlaceholderError(ConfigError):
+    def __init__(self, value: str, reason: str) -> None:
+        self.value = value
+        self.reason = reason
+        super().__init__(f"invalid placeholder in {value!r}: {reason}")

elementary_runtime/config/runtime.py

@@ -0,0 +1,60 @@
+from collections.abc import Mapping
+from pathlib import Path
+from typing import Annotated, Any, Literal
+
+import yaml
+from pydantic import AfterValidator, Field, ValidationError
+from typing_extensions import Self
+
+from elementary_runtime.common.validators import require_unique_attr
+from elementary_runtime.config.base import ConfigModel
+from elementary_runtime.config.connections import WarehouseConnection
+from elementary_runtime.config.env import expand_env
+from elementary_runtime.config.errors import ConfigError
+from elementary_runtime.config.sections import (
+    CloudConfig,
+    ConcurrencyConfig,
+    InstanceConfig,
+    LimitsConfig,
+)
+
+
+class RuntimeConfig(ConfigModel):
+    version: Literal[1]
+    cloud: CloudConfig
+    instance: InstanceConfig = Field(default_factory=InstanceConfig)
+    limits: LimitsConfig = Field(default_factory=LimitsConfig)
+    concurrency: ConcurrencyConfig = Field(default_factory=ConcurrencyConfig)
+    connections: Annotated[
+        list[WarehouseConnection],
+        Field(min_length=1),
+        AfterValidator(require_unique_attr("warehouse_connection_id")),
+    ]
+
+    @classmethod
+    def from_dict(cls, data: Mapping[str, Any]) -> Self:
+        try:
+            return cls.model_validate(data)
+        except ValidationError as exc:
+            raise ConfigError(str(exc)) from exc
+
+    @classmethod
+    def from_yaml(cls, path: str | Path) -> Self:
+        config_path = Path(path)
+        try:
+            raw = config_path.read_text(encoding="utf-8")
+        except OSError:
+            raise ConfigError(
+                f"unable to read configuration file {config_path}"
+            ) from None
+        try:
+            loaded = yaml.safe_load(raw)
+        except yaml.YAMLError as exc:
+            raise ConfigError(
+                f"invalid YAML in configuration file {config_path}: {exc}"
+            ) from exc
+        if loaded is None:
+            loaded = {}
+        if not isinstance(loaded, dict):
+            raise ConfigError("configuration root must be a mapping")
+        return cls.from_dict(expand_env(loaded))

elementary_runtime/config/secrets.py

@@ -0,0 +1,77 @@
+from dataclasses import dataclass
+from functools import cache
+from pathlib import Path
+from types import UnionType
+from typing import Annotated, Any, Union, get_args, get_origin
+
+from pydantic import AfterValidator, BaseModel, SecretStr
+
+FILE_SUFFIX = "_file"
+
+
+@dataclass(frozen=True, slots=True)
+class FileBacked:
+    pass
+
+
+def _require_non_empty_secret(value: SecretStr) -> SecretStr:
+    if not value.get_secret_value():
+        raise ValueError("value must not be empty")
+    return value
+
+
+FileBackedSecret = Annotated[
+    SecretStr,
+    FileBacked(),
+    AfterValidator(_require_non_empty_secret),
+]
+
+
+def read_secret_file(path: object) -> str:
+    if not isinstance(path, str) or not path:
+        raise ValueError("secret file path must be a non-empty string")
+    try:
+        return Path(path).read_text(encoding="utf-8").strip()
+    except OSError:
+        raise ValueError(f"unable to read secret file {path}") from None
+
+
+def _has_file_backed(annotation: object) -> bool:
+    origin = get_origin(annotation)
+    if origin is Annotated:
+        args = get_args(annotation)
+        return any(isinstance(arg, FileBacked) for arg in args[1:]) or _has_file_backed(
+            args[0]
+        )
+    if origin in {Union, UnionType}:
+        return any(_has_file_backed(arg) for arg in get_args(annotation))
+    return False
+
+
+@cache
+def file_backed_field_names(cls: type[BaseModel]) -> tuple[str, ...]:
+    return tuple(
+        name
+        for name, field in cls.model_fields.items()
+        if any(isinstance(marker, FileBacked) for marker in field.metadata)
+        or _has_file_backed(field.annotation)
+    )
+
+
+def resolve_file_backed(cls: type[BaseModel], data: Any) -> Any:
+    if not isinstance(data, dict):
+        return data
+    names = file_backed_field_names(cls)
+    if not names:
+        return data
+    resolved = dict(data)
+    for name in names:
+        file_key = f"{name}{FILE_SUFFIX}"
+        has_value = name in resolved
+        has_file = file_key in resolved
+        if has_value and has_file:
+            raise ValueError(f"{name} and {file_key} cannot both be set")
+        if not has_file:
+            continue
+        resolved[name] = read_secret_file(resolved.pop(file_key))
+    return resolved

elementary_runtime/config/sections.py

@@ -0,0 +1,98 @@
+from typing import Annotated
+
+from pydantic import Field, HttpUrl, TypeAdapter, model_validator
+from typing_extensions import Self
+
+from elementary_runtime.config.base import ConfigModel
+from elementary_runtime.config.secrets import FileBackedSecret
+from elementary_runtime.protocol import DisplayName, InstanceIdPrefix
+from elementary_runtime.protocol.limits import (
+    MAX_POLL_TASKS,
+    MAX_QUERY_TIMEOUT_SECONDS,
+    MAX_SAMPLE_ROW_LIMIT,
+)
+
+DEFAULT_API_URL = "https://app.elementary-data.com/runtime/v1"
+_DEFAULT_API_URL = TypeAdapter(HttpUrl).validate_python(DEFAULT_API_URL)
+DEFAULT_MAX_QUERY_TIMEOUT_SECONDS = 300
+DEFAULT_MAX_RESULT_BYTES = 10_485_760
+DEFAULT_MAX_SAMPLE_ROWS_RETURNED = 1000
+DEFAULT_MAX_CONCURRENT_TASKS = 8
+DEFAULT_POLL_INTERVAL_SECONDS = 10
+DEFAULT_HEARTBEAT_INTERVAL_SECONDS = 30
+
+MAX_QUERY_TIMEOUT_SECONDS_LIMIT = MAX_QUERY_TIMEOUT_SECONDS
+MAX_RESULT_BYTES_LIMIT = 1_073_741_824
+MAX_SAMPLE_ROWS_RETURNED_LIMIT = MAX_SAMPLE_ROW_LIMIT
+MAX_CONCURRENT_TASKS_LIMIT = 256
+POLL_INTERVAL_SECONDS_LIMIT = 3_600
+HEARTBEAT_INTERVAL_SECONDS_LIMIT = 3_600
+
+
+class CloudConfig(ConfigModel):
+    api_url: Annotated[HttpUrl, Field(description="Elementary Cloud runtime API")] = (
+        _DEFAULT_API_URL
+    )
+    token: FileBackedSecret
+
+
+class InstanceConfig(ConfigModel):
+    id_prefix: Annotated[
+        InstanceIdPrefix | None,
+        Field(description="prepended to the generated instance id"),
+    ] = None
+    display_name: DisplayName | None = None
+
+
+class LimitsConfig(ConfigModel):
+    max_query_timeout_seconds: Annotated[
+        int, Field(ge=1, le=MAX_QUERY_TIMEOUT_SECONDS_LIMIT)
+    ] = DEFAULT_MAX_QUERY_TIMEOUT_SECONDS
+    max_result_bytes: Annotated[
+        int,
+        Field(
+            ge=1,
+            le=MAX_RESULT_BYTES_LIMIT,
+            description="encoded result payload sent to Cloud",
+        ),
+    ] = DEFAULT_MAX_RESULT_BYTES
+    max_sample_rows_returned: Annotated[
+        int,
+        Field(
+            ge=0,
+            le=MAX_SAMPLE_ROWS_RETURNED_LIMIT,
+            description="rows included in the result sample",
+        ),
+    ] = DEFAULT_MAX_SAMPLE_ROWS_RETURNED
+
+
+class ConcurrencyConfig(ConfigModel):
+    max_concurrent_tasks: Annotated[int, Field(ge=1, le=MAX_CONCURRENT_TASKS_LIMIT)] = (
+        DEFAULT_MAX_CONCURRENT_TASKS
+    )
+    prefetch_count: Annotated[
+        int, Field(ge=1, le=MAX_POLL_TASKS, description="tasks requested per poll")
+    ] = DEFAULT_MAX_CONCURRENT_TASKS
+    poll_interval_seconds: Annotated[
+        int, Field(ge=1, le=POLL_INTERVAL_SECONDS_LIMIT)
+    ] = DEFAULT_POLL_INTERVAL_SECONDS
+    heartbeat_interval_seconds: Annotated[
+        int, Field(ge=1, le=HEARTBEAT_INTERVAL_SECONDS_LIMIT)
+    ] = DEFAULT_HEARTBEAT_INTERVAL_SECONDS
+
+    @model_validator(mode="before")
+    @classmethod
+    def _default_prefetch_count(cls, data: object) -> object:
+        if not isinstance(data, dict):
+            return data
+        data = dict(data)
+        if data.get("prefetch_count") is None:
+            max_tasks = data.get("max_concurrent_tasks", DEFAULT_MAX_CONCURRENT_TASKS)
+            data["prefetch_count"] = max_tasks
+        return data
+
+    @model_validator(mode="after")
+    def _prefetch_within_capacity(self) -> Self:
+        if self.prefetch_count > self.max_concurrent_tasks:
+            raise ValueError("prefetch_count cannot exceed max_concurrent_tasks")
+        return self

elementary_runtime/protocol/__init__.py

@@ -0,0 +1,162 @@
+from elementary_runtime.protocol.base import (
+    DisplayName,
+    ErrorMessage,
+    IdStr,
+    InstanceId,
+    InstanceIdPrefix,
+    NonEmptyStr,
+    ProtocolEnum,
+    ProtocolModel,
+    UtcDatetime,
+)
+from elementary_runtime.protocol.enums import (
+    ConnectionErrorCode,
+    ConnectionHealth,
+    ErrorCode,
+    ExceptionComponent,
+    ExceptionOperation,
+    HeartbeatStatus,
+    RuntimeStopReason,
+    SubmittedResultStatus,
+    TaskExecutionOutcome,
+    TaskType,
+    TelemetryEventType,
+    TestSourceType,
+)
+from elementary_runtime.protocol.errors import (
+    ErrorBody,
+    ErrorDetails,
+    ErrorResponse,
+    TaskErrorDetail,
+)
+from elementary_runtime.protocol.http import Headers, Paths
+from elementary_runtime.protocol.instances import (
+    ConnectionReport,
+    ObservedWarehouseIdentity,
+    RegisterInstanceRequest,
+)
+from elementary_runtime.protocol.tasks import (
+    TASK_MODELS,
+    CloudTestSource,
+    CustomSqlTestResult,
+    CustomSqlTestResultPayload,
+    CustomSqlTestTask,
+    CustomSqlTestTaskPayload,
+    HeartbeatTasksRequest,
+    HeartbeatTasksResponse,
+    PollTasksRequest,
+    PollTasksResponse,
+    ResultSample,
+    RunSqlResult,
+    RunSqlResultPayload,
+    RunSqlTask,
+    RunSqlTaskPayload,
+    SampleColumnType,
+    SampleHeader,
+    SqlExecutionStatus,
+    SqlQueryResultPayload,
+    SqlQueryTaskPayload,
+    SubmitResultsRequest,
+    SubmitResultsResponse,
+    SubmittedResult,
+    Task,
+    TaskEnvelope,
+    TaskHeartbeat,
+    TaskHeartbeatStatus,
+    TaskModels,
+    TaskResult,
+    TaskResultEnvelope,
+    WarehouseTaskEnvelope,
+)
+from elementary_runtime.protocol.telemetry import (
+    ExceptionFrame,
+    RuntimeExceptionEvent,
+    RuntimeExceptionFields,
+    RuntimeStartedEvent,
+    RuntimeStartedFields,
+    RuntimeStoppedEvent,
+    RuntimeStoppedFields,
+    TaskExecutionSummaryEvent,
+    TaskExecutionSummaryFields,
+    TelemetryEvent,
+    TelemetryEventBase,
+    TelemetryRequest,
+    TelemetryResponse,
+)
+
+__all__ = [
+    "TASK_MODELS",
+    "CloudTestSource",
+    "ConnectionErrorCode",
+    "ConnectionHealth",
+    "ConnectionReport",
+    "CustomSqlTestResult",
+    "CustomSqlTestResultPayload",
+    "CustomSqlTestTask",
+    "CustomSqlTestTaskPayload",
+    "DisplayName",
+    "ErrorBody",
+    "ErrorCode",
+    "ErrorDetails",
+    "ErrorMessage",
+    "ErrorResponse",
+    "ExceptionComponent",
+    "ExceptionFrame",
+    "ExceptionOperation",
+    "Headers",
+    "HeartbeatStatus",
+    "HeartbeatTasksRequest",
+    "HeartbeatTasksResponse",
+    "IdStr",
+    "InstanceId",
+    "InstanceIdPrefix",
+    "NonEmptyStr",
+    "ObservedWarehouseIdentity",
+    "Paths",
+    "PollTasksRequest",
+    "PollTasksResponse",
+    "ProtocolEnum",
+    "ProtocolModel",
+    "RegisterInstanceRequest",
+    "ResultSample",
+    "RunSqlResult",
+    "RunSqlResultPayload",
+    "RunSqlTask",
+    "RunSqlTaskPayload",
+    "RuntimeExceptionEvent",
+    "RuntimeExceptionFields",
+    "RuntimeStartedEvent",
+    "RuntimeStartedFields",
+    "RuntimeStopReason",
+    "RuntimeStoppedEvent",
+    "RuntimeStoppedFields",
+    "SampleColumnType",
+    "SampleHeader",
+    "SqlExecutionStatus",
+    "SqlQueryResultPayload",
+    "SqlQueryTaskPayload",
+    "SubmitResultsRequest",
+    "SubmitResultsResponse",
+    "SubmittedResult",
+    "SubmittedResultStatus",
+    "Task",
+    "TaskEnvelope",
+    "TaskErrorDetail",
+    "TaskExecutionOutcome",
+    "TaskExecutionSummaryEvent",
+    "TaskExecutionSummaryFields",
+    "TaskHeartbeat",
+    "TaskHeartbeatStatus",
+    "TaskModels",
+    "TaskResult",
+    "TaskResultEnvelope",
+    "TaskType",
+    "TelemetryEvent",
+    "TelemetryEventBase",
+    "TelemetryEventType",
+    "TelemetryRequest",
+    "TelemetryResponse",
+    "TestSourceType",
+    "UtcDatetime",
+    "WarehouseTaskEnvelope",
+]