pyspiral 0.6.9__cp312-abi3-macosx_11_0_arm64.whl → 0.7.12__cp312-abi3-macosx_11_0_arm64.whl

spiral/cli/telemetry.py

@@ -8,7 +8,7 @@ app = AsyncTyper(short_help="Client-side telemetry.")
 
 @app.command(help="Issue new telemetry export token.")
 def export():
-    res: IssueExportTokenResponse = state.settings.api.telemetry.issue_export_token()
+    res: IssueExportTokenResponse = state.spiral.api.telemetry.issue_export_token()
 
     command = f"export SPIRAL_OTEL_TOKEN={res.token}"
     pyperclip.copy(command)

spiral/cli/types.py

@@ -10,7 +10,7 @@ from spiral.cli import ERR_CONSOLE, state
 
 
 def ask_project(title="Select a project"):
-    projects = list(state.settings.api.project.list())
+    projects = list(state.spiral.api.project.list())
 
     if not projects:
         ERR_CONSOLE.print("No projects found")
@@ -29,7 +29,7 @@ ProjectArg = Annotated[ProjectId, Argument(help="Project ID", show_default=False
 
 
 def _org_default():
-    memberships = list(state.settings.api.organization.list_memberships())
+    memberships = list(state.spiral.api.organization.list_memberships())
 
     if not memberships:
         ERR_CONSOLE.print("No organizations found")

spiral/cli/workloads.py

@@ -17,7 +17,7 @@ def create(
     project: ProjectArg,
     name: Annotated[str | None, Option(help="Friendly name for the workload.")] = None,
 ):
-    res = state.settings.api.workload.create(project, CreateWorkloadRequest(name=name))
+    res = state.spiral.api.workload.create(project, CreateWorkloadRequest(name=name))
     CONSOLE.print(f"Created workload {res.workload.id}")
 
 
@@ -25,13 +25,13 @@ def create(
 def ls(
     project: ProjectArg,
 ):
-    workloads = list(state.settings.api.workload.list(project))
+    workloads = list(state.spiral.api.workload.list(project))
     CONSOLE.print(printer.table_of_models(Workload, workloads, fields=["id", "project_id", "name"]))
 
 
 @app.command(help="Issue new workflow credentials.")
 def issue_credentials(workload_id: Annotated[str, Argument(help="Workload ID.")]):
-    res: IssueWorkloadCredentialsResponse = state.settings.api.workload.issue_credentials(workload_id)
+    res: IssueWorkloadCredentialsResponse = state.spiral.api.workload.issue_credentials(workload_id)
 
     while True:
         choice = questionary.select(

spiral/client.py

@@ -6,12 +6,13 @@ import pyarrow as pa
 
 from spiral.api import SpiralAPI
 from spiral.api.projects import CreateProjectRequest, CreateProjectResponse
-from spiral.core.client import Operations
+from spiral.core.authn import Authn
+from spiral.core.client import Internal, KeyColumns
 from spiral.core.client import Spiral as CoreSpiral
+from spiral.core.config import ClientSettings
 from spiral.datetime_ import timestamp_micros
 from spiral.expressions import ExprLike
 from spiral.scan import Scan
-from spiral.settings import Settings, settings
 
 if TYPE_CHECKING:
     from spiral.iceberg import Iceberg
@@ -22,26 +23,74 @@ if TYPE_CHECKING:
 
 
 class Spiral:
-    def __init__(self, config: Settings | None = None):
-        self._config = config or settings()
+    """Main client for interacting with the Spiral data platform.
+
+    Configuration is loaded with the following priority (highest to lowest):
+    1. Explicit parameters.
+    2. Environment variables (`SPIRAL__*`)
+    3. Config file (`~/.spiral.toml`)
+    4. Default values (production URLs)
+
+    Examples:
+    ```python
+    # Default configuration
+    client = Spiral()
+
+    # With config overrides
+    client = Spiral(overrides={"limits.concurrency": "16"})
+    ```
+
+    Args:
+        config: Custom ClientSettings object. Defaults to global settings.
+        overrides: Configuration overrides using dot notation,
+            see the [Client Configuration](/python-client.md) page for a full list.
+    """
+
+    def __init__(
+        self,
+        config: ClientSettings | None = None,
+        overrides: dict[str, str] | None = None,
+    ):
+        self._overrides = overrides
+        self._config = config
         self._org = None
+        self._core = None
+        self._api = None
+
+    @property
+    def config(self) -> ClientSettings:
+        """Returns the client's configuration"""
+        return self.core.config()
 
     @property
-    def config(self) -> Settings:
-        return self._config
+    def authn(self) -> Authn:
+        """Get the authentication handler for this client."""
+        return self.core.authn()
 
     @property
     def api(self) -> SpiralAPI:
-        return self._config.api
+        if self._api is None:
+            self._api = SpiralAPI(self.authn, base_url=self.config.server_url)
+        return self._api
 
     @property
-    def _core(self) -> CoreSpiral:
-        return self._config.core
+    def core(self) -> CoreSpiral:
+        if self._core is None:
+            self._core = CoreSpiral(
+                config=self._config,
+                overrides=self._overrides,
+            )
+
+        return self._core
+
+    @property
+    def internal(self) -> Internal:
+        return self.core.internal(format=self.config.file_format)
 
     @property
     def organization(self) -> str:
         if self._org is None:
-            token = self._config.authn.token()
+            token = self.authn.token()
             if token is None:
                 raise ValueError("Authentication failed.")
             token_payload = jwt.decode(token.expose_secret(), options={"verify_signature": False})
@@ -79,25 +128,26 @@ class Spiral:
         """Open a table using an ID."""
         from spiral.table import Table
 
-        return Table(self, self._core.table(table_id))
+        return Table(self, self.core.table(table_id))
 
     def text_index(self, index_id: str) -> "TextIndex":
         """Open a text index using an ID."""
         from spiral.text_index import TextIndex
 
-        return TextIndex(self._core.text_index(index_id))
+        return TextIndex(self.core.text_index(index_id))
 
     def key_space_index(self, index_id: str) -> "KeySpaceIndex":
         """Open a key space index using an ID."""
         from spiral.key_space_index import KeySpaceIndex
 
-        return KeySpaceIndex(self._core.key_space_index(index_id))
+        return KeySpaceIndex(self.core.key_space_index(index_id))
 
     def scan(
         self,
         *projections: ExprLike,
         where: ExprLike | None = None,
         asof: datetime | int | None = None,
+        _key_columns: KeyColumns | None = None,
     ) -> Scan:
         """Starts a read transaction on the Spiral.
 
@@ -112,15 +162,16 @@ class Spiral:
             asof = timestamp_micros(asof)
 
         # Combine all projections into a single struct.
+        if not projections:
+            raise ValueError("At least one projection is required.")
         projection = se.merge(*projections)
         if where is not None:
             where = se.lift(where)
 
         return Scan(
-            self._core.scan(
-                projection.__expr__,
-                filter=where.__expr__ if where else None,
-                asof=asof,
+            self,
+            self.core.scan(
+                projection.__expr__, filter=where.__expr__ if where else None, asof=asof, key_columns=_key_columns
             ),
         )
 
@@ -155,17 +206,13 @@ class Spiral:
             freshness_window = timedelta(seconds=0)
         freshness_window_s = int(freshness_window.total_seconds())
 
-        return self._core.search(
+        return self.core.search(
             top_k=top_k,
             rank_by=rank_by.__expr__,
             filters=filters.__expr__ if filters else None,
             freshness_window_s=freshness_window_s,
         )
 
-    def _ops(self) -> Operations:
-        """Access maintenance operations."""
-        return self._core._ops(format=settings().file_format)
-
     @property
     def iceberg(self) -> "Iceberg":
         """

spiral/core/client/__init__.pyi

@@ -1,22 +1,36 @@
+from enum import Enum
 from typing import Any, Literal
 
 import pyarrow as pa
-from spiral.api.types import DatasetName, IndexName, ProjectId, RootUri, TableName
+from spiral.api.types import DatasetName, IndexName, ProjectId, RootUri, TableId, TableName
 from spiral.core.authn import Authn
-from spiral.core.table import ColumnGroupState, KeyRange, KeySpaceState, Scan, Snapshot, Table, Transaction
+from spiral.core.config import ClientSettings
+from spiral.core.table import ColumnGroupState, KeyRange, KeySpaceState, Scan, ScanState, Snapshot, Table, Transaction
 from spiral.core.table.spec import ColumnGroup, Schema
 from spiral.expressions import Expr
 
+# Only for typing, the actual definition is in Rust.
+class KeyColumns(Enum):
+    IfProjected = 0
+    Included = 1
+    Only = 2
+
 class Spiral:
     """A client for Spiral database"""
     def __init__(
         self,
-        api_url: str | None = None,
-        spfs_url: str | None = None,
-        authn: Authn | None = None,
+        config: ClientSettings | None = None,
+        overrides: dict[str, str] | None = None,
     ):
-        """Initialize the Spiral client."""
+        """Initialize the Spiral client.
+
+        Args:
+            config: Client configuration, defaults to the global config.
+            overrides: Configuration overrides using dot notation,
+                see the [Client Configuration](/python-client) page for a full list.
+        """
         ...
+
     def authn(self) -> Authn:
         """Get the current authentication context."""
         ...
@@ -26,10 +40,15 @@ class Spiral:
         projection: Expr,
         filter: Expr | None = None,
         asof: int | None = None,
+        key_columns: KeyColumns | None = None,
     ) -> Scan:
         """Construct a table scan."""
         ...
 
+    def load_scan(self, plan_state: ScanState) -> Scan:
+        """Load a scan from a serialized scan state."""
+        ...
+
     def transaction(self, table: Table, format: str | None = None, retries: int | None = 3) -> Transaction:
         """Being a table transaction."""
         ...
@@ -65,6 +84,22 @@ class Spiral:
         """Create a new table in the specified project."""
         ...
 
+    def move_table(
+        self,
+        table_id: TableId,
+        new_dataset: DatasetName,
+    ):
+        """Move a table to a dataset in the same project."""
+        ...
+
+    def rename_table(
+        self,
+        table_id: TableId,
+        new_table: TableName,
+    ):
+        """Rename a table."""
+        ...
+
     def text_index(self, index_id: str) -> TextIndex:
         """Get a text index."""
         ...
@@ -100,12 +135,12 @@ class Spiral:
         """Create a new key space index in the specified project."""
         ...
 
-    def _ops(self, *, format: str | None = None) -> Operations:
-        """Access maintenance operations.
+    def internal(self, *, format: str | None = None) -> Internal:
+        """Internal client APIs. It can change without notice."""
+        ...
 
-        IMPORTANT: This API is internal and is currently exposed for development & testing.
-            Maintenance operations are run by SpiralDB.
-        """
+    def config(self) -> ClientSettings:
+        """Client-side configuration."""
         ...
 
 class TextIndex:
@@ -158,8 +193,8 @@ class ShuffleConfig:
         max_batch_size: int | None = None,
     ): ...
 
-class Operations:
-    def flush_wal(self, table: Table, *, keep_latest_s: int | None = None) -> None:
+class Internal:
+    def flush_wal(self, table: Table) -> None:
         """
         Flush the write-ahead log of the table.
         """

spiral/core/config/__init__.pyi

@@ -0,0 +1,47 @@
+from spiral.core.authn import Token
+
+class ClientSettings:
+    """Client configuration loaded from ~/.spiral.toml and environment variables."""
+
+    @staticmethod
+    def load() -> ClientSettings:
+        """Load ClientSettings from ~/.spiral.toml and environment variables.
+
+        Configuration priority (highest to lowest):
+        1. Environment variables (SPIRAL__*)
+        2. Config file (~/.spiral.toml)
+        3. Default values
+        """
+        ...
+
+    @property
+    def server_url(self) -> str:
+        """The Spiral API endpoint URL."""
+        ...
+
+    @property
+    def spfs_url(self) -> str:
+        """The SpFS endpoint URL."""
+        ...
+
+    @property
+    def file_format(self) -> str:
+        """File format for table storage (vortex or parquet)."""
+        ...
+
+    @property
+    def token(self) -> Token | None:
+        """Authentication token (if provided via SPIRAL__TOKEN)."""
+        ...
+
+    @token.setter
+    def token(self, token: Token):
+        """Set the config's authentication token"""
+        ...
+
+    def to_json(self) -> str:
+        """Serialize to a JSON string"""
+        ...
+    @staticmethod
+    def from_json(json: str) -> ClientSettings:
+        """Deserialize from a JSON-formatted string"""

spiral/core/expr/__init__.pyi

@@ -0,0 +1,15 @@
+from pyarrow import Array, DataType, Scalar
+
+class Expr:
+    """Low level expression class."""
+
+def aux(name: str, data_type: DataType) -> Expr: ...
+
+# Array is correct (there is no ArrayData), see the table here:
+# https://arrow.apache.org/rust/arrow_pyarrow/index.html
+def scalar(array: Array[Scalar[DataType]]) -> Expr: ...
+def not_(expr: Expr) -> Expr: ...
+def is_null(expr: Expr) -> Expr: ...
+def binary(op: str, expr: Expr, Expr: Expr) -> Expr: ...
+def cast(_expr: Expr, _data_type: DataType) -> Expr: ...
+def array_lit(array: Array[Scalar[DataType]]) -> Expr: ...

spiral/core/expr/images/__init__.pyi

@@ -0,0 +1,3 @@
+from .. import Expr
+
+def encode_(images: Expr, format: str) -> Expr: ...

spiral/core/expr/list_/__init__.pyi

@@ -0,0 +1,4 @@
+from .. import Expr
+
+def contains(list: Expr, expr: Expr) -> Expr: ...
+def element_at(list: Expr, element: Expr) -> Expr: ...

spiral/core/expr/refs/__init__.pyi

@@ -0,0 +1,4 @@
+from .. import Expr
+
+def ref(expr: Expr, field: str | None) -> Expr: ...
+def deref(expr: Expr, field: str | None) -> Expr: ...

spiral/core/expr/str_/__init__.pyi

@@ -0,0 +1,3 @@
+from .. import Expr
+
+def substr(expr: Expr, begin: int, end: int | None) -> Expr: ...

spiral/core/expr/struct_/__init__.pyi

@@ -0,0 +1,6 @@
+from .. import Expr
+
+def getitem(expr: Expr, item: str) -> Expr: ...
+def select(expr: Expr, including: list[str] | None = None, excluding: list[str] | None = None) -> Expr: ...
+def pack(names: list[str], children: list[str], nullable: bool) -> Expr: ...
+def merge(names: list[Expr]) -> Expr: ...

spiral/core/expr/text/__init__.pyi

@@ -0,0 +1,5 @@
+from .. import Expr
+
+def field(expr: Expr, tokeneizer: str | None) -> Expr: ...
+def find(expr: Expr, term: str) -> Expr: ...
+def boost(expr: Expr, factor: float) -> Expr: ...

spiral/core/expr/udf/__init__.pyi

@@ -0,0 +1,14 @@
+from collections.abc import Callable
+
+from pyarrow import Array, DataType, Scalar
+
+from .. import Expr
+
+class UDF:
+    def __call__(self, args: list[Expr]) -> Expr: ...
+
+def create(
+    name: str,
+    return_type: Callable[[tuple[DataType, ...]], DataType],
+    invoke: Callable[[tuple[Array[Scalar[DataType]], ...]], Array[Scalar[DataType]]],
+) -> UDF: ...

spiral/core/expr/video/__init__.pyi

@@ -0,0 +1,3 @@
+from .. import Expr
+
+def read(expr: Expr, ranges: Expr, crops: Expr, format: str) -> Expr: ...

spiral/core/table/__init__.pyi

@@ -5,7 +5,7 @@ from spiral.core.client import Shard, ShuffleConfig
 
 from .manifests import FragmentManifest
 from .metastore import PyMetastore
-from .spec import ColumnGroup, Key, Schema, WriteAheadLog
+from .spec import ColumnGroup, Key, Operation, Schema, WriteAheadLog
 
 class KeyRange:
     """A right-exclusive range of keys."""
@@ -52,6 +52,21 @@ class Snapshot:
     table: Table
     wal: WriteAheadLog
 
+class ScanState:
+    def to_json(self) -> str: ...
+    @staticmethod
+    def from_json(json: str) -> ScanState: ...
+
+class MaterializablePlan:
+    pass
+
+class EvaluatedExecutablePlan:
+    pass
+
+class EvaluatedPlanStream:
+    def __next__(self) -> EvaluatedExecutablePlan: ...
+    def __iter__(self) -> EvaluatedPlanStream: ...
+
 class Scan:
     def key_schema(self) -> Schema: ...
     def schema(self) -> Schema: ...
@@ -62,10 +77,14 @@ class Scan:
     def column_groups(self) -> list[ColumnGroup]: ...
     def column_group_state(self, column_group: ColumnGroup) -> ColumnGroupState: ...
     def key_space_state(self, table_id: str) -> KeySpaceState: ...
+    def plan_state(self) -> ScanState: ...
+    def materializable_plan(self) -> MaterializablePlan: ...
     def to_record_batches(
         self,
+        key_range: KeyRange | None = None,
         key_table: pa.Table | pa.RecordBatch | None = None,
         batch_readahead: int | None = None,
+        progress: bool = True,
     ) -> pa.RecordBatchReader: ...
     def to_shuffled_record_batches(
         self,
@@ -78,6 +97,10 @@ class Scan:
         # If `infinite` is True, shards are shuffled after exhausted but not before the first pass.
         # Otherwise, shards are not shuffle and shuffle config is only used for shuffle buffer.
         ...
+
+    def evaluate_analyze(
+        self, key_table: pa.Table | pa.RecordBatch | None = None, batch_readahead: int | None = None
+    ) -> EvaluatedPlanStream: ...
     def metrics(self) -> dict[str, Any]: ...
 
 class KeySpaceState:
@@ -95,7 +118,19 @@ class Transaction:
     status: str
 
     def write(self, table: pa.RecordBatchReader, *, partition_size_bytes: int | None = None): ...
+    def writeback(
+        self,
+        scan: Scan,
+        *,
+        key_range: KeyRange | None = None,
+        partition_size_bytes: int | None = None,
+        batch_readahead: int | None = None,
+    ): ...
     def drop_columns(self, column_paths: list[str]): ...
-    def commit(self): ...
+    def ops(self) -> list[Operation]: ...
+    def take(self) -> list[Operation]: ...
+    def include(self, ops: list[Operation]): ...
+    def commit(self, *, compact: bool = False): ...
     def abort(self): ...
+    def is_empty(self) -> bool: ...
     def metrics(self) -> dict[str, Any]: ...

spiral/core/table/spec/__init__.pyi

@@ -62,6 +62,12 @@ class ColumnGroupMetadata:
     def apply_wal(self, wal: WriteAheadLog) -> ColumnGroupMetadata:
         """Applies the given WAL to the metadata."""
 
+class Operation:
+    # Base class for all operations in the WAL.
+    def to_json(self) -> str: ...
+    @staticmethod
+    def from_json(json: str) -> Operation: ...
+
 class LogEntry:
     ts: int
     operation: (
@@ -172,11 +178,7 @@ class KeySpaceWriteOp:
 
 class ColumnGroupWriteOp:
     column_group: ColumnGroup
-    level: FragmentLevel
     manifest_handle: ManifestHandle
-    key_span: KeySpan
-    key_extent: KeyExtent
-    column_ids: list[str]
 
 class SchemaEvolutionOp:
     column_group: ColumnGroup