pyspiral 0.4.4__cp310-abi3-macosx_11_0_arm64.whl → 0.6.0__cp310-abi3-macosx_11_0_arm64.whl

spiral/protogen/substrait/extensions/__init__.py

@@ -1,115 +0,0 @@
-# Generated by the protocol buffer compiler.  DO NOT EDIT!
-# sources: substrait/extensions/extensions.proto
-# plugin: python-betterproto
-# This file has been @generated
-
-from dataclasses import dataclass
-from typing import List
-
-import betterproto
-import betterproto.lib.google.protobuf as betterproto_lib_google_protobuf
-
-
-@dataclass(eq=False, repr=False)
-class SimpleExtensionUri(betterproto.Message):
-    extension_uri_anchor: int = betterproto.uint32_field(1)
-    """
-    A surrogate key used in the context of a single plan used to reference the
-     URI associated with an extension.
-    """
-
-    uri: str = betterproto.string_field(2)
-    """
-    The URI where this extension YAML can be retrieved. This is the "namespace"
-     of this extension.
-    """
-
-
-@dataclass(eq=False, repr=False)
-class SimpleExtensionDeclaration(betterproto.Message):
-    """
-    Describes a mapping between a specific extension entity and the uri where
-     that extension can be found.
-    """
-
-    extension_type: "SimpleExtensionDeclarationExtensionType" = (
-        betterproto.message_field(1, group="mapping_type")
-    )
-    extension_type_variation: "SimpleExtensionDeclarationExtensionTypeVariation" = (
-        betterproto.message_field(2, group="mapping_type")
-    )
-    extension_function: "SimpleExtensionDeclarationExtensionFunction" = (
-        betterproto.message_field(3, group="mapping_type")
-    )
-
-
-@dataclass(eq=False, repr=False)
-class SimpleExtensionDeclarationExtensionType(betterproto.Message):
-    """Describes a Type"""
-
-    extension_uri_reference: int = betterproto.uint32_field(1)
-    """
-    references the extension_uri_anchor defined for a specific extension URI.
-    """
-
-    type_anchor: int = betterproto.uint32_field(2)
-    """
-    A surrogate key used in the context of a single plan to reference a
-     specific extension type
-    """
-
-    name: str = betterproto.string_field(3)
-    """the name of the type in the defined extension YAML."""
-
-
-@dataclass(eq=False, repr=False)
-class SimpleExtensionDeclarationExtensionTypeVariation(betterproto.Message):
-    extension_uri_reference: int = betterproto.uint32_field(1)
-    """
-    references the extension_uri_anchor defined for a specific extension URI.
-    """
-
-    type_variation_anchor: int = betterproto.uint32_field(2)
-    """
-    A surrogate key used in the context of a single plan to reference a
-     specific type variation
-    """
-
-    name: str = betterproto.string_field(3)
-    """the name of the type in the defined extension YAML."""
-
-
-@dataclass(eq=False, repr=False)
-class SimpleExtensionDeclarationExtensionFunction(betterproto.Message):
-    extension_uri_reference: int = betterproto.uint32_field(1)
-    """
-    references the extension_uri_anchor defined for a specific extension URI.
-    """
-
-    function_anchor: int = betterproto.uint32_field(2)
-    """
-    A surrogate key used in the context of a single plan to reference a
-     specific function
-    """
-
-    name: str = betterproto.string_field(3)
-    """A function signature compound name"""
-
-
-@dataclass(eq=False, repr=False)
-class AdvancedExtension(betterproto.Message):
-    """
-    A generic object that can be used to embed additional extension information
-     into the serialized substrait plan.
-    """
-
-    optimization: List[
-        "betterproto_lib_google_protobuf.Any"
-    ] = betterproto.message_field(1)
-    """
-    An optimization is helpful information that don't influence semantics. May
-     be ignored by a consumer.
-    """
-
-    enhancement: "betterproto_lib_google_protobuf.Any" = betterproto.message_field(2)
-    """An enhancement alter semantics. Cannot be ignored by a consumer."""

spiral/tables/__init__.py

@@ -1,12 +0,0 @@
-from spiral import _lib
-from spiral.tables.client import Tables
-from spiral.tables.maintenance import Maintenance
-from spiral.tables.scan import Scan
-from spiral.tables.snapshot import Snapshot
-from spiral.tables.table import Table
-from spiral.tables.transaction import Transaction
-
-# Eagerly import the Spiral library
-assert _lib, "Spiral library"
-
-__all__ = ["Tables", "Table", "Snapshot", "Scan", "Transaction", "Maintenance"]

spiral/tables/client.py

@@ -1,130 +0,0 @@
-from datetime import datetime
-from typing import Any
-
-import pyarrow as pa
-
-from spiral.api import SpiralAPI
-from spiral.api.projects import TableResource
-from spiral.core.client import Spiral as CoreSpiral
-from spiral.core.table.spec import Schema
-from spiral.datetime_ import timestamp_micros
-from spiral.expressions import ExprLike
-from spiral.tables.scan import Scan
-from spiral.tables.table import Table
-from spiral.types_ import Uri
-
-
-class Tables:
-    """
-    Spiral Tables a powerful and flexible way for storing, analyzing,
-    and querying massive and/or multimodal datasets.
-
-    The data model will feel familiar to users of SQL- or DataFrame-style systems,
-    yet is designed to be more flexible, more powerful, and more useful in the context
-    of modern data processing. Tables are stored and queried directly from object storage.
-    """
-
-    def __init__(self, api: SpiralAPI, spiral: CoreSpiral, *, project_id: str | None = None):
-        self._api = api
-        self._spiral = spiral
-        self._project_id = project_id
-
-    def table(self, identifier: str) -> Table:
-        """Open a table with a `dataset.table` identifier, or `table` name using the `default` dataset."""
-        project_id, dataset, table = self._parse_identifier(identifier)
-        if project_id is None:
-            raise ValueError("Must provide a fully qualified table identifier.")
-
-        res = list(self._api.project.list_tables(project_id, dataset=dataset, table=table))
-        if len(res) == 0:
-            raise ValueError(f"Table not found: {project_id}.{dataset}.{table}")
-
-        res = res[0]
-        return Table(self, self._spiral.get_table(res.id), identifier=f"{res.project_id}.{res.dataset}.{res.table}")
-
-    def list_tables(self) -> list[TableResource]:
-        project_id = self._project_id
-        if project_id is None:
-            raise ValueError("Must provide a project ID to list tables.")
-        return list(self._api.project.list_tables(project_id))
-
-    def create_table(
-        self,
-        identifier: str,
-        *,
-        key_schema: pa.Schema | Any,
-        root_uri: Uri | None = None,
-        exist_ok: bool = False,
-    ) -> Table:
-        """Create a new table in the project.
-
-        Args:
-            identifier: The table identifier, in the form `project.dataset.table`, `dataset.table` or `table`.
-            key_schema: The schema of the table's keys.
-            root_uri: The root URI for the table.
-            exist_ok: If True, do not raise an error if the table already exists.
-        """
-        project_id, dataset, table = self._parse_identifier(identifier)
-        if project_id is None:
-            raise ValueError("Must provide a fully qualified table identifier.")
-
-        if not isinstance(key_schema, pa.Schema):
-            key_schema = pa.schema(key_schema)
-        key_schema = Schema.from_arrow(key_schema)
-
-        core_table = self._spiral.create_table(
-            project_id,
-            dataset=dataset,
-            table=table,
-            key_schema=key_schema,
-            root_uri=root_uri,
-            exist_ok=exist_ok,
-        )
-
-        return Table(self, core_table, identifier=f"{project_id}.{dataset}.{table}")
-
-    def _parse_identifier(self, identifier: str) -> tuple[str | None, str, str]:
-        parts = identifier.split(".")
-        if len(parts) == 1:
-            return self._project_id, "default", parts[0]
-        elif len(parts) == 2:
-            return self._project_id, parts[0], parts[1]
-        elif len(parts) == 3:
-            return parts[0], parts[1], parts[2]
-        else:
-            raise ValueError(f"Invalid table identifier: {identifier}")
-
-    def scan(
-        self,
-        *projections: ExprLike,
-        where: ExprLike | None = None,
-        asof: datetime | int | None = None,
-        exclude_keys: bool = False,
-    ) -> Scan:
-        """Starts a read transaction on the Spiral.
-
-        Args:
-            projections: a set of expressions that return struct arrays.
-            where: a query expression to apply to the data.
-            asof: only data written before the given timestamp will be returned, caveats around compaction.
-            exclude_keys: whether to exclude the key columns in the scan result, defaults to False.
-                Note that if a projection includes a key column, it will be included in the result.
-        """
-        from spiral import expressions as se
-
-        if isinstance(asof, datetime):
-            asof = timestamp_micros(asof)
-
-        # Combine all projections into a single struct.
-        projection = se.merge(*projections)
-        if where is not None:
-            where = se.lift(where)
-
-        return Scan(
-            self._spiral.open_table_scan(
-                projection.__expr__,
-                filter=where.__expr__ if where else None,
-                asof=asof,
-                exclude_keys=exclude_keys,
-            ),
-        )

spiral/tables/maintenance.py

@@ -1,12 +0,0 @@
-from spiral.core.table import TableMaintenance
-
-
-class Maintenance:
-    """Spiral table maintenance."""
-
-    def __init__(self, maintenance: TableMaintenance):
-        self._maintenance = maintenance
-
-    def flush_wal(self):
-        """Flush the write-ahead log."""
-        self._maintenance.flush_wal()

spiral/tables/snapshot.py

@@ -1,78 +0,0 @@
-from typing import TYPE_CHECKING
-
-from spiral.core.table import TableSnapshot
-from spiral.expressions import ExprLike
-from spiral.tables.scan import Scan
-from spiral.types_ import Timestamp
-
-if TYPE_CHECKING:
-    import duckdb
-    import polars as pl
-    import pyarrow.dataset
-
-    from spiral.tables import Tables
-    from spiral.tables.table import Table
-
-
-class Snapshot:
-    """Spiral table snapshot.
-
-    A snapshot represents a point-in-time view of a table.
-    """
-
-    def __init__(self, tables: "Tables", snapshot: TableSnapshot):
-        self._tables = tables
-        self._snapshot = snapshot
-
-    @property
-    def asof(self) -> Timestamp:
-        """Returns the asof timestamp of the snapshot."""
-        return self._snapshot.asof
-
-    @property
-    def client(self) -> "Tables":
-        """Returns the client used by the snapshot."""
-        return self._tables
-
-    @property
-    def table(self) -> "Table":
-        """Returns the table associated with the snapshot."""
-        from spiral.tables.table import Table
-
-        return Table(self._tables, self._snapshot.table)
-
-    def to_dataset(self) -> "pyarrow.dataset.Dataset":
-        """Returns a PyArrow Dataset representing the table."""
-        from .dataset import TableDataset
-
-        return TableDataset(self)
-
-    def to_polars(self) -> "pl.LazyFrame":
-        """Returns a Polars LazyFrame for the Spiral table."""
-        import polars as pl
-
-        return pl.scan_pyarrow_dataset(self.to_dataset())
-
-    def to_duckdb(self) -> "duckdb.DuckDBPyRelation":
-        """Returns a DuckDB relation for the Spiral table."""
-        import duckdb
-
-        return duckdb.from_arrow(self.to_dataset())
-
-    def scan(
-        self,
-        *projections: ExprLike,
-        where: ExprLike | None = None,
-        exclude_keys: bool = False,
-    ) -> Scan:
-        """Reads the snapshot. If projections are not provided, the entire table is read."""
-        if not projections:
-            # Use table as the default projection.
-            projections = [self._snapshot.table.__expr__]
-
-        return self._tables.scan(
-            *projections,
-            where=where,
-            asof=self._snapshot.asof,
-            exclude_keys=exclude_keys,
-        )

spiral/tables/table.py

@@ -1,145 +0,0 @@
-from datetime import datetime
-from typing import TYPE_CHECKING
-
-from spiral.core.table import Table as CoreTable
-from spiral.core.table.spec import Schema
-from spiral.expressions.base import Expr, ExprLike
-from spiral.settings import settings
-from spiral.tables.maintenance import Maintenance
-from spiral.tables.scan import Scan
-from spiral.tables.snapshot import Snapshot
-from spiral.tables.transaction import Transaction
-
-if TYPE_CHECKING:
-    from spiral.tables import Tables
-
-
-class Table(Expr):
-    """API for interacting with a SpiralDB's Table.
-
-    Different catalog implementations should ultimately construct a Table object.
-    """
-
-    # TODO(marko): Make identifier required.
-    def __init__(self, tables: "Tables", table: CoreTable, *, identifier: str | None = None):
-        super().__init__(table.__expr__)
-
-        self._tables = tables
-        self._table = table
-        self._identifier = identifier
-        self._key_schema = self._table.key_schema
-        self._key_columns = set(self._key_schema.names)
-
-    @property
-    def client(self) -> "Tables":
-        """Returns the client used by the table."""
-        return self._tables
-
-    @property
-    def table_id(self) -> str:
-        return self._table.id
-
-    @property
-    def identifier(self) -> str:
-        """Returns the fully qualified identifier of the table."""
-        return self._identifier or self._table.id
-
-    @property
-    def dataset(self) -> str | None:
-        """Returns the dataset of the table."""
-        if self._identifier is None:
-            return None
-        _, dataset, _ = self._identifier.split(".")
-        return dataset
-
-    @property
-    def name(self) -> str | None:
-        """Returns the name of the table."""
-        if self._identifier is None:
-            return None
-        _, _, name = self._identifier.split(".")
-        return name
-
-    @property
-    def last_modified_at(self) -> int:
-        return self._table.get_wal(asof=None).last_modified_at
-
-    def __str__(self):
-        return self.identifier
-
-    def __repr__(self):
-        return f'Table("{self.identifier}")'
-
-    def __getitem__(self, item: str) -> Expr:
-        return super().__getitem__(item)
-
-    def select(self, *paths: str, exclude: list[str] = None) -> "Expr":
-        # Override an expression select in the root column group to split between keys and columns.
-        if exclude is not None:
-            if set(exclude) & self._key_columns:
-                raise ValueError(
-                    "Cannot use 'exclude' arg with key columns. Use 'exclude_keys' and an explicit select of keys."
-                )
-
-        return super().select(*paths, exclude=exclude)
-
-    @property
-    def key_schema(self) -> Schema:
-        """Returns the key schema of the table."""
-        return self._key_schema
-
-    @property
-    def schema(self) -> Schema:
-        """Returns the FULL schema of the table.
-
-        NOTE: This can be expensive for large tables.
-        """
-        return self._table.get_schema(asof=None)
-
-    def scan(
-        self,
-        *projections: ExprLike,
-        where: ExprLike | None = None,
-        asof: datetime | int | None = None,
-        exclude_keys: bool = False,
-    ) -> Scan:
-        """Reads the table. If projections are not provided, the entire table is read."""
-        if not projections:
-            projections = [self]
-
-        return self._tables.scan(*projections, where=where, asof=asof, exclude_keys=exclude_keys)
-
-    def write(
-        self,
-        expr: ExprLike,
-        *,
-        partition_size_bytes: int | None = None,
-    ) -> None:
-        """Write an item to the table inside a single transaction.
-
-        :param expr: The expression to write. Must evaluate to a struct array.
-        :param partition_size_bytes: The maximum partition size in bytes.
-        """
-        with self.txn() as txn:
-            txn.write(
-                expr,
-                partition_size_bytes=partition_size_bytes,
-            )
-
-    def snapshot(self, asof: datetime | int | None = None) -> Snapshot:
-        """Returns a snapshot of the table at the given timestamp."""
-        if isinstance(asof, datetime):
-            asof = int(asof.timestamp() * 1_000_000)
-        return Snapshot(self._tables, self._table.get_snapshot(asof=asof))
-
-    def txn(self) -> Transaction:
-        """Begins a new transaction. Transaction must be committed for writes to become visible.
-
-        IMPORTANT: While transaction can be used to atomically write data to the table,
-             it is important that the primary key columns are unique within the transaction.
-        """
-        return Transaction(self._tables._spiral.open_transaction(self._table, settings().file_format))
-
-    def maintenance(self) -> Maintenance:
-        """Access maintenance operations for a table."""
-        return Maintenance(self._tables._spiral.open_maintenance(self._table, settings().file_format))