pyspiral 0.1.0__cp310-abi3-macosx_11_0_arm64.whl

spiral/proto/_/spiraldb/metastore/__init__.py

@@ -0,0 +1,499 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# sources: spiraldb/metastore.proto
+# plugin: python-betterproto
+# This file has been @generated
+
+from dataclasses import dataclass
+from typing import (
+    TYPE_CHECKING,
+    Dict,
+    List,
+    Optional,
+)
+
+import betterproto
+import grpclib
+from betterproto.grpc.grpclib_server import ServiceBase
+
+from ...spiral import table as __spiral_table__
+
+
+if TYPE_CHECKING:
+    import grpclib.server
+    from betterproto.grpc.grpclib_client import MetadataLike
+    from grpclib.metadata import Deadline
+
+
+class FileType(betterproto.Enum):
+    UNSPECIFIED = 0
+    FRAGMENT = 1
+    FRAGMENT_MANIFEST = 2
+    REFERENCE = 3
+
+
+@dataclass(eq=False, repr=False)
+class FileHandle(betterproto.Message):
+    uri: str = betterproto.string_field(1)
+    format: "__spiral_table__.FileFormat" = betterproto.enum_field(2)
+    spfs_token: Optional[str] = betterproto.string_field(3, optional=True)
+
+
+@dataclass(eq=False, repr=False)
+class GetReadToken(betterproto.Message):
+    pass
+
+
+@dataclass(eq=False, repr=False)
+class GetReadTokenRequest(betterproto.Message):
+    table_id: str = betterproto.string_field(1)
+
+
+@dataclass(eq=False, repr=False)
+class GetReadTokenResponse(betterproto.Message):
+    spfs_token: str = betterproto.string_field(1)
+
+
+@dataclass(eq=False, repr=False)
+class SignFileWrite(betterproto.Message):
+    pass
+
+
+@dataclass(eq=False, repr=False)
+class SignFileWriteRequest(betterproto.Message):
+    table_id: str = betterproto.string_field(1)
+    file_type: "FileType" = betterproto.enum_field(2)
+    file_format: "__spiral_table__.FileFormat" = betterproto.enum_field(3)
+
+
+@dataclass(eq=False, repr=False)
+class SignFileWriteResponse(betterproto.Message):
+    id: str = betterproto.string_field(1)
+    handle: "FileHandle" = betterproto.message_field(2)
+
+
+@dataclass(eq=False, repr=False)
+class AppendWal(betterproto.Message):
+    pass
+
+
+@dataclass(eq=False, repr=False)
+class AppendWalRequest(betterproto.Message):
+    table_id: str = betterproto.string_field(1)
+    prev_last_modified_at: int = betterproto.uint64_field(2)
+    """
+    Timestamp of the last modification in the log.
+     For simple compare and swap or conflict resolution.
+    """
+
+    entries: List["__spiral_table__.LogEntry"] = betterproto.message_field(3)
+
+
+@dataclass(eq=False, repr=False)
+class AppendWalResponse(betterproto.Message):
+    committed: "__spiral_table__.WriteAheadLog" = betterproto.message_field(
+        1, group="status"
+    )
+    """New log (includes the committed operations)."""
+
+    conflicted: "__spiral_table__.WriteAheadLog" = betterproto.message_field(
+        2, group="status"
+    )
+    """In case of a conflict, the existing log is returned."""
+
+
+@dataclass(eq=False, repr=False)
+class UpdateWal(betterproto.Message):
+    """NOTE: WAL is updated only on flush."""
+
+    pass
+
+
+@dataclass(eq=False, repr=False)
+class UpdateWalRequest(betterproto.Message):
+    table_id: str = betterproto.string_field(1)
+    prev_last_modified_at: int = betterproto.uint64_field(2)
+    """
+    Only sequential update is supported.
+    
+     Truncation does not modify the last modified at so we require
+     both fields to be passed for compare and swap.
+    
+     The assumption with this API is that all WAL updates will modify
+     log entries and therefore change at least one of the timestamps.
+    """
+
+    prev_truncated_up_to: int = betterproto.uint64_field(3)
+    new_wal: "__spiral_table__.WriteAheadLog" = betterproto.message_field(4)
+
+
+@dataclass(eq=False, repr=False)
+class UpdateWalResponse(betterproto.Message):
+    committed: "__spiral_table__.WriteAheadLog" = betterproto.message_field(
+        1, group="status"
+    )
+    conflicted: "__spiral_table__.WriteAheadLog" = betterproto.message_field(
+        2, group="status"
+    )
+
+
+@dataclass(eq=False, repr=False)
+class GetWal(betterproto.Message):
+    pass
+
+
+@dataclass(eq=False, repr=False)
+class GetWalRequest(betterproto.Message):
+    table_id: str = betterproto.string_field(1)
+
+
+@dataclass(eq=False, repr=False)
+class GetWalResponse(betterproto.Message):
+    wal: "__spiral_table__.WriteAheadLog" = betterproto.message_field(1)
+
+
+@dataclass(eq=False, repr=False)
+class UpdateColumnGroupMetadata(betterproto.Message):
+    """NOTE: Metadata is updated only on flush."""
+
+    pass
+
+
+@dataclass(eq=False, repr=False)
+class UpdateColumnGroupMetadataRequest(betterproto.Message):
+    table_id: str = betterproto.string_field(1)
+    prev_last_modified_at: int = betterproto.uint64_field(3)
+    """Used to compare and swap."""
+
+    new_metadata: "__spiral_table__.ColumnGroupMetadata" = betterproto.message_field(4)
+
+
+@dataclass(eq=False, repr=False)
+class UpdateColumnGroupMetadataResponse(betterproto.Message):
+    committed: "__spiral_table__.ColumnGroupMetadata" = betterproto.message_field(
+        1, group="status"
+    )
+    conflicted: "__spiral_table__.ColumnGroupMetadata" = betterproto.message_field(
+        2, group="status"
+    )
+
+
+@dataclass(eq=False, repr=False)
+class GetColumnGroupMetadata(betterproto.Message):
+    pass
+
+
+@dataclass(eq=False, repr=False)
+class GetColumnGroupMetadataRequest(betterproto.Message):
+    table_id: str = betterproto.string_field(1)
+    column_group: "__spiral_table__.ColumnGroup" = betterproto.message_field(2)
+
+
+@dataclass(eq=False, repr=False)
+class GetColumnGroupMetadataResponse(betterproto.Message):
+    metadata: "__spiral_table__.ColumnGroupMetadata" = betterproto.message_field(1)
+
+
+@dataclass(eq=False, repr=False)
+class GetColumnGroups(betterproto.Message):
+    pass
+
+
+@dataclass(eq=False, repr=False)
+class GetColumnGroupsRequest(betterproto.Message):
+    table_id: str = betterproto.string_field(1)
+
+
+@dataclass(eq=False, repr=False)
+class GetColumnGroupsResponse(betterproto.Message):
+    column_groups: List["__spiral_table__.ColumnGroup"] = betterproto.message_field(1)
+    last_modified_at: int = betterproto.uint64_field(2)
+
+
+class MetastoreServiceStub(betterproto.ServiceStub):
+    async def get_read_token(
+        self,
+        get_read_token_request: "GetReadTokenRequest",
+        *,
+        timeout: Optional[float] = None,
+        deadline: Optional["Deadline"] = None,
+        metadata: Optional["MetadataLike"] = None
+    ) -> "GetReadTokenResponse":
+        return await self._unary_unary(
+            "/spiraldb.metastore.MetastoreService/GetReadToken",
+            get_read_token_request,
+            GetReadTokenResponse,
+            timeout=timeout,
+            deadline=deadline,
+            metadata=metadata,
+        )
+
+    async def sign_write_file(
+        self,
+        sign_file_write_request: "SignFileWriteRequest",
+        *,
+        timeout: Optional[float] = None,
+        deadline: Optional["Deadline"] = None,
+        metadata: Optional["MetadataLike"] = None
+    ) -> "SignFileWriteResponse":
+        return await self._unary_unary(
+            "/spiraldb.metastore.MetastoreService/SignWriteFile",
+            sign_file_write_request,
+            SignFileWriteResponse,
+            timeout=timeout,
+            deadline=deadline,
+            metadata=metadata,
+        )
+
+    async def append_wal(
+        self,
+        append_wal_request: "AppendWalRequest",
+        *,
+        timeout: Optional[float] = None,
+        deadline: Optional["Deadline"] = None,
+        metadata: Optional["MetadataLike"] = None
+    ) -> "AppendWalResponse":
+        return await self._unary_unary(
+            "/spiraldb.metastore.MetastoreService/AppendWAL",
+            append_wal_request,
+            AppendWalResponse,
+            timeout=timeout,
+            deadline=deadline,
+            metadata=metadata,
+        )
+
+    async def update_wal(
+        self,
+        update_wal_request: "UpdateWalRequest",
+        *,
+        timeout: Optional[float] = None,
+        deadline: Optional["Deadline"] = None,
+        metadata: Optional["MetadataLike"] = None
+    ) -> "UpdateWalResponse":
+        return await self._unary_unary(
+            "/spiraldb.metastore.MetastoreService/UpdateWAL",
+            update_wal_request,
+            UpdateWalResponse,
+            timeout=timeout,
+            deadline=deadline,
+            metadata=metadata,
+        )
+
+    async def get_wal(
+        self,
+        get_wal_request: "GetWalRequest",
+        *,
+        timeout: Optional[float] = None,
+        deadline: Optional["Deadline"] = None,
+        metadata: Optional["MetadataLike"] = None
+    ) -> "GetWalResponse":
+        return await self._unary_unary(
+            "/spiraldb.metastore.MetastoreService/GetWAL",
+            get_wal_request,
+            GetWalResponse,
+            timeout=timeout,
+            deadline=deadline,
+            metadata=metadata,
+        )
+
+    async def update_column_group_metadata(
+        self,
+        update_column_group_metadata_request: "UpdateColumnGroupMetadataRequest",
+        *,
+        timeout: Optional[float] = None,
+        deadline: Optional["Deadline"] = None,
+        metadata: Optional["MetadataLike"] = None
+    ) -> "UpdateColumnGroupMetadataResponse":
+        return await self._unary_unary(
+            "/spiraldb.metastore.MetastoreService/UpdateColumnGroupMetadata",
+            update_column_group_metadata_request,
+            UpdateColumnGroupMetadataResponse,
+            timeout=timeout,
+            deadline=deadline,
+            metadata=metadata,
+        )
+
+    async def get_column_group_metadata(
+        self,
+        get_column_group_metadata_request: "GetColumnGroupMetadataRequest",
+        *,
+        timeout: Optional[float] = None,
+        deadline: Optional["Deadline"] = None,
+        metadata: Optional["MetadataLike"] = None
+    ) -> "GetColumnGroupMetadataResponse":
+        return await self._unary_unary(
+            "/spiraldb.metastore.MetastoreService/GetColumnGroupMetadata",
+            get_column_group_metadata_request,
+            GetColumnGroupMetadataResponse,
+            timeout=timeout,
+            deadline=deadline,
+            metadata=metadata,
+        )
+
+    async def get_column_groups(
+        self,
+        get_column_groups_request: "GetColumnGroupsRequest",
+        *,
+        timeout: Optional[float] = None,
+        deadline: Optional["Deadline"] = None,
+        metadata: Optional["MetadataLike"] = None
+    ) -> "GetColumnGroupsResponse":
+        return await self._unary_unary(
+            "/spiraldb.metastore.MetastoreService/GetColumnGroups",
+            get_column_groups_request,
+            GetColumnGroupsResponse,
+            timeout=timeout,
+            deadline=deadline,
+            metadata=metadata,
+        )
+
+
+class MetastoreServiceBase(ServiceBase):
+    async def get_read_token(
+        self, get_read_token_request: "GetReadTokenRequest"
+    ) -> "GetReadTokenResponse":
+        raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED)
+
+    async def sign_write_file(
+        self, sign_file_write_request: "SignFileWriteRequest"
+    ) -> "SignFileWriteResponse":
+        raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED)
+
+    async def append_wal(
+        self, append_wal_request: "AppendWalRequest"
+    ) -> "AppendWalResponse":
+        raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED)
+
+    async def update_wal(
+        self, update_wal_request: "UpdateWalRequest"
+    ) -> "UpdateWalResponse":
+        raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED)
+
+    async def get_wal(self, get_wal_request: "GetWalRequest") -> "GetWalResponse":
+        raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED)
+
+    async def update_column_group_metadata(
+        self, update_column_group_metadata_request: "UpdateColumnGroupMetadataRequest"
+    ) -> "UpdateColumnGroupMetadataResponse":
+        raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED)
+
+    async def get_column_group_metadata(
+        self, get_column_group_metadata_request: "GetColumnGroupMetadataRequest"
+    ) -> "GetColumnGroupMetadataResponse":
+        raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED)
+
+    async def get_column_groups(
+        self, get_column_groups_request: "GetColumnGroupsRequest"
+    ) -> "GetColumnGroupsResponse":
+        raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED)
+
+    async def __rpc_get_read_token(
+        self, stream: "grpclib.server.Stream[GetReadTokenRequest, GetReadTokenResponse]"
+    ) -> None:
+        request = await stream.recv_message()
+        response = await self.get_read_token(request)
+        await stream.send_message(response)
+
+    async def __rpc_sign_write_file(
+        self,
+        stream: "grpclib.server.Stream[SignFileWriteRequest, SignFileWriteResponse]",
+    ) -> None:
+        request = await stream.recv_message()
+        response = await self.sign_write_file(request)
+        await stream.send_message(response)
+
+    async def __rpc_append_wal(
+        self, stream: "grpclib.server.Stream[AppendWalRequest, AppendWalResponse]"
+    ) -> None:
+        request = await stream.recv_message()
+        response = await self.append_wal(request)
+        await stream.send_message(response)
+
+    async def __rpc_update_wal(
+        self, stream: "grpclib.server.Stream[UpdateWalRequest, UpdateWalResponse]"
+    ) -> None:
+        request = await stream.recv_message()
+        response = await self.update_wal(request)
+        await stream.send_message(response)
+
+    async def __rpc_get_wal(
+        self, stream: "grpclib.server.Stream[GetWalRequest, GetWalResponse]"
+    ) -> None:
+        request = await stream.recv_message()
+        response = await self.get_wal(request)
+        await stream.send_message(response)
+
+    async def __rpc_update_column_group_metadata(
+        self,
+        stream: "grpclib.server.Stream[UpdateColumnGroupMetadataRequest, UpdateColumnGroupMetadataResponse]",
+    ) -> None:
+        request = await stream.recv_message()
+        response = await self.update_column_group_metadata(request)
+        await stream.send_message(response)
+
+    async def __rpc_get_column_group_metadata(
+        self,
+        stream: "grpclib.server.Stream[GetColumnGroupMetadataRequest, GetColumnGroupMetadataResponse]",
+    ) -> None:
+        request = await stream.recv_message()
+        response = await self.get_column_group_metadata(request)
+        await stream.send_message(response)
+
+    async def __rpc_get_column_groups(
+        self,
+        stream: "grpclib.server.Stream[GetColumnGroupsRequest, GetColumnGroupsResponse]",
+    ) -> None:
+        request = await stream.recv_message()
+        response = await self.get_column_groups(request)
+        await stream.send_message(response)
+
+    def __mapping__(self) -> Dict[str, grpclib.const.Handler]:
+        return {
+            "/spiraldb.metastore.MetastoreService/GetReadToken": grpclib.const.Handler(
+                self.__rpc_get_read_token,
+                grpclib.const.Cardinality.UNARY_UNARY,
+                GetReadTokenRequest,
+                GetReadTokenResponse,
+            ),
+            "/spiraldb.metastore.MetastoreService/SignWriteFile": grpclib.const.Handler(
+                self.__rpc_sign_write_file,
+                grpclib.const.Cardinality.UNARY_UNARY,
+                SignFileWriteRequest,
+                SignFileWriteResponse,
+            ),
+            "/spiraldb.metastore.MetastoreService/AppendWAL": grpclib.const.Handler(
+                self.__rpc_append_wal,
+                grpclib.const.Cardinality.UNARY_UNARY,
+                AppendWalRequest,
+                AppendWalResponse,
+            ),
+            "/spiraldb.metastore.MetastoreService/UpdateWAL": grpclib.const.Handler(
+                self.__rpc_update_wal,
+                grpclib.const.Cardinality.UNARY_UNARY,
+                UpdateWalRequest,
+                UpdateWalResponse,
+            ),
+            "/spiraldb.metastore.MetastoreService/GetWAL": grpclib.const.Handler(
+                self.__rpc_get_wal,
+                grpclib.const.Cardinality.UNARY_UNARY,
+                GetWalRequest,
+                GetWalResponse,
+            ),
+            "/spiraldb.metastore.MetastoreService/UpdateColumnGroupMetadata": grpclib.const.Handler(
+                self.__rpc_update_column_group_metadata,
+                grpclib.const.Cardinality.UNARY_UNARY,
+                UpdateColumnGroupMetadataRequest,
+                UpdateColumnGroupMetadataResponse,
+            ),
+            "/spiraldb.metastore.MetastoreService/GetColumnGroupMetadata": grpclib.const.Handler(
+                self.__rpc_get_column_group_metadata,
+                grpclib.const.Cardinality.UNARY_UNARY,
+                GetColumnGroupMetadataRequest,
+                GetColumnGroupMetadataResponse,
+            ),
+            "/spiraldb.metastore.MetastoreService/GetColumnGroups": grpclib.const.Handler(
+                self.__rpc_get_column_groups,
+                grpclib.const.Cardinality.UNARY_UNARY,
+                GetColumnGroupsRequest,
+                GetColumnGroupsResponse,
+            ),
+        }

spiral/proto/scandal/__init__.py

@@ -0,0 +1,45 @@
+from spiral.proto._ import scandal as scandal_
+
+from spiral.proto._.scandal import (
+    Connectivity,
+    Delete,
+    DeleteRequest,
+    DeleteResponse,
+    Fetch,
+    FetchRequest,
+    FetchResponse,
+    Metadata,
+    MetadataParquet,
+    Put,
+    PutRequest,
+    PutResponse,
+    ScandalServiceStub,
+    ScandalServiceBase,
+    ServiceBase,
+    Sink,
+    Source,
+)
+
+__all__ = [
+    "Connectivity",
+    "Delete",
+    "DeleteRequest",
+    "DeleteResponse",
+    "Fetch",
+    "FetchRequest",
+    "FetchResponse",
+    "Metadata",
+    "MetadataParquet",
+    "Put",
+    "PutRequest",
+    "PutResponse",
+    "ScandalServiceStub",
+    "ScandalServiceBase",
+    "ServiceBase",
+    "Sink",
+    "Source",
+]
+
+from spiral.proto.util import patch_protos
+
+patch_protos(scandal_, globals())

spiral/proto/spiral/table/__init__.py

@@ -0,0 +1,96 @@
+import betterproto
+import pyarrow as pa
+
+from spiral.proto._.spiral import table as table_
+from spiral.proto._.spiral.table import (
+    ColumnGroup,
+    ConfigurationOp,
+    FileFormat,
+    KeyExtent,
+    KeySpan,
+    Level,
+    ManifestHandle,
+    Schema,
+    SchemaBreakOp,
+    SchemaEvolutionOp,
+    VersionedSchema,
+    WriteOp,
+)
+from spiral.proto._.spiral.table import (
+    ColumnGroupMetadata as ColumnGroupMetadata_,
+)
+from spiral.proto._.spiral.table import (
+    Entry as Entry_,
+)
+from spiral.proto._.spiral.table import (
+    WriteAheadLog as WriteAheadLog_,
+)
+
+__all__ = [
+    "ColumnGroup",
+    "ColumnGroupMetadata",
+    "ConfigurationOp",
+    "Entry",
+    "FileFormat",
+    "KeyExtent",
+    "KeySpan",
+    "Level",
+    "ManifestHandle",
+    "Schema",
+    "SchemaBreakOp",
+    "SchemaEvolutionOp",
+    "VersionedSchema",
+    "WriteAheadLog",
+    "WriteOp",
+]
+
+
+class ColumnGroupMetadata(ColumnGroupMetadata_):
+    @property
+    def latest_schema(self) -> pa.Schema:
+        if self.schema_versions:
+            return pa.ipc.read_schema(pa.py_buffer(self.schema_versions[-1].schema.arrow))
+        return pa.schema([])
+
+    def asof(self, asof: int | None) -> "ColumnGroupMetadata":
+        """Return the metadata as of the given timestamp."""
+        if asof is None:
+            return self.model_copy()
+
+        versions = [v for v in self.schema_versions if v.ts <= asof]
+        return self.model_copy(update=dict(schema_versions=versions))
+
+
+class Entry(Entry_):
+    @property
+    def column_group(self) -> tuple[str, ...] | None:
+        match betterproto.which_one_of(self, "operation"):
+            case _, op:
+                return tuple(op.column_group.parts)
+
+
+class WriteAheadLog(WriteAheadLog_):
+    # TODO(ngates): add a single `filter` function that takes these as kwargs
+
+    def asof(self, asof: int | None) -> "WriteAheadLog":
+        """Return the WAL with all entries after the given timestamp removed."""
+        if asof is None:
+            return self.model_copy()
+        return self.model_copy(update=dict(logs=[entry for entry in self.logs if entry.ts <= asof]))
+
+    def since(self, since: int | None) -> "WriteAheadLog":
+        """Return the WAL with all entries before the given timestamp removed."""
+        if since is None:
+            return self.model_copy()
+        return self.model_copy(update=dict(logs=[entry for entry in self.logs if entry.ts > since]))
+
+    def for_column_group(self, column_group_path: tuple[str, ...]) -> "WriteAheadLog":
+        """Return the WAL with all entries for the given column group."""
+        return self.model_copy(
+            update=dict(logs=[entry for entry in self.logs if tuple(entry.op.column_group.parts) == column_group_path])
+        )
+
+
+from spiral.proto.util import patch_protos
+
+patch_protos(table_, globals())