pyspiral 0.1.0__cp310-abi3-macosx_11_0_arm64.whl

spiral/proto/_/scandal/__init__.py

@@ -0,0 +1,223 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# sources: scandal/scandal.proto
+# plugin: python-betterproto
+# This file has been @generated
+
+from dataclasses import dataclass
+from typing import (
+    TYPE_CHECKING,
+    Dict,
+    List,
+    Optional,
+)
+
+import betterproto
+import grpclib
+from betterproto.grpc.grpclib_server import ServiceBase
+
+
+if TYPE_CHECKING:
+    import grpclib.server
+    from betterproto.grpc.grpclib_client import MetadataLike
+    from grpclib.metadata import Deadline
+
+
+@dataclass(eq=False, repr=False)
+class Source(betterproto.Message):
+    url: str = betterproto.string_field(1)
+    content_type: Optional[str] = betterproto.string_field(2, optional=True)
+    size: Optional[int] = betterproto.int64_field(3, optional=True)
+    parquet: "MetadataParquet" = betterproto.message_field(10, group="metadata")
+
+
+@dataclass(eq=False, repr=False)
+class Fetch(betterproto.Message):
+    """Let's make "fetch" happen."""
+
+    pass
+
+
+@dataclass(eq=False, repr=False)
+class FetchRequest(betterproto.Message):
+    uri: str = betterproto.string_field(1)
+    """
+    A signed request to read an spfs://<fsid>/path?token=<jwt> URI.
+      * Declares the MIME types the client can read directly.
+      * Declares whether the client has connectivity to the FileSystem.
+    """
+
+    connectivity: "Connectivity" = betterproto.message_field(2)
+    accepts: List[str] = betterproto.string_field(3)
+
+
+@dataclass(eq=False, repr=False)
+class FetchResponse(betterproto.Message):
+    sources: List["Source"] = betterproto.message_field(1)
+    """
+    The sources the client should use to attempt to read the file.
+     The client should try each source in order until it finds one that works.
+    """
+
+
+@dataclass(eq=False, repr=False)
+class Sink(betterproto.Message):
+    url: str = betterproto.string_field(1)
+
+
+@dataclass(eq=False, repr=False)
+class Put(betterproto.Message):
+    pass
+
+
+@dataclass(eq=False, repr=False)
+class PutRequest(betterproto.Message):
+    uri: str = betterproto.string_field(1)
+    connectivity: "Connectivity" = betterproto.message_field(2)
+
+
+@dataclass(eq=False, repr=False)
+class PutResponse(betterproto.Message):
+    sinks: List["Sink"] = betterproto.message_field(1)
+
+
+@dataclass(eq=False, repr=False)
+class Delete(betterproto.Message):
+    pass
+
+
+@dataclass(eq=False, repr=False)
+class DeleteRequest(betterproto.Message):
+    uri: str = betterproto.string_field(1)
+
+
+@dataclass(eq=False, repr=False)
+class DeleteResponse(betterproto.Message):
+    url: str = betterproto.string_field(1)
+    """Returns signed URL to delete the resource."""
+
+
+@dataclass(eq=False, repr=False)
+class Connectivity(betterproto.Message):
+    """
+    Information about the client's perceived connectivity to a FileSystem.
+    """
+
+    unreachable: bool = betterproto.bool_field(1)
+    round_trip_time_us: Optional[int] = betterproto.int32_field(2, optional=True)
+
+
+@dataclass(eq=False, repr=False)
+class Metadata(betterproto.Message):
+    pass
+
+
+@dataclass(eq=False, repr=False)
+class MetadataParquet(betterproto.Message):
+    pass
+
+
+class ScandalServiceStub(betterproto.ServiceStub):
+    async def fetch(
+        self,
+        fetch_request: "FetchRequest",
+        *,
+        timeout: Optional[float] = None,
+        deadline: Optional["Deadline"] = None,
+        metadata: Optional["MetadataLike"] = None
+    ) -> "FetchResponse":
+        return await self._unary_unary(
+            "/scandal.ScandalService/Fetch",
+            fetch_request,
+            FetchResponse,
+            timeout=timeout,
+            deadline=deadline,
+            metadata=metadata,
+        )
+
+    async def put(
+        self,
+        put_request: "PutRequest",
+        *,
+        timeout: Optional[float] = None,
+        deadline: Optional["Deadline"] = None,
+        metadata: Optional["MetadataLike"] = None
+    ) -> "PutResponse":
+        return await self._unary_unary(
+            "/scandal.ScandalService/Put",
+            put_request,
+            PutResponse,
+            timeout=timeout,
+            deadline=deadline,
+            metadata=metadata,
+        )
+
+    async def delete(
+        self,
+        delete_request: "DeleteRequest",
+        *,
+        timeout: Optional[float] = None,
+        deadline: Optional["Deadline"] = None,
+        metadata: Optional["MetadataLike"] = None
+    ) -> "DeleteResponse":
+        return await self._unary_unary(
+            "/scandal.ScandalService/Delete",
+            delete_request,
+            DeleteResponse,
+            timeout=timeout,
+            deadline=deadline,
+            metadata=metadata,
+        )
+
+
+class ScandalServiceBase(ServiceBase):
+    async def fetch(self, fetch_request: "FetchRequest") -> "FetchResponse":
+        raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED)
+
+    async def put(self, put_request: "PutRequest") -> "PutResponse":
+        raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED)
+
+    async def delete(self, delete_request: "DeleteRequest") -> "DeleteResponse":
+        raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED)
+
+    async def __rpc_fetch(
+        self, stream: "grpclib.server.Stream[FetchRequest, FetchResponse]"
+    ) -> None:
+        request = await stream.recv_message()
+        response = await self.fetch(request)
+        await stream.send_message(response)
+
+    async def __rpc_put(
+        self, stream: "grpclib.server.Stream[PutRequest, PutResponse]"
+    ) -> None:
+        request = await stream.recv_message()
+        response = await self.put(request)
+        await stream.send_message(response)
+
+    async def __rpc_delete(
+        self, stream: "grpclib.server.Stream[DeleteRequest, DeleteResponse]"
+    ) -> None:
+        request = await stream.recv_message()
+        response = await self.delete(request)
+        await stream.send_message(response)
+
+    def __mapping__(self) -> Dict[str, grpclib.const.Handler]:
+        return {
+            "/scandal.ScandalService/Fetch": grpclib.const.Handler(
+                self.__rpc_fetch,
+                grpclib.const.Cardinality.UNARY_UNARY,
+                FetchRequest,
+                FetchResponse,
+            ),
+            "/scandal.ScandalService/Put": grpclib.const.Handler(
+                self.__rpc_put,
+                grpclib.const.Cardinality.UNARY_UNARY,
+                PutRequest,
+                PutResponse,
+            ),
+            "/scandal.ScandalService/Delete": grpclib.const.Handler(
+                self.__rpc_delete,
+                grpclib.const.Cardinality.UNARY_UNARY,
+                DeleteRequest,
+                DeleteResponse,
+            ),
+        }

spiral/proto/_/spfs/__init__.py

@@ -0,0 +1,36 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# sources: spfs/spfs.proto
+# plugin: python-betterproto
+# This file has been @generated
+
+from dataclasses import dataclass
+
+import betterproto
+
+
+@dataclass(eq=False, repr=False)
+class FileMetadata(betterproto.Message):
+    protobuf: "ProtobufFileSpecificMetadata" = betterproto.message_field(
+        1, group="format_specific"
+    )
+    parquet: "ParquetFileSpecificMetadata" = betterproto.message_field(
+        2, group="format_specific"
+    )
+    vortex: "VortexFileSpecificMetadata" = betterproto.message_field(
+        3, group="format_specific"
+    )
+
+
+@dataclass(eq=False, repr=False)
+class ProtobufFileSpecificMetadata(betterproto.Message):
+    pass
+
+
+@dataclass(eq=False, repr=False)
+class ParquetFileSpecificMetadata(betterproto.Message):
+    metadata_size_bytes: int = betterproto.uint32_field(1)
+
+
+@dataclass(eq=False, repr=False)
+class VortexFileSpecificMetadata(betterproto.Message):
+    metadata_size_bytes: int = betterproto.uint32_field(1)

spiral/proto/_/spiral/table/__init__.py

@@ -0,0 +1,225 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# sources: spiral/table/common.proto, spiral/table/metadata.proto, spiral/table/statistics.proto, spiral/table/wal.proto
+# plugin: python-betterproto
+# This file has been @generated
+
+from dataclasses import dataclass
+from typing import (
+    List,
+    Optional,
+)
+
+import betterproto
+
+
+class FileFormat(betterproto.Enum):
+    UNSPECIFIED = 0
+    PARQUET = 1
+    PROTOBUF = 2
+    BINARY_ARRAY = 3
+    VORTEX = 4
+
+
+class Level(betterproto.Enum):
+    UNSPECIFIED = 0
+    L0 = 1
+    L1 = 2
+
+
+@dataclass(eq=False, repr=False)
+class ApproximateSetMembership(betterproto.Message):
+    bloom_filter: "BloomFilter" = betterproto.message_field(
+        2, group="membership_strategy"
+    )
+
+
+@dataclass(eq=False, repr=False)
+class BloomFilter(betterproto.Message):
+    bit_vec: bytes = betterproto.bytes_field(1)
+    bitmap_bits: int = betterproto.uint64_field(2)
+    k_num: int = betterproto.uint32_field(3)
+
+
+@dataclass(eq=False, repr=False)
+class Schema(betterproto.Message):
+    arrow: bytes = betterproto.bytes_field(1)
+
+
+@dataclass(eq=False, repr=False)
+class ColumnGroup(betterproto.Message):
+    parts: List[str] = betterproto.string_field(1)
+
+
+@dataclass(eq=False, repr=False)
+class KeySpan(betterproto.Message):
+    """/ Span of keys defined as indices into a key space (exclusive end)."""
+
+    begin: int = betterproto.uint64_field(1)
+    end: int = betterproto.uint64_field(2)
+
+
+@dataclass(eq=False, repr=False)
+class KeyExtent(betterproto.Message):
+    """/ Extent of keys in terms of min and max (an inclusive range)."""
+
+    min: bytes = betterproto.bytes_field(1)
+    max: bytes = betterproto.bytes_field(2)
+
+
+@dataclass(eq=False, repr=False)
+class KeyMap(betterproto.Message):
+    """/ Displacement map."""
+
+    bitmap: bytes = betterproto.bytes_field(1)
+
+
+@dataclass(eq=False, repr=False)
+class ManifestHandle(betterproto.Message):
+    """
+    / Handle to a manifest file, with additional metadata to accelerate reads.
+    """
+
+    id: str = betterproto.string_field(1)
+    format: "FileFormat" = betterproto.enum_field(2)
+    file_size: int = betterproto.uint64_field(3)
+    spfs_format_metadata: Optional[bytes] = betterproto.bytes_field(4, optional=True)
+
+
+@dataclass(eq=False, repr=False)
+class VersionedSchema(betterproto.Message):
+    """/ Timestamped schema."""
+
+    ts: int = betterproto.uint64_field(1)
+    schema: "Schema" = betterproto.message_field(2)
+    column_ids: List[str] = betterproto.string_field(3)
+    """
+    List of column IDs, in the same order as columns in the schema.
+     Must have the same length as the number of columns in the schema.
+    """
+
+
+@dataclass(eq=False, repr=False)
+class WriteAheadLog(betterproto.Message):
+    """
+    / Table's Write Ahead Log stores sequence of operations and table metadata.
+    """
+
+    entries: List["LogEntry"] = betterproto.message_field(1)
+    truncated_up_to: int = betterproto.uint64_field(3)
+    """
+    Timestamp of the latest entry that has been removed from the log.
+     Entries with ts <= truncated_up_to are NOT present in the log.
+    """
+
+    ks_manifest_handle: Optional["ManifestHandle"] = betterproto.message_field(
+        2, optional=True
+    )
+    """An absent manifest means that one has not yet been written."""
+
+
+@dataclass(eq=False, repr=False)
+class LogEntry(betterproto.Message):
+    ts: int = betterproto.uint64_field(1)
+    key_space_write: "KeySpaceWriteOp" = betterproto.message_field(2, group="operation")
+    fragment_set_write: "FragmentSetWriteOp" = betterproto.message_field(
+        3, group="operation"
+    )
+    configuration: "ConfigurationOp" = betterproto.message_field(4, group="operation")
+    schema_evolution: "SchemaEvolutionOp" = betterproto.message_field(
+        5, group="operation"
+    )
+    schema_break: "SchemaBreakOp" = betterproto.message_field(6, group="operation")
+    compact_key_space: "CompactKeySpaceOp" = betterproto.message_field(
+        7, group="operation"
+    )
+    compact_column_group: "CompactColumnGroupOp" = betterproto.message_field(
+        8, group="operation"
+    )
+
+
+@dataclass(eq=False, repr=False)
+class KeySpaceWriteOp(betterproto.Message):
+    ks_id: str = betterproto.string_field(1)
+    manifest_handle: "ManifestHandle" = betterproto.message_field(2)
+
+
+@dataclass(eq=False, repr=False)
+class FragmentSetWriteOp(betterproto.Message):
+    column_group: "ColumnGroup" = betterproto.message_field(1)
+    fs_id: str = betterproto.string_field(2)
+    fs_level: "Level" = betterproto.enum_field(3)
+    manifest_handle: "ManifestHandle" = betterproto.message_field(4)
+    key_span: "KeySpan" = betterproto.message_field(5)
+    key_extent: "KeyExtent" = betterproto.message_field(6)
+    column_ids: List[str] = betterproto.string_field(7)
+
+
+@dataclass(eq=False, repr=False)
+class ConfigurationOp(betterproto.Message):
+    column_group: "ColumnGroup" = betterproto.message_field(1)
+    immutable_schema: Optional[bool] = betterproto.bool_field(2, optional=True)
+    """All column group configuration is stored in column group metadata."""
+
+
+@dataclass(eq=False, repr=False)
+class SchemaEvolutionOp(betterproto.Message):
+    column_group: "ColumnGroup" = betterproto.message_field(1)
+    new_schema: "VersionedSchema" = betterproto.message_field(2)
+
+
+@dataclass(eq=False, repr=False)
+class SchemaBreakOp(betterproto.Message):
+    column_group: "ColumnGroup" = betterproto.message_field(1)
+    removed_column_names: List[str] = betterproto.string_field(2)
+
+
+@dataclass(eq=False, repr=False)
+class CompactKeySpaceOp(betterproto.Message):
+    from_ks_ids: List[str] = betterproto.string_field(1)
+    into_ks_ids: List[str] = betterproto.string_field(2)
+
+
+@dataclass(eq=False, repr=False)
+class CompactColumnGroupOp(betterproto.Message):
+    column_group: "ColumnGroup" = betterproto.message_field(1)
+    from_fragment_ids: List[str] = betterproto.string_field(2)
+    """
+    NOTE: While key space compaction always compacts a full key space,
+     column group compaction can compact only a subset of the fragment set,
+     therefore the event specifies the fragment ids.
+    """
+
+    into_fs_ids: List[str] = betterproto.string_field(3)
+
+
+@dataclass(eq=False, repr=False)
+class ColumnGroupMetadata(betterproto.Message):
+    column_group: "ColumnGroup" = betterproto.message_field(1)
+    manifest_handle: Optional["ManifestHandle"] = betterproto.message_field(
+        2, optional=True
+    )
+    """An absent manifest means that one has not yet been written."""
+
+    last_modified_at: int = betterproto.uint64_field(3)
+    """Timestamp of the last WAL entry that modified this metadata."""
+
+    schema_versions: List["VersionedSchema"] = betterproto.message_field(4)
+    """
+    TODO(marko): Add config that truncates this list and breaks time travel.
+     Versions of the schema. Higher index is more recent, last element is latest
+     schema. Stored to support time travel through the schema, e.g. reading
+     deleted column.
+    """
+
+    immutable_schema: bool = betterproto.bool_field(5)
+    """
+    True if schema can NOT be evolved on write.
+     An explicit schema evolution is required.
+    """
+
+    schema_salt: int = betterproto.int32_field(6)
+    """
+    Schema salt is used to compute column IDs. It is modified on breaking
+     change, e.g. column deletion. This ensures that if the previously existing
+     column is added again, it will have a different ID.
+    """