pyspiral 0.2.5__cp310-abi3-macosx_11_0_arm64.whl → 0.4.0__cp310-abi3-macosx_11_0_arm64.whl

spiral/cli/project.py

@@ -1,107 +0,0 @@
-from typing import Annotated
-
-import rich
-import typer
-from typer import Option
-
-from spiral.api.organizations import OrganizationRole
-from spiral.api.projects import CreateProject, Grant, GrantRole, ListGrants, Project
-from spiral.cli import AsyncTyper, OptionalStr, printer, state
-from spiral.cli.org import current_org_id
-from spiral.cli.types import ProjectArg
-
-app = AsyncTyper()
-
-
-@app.command(help="List projects.")
-def ls():
-    projects = list(state.settings.api.project.list())
-    rich.print(printer.table_of_models(Project, projects))
-
-
-@app.command(help="Create a new project.")
-def create(
-    id_prefix: Annotated[
-        OptionalStr, Option(help="An optional ID prefix to which a random number will be appended.")
-    ] = None,
-    org_id: Annotated[OptionalStr, Option(help="Organization ID in which to create the project.")] = None,
-    name: Annotated[OptionalStr, Option(help="Friendly name for the project.")] = None,
-):
-    res = state.settings.api.project.create(
-        CreateProject.Request(organization_id=org_id or current_org_id(), id_prefix=id_prefix, name=name)
-    )
-    rich.print(f"Created project {res.project.id}")
-
-
-@app.command(help="Grant a role on a project.")
-def grant(
-    project: ProjectArg,
-    role: Annotated[str, Option(help="Role to grant.")],
-    org_id: Annotated[
-        OptionalStr, Option(help="Pass an organization ID to grant a role to an organization user(s).")
-    ] = None,
-    user_id: Annotated[
-        OptionalStr, Option(help="Pass a user ID when using --org-id to grant a role to grant a role to a user.")
-    ] = None,
-    org_role: Annotated[
-        OptionalStr,
-        Option(help="Pass an organization role when using --org-id to grant a role to all users with that role."),
-    ] = None,
-    workload_id: Annotated[OptionalStr, Option(help="Pass a workload ID to grant a role to a workload.")] = None,
-    github: Annotated[
-        OptionalStr, Option(help="Pass an `<org>/<repo>` string to grant a role to a job running in GitHub Actions.")
-    ] = None,
-    modal: Annotated[
-        OptionalStr,
-        Option(help="Pass a `<workspace_id>/<env_name>` string to grant a role to a job running in Modal environment."),
-    ] = None,
-    conditions: list[str] = Option(
-        default_factory=list,
-        help="`<key>=<value>` token conditions to apply to the grant when using --github or --modal.",
-    ),
-):
-    # Check mutual exclusion
-    if sum(int(bool(opt)) for opt in {org_id, workload_id, github, modal}) != 1:
-        raise typer.BadParameter("Only one of --org-id, --github or --modal may be specified.")
-
-    if github:
-        org, repo = github.split("/", 1)
-        conditions = {GrantRole.GitHubClaim(k): v for k, v in dict(c.split("=", 1) for c in conditions).items()}
-        principal = GrantRole.GitHubPrincipal(org=org, repo=repo, conditions=conditions)
-    elif modal:
-        workspace_id, environment_name = modal.split("/", 1)
-        conditions = {GrantRole.ModalClaim(k): v for k, v in dict(c.split("=", 1) for c in conditions).items()}
-        principal = GrantRole.ModalPrincipal(
-            workspace_id=workspace_id, environment_name=environment_name, conditions=conditions
-        )
-    elif org_id:
-        # Check mutual exclusion
-        if sum(int(bool(opt)) for opt in {user_id, org_role}) != 1:
-            raise typer.BadParameter("Only one of --user-id or --org-role may be specified.")
-
-        if user_id is not None:
-            principal = GrantRole.OrgUserPrincipal(org_id=org_id, user_id=user_id)
-        elif org_role is None:
-            principal = GrantRole.OrgRolePrincipal(org_id=org_id, role=OrganizationRole(org_role))
-        else:
-            raise NotImplementedError("Only user or role principal is supported at this time.")
-    elif workload_id:
-        principal = GrantRole.WorkloadPrincipal(workload_id=workload_id)
-    else:
-        raise NotImplementedError("Only organization, GitHub or Modal principal is supported at this time.")
-
-    state.settings.api.project.grant_role(
-        GrantRole.Request(
-            project_id=project,
-            role_id=role,
-            principal=principal,
-        )
-    )
-
-    rich.print(f"Granted role {role} on project {project}")
-
-
-@app.command(help="List project grants.")
-def grants(project: ProjectArg):
-    project_grants = list(state.settings.api.project.list_grants(ListGrants.Request(project_id=project)))
-    rich.print(printer.table_of_models(Grant, project_grants, title="Project Grants"))

spiral/cli/table.py

@@ -1,20 +0,0 @@
-from typing import Annotated
-
-import rich
-from typer import Option
-
-from spiral.api.tables import ListTables, Table
-from spiral.cli import AsyncTyper, OptionalStr, printer, state
-from spiral.cli.types import ProjectArg
-
-app = AsyncTyper()
-
-
-@app.command(help="List tables.")
-def ls(
-    project: ProjectArg,
-    dataset: Annotated[OptionalStr, Option(help="Filter by dataset name.")] = None,
-):
-    """List tables."""
-    tables = list(state.settings.api.table.list(ListTables.Request(project_id=project, dataset=dataset)))
-    rich.print(printer.table_of_models(Table, tables, fields=["id", "project_id", "dataset", "table"]))

spiral/cli/token.py

@@ -1,27 +0,0 @@
-from typing import Annotated
-
-import rich
-from typer import Argument, Option
-
-from spiral.api.tokens import ListTokens, RevokeToken, Token
-from spiral.cli import AsyncTyper, OptionalStr, printer, state
-from spiral.cli.types import ProjectArg
-
-app = AsyncTyper()
-
-
-@app.command(help="List tokens.")
-def ls(
-    project: ProjectArg,
-    on_behalf_of: Annotated[OptionalStr, Option(help="Filter by on behalf of.")] = None,
-):
-    tokens = list(state.settings.api.token.list(ListTokens.Request(project_id=project, on_behalf_of=on_behalf_of)))
-    rich.print(printer.table_of_models(Token, tokens, fields=["id", "project_id", "on_behalf_of"]))
-
-
-@app.command(help="Revoke a token.")
-def revoke(token_id: Annotated[str, Argument(help="Token ID.")]):
-    res = state.settings.api.token.revoke(RevokeToken.Request(token_id=token_id))
-    rich.print(
-        f"Revoked token {res.token.id} for project {res.token.project_id} acting on behalf of {res.token.on_behalf_of}"
-    )

spiral/config.py

@@ -1,26 +0,0 @@
-import os
-
-from pydantic_settings import BaseSettings, SettingsConfigDict
-
-FILE_FORMAT = os.environ.get("SPIRAL_FILE_FORMAT", "parquet")
-
-
-class Config(BaseSettings):
-    model_config = SettingsConfigDict(
-        env_nested_delimiter="__",
-        env_prefix="SPIRAL_CORE__",
-        frozen=True,
-    )
-
-    partition_file_min_size: int = 256 * 1024 * 1024  # 256MB
-    flush_wal_on_write: bool = False
-
-    # TODO(marko): Support config. Unused after migration to Rust.
-    # #: Defaults to ThreadPoolExecutor's default (based on os.cpu_count().
-    # scan_num_threads: int | None = 61  # 61 is used by Golang and Tokio, for some reason...
-    #
-    # #: The duration of WAL that is preserved to allow for txn conflict resolution.
-    # transaction_window: int = 0 if DEV else timedelta(days=1).total_seconds()
-    #
-    # #: Truncation length of string statistics.
-    # string_truncation_length: int = 1024

spiral/core/core/__init__.pyi

@@ -1,53 +0,0 @@
-from typing import Any
-
-import pyarrow as pa
-from spiral.core.manifests import FragmentManifest
-from spiral.core.metastore import PyMetastore
-from spiral.core.spec import ColumnGroup, ColumnGroupMetadata, KeyRange, Schema, WriteAheadLog
-from spiral.expressions import Expr
-
-class Table:
-    def __init__(self, metastore: PyMetastore): ...
-
-    id: str
-    root_uri: str
-    key_schema: Schema
-    metastore: PyMetastore
-
-    def get_wal(self, *, asof: int | None) -> WriteAheadLog: ...
-    def get_column_group_metadata(self, column_group: ColumnGroup, *, asof: int | None) -> ColumnGroupMetadata: ...
-    def list_column_groups(self, *, asof: int | None) -> list[ColumnGroup] | None: ...
-    def get_schema(self, *, asof: int | None) -> Schema: ...
-
-class TableScan:
-    def __init__(
-        self,
-        projection: Expr,
-        filter: Expr | None = None,
-        asof: int | None = None,
-        exclude_keys: bool = False,
-        aux_schema: pa.Schema | None = None,
-    ) -> TableScan: ...
-    def key_schema(self) -> Schema: ...
-    def schema(self) -> Schema: ...
-    def is_empty(self) -> bool: ...
-    def split(self) -> list[KeyRange]: ...
-    def table_ids(self) -> list[str]: ...
-    def column_groups(self) -> list[ColumnGroup]: ...
-    def to_record_batches(self, aux_table: pa.Table | pa.RecordBatch | None = None) -> pa.RecordBatchReader: ...
-    def column_group_scan(self, column_group: ColumnGroup) -> ColumnGroupScan: ...
-    def key_space_scan(self, table_id: str) -> KeySpaceScan: ...
-    def metrics(self) -> dict[str, Any]: ...
-
-class KeySpaceScan:
-    manifest: FragmentManifest
-
-    def key_schema(self) -> Schema: ...
-
-class ColumnGroupScan:
-    manifest: FragmentManifest
-
-    def schema(self) -> Schema: ...
-
-def write(table: Table, expr: Expr, format: str = "parquet", *, partition_size: int | None = None): ...
-def flush_wal(table: Table, manifest_format: str = "parquet"): ...

spiral/core/metastore/__init__.pyi

@@ -1,91 +0,0 @@
-"""The SpiralDB metastore API."""
-
-from collections.abc import Callable
-
-from spiral.core.spec import ColumnGroup, ColumnGroupMetadata, FileFormat, LogEntry, Schema, WriteAheadLog
-from spiral.types_ import Timestamp, Uri
-from spiraldb.proto.spiral.table import ManifestHandle
-
-class FileHandle:
-    def __init__(self, *, uri: str, format: FileFormat, spfs_token: str | None): ...
-
-    uri: str
-    format: FileFormat
-    spfs_token: str | None
-
-class FileRef:
-    def __init__(self, *, id: str, file_type: FileType, file_format: FileFormat): ...
-
-    id: str
-    file_type: FileType
-    file_format: FileFormat
-
-    def resolve_uri(self, root_uri: str) -> str:
-        """Resolves the file reference URI given the root URI."""
-
-class FileType:
-    FragmentFile: FileType
-    FragmentManifest: FileType
-    ReferenceFile: FileType
-
-    def __int__(self) -> int:
-        """Returns the protobuf enum int value."""
-
-class PyMetastore:
-    """Rust implementation of the metastore API."""
-
-    @property
-    def table_id(self) -> str: ...
-    @property
-    def root_uri(self) -> Uri: ...
-    @property
-    def key_schema(self) -> Schema: ...
-    def get_wal(self) -> WriteAheadLog:
-        """Return the log for the table."""
-        ...
-
-    def append_wal(self, prev_last_modified_at: Timestamp, entries: list[LogEntry]) -> WriteAheadLog:
-        """Append additional entries into the write-ahead log given the previous write-ahead log timestamp.
-
-        The given entries should have a timestamp of zero and will be assigned an actual timestamp by the server.
-
-        This API is designed to support both a trivial compare-and-swap on the WAL, and also to support more advanced
-        conflict resolution within the metastore.
-        """
-        ...
-
-    def update_wal(
-        self,
-        prev_ks_manifest_handle_id: str,
-        truncate_ts_max: Timestamp | None = None,
-        new_ks_manifest_handle: ManifestHandle | None = None,
-    ) -> WriteAheadLog:
-        """Update the write-ahead log atomically.
-
-        Supports WAL truncation and manifest handle updates necessary for flushing.
-        """
-        ...
-
-    def get_column_group_metadata(self, column_group: ColumnGroup) -> ColumnGroupMetadata:
-        """Return the metadata for column group."""
-        ...
-
-    def update_column_group_metadata(
-        self, prev_last_modified_at: Timestamp, column_group_metadata: ColumnGroupMetadata
-    ) -> ColumnGroupMetadata:
-        """Update the column group metadata to the metastore given the previous metadata timestamp."""
-        ...
-
-    def list_column_groups(self) -> tuple[list[ColumnGroup], Timestamp]:
-        """List all column groups in the table, or None if no index is available."""
-        ...
-
-    @staticmethod
-    def http(
-        table_id: str, root_uri: str, key_schema: Schema, base_url: str, token_provider: Callable[[], str]
-    ) -> PyMetastore:
-        """Construct a PyMetastore backed by an HTTP metastore service."""
-
-    @staticmethod
-    def test(table_id: str, root_uri: str, key_schema: Schema) -> PyMetastore:
-        """Construct a PyMetastore backed by an in-memory mock metastore service."""

spiral/proto/_/spfs/__init__.py

@@ -1,36 +0,0 @@
-# Generated by the protocol buffer compiler.  DO NOT EDIT!
-# sources: spfs/spfs.proto
-# plugin: python-betterproto
-# This file has been @generated
-
-from dataclasses import dataclass
-
-import betterproto
-
-
-@dataclass(eq=False, repr=False)
-class FileMetadata(betterproto.Message):
-    protobuf: "ProtobufFileSpecificMetadata" = betterproto.message_field(
-        1, group="format_specific"
-    )
-    parquet: "ParquetFileSpecificMetadata" = betterproto.message_field(
-        2, group="format_specific"
-    )
-    vortex: "VortexFileSpecificMetadata" = betterproto.message_field(
-        3, group="format_specific"
-    )
-
-
-@dataclass(eq=False, repr=False)
-class ProtobufFileSpecificMetadata(betterproto.Message):
-    pass
-
-
-@dataclass(eq=False, repr=False)
-class ParquetFileSpecificMetadata(betterproto.Message):
-    metadata_size_bytes: int = betterproto.uint32_field(1)
-
-
-@dataclass(eq=False, repr=False)
-class VortexFileSpecificMetadata(betterproto.Message):
-    metadata_size_bytes: int = betterproto.uint32_field(1)

spiral/proto/_/spiral/table/__init__.py

@@ -1,225 +0,0 @@
-# Generated by the protocol buffer compiler.  DO NOT EDIT!
-# sources: spiral/table/common.proto, spiral/table/metadata.proto, spiral/table/statistics.proto, spiral/table/wal.proto
-# plugin: python-betterproto
-# This file has been @generated
-
-from dataclasses import dataclass
-from typing import (
-    List,
-    Optional,
-)
-
-import betterproto
-
-
-class FileFormat(betterproto.Enum):
-    UNSPECIFIED = 0
-    PARQUET = 1
-    PROTOBUF = 2
-    BINARY_ARRAY = 3
-    VORTEX = 4
-
-
-class Level(betterproto.Enum):
-    UNSPECIFIED = 0
-    L0 = 1
-    L1 = 2
-
-
-@dataclass(eq=False, repr=False)
-class ApproximateSetMembership(betterproto.Message):
-    bloom_filter: "BloomFilter" = betterproto.message_field(
-        2, group="membership_strategy"
-    )
-
-
-@dataclass(eq=False, repr=False)
-class BloomFilter(betterproto.Message):
-    bit_vec: bytes = betterproto.bytes_field(1)
-    bitmap_bits: int = betterproto.uint64_field(2)
-    k_num: int = betterproto.uint32_field(3)
-
-
-@dataclass(eq=False, repr=False)
-class Schema(betterproto.Message):
-    arrow: bytes = betterproto.bytes_field(1)
-
-
-@dataclass(eq=False, repr=False)
-class ColumnGroup(betterproto.Message):
-    parts: List[str] = betterproto.string_field(1)
-
-
-@dataclass(eq=False, repr=False)
-class KeySpan(betterproto.Message):
-    """/ Span of keys defined as indices into a key space (exclusive end)."""
-
-    begin: int = betterproto.uint64_field(1)
-    end: int = betterproto.uint64_field(2)
-
-
-@dataclass(eq=False, repr=False)
-class KeyExtent(betterproto.Message):
-    """/ Extent of keys in terms of min and max (an inclusive range)."""
-
-    min: bytes = betterproto.bytes_field(1)
-    max: bytes = betterproto.bytes_field(2)
-
-
-@dataclass(eq=False, repr=False)
-class KeyMap(betterproto.Message):
-    """/ Displacement map."""
-
-    bitmap: bytes = betterproto.bytes_field(1)
-
-
-@dataclass(eq=False, repr=False)
-class ManifestHandle(betterproto.Message):
-    """
-    / Handle to a manifest file, with additional metadata to accelerate reads.
-    """
-
-    id: str = betterproto.string_field(1)
-    format: "FileFormat" = betterproto.enum_field(2)
-    file_size: int = betterproto.uint64_field(3)
-    spfs_format_metadata: Optional[bytes] = betterproto.bytes_field(4, optional=True)
-
-
-@dataclass(eq=False, repr=False)
-class VersionedSchema(betterproto.Message):
-    """/ Timestamped schema."""
-
-    ts: int = betterproto.uint64_field(1)
-    schema: "Schema" = betterproto.message_field(2)
-    column_ids: List[str] = betterproto.string_field(3)
-    """
-    List of column IDs, in the same order as columns in the schema.
-     Must have the same length as the number of columns in the schema.
-    """
-
-
-@dataclass(eq=False, repr=False)
-class WriteAheadLog(betterproto.Message):
-    """
-    / Table's Write Ahead Log stores sequence of operations and table metadata.
-    """
-
-    entries: List["LogEntry"] = betterproto.message_field(1)
-    truncated_up_to: int = betterproto.uint64_field(3)
-    """
-    Timestamp of the latest entry that has been removed from the log.
-     Entries with ts <= truncated_up_to are NOT present in the log.
-    """
-
-    ks_manifest_handle: Optional["ManifestHandle"] = betterproto.message_field(
-        2, optional=True
-    )
-    """An absent manifest means that one has not yet been written."""
-
-
-@dataclass(eq=False, repr=False)
-class LogEntry(betterproto.Message):
-    ts: int = betterproto.uint64_field(1)
-    key_space_write: "KeySpaceWriteOp" = betterproto.message_field(2, group="operation")
-    fragment_set_write: "FragmentSetWriteOp" = betterproto.message_field(
-        3, group="operation"
-    )
-    configuration: "ConfigurationOp" = betterproto.message_field(4, group="operation")
-    schema_evolution: "SchemaEvolutionOp" = betterproto.message_field(
-        5, group="operation"
-    )
-    schema_break: "SchemaBreakOp" = betterproto.message_field(6, group="operation")
-    compact_key_space: "CompactKeySpaceOp" = betterproto.message_field(
-        7, group="operation"
-    )
-    compact_column_group: "CompactColumnGroupOp" = betterproto.message_field(
-        8, group="operation"
-    )
-
-
-@dataclass(eq=False, repr=False)
-class KeySpaceWriteOp(betterproto.Message):
-    ks_id: str = betterproto.string_field(1)
-    manifest_handle: "ManifestHandle" = betterproto.message_field(2)
-
-
-@dataclass(eq=False, repr=False)
-class FragmentSetWriteOp(betterproto.Message):
-    column_group: "ColumnGroup" = betterproto.message_field(1)
-    fs_id: str = betterproto.string_field(2)
-    fs_level: "Level" = betterproto.enum_field(3)
-    manifest_handle: "ManifestHandle" = betterproto.message_field(4)
-    key_span: "KeySpan" = betterproto.message_field(5)
-    key_extent: "KeyExtent" = betterproto.message_field(6)
-    column_ids: List[str] = betterproto.string_field(7)
-
-
-@dataclass(eq=False, repr=False)
-class ConfigurationOp(betterproto.Message):
-    column_group: "ColumnGroup" = betterproto.message_field(1)
-    immutable_schema: Optional[bool] = betterproto.bool_field(2, optional=True)
-    """All column group configuration is stored in column group metadata."""
-
-
-@dataclass(eq=False, repr=False)
-class SchemaEvolutionOp(betterproto.Message):
-    column_group: "ColumnGroup" = betterproto.message_field(1)
-    new_schema: "VersionedSchema" = betterproto.message_field(2)
-
-
-@dataclass(eq=False, repr=False)
-class SchemaBreakOp(betterproto.Message):
-    column_group: "ColumnGroup" = betterproto.message_field(1)
-    removed_column_names: List[str] = betterproto.string_field(2)
-
-
-@dataclass(eq=False, repr=False)
-class CompactKeySpaceOp(betterproto.Message):
-    from_ks_ids: List[str] = betterproto.string_field(1)
-    into_ks_ids: List[str] = betterproto.string_field(2)
-
-
-@dataclass(eq=False, repr=False)
-class CompactColumnGroupOp(betterproto.Message):
-    column_group: "ColumnGroup" = betterproto.message_field(1)
-    from_fragment_ids: List[str] = betterproto.string_field(2)
-    """
-    NOTE: While key space compaction always compacts a full key space,
-     column group compaction can compact only a subset of the fragment set,
-     therefore the event specifies the fragment ids.
-    """
-
-    into_fs_ids: List[str] = betterproto.string_field(3)
-
-
-@dataclass(eq=False, repr=False)
-class ColumnGroupMetadata(betterproto.Message):
-    column_group: "ColumnGroup" = betterproto.message_field(1)
-    manifest_handle: Optional["ManifestHandle"] = betterproto.message_field(
-        2, optional=True
-    )
-    """An absent manifest means that one has not yet been written."""
-
-    last_modified_at: int = betterproto.uint64_field(3)
-    """Timestamp of the last WAL entry that modified this metadata."""
-
-    schema_versions: List["VersionedSchema"] = betterproto.message_field(4)
-    """
-    TODO(marko): Add config that truncates this list and breaks time travel.
-     Versions of the schema. Higher index is more recent, last element is latest
-     schema. Stored to support time travel through the schema, e.g. reading
-     deleted column.
-    """
-
-    immutable_schema: bool = betterproto.bool_field(5)
-    """
-    True if schema can NOT be evolved on write.
-     An explicit schema evolution is required.
-    """
-
-    schema_salt: int = betterproto.int32_field(6)
-    """
-    Schema salt is used to compute column IDs. It is modified on breaking
-     change, e.g. column deletion. This ensures that if the previously existing
-     column is added again, it will have a different ID.
-    """