pyspiral 0.5.0__cp310-abi3-macosx_11_0_arm64.whl → 0.6.0__cp310-abi3-macosx_11_0_arm64.whl

spiral/cli/iceberg/namespaces.py

@@ -1,47 +0,0 @@
-import sys
-from typing import Annotated
-
-import pyiceberg.exceptions
-import rich
-import typer
-from typer import Argument
-
-from spiral.cli import AsyncTyper, state
-from spiral.cli.types import ProjectArg
-
-app = AsyncTyper(short_help="Apache Iceberg Namespaces.")
-
-
-@app.command(help="List namespaces.")
-def ls(
-    project: ProjectArg,
-    namespace: Annotated[str | None, Argument(help="List only namespaces under this namespace.")] = None,
-):
-    """List Iceberg namespaces."""
-    catalog = state.spiral.iceberg.catalog()
-
-    if namespace is None:
-        try:
-            namespaces = catalog.list_namespaces(project)
-        except pyiceberg.exceptions.ForbiddenError:
-            print(
-                f"The project, {repr(project)}, does not exist or you lack the "
-                f"`iceberg:view` permission to list namespaces in it.",
-                file=sys.stderr,
-            )
-            raise typer.Exit(code=1)
-    else:
-        try:
-            namespaces = catalog.list_namespaces((project, namespace))
-        except pyiceberg.exceptions.ForbiddenError:
-            print(
-                f"The namespace, {repr(project)}.{repr(namespace)}, does not exist or you lack the "
-                f"`iceberg:view` permission to list namespaces in it.",
-                file=sys.stderr,
-            )
-            raise typer.Exit(code=1)
-
-    table = rich.table.Table("Namespace ID", title="Iceberg namespaces")
-    for ns in namespaces:
-        table.add_row(".".join(ns))
-    rich.print(table)

spiral/cli/iceberg/tables.py

@@ -1,60 +0,0 @@
-import sys
-from typing import Annotated
-
-import pyiceberg.exceptions
-import rich
-import typer
-from typer import Argument
-
-from spiral.cli import AsyncTyper, state
-from spiral.cli.types import ProjectArg
-
-app = AsyncTyper(short_help="Apache Iceberg Tables.")
-
-
-@app.command(help="List tables.")
-def ls(
-    project: ProjectArg,
-    namespace: Annotated[str | None, Argument(help="Show only tables in the given namespace.")] = None,
-):
-    catalog = state.spiral.iceberg.catalog()
-
-    try:
-        if namespace is None:
-            tables = catalog.list_tables(project)
-        else:
-            tables = catalog.list_tables((project, namespace))
-    except pyiceberg.exceptions.ForbiddenError:
-        print(
-            f"The namespace, {repr(project)}.{repr(namespace)}, does not exist or you lack the "
-            f"`iceberg:view` permission to list tables in it.",
-            file=sys.stderr,
-        )
-        raise typer.Exit(code=1)
-
-    rich_table = rich.table.Table("table id", title="Iceberg tables")
-    for table in tables:
-        rich_table.add_row(".".join(table))
-    rich.print(rich_table)
-
-
-@app.command(help="Show the table schema.")
-def schema(
-    project: ProjectArg,
-    namespace: Annotated[str, Argument(help="Table namespace.")],
-    table: Annotated[str, Argument(help="Table name.")],
-):
-    catalog = state.spiral.iceberg.catalog()
-
-    try:
-        tbl = catalog.load_table((project, namespace, table))
-    except pyiceberg.exceptions.NoSuchTableError:
-        print(f"No table {repr(table)} found in {repr(project)}.{repr(namespace)}", file=sys.stderr)
-        raise typer.Exit(code=1)
-
-    rich_table = rich.table.Table(
-        "Field ID", "Field name", "Type", "Required", "Doc", title=f"{project}.{namespace}.{table}"
-    )
-    for col in tbl.schema().columns:
-        rich_table.add_row(str(col.field_id), col.name, str(col.field_type), str(col.required), col.doc)
-    rich.print(rich_table)

spiral/cli/indexes/__init__.py

@@ -1,40 +0,0 @@
-from typing import Annotated
-
-import rich
-from typer import Option
-
-from spiral.api.text_indexes import SyncIndexRequest
-from spiral.cli import AsyncTyper, state
-from spiral.cli.indexes.args import get_text_index_id
-from spiral.cli.types import ProjectArg
-
-from ...api.workers import ResourceClass
-from . import workers
-
-app = AsyncTyper(short_help="Indexes.")
-app.add_typer(workers.app, name="workers")
-
-
-@app.command(help="List indexes.")
-def ls(
-    project: ProjectArg,
-):
-    """List indexes."""
-    indexes = state.spiral.project(project).indexes.list_indexes()
-
-    rich_table = rich.table.Table("id", "name", title="Indexes")
-    for index in indexes:
-        rich_table.add_row(index.id, index.name)
-    rich.print(rich_table)
-
-
-@app.command(help="Trigger a sync job for the index.")
-def sync(
-    project: ProjectArg,
-    name: Annotated[str | None, Option(help="Index name.")] = None,
-    resources: Annotated[ResourceClass, Option(help="Resources to use for the sync job.")] = ResourceClass.SMALL,
-):
-    """Trigger a sync job for the index."""
-    index_id = get_text_index_id(project, name)
-    response = state.spiral.api.text_indexes.sync_index(index_id, SyncIndexRequest(resources=resources))
-    rich.print(f"Triggered sync job {response.worker_id} for index {index_id}.")

spiral/cli/indexes/args.py

@@ -1,39 +0,0 @@
-from typing import Annotated
-
-import questionary
-import rich
-import typer
-from questionary import Choice
-from typer import Option
-
-from spiral.api.projects import TextIndexResource
-from spiral.api.types import IndexId
-from spiral.cli import state
-from spiral.cli.types import ProjectArg
-
-
-def ask_index(project_id, title="Select an index"):
-    indexes: list[TextIndexResource] = list(state.spiral.project(project_id).indexes.list_indexes())
-
-    if not indexes:
-        rich.print("[red]No indexes found[/red]")
-        raise typer.Exit(1)
-
-    return questionary.select(
-        title,
-        choices=[Choice(title=index.name, value=index.id) for index in sorted(indexes, key=lambda t: (t.name, t.id))],
-    ).ask()
-
-
-def get_text_index_id(
-    project: ProjectArg,
-    name: Annotated[str | None, Option(help="Index name.")] = None,
-) -> IndexId:
-    if name is None:
-        return ask_index(project)
-
-    indexes: list[TextIndexResource] = list(state.spiral.project(project).indexes.list_indexes())
-    for index in indexes:
-        if index.name == name:
-            return index.id
-    raise ValueError(f"Index not found: {name}")

spiral/cli/indexes/workers.py

@@ -1,59 +0,0 @@
-from typing import Annotated
-
-import rich
-from typer import Option
-
-from spiral.api.text_indexes import CreateWorkerRequest
-from spiral.api.workers import CPU, GcpRegion, Memory
-from spiral.cli import AsyncTyper, state
-from spiral.cli.indexes.args import get_text_index_id
-from spiral.cli.types import ProjectArg
-
-app = AsyncTyper(short_help="Text Search Workers.")
-
-
-@app.command(name="serve", help="Create a search worker.")
-def serve(
-    project: ProjectArg,
-    index: Annotated[str | None, Option(help="Index name.")] = None,
-    region: Annotated[GcpRegion, Option(help="GCP region for the worker.")] = GcpRegion.US_EAST4,
-    cpu: Annotated[CPU, Option(help="CPU resources for the worker.")] = CPU.ONE,
-    memory: Annotated[Memory, Option(help="Memory resources for the worker in MB.")] = Memory.MB_512,
-):
-    """Create a new text search worker."""
-    index_id = get_text_index_id(project, index)
-    request = CreateWorkerRequest(cpu=cpu, memory=memory, region=region)
-    response = state.spiral.api.text_indexes.create_worker(index_id, request)
-    rich.print(f"Created worker {response.worker_id} for {index_id}.")
-
-
-@app.command(name="shutdown", help="Shutdown a search worker.")
-def shutdown(worker_id: str):
-    """Shutdown a worker."""
-    state.spiral.api.text_indexes.shutdown_worker(worker_id)
-    rich.print(f"Requested worker {worker_id} to shutdown.")
-
-
-@app.command(name="ls", help="List search workers.")
-def ls(
-    project: ProjectArg,
-    index: Annotated[str | None, Option(help="Index name.")] = None,
-):
-    """List text search workers."""
-    index_id = get_text_index_id(project, index)
-    worker_ids = state.spiral.api.text_indexes.list_workers(index_id)
-
-    rich_table = rich.table.Table("Worker ID", "URL", title=f"Text Search Workers for {index_id}")
-    for worker_id in worker_ids:
-        try:
-            worker = state.spiral.api.text_indexes.get_worker(worker_id)
-            rich_table.add_row(
-                worker_id,
-                worker.url,
-            )
-        except Exception:
-            rich_table.add_row(
-                worker_id,
-                "Unavailable",
-            )
-    rich.print(rich_table)

spiral/cli/tables/__init__.py

@@ -1,88 +0,0 @@
-from typing import Annotated
-
-import rich
-from typer import Argument, Option
-
-from spiral import Spiral
-from spiral.cli import AsyncTyper
-from spiral.cli.tables.args import get_table
-from spiral.cli.types import ProjectArg
-
-app = AsyncTyper(short_help="Spiral Tables.")
-
-
-@app.command(help="List tables.")
-def ls(
-    project: ProjectArg,
-):
-    tables = Spiral().project(project).tables.list_tables()
-
-    rich_table = rich.table.Table("id", "dataset", "name", title="Spiral tables")
-    for table in tables:
-        rich_table.add_row(table.id, table.dataset, table.table)
-    rich.print(rich_table)
-
-
-@app.command(help="Show the table key schema.")
-def key_schema(
-    project: ProjectArg,
-    table: Annotated[str | None, Option(help="Table name.")] = None,
-    dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
-):
-    _, table = get_table(project, table, dataset)
-    rich.print(table.key_schema)
-
-
-@app.command(help="Compute the full table schema.")
-def schema(
-    project: ProjectArg,
-    table: Annotated[str | None, Option(help="Table name.")] = None,
-    dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
-):
-    _, table = get_table(project, table, dataset)
-    rich.print(table.schema)
-
-
-@app.command(help="Flush Write-Ahead-Log.")
-def flush(
-    project: ProjectArg,
-    table: Annotated[str | None, Option(help="Table name.")] = None,
-    dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
-):
-    identifier, table = get_table(project, table, dataset)
-    table.maintenance().flush_wal()
-    print(f"Flushed WAL for table {identifier} in project {project}.")
-
-
-@app.command(help="Display scan.")
-def debug(
-    project: ProjectArg,
-    table: Annotated[str | None, Option(help="Table name.")] = None,
-    dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
-    column_group: Annotated[str, Argument(help="Dot-separated column group path.")] = ".",
-):
-    _, table = get_table(project, table, dataset)
-    if column_group != ".":
-        projection = table[column_group]
-    else:
-        projection = table
-    scan = table.scan(projection)
-
-    scan._debug()
-
-
-@app.command(help="Display manifests.")
-def manifests(
-    project: ProjectArg,
-    table: Annotated[str | None, Option(help="Table name.")] = None,
-    dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
-    column_group: Annotated[str, Argument(help="Dot-separated column group path.")] = ".",
-):
-    _, table = get_table(project, table, dataset)
-    if column_group != ".":
-        projection = table[column_group]
-    else:
-        projection = table
-    scan = projection.scan()
-
-    scan._dump_manifests()

spiral/cli/tables/args.py

@@ -1,42 +0,0 @@
-from typing import Annotated
-
-import questionary
-import rich
-import typer
-from questionary import Choice
-from typer import Option
-
-from spiral.api.projects import TableResource
-from spiral.cli import state
-from spiral.cli.types import ProjectArg
-from spiral.tables import Table
-
-
-def ask_table(project_id, title="Select a table"):
-    tables: list[TableResource] = list(state.spiral.project(project_id).tables.list_tables())
-
-    if not tables:
-        rich.print("[red]No tables found[/red]")
-        raise typer.Exit(1)
-
-    return questionary.select(
-        title,
-        choices=[
-            Choice(title=f"{table.dataset}.{table.table}", value=f"{table.project_id}.{table.dataset}.{table.table}")
-            for table in sorted(tables, key=lambda t: (t.dataset, t.table))
-        ],
-    ).ask()
-
-
-def get_table(
-    project: ProjectArg,
-    table: Annotated[str | None, Option(help="Table name.")] = None,
-    dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
-) -> (str, Table):
-    if table is None:
-        identifier = ask_table(project)
-    else:
-        identifier = table
-        if dataset is not None:
-            identifier = f"{dataset}.{table}"
-    return identifier, state.spiral.project(project).tables.table(identifier)

spiral/core/index/__init__.pyi

@@ -1,7 +0,0 @@
-import pyarrow as pa
-
-class TextIndex:
-    id: str
-
-class SearchScan:
-    def to_record_batches(self) -> pa.RecordBatchReader: ...

spiral/iceberg/__init__.py

@@ -1,3 +0,0 @@
-from spiral.iceberg.client import Iceberg
-
-__all__ = ["Iceberg"]

spiral/indexes/__init__.py

@@ -1,5 +0,0 @@
-from spiral.indexes.client import Indexes
-from spiral.indexes.index import TextIndex
-from spiral.indexes.scan import SearchScan
-
-__all__ = ["Indexes", "SearchScan", "TextIndex"]

spiral/indexes/client.py

@@ -1,137 +0,0 @@
-import datetime
-
-from spiral.api import SpiralAPI
-from spiral.api.projects import TextIndexResource
-from spiral.core.client import Spiral as CoreSpiral
-from spiral.expressions.base import ExprLike
-from spiral.indexes.index import TextIndex
-from spiral.indexes.scan import SearchScan
-from spiral.types_ import Uri
-
-
-class Indexes:
-    def __init__(self, api: SpiralAPI, spiral: CoreSpiral, *, project_id: str | None = None):
-        self._api = api
-        self._spiral = spiral
-        self._project_id = project_id
-
-    def index(self, identifier: str) -> TextIndex:
-        """Returns the index with the given identifier."""
-        project_id, index_name = self._parse_identifier(identifier)
-        if project_id is None:
-            raise ValueError("Must provide a fully qualified index identifier.")
-
-        res = list(self._api.project.list_text_indexes(project_id, name=index_name))
-        if len(res) == 0:
-            raise ValueError(f"Index not found: {project_id}.{index_name}")
-        res = res[0]
-
-        return TextIndex(self, self._spiral.get_text_index(res.id), index_name)
-
-    def list_indexes(self) -> list[TextIndexResource]:
-        project_id = self._project_id
-        if project_id is None:
-            raise ValueError("Must provide a project ID to list indexes.")
-        return list(self._api.project.list_text_indexes(project_id))
-
-    def create_text_index(
-        self,
-        identifier: str,
-        # At least one projection is required. All projections must reference the same table!
-        # NOTE(marko): Indexes are currently independent of tables.
-        #   That will likely change with the new root resource such as documents.
-        *projections: ExprLike,
-        where: ExprLike | None = None,
-        root_uri: Uri | None = None,
-        exist_ok: bool = False,
-    ) -> TextIndex:
-        """Creates a text index over the table projection.
-
-        See `se.text.field` for how to create and configure indexable fields.
-
-        Args:
-            identifier: The index identifier, in the form `project.index` or `index`.
-            projections: At least one projection expression is required.
-                All projections must reference the same table.
-            where: An optional filter expression to apply to the index.
-            root_uri: The root URI for the index.
-            exist_ok: If True, do not raise an error if the index already exists.
-        """
-        from spiral import expressions as se
-
-        project_id, index_name = self._parse_identifier(identifier)
-        if project_id is None:
-            raise ValueError("Must provide a fully qualified index identifier.")
-
-        if not projections:
-            raise ValueError("At least one projection is required.")
-        projection = se.merge(*projections)
-        if where is not None:
-            where = se.lift(where)
-
-        core_index = self._spiral.create_text_index(
-            project_id,
-            index_name,
-            projection.__expr__,
-            where.__expr__ if where else None,
-            root_uri=root_uri,
-            # TODO(marko): Validate that if an index exists, it's the same?
-            exist_ok=exist_ok,
-        )
-
-        return TextIndex(self, core_index, index_name)
-
-    def _parse_identifier(self, identifier: str) -> tuple[str | None, str]:
-        parts = identifier.split(".")
-        if len(parts) == 1:
-            return self._project_id, parts[0]
-        elif len(parts) == 2:
-            return parts[0], parts[1]
-        else:
-            raise ValueError(f"Invalid index identifier: {identifier}")
-
-    def search(
-        self,
-        *rank_by: ExprLike,
-        where: ExprLike | None = None,
-        top_k: int = 10,
-        # Do not refresh the index if freshness does not exceed the freshness window.
-        # NOTE(marko): The current implementation fails the query if the index is stale.
-        freshness_window: datetime.timedelta | None = None,
-    ) -> SearchScan:
-        """Queries the index with the given rank by and where clauses.
-
-        Rank by expressions are combined for scoring.
-            See `se.text.find` and `se.text.boost` for scoring expressions.
-        The `where` expression is used to filter the results.
-            It must return a boolean value and use only conjunctions (ANDs). Expressions in where statement
-            are considered either a `must` or `must_not` clause in search terminology.
-
-        Args:
-            rank_by: At least one rank by expression is required.
-                These expressions are used to score the results.
-            where: An optional filter expression to apply to the index.
-                It must return a boolean value and use only conjunctions (ANDs).
-            top_k: The number of top results to return.
-            freshness_window: If provided, the index will not be refreshed if its freshness does not exceed this window.
-        """
-        from spiral import expressions as se
-
-        if not rank_by:
-            raise ValueError("At least one rank by expression is required.")
-        rank_by = se.or_(*rank_by)
-        if where is not None:
-            where = se.lift(where)
-
-        if freshness_window is None:
-            freshness_window = datetime.timedelta(seconds=0)
-        freshness_window_s = int(freshness_window.total_seconds())
-
-        return SearchScan(
-            self._spiral.open_search_scan(
-                rank_by.__expr__,
-                top_k=top_k,
-                freshness_window_s=freshness_window_s,
-                filter=where.__expr__ if where else None,
-            )
-        )

spiral/indexes/index.py

@@ -1,28 +0,0 @@
-from typing import TYPE_CHECKING
-
-from spiral.core.index import TextIndex as CoreTextIndex
-from spiral.expressions import Expr
-
-if TYPE_CHECKING:
-    from spiral.indexes import Indexes
-
-
-class TextIndex(Expr):
-    def __init__(self, indexes: "Indexes", index: CoreTextIndex, name: str):
-        super().__init__(index.__expr__)
-
-        self._indexes = indexes
-        self._index = index
-        self._name = name
-
-    @property
-    def client(self) -> "Indexes":
-        return self._indexes
-
-    @property
-    def index_id(self) -> str:
-        return self._index.id
-
-    @property
-    def name(self) -> str:
-        return self._name

spiral/indexes/scan.py

@@ -1,22 +0,0 @@
-import pyarrow as pa
-
-from spiral.core.index import SearchScan as CoreSearchScan
-from spiral.settings import CI, DEV
-
-
-class SearchScan:
-    def __init__(self, scan: CoreSearchScan):
-        self._scan = scan
-
-    def to_record_batches(self) -> pa.RecordBatchReader:
-        """Read all results as a record batch reader."""
-        return self._scan.to_record_batches()
-
-    def to_table(self) -> pa.Table:
-        """Read all results as a table."""
-        # NOTE: Evaluates fully on Rust side which improved debuggability.
-        if DEV and not CI:
-            rb = self._scan.to_record_batch()
-            return pa.Table.from_batches([rb])
-
-        return self.to_record_batches().read_all()

spiral/protogen/_/spiral/table/__init__.py

@@ -1,22 +0,0 @@
-# Generated by the protocol buffer compiler.  DO NOT EDIT!
-# sources: spiral/table/statistics.proto
-# plugin: python-betterproto
-# This file has been @generated
-
-from dataclasses import dataclass
-
-import betterproto
-
-
-@dataclass(eq=False, repr=False)
-class ApproximateSetMembership(betterproto.Message):
-    bloom_filter: "BloomFilter" = betterproto.message_field(
-        2, group="membership_strategy"
-    )
-
-
-@dataclass(eq=False, repr=False)
-class BloomFilter(betterproto.Message):
-    bit_vec: bytes = betterproto.bytes_field(1)
-    bitmap_bits: int = betterproto.uint64_field(2)
-    k_num: int = betterproto.uint32_field(3)