pyspiral 0.8.9__cp311-abi3-macosx_11_0_arm64.whl → 0.9.9__cp311-abi3-macosx_11_0_arm64.whl

spiral/cli/app.py

@@ -20,6 +20,7 @@ from spiral.cli import (
     tables,
     telemetry,
     text,
+    transactions,
     workloads,
 )
 from spiral.settings import LOG_DIR, PACKAGE_NAME
@@ -77,18 +78,23 @@ app.add_typer(projects.app, name="projects")
 app.add_typer(fs.app, name="fs")
 app.add_typer(workloads.app, name="workloads")
 app.add_typer(tables.app, name="tables")
-app.add_typer(key_spaces.app, name="ks")
-app.add_typer(text.app, name="text")
-app.add_typer(telemetry.app, name="telemetry")
 app.add_typer(iceberg.app, name="iceberg")
+app.add_typer(telemetry.app, name="telemetry")
 app.command("login")(login.command)
+app.command("whoami")(login.whoami)
 app.command("console")(console.command)
 
 
 # Register unless we're building docs. Because Typer docs command does not skip hidden commands...
 if not bool(os.environ.get("SPIRAL_DOCS", False)):
     app.add_typer(admin.app, name="admin", hidden=True)
-    app.command("whoami", hidden=True)(login.whoami)
+
+    # Hide some "beta" commands.
+    app.add_typer(transactions.app, name="txn", hidden=True)
+    app.add_typer(key_spaces.app, name="ks", hidden=True)
+    app.add_typer(text.app, name="text", hidden=True)
+
+    # Hidden because there isn't really a logout. This just removes the stored credentials.
     app.command("logout", hidden=True)(login.logout)
 
 

spiral/cli/chooser.py

@@ -0,0 +1,30 @@
+from collections.abc import Callable
+from typing import Any, cast
+
+import questionary
+import typer
+from questionary import Choice
+
+
+def choose(title: str, choices: list[Choice] | list[str]) -> Any:
+    """Interactively select one of the choices exiting the process if ctrl-c is pressed."""
+    if choices and isinstance(choices[0], Choice):
+        for choice in choices:
+            assert isinstance(choice, Choice)
+            choice.value = (choice.value,)
+    else:
+        choices = cast(list[str], choices)
+        choices = [Choice(choice, value=(choice,)) for choice in choices]
+
+    maybe_selection = questionary.select(title, choices=choices).ask()
+    if maybe_selection is None:
+        raise typer.Exit(2)
+    return maybe_selection[0]
+
+
+def input_text(message: str, validate: Callable[[str], bool | str]) -> str:
+    """Interactively receive string input which passes the validation exiting the process if ctrl-c is pressed."""
+    maybe_text = questionary.text(message, validate=validate).ask()
+    if maybe_text is None:
+        raise typer.Exit(2)
+    return maybe_text[0]

spiral/cli/fs.py

@@ -13,7 +13,8 @@ from spiral.api.filesystems import (
     UpstreamFileSystem,
 )
 from spiral.cli import CONSOLE, AsyncTyper, state
-from spiral.cli.types import ProjectArg, ask_project
+from spiral.cli.chooser import choose
+from spiral.cli.types_ import ProjectArg, ask_project
 
 app = AsyncTyper(short_help="File Systems.")
 
@@ -26,7 +27,7 @@ def show(project: ProjectArg):
 
 def ask_provider():
     res = state.spiral.api.file_systems.list_providers()
-    return questionary.select("Select a file system provider", choices=res).ask()
+    return choose("Select a file system provider", choices=res)
 
 
 @app.command(help="Update a project's default file system.")

spiral/cli/iceberg.py

@@ -6,7 +6,7 @@ import typer
 from typer import Argument
 
 from spiral.cli import CONSOLE, ERR_CONSOLE, AsyncTyper, state
-from spiral.cli.types import ProjectArg
+from spiral.cli.types_ import ProjectArg
 
 app = AsyncTyper(short_help="Apache Iceberg Catalog.")
 

spiral/cli/key_spaces.py

@@ -1,6 +1,5 @@
 from typing import Annotated
 
-import questionary
 import rich
 import typer
 from questionary import Choice
@@ -11,7 +10,8 @@ from spiral.api.projects import KeySpaceIndexResource
 from spiral.api.types import IndexId
 from spiral.api.workers import ResourceClass
 from spiral.cli import CONSOLE, AsyncTyper, state
-from spiral.cli.types import ProjectArg
+from spiral.cli.chooser import choose
+from spiral.cli.types_ import ProjectArg
 
 app = AsyncTyper(short_help="Key Space Indexes.")
 
@@ -23,10 +23,10 @@ def ask_index(project_id, title="Select an index"):
         CONSOLE.print("[red]No indexes found[/red]")
         raise typer.Exit(1)
 
-    return questionary.select(
+    return choose(
         title,
         choices=[Choice(title=index.name, value=index.id) for index in sorted(indexes, key=lambda t: (t.name, t.id))],
-    ).ask()
+    )
 
 
 def get_index_id(

spiral/cli/orgs.py

@@ -6,7 +6,7 @@ from rich.table import Table
 
 from spiral.api.organizations import InviteUserRequest, OrgRole, PortalLinkIntent, PortalLinkRequest
 from spiral.cli import CONSOLE, ERR_CONSOLE, AsyncTyper, state
-from spiral.cli.types import OrganizationArg
+from spiral.cli.types_ import OrganizationArg
 from spiral.core.authn import DeviceCodeAuth
 
 app = AsyncTyper(short_help="Org admin.")

spiral/cli/projects.py

@@ -21,14 +21,14 @@ from spiral.api.projects import (
     WorkloadPrincipalConditions,
 )
 from spiral.cli import CONSOLE, AsyncTyper, printer, state
-from spiral.cli.types import ProjectArg
+from spiral.cli.types_ import ProjectArg
 
 app = AsyncTyper(short_help="Projects and grants.")
 
 
 @app.command(help="List projects.")
 def ls():
-    projects = list(state.spiral.api.projects.list())
+    projects = list(sorted(state.spiral.api.projects.list(), key=lambda p: p.id))
     CONSOLE.print(printer.table_of_models(Project, projects))
 
 

spiral/cli/tables.py

@@ -11,8 +11,9 @@ from typer import Argument, Option
 from spiral import Spiral
 from spiral.api.projects import TableResource
 from spiral.cli import CONSOLE, ERR_CONSOLE, AsyncTyper, state
-from spiral.cli.types import ProjectArg
-from spiral.debug.manifests import display_manifests
+from spiral.cli.chooser import choose, input_text
+from spiral.cli.types_ import ProjectArg
+from spiral.debug.manifests import display_manifests, display_scan_manifests
 from spiral.table import Table
 
 app = AsyncTyper(short_help="Spiral Tables.")
@@ -25,13 +26,13 @@ def ask_table(project_id: str, title: str = "Select a table") -> str:
         ERR_CONSOLE.print("No tables found")
         raise typer.Exit(1)
 
-    return questionary.select(  # pyright: ignore[reportAny]
+    return choose(  # pyright: ignore[reportAny]
         title,
         choices=[
             Choice(title=f"{table.dataset}.{table.table}", value=f"{table.dataset}.{table.table}")
             for table in sorted(tables, key=lambda t: (t.dataset, t.table))
         ],
-    ).ask()
+    )
 
 
 def get_table(
@@ -89,7 +90,7 @@ def validate_non_empty_str(text: str) -> bool | str:
 
 
 def get_string(message: str, validate: Callable[[str], bool | str] = validate_non_empty_str) -> str:
-    return questionary.text(message, validate=validate).ask()
+    return input_text(message, validate=validate)
 
 
 @app.command(help="Move table to a different dataset.")
@@ -182,27 +183,53 @@ def flush(
     CONSOLE.print(f"Flushed WAL for table {identifier} in project {project}.")
 
 
-@app.command(help="Display all manifests.")
+@app.command(help="Truncate column group metadata.")
+def truncate(
+    project: ProjectArg,
+    table: Annotated[str | None, Option(help="Table name.")] = None,
+    dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
+):
+    identifier, t = get_table(project, table, dataset)
+
+    # Ask for confirmation
+    confirm = questionary.confirm(
+        f"Are you sure you want to truncate metadata for table '{identifier}'? This will break as-of queries."
+    ).ask()
+    if not confirm:
+        CONSOLE.print("Aborted.")
+        raise typer.Exit(0)
+
+    state.spiral.internal.truncate_metadata(t.core)
+    CONSOLE.print(f"Truncated metadata for table {identifier} in project {project}.")
+
+
+@app.command(help="Display all fragments from metadata.", deprecated="Use 'fragments' instead.")
 def manifests(
     project: ProjectArg,
     table: Annotated[str | None, Option(help="Table name.")] = None,
     dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
     max_rows: Annotated[int | None, Option(help="Maximum number of rows to show per manifest.")] = None,
 ):
-    _, t = get_table(project, table, dataset)
-    s = t.snapshot()
+    fragments(project, table, dataset, max_rows)
 
-    key_space_state = state.spiral.internal.key_space_state(s.core)  # pyright: ignore[reportPrivateUsage]
-    key_space_manifest = key_space_state.manifest
 
-    column_groups_states = state.spiral.internal.column_groups_states(s.core, key_space_state)  # pyright: ignore[reportPrivateUsage]
-    display_manifests(
-        key_space_manifest, [(x.column_group, x.manifest) for x in column_groups_states], t.key_schema, max_rows
-    )
+@app.command(help="Display all fragments from metadata.")
+def fragments(
+    project: ProjectArg,
+    table: Annotated[str | None, Option(help="Table name.")] = None,
+    dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
+    max_rows: Annotated[int | None, Option(help="Maximum number of fragments to show per manifest.")] = None,
+):
+    _, t = get_table(project, table, dataset)
+    s = t.snapshot()
+    cgs = s.core.column_groups()
+    key_space_manifest = state.spiral.internal.key_space_manifest(s.core)
+    column_group_manifests = [(cg, state.spiral.internal.column_group_manifest(s.core, cg)) for cg in cgs]
+    display_manifests(key_space_manifest, column_group_manifests, t.key_schema, max_rows)
 
 
-@app.command(help="Visualize the scan of a given column group.")
-def debug_scan(
+@app.command(help="Display the fragments used in a scan of a given column group.")
+def fragments_scan(
     project: ProjectArg,
     table: Annotated[str | None, Option(help="Table name.")] = None,
     dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
@@ -210,11 +237,11 @@ def debug_scan(
 ):
     _, t = get_table(project, table, dataset)
     scan = state.spiral.scan(t[column_group] if column_group != "." else t)
-    scan._debug()  # pyright: ignore[reportPrivateUsage]
+    display_scan_manifests(scan.core)
 
 
-@app.command(help="Display the manifests for a scan of a given column group.")
-def dump_scan(
+@app.command(help="Visualize the scan of a given column group.")
+def debug_scan(
     project: ProjectArg,
     table: Annotated[str | None, Option(help="Table name.")] = None,
     dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
@@ -222,4 +249,4 @@ def dump_scan(
 ):
     _, t = get_table(project, table, dataset)
     scan = state.spiral.scan(t[column_group] if column_group != "." else t)
-    scan._dump_manifests()  # pyright: ignore[reportPrivateUsage]
+    scan._debug()  # pyright: ignore[reportPrivateUsage]

spiral/cli/telemetry.py

@@ -1,4 +1,4 @@
-import pyperclip
+import typer
 
 from spiral.api.telemetry import IssueExportTokenResponse
 from spiral.cli import CONSOLE, AsyncTyper, state
@@ -7,11 +7,18 @@ app = AsyncTyper(short_help="Client-side telemetry.")
 
 
 @app.command(help="Issue new telemetry export token.")
-def export():
+def export(
+    print_token: bool = typer.Option(False, "--print", help="Print the token instead of copying to clipboard."),
+):
     res: IssueExportTokenResponse = state.spiral.api.telemetry.issue_export_token()
 
-    command = f"export SPIRAL_OTEL_TOKEN={res.token}"
-    pyperclip.copy(command)
+    if print_token:
+        print(res.token)
+    else:
+        import pyperclip
 
-    CONSOLE.print("Export command copied to clipboard! Paste and run to set [green]SPIRAL_OTEL_TOKEN[/green].")
-    CONSOLE.print("[dim]Token is valid for 1h.[/dim]")
+        command = f"export SPIRAL_OTEL_TOKEN={res.token}"
+        pyperclip.copy(command)
+
+        CONSOLE.print("Export command copied to clipboard! Paste and run to set [green]SPIRAL_OTEL_TOKEN[/green].")
+        CONSOLE.print("[dim]Token is valid for 1h.[/dim]")

spiral/cli/text.py

@@ -1,6 +1,5 @@
 from typing import Annotated
 
-import questionary
 import rich
 import typer
 from questionary import Choice
@@ -11,7 +10,8 @@ from spiral.api.text_indexes import CreateWorkerRequest, SyncIndexRequest
 from spiral.api.types import IndexId
 from spiral.api.workers import CPU, GcpRegion, Memory, ResourceClass
 from spiral.cli import CONSOLE, ERR_CONSOLE, AsyncTyper, state
-from spiral.cli.types import ProjectArg
+from spiral.cli.chooser import choose
+from spiral.cli.types_ import ProjectArg
 
 app = AsyncTyper(short_help="Text Indexes.")
 
@@ -23,10 +23,10 @@ def ask_index(project_id, title="Select an index"):
         ERR_CONSOLE.print("No indexes found")
         raise typer.Exit(1)
 
-    return questionary.select(
+    return choose(
         title,
         choices=[Choice(title=index.name, value=index.id) for index in sorted(indexes, key=lambda t: (t.name, t.id))],
-    ).ask()
+    )
 
 
 def get_index_id(

spiral/cli/transactions.py

@@ -0,0 +1,84 @@
+from datetime import UTC
+from typing import Annotated
+
+import questionary
+import rich
+import rich.table
+import typer
+from typer import Option
+
+from spiral.cli import CONSOLE, AsyncTyper, state
+from spiral.cli.tables import get_table
+from spiral.cli.types_ import ProjectArg
+
+app = AsyncTyper(short_help="Table Transactions.")
+
+
+@app.command("ls", help="List transactions for a table.")
+def ls(
+    project: ProjectArg,
+    table: Annotated[str | None, Option(help="Table name.")] = None,
+    dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
+    since: Annotated[
+        int | None, Option(help="List transactions committed after this timestamp (microseconds).")
+    ] = None,
+):
+    from datetime import datetime
+
+    identifier, t = get_table(project, table, dataset)
+
+    # Get transactions from the API
+    transactions = state.spiral.api.tables.list_transactions(
+        table_id=t.table_id,
+        since=since,
+    )
+
+    if not transactions:
+        CONSOLE.print("No transactions found.")
+        return
+
+    # Create a rich table to display transactions
+    rich_table = rich.table.Table(
+        "Table ID", "Tx Index", "Committed At", "Operations", title=f"Transactions for {identifier}"
+    )
+
+    for txn in transactions:
+        # Convert timestamp to readable format
+        dt = datetime.fromtimestamp(txn.committed_at / 1_000_000, tz=UTC)
+        committed_str = dt.strftime("%Y-%m-%d %H:%M:%S UTC")
+
+        # Count operations by type
+        op_counts: dict[str, int] = {}
+        for op in txn.operations:
+            # Operation is a dict with a single key indicating the type
+            op_type = op["type"]
+            op_counts[op_type] = op_counts.get(op_type, 0) + 1
+
+        op_summary = ", ".join(f"{count}x {op_type}" for op_type, count in op_counts.items())
+
+        rich_table.add_row(t.table_id, str(txn.txn_idx), committed_str, op_summary)
+
+    CONSOLE.print(rich_table)
+
+
+@app.command("revert", help="Revert a transaction by table ID and transaction index.")
+def revert(
+    table_id: str,
+    txn_idx: int,
+):
+    # Ask for confirmation
+    CONSOLE.print(
+        "[yellow]Only transactions still in WAL can be reverted. "
+        "It is recommended to only revert the latest transaction. "
+        "Reverting historical transactions may break a table.[/yellow]"
+    )
+    confirm = questionary.confirm(
+        f"Are you sure you want to revert transaction {txn_idx} for table '{table_id}'?\n"
+    ).ask()
+    if not confirm:
+        CONSOLE.print("Aborted.")
+        raise typer.Exit(0)
+
+    # Revert the transaction
+    state.spiral.api.tables.revert_transaction(table_id=table_id, txn_idx=txn_idx)
+    CONSOLE.print(f"Successfully reverted transaction {txn_idx} for table {table_id}.")

spiral/cli/{types.py → types_.py}

@@ -1,12 +1,12 @@
 from typing import Annotated
 
-import questionary
 import typer
 from questionary import Choice
 from typer import Argument
 
 from spiral.api.types import OrgId, ProjectId
 from spiral.cli import ERR_CONSOLE, state
+from spiral.cli.chooser import choose
 
 
 def ask_project(title="Select a project"):
@@ -16,13 +16,13 @@ def ask_project(title="Select a project"):
         ERR_CONSOLE.print("No projects found")
         raise typer.Exit(1)
 
-    return questionary.select(
+    return choose(
         title,
-        choices=[
+        [
             Choice(title=f"{project.id} - {project.name}" if project.name else project.id, value=project.id)
             for project in projects
         ],
-    ).ask()
+    )
 
 
 ProjectArg = Annotated[ProjectId, Argument(help="Project ID", show_default=False, default_factory=ask_project)]
@@ -35,7 +35,7 @@ def _org_default():
         ERR_CONSOLE.print("No organizations found")
         raise typer.Exit(1)
 
-    return questionary.select(
+    return choose(
         "Select an organization",
         choices=[
             Choice(
@@ -44,7 +44,7 @@ def _org_default():
             )
             for m in memberships
         ],
-    ).ask()
+    )
 
 
 OrganizationArg = Annotated[OrgId, Argument(help="Organization ID", show_default=False, default_factory=_org_default)]

spiral/cli/workloads.py

@@ -1,7 +1,6 @@
 from typing import Annotated
 
 import pyperclip
-import questionary
 from questionary import Choice
 from typer import Argument, Option
 
@@ -12,7 +11,8 @@ from spiral.api.workloads import (
     Workload,
 )
 from spiral.cli import CONSOLE, ERR_CONSOLE, AsyncTyper, printer, state
-from spiral.cli.types import ProjectArg
+from spiral.cli.chooser import choose
+from spiral.cli.types_ import ProjectArg
 
 app = AsyncTyper()
 
@@ -54,14 +54,14 @@ def issue_creds(
         CONSOLE.print(f"[green]SPIRAL_CLIENT_SECRET[/green] {res.client_secret}")
     else:
         while True:
-            choice = questionary.select(
+            choice = choose(
                 "What would you like to do with the secret? You will not be able to see this secret again!",
                 choices=[
                     Choice(title="Copy to clipboard", value=1),
                     Choice(title="Print to console", value=2),
                     Choice(title="Exit", value=3),
                 ],
-            ).ask()
+            )
 
             if choice == 1:
                 pyperclip.copy(res.client_secret)

spiral/client.py

@@ -8,7 +8,7 @@ import pyarrow as pa
 from spiral.api import SpiralAPI
 from spiral.api.projects import CreateProjectRequest, CreateProjectResponse
 from spiral.core.authn import Authn
-from spiral.core.client import Internal, Shard
+from spiral.core.client import Internal, KeyColumns, Shard
 from spiral.core.client import Spiral as CoreSpiral
 from spiral.core.config import ClientSettings
 from spiral.datetime_ import timestamp_micros
@@ -157,6 +157,7 @@ class Spiral:
         where: ExprLike | None = None,
         asof: datetime | int | None = None,
         shard: Shard | None = None,
+        hide_progress_bar: bool = False,
     ) -> Scan:
         """Starts a read transaction on the Spiral.
 
@@ -167,7 +168,61 @@ class Spiral:
             shard: if provided, opens the scan only for the given shard.
                 While shards can be provided when executing the scan, providing a shard here
                 optimizes the scan planning phase and can significantly reduce metadata download.
+            hide_progress_bar: if True, disables the progress bar during scan building.
         """
+        return self._scan_internal(
+            *projections,
+            where=where,
+            asof=asof,
+            shard=shard,
+            hide_progress_bar=hide_progress_bar,
+        )
+
+    def scan_keys(
+        self,
+        *projections: ExprLike,
+        where: ExprLike | None = None,
+        asof: datetime | int | None = None,
+        shard: Shard | None = None,
+        hide_progress_bar: bool = False,
+    ) -> Scan:
+        """Starts a keys-only read transaction on the Spiral.
+
+        To determine which keys are present in at least one column group of the table, key scan the
+        table itself:
+
+        ```
+        sp.scan_keys(table)
+        ```
+
+        Args:
+            projections: scan the keys of the column groups referenced by these expressions.
+            where: a query expression to apply to the data.
+            asof: execute the scan on the version of the table as of the given timestamp.
+            shard: if provided, opens the scan only for the given shard.
+                While shards can be provided when executing the scan, providing a shard here
+                optimizes the scan planning phase and can significantly reduce metadata download.
+            hide_progress_bar: if True, disables the progress bar during scan building.
+
+        """
+        return self._scan_internal(
+            *projections,
+            where=where,
+            asof=asof,
+            shard=shard,
+            hide_progress_bar=hide_progress_bar,
+            key_columns=KeyColumns.Only,
+        )
+
+    def _scan_internal(
+        self,
+        *projections: ExprLike,
+        where: ExprLike | None = None,
+        asof: datetime | int | None = None,
+        shard: Shard | None = None,
+        key_columns: KeyColumns | None = None,
+        hide_progress_bar: bool = False,
+    ) -> Scan:
         from spiral import expressions as se
 
         if isinstance(asof, datetime):
@@ -182,7 +237,14 @@ class Spiral:
 
         return Scan(
             self,
-            self.core.scan(projection.__expr__, filter=where.__expr__ if where else None, asof=asof, shard=shard),
+            self.core.scan(
+                projection.__expr__,
+                filter=where.__expr__ if where else None,
+                asof=asof,
+                shard=shard,
+                key_columns=key_columns,
+                progress=(not hide_progress_bar),
+            ),
         )
 
     # TODO(marko): This should be query, and search should be query + scan.
@@ -223,16 +285,16 @@ class Spiral:
             freshness_window_s=freshness_window_s,
         )
 
-    def resume_scan(self, state_json: str) -> Scan:
-        """Resumes a previously started scan using its scan state.
+    def resume_scan(self, context_bytes: bytes) -> Scan:
+        """Resumes a previously started scan using its scan context.
 
         Args:
-            state_json: The scan state returned by a previous scan.
+            context_bytes: The compressed scan context returned by a previous scan.
         """
-        from spiral.core.table import ScanState
+        from spiral.core.table import ScanContext
 
-        state = ScanState.from_json(state_json)
-        return Scan(self, self.core.load_scan(state))
+        context = ScanContext.from_bytes_compressed(context_bytes)
+        return Scan(self, self.core.load_scan(context))
 
     def compute_shards(
         self,