pyspiral 0.4.4__cp310-abi3-macosx_11_0_arm64.whl → 0.5.0__cp310-abi3-macosx_11_0_arm64.whl

{pyspiral-0.4.4.dist-info → pyspiral-0.5.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyspiral
-Version: 0.4.4
+Version: 0.5.0
 Classifier: Intended Audience :: Science/Research
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python
@@ -13,22 +13,23 @@ Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Rust
 Classifier: License :: Other/Proprietary License
 Requires-Dist: betterproto==2.0.0b7
+Requires-Dist: datasets>=4.0.0
 Requires-Dist: google-re2>=1.1.20240702
 Requires-Dist: grpclib>=0.4.7
 Requires-Dist: hishel>=0.0.30
 Requires-Dist: httpx>=0.27.0
+Requires-Dist: nanoid>=2.0.0
 Requires-Dist: numpy>=2
 Requires-Dist: pyarrow>=21.0.0
 Requires-Dist: pydantic-settings>=2.3.4
 Requires-Dist: pydantic[email]>=2.5.3
 Requires-Dist: pyjwt[crypto]>=2.9.0
+Requires-Dist: pyperclip>=1.9.0
 Requires-Dist: questionary>=2.0.1
+Requires-Dist: sqlglot[rs]>=25.25.1
 Requires-Dist: tqdm>=4.66.5
 Requires-Dist: typer>=0.16
 Requires-Dist: xxhash>=3.4.1
-Requires-Dist: nanoid>=2.0.0
-Requires-Dist: sqlglot[rs]>=25.25.1
-Requires-Dist: pyperclip>=1.9.0
 Requires-Dist: polars>=1.31.0 ; extra == 'polars'
 Requires-Dist: duckdb>=1.3.2 ; extra == 'duckdb'
 Requires-Dist: pyiceberg>=0.9.1 ; extra == 'pyiceberg'

{pyspiral-0.4.4.dist-info → pyspiral-0.5.0.dist-info}/RECORD

@@ -1,17 +1,19 @@
-pyspiral-0.4.4.dist-info/METADATA,sha256=lVusDQ4LmIf3tRllkl2_hUNn1y0P_yMfNKhkRbpNLW0,1610
-pyspiral-0.4.4.dist-info/WHEEL,sha256=Mdosfxua6Dx1zYgObRH97e3wyiELqBbLtoRJj4RUSQE,103
-pyspiral-0.4.4.dist-info/entry_points.txt,sha256=uft7u-a6g40NLt4Q6BleWbK4NY0M8nZuYPpP8DV0EOk,45
+pyspiral-0.5.0.dist-info/METADATA,sha256=J_bR2LwG0i6M8wB0-60AXJm8MutIfqs9QUbaINmBeTI,1641
+pyspiral-0.5.0.dist-info/WHEEL,sha256=Mdosfxua6Dx1zYgObRH97e3wyiELqBbLtoRJj4RUSQE,103
+pyspiral-0.5.0.dist-info/entry_points.txt,sha256=uft7u-a6g40NLt4Q6BleWbK4NY0M8nZuYPpP8DV0EOk,45
 spiral/__init__.py,sha256=Jv1vbcnnmcTsBLN5mSNjnX3ae4C_mgojXDSBFaqIhN0,208
-spiral/_lib.abi3.so,sha256=5-2gLMM7XOEgX6lXbIBu-_ucQsC5zUZWqZaH7oXkdSE,59883120
-spiral/adbc.py,sha256=HcvR60uQeEK2oggSAK6y5VYtIrACIiCQ-85MEf18EZc,14199
-spiral/api/__init__.py,sha256=_7BS1RhqEFjnt3XwFWZNCHVEQeSKpezPevAiGCsvDbE,1776
+spiral/_lib.abi3.so,sha256=4J6RSb9LxUWqUnRZeyccSneyjqEFfIrJ6lMCnsNUqbc,60398416
+spiral/adbc.py,sha256=RIIWBew7zPoQa_h3I-A-nX9cUMDM3D3Je0mqE9aDX9k,14885
+spiral/api/__init__.py,sha256=nzZK3r1K2GSaqaLoIba4WuyUyG7ApjLZ4fyJovT1unA,2000
 spiral/api/admin.py,sha256=A1iVR1XYJSObZivPAD5UzmPuMgupXc9kaHNYYa_kwfs,585
-spiral/api/client.py,sha256=9-L6T8niQAXo90jRxllJD4hXXmcGfHj7CW9X3XTYa5Q,4551
+spiral/api/client.py,sha256=XQaOd1DgLTDOxW_uZUeTX5UCnUpPV1unjxbzWLlPJaM,4650
 spiral/api/filesystems.py,sha256=EA4iqhTeaIlvObvEUxHmZl0pQ24IOxUVWM3GPhFLw8o,4969
 spiral/api/organizations.py,sha256=B-8zZ7lFJANGK7dUNbo_aU-cgI959JBP9VcWb6wdgi0,1895
-spiral/api/projects.py,sha256=JBGof9A2Ivasu2jrULMjHBwlna0M8WRrTNqU-Es4GJ8,5673
+spiral/api/projects.py,sha256=mvyp4tnUhItNnPnyWsj0WrSf2ca3mggL7KjXs5Fllco,5671
 spiral/api/telemetry.py,sha256=tfdA3E_EWJwFVxkQfkm8tiYGRubnx2LuE5nbfsk1oG4,474
-spiral/api/types.py,sha256=zx-BRKsi1GHg9aL9gMUaVQWYYMXJcP0A8OQUc7jSIAc,653
+spiral/api/text_indexes.py,sha256=f8AcrYaBKsaka5A4okuExl96A-2rBbsgru98l_xogwo,1826
+spiral/api/types.py,sha256=lGdiKViRgIEJXD2ubwnyEIEwHkfRumlZjVEaHMV3Tm8,682
+spiral/api/workers.py,sha256=0wZNUHMioDT53P1OBJfpjyDfIodHwwT6858z2IlRIM4,636
 spiral/api/workloads.py,sha256=XAyXV7vgZcoyyoPoGvOT4jTpyFKFMvrrAfhL6d1h1kE,1748
 spiral/arrow_.py,sha256=T1LZ7bh9aMDbXfpUsf0dR0E1roTQyAYSgZ2mL4s8J_4,7681
 spiral/cli/__init__.py,sha256=ooAFz_iCpVCKHE0TiVElIynbP2PtTgD9cUw46Vh1lcw,2145
@@ -23,21 +25,24 @@ spiral/cli/fs.py,sha256=dVPoAoAbuQ9yJlfI-JiFgS9VdnPmeBMygVHgehJRj34,4367
 spiral/cli/iceberg/__init__.py,sha256=IQV_gwCFSj6Ubxs58VM9Pal1ymgG2bxdDgOPuk9E5bs,214
 spiral/cli/iceberg/namespaces.py,sha256=x9pvHlcXtcATYYjqimHa6CtkyL3taQUJ--ni_Bfoemc,1510
 spiral/cli/iceberg/tables.py,sha256=nSR4-t54otJfCmubB6vXnbOkbqPVGV0sHBlc-t9cIVg,1930
-spiral/cli/indexes/__init__.py,sha256=-USfxCIdckzZKBNQ-DXqe3V5ttWVo_Fsa1Mfcx5hdIw,467
+spiral/cli/indexes/__init__.py,sha256=yNMBZh3kAz1NXsridvbY4-4jNIJAgntBhDnu1EfaZTI,1291
+spiral/cli/indexes/args.py,sha256=B4zAFnVZKPHq-_z8qnHYwgs1v4c-w_iypbA5MdXL67s,1139
+spiral/cli/indexes/workers.py,sha256=mu-7-Asz1txNtCkoei6q-m-lE5YhLUjCDwstKJcew40,2169
 spiral/cli/login.py,sha256=InKMnpV8NATW5RPgB3ZL-DSVPzUuUByyK4Fx7pZEgfg,607
 spiral/cli/orgs.py,sha256=V-4ZTT3FwFQLcs1-BenC8uCgvWOJcxkZPSdCPfsexhc,2848
 spiral/cli/printer.py,sha256=W83KAE-7meoDD1yRltLQrZqrA2olGapBGy_2USWkY08,1778
 spiral/cli/projects.py,sha256=TKXu_VzkIUccwXzdlg-wQMkrB-Py33g052NrbuJx-D4,5096
 spiral/cli/state.py,sha256=10wTIVQ0SJkY67Z6-KQ1LFlt3aVIPmZhoHFdTwp4kNA,130
-spiral/cli/tables/__init__.py,sha256=lkGLDeU28IVnuxJdlYSUh6QSB9fQ4_1MeZJL73iXcHo,3660
+spiral/cli/tables/__init__.py,sha256=DTxviiflWZkUDmPhuGzXmhW2mNRgPNkJcaP7olFtXpc,2678
+spiral/cli/tables/args.py,sha256=bxKQoJuWhCTHlDYz_WRQ6_Kp9XgRBH1UIMgUMzlxliA,1262
 spiral/cli/telemetry.py,sha256=ABDCyV5QJGOIJp4AxvK0LG5xNPIysP37K5haL38T7P4,586
 spiral/cli/types.py,sha256=YG1eHhRLaqlVU_18DQBuF_YMsabhMZLBY0V9CvbSxjY,1369
 spiral/cli/workloads.py,sha256=SbxgwiBlX1AuqpOLV3gs7DFkH-Tbeend7qJTwq0Je84,1994
 spiral/client.py,sha256=K-OuMOTgYxOA9vef5jSANjmPRBfGrzQ65fg6Fd-rHMY,2683
 spiral/core/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 spiral/core/client/__init__.pyi,sha256=Tn1OJmkO1rQUsPE9BtfEyxIjoife6s16qOd8XiyHi2c,3475
-spiral/core/index/__init__.pyi,sha256=NPOG1ztFO6siBGpmJU3boRzX26xfxw--2TiCydosGvo,314
-spiral/core/table/__init__.pyi,sha256=agrxN1dYx--dte_edQOKgAXT8yPDeh_cHA8dYAOodbE,3290
+spiral/core/index/__init__.pyi,sha256=MBq-jBuTmBreYMJ4AJFAe4e-ByRMM7JVssuEJMLVfQs,131
+spiral/core/table/__init__.pyi,sha256=XhN9xpDdBnugRhtiP9ThC3DSADoLMDL9lxSjjY7fJ28,3296
 spiral/core/table/manifests/__init__.pyi,sha256=3V59-K1qr1z2dGfgRKXaHSVheK8NNw8Q8PFhfbeQd_4,1065
 spiral/core/table/metastore/__init__.pyi,sha256=dMqySDnsjPUTBuFU2MaQGyocKEoGkWpeTQmUP2iIKbc,1880
 spiral/core/table/spec/__init__.pyi,sha256=D4GQp9RWwyLKTlRW7eDXcQE-xA5rF2iBcXZ8y7b48EE,5595
@@ -61,7 +66,7 @@ spiral/iceberg/__init__.py,sha256=jSIlTxWauAbJV5gsWglZisFbnfNNzLYN90scoYcdWzc,65
 spiral/iceberg/client.py,sha256=E6FyE_h2HLgDW1cAFg1XgglJr6rbVOCWjRtRmqoMVkM,1003
 spiral/indexes/__init__.py,sha256=TXLQ-_3xso3lFIp2lM58_ip9OPNwPKFv1FdsWiUF-d8,178
 spiral/indexes/client.py,sha256=NsFBILEHMjyCUruFrUEKucRQRrN4OvqgbL4pmzWs07g,5600
-spiral/indexes/index.py,sha256=4CmSFlZYp46B2CjqtiyZ7VF5EH3duiutz3nWFnyApLA,973
+spiral/indexes/index.py,sha256=TJB1-hSiPNrzJ_VYFTZGHvIQ_vmoILPK7tyi_9oS1nA,638
 spiral/indexes/scan.py,sha256=B2m-UgNuawNB90HXK33GTQfMy2WLdNNxiiB6cIjFW2Y,697
 spiral/project.py,sha256=0uJ1Jb88Ie-cCNnSdX3QfFtCUqrjLka4zCm_TxCpVak,1189
 spiral/protogen/_/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -83,16 +88,16 @@ spiral/server.py,sha256=ztBmB5lBnUz-smQxR_tC8AI5SOhz17wH0MI3GuzDUdM,600
 spiral/settings.py,sha256=PIQV2ljtB3pEOWoMRVSRzSGJNrXviO2JBgZ5ZY_Nq2E,2794
 spiral/substrait_.py,sha256=2BYvwFGcCwJ0JXNhXOLdPuhM1PqFyaeSqFpQCtv-M4E,12581
 spiral/tables/__init__.py,sha256=iiP7BkHA117em37_e75jtdvoZC10xCXtld18gRnPbTw,430
-spiral/tables/client.py,sha256=l_wJJRf3BPD5lg4Q1Ll2lAqQIuBCnKwC6JtsAui91Tc,4915
+spiral/tables/client.py,sha256=MVgfeVF7P4kXnjOTQExp0VmywyuuRQ2IZVxdwVY3xgk,5015
 spiral/tables/dataset.py,sha256=DuHeKVCJfXLsbxmde9QW6yvesW5uhswG6qAxV5X0ZgA,7890
 spiral/tables/debug/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-spiral/tables/debug/manifests.py,sha256=E_-DiMBg2EPL97cl9hLWhiqEsFtjEBgh_C7jZy8EWYc,2594
+spiral/tables/debug/manifests.py,sha256=t7E0AchHrzOv9vAQpE77Qp3rLc5VTzRRxlByt5OWnUM,2596
 spiral/tables/debug/metrics.py,sha256=XdRDcjggtsLNGCAjam6IxG9072pz_d2C8iLApNRFUtk,2044
-spiral/tables/debug/scan.py,sha256=-IWX_UjO4QP9Hj7PtZ1rLlbswJcryOin56GT-exqFm4,8942
+spiral/tables/debug/scan.py,sha256=EEG2gzbBpUyrtk4jQXh-ENk8aySrKf0CZMym_wHuWm4,8947
 spiral/tables/maintenance.py,sha256=7Xa2Jdu_OY1Qu6iN1sPVdywVZtk_Mv3EaC3G93cmQvI,305
 spiral/tables/scan.py,sha256=3lPf5fSyF1fHGdGJ-pvu5HxPWoonf_XL7neWTqzB-0I,7582
 spiral/tables/snapshot.py,sha256=2NTuVEp2uJ1pV3Q5tLj7FOzPSc9axlfb6uOITwHnj0g,2229
 spiral/tables/table.py,sha256=VM93Rsm67sJFendI1_VhlkFORIdBGfhCMBUBK4dve9I,4910
 spiral/tables/transaction.py,sha256=3a64R-mf_cmR54BNn8U-05jmWonp6Ivxhe6u01Dyjzo,1573
 spiral/types_.py,sha256=W_jyO7F6rpPiH69jhgSgV7OxQZbOlb1Ho3InpKUP6Eo,155
-pyspiral-0.4.4.dist-info/RECORD,,
+pyspiral-0.5.0.dist-info/RECORD,,

spiral/adbc.py

@@ -35,6 +35,7 @@ from spiral.protogen._.arrow.flight.protocol.sql import (
     SqlInfo,
     SqlSupportedTransaction,
 )
+from spiral.tables import Snapshot
 
 log = logging.getLogger(__name__)
 logging.getLogger("sqlx").setLevel(logging.WARNING)
@@ -64,7 +65,6 @@ def debuggable(func):
     return wrapper_decorator
 
 
-# TODO(marko): This should work for Iceberg tables.
 class ADBCServerBase:
     def get_sql_info(self, _req: CommandGetSqlInfo) -> pa.RecordBatchReader:
         """Default implementation that reports no support for any complex features."""
@@ -143,6 +143,17 @@ class SpiralADBCServer(ADBCServerBase):
 
         self.pool = ThreadPoolExecutor()
 
+    def open_snapshot(self, tbl) -> Snapshot:
+        """Open a table in the Spiral project and return it as a PyArrow Dataset."""
+        if tbl.catalog is None or tbl.catalog == "":
+            raise FlightError("Project (Data Catalog) must be specified to open a table.")
+
+        project = tbl.catalog
+        dataset = tbl.db or "default"
+        table = tbl.name
+
+        return self.sp.project(project).tables.table(f"{dataset}.{table}").snapshot()
+
     def get_catalogs(self, req: CommandGetCatalogs) -> pa.RecordBatchReader:
         schema = pa.schema([pa.field("catalog_name", pa.string(), nullable=False)])
 
@@ -170,15 +181,16 @@ class SpiralADBCServer(ADBCServerBase):
             if req.catalog == "":
                 # Empty string means databases _without_ a catalog, which we don't support
                 return
+            catalog = req.catalog
 
             # Otherwise, catalog is either the project ID, or None.
-            if req.catalog is None:
+            if catalog is None:
                 projects = self.sp.list_projects()
             else:
                 projects = [self.sp.project(req.catalog)]
 
             for project in projects:
-                datasets = {dt.dataset for dt in project.tables.list_tables()}
+                datasets = {tbl.dataset for tbl in project.tables.list_tables()}
 
                 batch = pa.RecordBatch.from_arrays(
                     [
@@ -219,6 +231,7 @@ class SpiralADBCServer(ADBCServerBase):
                 projects = list(self.sp.list_projects())
             else:
                 projects = [self.sp.project(req.catalog)]
+            projects = sorted(projects, key=lambda p: p.id)
 
             def _process_project(project):
                 tables: list[TableResource] = project.tables.list_tables()
@@ -248,12 +261,13 @@ class SpiralADBCServer(ADBCServerBase):
     def statement_query(self, req: CommandStatementQuery, limit: int | None = None) -> pa.RecordBatchReader:
         # Extract the tables from the query, and bring them into the Python locals scope.
         expr = sqlglot.parse_one(req.query, dialect="duckdb")
+        datasets = {}
         for tbl in expr.find_all(exp.Table):
             # We swap the three-part identifier out for a single identifier
-            # This lets us insert a PyArrow Dataset into Python locals such that
-            # DuckDB will pick up on it for the query.
-            name = exp.table_name(tbl)
-            locals()[name] = self.sp.tables.table(f"{tbl.catalog}.{tbl.db}.{tbl.name}").snapshot().to_dataset()
+            # This lets us register a PyArrow Dataset with DuckDB for the query.
+            snapshot = self.open_snapshot(tbl)
+            name = snapshot.table.table_id
+            datasets[name] = snapshot.to_dataset()
             tbl.replace(exp.table_(table=name))
 
         try:
@@ -262,7 +276,11 @@ class SpiralADBCServer(ADBCServerBase):
             raise FlightError("DuckDB is required for SQL queries.")
 
         try:
-            sql = duckdb.sql(expr.sql(dialect="duckdb"))
+            # Create a DuckDB connection and register the datasets
+            conn = duckdb.connect()
+            for name, dataset in datasets.items():
+                conn.register(name, dataset)
+            sql = conn.sql(expr.sql(dialect="duckdb"))
         except Exception as e:
             raise FlightError(str(e))
 

spiral/api/__init__.py

@@ -3,6 +3,8 @@ from typing import TYPE_CHECKING
 
 import httpx
 
+from spiral.api.text_indexes import TextIndexesService
+
 from .client import _Client
 
 if TYPE_CHECKING:
@@ -57,6 +59,12 @@ class SpiralAPI:
 
         return WorkloadService(self.client)
 
+    @property
+    def text_indexes(self) -> "TextIndexesService":
+        from .text_indexes import TextIndexesService
+
+        return TextIndexesService(self.client)
+
     @property
     def telemetry(self) -> "TelemetryService":
         from .telemetry import TelemetryService

spiral/api/client.py

@@ -146,6 +146,10 @@ class _Client:
             # Enrich the exception with the response body
             raise SpiralHTTPError(body=resp.text, code=resp.status_code) from e
 
+        if response_cls == type[None]:
+            assert resp.text == ""
+            return None
+
         return TypeAdapter(response_cls).validate_python(resp.json())
 
     def paged(

spiral/api/projects.py

@@ -192,6 +192,6 @@ class ProjectService(ServiceBase):
         """Get a grant."""
         return self.client.get(f"/v1/grants/{grant_id}", Grant)
 
-    def revoke_grant(self, grant_id: str) -> None:
+    def revoke_grant(self, grant_id: str):
         """Revoke a grant."""
-        return self.client.delete(f"/v1/grants/{grant_id}", None)
+        return self.client.delete(f"/v1/grants/{grant_id}", type[None])

spiral/api/text_indexes.py

@@ -0,0 +1,56 @@
+from pydantic import BaseModel
+
+from .client import Paged, PagedResponse, ServiceBase
+from .types import IndexId, ProjectId, WorkerId
+from .workers import CPU, GcpRegion, Memory, ResourceClass
+
+
+class TextSearchWorker(BaseModel):
+    worker_id: WorkerId
+    project_id: ProjectId
+    index_id: IndexId
+    url: str | None
+
+
+class CreateWorkerRequest(BaseModel):
+    cpu: CPU
+    memory: Memory
+    region: GcpRegion
+
+
+class CreateWorkerResponse(BaseModel):
+    worker_id: WorkerId
+
+
+class SyncIndexRequest(BaseModel):
+    """Request to sync a text index."""
+
+    resources: ResourceClass
+
+
+class SyncIndexResponse(BaseModel):
+    worker_id: WorkerId
+
+
+class TextIndexesService(ServiceBase):
+    """Service for workload operations."""
+
+    def create_worker(self, index_id: IndexId, request: CreateWorkerRequest) -> CreateWorkerResponse:
+        """Create a new search worker."""
+        return self.client.post(f"/v1/text-indexes/{index_id}/workers", request, CreateWorkerResponse)
+
+    def list_workers(self, index_id: IndexId) -> Paged[WorkerId]:
+        """List text index workers for the given index."""
+        return self.client.paged(f"/v1/text-indexes/{index_id}/workers", PagedResponse[WorkerId])
+
+    def get_worker(self, worker_id: WorkerId) -> TextSearchWorker:
+        """Get a text index worker."""
+        return self.client.get(f"/v1/text-index-workers/{worker_id}", TextSearchWorker)
+
+    def shutdown_worker(self, worker_id: WorkerId) -> None:
+        """Shutdown a text index worker."""
+        return self.client.delete(f"/v1/text-index-workers/{worker_id}", type[None])
+
+    def sync_index(self, index_id: IndexId, request: SyncIndexRequest) -> SyncIndexResponse:
+        """Start a job to sync an index."""
+        return self.client.post(f"/v1/text-indexes/{index_id}/sync", request, SyncIndexResponse)

spiral/api/types.py

@@ -13,6 +13,8 @@ UserId = str
 OrgId = str
 ProjectId = str
 RoleId = str
+IndexId = str
+WorkerId = str
 
 RootUri = Annotated[str, AfterValidator(_validate_root_uri)]
 DatasetName = Annotated[str, StringConstraints(max_length=128, pattern=r"^[a-zA-Z_][a-zA-Z0-9_-]+$")]

spiral/api/workers.py

@@ -0,0 +1,40 @@
+from enum import Enum, IntEnum
+
+
+class CPU(IntEnum):
+    ONE = 1
+    TWO = 2
+    FOUR = 4
+    EIGHT = 8
+
+    def __str__(self):
+        return str(self.value)
+
+
+class Memory(str, Enum):
+    MB_512 = "512Mi"
+    GB_1 = "1Gi"
+    GB_2 = "2Gi"
+    GB_4 = "4Gi"
+    GB_8 = "8Gi"
+
+    def __str__(self):
+        return self.value
+
+
+class GcpRegion(str, Enum):
+    US_EAST4 = "us-east4"
+    EUROPE_WEST4 = "europe-west4"
+
+    def __str__(self):
+        return self.value
+
+
+class ResourceClass(str, Enum):
+    """Resource class for text index sync."""
+
+    SMALL = "small"
+    LARGE = "large"
+
+    def __str__(self):
+        return self.value

spiral/cli/indexes/__init__.py

@@ -1,9 +1,18 @@
+from typing import Annotated
+
 import rich
+from typer import Option
 
+from spiral.api.text_indexes import SyncIndexRequest
 from spiral.cli import AsyncTyper, state
+from spiral.cli.indexes.args import get_text_index_id
 from spiral.cli.types import ProjectArg
 
+from ...api.workers import ResourceClass
+from . import workers
+
 app = AsyncTyper(short_help="Indexes.")
+app.add_typer(workers.app, name="workers")
 
 
 @app.command(help="List indexes.")
@@ -17,3 +26,15 @@ def ls(
     for index in indexes:
         rich_table.add_row(index.id, index.name)
     rich.print(rich_table)
+
+
+@app.command(help="Trigger a sync job for the index.")
+def sync(
+    project: ProjectArg,
+    name: Annotated[str | None, Option(help="Index name.")] = None,
+    resources: Annotated[ResourceClass, Option(help="Resources to use for the sync job.")] = ResourceClass.SMALL,
+):
+    """Trigger a sync job for the index."""
+    index_id = get_text_index_id(project, name)
+    response = state.spiral.api.text_indexes.sync_index(index_id, SyncIndexRequest(resources=resources))
+    rich.print(f"Triggered sync job {response.worker_id} for index {index_id}.")

spiral/cli/indexes/args.py

@@ -0,0 +1,39 @@
+from typing import Annotated
+
+import questionary
+import rich
+import typer
+from questionary import Choice
+from typer import Option
+
+from spiral.api.projects import TextIndexResource
+from spiral.api.types import IndexId
+from spiral.cli import state
+from spiral.cli.types import ProjectArg
+
+
+def ask_index(project_id, title="Select an index"):
+    indexes: list[TextIndexResource] = list(state.spiral.project(project_id).indexes.list_indexes())
+
+    if not indexes:
+        rich.print("[red]No indexes found[/red]")
+        raise typer.Exit(1)
+
+    return questionary.select(
+        title,
+        choices=[Choice(title=index.name, value=index.id) for index in sorted(indexes, key=lambda t: (t.name, t.id))],
+    ).ask()
+
+
+def get_text_index_id(
+    project: ProjectArg,
+    name: Annotated[str | None, Option(help="Index name.")] = None,
+) -> IndexId:
+    if name is None:
+        return ask_index(project)
+
+    indexes: list[TextIndexResource] = list(state.spiral.project(project).indexes.list_indexes())
+    for index in indexes:
+        if index.name == name:
+            return index.id
+    raise ValueError(f"Index not found: {name}")

spiral/cli/indexes/workers.py

@@ -0,0 +1,59 @@
+from typing import Annotated
+
+import rich
+from typer import Option
+
+from spiral.api.text_indexes import CreateWorkerRequest
+from spiral.api.workers import CPU, GcpRegion, Memory
+from spiral.cli import AsyncTyper, state
+from spiral.cli.indexes.args import get_text_index_id
+from spiral.cli.types import ProjectArg
+
+app = AsyncTyper(short_help="Text Search Workers.")
+
+
+@app.command(name="serve", help="Create a search worker.")
+def serve(
+    project: ProjectArg,
+    index: Annotated[str | None, Option(help="Index name.")] = None,
+    region: Annotated[GcpRegion, Option(help="GCP region for the worker.")] = GcpRegion.US_EAST4,
+    cpu: Annotated[CPU, Option(help="CPU resources for the worker.")] = CPU.ONE,
+    memory: Annotated[Memory, Option(help="Memory resources for the worker in MB.")] = Memory.MB_512,
+):
+    """Create a new text search worker."""
+    index_id = get_text_index_id(project, index)
+    request = CreateWorkerRequest(cpu=cpu, memory=memory, region=region)
+    response = state.spiral.api.text_indexes.create_worker(index_id, request)
+    rich.print(f"Created worker {response.worker_id} for {index_id}.")
+
+
+@app.command(name="shutdown", help="Shutdown a search worker.")
+def shutdown(worker_id: str):
+    """Shutdown a worker."""
+    state.spiral.api.text_indexes.shutdown_worker(worker_id)
+    rich.print(f"Requested worker {worker_id} to shutdown.")
+
+
+@app.command(name="ls", help="List search workers.")
+def ls(
+    project: ProjectArg,
+    index: Annotated[str | None, Option(help="Index name.")] = None,
+):
+    """List text search workers."""
+    index_id = get_text_index_id(project, index)
+    worker_ids = state.spiral.api.text_indexes.list_workers(index_id)
+
+    rich_table = rich.table.Table("Worker ID", "URL", title=f"Text Search Workers for {index_id}")
+    for worker_id in worker_ids:
+        try:
+            worker = state.spiral.api.text_indexes.get_worker(worker_id)
+            rich_table.add_row(
+                worker_id,
+                worker.url,
+            )
+        except Exception:
+            rich_table.add_row(
+                worker_id,
+                "Unavailable",
+            )
+    rich.print(rich_table)

spiral/cli/tables/__init__.py

@@ -1,35 +1,16 @@
 from typing import Annotated
 
-import questionary
 import rich
-import typer
-from questionary import Choice
 from typer import Argument, Option
 
 from spiral import Spiral
-from spiral.cli import AsyncTyper, state
+from spiral.cli import AsyncTyper
+from spiral.cli.tables.args import get_table
 from spiral.cli.types import ProjectArg
-from spiral.tables import Table
 
 app = AsyncTyper(short_help="Spiral Tables.")
 
 
-def ask_table(project_id, title="Select a table"):
-    tables = list(state.spiral.project(project_id).tables.list_tables())
-
-    if not tables:
-        rich.print("[red]No tables found[/red]")
-        raise typer.Exit(1)
-
-    return questionary.select(
-        title,
-        choices=[
-            Choice(title=f"{table.dataset}.{table.table}", value=f"{table.project_id}.{table.dataset}.{table.table}")
-            for table in tables
-        ],
-    ).ask()
-
-
 @app.command(help="List tables.")
 def ls(
     project: ProjectArg,
@@ -48,7 +29,7 @@ def key_schema(
     table: Annotated[str | None, Option(help="Table name.")] = None,
     dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
 ):
-    _, table = _get_table(project, table, dataset)
+    _, table = get_table(project, table, dataset)
     rich.print(table.key_schema)
 
 
@@ -58,7 +39,7 @@ def schema(
     table: Annotated[str | None, Option(help="Table name.")] = None,
     dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
 ):
-    _, table = _get_table(project, table, dataset)
+    _, table = get_table(project, table, dataset)
     rich.print(table.schema)
 
 
@@ -68,7 +49,7 @@ def flush(
     table: Annotated[str | None, Option(help="Table name.")] = None,
     dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
 ):
-    identifier, table = _get_table(project, table, dataset)
+    identifier, table = get_table(project, table, dataset)
     table.maintenance().flush_wal()
     print(f"Flushed WAL for table {identifier} in project {project}.")
 
@@ -80,7 +61,7 @@ def debug(
     dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
     column_group: Annotated[str, Argument(help="Dot-separated column group path.")] = ".",
 ):
-    _, table = _get_table(project, table, dataset)
+    _, table = get_table(project, table, dataset)
     if column_group != ".":
         projection = table[column_group]
     else:
@@ -97,7 +78,7 @@ def manifests(
     dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
     column_group: Annotated[str, Argument(help="Dot-separated column group path.")] = ".",
 ):
-    _, table = _get_table(project, table, dataset)
+    _, table = get_table(project, table, dataset)
     if column_group != ".":
         projection = table[column_group]
     else:
@@ -105,17 +86,3 @@ def manifests(
     scan = projection.scan()
 
     scan._dump_manifests()
-
-
-def _get_table(
-    project: ProjectArg,
-    table: Annotated[str | None, Option(help="Table name.")] = None,
-    dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
-) -> (str, Table):
-    if table is None:
-        identifier = ask_table(project)
-    else:
-        identifier = table
-        if dataset is not None:
-            identifier = f"{dataset}.{table}"
-    return identifier, state.spiral.project(project).tables.table(identifier)

spiral/cli/tables/args.py

@@ -0,0 +1,42 @@
+from typing import Annotated
+
+import questionary
+import rich
+import typer
+from questionary import Choice
+from typer import Option
+
+from spiral.api.projects import TableResource
+from spiral.cli import state
+from spiral.cli.types import ProjectArg
+from spiral.tables import Table
+
+
+def ask_table(project_id, title="Select a table"):
+    tables: list[TableResource] = list(state.spiral.project(project_id).tables.list_tables())
+
+    if not tables:
+        rich.print("[red]No tables found[/red]")
+        raise typer.Exit(1)
+
+    return questionary.select(
+        title,
+        choices=[
+            Choice(title=f"{table.dataset}.{table.table}", value=f"{table.project_id}.{table.dataset}.{table.table}")
+            for table in sorted(tables, key=lambda t: (t.dataset, t.table))
+        ],
+    ).ask()
+
+
+def get_table(
+    project: ProjectArg,
+    table: Annotated[str | None, Option(help="Table name.")] = None,
+    dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
+) -> (str, Table):
+    if table is None:
+        identifier = ask_table(project)
+    else:
+        identifier = table
+        if dataset is not None:
+            identifier = f"{dataset}.{table}"
+    return identifier, state.spiral.project(project).tables.table(identifier)

spiral/core/index/__init__.pyi

@@ -1,15 +1,7 @@
 import pyarrow as pa
 
-class IndexStatus:
-    status: str
-    staleness_s: int | None
-    # An extent of keys that are indexed.
-    # key_extent: KeyExtent | None
-
 class TextIndex:
     id: str
 
-    def status(self) -> IndexStatus: ...
-
 class SearchScan:
     def to_record_batches(self) -> pa.RecordBatchReader: ...

spiral/core/table/__init__.pyi

@@ -69,16 +69,16 @@ class TableScan:
         shuffle_buffer_size: int | None = None,
         shuffle_pool_num_rows: int | None = None,
     ) -> pa.RecordBatchReader: ...
-    def column_group_scan(self, column_group: ColumnGroup) -> ColumnGroupScan: ...
-    def key_space_scan(self, table_id: str) -> KeySpaceScan: ...
+    def column_group_state(self, column_group: ColumnGroup) -> ColumnGroupState: ...
+    def key_space_state(self, table_id: str) -> KeySpaceState: ...
     def metrics(self) -> dict[str, Any]: ...
 
-class KeySpaceScan:
+class KeySpaceState:
     manifest: FragmentManifest
 
     def key_schema(self) -> Schema: ...
 
-class ColumnGroupScan:
+class ColumnGroupState:
     manifest: FragmentManifest
 
     def schema(self) -> Schema: ...

spiral/indexes/index.py

@@ -1,4 +1,3 @@
-import datetime
 from typing import TYPE_CHECKING
 
 from spiral.core.index import TextIndex as CoreTextIndex
@@ -27,8 +26,3 @@ class TextIndex(Expr):
     @property
     def name(self) -> str:
         return self._name
-
-    def status(self) -> (str, datetime.timedelta | None):
-        """Fetch the status of the index. If status is ready, returns the staleness of the index."""
-        status = self._index.status()
-        return status.status, datetime.timedelta(seconds=status.staleness_s) if status.staleness_s is not None else None

spiral/tables/client.py

@@ -25,6 +25,9 @@ class Tables:
     """
 
     def __init__(self, api: SpiralAPI, spiral: CoreSpiral, *, project_id: str | None = None):
+        if project_id == "":
+            raise ValueError("Project ID cannot be an empty string.")
+
         self._api = api
         self._spiral = spiral
         self._project_id = project_id

spiral/tables/debug/manifests.py

@@ -10,14 +10,14 @@ def display_manifests(scan: TableScan):
         raise NotImplementedError("Multiple table scans are not supported.")
     table_id = scan.table_ids()[0]
 
-    key_space_manifest: FragmentManifest = scan.key_space_scan(table_id).manifest
+    key_space_manifest: FragmentManifest = scan.key_space_state(table_id).manifest
     _table_of_fragments(
         key_space_manifest,
         title="Key Space manifest",
     )
 
     for column_group in scan.column_groups():
-        column_group_manifest: FragmentManifest = scan.column_group_scan(column_group).manifest
+        column_group_manifest: FragmentManifest = scan.column_group_state(column_group).manifest
         _table_of_fragments(
             column_group_manifest,
             title=f"Column Group manifest for {str(column_group)}",

spiral/tables/debug/scan.py

@@ -15,17 +15,17 @@ def show_scan(scan: TableScan):
     column_groups = scan.column_groups()
 
     splits = scan.split()
-    key_space_scan = scan.key_space_scan(table_id)
+    key_space_state = scan.key_space_state(table_id)
 
     # Collect all key bounds from all manifests. This makes sure all visualizations are aligned.
     key_points = set()
-    key_space_manifest = key_space_scan.manifest
+    key_space_manifest = key_space_state.manifest
     for i in range(len(key_space_manifest)):
         fragment_file = key_space_manifest[i]
         key_points.add(fragment_file.key_extent.min)
         key_points.add(fragment_file.key_extent.max)
     for cg in column_groups:
-        cg_scan = scan.column_group_scan(cg)
+        cg_scan = scan.column_group_state(cg)
         cg_manifest = cg_scan.manifest
         for i in range(len(cg_manifest)):
             fragment_file = cg_manifest[i]
@@ -39,7 +39,7 @@ def show_scan(scan: TableScan):
 
     show_manifest(key_space_manifest, scope="Key space", key_points=key_points, splits=splits)
     for cg in scan.column_groups():
-        cg_scan = scan.column_group_scan(cg)
+        cg_scan = scan.column_group_state(cg)
         # Skip table id from the start of the column group.
         show_manifest(cg_scan.manifest, scope=".".join(cg.path[1:]), key_points=key_points, splits=splits)