pyspiral 0.4.3__cp310-abi3-macosx_11_0_arm64.whl → 0.5.0__cp310-abi3-macosx_11_0_arm64.whl

{pyspiral-0.4.3.dist-info → pyspiral-0.5.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyspiral
-Version: 0.4.3
+Version: 0.5.0
 Classifier: Intended Audience :: Science/Research
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python
@@ -13,22 +13,23 @@ Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Rust
 Classifier: License :: Other/Proprietary License
 Requires-Dist: betterproto==2.0.0b7
+Requires-Dist: datasets>=4.0.0
 Requires-Dist: google-re2>=1.1.20240702
 Requires-Dist: grpclib>=0.4.7
 Requires-Dist: hishel>=0.0.30
 Requires-Dist: httpx>=0.27.0
+Requires-Dist: nanoid>=2.0.0
 Requires-Dist: numpy>=2
 Requires-Dist: pyarrow>=21.0.0
 Requires-Dist: pydantic-settings>=2.3.4
 Requires-Dist: pydantic[email]>=2.5.3
 Requires-Dist: pyjwt[crypto]>=2.9.0
+Requires-Dist: pyperclip>=1.9.0
 Requires-Dist: questionary>=2.0.1
+Requires-Dist: sqlglot[rs]>=25.25.1
 Requires-Dist: tqdm>=4.66.5
 Requires-Dist: typer>=0.16
 Requires-Dist: xxhash>=3.4.1
-Requires-Dist: nanoid>=2.0.0
-Requires-Dist: sqlglot[rs]>=25.25.1
-Requires-Dist: pyperclip>=1.9.0
 Requires-Dist: polars>=1.31.0 ; extra == 'polars'
 Requires-Dist: duckdb>=1.3.2 ; extra == 'duckdb'
 Requires-Dist: pyiceberg>=0.9.1 ; extra == 'pyiceberg'

{pyspiral-0.4.3.dist-info → pyspiral-0.5.0.dist-info}/RECORD

@@ -1,17 +1,19 @@
-pyspiral-0.4.3.dist-info/METADATA,sha256=vSc4ZGaqT4FiTed4rIQdo6sMQBXs690OKn-z6o1gfLQ,1610
-pyspiral-0.4.3.dist-info/WHEEL,sha256=Mdosfxua6Dx1zYgObRH97e3wyiELqBbLtoRJj4RUSQE,103
-pyspiral-0.4.3.dist-info/entry_points.txt,sha256=uft7u-a6g40NLt4Q6BleWbK4NY0M8nZuYPpP8DV0EOk,45
+pyspiral-0.5.0.dist-info/METADATA,sha256=J_bR2LwG0i6M8wB0-60AXJm8MutIfqs9QUbaINmBeTI,1641
+pyspiral-0.5.0.dist-info/WHEEL,sha256=Mdosfxua6Dx1zYgObRH97e3wyiELqBbLtoRJj4RUSQE,103
+pyspiral-0.5.0.dist-info/entry_points.txt,sha256=uft7u-a6g40NLt4Q6BleWbK4NY0M8nZuYPpP8DV0EOk,45
 spiral/__init__.py,sha256=Jv1vbcnnmcTsBLN5mSNjnX3ae4C_mgojXDSBFaqIhN0,208
-spiral/_lib.abi3.so,sha256=NUSuOVRTgkgwtCLTg5IaYwewnboJsyVJEX4L95NeINE,59953376
-spiral/adbc.py,sha256=HcvR60uQeEK2oggSAK6y5VYtIrACIiCQ-85MEf18EZc,14199
-spiral/api/__init__.py,sha256=_7BS1RhqEFjnt3XwFWZNCHVEQeSKpezPevAiGCsvDbE,1776
+spiral/_lib.abi3.so,sha256=4J6RSb9LxUWqUnRZeyccSneyjqEFfIrJ6lMCnsNUqbc,60398416
+spiral/adbc.py,sha256=RIIWBew7zPoQa_h3I-A-nX9cUMDM3D3Je0mqE9aDX9k,14885
+spiral/api/__init__.py,sha256=nzZK3r1K2GSaqaLoIba4WuyUyG7ApjLZ4fyJovT1unA,2000
 spiral/api/admin.py,sha256=A1iVR1XYJSObZivPAD5UzmPuMgupXc9kaHNYYa_kwfs,585
-spiral/api/client.py,sha256=9-L6T8niQAXo90jRxllJD4hXXmcGfHj7CW9X3XTYa5Q,4551
-spiral/api/filesystems.py,sha256=5Ky_otnresGj7WdsR8Xi7DDM3lkB8UES6Lru_xWAGDM,4559
+spiral/api/client.py,sha256=XQaOd1DgLTDOxW_uZUeTX5UCnUpPV1unjxbzWLlPJaM,4650
+spiral/api/filesystems.py,sha256=EA4iqhTeaIlvObvEUxHmZl0pQ24IOxUVWM3GPhFLw8o,4969
 spiral/api/organizations.py,sha256=B-8zZ7lFJANGK7dUNbo_aU-cgI959JBP9VcWb6wdgi0,1895
-spiral/api/projects.py,sha256=JBGof9A2Ivasu2jrULMjHBwlna0M8WRrTNqU-Es4GJ8,5673
+spiral/api/projects.py,sha256=mvyp4tnUhItNnPnyWsj0WrSf2ca3mggL7KjXs5Fllco,5671
 spiral/api/telemetry.py,sha256=tfdA3E_EWJwFVxkQfkm8tiYGRubnx2LuE5nbfsk1oG4,474
-spiral/api/types.py,sha256=zx-BRKsi1GHg9aL9gMUaVQWYYMXJcP0A8OQUc7jSIAc,653
+spiral/api/text_indexes.py,sha256=f8AcrYaBKsaka5A4okuExl96A-2rBbsgru98l_xogwo,1826
+spiral/api/types.py,sha256=lGdiKViRgIEJXD2ubwnyEIEwHkfRumlZjVEaHMV3Tm8,682
+spiral/api/workers.py,sha256=0wZNUHMioDT53P1OBJfpjyDfIodHwwT6858z2IlRIM4,636
 spiral/api/workloads.py,sha256=XAyXV7vgZcoyyoPoGvOT4jTpyFKFMvrrAfhL6d1h1kE,1748
 spiral/arrow_.py,sha256=T1LZ7bh9aMDbXfpUsf0dR0E1roTQyAYSgZ2mL4s8J_4,7681
 spiral/cli/__init__.py,sha256=ooAFz_iCpVCKHE0TiVElIynbP2PtTgD9cUw46Vh1lcw,2145
@@ -23,34 +25,37 @@ spiral/cli/fs.py,sha256=dVPoAoAbuQ9yJlfI-JiFgS9VdnPmeBMygVHgehJRj34,4367
 spiral/cli/iceberg/__init__.py,sha256=IQV_gwCFSj6Ubxs58VM9Pal1ymgG2bxdDgOPuk9E5bs,214
 spiral/cli/iceberg/namespaces.py,sha256=x9pvHlcXtcATYYjqimHa6CtkyL3taQUJ--ni_Bfoemc,1510
 spiral/cli/iceberg/tables.py,sha256=nSR4-t54otJfCmubB6vXnbOkbqPVGV0sHBlc-t9cIVg,1930
-spiral/cli/indexes/__init__.py,sha256=-USfxCIdckzZKBNQ-DXqe3V5ttWVo_Fsa1Mfcx5hdIw,467
+spiral/cli/indexes/__init__.py,sha256=yNMBZh3kAz1NXsridvbY4-4jNIJAgntBhDnu1EfaZTI,1291
+spiral/cli/indexes/args.py,sha256=B4zAFnVZKPHq-_z8qnHYwgs1v4c-w_iypbA5MdXL67s,1139
+spiral/cli/indexes/workers.py,sha256=mu-7-Asz1txNtCkoei6q-m-lE5YhLUjCDwstKJcew40,2169
 spiral/cli/login.py,sha256=InKMnpV8NATW5RPgB3ZL-DSVPzUuUByyK4Fx7pZEgfg,607
 spiral/cli/orgs.py,sha256=V-4ZTT3FwFQLcs1-BenC8uCgvWOJcxkZPSdCPfsexhc,2848
 spiral/cli/printer.py,sha256=W83KAE-7meoDD1yRltLQrZqrA2olGapBGy_2USWkY08,1778
 spiral/cli/projects.py,sha256=TKXu_VzkIUccwXzdlg-wQMkrB-Py33g052NrbuJx-D4,5096
 spiral/cli/state.py,sha256=10wTIVQ0SJkY67Z6-KQ1LFlt3aVIPmZhoHFdTwp4kNA,130
-spiral/cli/tables/__init__.py,sha256=lkGLDeU28IVnuxJdlYSUh6QSB9fQ4_1MeZJL73iXcHo,3660
+spiral/cli/tables/__init__.py,sha256=DTxviiflWZkUDmPhuGzXmhW2mNRgPNkJcaP7olFtXpc,2678
+spiral/cli/tables/args.py,sha256=bxKQoJuWhCTHlDYz_WRQ6_Kp9XgRBH1UIMgUMzlxliA,1262
 spiral/cli/telemetry.py,sha256=ABDCyV5QJGOIJp4AxvK0LG5xNPIysP37K5haL38T7P4,586
 spiral/cli/types.py,sha256=YG1eHhRLaqlVU_18DQBuF_YMsabhMZLBY0V9CvbSxjY,1369
 spiral/cli/workloads.py,sha256=SbxgwiBlX1AuqpOLV3gs7DFkH-Tbeend7qJTwq0Je84,1994
 spiral/client.py,sha256=K-OuMOTgYxOA9vef5jSANjmPRBfGrzQ65fg6Fd-rHMY,2683
 spiral/core/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 spiral/core/client/__init__.pyi,sha256=Tn1OJmkO1rQUsPE9BtfEyxIjoife6s16qOd8XiyHi2c,3475
-spiral/core/index/__init__.pyi,sha256=NPOG1ztFO6siBGpmJU3boRzX26xfxw--2TiCydosGvo,314
-spiral/core/table/__init__.pyi,sha256=dwOaxcOl6ZIlxoLjOnC3CNUgGetWfnEV1Jx06aCH8M8,3265
+spiral/core/index/__init__.pyi,sha256=MBq-jBuTmBreYMJ4AJFAe4e-ByRMM7JVssuEJMLVfQs,131
+spiral/core/table/__init__.pyi,sha256=XhN9xpDdBnugRhtiP9ThC3DSADoLMDL9lxSjjY7fJ28,3296
 spiral/core/table/manifests/__init__.pyi,sha256=3V59-K1qr1z2dGfgRKXaHSVheK8NNw8Q8PFhfbeQd_4,1065
 spiral/core/table/metastore/__init__.pyi,sha256=dMqySDnsjPUTBuFU2MaQGyocKEoGkWpeTQmUP2iIKbc,1880
 spiral/core/table/spec/__init__.pyi,sha256=D4GQp9RWwyLKTlRW7eDXcQE-xA5rF2iBcXZ8y7b48EE,5595
 spiral/datetime_.py,sha256=1TA1RYIRU22qcUuipIjVhAtGnPDVn2z9WttuhkmfkwY,964
-spiral/expressions/__init__.py,sha256=hLh2qfHxM7hJg6GidRApqczwE80vatizkKN6YDUAAUA,6570
+spiral/expressions/__init__.py,sha256=T8PIb0_UB9kynK0dpWbUD4No5lKRTG-wKnao8xOcXjY,6381
 spiral/expressions/base.py,sha256=q_W9XslcdFQtOIE_d1VkEmLickaXKOAoIcFeMoh-nqQ,4751
 spiral/expressions/http.py,sha256=begUydWoFHEqjeLkATvI_v66Ez6_rR-OQBWO5cHbb9c,2742
 spiral/expressions/io.py,sha256=gJ2a0FKMmdxarWKENulPRwH7KDvSJTIh_OUxX306xAM,3045
 spiral/expressions/list_.py,sha256=MMt5lf5H1M3O-x6N_PvqOLGq9NOk6Ukv0fPWwPC_uy4,1809
-spiral/expressions/mp4.py,sha256=R-fcVYRI6KaH1Nwpmqnsc1VYd9wA7Nuiy2UDcNxEzpw,2165
+spiral/expressions/mp4.py,sha256=_xGVnkygddzxP9a8OACJ8_KXnejuVbYCVKBCXBQ798Y,2151
 spiral/expressions/png.py,sha256=KO8X0OmMzUFwpg2I_j0JTyldPzVXDWIMzjWMWDV9vIY,506
 spiral/expressions/qoi.py,sha256=gvIbb6fXb_Bb080sn9wkpbGGrPs2UEcTXCfuv4-kcYQ,506
-spiral/expressions/refs.py,sha256=ISMtJtUL--BjHF6rsvgN3Um4QcvVqQE9URngOxjQrhw,2115
+spiral/expressions/refs.py,sha256=omeHBQ5o6N4xgZ3x5Xz7IRrWwYBBtQY8DYK0NNAxeGo,2109
 spiral/expressions/str_.py,sha256=tY8RXW3JWvr1-bEfCZtk5FAf11wKJnXPuA9EoeJ9tA4,1265
 spiral/expressions/struct.py,sha256=pGAnCDh6AK0BK1XfZ1qG4ce4ranIQEE1HQsgmzBcfwQ,2038
 spiral/expressions/text.py,sha256=-02gBWYoyNQ3qQ1--9HTa8IryUDojYQVIp8C7rgnOWQ,1893
@@ -61,7 +66,7 @@ spiral/iceberg/__init__.py,sha256=jSIlTxWauAbJV5gsWglZisFbnfNNzLYN90scoYcdWzc,65
 spiral/iceberg/client.py,sha256=E6FyE_h2HLgDW1cAFg1XgglJr6rbVOCWjRtRmqoMVkM,1003
 spiral/indexes/__init__.py,sha256=TXLQ-_3xso3lFIp2lM58_ip9OPNwPKFv1FdsWiUF-d8,178
 spiral/indexes/client.py,sha256=NsFBILEHMjyCUruFrUEKucRQRrN4OvqgbL4pmzWs07g,5600
-spiral/indexes/index.py,sha256=4CmSFlZYp46B2CjqtiyZ7VF5EH3duiutz3nWFnyApLA,973
+spiral/indexes/index.py,sha256=TJB1-hSiPNrzJ_VYFTZGHvIQ_vmoILPK7tyi_9oS1nA,638
 spiral/indexes/scan.py,sha256=B2m-UgNuawNB90HXK33GTQfMy2WLdNNxiiB6cIjFW2Y,697
 spiral/project.py,sha256=0uJ1Jb88Ie-cCNnSdX3QfFtCUqrjLka4zCm_TxCpVak,1189
 spiral/protogen/_/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -81,18 +86,18 @@ spiral/protogen/util.py,sha256=smnvVo6nYH3FfDm9jqhNLaXz4bbTBaQezHQDCTvZyiQ,1486
 spiral/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 spiral/server.py,sha256=ztBmB5lBnUz-smQxR_tC8AI5SOhz17wH0MI3GuzDUdM,600
 spiral/settings.py,sha256=PIQV2ljtB3pEOWoMRVSRzSGJNrXviO2JBgZ5ZY_Nq2E,2794
-spiral/substrait_.py,sha256=RNSmfbGFT_5uyo8AFtzS9A7IHW3DkacMTw2vKnj0Das,12762
+spiral/substrait_.py,sha256=2BYvwFGcCwJ0JXNhXOLdPuhM1PqFyaeSqFpQCtv-M4E,12581
 spiral/tables/__init__.py,sha256=iiP7BkHA117em37_e75jtdvoZC10xCXtld18gRnPbTw,430
-spiral/tables/client.py,sha256=l_wJJRf3BPD5lg4Q1Ll2lAqQIuBCnKwC6JtsAui91Tc,4915
+spiral/tables/client.py,sha256=MVgfeVF7P4kXnjOTQExp0VmywyuuRQ2IZVxdwVY3xgk,5015
 spiral/tables/dataset.py,sha256=DuHeKVCJfXLsbxmde9QW6yvesW5uhswG6qAxV5X0ZgA,7890
 spiral/tables/debug/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-spiral/tables/debug/manifests.py,sha256=E_-DiMBg2EPL97cl9hLWhiqEsFtjEBgh_C7jZy8EWYc,2594
+spiral/tables/debug/manifests.py,sha256=t7E0AchHrzOv9vAQpE77Qp3rLc5VTzRRxlByt5OWnUM,2596
 spiral/tables/debug/metrics.py,sha256=XdRDcjggtsLNGCAjam6IxG9072pz_d2C8iLApNRFUtk,2044
-spiral/tables/debug/scan.py,sha256=-IWX_UjO4QP9Hj7PtZ1rLlbswJcryOin56GT-exqFm4,8942
+spiral/tables/debug/scan.py,sha256=EEG2gzbBpUyrtk4jQXh-ENk8aySrKf0CZMym_wHuWm4,8947
 spiral/tables/maintenance.py,sha256=7Xa2Jdu_OY1Qu6iN1sPVdywVZtk_Mv3EaC3G93cmQvI,305
 spiral/tables/scan.py,sha256=3lPf5fSyF1fHGdGJ-pvu5HxPWoonf_XL7neWTqzB-0I,7582
 spiral/tables/snapshot.py,sha256=2NTuVEp2uJ1pV3Q5tLj7FOzPSc9axlfb6uOITwHnj0g,2229
-spiral/tables/table.py,sha256=4B2drwwfaoL6aIJ-5Ll-Bqza-EBeDIfMkuszSOZqSpk,5326
+spiral/tables/table.py,sha256=VM93Rsm67sJFendI1_VhlkFORIdBGfhCMBUBK4dve9I,4910
 spiral/tables/transaction.py,sha256=3a64R-mf_cmR54BNn8U-05jmWonp6Ivxhe6u01Dyjzo,1573
 spiral/types_.py,sha256=W_jyO7F6rpPiH69jhgSgV7OxQZbOlb1Ho3InpKUP6Eo,155
-pyspiral-0.4.3.dist-info/RECORD,,
+pyspiral-0.5.0.dist-info/RECORD,,

spiral/adbc.py

@@ -35,6 +35,7 @@ from spiral.protogen._.arrow.flight.protocol.sql import (
     SqlInfo,
     SqlSupportedTransaction,
 )
+from spiral.tables import Snapshot
 
 log = logging.getLogger(__name__)
 logging.getLogger("sqlx").setLevel(logging.WARNING)
@@ -64,7 +65,6 @@ def debuggable(func):
     return wrapper_decorator
 
 
-# TODO(marko): This should work for Iceberg tables.
 class ADBCServerBase:
     def get_sql_info(self, _req: CommandGetSqlInfo) -> pa.RecordBatchReader:
         """Default implementation that reports no support for any complex features."""
@@ -143,6 +143,17 @@ class SpiralADBCServer(ADBCServerBase):
 
         self.pool = ThreadPoolExecutor()
 
+    def open_snapshot(self, tbl) -> Snapshot:
+        """Open a table in the Spiral project and return it as a PyArrow Dataset."""
+        if tbl.catalog is None or tbl.catalog == "":
+            raise FlightError("Project (Data Catalog) must be specified to open a table.")
+
+        project = tbl.catalog
+        dataset = tbl.db or "default"
+        table = tbl.name
+
+        return self.sp.project(project).tables.table(f"{dataset}.{table}").snapshot()
+
     def get_catalogs(self, req: CommandGetCatalogs) -> pa.RecordBatchReader:
         schema = pa.schema([pa.field("catalog_name", pa.string(), nullable=False)])
 
@@ -170,15 +181,16 @@ class SpiralADBCServer(ADBCServerBase):
             if req.catalog == "":
                 # Empty string means databases _without_ a catalog, which we don't support
                 return
+            catalog = req.catalog
 
             # Otherwise, catalog is either the project ID, or None.
-            if req.catalog is None:
+            if catalog is None:
                 projects = self.sp.list_projects()
             else:
                 projects = [self.sp.project(req.catalog)]
 
             for project in projects:
-                datasets = {dt.dataset for dt in project.tables.list_tables()}
+                datasets = {tbl.dataset for tbl in project.tables.list_tables()}
 
                 batch = pa.RecordBatch.from_arrays(
                     [
@@ -219,6 +231,7 @@ class SpiralADBCServer(ADBCServerBase):
                 projects = list(self.sp.list_projects())
             else:
                 projects = [self.sp.project(req.catalog)]
+            projects = sorted(projects, key=lambda p: p.id)
 
             def _process_project(project):
                 tables: list[TableResource] = project.tables.list_tables()
@@ -248,12 +261,13 @@ class SpiralADBCServer(ADBCServerBase):
     def statement_query(self, req: CommandStatementQuery, limit: int | None = None) -> pa.RecordBatchReader:
         # Extract the tables from the query, and bring them into the Python locals scope.
         expr = sqlglot.parse_one(req.query, dialect="duckdb")
+        datasets = {}
         for tbl in expr.find_all(exp.Table):
             # We swap the three-part identifier out for a single identifier
-            # This lets us insert a PyArrow Dataset into Python locals such that
-            # DuckDB will pick up on it for the query.
-            name = exp.table_name(tbl)
-            locals()[name] = self.sp.tables.table(f"{tbl.catalog}.{tbl.db}.{tbl.name}").snapshot().to_dataset()
+            # This lets us register a PyArrow Dataset with DuckDB for the query.
+            snapshot = self.open_snapshot(tbl)
+            name = snapshot.table.table_id
+            datasets[name] = snapshot.to_dataset()
             tbl.replace(exp.table_(table=name))
 
         try:
@@ -262,7 +276,11 @@ class SpiralADBCServer(ADBCServerBase):
             raise FlightError("DuckDB is required for SQL queries.")
 
         try:
-            sql = duckdb.sql(expr.sql(dialect="duckdb"))
+            # Create a DuckDB connection and register the datasets
+            conn = duckdb.connect()
+            for name, dataset in datasets.items():
+                conn.register(name, dataset)
+            sql = conn.sql(expr.sql(dialect="duckdb"))
         except Exception as e:
             raise FlightError(str(e))
 

spiral/api/__init__.py

@@ -3,6 +3,8 @@ from typing import TYPE_CHECKING
 
 import httpx
 
+from spiral.api.text_indexes import TextIndexesService
+
 from .client import _Client
 
 if TYPE_CHECKING:
@@ -57,6 +59,12 @@ class SpiralAPI:
 
         return WorkloadService(self.client)
 
+    @property
+    def text_indexes(self) -> "TextIndexesService":
+        from .text_indexes import TextIndexesService
+
+        return TextIndexesService(self.client)
+
     @property
     def telemetry(self) -> "TelemetryService":
         from .telemetry import TelemetryService

spiral/api/client.py

@@ -146,6 +146,10 @@ class _Client:
             # Enrich the exception with the response body
             raise SpiralHTTPError(body=resp.text, code=resp.status_code) from e
 
+        if response_cls == type[None]:
+            assert resp.text == ""
+            return None
+
         return TypeAdapter(response_cls).validate_python(resp.json())
 
     def paged(

spiral/api/filesystems.py

@@ -1,4 +1,5 @@
 from enum import Enum
+from types import NoneType
 from typing import Annotated, Literal
 
 from pydantic import AfterValidator, BaseModel, Field
@@ -17,6 +18,7 @@ def _validate_directory_path(path: str) -> str:
 
 DirectoryPath = Annotated[str, AfterValidator(_validate_directory_path)]
 FilePath = str  # Path or directory
+FsLoc = str
 
 
 class BuiltinFileSystem(BaseModel):
@@ -120,6 +122,12 @@ class CreateMountResponse(BaseModel):
     mount: Mount
 
 
+class GetMountAndFileSystemResponse(BaseModel):
+    mount: Mount
+    file_system: FileSystem
+    fs_loc: FsLoc
+
+
 class FileSystemService(ServiceBase):
     """Service for file system operations."""
 
@@ -148,6 +156,10 @@ class FileSystemService(ServiceBase):
         """Get a mount."""
         return self.client.get(f"/v1/mounts/{mount_id}", Mount)
 
+    def get_mount_and_file_system(self, mount_id: str) -> GetMountAndFileSystemResponse:
+        """Get the mount and its associated file system."""
+        return self.client.get(f"/v1/mounts/{mount_id}/with-filesystem", GetMountAndFileSystemResponse)
+
     def remove_mount(self, mount_id: str) -> None:
         """Remove mount."""
-        return self.client.delete(f"/v1/mounts/{mount_id}", None)
+        return self.client.delete(f"/v1/mounts/{mount_id}", NoneType)

spiral/api/projects.py

@@ -192,6 +192,6 @@ class ProjectService(ServiceBase):
         """Get a grant."""
         return self.client.get(f"/v1/grants/{grant_id}", Grant)
 
-    def revoke_grant(self, grant_id: str) -> None:
+    def revoke_grant(self, grant_id: str):
         """Revoke a grant."""
-        return self.client.delete(f"/v1/grants/{grant_id}", None)
+        return self.client.delete(f"/v1/grants/{grant_id}", type[None])

spiral/api/text_indexes.py

@@ -0,0 +1,56 @@
+from pydantic import BaseModel
+
+from .client import Paged, PagedResponse, ServiceBase
+from .types import IndexId, ProjectId, WorkerId
+from .workers import CPU, GcpRegion, Memory, ResourceClass
+
+
+class TextSearchWorker(BaseModel):
+    worker_id: WorkerId
+    project_id: ProjectId
+    index_id: IndexId
+    url: str | None
+
+
+class CreateWorkerRequest(BaseModel):
+    cpu: CPU
+    memory: Memory
+    region: GcpRegion
+
+
+class CreateWorkerResponse(BaseModel):
+    worker_id: WorkerId
+
+
+class SyncIndexRequest(BaseModel):
+    """Request to sync a text index."""
+
+    resources: ResourceClass
+
+
+class SyncIndexResponse(BaseModel):
+    worker_id: WorkerId
+
+
+class TextIndexesService(ServiceBase):
+    """Service for workload operations."""
+
+    def create_worker(self, index_id: IndexId, request: CreateWorkerRequest) -> CreateWorkerResponse:
+        """Create a new search worker."""
+        return self.client.post(f"/v1/text-indexes/{index_id}/workers", request, CreateWorkerResponse)
+
+    def list_workers(self, index_id: IndexId) -> Paged[WorkerId]:
+        """List text index workers for the given index."""
+        return self.client.paged(f"/v1/text-indexes/{index_id}/workers", PagedResponse[WorkerId])
+
+    def get_worker(self, worker_id: WorkerId) -> TextSearchWorker:
+        """Get a text index worker."""
+        return self.client.get(f"/v1/text-index-workers/{worker_id}", TextSearchWorker)
+
+    def shutdown_worker(self, worker_id: WorkerId) -> None:
+        """Shutdown a text index worker."""
+        return self.client.delete(f"/v1/text-index-workers/{worker_id}", type[None])
+
+    def sync_index(self, index_id: IndexId, request: SyncIndexRequest) -> SyncIndexResponse:
+        """Start a job to sync an index."""
+        return self.client.post(f"/v1/text-indexes/{index_id}/sync", request, SyncIndexResponse)

spiral/api/types.py

@@ -13,6 +13,8 @@ UserId = str
 OrgId = str
 ProjectId = str
 RoleId = str
+IndexId = str
+WorkerId = str
 
 RootUri = Annotated[str, AfterValidator(_validate_root_uri)]
 DatasetName = Annotated[str, StringConstraints(max_length=128, pattern=r"^[a-zA-Z_][a-zA-Z0-9_-]+$")]

spiral/api/workers.py

@@ -0,0 +1,40 @@
+from enum import Enum, IntEnum
+
+
+class CPU(IntEnum):
+    ONE = 1
+    TWO = 2
+    FOUR = 4
+    EIGHT = 8
+
+    def __str__(self):
+        return str(self.value)
+
+
+class Memory(str, Enum):
+    MB_512 = "512Mi"
+    GB_1 = "1Gi"
+    GB_2 = "2Gi"
+    GB_4 = "4Gi"
+    GB_8 = "8Gi"
+
+    def __str__(self):
+        return self.value
+
+
+class GcpRegion(str, Enum):
+    US_EAST4 = "us-east4"
+    EUROPE_WEST4 = "europe-west4"
+
+    def __str__(self):
+        return self.value
+
+
+class ResourceClass(str, Enum):
+    """Resource class for text index sync."""
+
+    SMALL = "small"
+    LARGE = "large"
+
+    def __str__(self):
+        return self.value

spiral/cli/indexes/__init__.py

@@ -1,9 +1,18 @@
+from typing import Annotated
+
 import rich
+from typer import Option
 
+from spiral.api.text_indexes import SyncIndexRequest
 from spiral.cli import AsyncTyper, state
+from spiral.cli.indexes.args import get_text_index_id
 from spiral.cli.types import ProjectArg
 
+from ...api.workers import ResourceClass
+from . import workers
+
 app = AsyncTyper(short_help="Indexes.")
+app.add_typer(workers.app, name="workers")
 
 
 @app.command(help="List indexes.")
@@ -17,3 +26,15 @@ def ls(
     for index in indexes:
         rich_table.add_row(index.id, index.name)
     rich.print(rich_table)
+
+
+@app.command(help="Trigger a sync job for the index.")
+def sync(
+    project: ProjectArg,
+    name: Annotated[str | None, Option(help="Index name.")] = None,
+    resources: Annotated[ResourceClass, Option(help="Resources to use for the sync job.")] = ResourceClass.SMALL,
+):
+    """Trigger a sync job for the index."""
+    index_id = get_text_index_id(project, name)
+    response = state.spiral.api.text_indexes.sync_index(index_id, SyncIndexRequest(resources=resources))
+    rich.print(f"Triggered sync job {response.worker_id} for index {index_id}.")

spiral/cli/indexes/args.py

@@ -0,0 +1,39 @@
+from typing import Annotated
+
+import questionary
+import rich
+import typer
+from questionary import Choice
+from typer import Option
+
+from spiral.api.projects import TextIndexResource
+from spiral.api.types import IndexId
+from spiral.cli import state
+from spiral.cli.types import ProjectArg
+
+
+def ask_index(project_id, title="Select an index"):
+    indexes: list[TextIndexResource] = list(state.spiral.project(project_id).indexes.list_indexes())
+
+    if not indexes:
+        rich.print("[red]No indexes found[/red]")
+        raise typer.Exit(1)
+
+    return questionary.select(
+        title,
+        choices=[Choice(title=index.name, value=index.id) for index in sorted(indexes, key=lambda t: (t.name, t.id))],
+    ).ask()
+
+
+def get_text_index_id(
+    project: ProjectArg,
+    name: Annotated[str | None, Option(help="Index name.")] = None,
+) -> IndexId:
+    if name is None:
+        return ask_index(project)
+
+    indexes: list[TextIndexResource] = list(state.spiral.project(project).indexes.list_indexes())
+    for index in indexes:
+        if index.name == name:
+            return index.id
+    raise ValueError(f"Index not found: {name}")

spiral/cli/indexes/workers.py

@@ -0,0 +1,59 @@
+from typing import Annotated
+
+import rich
+from typer import Option
+
+from spiral.api.text_indexes import CreateWorkerRequest
+from spiral.api.workers import CPU, GcpRegion, Memory
+from spiral.cli import AsyncTyper, state
+from spiral.cli.indexes.args import get_text_index_id
+from spiral.cli.types import ProjectArg
+
+app = AsyncTyper(short_help="Text Search Workers.")
+
+
+@app.command(name="serve", help="Create a search worker.")
+def serve(
+    project: ProjectArg,
+    index: Annotated[str | None, Option(help="Index name.")] = None,
+    region: Annotated[GcpRegion, Option(help="GCP region for the worker.")] = GcpRegion.US_EAST4,
+    cpu: Annotated[CPU, Option(help="CPU resources for the worker.")] = CPU.ONE,
+    memory: Annotated[Memory, Option(help="Memory resources for the worker in MB.")] = Memory.MB_512,
+):
+    """Create a new text search worker."""
+    index_id = get_text_index_id(project, index)
+    request = CreateWorkerRequest(cpu=cpu, memory=memory, region=region)
+    response = state.spiral.api.text_indexes.create_worker(index_id, request)
+    rich.print(f"Created worker {response.worker_id} for {index_id}.")
+
+
+@app.command(name="shutdown", help="Shutdown a search worker.")
+def shutdown(worker_id: str):
+    """Shutdown a worker."""
+    state.spiral.api.text_indexes.shutdown_worker(worker_id)
+    rich.print(f"Requested worker {worker_id} to shutdown.")
+
+
+@app.command(name="ls", help="List search workers.")
+def ls(
+    project: ProjectArg,
+    index: Annotated[str | None, Option(help="Index name.")] = None,
+):
+    """List text search workers."""
+    index_id = get_text_index_id(project, index)
+    worker_ids = state.spiral.api.text_indexes.list_workers(index_id)
+
+    rich_table = rich.table.Table("Worker ID", "URL", title=f"Text Search Workers for {index_id}")
+    for worker_id in worker_ids:
+        try:
+            worker = state.spiral.api.text_indexes.get_worker(worker_id)
+            rich_table.add_row(
+                worker_id,
+                worker.url,
+            )
+        except Exception:
+            rich_table.add_row(
+                worker_id,
+                "Unavailable",
+            )
+    rich.print(rich_table)

spiral/cli/tables/__init__.py

@@ -1,35 +1,16 @@
 from typing import Annotated
 
-import questionary
 import rich
-import typer
-from questionary import Choice
 from typer import Argument, Option
 
 from spiral import Spiral
-from spiral.cli import AsyncTyper, state
+from spiral.cli import AsyncTyper
+from spiral.cli.tables.args import get_table
 from spiral.cli.types import ProjectArg
-from spiral.tables import Table
 
 app = AsyncTyper(short_help="Spiral Tables.")
 
 
-def ask_table(project_id, title="Select a table"):
-    tables = list(state.spiral.project(project_id).tables.list_tables())
-
-    if not tables:
-        rich.print("[red]No tables found[/red]")
-        raise typer.Exit(1)
-
-    return questionary.select(
-        title,
-        choices=[
-            Choice(title=f"{table.dataset}.{table.table}", value=f"{table.project_id}.{table.dataset}.{table.table}")
-            for table in tables
-        ],
-    ).ask()
-
-
 @app.command(help="List tables.")
 def ls(
     project: ProjectArg,
@@ -48,7 +29,7 @@ def key_schema(
     table: Annotated[str | None, Option(help="Table name.")] = None,
     dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
 ):
-    _, table = _get_table(project, table, dataset)
+    _, table = get_table(project, table, dataset)
     rich.print(table.key_schema)
 
 
@@ -58,7 +39,7 @@ def schema(
     table: Annotated[str | None, Option(help="Table name.")] = None,
     dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
 ):
-    _, table = _get_table(project, table, dataset)
+    _, table = get_table(project, table, dataset)
     rich.print(table.schema)
 
 
@@ -68,7 +49,7 @@ def flush(
     table: Annotated[str | None, Option(help="Table name.")] = None,
     dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
 ):
-    identifier, table = _get_table(project, table, dataset)
+    identifier, table = get_table(project, table, dataset)
     table.maintenance().flush_wal()
     print(f"Flushed WAL for table {identifier} in project {project}.")
 
@@ -80,7 +61,7 @@ def debug(
     dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
     column_group: Annotated[str, Argument(help="Dot-separated column group path.")] = ".",
 ):
-    _, table = _get_table(project, table, dataset)
+    _, table = get_table(project, table, dataset)
     if column_group != ".":
         projection = table[column_group]
     else:
@@ -97,7 +78,7 @@ def manifests(
     dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
     column_group: Annotated[str, Argument(help="Dot-separated column group path.")] = ".",
 ):
-    _, table = _get_table(project, table, dataset)
+    _, table = get_table(project, table, dataset)
     if column_group != ".":
         projection = table[column_group]
     else:
@@ -105,17 +86,3 @@ def manifests(
     scan = projection.scan()
 
     scan._dump_manifests()
-
-
-def _get_table(
-    project: ProjectArg,
-    table: Annotated[str | None, Option(help="Table name.")] = None,
-    dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
-) -> (str, Table):
-    if table is None:
-        identifier = ask_table(project)
-    else:
-        identifier = table
-        if dataset is not None:
-            identifier = f"{dataset}.{table}"
-    return identifier, state.spiral.project(project).tables.table(identifier)

spiral/cli/tables/args.py

@@ -0,0 +1,42 @@
+from typing import Annotated
+
+import questionary
+import rich
+import typer
+from questionary import Choice
+from typer import Option
+
+from spiral.api.projects import TableResource
+from spiral.cli import state
+from spiral.cli.types import ProjectArg
+from spiral.tables import Table
+
+
+def ask_table(project_id, title="Select a table"):
+    tables: list[TableResource] = list(state.spiral.project(project_id).tables.list_tables())
+
+    if not tables:
+        rich.print("[red]No tables found[/red]")
+        raise typer.Exit(1)
+
+    return questionary.select(
+        title,
+        choices=[
+            Choice(title=f"{table.dataset}.{table.table}", value=f"{table.project_id}.{table.dataset}.{table.table}")
+            for table in sorted(tables, key=lambda t: (t.dataset, t.table))
+        ],
+    ).ask()
+
+
+def get_table(
+    project: ProjectArg,
+    table: Annotated[str | None, Option(help="Table name.")] = None,
+    dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
+) -> (str, Table):
+    if table is None:
+        identifier = ask_table(project)
+    else:
+        identifier = table
+        if dataset is not None:
+            identifier = f"{dataset}.{table}"
+    return identifier, state.spiral.project(project).tables.table(identifier)

spiral/core/index/__init__.pyi

@@ -1,15 +1,7 @@
 import pyarrow as pa
 
-class IndexStatus:
-    status: str
-    staleness_s: int | None
-    # An extent of keys that are indexed.
-    # key_extent: KeyExtent | None
-
 class TextIndex:
     id: str
 
-    def status(self) -> IndexStatus: ...
-
 class SearchScan:
     def to_record_batches(self) -> pa.RecordBatchReader: ...

spiral/core/table/__init__.pyi

@@ -36,6 +36,7 @@ class Table:
 
     id: str
     root_uri: str
+    mount_id: str | None
     key_schema: Schema
     metastore: PyMetastore
 
@@ -68,16 +69,16 @@ class TableScan:
         shuffle_buffer_size: int | None = None,
         shuffle_pool_num_rows: int | None = None,
     ) -> pa.RecordBatchReader: ...
-    def column_group_scan(self, column_group: ColumnGroup) -> ColumnGroupScan: ...
-    def key_space_scan(self, table_id: str) -> KeySpaceScan: ...
+    def column_group_state(self, column_group: ColumnGroup) -> ColumnGroupState: ...
+    def key_space_state(self, table_id: str) -> KeySpaceState: ...
     def metrics(self) -> dict[str, Any]: ...
 
-class KeySpaceScan:
+class KeySpaceState:
     manifest: FragmentManifest
 
     def key_schema(self) -> Schema: ...
 
-class ColumnGroupScan:
+class ColumnGroupState:
     manifest: FragmentManifest
 
     def schema(self) -> Schema: ...

spiral/expressions/__init__.py

@@ -47,7 +47,7 @@ __all__ = [
     "not_",
     "or_",
     "pack",
-    "keyed",
+    "aux",
     "ref",
     "refs",
     "scalar",
@@ -116,26 +116,17 @@ def lift(expr: ExprLike) -> Expr:
     return scalar(expr)
 
 
-def key(name: builtins.str) -> Expr:
-    """Create a variable expression referencing a key column.
+def aux(name: builtins.str, dtype: pa.DataType) -> Expr:
+    """Create a variable expression referencing a column in the auxiliary table.
 
-    Args:
-        name: variable name
-    """
-    return Expr(_lib.expr.keyed(name))
-
-
-def keyed(name: builtins.str, dtype: pa.DataType | None = None) -> Expr:
-    """Create a variable expression referencing a column in the key table.
-
-    Key table is optionally given to `Scan#to_record_batches` function when reading only specific keys
+    Auxiliary table is optionally given to `Scan#to_record_batches` function when reading only specific keys
     or doing cell pushdown.
 
     Args:
         name: variable name
-        dtype: must match dtype of the column in the key table.
+        dtype: must match dtype of the column in the auxiliary table.
     """
-    return Expr(_lib.expr.keyed(name, dtype))
+    return Expr(_lib.expr.aux(name, dtype))
 
 
 def scalar(value: Any) -> Expr:

spiral/expressions/mp4.py

@@ -20,13 +20,13 @@ def read(expr: ExprLike | str, frames: ExprLike | str, crop: ExprLike | str):
 
     Args:
         expr: The referenced `Mp4` bytes.
-            A str is assumed to be the `se.keyed` expression.
+            A str is assumed to be the `se.aux` expression.
         frames: The range of frames to read. Each element must be a list of two uint32,
             frame start and frame end, or null / empty list to read all frames.
-            A str is assumed to be the `se.keyed` expression.
+            A str is assumed to be the `se.aux` expression.
         crop: The crop of the frames to read. Each element must be a list of four uint32,
             x, y, width, height or null / empty list to read full frames.
-            A str is assumed to be the `se.keyed` expression.
+            A str is assumed to be the `se.aux` expression.
 
     Returns:
         An array where each element is a decoded cropped video with fields:
@@ -36,17 +36,17 @@ def read(expr: ExprLike | str, frames: ExprLike | str, crop: ExprLike | str):
             frames: Number of frames with type `pa.uint32()`.
     """
     from spiral import _lib
-    from spiral.expressions import keyed, lift
+    from spiral.expressions import aux, lift
 
     if isinstance(expr, str):
-        expr = keyed(
+        expr = aux(
             expr,
             pa.struct([("__ref__", pa.struct([("id", pa.string()), ("begin", pa.uint64()), ("end", pa.uint64())]))]),
         )
     if isinstance(frames, str):
-        frames = keyed(frames, pa.list_(pa.uint32()))
+        frames = aux(frames, pa.list_(pa.uint32()))
     if isinstance(crop, str):
-        crop = keyed(crop, pa.list_(pa.uint32()))
+        crop = aux(crop, pa.list_(pa.uint32()))
 
     expr = lift(expr)
     frames = lift(frames)

spiral/expressions/refs.py

@@ -29,15 +29,15 @@ def deref(expr: ExprLike | str, field: str | None = None) -> Expr:
     column back into their original form, e.g. binary.
 
     Args:
-        expr: The expression to de-reference. A str is assumed to be the `se.keyed` expression.
+        expr: The expression to de-reference. A str is assumed to be the `se.aux` expression.
         field: If the expr evaluates into struct, the field name of that struct that should be de-referenced.
             If `None`, the expr must evaluate into a reference type.
     """
     from spiral import _lib
-    from spiral.expressions import keyed, lift
+    from spiral.expressions import aux, lift
 
     if isinstance(expr, str):
-        expr = keyed(
+        expr = aux(
             expr,
             pa.struct([("__ref__", pa.struct([("id", pa.string()), ("begin", pa.uint64()), ("end", pa.uint64())]))]),
         )

spiral/indexes/index.py

@@ -1,4 +1,3 @@
-import datetime
 from typing import TYPE_CHECKING
 
 from spiral.core.index import TextIndex as CoreTextIndex
@@ -27,8 +26,3 @@ class TextIndex(Expr):
     @property
     def name(self) -> str:
         return self._name
-
-    def status(self) -> (str, datetime.timedelta | None):
-        """Fetch the status of the index. If status is ready, returns the staleness of the index."""
-        status = self._index.status()
-        return status.status, datetime.timedelta(seconds=status.staleness_s) if status.staleness_s is not None else None

spiral/substrait_.py

@@ -248,11 +248,6 @@ class SubstraitConverter:
             case "struct_field", ref:
                 ref: ExpressionReferenceSegmentStructField
                 field_name = scope_type.field(ref.field).name
-
-                if field_name in self.key_names:
-                    # This is a key column, so we need to select it from the scope.
-                    return se.key(field_name)
-
                 scope = se.getitem(scope, field_name)
                 scope_type = scope_type.field(ref.field).type
                 if ref.is_set("child"):

spiral/tables/client.py

@@ -25,6 +25,9 @@ class Tables:
     """
 
     def __init__(self, api: SpiralAPI, spiral: CoreSpiral, *, project_id: str | None = None):
+        if project_id == "":
+            raise ValueError("Project ID cannot be an empty string.")
+
         self._api = api
         self._spiral = spiral
         self._project_id = project_id

spiral/tables/debug/manifests.py

@@ -10,14 +10,14 @@ def display_manifests(scan: TableScan):
         raise NotImplementedError("Multiple table scans are not supported.")
     table_id = scan.table_ids()[0]
 
-    key_space_manifest: FragmentManifest = scan.key_space_scan(table_id).manifest
+    key_space_manifest: FragmentManifest = scan.key_space_state(table_id).manifest
     _table_of_fragments(
         key_space_manifest,
         title="Key Space manifest",
     )
 
     for column_group in scan.column_groups():
-        column_group_manifest: FragmentManifest = scan.column_group_scan(column_group).manifest
+        column_group_manifest: FragmentManifest = scan.column_group_state(column_group).manifest
         _table_of_fragments(
             column_group_manifest,
             title=f"Column Group manifest for {str(column_group)}",

spiral/tables/debug/scan.py

@@ -15,17 +15,17 @@ def show_scan(scan: TableScan):
     column_groups = scan.column_groups()
 
     splits = scan.split()
-    key_space_scan = scan.key_space_scan(table_id)
+    key_space_state = scan.key_space_state(table_id)
 
     # Collect all key bounds from all manifests. This makes sure all visualizations are aligned.
     key_points = set()
-    key_space_manifest = key_space_scan.manifest
+    key_space_manifest = key_space_state.manifest
     for i in range(len(key_space_manifest)):
         fragment_file = key_space_manifest[i]
         key_points.add(fragment_file.key_extent.min)
         key_points.add(fragment_file.key_extent.max)
     for cg in column_groups:
-        cg_scan = scan.column_group_scan(cg)
+        cg_scan = scan.column_group_state(cg)
         cg_manifest = cg_scan.manifest
         for i in range(len(cg_manifest)):
             fragment_file = cg_manifest[i]
@@ -39,7 +39,7 @@ def show_scan(scan: TableScan):
 
     show_manifest(key_space_manifest, scope="Key space", key_points=key_points, splits=splits)
     for cg in scan.column_groups():
-        cg_scan = scan.column_group_scan(cg)
+        cg_scan = scan.column_group_state(cg)
         # Skip table id from the start of the column group.
         show_manifest(cg_scan.manifest, scope=".".join(cg.path[1:]), key_points=key_points, splits=splits)
 

spiral/tables/table.py

@@ -71,11 +71,6 @@ class Table(Expr):
         return f'Table("{self.identifier}")'
 
     def __getitem__(self, item: str) -> Expr:
-        from spiral import expressions as se
-
-        if item in self._key_columns:
-            return se.key(name=item)
-
         return super().__getitem__(item)
 
     def select(self, *paths: str, exclude: list[str] = None) -> "Expr":
@@ -86,14 +81,7 @@ class Table(Expr):
                     "Cannot use 'exclude' arg with key columns. Use 'exclude_keys' and an explicit select of keys."
                 )
 
-        key_paths = set(paths) & self._key_columns
-        other_paths = set(paths) - key_paths
-        if not key_paths:
-            return super().select(*paths, exclude=exclude)
-
-        from spiral import expressions as se
-
-        return se.merge(se.pack({key: se.key(key) for key in key_paths}), super().select(*other_paths, exclude=exclude))
+        return super().select(*paths, exclude=exclude)
 
     @property
     def key_schema(self) -> Schema: