pyspiral 0.4.0__pp310-pypy310_pp73-macosx_10_12_x86_64.whl

pyspiral-0.4.0.dist-info/METADATA

@@ -0,0 +1,46 @@
+Metadata-Version: 2.4
+Name: pyspiral
+Version: 0.4.0
+Classifier: Intended Audience :: Science/Research
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Rust
+Classifier: License :: Other/Proprietary License
+Requires-Dist: betterproto==2.0.0b7
+Requires-Dist: google-re2>=1.1.20240702
+Requires-Dist: grpclib>=0.4.7
+Requires-Dist: hishel>=0.0.30
+Requires-Dist: httpx>=0.27.0
+Requires-Dist: numpy>=2
+Requires-Dist: pyarrow>=21.0.0
+Requires-Dist: pydantic-settings>=2.3.4
+Requires-Dist: pydantic[email]>=2.5.3
+Requires-Dist: pyjwt[crypto]>=2.9.0
+Requires-Dist: questionary>=2.0.1
+Requires-Dist: tqdm>=4.66.5
+Requires-Dist: typer>=0.16
+Requires-Dist: xxhash>=3.4.1
+Requires-Dist: nanoid>=2.0.0
+Requires-Dist: sqlglot[rs]>=25.25.1
+Requires-Dist: pyperclip>=1.9.0
+Requires-Dist: polars>=1.31.0 ; extra == 'polars'
+Requires-Dist: duckdb>=1.3.2 ; extra == 'duckdb'
+Requires-Dist: pyiceberg>=0.9.1 ; extra == 'pyiceberg'
+Provides-Extra: polars
+Provides-Extra: duckdb
+Provides-Extra: pyiceberg
+Summary: Python client for Spiral.
+Home-Page: https://spiraldb.com
+Author-email: SpiralDB <hello@spiraldb.com>
+License: Proprietary License
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
+
+# PySpiral
+

pyspiral-0.4.0.dist-info/RECORD

@@ -0,0 +1,98 @@
+pyspiral-0.4.0.dist-info/METADATA,sha256=B17wQZ-DamgU1NYanta7NGFB-8iPqpYmASeHv5QBQgk,1610
+pyspiral-0.4.0.dist-info/WHEEL,sha256=JiwhipTW0_LsGDAT-A3mDb2ZFAL_o8nDI5ZvqBdvwBg,113
+pyspiral-0.4.0.dist-info/entry_points.txt,sha256=uft7u-a6g40NLt4Q6BleWbK4NY0M8nZuYPpP8DV0EOk,45
+spiral/__init__.py,sha256=Jv1vbcnnmcTsBLN5mSNjnX3ae4C_mgojXDSBFaqIhN0,208
+spiral/_lib.pypy310-pp73-darwin.so,sha256=-KxxizeEA9H_eAvIutS_wHZdCySET1QpQkc5fYUKtrg,66358892
+spiral/adbc.py,sha256=HcvR60uQeEK2oggSAK6y5VYtIrACIiCQ-85MEf18EZc,14199
+spiral/api/__init__.py,sha256=_7BS1RhqEFjnt3XwFWZNCHVEQeSKpezPevAiGCsvDbE,1776
+spiral/api/admin.py,sha256=A1iVR1XYJSObZivPAD5UzmPuMgupXc9kaHNYYa_kwfs,585
+spiral/api/client.py,sha256=9-L6T8niQAXo90jRxllJD4hXXmcGfHj7CW9X3XTYa5Q,4551
+spiral/api/filesystems.py,sha256=5Ky_otnresGj7WdsR8Xi7DDM3lkB8UES6Lru_xWAGDM,4559
+spiral/api/organizations.py,sha256=B-8zZ7lFJANGK7dUNbo_aU-cgI959JBP9VcWb6wdgi0,1895
+spiral/api/projects.py,sha256=JBGof9A2Ivasu2jrULMjHBwlna0M8WRrTNqU-Es4GJ8,5673
+spiral/api/telemetry.py,sha256=tfdA3E_EWJwFVxkQfkm8tiYGRubnx2LuE5nbfsk1oG4,474
+spiral/api/types.py,sha256=zx-BRKsi1GHg9aL9gMUaVQWYYMXJcP0A8OQUc7jSIAc,653
+spiral/api/workloads.py,sha256=XAyXV7vgZcoyyoPoGvOT4jTpyFKFMvrrAfhL6d1h1kE,1748
+spiral/arrow_.py,sha256=T1LZ7bh9aMDbXfpUsf0dR0E1roTQyAYSgZ2mL4s8J_4,7681
+spiral/cli/__init__.py,sha256=ooAFz_iCpVCKHE0TiVElIynbP2PtTgD9cUw46Vh1lcw,2145
+spiral/cli/__main__.py,sha256=kNaKM2xgJo7GRogf83nYldLM-RGUR6vymdGwZxywQu0,71
+spiral/cli/admin.py,sha256=7WbU_tr05clUjmZ-RkKTlvcf1pbXIElRfHRJlCItFGk,326
+spiral/cli/app.py,sha256=-k0rrLbfJRLay_2_MOCt57PLcx0VnNMCkrnKV7j7nos,1725
+spiral/cli/console.py,sha256=6JHbAQV6MFWz3P-VzqPOjhHpkIQagsCdzTMvmuDKMkU,2580
+spiral/cli/fs.py,sha256=dVPoAoAbuQ9yJlfI-JiFgS9VdnPmeBMygVHgehJRj34,4367
+spiral/cli/iceberg/__init__.py,sha256=IQV_gwCFSj6Ubxs58VM9Pal1ymgG2bxdDgOPuk9E5bs,214
+spiral/cli/iceberg/namespaces.py,sha256=x9pvHlcXtcATYYjqimHa6CtkyL3taQUJ--ni_Bfoemc,1510
+spiral/cli/iceberg/tables.py,sha256=nSR4-t54otJfCmubB6vXnbOkbqPVGV0sHBlc-t9cIVg,1930
+spiral/cli/indexes/__init__.py,sha256=-USfxCIdckzZKBNQ-DXqe3V5ttWVo_Fsa1Mfcx5hdIw,467
+spiral/cli/login.py,sha256=InKMnpV8NATW5RPgB3ZL-DSVPzUuUByyK4Fx7pZEgfg,607
+spiral/cli/orgs.py,sha256=V-4ZTT3FwFQLcs1-BenC8uCgvWOJcxkZPSdCPfsexhc,2848
+spiral/cli/printer.py,sha256=W83KAE-7meoDD1yRltLQrZqrA2olGapBGy_2USWkY08,1778
+spiral/cli/projects.py,sha256=TKXu_VzkIUccwXzdlg-wQMkrB-Py33g052NrbuJx-D4,5096
+spiral/cli/state.py,sha256=10wTIVQ0SJkY67Z6-KQ1LFlt3aVIPmZhoHFdTwp4kNA,130
+spiral/cli/tables/__init__.py,sha256=lkGLDeU28IVnuxJdlYSUh6QSB9fQ4_1MeZJL73iXcHo,3660
+spiral/cli/telemetry.py,sha256=ABDCyV5QJGOIJp4AxvK0LG5xNPIysP37K5haL38T7P4,586
+spiral/cli/types.py,sha256=YG1eHhRLaqlVU_18DQBuF_YMsabhMZLBY0V9CvbSxjY,1369
+spiral/cli/workloads.py,sha256=SbxgwiBlX1AuqpOLV3gs7DFkH-Tbeend7qJTwq0Je84,1994
+spiral/client.py,sha256=K-OuMOTgYxOA9vef5jSANjmPRBfGrzQ65fg6Fd-rHMY,2683
+spiral/core/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+spiral/core/client/__init__.pyi,sha256=FFpUCPyQjH_VxldflqWmvniSugLuDiQv11vMim9HhDY,3466
+spiral/core/index/__init__.pyi,sha256=NPOG1ztFO6siBGpmJU3boRzX26xfxw--2TiCydosGvo,314
+spiral/core/table/__init__.pyi,sha256=dwOaxcOl6ZIlxoLjOnC3CNUgGetWfnEV1Jx06aCH8M8,3265
+spiral/core/table/manifests/__init__.pyi,sha256=3V59-K1qr1z2dGfgRKXaHSVheK8NNw8Q8PFhfbeQd_4,1065
+spiral/core/table/metastore/__init__.pyi,sha256=dMqySDnsjPUTBuFU2MaQGyocKEoGkWpeTQmUP2iIKbc,1880
+spiral/core/table/spec/__init__.pyi,sha256=D4GQp9RWwyLKTlRW7eDXcQE-xA5rF2iBcXZ8y7b48EE,5595
+spiral/datetime_.py,sha256=1TA1RYIRU22qcUuipIjVhAtGnPDVn2z9WttuhkmfkwY,964
+spiral/expressions/__init__.py,sha256=QAEtghxWoT-obmMsHpWwPloRh5EZFpeQ7H_i9F4PG5c,6556
+spiral/expressions/base.py,sha256=q_W9XslcdFQtOIE_d1VkEmLickaXKOAoIcFeMoh-nqQ,4751
+spiral/expressions/http.py,sha256=begUydWoFHEqjeLkATvI_v66Ez6_rR-OQBWO5cHbb9c,2742
+spiral/expressions/io.py,sha256=gJ2a0FKMmdxarWKENulPRwH7KDvSJTIh_OUxX306xAM,3045
+spiral/expressions/list_.py,sha256=MMt5lf5H1M3O-x6N_PvqOLGq9NOk6Ukv0fPWwPC_uy4,1809
+spiral/expressions/mp4.py,sha256=R-fcVYRI6KaH1Nwpmqnsc1VYd9wA7Nuiy2UDcNxEzpw,2165
+spiral/expressions/png.py,sha256=KO8X0OmMzUFwpg2I_j0JTyldPzVXDWIMzjWMWDV9vIY,506
+spiral/expressions/qoi.py,sha256=gvIbb6fXb_Bb080sn9wkpbGGrPs2UEcTXCfuv4-kcYQ,506
+spiral/expressions/refs.py,sha256=ISMtJtUL--BjHF6rsvgN3Um4QcvVqQE9URngOxjQrhw,2115
+spiral/expressions/str_.py,sha256=tY8RXW3JWvr1-bEfCZtk5FAf11wKJnXPuA9EoeJ9tA4,1265
+spiral/expressions/struct.py,sha256=pGAnCDh6AK0BK1XfZ1qG4ce4ranIQEE1HQsgmzBcfwQ,2038
+spiral/expressions/text.py,sha256=-02gBWYoyNQ3qQ1--9HTa8IryUDojYQVIp8C7rgnOWQ,1893
+spiral/expressions/tiff.py,sha256=fQwIn0kLFBM2Y3YYIHmTgb_EIRHKT2fNc77nioDQQw4,8044
+spiral/expressions/udf.py,sha256=r6398z2Aj7KnXtwEvCiGNbgOXa6xsb_bnnG-FEvFxV4,1370
+spiral/grpc_.py,sha256=f3czdP1Mxme42Y5--a5ogYq1TTiWn-J_MlGjwJ2mWwM,1015
+spiral/iceberg/__init__.py,sha256=jSIlTxWauAbJV5gsWglZisFbnfNNzLYN90scoYcdWzc,65
+spiral/iceberg/client.py,sha256=E6FyE_h2HLgDW1cAFg1XgglJr6rbVOCWjRtRmqoMVkM,1003
+spiral/indexes/__init__.py,sha256=TXLQ-_3xso3lFIp2lM58_ip9OPNwPKFv1FdsWiUF-d8,178
+spiral/indexes/client.py,sha256=NsFBILEHMjyCUruFrUEKucRQRrN4OvqgbL4pmzWs07g,5600
+spiral/indexes/index.py,sha256=4CmSFlZYp46B2CjqtiyZ7VF5EH3duiutz3nWFnyApLA,973
+spiral/indexes/scan.py,sha256=B2m-UgNuawNB90HXK33GTQfMy2WLdNNxiiB6cIjFW2Y,697
+spiral/project.py,sha256=0uJ1Jb88Ie-cCNnSdX3QfFtCUqrjLka4zCm_TxCpVak,1189
+spiral/protogen/_/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+spiral/protogen/_/arrow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+spiral/protogen/_/arrow/flight/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+spiral/protogen/_/arrow/flight/protocol/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+spiral/protogen/_/arrow/flight/protocol/sql/__init__.py,sha256=_xhj9QkWEW1qZ-iVxcQ8k4EjYr7KJ5ofitJGqVUGQi4,79921
+spiral/protogen/_/scandal/__init__.py,sha256=X5YJqErZDIXxTESw8fLqJp3P2wZlqAglBzPs3LpTd-w,5145
+spiral/protogen/_/spiral/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+spiral/protogen/_/spiral/table/__init__.py,sha256=o_aNyTuJBIRY6MlAWceMsjbfaSUuZphRxiG_IXmC0mU,629
+spiral/protogen/_/substrait/__init__.py,sha256=pV4-T-lwAHKkfFrNYSUGY4IkbIvuKjSo_imzF7BLj_s,126526
+spiral/protogen/_/substrait/extensions/__init__.py,sha256=yD7dg0TBqn-GK_L0qeVof1GKnwSLg_kPyQSV3kcSljs,3655
+spiral/protogen/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+spiral/protogen/substrait/__init__.py,sha256=pV4-T-lwAHKkfFrNYSUGY4IkbIvuKjSo_imzF7BLj_s,126526
+spiral/protogen/substrait/extensions/__init__.py,sha256=yD7dg0TBqn-GK_L0qeVof1GKnwSLg_kPyQSV3kcSljs,3655
+spiral/protogen/util.py,sha256=smnvVo6nYH3FfDm9jqhNLaXz4bbTBaQezHQDCTvZyiQ,1486
+spiral/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+spiral/server.py,sha256=ztBmB5lBnUz-smQxR_tC8AI5SOhz17wH0MI3GuzDUdM,600
+spiral/settings.py,sha256=PIQV2ljtB3pEOWoMRVSRzSGJNrXviO2JBgZ5ZY_Nq2E,2794
+spiral/substrait_.py,sha256=RNSmfbGFT_5uyo8AFtzS9A7IHW3DkacMTw2vKnj0Das,12762
+spiral/tables/__init__.py,sha256=iiP7BkHA117em37_e75jtdvoZC10xCXtld18gRnPbTw,430
+spiral/tables/client.py,sha256=l_wJJRf3BPD5lg4Q1Ll2lAqQIuBCnKwC6JtsAui91Tc,4915
+spiral/tables/dataset.py,sha256=DuHeKVCJfXLsbxmde9QW6yvesW5uhswG6qAxV5X0ZgA,7890
+spiral/tables/debug/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+spiral/tables/debug/manifests.py,sha256=E_-DiMBg2EPL97cl9hLWhiqEsFtjEBgh_C7jZy8EWYc,2594
+spiral/tables/debug/metrics.py,sha256=XdRDcjggtsLNGCAjam6IxG9072pz_d2C8iLApNRFUtk,2044
+spiral/tables/debug/scan.py,sha256=-IWX_UjO4QP9Hj7PtZ1rLlbswJcryOin56GT-exqFm4,8942
+spiral/tables/maintenance.py,sha256=7Xa2Jdu_OY1Qu6iN1sPVdywVZtk_Mv3EaC3G93cmQvI,305
+spiral/tables/scan.py,sha256=3lPf5fSyF1fHGdGJ-pvu5HxPWoonf_XL7neWTqzB-0I,7582
+spiral/tables/snapshot.py,sha256=2NTuVEp2uJ1pV3Q5tLj7FOzPSc9axlfb6uOITwHnj0g,2229
+spiral/tables/table.py,sha256=4B2drwwfaoL6aIJ-5Ll-Bqza-EBeDIfMkuszSOZqSpk,5326
+spiral/tables/transaction.py,sha256=3a64R-mf_cmR54BNn8U-05jmWonp6Ivxhe6u01Dyjzo,1573
+spiral/types_.py,sha256=W_jyO7F6rpPiH69jhgSgV7OxQZbOlb1Ho3InpKUP6Eo,155
+pyspiral-0.4.0.dist-info/RECORD,,

pyspiral-0.4.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: maturin (1.9.3)
+Root-Is-Purelib: false
+Tag: pp310-pypy310_pp73-macosx_10_12_x86_64

pyspiral-0.4.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+spiral=spiral.cli.app:main

spiral/__init__.py

@@ -0,0 +1,10 @@
+"""Python client for Spiral"""
+
+# This is here to make sure we load the native extension first
+from spiral import _lib
+
+assert _lib
+
+from spiral.client import Spiral  # noqa: E402, I001
+
+__all__ = ["Spiral"]

spiral/adbc.py

@@ -0,0 +1,393 @@
+import abc
+import functools
+import logging
+from concurrent.futures import ThreadPoolExecutor
+from urllib.parse import urlparse
+
+import pyarrow as pa
+import pyarrow.compute as pc
+import sqlglot
+import sqlglot.expressions as exp
+from betterproto.lib.google.protobuf import Any
+from pyarrow.flight import (
+    Action,
+    FlightDescriptor,
+    FlightEndpoint,
+    FlightError,
+    FlightInfo,
+    FlightMetadataWriter,
+    FlightServerBase,
+    MetadataRecordBatchReader,
+    RecordBatchStream,
+    ServerCallContext,
+    Ticket,
+)
+
+from spiral import Spiral
+from spiral.api.projects import TableResource
+from spiral.protogen._.arrow.flight.protocol import sql as rpc
+from spiral.protogen._.arrow.flight.protocol.sql import (
+    CommandGetCatalogs,
+    CommandGetDbSchemas,
+    CommandGetSqlInfo,
+    CommandGetTables,
+    CommandStatementQuery,
+    SqlInfo,
+    SqlSupportedTransaction,
+)
+
+log = logging.getLogger(__name__)
+logging.getLogger("sqlx").setLevel(logging.WARNING)
+
+
+def debuggable(func):
+    """A decorator to enable GUI (i.e. PyCharm) debugging in the
+    decorated Arrow Flight RPC Server function.
+
+    See: https://github.com/apache/arrow/issues/36844
+    for more details...
+    """
+
+    @functools.wraps(func)
+    def wrapper_decorator(*args, **kwargs):
+        try:
+            import pydevd
+
+            pydevd.connected = True
+            pydevd.settrace(suspend=False)
+        except ImportError:
+            # Not running in debugger
+            pass
+        value = func(*args, **kwargs)
+        return value
+
+    return wrapper_decorator
+
+
+# TODO(marko): This should work for Iceberg tables.
+class ADBCServerBase:
+    def get_sql_info(self, _req: CommandGetSqlInfo) -> pa.RecordBatchReader:
+        """Default implementation that reports no support for any complex features."""
+        info = {
+            SqlInfo.FLIGHT_SQL_SERVER_NAME: "Spiral ADBC Server",
+            SqlInfo.FLIGHT_SQL_SERVER_VERSION: "0.0.1",
+            SqlInfo.FLIGHT_SQL_SERVER_ARROW_VERSION: pa.__version__,
+            SqlInfo.FLIGHT_SQL_SERVER_READ_ONLY: True,
+            SqlInfo.FLIGHT_SQL_SERVER_TRANSACTION: SqlSupportedTransaction.NONE.value,
+        }
+
+        # See https://github.com/apache/arrow-adbc/blob/38c21c2311a59803559cb0091b3f34180c28b25f/rust/core/src/schemas.rs#L35
+        union_fields = [
+            pa.field("string_value", pa.string()),
+            pa.field("bool_value", pa.bool_()),
+            pa.field("int64_value", pa.int64()),
+            pa.field("int32_bitmask", pa.int32()),
+            pa.field("string_list", pa.list_(pa.string())),
+            pa.field(
+                "int32_to_int32_list_map",
+                pa.map_(pa.int32(), pa.list_(pa.int32()), keys_sorted=False),
+            ),
+        ]
+        schema = pa.schema(
+            [
+                pa.field("info_name", pa.uint32(), nullable=False),
+                pa.field("info_value", pa.dense_union(union_fields), nullable=False),
+            ]
+        )
+
+        # PyArrow doesn't support creating a dense union for us :(
+        types = []
+        offsets = []
+        ints = []
+        bools = []
+        strs = []
+        for value in info.values():
+            if isinstance(value, str):
+                types.append(0)
+                offsets.append(len(strs))
+                strs.append(value)
+            elif isinstance(value, bool):
+                types.append(1)
+                offsets.append(len(bools))
+                bools.append(value)
+            else:
+                types.append(1)
+                offsets.append(len(ints))
+                ints.append(value)
+
+        values = pa.UnionArray.from_dense(
+            pa.array(types, type=pa.int8()),
+            pa.array(offsets, type=pa.int32()),
+            [pa.array(data, type=f.type) for data, f in zip([strs, bools, ints, [], [], []], union_fields)],
+            [f.name for f in union_fields],
+        )
+
+        return pa.table(data=[pa.array(list(info.keys()), type=pa.uint32()), values], schema=schema).to_reader()
+
+    @abc.abstractmethod
+    def get_catalogs(self, req: CommandGetCatalogs) -> pa.RecordBatchReader: ...
+
+    @abc.abstractmethod
+    def get_db_schemas(self, req: CommandGetDbSchemas) -> pa.RecordBatchReader: ...
+
+    @abc.abstractmethod
+    def get_tables(self, req: CommandGetTables) -> pa.RecordBatchReader: ...
+
+    @abc.abstractmethod
+    def statement_query(self, req: CommandStatementQuery, limit: int | None = None) -> pa.RecordBatchReader: ...
+
+
+class SpiralADBCServer(ADBCServerBase):
+    def __init__(self, spiral: Spiral):
+        self.sp = spiral
+
+        self.pool = ThreadPoolExecutor()
+
+    def get_catalogs(self, req: CommandGetCatalogs) -> pa.RecordBatchReader:
+        schema = pa.schema([pa.field("catalog_name", pa.string(), nullable=False)])
+
+        @debuggable
+        def batches():
+            yield pa.RecordBatch.from_arrays(
+                [[p.id for p in self.sp.list_projects()]],
+                schema=schema,
+            )
+
+        return pa.RecordBatchReader.from_batches(schema, batches())
+
+    def get_db_schemas(self, req: CommandGetDbSchemas) -> pa.RecordBatchReader:
+        """Get the schemas from the database."""
+
+        schema = pa.schema(
+            [
+                pa.field("catalog_name", pa.string()),
+                pa.field("db_schema_name", pa.string(), nullable=False),
+            ]
+        )
+
+        @debuggable
+        def batches():
+            if req.catalog == "":
+                # Empty string means databases _without_ a catalog, which we don't support
+                return
+
+            # Otherwise, catalog is either the project ID, or None.
+            if req.catalog is None:
+                projects = self.sp.list_projects()
+            else:
+                projects = [self.sp.project(req.catalog)]
+
+            for project in projects:
+                datasets = {dt.dataset for dt in project.tables.list_tables()}
+
+                batch = pa.RecordBatch.from_arrays(
+                    [
+                        [project.id] * len(datasets),
+                        list(datasets),
+                    ],
+                    schema=schema,
+                )
+
+                if req.db_schema_filter_pattern:
+                    mask = pc.match_like(batch["db_schema_name"], req.db_schema_filter_pattern)
+                    batch = batch.filter(mask)
+
+                yield batch
+
+        return pa.RecordBatchReader.from_batches(schema, batches())
+
+    def get_tables(self, req: CommandGetTables) -> pa.RecordBatchReader:
+        schema = pa.schema(
+            [
+                pa.field("catalog_name", pa.string()),
+                pa.field("db_schema_name", pa.string()),
+                pa.field("table_name", pa.string(), nullable=False),
+                pa.field("table_type", pa.string(), nullable=False),
+            ]
+            + [pa.field("table_schema", pa.binary(), nullable=False)]
+            if req.include_schema
+            else []
+        )
+
+        @debuggable
+        def batches():
+            if req.catalog == "":
+                # Empty string means databases _without_ a catalog, which we don't support
+                return
+
+            if req.catalog is None:
+                projects = list(self.sp.list_projects())
+            else:
+                projects = [self.sp.project(req.catalog)]
+
+            def _process_project(project):
+                tables: list[TableResource] = project.tables.list_tables()
+
+                rows = []
+                for table in tables:
+                    row = {
+                        "catalog_name": project.id,
+                        "db_schema_name": table.dataset,
+                        "table_name": table.table,
+                        "table_type": "TABLE",
+                    }
+
+                    if req.include_schema:
+                        open_table = project.tables.table(f"{table.project_id}.{table.dataset}.{table.table}")
+                        row["table_schema"] = open_table.snapshot().to_dataset().schema.serialize().to_pybytes()
+
+                    rows.append(row)
+
+                return pa.RecordBatch.from_pylist(rows, schema=schema)
+
+            yield from self.pool.map(_process_project, projects)
+
+        return pa.RecordBatchReader.from_batches(schema, batches())
+
+    @debuggable
+    def statement_query(self, req: CommandStatementQuery, limit: int | None = None) -> pa.RecordBatchReader:
+        # Extract the tables from the query, and bring them into the Python locals scope.
+        expr = sqlglot.parse_one(req.query, dialect="duckdb")
+        for tbl in expr.find_all(exp.Table):
+            # We swap the three-part identifier out for a single identifier
+            # This lets us insert a PyArrow Dataset into Python locals such that
+            # DuckDB will pick up on it for the query.
+            name = exp.table_name(tbl)
+            locals()[name] = self.sp.tables.table(f"{tbl.catalog}.{tbl.db}.{tbl.name}").snapshot().to_dataset()
+            tbl.replace(exp.table_(table=name))
+
+        try:
+            import duckdb
+        except ImportError:
+            raise FlightError("DuckDB is required for SQL queries.")
+
+        try:
+            sql = duckdb.sql(expr.sql(dialect="duckdb"))
+        except Exception as e:
+            raise FlightError(str(e))
+
+        if limit is not None:
+            sql = sql.limit(limit)
+
+        return sql.fetch_arrow_reader(batch_size=1_000)
+
+
+class ADBCFlightServer(FlightServerBase):
+    """An implementation of a FlightSQL ADBC server."""
+
+    def __init__(self, abdc: ADBCServerBase, *, location=None, **kwargs):
+        super().__init__(location=location, **kwargs)
+        self.location = location
+        self.adbc = abdc
+
+        self.host = "localhost"
+        self.tls = False
+        if location:
+            parts = urlparse(location)
+            self.host = parts.hostname
+            self.tls = parts.scheme.endswith("s")
+
+    @debuggable
+    def do_action(self, context: ServerCallContext, action: Action):
+        log.info("DoAction %s: %s", context.peer(), action)
+        super().do_action(context, action)
+
+    @debuggable
+    def do_exchange(self, context: ServerCallContext, descriptor: FlightDescriptor, reader, writer):
+        log.info("DoExchange %s: %s", context.peer(), descriptor)
+        super().do_exchange(context, descriptor, reader, writer)
+
+    @debuggable
+    def do_get(self, context: ServerCallContext, ticket: Ticket):
+        log.info("DoGet %s: %s", context.peer(), ticket)
+        req = self.parse_command(ticket.ticket)
+        match req:
+            case CommandGetSqlInfo():
+                return RecordBatchStream(self.adbc.get_sql_info(req))
+            case CommandGetCatalogs():
+                return RecordBatchStream(self.adbc.get_catalogs(req))
+            case CommandGetDbSchemas():
+                return RecordBatchStream(self.adbc.get_db_schemas(req))
+            case CommandGetTables():
+                return RecordBatchStream(self.adbc.get_tables(req))
+            case CommandStatementQuery():
+                return RecordBatchStream(self.adbc.statement_query(req))
+            case _:
+                raise NotImplementedError(f"Unsupported do_Get: {req}")
+
+    @debuggable
+    def do_put(
+        self,
+        context: ServerCallContext,
+        descriptor: FlightDescriptor,
+        reader: MetadataRecordBatchReader,
+        writer: FlightMetadataWriter,
+    ):
+        log.info("DoPut %s: %s", context.peer(), descriptor)
+        super().do_put(context, descriptor, reader, writer)
+
+    @debuggable
+    def get_flight_info(self, context: ServerCallContext, descriptor: FlightDescriptor) -> FlightInfo:
+        log.info("GetFlightInfo %s: %s", context.peer(), descriptor)
+        req = self.parse_command(descriptor.command)
+        match req:
+            case CommandGetSqlInfo():
+                # Each metadata type contributes to the schema.
+                schema = self.adbc.get_sql_info(req).schema
+            case CommandGetCatalogs():
+                schema = self.adbc.get_catalogs(req).schema
+            case CommandGetDbSchemas():
+                schema = self.adbc.get_db_schemas(req).schema
+            case CommandGetTables():
+                schema = self.adbc.get_tables(req).schema
+            case CommandStatementQuery():
+                schema = self.adbc.statement_query(req, limit=0).schema
+            case _:
+                raise NotImplementedError(f"Unsupported command: {req}")
+
+        return self._make_flight_info(self.descriptor_to_key(descriptor), descriptor, schema)
+
+    @staticmethod
+    def parse_command(command: bytes):
+        command = Any().parse(command)
+
+        if not command.type_url.startswith("type.googleapis.com/arrow.flight.protocol.sql."):
+            raise NotImplementedError(f"Unsupported command: {command.type_url}")
+
+        proto_cls_name = command.type_url[len("type.googleapis.com/arrow.flight.protocol.sql.") :]
+        proto_cls = getattr(rpc, proto_cls_name)
+        return proto_cls().parse(command.value)
+
+    @staticmethod
+    def descriptor_to_key(descriptor):
+        return descriptor.command
+
+    @debuggable
+    def get_schema(self, context: ServerCallContext, descriptor: FlightDescriptor):
+        log.info("GetSchema %s: %s", context.peer(), descriptor)
+        return super().get_schema(context, descriptor)
+
+    @debuggable
+    def list_actions(self, context: ServerCallContext):
+        log.info("ListActions %s", context.peer())
+        super().list_actions(context)
+
+    @debuggable
+    def list_flights(self, context: ServerCallContext, criteria):
+        log.info("ListFlights %s: %s", context.peer(), criteria)
+        super().list_flights(context, criteria)
+
+    def _make_flight_info(self, key, descriptor, schema: pa.Schema):
+        # If we pass zero locations, the FlightSQL client should attempt to use the original connection.
+        endpoints = [FlightEndpoint(key, [])]
+        return FlightInfo(schema, descriptor, endpoints, -1, -1)
+
+
+if __name__ == "__main__":
+    import logging
+
+    logging.basicConfig()
+    logging.getLogger("spiral").setLevel(logging.DEBUG)
+
+    server = ADBCFlightServer(SpiralADBCServer(Spiral()), location="grpc://localhost:5005")
+    server.serve()

spiral/api/__init__.py

@@ -0,0 +1,64 @@
+import os
+from typing import TYPE_CHECKING
+
+import httpx
+
+from .client import _Client
+
+if TYPE_CHECKING:
+    from spiral.core.client import Authn
+
+    from .admin import AdminService
+    from .filesystems import FileSystemService
+    from .organizations import OrganizationService
+    from .projects import ProjectService
+    from .telemetry import TelemetryService
+    from .workloads import WorkloadService
+
+
+class SpiralAPI:
+    def __init__(self, authn: "Authn", base_url: str | None = None):
+        self.base_url = base_url or os.environ.get("SPIRAL_URL", "https://api.spiraldb.com")
+        self.client = _Client(
+            httpx.Client(
+                base_url=self.base_url,
+                timeout=None if ("PYTEST_VERSION" in os.environ or bool(os.environ.get("SPIRAL_DEV", None))) else 60,
+            ),
+            authn,
+        )
+
+    @property
+    def _admin(self) -> "AdminService":
+        from .admin import AdminService
+
+        return AdminService(self.client)
+
+    @property
+    def organization(self) -> "OrganizationService":
+        from .organizations import OrganizationService
+
+        return OrganizationService(self.client)
+
+    @property
+    def project(self) -> "ProjectService":
+        from .projects import ProjectService
+
+        return ProjectService(self.client)
+
+    @property
+    def file_system(self) -> "FileSystemService":
+        from .filesystems import FileSystemService
+
+        return FileSystemService(self.client)
+
+    @property
+    def workload(self) -> "WorkloadService":
+        from .workloads import WorkloadService
+
+        return WorkloadService(self.client)
+
+    @property
+    def telemetry(self) -> "TelemetryService":
+        from .telemetry import TelemetryService
+
+        return TelemetryService(self.client)

spiral/api/admin.py

@@ -0,0 +1,15 @@
+from .client import Paged, PagedResponse, ServiceBase
+from .organizations import OrgMembership
+from .types import OrgId
+
+
+class AdminService(ServiceBase):
+    def sync_memberships(self, org_id: OrgId | None = None) -> Paged[OrgMembership]:
+        params = {}
+        if org_id:
+            params["org_id"] = str(org_id)
+        return self.client.paged("/v1/admin/sync-memberships", PagedResponse[OrgMembership], params=params)
+
+    def sync_orgs(self) -> Paged[OrgId]:
+        params = {}
+        return self.client.paged("/v1/admin/sync-orgs", PagedResponse[OrgId], params=params)