pyspiral 0.6.2__cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl → 0.6.4__cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl

{pyspiral-0.6.2.dist-info → pyspiral-0.6.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyspiral
-Version: 0.6.2
+Version: 0.6.4
 Classifier: Intended Audience :: Science/Research
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python
@@ -12,7 +12,7 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Rust
 Classifier: License :: Other/Proprietary License
-Requires-Dist: betterproto2>=0.8.0
+Requires-Dist: betterproto2>=0.9.0
 Requires-Dist: google-re2>=1.1.20240702
 Requires-Dist: grpclib>=0.4.7
 Requires-Dist: hishel>=0.0.30
@@ -21,7 +21,7 @@ Requires-Dist: nanoid>=2.0.0
 Requires-Dist: numpy>=2
 Requires-Dist: pyarrow>=21.0.0
 Requires-Dist: pydantic-settings>=2.3.4
-Requires-Dist: pydantic[email]>=2.5.3
+Requires-Dist: pydantic[email]>=2.5.3,<2.12
 Requires-Dist: pyjwt[crypto]>=2.9.0
 Requires-Dist: pyperclip>=1.9.0
 Requires-Dist: questionary>=2.0.1

{pyspiral-0.6.2.dist-info → pyspiral-0.6.4.dist-info}/RECORD

@@ -1,13 +1,13 @@
-pyspiral-0.6.2.dist-info/METADATA,sha256=n0a5SuYMybj-eUwB9No9IFxSk9Cn_pDZfjpx0HuDxRw,1836
-pyspiral-0.6.2.dist-info/WHEEL,sha256=PxcKzGLVtZeSnGJDErQ-Emkn2AvBXbmzIogfnaf7-q8,130
-pyspiral-0.6.2.dist-info/entry_points.txt,sha256=uft7u-a6g40NLt4Q6BleWbK4NY0M8nZuYPpP8DV0EOk,45
+pyspiral-0.6.4.dist-info/METADATA,sha256=NHgJTgogXMcDIHbb3I6ONsKdj23IfS2Vwa60m6YgGZo,1842
+pyspiral-0.6.4.dist-info/WHEEL,sha256=sHl2MPySRQtLBS4t9I9tl1bAeFFBhTGABHYdwnegkVM,130
+pyspiral-0.6.4.dist-info/entry_points.txt,sha256=uft7u-a6g40NLt4Q6BleWbK4NY0M8nZuYPpP8DV0EOk,45
 spiral/__init__.py,sha256=5c0faqg-kHZBDwriQ7LzLAMcFolIucp-IA1EzNvCZ3k,711
-spiral/_lib.abi3.so,sha256=oOsM8f904rj2leVKR8cCOkLQ7uxcf36tKVIN0bCVOok,55184120
+spiral/_lib.abi3.so,sha256=oq_VegFDXsy3-ap1Coy5wf_JHD4s6s8PZ8VguB65bmQ,50403064
 spiral/adbc.py,sha256=7IxfWIeQN-fh0W5OdN_PP2x3pzQYg6ZUOLsHg3jktqw,14842
 spiral/api/__init__.py,sha256=ULBlVq3PnfNOO6T5naE_ULmmii-83--qTuN2PpAUQN0,2241
 spiral/api/admin.py,sha256=A1iVR1XYJSObZivPAD5UzmPuMgupXc9kaHNYYa_kwfs,585
-spiral/api/client.py,sha256=c63u4Nv0XqXW3BpGAofMk44d-1_4RymKwbcMzq9qxeY,4649
-spiral/api/filesystems.py,sha256=EA4iqhTeaIlvObvEUxHmZl0pQ24IOxUVWM3GPhFLw8o,4969
+spiral/api/client.py,sha256=xGc3RKRrerDGGt3QA7Y_friEa-OkZXSQI2Yd1KeFDdw,4668
+spiral/api/filesystems.py,sha256=yEHgHfo7t1_becm0UFedc3nd49_G77hHjYwtYQ6P9XU,4240
 spiral/api/key_space_indexes.py,sha256=-38rZXTdkL4mLhp9h3CtqyIyutzzq88tV6bhK05MqYE,640
 spiral/api/organizations.py,sha256=B-8zZ7lFJANGK7dUNbo_aU-cgI959JBP9VcWb6wdgi0,1895
 spiral/api/projects.py,sha256=62Y1lqI_TpUh3WKQqrjbLWJHiZsI_X3g8u2RTbUwkoA,6162
@@ -20,12 +20,12 @@ spiral/arrow_.py,sha256=T1LZ7bh9aMDbXfpUsf0dR0E1roTQyAYSgZ2mL4s8J_4,7681
 spiral/cli/__init__.py,sha256=LutjpWZu5Rvmba8C8bPa5vOCv74JuAoE1kvz0nd48dE,2476
 spiral/cli/__main__.py,sha256=kNaKM2xgJo7GRogf83nYldLM-RGUR6vymdGwZxywQu0,71
 spiral/cli/admin.py,sha256=-ubYqs8nKjnQStbQ68jpWx_9xh0TsaxI0wM1Hfko8_U,319
-spiral/cli/app.py,sha256=HWCjMJLzSz_JaiLF046jzC9A4-yvzS6506D3cOR2Vgc,1773
+spiral/cli/app.py,sha256=lv37s8nvptxrJuloe9W603Oz5-1n5_BPzbbKdIvBkb4,2759
 spiral/cli/console.py,sha256=6JHbAQV6MFWz3P-VzqPOjhHpkIQagsCdzTMvmuDKMkU,2580
-spiral/cli/fs.py,sha256=UREIJhjr6MfIdcKK7pjUKICd0wsQULhQiWRVWUnQ0dc,4376
+spiral/cli/fs.py,sha256=vaPcSc2YghhHeipxNitIdsHaBhFwlwkvPFqYsFSN9P0,2927
 spiral/cli/iceberg.py,sha256=Q14tcGcn1LixbFCYP0GhfYwFFXTmmi8tqBPYwalJEyE,3248
 spiral/cli/key_spaces.py,sha256=x3IFRP5d47pKiAHeWExYMOBaT2TwxbWjVM01SUqKrwI,2943
-spiral/cli/login.py,sha256=TgTr37ImgG1NKN8VbtqkxVAYaZFpMXMwPAb23gVldEw,649
+spiral/cli/login.py,sha256=iyWQR2n2cOcg2-6NMaD2uCSQfvsoz7wMeyT9s7h80Fc,698
 spiral/cli/orgs.py,sha256=fmOuLxpeIFfKqePRi292Gv9k-EF5pPn_tbKd2BLl2Ig,2869
 spiral/cli/printer.py,sha256=HcvSUpaMItzmhBUfIHROK1Z3SL8J8wDopS3Qo8H00uw,1781
 spiral/cli/projects.py,sha256=UYrBlLcFacuXExdLX1sZByfvkz9MRtk_0oRAZvqHa0w,5105
@@ -37,20 +37,21 @@ spiral/cli/types.py,sha256=XYzo1GgX7dBBItoBSrHI4vO5C2lLmS2sktb-2GnGH3E,1362
 spiral/cli/workloads.py,sha256=2_SLfQTFN6y73R9H0i9dk8VIOVagKxSxOpHXC56yptY,2015
 spiral/client.py,sha256=Po9xgCH3NwVsCeRZMm3eJUPV77Rknyj-9MfCS1TbdTg,6623
 spiral/core/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+spiral/core/_tools/__init__.pyi,sha256=b2KLfTOQ67pjfbYt07o0IGiTu5o2bZw69lllV8v0Dps,143
 spiral/core/authn/__init__.pyi,sha256=Jw_8ywTMDTwgAtGxMtFED63rU0jOgrv-eZtaZ5sR5t4,757
-spiral/core/client/__init__.pyi,sha256=iEhZgbySG5LScfrtkiiHW1iHghgehsrVmPP-v5Pv_vk,5740
-spiral/core/table/__init__.pyi,sha256=sjjShdgM_Uh8Roou1k02MnrqYpdAX4QuyRlIRlnyp1M,3073
+spiral/core/client/__init__.pyi,sha256=6D4eu78eHW9Zl5Fx3UIRbk9ywvlZJephal56H1LDjko,6095
+spiral/core/table/__init__.pyi,sha256=ajxO2N92hTQ4evsl7QBWB8ivz-cDNxXnAv0jytRw0ZY,3183
 spiral/core/table/manifests/__init__.pyi,sha256=eVfDpmhYSjafIvvALqAkZe5baN3Y1HpKpxYEbjwd4gQ,1043
 spiral/core/table/metastore/__init__.pyi,sha256=rc3u9MwEKRvL2kxOc8lBorddFRnM8o_o1frqtae86a4,1697
 spiral/core/table/spec/__init__.pyi,sha256=0NyGeyEhV_ebwKWVU3sqSvdF2D9v8kEVwo6wYAHF99M,5579
 spiral/dataset.py,sha256=NNqG-oOrhbmNC2OMZ9AYAm4YkwwBozeRI6zXtz4cspA,8008
 spiral/datetime_.py,sha256=1TA1RYIRU22qcUuipIjVhAtGnPDVn2z9WttuhkmfkwY,964
 spiral/debug/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-spiral/debug/manifests.py,sha256=oaPB4534pQdqvPXCZetVNSvvhpdXTrv_1pN-_bAkeAo,2893
+spiral/debug/manifests.py,sha256=7f1O3ba9mrA5nXpOF9cEIQuUAteP5wiBkFy_diQJ7No,3216
 spiral/debug/metrics.py,sha256=XdRDcjggtsLNGCAjam6IxG9072pz_d2C8iLApNRFUtk,2044
-spiral/debug/scan.py,sha256=9bMmVQFs5M6Rldm0fmrmmvn9LbSSTKBV5tIu37mEn78,8938
-spiral/expressions/__init__.py,sha256=T8PIb0_UB9kynK0dpWbUD4No5lKRTG-wKnao8xOcXjY,6381
-spiral/expressions/base.py,sha256=OOUDrbkLBE0lSkAmM-6FP2F2N8zhN_in3S_UDrWLDeQ,4805
+spiral/debug/scan.py,sha256=UEm_aRnql5pwDPTpZgakMLNjlzkKL4RurBFFqH_BLAQ,9526
+spiral/expressions/__init__.py,sha256=QQrnKrrWnDk7W5I9yhPS21ME0cUkEou30hga8MVkt1I,6396
+spiral/expressions/base.py,sha256=4qlXbi4IusZi5b4QEadWhXtmuYd0ETzOB1NWMWYIsTs,5163
 spiral/expressions/http.py,sha256=begUydWoFHEqjeLkATvI_v66Ez6_rR-OQBWO5cHbb9c,2742
 spiral/expressions/io.py,sha256=gJ2a0FKMmdxarWKENulPRwH7KDvSJTIh_OUxX306xAM,3045
 spiral/expressions/list_.py,sha256=MMt5lf5H1M3O-x6N_PvqOLGq9NOk6Ukv0fPWwPC_uy4,1809
@@ -65,35 +66,36 @@ spiral/expressions/tiff.py,sha256=fQwIn0kLFBM2Y3YYIHmTgb_EIRHKT2fNc77nioDQQw4,80
 spiral/expressions/udf.py,sha256=yb9MIcrFftpNDxgBF228cvdv6TY-hEFikYz2fq_nzWo,1353
 spiral/grpc_.py,sha256=f3czdP1Mxme42Y5--a5ogYq1TTiWn-J_MlGjwJ2mWwM,1015
 spiral/iceberg.py,sha256=JGq62Qnf296r9_hRAoH85GQq45-uSBjwXWw_CvPi6G4,930
+spiral/iterable_dataset.py,sha256=Eekg9ad8tcwXcloHWReBbvCSr5ZappRHn2ldKTvwqS0,4622
 spiral/key_space_index.py,sha256=NAB_nONEjpMYbse8suz42w7Qb5OPHuKN9h9CT2NJe08,1460
 spiral/project.py,sha256=CO_Pn6vPqaonNvRdCNRFcBWr4TqO2AsAUTH5xawIeCE,7283
 spiral/protogen/_/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 spiral/protogen/_/arrow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 spiral/protogen/_/arrow/flight/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 spiral/protogen/_/arrow/flight/protocol/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-spiral/protogen/_/arrow/flight/protocol/sql/__init__.py,sha256=yt4_UDWfOaVpyCBeQa2aVXIfZzRSrcfIQHsXFCWv0qI,90023
+spiral/protogen/_/arrow/flight/protocol/sql/__init__.py,sha256=ooZZsDCRFpktUCH11OdxMRa_GLQYnY9w-1fBr5a7vBk,90023
 spiral/protogen/_/google/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-spiral/protogen/_/google/protobuf/__init__.py,sha256=lsfhHEPczGOxrOmkstAqh64V0Kt8hQE_6N0tIpc27HU,75116
+spiral/protogen/_/google/protobuf/__init__.py,sha256=H0FVEXusqww2j5dl7Ee05tR6qMG_hQioUp1qFfDgnco,80036
 spiral/protogen/_/message_pool.py,sha256=4-cRhhiM6bmfpUJZ8qxc8LEyqHBHpLCcotjbyZxl7JM,71
 spiral/protogen/_/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-spiral/protogen/_/scandal/__init__.py,sha256=-4m9DHPjtLFzpnesaAv8W_p8R_kfGjA5z3l0GPPbjD8,4965
-spiral/protogen/_/spfs/__init__.py,sha256=4lnc88HhuH4t-JR9NjXz5r5WVESxCEbyUpV7Xfc-SBI,2028
-spiral/protogen/_/spql/__init__.py,sha256=JJBlNacSIIoo5cazHFyLtdkGRLYgwNru1FstFpuPGg8,1548
-spiral/protogen/_/substrait/__init__.py,sha256=-PqWiWMN0hl3Gntj5l1wpEhOMdGDgv3PskGPouaRct8,209839
-spiral/protogen/_/substrait/extensions/__init__.py,sha256=sCMvwWCXWu2cSGiTEH0hRjkn0WTsePLDIxRBBNpENJs,5326
+spiral/protogen/_/scandal/__init__.py,sha256=liUQAICLd2sPccCmqo0_c1duSbNj_m8p_IgmdnHsB3E,4965
+spiral/protogen/_/spfs/__init__.py,sha256=zMMEDIfPXQNBkisLI-iMWbJABye-vK42Gf2BUQQYR_c,2028
+spiral/protogen/_/spql/__init__.py,sha256=PEC4bI-PHdJ4Zd8Jb1k6Xk2iFYoYqIUbTGlL2JVGnT0,1548
+spiral/protogen/_/substrait/__init__.py,sha256=eVv-5CRJv8KstANM-_U3WzCVmkmZ8_BTOFP-1f2sSX4,209839
+spiral/protogen/_/substrait/extensions/__init__.py,sha256=nhnEnho70GAT8WPj2xtwJUzk5GJ6X2e-HTvyk7emGsk,5326
 spiral/protogen/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 spiral/protogen/util.py,sha256=smnvVo6nYH3FfDm9jqhNLaXz4bbTBaQezHQDCTvZyiQ,1486
 spiral/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-spiral/scan.py,sha256=20-NSGsoXYf6uKQ7yEdbbwT8ijIK7KxKTctycsl0AIk,7073
+spiral/scan.py,sha256=SCDZ9UXA4g0Jq9BQ9Zt7cEK2NBq64Hqh_SttR4tF6jo,6252
 spiral/server.py,sha256=ztBmB5lBnUz-smQxR_tC8AI5SOhz17wH0MI3GuzDUdM,600
-spiral/settings.py,sha256=Nap68xM-1ZvF3yDhkyRnNDIAVMIgxmIksglg_1iT0-0,3069
-spiral/snapshot.py,sha256=_l2wrqUXz2RARjIDxOWw4aQpegJohvggIoWuCllzStA,1506
+spiral/settings.py,sha256=cLD1MSyzIM1CRRB2ncYB9u849cxGAqYlycvNExU6TGo,3096
+spiral/snapshot.py,sha256=tYEIKYYqS9Eusb8rsrG46VD7fiNPA9yVOR5ajMMtT_g,2018
 spiral/streaming_/__init__.py,sha256=s7MlW2ERsuZmZGExLFL6RcZon2e0tNBocBg5ANgki7k,61
 spiral/streaming_/reader.py,sha256=Kpqknv2jn12jUhHOEEDArj0JZwrWb8XjoOGs9HrdVyA,4047
-spiral/streaming_/stream.py,sha256=nxJEisPfZ2-Ebkm83hz_3v8NH27FxBku-1jw7UDlQuM,5881
+spiral/streaming_/stream.py,sha256=-prGp73h0XDsdKW0mAEamy4AXhd1oF5fBbNbbY1k2-A,5931
 spiral/substrait_.py,sha256=AKeOD4KIXvz2J4TYxnIneOiHddtBIyOhuNxVO_uH0eg,12592
-spiral/table.py,sha256=ZQFq5tuovDjQcpi38b5FUMuHNGI5XV0MnZbC6vbza1o,10312
+spiral/table.py,sha256=ru1G7CXZGD-k4sg621qe-IEAU9kU1WujZ32AXAmdvx4,8861
 spiral/text_index.py,sha256=FQ9rgIEGLSJryS9lFdMhKtPFey18BXoWbPXyvZPJJ04,442
 spiral/transaction.py,sha256=nSykH4UGs9hGtWuSWK9YyT9jfEuvzfkKoUgMM5Xt4zU,1841
 spiral/types_.py,sha256=W_jyO7F6rpPiH69jhgSgV7OxQZbOlb1Ho3InpKUP6Eo,155
-pyspiral-0.6.2.dist-info/RECORD,,
+pyspiral-0.6.4.dist-info/RECORD,,

{pyspiral-0.6.2.dist-info → pyspiral-0.6.4.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: maturin (1.9.4)
+Generator: maturin (1.9.6)
 Root-Is-Purelib: false
 Tag: cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64

spiral/api/client.py

@@ -129,7 +129,7 @@ class _Client:
     ) -> ResponseT:
         req_data: dict[str, Any] = {}
         if req is not None:
-            req_data = dict(json=TypeAdapter(req.__class__).dump_python(req, mode="json"))
+            req_data = dict(json=TypeAdapter(req.__class__).dump_python(req, mode="json", exclude_none=True))
 
         token = self.authn.token()
         resp = self.http.request(

spiral/api/filesystems.py

@@ -43,10 +43,13 @@ class S3FileSystem(BaseModel):
     """File system backed by an S3-compatible bucket."""
 
     type: Literal["s3"] = "s3"
-    endpoint: str = "https://s3.amazonaws.com"
-    region: str = "auto"
+    endpoint: str | None = None
+    region: str
     bucket: str
-    directory: DirectoryPath | None
+    directory: DirectoryPath | None = None
+    # ARN of the role to assume when accessing the bucket https://docs.spiraldb.com/filesystems#aws
+    # role_arn: str | None = None
+    role_arn: str  # TODO(marko): Make optional once we support third-party S3-compatible storage.
 
 
 class GCSFileSystem(BaseModel):
@@ -55,7 +58,7 @@ class GCSFileSystem(BaseModel):
     type: Literal["gcs"] = "gcs"
     region: str
     bucket: str
-    directory: DirectoryPath | None
+    directory: DirectoryPath | None = None
 
 
 FileSystem = Annotated[
@@ -78,40 +81,6 @@ class Mount(BaseModel):
     principal: str
 
 
-class AWSSecretAccessKey(BaseModel):
-    """AWS secret access key credentials to be used with an S3 file system.
-    The access key must have read/write access to the bucket specified in the file system.
-    """
-
-    access_key_id: str
-    secret_access_key: str
-
-
-class UpdateS3FileSystem(S3FileSystem):
-    credentials: AWSSecretAccessKey
-
-
-class GCPServiceAccount(BaseModel):
-    """Google Cloud Platform service account credentials to be used with a GCS file system.
-    The service account must have read/write access to the bucket specified in the file system.
-    """
-
-    service_account: str
-
-
-class UpdateGCSFileSystem(GCSFileSystem):
-    credentials: GCPServiceAccount
-
-
-UpdateFileSystemRequest = Annotated[
-    BuiltinFileSystem | UpstreamFileSystem | UpdateS3FileSystem | UpdateGCSFileSystem, Field(discriminator="type")
-]
-
-
-class UpdateFileSystemResponse(BaseModel):
-    file_system: FileSystem
-
-
 class CreateMountRequest(BaseModel):
     directory: DirectoryPath
     mode: Mode
@@ -136,9 +105,9 @@ class FileSystemService(ServiceBase):
         response = self.client.get("/v1/file-systems/builtin-providers", dict)
         return response.get("providers", [])
 
-    def update_file_system(self, project_id: ProjectId, request: UpdateFileSystemRequest) -> UpdateFileSystemResponse:
+    def update_file_system(self, project_id: ProjectId, request: FileSystem) -> FileSystem:
         """Update project's default file system."""
-        return self.client.post(f"/v1/file-systems/{project_id}", request, UpdateFileSystemResponse)
+        return self.client.post(f"/v1/file-systems/{project_id}", request, FileSystem)
 
     def get_file_system(self, project_id: ProjectId) -> FileSystem:
         """Get project's default file system."""

spiral/cli/app.py

@@ -1,6 +1,10 @@
 import logging
 import os
+from importlib import metadata
 from logging.handlers import RotatingFileHandler
+from typing import Annotated
+
+import typer
 
 from spiral.cli import (
     AsyncTyper,
@@ -18,16 +22,48 @@ from spiral.cli import (
     text,
     workloads,
 )
-from spiral.settings import LOG_DIR, Settings
+from spiral.settings import LOG_DIR, PACKAGE_NAME, Settings
 
 app = AsyncTyper(name="spiral")
 
 
-@app.callback()
-def _callback(verbose: bool = False):
-    if verbose:
+def version_callback(ctx: typer.Context, value: bool):
+    """
+    Display the version of the Spiral CLI.
+    """
+    # True when generating completion, we can just return
+    if ctx.resilient_parsing:
+        return
+
+    if value:
+        ver = metadata.version(PACKAGE_NAME)
+        print(f"spiral {ver}")
+        raise typer.Exit()
+
+
+def verbose_callback(ctx: typer.Context, value: bool):
+    """
+    Use more verbose output.
+    """
+    # True when generating completion, we can just return
+    if ctx.resilient_parsing:
+        return
+
+    if value:
         logging.getLogger().setLevel(level=logging.INFO)
 
+
+@app.callback(invoke_without_command=True)
+def _callback(
+    ctx: typer.Context,
+    version: Annotated[
+        bool | None,
+        typer.Option("--version", callback=version_callback, help=version_callback.__doc__, is_eager=True),
+    ] = None,
+    verbose: Annotated[
+        bool | None, typer.Option("--verbose", callback=verbose_callback, help=verbose_callback.__doc__)
+    ] = None,
+):
     # Load the settings (we reload in the callback to support testing under different env vars)
     state.settings = Settings()
 
@@ -42,12 +78,12 @@ app.add_typer(text.app, name="text")
 app.add_typer(telemetry.app, name="telemetry")
 app.command("console")(console.command)
 app.command("login")(login.command)
-app.command("whoami")(login.whoami)
 
 # Register unless we're building docs. Because Typer docs command does not skip hidden commands...
 if not bool(os.environ.get("SPIRAL_DOCS", False)):
-    app.add_typer(workloads.app, name="workloads", hidden=True)
     app.add_typer(admin.app, name="admin", hidden=True)
+    app.add_typer(workloads.app, name="workloads", hidden=True)
+    app.command("whoami", hidden=True)(login.whoami)
     app.command("logout", hidden=True)(login.logout)
 
 

spiral/cli/fs.py

@@ -1,15 +1,12 @@
-from typing import Annotated
+from typing import Literal
 
 import questionary
-from pydantic import SecretStr
 from typer import Option
 
 from spiral.api.filesystems import (
-    AWSSecretAccessKey,
     BuiltinFileSystem,
-    GCPServiceAccount,
-    UpdateGCSFileSystem,
-    UpdateS3FileSystem,
+    GCSFileSystem,
+    S3FileSystem,
     UpstreamFileSystem,
 )
 from spiral.cli import CONSOLE, AsyncTyper, state
@@ -21,11 +18,7 @@ app = AsyncTyper(short_help="File Systems.")
 @app.command(help="Show the file system configured for project.")
 def show(project: ProjectArg):
     file_system = state.settings.api.file_system.get_file_system(project)
-    match file_system:
-        case BuiltinFileSystem(provider=provider):
-            CONSOLE.print(f"provider: {provider}")
-        case _:
-            CONSOLE.print(file_system)
+    CONSOLE.print(file_system)
 
 
 def ask_provider():
@@ -33,76 +26,48 @@ def ask_provider():
     return questionary.select("Select a file system provider", choices=res).ask()
 
 
-BuiltinProviderOpt = Annotated[
-    str,
-    Option(help="Built-in provider to use for the file system.", show_default=False, default_factory=ask_provider),
-]
-
-
 @app.command(help="Update a project's default file system.")
 def update(
     project: ProjectArg,
-    builtin: bool = Option(False, help="Use a built-in file system provider."),
-    upstream: bool = Option(
-        False, help="Use another project as default file system. Only if another project is an external provider."
-    ),
-    s3: bool = Option(False, help="Use S3 compatible provider."),
-    gcs: bool = Option(False, help="Use GCS provider."),
-    provider: str = Option(None, help="Built-in provider to use for the file system."),
-    endpoint: str = Option(None, help="Endpoint for S3 provider."),
-    region: str = Option(None, help="Region for S3 or GCS provider. Required for GCS."),
-    bucket: str = Option(None, help="Bucket name for S3 or GCS provider."),
-    directory: str = Option(None, help="Directory for S3 or GCS provider."),
-    access_key_id: str = Option(None, help="Access key ID for S3 provider. Required for S3."),
-    secret_access_key: str = Option(None, help="Secret access key for S3 provider. Required for S3."),
-    credentials_path: str = Option(
-        None, help="Path to service account credentials file for GCS provider. Required for GCS."
+    type_: Literal["builtin", "s3", "gcs", "upstream"] = Option(None, "--type", help="Type of the file system."),
+    provider: str = Option(None, help="Provider, when using `builtin` type."),
+    endpoint: str = Option(None, help="Endpoint, when using `s3` type."),
+    region: str = Option(
+        None, help="Region, when using `s3` or `gcs` type (defaults to `auto` for `s3` when `endpoint` is set)."
     ),
+    bucket: str = Option(None, help="Bucket, when using `s3` or `gcs` type."),
+    role_arn: str = Option(None, help="Role ARN to assume, when using `s3` type."),
 ):
-    if not any([builtin, s3, gcs, upstream]):
-        raise ValueError("Must specify one of --builtin, --upstream, --s3, or --gcs.")
-
-    if builtin:
+    if type_ == "builtin":
         provider = provider or ask_provider()
         file_system = BuiltinFileSystem(provider=provider)
 
-    elif upstream:
+    elif type_ == "upstream":
         upstream_project = ask_project(title="Select a project to use as file system.")
         file_system = UpstreamFileSystem(project_id=upstream_project)
 
-    elif s3:
-        if access_key_id is None or secret_access_key is None:
-            raise ValueError("--access-key-id and --secret-access-key are required for S3 provider.")
-        credentials = AWSSecretAccessKey(access_key_id=access_key_id, secret_access_key=secret_access_key)
-
+    elif type_ == "s3":
+        if role_arn is None:
+            raise ValueError("--role-arn is required for S3 provider.")
+        if not role_arn.startswith("arn:aws:iam::") or ":role/" not in role_arn:
+            raise ValueError("Invalid role ARN format. Expected `arn:aws:iam::<account>:role/<role_name>`")
         if bucket is None:
             raise ValueError("--bucket is required for S3 provider.")
-        file_system = UpdateS3FileSystem(bucket=bucket, credentials=credentials)
+        region = region or ("auto" if endpoint else None)
+        file_system = S3FileSystem(bucket=bucket, role_arn=role_arn, region=region)
         if endpoint:
             file_system.endpoint = endpoint
-        if region:
-            file_system.region = region
-        if directory:
-            file_system.directory = directory
-
-    elif gcs:
-        if credentials_path is None:
-            raise ValueError("--credentials-path is required for GCS provider.")
-        with open(credentials_path) as f:
-            service_account = f.read()
-        credentials = GCPServiceAccount(credentials=SecretStr(service_account))
 
+    elif type_ == "gcs":
         if region is None or bucket is None:
             raise ValueError("--region and --bucket is required for GCS provider.")
-        file_system = UpdateGCSFileSystem(bucket=bucket, region=region, credentials=credentials)
-        if directory:
-            file_system.directory = directory
+        file_system = GCSFileSystem(bucket=bucket, region=region)
 
     else:
-        raise ValueError("Must specify either --s3 or --gcs.")
+        raise ValueError(f"Unknown file system type: {type_}")
 
-    res = state.settings.api.file_system.update_file_system(project, file_system)
-    CONSOLE.print(res.file_system)
+    fs = state.settings.api.file_system.update_file_system(project, file_system)
+    CONSOLE.print(fs)
 
 
 @app.command(help="Lists the available built-in file system providers.")

spiral/cli/login.py

@@ -3,10 +3,11 @@ import jwt
 from spiral.cli import CONSOLE, state
 
 
-def command(org_id: str | None = None, force: bool = False):
+def command(org_id: str | None = None, force: bool = False, show_token: bool = False):
     token = state.settings.device_code_auth.authenticate(force=force, org_id=org_id)
     CONSOLE.print("Successfully logged in.")
-    CONSOLE.print(token.expose_secret(), soft_wrap=True)
+    if show_token:
+        CONSOLE.print(token.expose_secret(), soft_wrap=True)
 
 
 def whoami():

spiral/core/_tools/__init__.pyi

@@ -0,0 +1,5 @@
+from ..table.spec import Schema
+
+def pretty_key(key: bytes, schema: Schema) -> str:
+    """Represent a key in a human-readable way."""
+    ...

spiral/core/client/__init__.pyi

@@ -136,11 +136,18 @@ class ShuffleStrategy:
     # Externally provided shards to shuffle before reading rows.
     shards: list[Shard] | None
 
+    # Maximum number of rows to return in a single batch.
+    # If None, it is derived from the shuffle buffer size.
+    # IMPORTANT: The returned batch may be smaller than this size.
+    max_batch_size: int | None
+
     def __init__(
         self,
+        shuffle_buffer_size: int,
+        *,
         seed: int | None = None,
-        shuffle_buffer_size: int | None = None,
         shards: list[Shard] | None = None,
+        max_batch_size: int | None = None,
     ): ...
 
 class Operations:
@@ -205,3 +212,7 @@ class Operations:
         """
         ...
     def metrics(self) -> dict[str, Any]: ...
+
+def flush_telemetry() -> None:
+    """Flush telemetry data to the configured exporter."""
+    ...

spiral/core/table/__init__.pyi

@@ -70,6 +70,9 @@ class Scan:
         self,
         strategy: ShuffleStrategy | None = None,
         batch_readahead: int | None = None,
+        num_workers: int | None = None,
+        worker_id: int | None = None,
+        infinite: bool = False,
     ) -> pa.RecordBatchReader: ...
     def metrics(self) -> dict[str, Any]: ...
     def _prepare_shard(

spiral/debug/manifests.py

@@ -1,3 +1,6 @@
+from rich.console import Console
+from rich.table import Table
+
 from spiral import datetime_
 from spiral.core.table import Scan
 from spiral.core.table.manifests import FragmentManifest
@@ -42,23 +45,26 @@ def _table_of_fragments(manifest: FragmentManifest, title: str):
     avg_size = total_size / fragment_count if fragment_count > 0 else 0
 
     # Print title and summary
-    print(f"\n\n{title}")
-    print(
+    console = Console()
+    console.print(f"\n\n{title}")
+    console.print(
         f"{fragment_count} fragments, "
         f"total: {_format_bytes(total_size)}, "
         f"avg: {_format_bytes(int(avg_size))}, "
         f"metadata: {_format_bytes(total_metadata_size)}"
     )
-    print("=" * 120)
 
-    # Print header
-    print(
-        f"{'ID':<30} {'Size (Metadata)':<20} {'Format':<10} {'Key Span':<10} "
-        f"{'Level':<5} {'Committed At':<20} {'Compacted At':<20}"
-    )
-    print("=" * 120)
+    # Create rich table
+    table = Table(title=None, show_header=True, header_style="bold")
+    table.add_column("ID", style="cyan", no_wrap=True)
+    table.add_column("Size (Metadata)", justify="right")
+    table.add_column("Format", justify="center")
+    table.add_column("Key Span", justify="center")
+    table.add_column("Level", justify="center")
+    table.add_column("Committed At", justify="center")
+    table.add_column("Compacted At", justify="center")
 
-    # Print each fragment
+    # Add each fragment as a row
     for fragment in manifest:
         committed_str = str(datetime_.from_timestamp_micros(fragment.committed_at)) if fragment.committed_at else "N/A"
         compacted_str = str(datetime_.from_timestamp_micros(fragment.compacted_at)) if fragment.compacted_at else "N/A"
@@ -68,12 +74,14 @@ def _table_of_fragments(manifest: FragmentManifest, title: str):
         )
         key_span = f"{fragment.key_span.begin}..{fragment.key_span.end}"
 
-        print(
-            f"{fragment.id:<30} "
-            f"{size_with_metadata:<20} "
-            f"{str(fragment.format):<10} "
-            f"{key_span:<10} "
-            f"{str(fragment.level):<5} "
-            f"{committed_str:<20} "
-            f"{compacted_str:<20}"
+        table.add_row(
+            fragment.id,
+            size_with_metadata,
+            str(fragment.format),
+            key_span,
+            str(fragment.level),
+            committed_str,
+            compacted_str,
         )
+
+    console.print(table)

spiral/debug/scan.py

@@ -146,14 +146,32 @@ def _get_fragment_color(manifest_file: FragmentFile, color_index, total_colors):
         return cm.viridis(color_index / total_colors)
 
 
+def _get_human_size(size_bytes: int) -> str:
+    # Convert bytes to a human-readable format
+    for unit in ["B", "KB", "MB", "GB", "TB"]:
+        if size_bytes < 1024:
+            return f"{size_bytes:.2f} {unit}"
+        size_bytes /= 1024
+    return f"{size_bytes:.2f} PB"
+
+
+def _maybe_truncate(text, max_length: int = 30) -> str:
+    text = str(text)
+    if len(text) <= max_length:
+        return text
+
+    half_length = (max_length - 3) // 2
+    return text[:half_length] + "..." + text[-half_length:]
+
+
 def _get_fragment_legend(manifest_file: FragmentFile):
     return "\n".join(
         [
             f"id: {manifest_file.id}",
-            f"size: {manifest_file.size_bytes:,} bytes",
+            f"size: {_get_human_size(manifest_file.size_bytes)} ({manifest_file.size_bytes} bytes)",
             f"key_span: {manifest_file.key_span}",
-            f"key_min: {manifest_file.key_extent.min}",
-            f"key_max: {manifest_file.key_extent.max}",
+            f"key_min: {_maybe_truncate(manifest_file.key_extent.min)}",
+            f"key_max: {_maybe_truncate(manifest_file.key_extent.max)}",
             f"format: {manifest_file.format}",
             f"level: {manifest_file.level}",
             f"committed_at: {_format_timestamp(manifest_file.committed_at)}",

spiral/expressions/__init__.py

@@ -92,7 +92,7 @@ def lift(expr: ExprLike) -> Expr:
         return lift(pa.array(expr))
 
     # Unpack tables and chunked arrays
-    if isinstance(expr, pa.Table):
+    if isinstance(expr, pa.Table | pa.RecordBatch):
         expr = expr.to_struct_array()
     if isinstance(expr, pa.ChunkedArray):
         expr = expr.combine_chunks()
@@ -104,7 +104,7 @@ def lift(expr: ExprLike) -> Expr:
         if isinstance(expr, pa.StructArray) and expr.null_count != 0:
             # raise ValueError("lift: cannot lift a struct array with nulls.")
             warnings.warn("found a struct array with nulls", stacklevel=2)
-        if isinstance(expr, pa.StructScalar) and not expr.is_valid():
+        if isinstance(expr, pa.StructScalar) and not expr.is_valid:
             # raise ValueError("lift: cannot lift a struct scalar with nulls.")
             warnings.warn("found a struct scalar with nulls", stacklevel=2)
         return lift(arrow_.nest_structs(expr))