pyspiral 0.6.11__cp312-abi3-manylinux_2_28_aarch64.whl → 0.6.13__cp312-abi3-manylinux_2_28_aarch64.whl

{pyspiral-0.6.11.dist-info → pyspiral-0.6.13.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyspiral
-Version: 0.6.11
+Version: 0.6.13
 Classifier: Intended Audience :: Science/Research
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python
@@ -31,15 +31,18 @@ Requires-Dist: typer>=0.16
 Requires-Dist: xxhash>=3.4.1
 Requires-Dist: polars>=1.31.0 ; extra == 'polars'
 Requires-Dist: duckdb>=1.3.2 ; extra == 'duckdb'
-Requires-Dist: datasets>=4.0.0 ; extra == 'datasets'
-Requires-Dist: pyiceberg>=0.9.1 ; extra == 'pyiceberg'
+Requires-Dist: pyiceberg[s3fs]>=0.9.1 ; extra == 'iceberg'
+Requires-Dist: datasets>=4.0.0 ; extra == 'huggingface'
 Requires-Dist: mosaicml-streaming>=0.13.0 ; extra == 'streaming'
 Requires-Dist: vortex-data>=0.52.1 ; extra == 'streaming'
+Requires-Dist: dask>=2025.10.0 ; extra == 'dask'
+Requires-Dist: distributed>=2025.10.0 ; extra == 'dask'
 Provides-Extra: polars
 Provides-Extra: duckdb
-Provides-Extra: datasets
-Provides-Extra: pyiceberg
+Provides-Extra: iceberg
+Provides-Extra: huggingface
 Provides-Extra: streaming
+Provides-Extra: dask
 Summary: Python client for Spiral.
 Home-Page: https://spiraldb.com
 Author-email: SpiralDB <hello@spiraldb.com>

{pyspiral-0.6.11.dist-info → pyspiral-0.6.13.dist-info}/RECORD

@@ -1,8 +1,8 @@
-pyspiral-0.6.11.dist-info/METADATA,sha256=_Km6gYKZorM7zvU5kOO6lUSY39ei8orw_8vz30yF1Gk,1843
-pyspiral-0.6.11.dist-info/WHEEL,sha256=I5JYpyYzeAl2SOerY_wvkm-HJti0rDQc6zMeJs35MpM,108
-pyspiral-0.6.11.dist-info/entry_points.txt,sha256=uft7u-a6g40NLt4Q6BleWbK4NY0M8nZuYPpP8DV0EOk,45
-spiral/__init__.py,sha256=n4JNLrO3wyw_k_U_JKyNiGON0wEpfvqxDhDdB2P6dhM,1007
-spiral/_lib.abi3.so,sha256=lNiMj4FtAFVhmeisqNtLg9DUA3in95UYoct6lnagkC4,57478712
+pyspiral-0.6.13.dist-info/METADATA,sha256=AvViHjB1v9OqYTKONYf_DbfCB0HktAy-numkhQhuq20,1977
+pyspiral-0.6.13.dist-info/WHEEL,sha256=I5JYpyYzeAl2SOerY_wvkm-HJti0rDQc6zMeJs35MpM,108
+pyspiral-0.6.13.dist-info/entry_points.txt,sha256=uft7u-a6g40NLt4Q6BleWbK4NY0M8nZuYPpP8DV0EOk,45
+spiral/__init__.py,sha256=gAysTwG_oEeKVMdCOfOzDhl0bM2miiK8Ds2vvUihBWw,1153
+spiral/_lib.abi3.so,sha256=x-kkHeRPwdf73BSVaVwxA6K71a0mtTXCV_omc2t0j7g,61221816
 spiral/adbc.py,sha256=7IxfWIeQN-fh0W5OdN_PP2x3pzQYg6ZUOLsHg3jktqw,14842
 spiral/api/__init__.py,sha256=ULBlVq3PnfNOO6T5naE_ULmmii-83--qTuN2PpAUQN0,2241
 spiral/api/admin.py,sha256=A1iVR1XYJSObZivPAD5UzmPuMgupXc9kaHNYYa_kwfs,585
@@ -23,53 +23,59 @@ spiral/cli/admin.py,sha256=-ubYqs8nKjnQStbQ68jpWx_9xh0TsaxI0wM1Hfko8_U,319
 spiral/cli/app.py,sha256=smzGj5a2RwhM9RQChmlEeKZLN4Fk60-bP7Lm5_Is1Rw,2760
 spiral/cli/console.py,sha256=6JHbAQV6MFWz3P-VzqPOjhHpkIQagsCdzTMvmuDKMkU,2580
 spiral/cli/fs.py,sha256=vaPcSc2YghhHeipxNitIdsHaBhFwlwkvPFqYsFSN9P0,2927
-spiral/cli/iceberg.py,sha256=Q14tcGcn1LixbFCYP0GhfYwFFXTmmi8tqBPYwalJEyE,3248
-spiral/cli/key_spaces.py,sha256=x3IFRP5d47pKiAHeWExYMOBaT2TwxbWjVM01SUqKrwI,2943
+spiral/cli/iceberg.py,sha256=wdMyl0j821MLnXNZ6Kwm65ogh98C-pjMJm3Y6YqlnTI,3249
+spiral/cli/key_spaces.py,sha256=Xaw7WH-Qw_j6AxisdIoKfjAgVRXLM9qBFzuCTjPAFLI,3516
 spiral/cli/login.py,sha256=2tw6uN5rEpiMMAmjQSB3-JUPf3C0Wc1eTGCDxhYtJps,731
 spiral/cli/orgs.py,sha256=fmOuLxpeIFfKqePRi292Gv9k-EF5pPn_tbKd2BLl2Ig,2869
 spiral/cli/printer.py,sha256=aosc763hDFgoXJGkiANmNyO3kAsecAS1JWgjEhn8GCM,1784
 spiral/cli/projects.py,sha256=1M1nGrBT-t0aY9RV5Cnmzy7YrhIvmHwdkpa3y9j8rG8,5756
 spiral/cli/state.py,sha256=10wTIVQ0SJkY67Z6-KQ1LFlt3aVIPmZhoHFdTwp4kNA,130
-spiral/cli/tables.py,sha256=fFte_wMNcB0V-fmfSXfSbtV4UlAi-Xw5nYDJ0b62CGk,6360
+spiral/cli/tables.py,sha256=qm3izcysElJrQlerNZdfx5RWSVXtyVfkP3o_H51ltFw,6366
 spiral/cli/telemetry.py,sha256=Uxo1Q1FkKJ6n6QNGOUmL3j_pRRWRx0qWIhoP-U9BuR0,589
 spiral/cli/text.py,sha256=DlWGe4JrkdERAiqyITNpk91Wqb63Re99rNYlIFsIamc,4031
 spiral/cli/types.py,sha256=XYzo1GgX7dBBItoBSrHI4vO5C2lLmS2sktb-2GnGH3E,1362
 spiral/cli/workloads.py,sha256=2_SLfQTFN6y73R9H0i9dk8VIOVagKxSxOpHXC56yptY,2015
-spiral/client.py,sha256=N4sQLxtQ6GYCnj00hm4VX1vUVUqzQdHhl_KfQwp-1LQ,6345
+spiral/client.py,sha256=zMp-xXGL4R1Py_rYrC5o3jFLam1oA74azi50dvMP-_o,6329
 spiral/core/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 spiral/core/_tools/__init__.pyi,sha256=b2KLfTOQ67pjfbYt07o0IGiTu5o2bZw69lllV8v0Dps,143
 spiral/core/authn/__init__.pyi,sha256=z_GWyIS62fuiYQrYO8hzw4W8oGaiciqS1u5qtAt54VY,769
-spiral/core/client/__init__.pyi,sha256=1HK3SOMT1QKmD5Hai58ZFjiEZK0QzyYtP84hse8SBEI,6666
-spiral/core/table/__init__.pyi,sha256=QqG_pMlPhMtXG-56dXyQjOWVKMugPP0nnYnvYaY0Q10,3288
+spiral/core/client/__init__.pyi,sha256=ajF8XaxThnTdsPpw1k3pPLEurIaDg9yeXqwIRJNlJTY,6665
+spiral/core/expr/__init__.pyi,sha256=3HSKjkotiEkxBvGBALXEBIie0JiyI9bCpehwA3nMQkU,571
+spiral/core/expr/images/__init__.pyi,sha256=wnE_wZXq7a4iqTg3SVm-ssxGw1WQZyk5dGOPaP4Btko,73
+spiral/core/expr/list_/__init__.pyi,sha256=Q_9c87eIQfZbqlaw_rq3fvs93YEsW7K5VYk6VZ4g6mU,126
+spiral/core/expr/refs/__init__.pyi,sha256=nZZP3l_Z6bLx6V8lTcH3Jgo--xwfADOU2XdTAvM5IMk,127
+spiral/core/expr/str_/__init__.pyi,sha256=Bm6fZK-d4fNbJuuBhVoWMACXUbQQ2SjlhgrOpdOHIPM,86
+spiral/core/expr/struct_/__init__.pyi,sha256=MXckd98eV_x3X0RhEWvlkA3DcDXRtLs5pNnTQkc09nE,296
+spiral/core/expr/text/__init__.pyi,sha256=ed83n1xcsGY7_QDhMmJGnSQ20UrJFXcdv1AveSEcS1c,175
+spiral/core/expr/udf/__init__.pyi,sha256=zsZs081KVhY3-1JidqTkWMW81Qd_ScoTGZvasIhIK-4,358
+spiral/core/expr/video/__init__.pyi,sha256=nQJEcSsigZuRpMjkI_O4EEtMK_n2zRvorcL_KEeD5vU,95
+spiral/core/table/__init__.pyi,sha256=YBL12_JPTWz2mNbqlDqbT1exxVJYzwfXdHCi6Z37JxA,3841
 spiral/core/table/manifests/__init__.pyi,sha256=eVfDpmhYSjafIvvALqAkZe5baN3Y1HpKpxYEbjwd4gQ,1043
 spiral/core/table/metastore/__init__.pyi,sha256=rc3u9MwEKRvL2kxOc8lBorddFRnM8o_o1frqtae86a4,1697
-spiral/core/table/spec/__init__.pyi,sha256=OFYJXPXix7gskYJIMog7IniZslEPJ0xvL-sUSFDPbXs,5643
-spiral/dataloader.py,sha256=FFZhIflQPEygXe-xBLifQnnxANi4CFooaHRm4i-EGHo,10335
+spiral/core/table/spec/__init__.pyi,sha256=twzX4vFmgBxInZWq_nyP6DR9OQjjOVrbZMn97kndeS8,5808
+spiral/dataloader.py,sha256=W9siY4BF4p_rwTTSS4KgsaQsPLxxza6XmQhrdBzzMJ8,10592
 spiral/dataset.py,sha256=PMLoXnXuEUciP6-NXqTmQLXu0UIH7OcC4-iZtY_iuO8,7973
 spiral/datetime_.py,sha256=elXaUWtZuuLVcu9E0aXnvYRPB9XWqZbLDToozQYQYjU,950
 spiral/debug/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 spiral/debug/manifests.py,sha256=7f1O3ba9mrA5nXpOF9cEIQuUAteP5wiBkFy_diQJ7No,3216
 spiral/debug/metrics.py,sha256=XdRDcjggtsLNGCAjam6IxG9072pz_d2C8iLApNRFUtk,2044
 spiral/debug/scan.py,sha256=UEm_aRnql5pwDPTpZgakMLNjlzkKL4RurBFFqH_BLAQ,9526
-spiral/expressions/__init__.py,sha256=KhwFjVKoFgx1S6hkVcE8aZjoHY_1N-BgQ2rGEZfPQvM,7957
-spiral/expressions/base.py,sha256=915gpvZZCTRCO5q93pwwmhf-R6C23LQsyDt4Q2dHk9s,5290
-spiral/expressions/http.py,sha256=begUydWoFHEqjeLkATvI_v66Ez6_rR-OQBWO5cHbb9c,2742
-spiral/expressions/io.py,sha256=gJ2a0FKMmdxarWKENulPRwH7KDvSJTIh_OUxX306xAM,3045
+spiral/enrichment.py,sha256=e2yzNWTTG73uEkLTc4ccTNRQ94cBtM04eGzlJ2-kBOI,5851
+spiral/expressions/__init__.py,sha256=Fp7Xx3exh9KJad92tgd_TGGIpYLQTHqWjW-pexzQibU,7981
+spiral/expressions/base.py,sha256=PvhJkcUSsPSIaxirHVzM9zlqyBXiaiia1HXohXdOmL4,5377
+spiral/expressions/http.py,sha256=WfHVLqz_LjBr78mN3ARBRQqgBrkao7-S73JxjC4Xwvo,356
 spiral/expressions/list_.py,sha256=MMt5lf5H1M3O-x6N_PvqOLGq9NOk6Ukv0fPWwPC_uy4,1809
-spiral/expressions/mp4.py,sha256=_xGVnkygddzxP9a8OACJ8_KXnejuVbYCVKBCXBQ798Y,2151
-spiral/expressions/png.py,sha256=KO8X0OmMzUFwpg2I_j0JTyldPzVXDWIMzjWMWDV9vIY,506
-spiral/expressions/qoi.py,sha256=gvIbb6fXb_Bb080sn9wkpbGGrPs2UEcTXCfuv4-kcYQ,506
-spiral/expressions/refs.py,sha256=omeHBQ5o6N4xgZ3x5Xz7IRrWwYBBtQY8DYK0NNAxeGo,2109
+spiral/expressions/s3.py,sha256=bkd0HANerNKlOblp2z7JJOSWjF9Bw9lZe1A-KTrUEgk,378
 spiral/expressions/str_.py,sha256=tY8RXW3JWvr1-bEfCZtk5FAf11wKJnXPuA9EoeJ9tA4,1265
 spiral/expressions/struct.py,sha256=pGAnCDh6AK0BK1XfZ1qG4ce4ranIQEE1HQsgmzBcfwQ,2038
 spiral/expressions/text.py,sha256=-02gBWYoyNQ3qQ1--9HTa8IryUDojYQVIp8C7rgnOWQ,1893
-spiral/expressions/tiff.py,sha256=fQwIn0kLFBM2Y3YYIHmTgb_EIRHKT2fNc77nioDQQw4,8044
-spiral/expressions/udf.py,sha256=yb9MIcrFftpNDxgBF228cvdv6TY-hEFikYz2fq_nzWo,1353
+spiral/expressions/tiff.py,sha256=4dngO97bT1QY0By7-PxOQVmSwQC3PQAiixVhLJ-4HMQ,7986
+spiral/expressions/udf.py,sha256=XOxa7Kocb4Cg4q_qFvRT6hVnVzi22CQenqrvS-TL-VY,1936
 spiral/grpc_.py,sha256=f3czdP1Mxme42Y5--a5ogYq1TTiWn-J_MlGjwJ2mWwM,1015
 spiral/iceberg.py,sha256=JGq62Qnf296r9_hRAoH85GQq45-uSBjwXWw_CvPi6G4,930
 spiral/iterable_dataset.py,sha256=Eekg9ad8tcwXcloHWReBbvCSr5ZappRHn2ldKTvwqS0,4622
 spiral/key_space_index.py,sha256=NAB_nONEjpMYbse8suz42w7Qb5OPHuKN9h9CT2NJe08,1460
-spiral/project.py,sha256=CO_Pn6vPqaonNvRdCNRFcBWr4TqO2AsAUTH5xawIeCE,7283
+spiral/project.py,sha256=VsokZgS0TqIel7UAXMyoBToxn-l_D3ivGwc41x7HLF0,7277
 spiral/protogen/_/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 spiral/protogen/_/arrow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 spiral/protogen/_/arrow/flight/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -87,16 +93,16 @@ spiral/protogen/_/substrait/extensions/__init__.py,sha256=nhnEnho70GAT8WPj2xtwJU
 spiral/protogen/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 spiral/protogen/util.py,sha256=smnvVo6nYH3FfDm9jqhNLaXz4bbTBaQezHQDCTvZyiQ,1486
 spiral/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-spiral/scan.py,sha256=4PUlI_DHbO1WTttLia6DinhGtOWsCiqek4ZljoEiRZc,10523
+spiral/scan.py,sha256=csbk5ePbU-RlEVIF7isccF2zRBB8L8ZY_HEpalMjgLY,12340
 spiral/server.py,sha256=ztBmB5lBnUz-smQxR_tC8AI5SOhz17wH0MI3GuzDUdM,600
-spiral/settings.py,sha256=JRQSwjJyNaCqQdQLxiqB_O_LZRQXMLyshJBrI2LZHwM,3113
+spiral/settings.py,sha256=sUhMMBCXaPvUYztN_gztD9TjeUYJwVeEcJrq4FLy6M0,3232
 spiral/snapshot.py,sha256=cTobi5jtiANxalGA-isokQHblNmXGtuUvgUGGNVybsI,1555
 spiral/streaming_/__init__.py,sha256=s7MlW2ERsuZmZGExLFL6RcZon2e0tNBocBg5ANgki7k,61
 spiral/streaming_/reader.py,sha256=tl_lC9xgh1-QFhsZn4xQT7It3PVTzHCEUT2BG2dWBRQ,4166
-spiral/streaming_/stream.py,sha256=nXnygiuCxi1D3PhaxV8Ujif4J9ly_OczA7CZ3W4WN2w,5913
+spiral/streaming_/stream.py,sha256=DM1hBDHnWm1ZFKZ-hZ4zxeSXITcUI6kWzwdJZvywI8o,5915
 spiral/substrait_.py,sha256=AKeOD4KIXvz2J4TYxnIneOiHddtBIyOhuNxVO_uH0eg,12592
-spiral/table.py,sha256=G05b6M0uVmT5ew5GxuzsVB4rQzg25W3zGMTftL07pJU,11026
+spiral/table.py,sha256=prjDBcm6Qerdq3ypXzfbXb7ngAcO0j-Z9aTeZvzKoqs,12209
 spiral/text_index.py,sha256=FQ9rgIEGLSJryS9lFdMhKtPFey18BXoWbPXyvZPJJ04,442
-spiral/transaction.py,sha256=h6YdAwOYX6qq-tXYV4i9yhy1Nq1tIfRphY_fk7Q_yLQ,1854
+spiral/transaction.py,sha256=hQm6DfCklMDpIYJ9qA2wR45cCuUPGCiJy1tHGE3AsEY,3418
 spiral/types_.py,sha256=W_jyO7F6rpPiH69jhgSgV7OxQZbOlb1Ho3InpKUP6Eo,155
-pyspiral-0.6.11.dist-info/RECORD,,
+pyspiral-0.6.13.dist-info/RECORD,,

spiral/__init__.py

@@ -1,14 +1,18 @@
 """Python client for Spiral"""
 
+import importlib
+
 # This is here to make sure we load the native extension first
 from spiral import _lib
 
 # Eagerly import the Spiral library
 assert _lib, "Spiral library"
 
+
 from spiral.client import Spiral  # noqa: E402
 from spiral.core.client import Shard, ShuffleConfig  # noqa: E402
 from spiral.dataloader import SpiralDataLoader, World  # noqa: E402
+from spiral.enrichment import Enrichment  # noqa: E402
 from spiral.iceberg import Iceberg  # noqa: E402
 from spiral.key_space_index import KeySpaceIndex  # noqa: E402
 from spiral.project import Project  # noqa: E402
@@ -24,6 +28,7 @@ __all__ = [
     "Table",
     "Snapshot",
     "Transaction",
+    "Enrichment",
     "Scan",
     "Shard",
     "ShuffleConfig",
@@ -33,3 +38,5 @@ __all__ = [
     "World",
     "Iceberg",
 ]
+
+__version__ = importlib.metadata.version("pyspiral")

spiral/cli/iceberg.py

@@ -8,7 +8,7 @@ from typer import Argument
 from spiral.cli import CONSOLE, ERR_CONSOLE, AsyncTyper, state
 from spiral.cli.types import ProjectArg
 
-app = AsyncTyper(short_help="Apache Iceberg Catalog")
+app = AsyncTyper(short_help="Apache Iceberg Catalog.")
 
 
 @app.command(help="List namespaces.")

spiral/cli/key_spaces.py

@@ -64,7 +64,7 @@ def show(
     """Show index partitions."""
     index_id = get_index_id(project, name)
     index = state.spiral.key_space_index(index_id)
-    shards = state.spiral._ops().compute_shards(index.core)
+    shards = state.spiral.internal.compute_shards(index.core)
 
     rich_table = rich.table.Table("Begin", "End", "Cardinality", title=f"Index {index.name} Partitions")
     for partition in shards:
@@ -87,3 +87,17 @@ def sync(
     index_id = get_index_id(project, name)
     response = state.spiral.api.key_space_indexes.sync_index(index_id, SyncIndexRequest(resources=resources))
     CONSOLE.print(f"Triggered sync job {response.worker_id} for index {index_id}.")
+
+
+# TODO(marko): This will be removed.
+@app.command(help="Run a sync and wait for it to complete.")
+def sync_local(
+    project: ProjectArg,
+    name: Annotated[str | None, Option(help="Index name.")] = None,
+):
+    """Run a sync and wait for it to complete."""
+    index_id = get_index_id(project, name)
+    index = state.spiral.key_space_index(index_id)
+    snapshot = state.spiral.table(index.table_id).snapshot()
+    state.spiral.internal.update_key_space_index(index.core, snapshot.core)
+    CONSOLE.print(f"Index {index.name} is up to date as-of {snapshot.asof}.")

spiral/cli/tables.py

@@ -130,7 +130,7 @@ def flush(
     keep_latest_s = int(duration.total_seconds()) if duration is not None else None
 
     identifier, t = get_table(project, table, dataset)
-    state.spiral._ops().flush_wal(t.core, keep_latest_s=keep_latest_s)  # pyright: ignore[reportPrivateUsage]
+    state.spiral.internal.flush_wal(t.core, keep_latest_s=keep_latest_s)  # pyright: ignore[reportPrivateUsage]
     CONSOLE.print(f"Flushed WAL for table {identifier} in project {project}.")
 
 
@@ -143,10 +143,10 @@ def manifests(
     _, t = get_table(project, table, dataset)
     s = t.snapshot()
 
-    key_space_state = state.spiral._ops().key_space_state(s.core)  # pyright: ignore[reportPrivateUsage]
+    key_space_state = state.spiral.internal.key_space_state(s.core)  # pyright: ignore[reportPrivateUsage]
     key_space_manifest = key_space_state.manifest
 
-    column_groups_states = state.spiral._ops().column_groups_states(s.core, key_space_state)  # pyright: ignore[reportPrivateUsage]
+    column_groups_states = state.spiral.internal.column_groups_states(s.core, key_space_state)  # pyright: ignore[reportPrivateUsage]
     display_manifests(key_space_manifest, [(x.column_group, x.manifest) for x in column_groups_states])
 
 

spiral/client.py

@@ -6,7 +6,7 @@ import pyarrow as pa
 
 from spiral.api import SpiralAPI
 from spiral.api.projects import CreateProjectRequest, CreateProjectResponse
-from spiral.core.client import Operations
+from spiral.core.client import Internal
 from spiral.core.client import Spiral as CoreSpiral
 from spiral.datetime_ import timestamp_micros
 from spiral.expressions import ExprLike
@@ -35,9 +35,13 @@ class Spiral:
         return self._config.api
 
     @property
-    def _core(self) -> CoreSpiral:
+    def core(self) -> CoreSpiral:
         return self._config.core
 
+    @property
+    def internal(self) -> Internal:
+        return self.core.internal(format=settings().file_format)
+
     @property
     def organization(self) -> str:
         if self._org is None:
@@ -79,19 +83,19 @@ class Spiral:
         """Open a table using an ID."""
         from spiral.table import Table
 
-        return Table(self, self._core.table(table_id))
+        return Table(self, self.core.table(table_id))
 
     def text_index(self, index_id: str) -> "TextIndex":
         """Open a text index using an ID."""
         from spiral.text_index import TextIndex
 
-        return TextIndex(self._core.text_index(index_id))
+        return TextIndex(self.core.text_index(index_id))
 
     def key_space_index(self, index_id: str) -> "KeySpaceIndex":
         """Open a key space index using an ID."""
         from spiral.key_space_index import KeySpaceIndex
 
-        return KeySpaceIndex(self._core.key_space_index(index_id))
+        return KeySpaceIndex(self.core.key_space_index(index_id))
 
     def scan(
         self,
@@ -117,7 +121,8 @@ class Spiral:
             where = se.lift(where)
 
         return Scan(
-            self._core.scan(
+            self,
+            self.core.scan(
                 projection.__expr__,
                 filter=where.__expr__ if where else None,
                 asof=asof,
@@ -155,17 +160,13 @@ class Spiral:
             freshness_window = timedelta(seconds=0)
         freshness_window_s = int(freshness_window.total_seconds())
 
-        return self._core.search(
+        return self.core.search(
             top_k=top_k,
             rank_by=rank_by.__expr__,
             filters=filters.__expr__ if filters else None,
             freshness_window_s=freshness_window_s,
         )
 
-    def _ops(self) -> Operations:
-        """Access maintenance operations."""
-        return self._core._ops(format=settings().file_format)
-
     @property
     def iceberg(self) -> "Iceberg":
         """

spiral/core/client/__init__.pyi

@@ -3,7 +3,7 @@ from typing import Any, Literal
 import pyarrow as pa
 from spiral.api.types import DatasetName, IndexName, ProjectId, RootUri, TableName
 from spiral.core.authn import Authn
-from spiral.core.table import ColumnGroupState, KeyRange, KeySpaceState, Scan, Snapshot, Table, Transaction
+from spiral.core.table import ColumnGroupState, KeyRange, KeySpaceState, Scan, ScanState, Snapshot, Table, Transaction
 from spiral.core.table.spec import ColumnGroup, Schema
 from spiral.expressions import Expr
 
@@ -30,6 +30,10 @@ class Spiral:
         """Construct a table scan."""
         ...
 
+    def load_scan(self, scan_state: ScanState) -> Scan:
+        """Load a scan from a serialized scan state."""
+        ...
+
     def transaction(self, table: Table, format: str | None = None, retries: int | None = 3) -> Transaction:
         """Being a table transaction."""
         ...
@@ -100,12 +104,8 @@ class Spiral:
         """Create a new key space index in the specified project."""
         ...
 
-    def _ops(self, *, format: str | None = None) -> Operations:
-        """Access maintenance operations.
-
-        IMPORTANT: This API is internal and is currently exposed for development & testing.
-            Maintenance operations are run by SpiralDB.
-        """
+    def internal(self, *, format: str | None = None) -> Internal:
+        """Internal client APIs. It can change without notice."""
         ...
 
 class TextIndex:
@@ -158,7 +158,7 @@ class ShuffleConfig:
         max_batch_size: int | None = None,
     ): ...
 
-class Operations:
+class Internal:
     def flush_wal(self, table: Table, *, keep_latest_s: int | None = None) -> None:
         """
         Flush the write-ahead log of the table.

spiral/core/expr/__init__.pyi

@@ -0,0 +1,15 @@
+from pyarrow import Array, DataType, Scalar
+
+class Expr:
+    """Low level expression class."""
+
+def aux(name: str, data_type: DataType) -> Expr: ...
+
+# Array is correct (there is no ArrayData), see the table here:
+# https://arrow.apache.org/rust/arrow_pyarrow/index.html
+def scalar(array: Array[Scalar[DataType]]) -> Expr: ...
+def not_(expr: Expr) -> Expr: ...
+def is_null(expr: Expr) -> Expr: ...
+def binary(op: str, expr: Expr, Expr: Expr) -> Expr: ...
+def cast(_expr: Expr, _data_type: DataType) -> Expr: ...
+def array_lit(array: Array[Scalar[DataType]]) -> Expr: ...

spiral/core/expr/images/__init__.pyi

@@ -0,0 +1,3 @@
+from .. import Expr
+
+def encode_(images: Expr, format: str) -> Expr: ...

spiral/core/expr/list_/__init__.pyi

@@ -0,0 +1,4 @@
+from .. import Expr
+
+def contains(list: Expr, expr: Expr) -> Expr: ...
+def element_at(list: Expr, element: Expr) -> Expr: ...

spiral/core/expr/refs/__init__.pyi

@@ -0,0 +1,4 @@
+from .. import Expr
+
+def ref(expr: Expr, field: str | None) -> Expr: ...
+def deref(expr: Expr, field: str | None) -> Expr: ...

spiral/core/expr/str_/__init__.pyi

@@ -0,0 +1,3 @@
+from .. import Expr
+
+def substr(expr: Expr, begin: int, end: int | None) -> Expr: ...

spiral/core/expr/struct_/__init__.pyi

@@ -0,0 +1,6 @@
+from .. import Expr
+
+def getitem(expr: Expr, item: str) -> Expr: ...
+def select(expr: Expr, including: list[str] | None = None, excluding: list[str] | None = None) -> Expr: ...
+def pack(names: list[str], children: list[str], nullable: bool) -> Expr: ...
+def merge(names: list[Expr]) -> Expr: ...

spiral/core/expr/text/__init__.pyi

@@ -0,0 +1,5 @@
+from .. import Expr
+
+def field(expr: Expr, tokeneizer: str | None) -> Expr: ...
+def find(expr: Expr, term: str) -> Expr: ...
+def boost(expr: Expr, factor: float) -> Expr: ...

spiral/core/expr/udf/__init__.pyi

@@ -0,0 +1,14 @@
+from collections.abc import Callable
+
+from pyarrow import Array, DataType, Scalar
+
+from .. import Expr
+
+class UDF:
+    def __call__(self, args: list[Expr]) -> Expr: ...
+
+def create(
+    name: str,
+    return_type: Callable[[tuple[DataType, ...]], DataType],
+    invoke: Callable[[tuple[Array[Scalar[DataType]], ...]], Array[Scalar[DataType]]],
+) -> UDF: ...

spiral/core/expr/video/__init__.pyi

@@ -0,0 +1,3 @@
+from .. import Expr
+
+def read(expr: Expr, ranges: Expr, crops: Expr, format: str) -> Expr: ...

spiral/core/table/__init__.pyi

@@ -5,7 +5,7 @@ from spiral.core.client import Shard, ShuffleConfig
 
 from .manifests import FragmentManifest
 from .metastore import PyMetastore
-from .spec import ColumnGroup, Key, Schema, WriteAheadLog
+from .spec import ColumnGroup, Key, Operation, Schema, WriteAheadLog
 
 class KeyRange:
     """A right-exclusive range of keys."""
@@ -52,6 +52,11 @@ class Snapshot:
     table: Table
     wal: WriteAheadLog
 
+class ScanState:
+    def to_json(self) -> str: ...
+    @staticmethod
+    def from_json(json: str) -> ScanState: ...
+
 class Scan:
     def key_schema(self) -> Schema: ...
     def schema(self) -> Schema: ...
@@ -62,8 +67,10 @@ class Scan:
     def column_groups(self) -> list[ColumnGroup]: ...
     def column_group_state(self, column_group: ColumnGroup) -> ColumnGroupState: ...
     def key_space_state(self, table_id: str) -> KeySpaceState: ...
+    def scan_state(self) -> ScanState: ...
     def to_record_batches(
         self,
+        key_range: KeyRange | None = None,
         key_table: pa.Table | pa.RecordBatch | None = None,
         batch_readahead: int | None = None,
     ) -> pa.RecordBatchReader: ...
@@ -95,7 +102,18 @@ class Transaction:
     status: str
 
     def write(self, table: pa.RecordBatchReader, *, partition_size_bytes: int | None = None): ...
+    def writeback(
+        self,
+        scan: Scan,
+        *,
+        key_range: KeyRange | None = None,
+        partition_size_bytes: int | None = None,
+        batch_readahead: int | None = None,
+    ): ...
     def drop_columns(self, column_paths: list[str]): ...
+    def take(self) -> list[Operation]: ...
+    def include(self, ops: list[Operation]): ...
     def commit(self): ...
     def abort(self): ...
+    def is_empty(self) -> bool: ...
     def metrics(self) -> dict[str, Any]: ...

spiral/core/table/spec/__init__.pyi

@@ -62,6 +62,12 @@ class ColumnGroupMetadata:
     def apply_wal(self, wal: WriteAheadLog) -> ColumnGroupMetadata:
         """Applies the given WAL to the metadata."""
 
+class Operation:
+    # Base class for all operations in the WAL.
+    def to_json(self) -> str: ...
+    @staticmethod
+    def from_json(json: str) -> Operation: ...
+
 class LogEntry:
     ts: int
     operation: (

spiral/dataloader.py

@@ -88,22 +88,24 @@ class SpiralDataLoader:
     - map_workers for parallel post-processing (tokenization, decoding, etc.)
     - Built-in checkpoint support via skip_samples
     - Explicit shard-based architecture for distributed training
-    """
 
-    # Example usage:
-    #
-    # Simple usage:
-    #   loader = SpiralDataLoader(scan, batch_size=32)
-    #   for batch in loader:
-    #       train_step(batch)
-    #
-    # With parallel transforms:
-    #   loader = SpiralDataLoader(
-    #       scan,
-    #       batch_size=32,
-    #       transform_fn=tokenize_batch,
-    #       map_workers=4,
-    #   )
+    Simple usage:
+    ```python
+    loader = SpiralDataLoader(scan, batch_size=32)
+    for batch in loader:
+        train_step(batch)
+    ```
+
+    With parallel transforms:
+    ```python
+    loader = SpiralDataLoader(
+        scan,
+        batch_size=32,
+        transform_fn=tokenize_batch,
+        map_workers=4,
+    )
+    ```
+    """
 
     def __init__(
         self,
@@ -119,6 +121,7 @@ class SpiralDataLoader:
         # TODO(os): accept vortex arrays here instead of Arrow
         transform_fn: Callable[[pa.RecordBatch], Any] | None = None,
         map_workers: int = 0,
+        infinite: bool = False,
     ):
         """Initialize SpiralDataLoader.
 
@@ -143,6 +146,9 @@ class SpiralDataLoader:
             map_workers: Number of worker processes for parallel transform_fn
                 application. 0 means single-process (no parallelism). Use this for
                 CPU-bound transforms like tokenization or audio decoding.
+            infinite: Whether to cycle through the dataset infinitely. If True,
+                the dataloader will repeat the dataset indefinitely. If False,
+                the dataloader will stop after going through the dataset once.
         """
         self.scan = scan
         self.shards = shards if shards is not None else scan.shards()
@@ -155,6 +161,7 @@ class SpiralDataLoader:
         self.batch_readahead = batch_readahead
         self.transform_fn = transform_fn
         self.map_workers = map_workers
+        self.infinite = infinite
 
         self._samples_yielded = 0
 
@@ -174,7 +181,7 @@ class SpiralDataLoader:
             shuffle=shuffle,
             max_batch_size=self.batch_size,
             batch_readahead=self.batch_readahead,
-            infinite=False,
+            infinite=self.infinite,
         )
 
         if self.skip_samples > 0:
@@ -220,16 +227,21 @@ class SpiralDataLoader:
 
         Returns:
             Dictionary containing samples_yielded, seed, and shards.
+
+        Example checkpoint:
+        ```python
+        loader = SpiralDataLoader(scan, batch_size=32, seed=42)
+        for i, batch in enumerate(loader):
+            if i == 10:
+                checkpoint = loader.state_dict()
+                break
+        ```
+
+        Example resume:
+        ```python
+        loader = SpiralDataLoader.from_state_dict(scan, checkpoint, batch_size=32)
+        ```
         """
-        # Example usage:
-        #   loader = SpiralDataLoader(scan, batch_size=32, seed=42)
-        #   for i, batch in enumerate(loader):
-        #       if i == 10:
-        #           checkpoint = loader.state_dict()
-        #           break
-        #
-        #   # Resume later with exact same shards
-        #   loader = SpiralDataLoader.from_state_dict(scan, checkpoint, batch_size=32)
         return {
             "samples_yielded": self._samples_yielded,
             "seed": self.seed,
@@ -257,20 +269,22 @@ class SpiralDataLoader:
 
         Returns:
             New SpiralDataLoader instance configured to resume from the checkpoint.
+
+        Save checkpoint during training:
+        ```python
+        loader = scan.to_distributed_data_loader(scan, batch_size=32, seed=42)
+        checkpoint = loader.state_dict()
+        ```
+
+        Resume later using the same shards from checkpoint:
+        ```python
+        resumed_loader = SpiralDataLoader.from_state_dict(
+            scan,
+            checkpoint,
+            batch_size=32,
+            transform_fn=my_transform,
+        )
         """
-        # Example usage:
-        #
-        # Save checkpoint during training:
-        #   loader = scan.to_distributed_data_loader(scan, batch_size=32, seed=42)
-        #   checkpoint = loader.state_dict()
-        #
-        # Resume later using the same shards from checkpoint:
-        #   resumed_loader = SpiralDataLoader.from_state_dict(
-        #       scan,
-        #       checkpoint,
-        #       batch_size=32,
-        #       transform_fn=my_transform,
-        #   )
 
         # Extract resume parameters from state
         seed = state.get("seed", 42)