pyspiral 0.6.10__cp312-abi3-macosx_11_0_arm64.whl → 0.6.12__cp312-abi3-macosx_11_0_arm64.whl

{pyspiral-0.6.10.dist-info → pyspiral-0.6.12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyspiral
-Version: 0.6.10
+Version: 0.6.12
 Classifier: Intended Audience :: Science/Research
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python

{pyspiral-0.6.10.dist-info → pyspiral-0.6.12.dist-info}/RECORD

@@ -1,8 +1,8 @@
-pyspiral-0.6.10.dist-info/METADATA,sha256=WpZleQ1yllvMlxL6zo5Ypw5Bzlebi10H7XxHM8EnstY,1843
-pyspiral-0.6.10.dist-info/WHEEL,sha256=KQvxBiy7GLcML6Ad3w_ZPrgSvER1uXd7aYb6wy6b44Y,103
-pyspiral-0.6.10.dist-info/entry_points.txt,sha256=uft7u-a6g40NLt4Q6BleWbK4NY0M8nZuYPpP8DV0EOk,45
+pyspiral-0.6.12.dist-info/METADATA,sha256=ANXjtdzd8s_zdWLd-mTm0X07pWbjlpkjQP8X5yP4qpY,1843
+pyspiral-0.6.12.dist-info/WHEEL,sha256=KQvxBiy7GLcML6Ad3w_ZPrgSvER1uXd7aYb6wy6b44Y,103
+pyspiral-0.6.12.dist-info/entry_points.txt,sha256=uft7u-a6g40NLt4Q6BleWbK4NY0M8nZuYPpP8DV0EOk,45
 spiral/__init__.py,sha256=n4JNLrO3wyw_k_U_JKyNiGON0wEpfvqxDhDdB2P6dhM,1007
-spiral/_lib.abi3.so,sha256=ptsuCD4f03_iVGiwI6y5VTZThQxOvPtOW-6xBMfMHSI,66386640
+spiral/_lib.abi3.so,sha256=ws3Kwbb_S3lcrGFbBHeYwfNVPWHi0uSPjJyzIS1r5W8,70093168
 spiral/adbc.py,sha256=7IxfWIeQN-fh0W5OdN_PP2x3pzQYg6ZUOLsHg3jktqw,14842
 spiral/api/__init__.py,sha256=ULBlVq3PnfNOO6T5naE_ULmmii-83--qTuN2PpAUQN0,2241
 spiral/api/admin.py,sha256=A1iVR1XYJSObZivPAD5UzmPuMgupXc9kaHNYYa_kwfs,585
@@ -24,52 +24,56 @@ spiral/cli/app.py,sha256=smzGj5a2RwhM9RQChmlEeKZLN4Fk60-bP7Lm5_Is1Rw,2760
 spiral/cli/console.py,sha256=6JHbAQV6MFWz3P-VzqPOjhHpkIQagsCdzTMvmuDKMkU,2580
 spiral/cli/fs.py,sha256=vaPcSc2YghhHeipxNitIdsHaBhFwlwkvPFqYsFSN9P0,2927
 spiral/cli/iceberg.py,sha256=Q14tcGcn1LixbFCYP0GhfYwFFXTmmi8tqBPYwalJEyE,3248
-spiral/cli/key_spaces.py,sha256=x3IFRP5d47pKiAHeWExYMOBaT2TwxbWjVM01SUqKrwI,2943
+spiral/cli/key_spaces.py,sha256=TF1tbRnrjemp4aMAbLc7o4_jPChIumaQGPuvfW0sR5o,2945
 spiral/cli/login.py,sha256=2tw6uN5rEpiMMAmjQSB3-JUPf3C0Wc1eTGCDxhYtJps,731
 spiral/cli/orgs.py,sha256=fmOuLxpeIFfKqePRi292Gv9k-EF5pPn_tbKd2BLl2Ig,2869
 spiral/cli/printer.py,sha256=aosc763hDFgoXJGkiANmNyO3kAsecAS1JWgjEhn8GCM,1784
 spiral/cli/projects.py,sha256=1M1nGrBT-t0aY9RV5Cnmzy7YrhIvmHwdkpa3y9j8rG8,5756
 spiral/cli/state.py,sha256=10wTIVQ0SJkY67Z6-KQ1LFlt3aVIPmZhoHFdTwp4kNA,130
-spiral/cli/tables.py,sha256=fFte_wMNcB0V-fmfSXfSbtV4UlAi-Xw5nYDJ0b62CGk,6360
+spiral/cli/tables.py,sha256=qm3izcysElJrQlerNZdfx5RWSVXtyVfkP3o_H51ltFw,6366
 spiral/cli/telemetry.py,sha256=Uxo1Q1FkKJ6n6QNGOUmL3j_pRRWRx0qWIhoP-U9BuR0,589
 spiral/cli/text.py,sha256=DlWGe4JrkdERAiqyITNpk91Wqb63Re99rNYlIFsIamc,4031
 spiral/cli/types.py,sha256=XYzo1GgX7dBBItoBSrHI4vO5C2lLmS2sktb-2GnGH3E,1362
 spiral/cli/workloads.py,sha256=2_SLfQTFN6y73R9H0i9dk8VIOVagKxSxOpHXC56yptY,2015
-spiral/client.py,sha256=N4sQLxtQ6GYCnj00hm4VX1vUVUqzQdHhl_KfQwp-1LQ,6345
+spiral/client.py,sha256=pw6vB85oLVbBudc_HRzmLCItcecsTjNM5SMu_kVOMCo,6568
 spiral/core/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 spiral/core/_tools/__init__.pyi,sha256=b2KLfTOQ67pjfbYt07o0IGiTu5o2bZw69lllV8v0Dps,143
 spiral/core/authn/__init__.pyi,sha256=z_GWyIS62fuiYQrYO8hzw4W8oGaiciqS1u5qtAt54VY,769
-spiral/core/client/__init__.pyi,sha256=1HK3SOMT1QKmD5Hai58ZFjiEZK0QzyYtP84hse8SBEI,6666
-spiral/core/table/__init__.pyi,sha256=QqG_pMlPhMtXG-56dXyQjOWVKMugPP0nnYnvYaY0Q10,3288
+spiral/core/client/__init__.pyi,sha256=ajF8XaxThnTdsPpw1k3pPLEurIaDg9yeXqwIRJNlJTY,6665
+spiral/core/expr/__init__.pyi,sha256=3HSKjkotiEkxBvGBALXEBIie0JiyI9bCpehwA3nMQkU,571
+spiral/core/expr/images/__init__.pyi,sha256=wnE_wZXq7a4iqTg3SVm-ssxGw1WQZyk5dGOPaP4Btko,73
+spiral/core/expr/list_/__init__.pyi,sha256=Q_9c87eIQfZbqlaw_rq3fvs93YEsW7K5VYk6VZ4g6mU,126
+spiral/core/expr/refs/__init__.pyi,sha256=nZZP3l_Z6bLx6V8lTcH3Jgo--xwfADOU2XdTAvM5IMk,127
+spiral/core/expr/str_/__init__.pyi,sha256=Bm6fZK-d4fNbJuuBhVoWMACXUbQQ2SjlhgrOpdOHIPM,86
+spiral/core/expr/struct_/__init__.pyi,sha256=MXckd98eV_x3X0RhEWvlkA3DcDXRtLs5pNnTQkc09nE,296
+spiral/core/expr/text/__init__.pyi,sha256=ed83n1xcsGY7_QDhMmJGnSQ20UrJFXcdv1AveSEcS1c,175
+spiral/core/expr/udf/__init__.pyi,sha256=zsZs081KVhY3-1JidqTkWMW81Qd_ScoTGZvasIhIK-4,358
+spiral/core/expr/video/__init__.pyi,sha256=nQJEcSsigZuRpMjkI_O4EEtMK_n2zRvorcL_KEeD5vU,95
+spiral/core/table/__init__.pyi,sha256=HN4ag8E1QDF_VgekJZqjhuQLhorU3ivjIOBHai2OEVc,3672
 spiral/core/table/manifests/__init__.pyi,sha256=eVfDpmhYSjafIvvALqAkZe5baN3Y1HpKpxYEbjwd4gQ,1043
 spiral/core/table/metastore/__init__.pyi,sha256=rc3u9MwEKRvL2kxOc8lBorddFRnM8o_o1frqtae86a4,1697
-spiral/core/table/spec/__init__.pyi,sha256=OFYJXPXix7gskYJIMog7IniZslEPJ0xvL-sUSFDPbXs,5643
-spiral/dataloader.py,sha256=FFZhIflQPEygXe-xBLifQnnxANi4CFooaHRm4i-EGHo,10335
+spiral/core/table/spec/__init__.pyi,sha256=PgacM_fZmkHuplj7IbYrj5KfFI3-VPYnyuzI2w7A70Y,5717
+spiral/dataloader.py,sha256=2haLoI6KLrzXfPozAgEa-eCOSDsNldJ1qwCmFpNMyTQ,10281
 spiral/dataset.py,sha256=PMLoXnXuEUciP6-NXqTmQLXu0UIH7OcC4-iZtY_iuO8,7973
 spiral/datetime_.py,sha256=elXaUWtZuuLVcu9E0aXnvYRPB9XWqZbLDToozQYQYjU,950
 spiral/debug/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 spiral/debug/manifests.py,sha256=7f1O3ba9mrA5nXpOF9cEIQuUAteP5wiBkFy_diQJ7No,3216
 spiral/debug/metrics.py,sha256=XdRDcjggtsLNGCAjam6IxG9072pz_d2C8iLApNRFUtk,2044
 spiral/debug/scan.py,sha256=UEm_aRnql5pwDPTpZgakMLNjlzkKL4RurBFFqH_BLAQ,9526
-spiral/expressions/__init__.py,sha256=KhwFjVKoFgx1S6hkVcE8aZjoHY_1N-BgQ2rGEZfPQvM,7957
-spiral/expressions/base.py,sha256=915gpvZZCTRCO5q93pwwmhf-R6C23LQsyDt4Q2dHk9s,5290
-spiral/expressions/http.py,sha256=begUydWoFHEqjeLkATvI_v66Ez6_rR-OQBWO5cHbb9c,2742
-spiral/expressions/io.py,sha256=gJ2a0FKMmdxarWKENulPRwH7KDvSJTIh_OUxX306xAM,3045
+spiral/expressions/__init__.py,sha256=UNxK5qQNrl-BuHsjKcWDj35w5lJviLkGFUQj8OhLID0,7919
+spiral/expressions/base.py,sha256=PvhJkcUSsPSIaxirHVzM9zlqyBXiaiia1HXohXdOmL4,5377
 spiral/expressions/list_.py,sha256=MMt5lf5H1M3O-x6N_PvqOLGq9NOk6Ukv0fPWwPC_uy4,1809
-spiral/expressions/mp4.py,sha256=_xGVnkygddzxP9a8OACJ8_KXnejuVbYCVKBCXBQ798Y,2151
-spiral/expressions/png.py,sha256=KO8X0OmMzUFwpg2I_j0JTyldPzVXDWIMzjWMWDV9vIY,506
-spiral/expressions/qoi.py,sha256=gvIbb6fXb_Bb080sn9wkpbGGrPs2UEcTXCfuv4-kcYQ,506
-spiral/expressions/refs.py,sha256=omeHBQ5o6N4xgZ3x5Xz7IRrWwYBBtQY8DYK0NNAxeGo,2109
+spiral/expressions/s3.py,sha256=D-kuLifIEY314Q8rB2-ZP8U-IT0FywtbJDMuyusBKiQ,414
 spiral/expressions/str_.py,sha256=tY8RXW3JWvr1-bEfCZtk5FAf11wKJnXPuA9EoeJ9tA4,1265
 spiral/expressions/struct.py,sha256=pGAnCDh6AK0BK1XfZ1qG4ce4ranIQEE1HQsgmzBcfwQ,2038
 spiral/expressions/text.py,sha256=-02gBWYoyNQ3qQ1--9HTa8IryUDojYQVIp8C7rgnOWQ,1893
-spiral/expressions/tiff.py,sha256=fQwIn0kLFBM2Y3YYIHmTgb_EIRHKT2fNc77nioDQQw4,8044
-spiral/expressions/udf.py,sha256=yb9MIcrFftpNDxgBF228cvdv6TY-hEFikYz2fq_nzWo,1353
+spiral/expressions/tiff.py,sha256=4dngO97bT1QY0By7-PxOQVmSwQC3PQAiixVhLJ-4HMQ,7986
+spiral/expressions/udf.py,sha256=yvZCuGK9S9Sa9I18h-apUxsDni2B7E9WEqPrxHBjUWE,1657
 spiral/grpc_.py,sha256=f3czdP1Mxme42Y5--a5ogYq1TTiWn-J_MlGjwJ2mWwM,1015
 spiral/iceberg.py,sha256=JGq62Qnf296r9_hRAoH85GQq45-uSBjwXWw_CvPi6G4,930
 spiral/iterable_dataset.py,sha256=Eekg9ad8tcwXcloHWReBbvCSr5ZappRHn2ldKTvwqS0,4622
 spiral/key_space_index.py,sha256=NAB_nONEjpMYbse8suz42w7Qb5OPHuKN9h9CT2NJe08,1460
-spiral/project.py,sha256=CO_Pn6vPqaonNvRdCNRFcBWr4TqO2AsAUTH5xawIeCE,7283
+spiral/project.py,sha256=VsokZgS0TqIel7UAXMyoBToxn-l_D3ivGwc41x7HLF0,7277
 spiral/protogen/_/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 spiral/protogen/_/arrow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 spiral/protogen/_/arrow/flight/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -87,16 +91,16 @@ spiral/protogen/_/substrait/extensions/__init__.py,sha256=nhnEnho70GAT8WPj2xtwJU
 spiral/protogen/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 spiral/protogen/util.py,sha256=smnvVo6nYH3FfDm9jqhNLaXz4bbTBaQezHQDCTvZyiQ,1486
 spiral/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-spiral/scan.py,sha256=4PUlI_DHbO1WTttLia6DinhGtOWsCiqek4ZljoEiRZc,10523
+spiral/scan.py,sha256=fXZq0NL9YApt-UwkGpaT0ETn-rK-1_tltq7nqsImZI4,11199
 spiral/server.py,sha256=ztBmB5lBnUz-smQxR_tC8AI5SOhz17wH0MI3GuzDUdM,600
 spiral/settings.py,sha256=JRQSwjJyNaCqQdQLxiqB_O_LZRQXMLyshJBrI2LZHwM,3113
 spiral/snapshot.py,sha256=cTobi5jtiANxalGA-isokQHblNmXGtuUvgUGGNVybsI,1555
 spiral/streaming_/__init__.py,sha256=s7MlW2ERsuZmZGExLFL6RcZon2e0tNBocBg5ANgki7k,61
 spiral/streaming_/reader.py,sha256=tl_lC9xgh1-QFhsZn4xQT7It3PVTzHCEUT2BG2dWBRQ,4166
-spiral/streaming_/stream.py,sha256=nXnygiuCxi1D3PhaxV8Ujif4J9ly_OczA7CZ3W4WN2w,5913
+spiral/streaming_/stream.py,sha256=DM1hBDHnWm1ZFKZ-hZ4zxeSXITcUI6kWzwdJZvywI8o,5915
 spiral/substrait_.py,sha256=AKeOD4KIXvz2J4TYxnIneOiHddtBIyOhuNxVO_uH0eg,12592
-spiral/table.py,sha256=G05b6M0uVmT5ew5GxuzsVB4rQzg25W3zGMTftL07pJU,11026
+spiral/table.py,sha256=dwQr1EAACbfxG8fISFqRrUEAE2P2y6xsx0vFK9Gwyfc,11662
 spiral/text_index.py,sha256=FQ9rgIEGLSJryS9lFdMhKtPFey18BXoWbPXyvZPJJ04,442
-spiral/transaction.py,sha256=h6YdAwOYX6qq-tXYV4i9yhy1Nq1tIfRphY_fk7Q_yLQ,1854
+spiral/transaction.py,sha256=M_Tf-TijVBluuInWk6XSFNCR2dKN4S9EdsHM3QD20ng,2948
 spiral/types_.py,sha256=W_jyO7F6rpPiH69jhgSgV7OxQZbOlb1Ho3InpKUP6Eo,155
-pyspiral-0.6.10.dist-info/RECORD,,
+pyspiral-0.6.12.dist-info/RECORD,,

spiral/cli/key_spaces.py

@@ -64,7 +64,7 @@ def show(
     """Show index partitions."""
     index_id = get_index_id(project, name)
     index = state.spiral.key_space_index(index_id)
-    shards = state.spiral._ops().compute_shards(index.core)
+    shards = state.spiral.internal.compute_shards(index.core)
 
     rich_table = rich.table.Table("Begin", "End", "Cardinality", title=f"Index {index.name} Partitions")
     for partition in shards:

spiral/cli/tables.py

@@ -130,7 +130,7 @@ def flush(
     keep_latest_s = int(duration.total_seconds()) if duration is not None else None
 
     identifier, t = get_table(project, table, dataset)
-    state.spiral._ops().flush_wal(t.core, keep_latest_s=keep_latest_s)  # pyright: ignore[reportPrivateUsage]
+    state.spiral.internal.flush_wal(t.core, keep_latest_s=keep_latest_s)  # pyright: ignore[reportPrivateUsage]
     CONSOLE.print(f"Flushed WAL for table {identifier} in project {project}.")
 
 
@@ -143,10 +143,10 @@ def manifests(
     _, t = get_table(project, table, dataset)
     s = t.snapshot()
 
-    key_space_state = state.spiral._ops().key_space_state(s.core)  # pyright: ignore[reportPrivateUsage]
+    key_space_state = state.spiral.internal.key_space_state(s.core)  # pyright: ignore[reportPrivateUsage]
     key_space_manifest = key_space_state.manifest
 
-    column_groups_states = state.spiral._ops().column_groups_states(s.core, key_space_state)  # pyright: ignore[reportPrivateUsage]
+    column_groups_states = state.spiral.internal.column_groups_states(s.core, key_space_state)  # pyright: ignore[reportPrivateUsage]
     display_manifests(key_space_manifest, [(x.column_group, x.manifest) for x in column_groups_states])
 
 

spiral/client.py

@@ -6,11 +6,11 @@ import pyarrow as pa
 
 from spiral.api import SpiralAPI
 from spiral.api.projects import CreateProjectRequest, CreateProjectResponse
-from spiral.core.client import Operations
+from spiral.core.client import Internal
 from spiral.core.client import Spiral as CoreSpiral
 from spiral.datetime_ import timestamp_micros
 from spiral.expressions import ExprLike
-from spiral.scan import Scan
+from spiral.scan import Scan, ScanState
 from spiral.settings import Settings, settings
 
 if TYPE_CHECKING:
@@ -35,9 +35,13 @@ class Spiral:
         return self._config.api
 
     @property
-    def _core(self) -> CoreSpiral:
+    def core(self) -> CoreSpiral:
         return self._config.core
 
+    @property
+    def internal(self) -> Internal:
+        return self.core.internal(format=settings().file_format)
+
     @property
     def organization(self) -> str:
         if self._org is None:
@@ -79,19 +83,19 @@ class Spiral:
         """Open a table using an ID."""
         from spiral.table import Table
 
-        return Table(self, self._core.table(table_id))
+        return Table(self, self.core.table(table_id))
 
     def text_index(self, index_id: str) -> "TextIndex":
         """Open a text index using an ID."""
         from spiral.text_index import TextIndex
 
-        return TextIndex(self._core.text_index(index_id))
+        return TextIndex(self.core.text_index(index_id))
 
     def key_space_index(self, index_id: str) -> "KeySpaceIndex":
         """Open a key space index using an ID."""
         from spiral.key_space_index import KeySpaceIndex
 
-        return KeySpaceIndex(self._core.key_space_index(index_id))
+        return KeySpaceIndex(self.core.key_space_index(index_id))
 
     def scan(
         self,
@@ -117,13 +121,21 @@ class Spiral:
             where = se.lift(where)
 
         return Scan(
-            self._core.scan(
+            self.core.scan(
                 projection.__expr__,
                 filter=where.__expr__ if where else None,
                 asof=asof,
             ),
         )
 
+    def load_scan(self, scan_state: ScanState) -> Scan:
+        """Load a scan from a serialized scan state.
+
+        Args:
+            scan_state: The serialized scan state.
+        """
+        return Scan(self.core.load_scan(scan_state.core))
+
     # TODO(marko): This should be query, and search should be query + scan.
     def search(
         self,
@@ -155,17 +167,13 @@ class Spiral:
             freshness_window = timedelta(seconds=0)
         freshness_window_s = int(freshness_window.total_seconds())
 
-        return self._core.search(
+        return self.core.search(
             top_k=top_k,
             rank_by=rank_by.__expr__,
             filters=filters.__expr__ if filters else None,
             freshness_window_s=freshness_window_s,
         )
 
-    def _ops(self) -> Operations:
-        """Access maintenance operations."""
-        return self._core._ops(format=settings().file_format)
-
     @property
     def iceberg(self) -> "Iceberg":
         """

spiral/core/client/__init__.pyi

@@ -3,7 +3,7 @@ from typing import Any, Literal
 import pyarrow as pa
 from spiral.api.types import DatasetName, IndexName, ProjectId, RootUri, TableName
 from spiral.core.authn import Authn
-from spiral.core.table import ColumnGroupState, KeyRange, KeySpaceState, Scan, Snapshot, Table, Transaction
+from spiral.core.table import ColumnGroupState, KeyRange, KeySpaceState, Scan, ScanState, Snapshot, Table, Transaction
 from spiral.core.table.spec import ColumnGroup, Schema
 from spiral.expressions import Expr
 
@@ -30,6 +30,10 @@ class Spiral:
         """Construct a table scan."""
         ...
 
+    def load_scan(self, scan_state: ScanState) -> Scan:
+        """Load a scan from a serialized scan state."""
+        ...
+
     def transaction(self, table: Table, format: str | None = None, retries: int | None = 3) -> Transaction:
         """Being a table transaction."""
         ...
@@ -100,12 +104,8 @@ class Spiral:
         """Create a new key space index in the specified project."""
         ...
 
-    def _ops(self, *, format: str | None = None) -> Operations:
-        """Access maintenance operations.
-
-        IMPORTANT: This API is internal and is currently exposed for development & testing.
-            Maintenance operations are run by SpiralDB.
-        """
+    def internal(self, *, format: str | None = None) -> Internal:
+        """Internal client APIs. It can change without notice."""
         ...
 
 class TextIndex:
@@ -158,7 +158,7 @@ class ShuffleConfig:
         max_batch_size: int | None = None,
     ): ...
 
-class Operations:
+class Internal:
     def flush_wal(self, table: Table, *, keep_latest_s: int | None = None) -> None:
         """
         Flush the write-ahead log of the table.

spiral/core/expr/__init__.pyi

@@ -0,0 +1,15 @@
+from pyarrow import Array, DataType, Scalar
+
+class Expr:
+    """Low level expression class."""
+
+def aux(name: str, data_type: DataType) -> Expr: ...
+
+# Array is correct (there is no ArrayData), see the table here:
+# https://arrow.apache.org/rust/arrow_pyarrow/index.html
+def scalar(array: Array[Scalar[DataType]]) -> Expr: ...
+def not_(expr: Expr) -> Expr: ...
+def is_null(expr: Expr) -> Expr: ...
+def binary(op: str, expr: Expr, Expr: Expr) -> Expr: ...
+def cast(_expr: Expr, _data_type: DataType) -> Expr: ...
+def array_lit(array: Array[Scalar[DataType]]) -> Expr: ...

spiral/core/expr/images/__init__.pyi

@@ -0,0 +1,3 @@
+from .. import Expr
+
+def encode_(images: Expr, format: str) -> Expr: ...

spiral/core/expr/list_/__init__.pyi

@@ -0,0 +1,4 @@
+from .. import Expr
+
+def contains(list: Expr, expr: Expr) -> Expr: ...
+def element_at(list: Expr, element: Expr) -> Expr: ...

spiral/core/expr/refs/__init__.pyi

@@ -0,0 +1,4 @@
+from .. import Expr
+
+def ref(expr: Expr, field: str | None) -> Expr: ...
+def deref(expr: Expr, field: str | None) -> Expr: ...

spiral/core/expr/str_/__init__.pyi

@@ -0,0 +1,3 @@
+from .. import Expr
+
+def substr(expr: Expr, begin: int, end: int | None) -> Expr: ...

spiral/core/expr/struct_/__init__.pyi

@@ -0,0 +1,6 @@
+from .. import Expr
+
+def getitem(expr: Expr, item: str) -> Expr: ...
+def select(expr: Expr, including: list[str] | None = None, excluding: list[str] | None = None) -> Expr: ...
+def pack(names: list[str], children: list[str], nullable: bool) -> Expr: ...
+def merge(names: list[Expr]) -> Expr: ...

spiral/core/expr/text/__init__.pyi

@@ -0,0 +1,5 @@
+from .. import Expr
+
+def field(expr: Expr, tokeneizer: str | None) -> Expr: ...
+def find(expr: Expr, term: str) -> Expr: ...
+def boost(expr: Expr, factor: float) -> Expr: ...

spiral/core/expr/udf/__init__.pyi

@@ -0,0 +1,14 @@
+from collections.abc import Callable
+
+from pyarrow import Array, DataType, Scalar
+
+from .. import Expr
+
+class UDF:
+    def __call__(self, args: list[Expr]) -> Expr: ...
+
+def create(
+    name: str,
+    return_type: Callable[[tuple[DataType, ...]], DataType],
+    invoke: Callable[[tuple[Array[Scalar[DataType]], ...]], Array[Scalar[DataType]]],
+) -> UDF: ...

spiral/core/expr/video/__init__.pyi

@@ -0,0 +1,3 @@
+from .. import Expr
+
+def read(expr: Expr, ranges: Expr, crops: Expr, format: str) -> Expr: ...

spiral/core/table/__init__.pyi

@@ -5,7 +5,7 @@ from spiral.core.client import Shard, ShuffleConfig
 
 from .manifests import FragmentManifest
 from .metastore import PyMetastore
-from .spec import ColumnGroup, Key, Schema, WriteAheadLog
+from .spec import ColumnGroup, Key, Operation, Schema, WriteAheadLog
 
 class KeyRange:
     """A right-exclusive range of keys."""
@@ -52,6 +52,11 @@ class Snapshot:
     table: Table
     wal: WriteAheadLog
 
+class ScanState:
+    def to_json(self) -> str: ...
+    @staticmethod
+    def from_json(json: str) -> ScanState: ...
+
 class Scan:
     def key_schema(self) -> Schema: ...
     def schema(self) -> Schema: ...
@@ -62,6 +67,7 @@ class Scan:
     def column_groups(self) -> list[ColumnGroup]: ...
     def column_group_state(self, column_group: ColumnGroup) -> ColumnGroupState: ...
     def key_space_state(self, table_id: str) -> KeySpaceState: ...
+    def scan_state(self) -> ScanState: ...
     def to_record_batches(
         self,
         key_table: pa.Table | pa.RecordBatch | None = None,
@@ -95,7 +101,10 @@ class Transaction:
     status: str
 
     def write(self, table: pa.RecordBatchReader, *, partition_size_bytes: int | None = None): ...
+    def writeback(self, scan: Scan, *, partition_size_bytes: int | None = None, batch_readahead: int | None = None): ...
     def drop_columns(self, column_paths: list[str]): ...
+    def take(self) -> list[Operation]: ...
+    def include(self, ops: list[Operation]): ...
     def commit(self): ...
     def abort(self): ...
     def metrics(self) -> dict[str, Any]: ...

spiral/core/table/spec/__init__.pyi

@@ -62,6 +62,10 @@ class ColumnGroupMetadata:
     def apply_wal(self, wal: WriteAheadLog) -> ColumnGroupMetadata:
         """Applies the given WAL to the metadata."""
 
+class Operation:
+    # Base class for all operations in the WAL.
+    ...
+
 class LogEntry:
     ts: int
     operation: (

spiral/dataloader.py

@@ -88,22 +88,24 @@ class SpiralDataLoader:
     - map_workers for parallel post-processing (tokenization, decoding, etc.)
     - Built-in checkpoint support via skip_samples
     - Explicit shard-based architecture for distributed training
-    """
 
-    # Example usage:
-    #
-    # Simple usage:
-    #   loader = SpiralDataLoader(scan, batch_size=32)
-    #   for batch in loader:
-    #       train_step(batch)
-    #
-    # With parallel transforms:
-    #   loader = SpiralDataLoader(
-    #       scan,
-    #       batch_size=32,
-    #       transform_fn=tokenize_batch,
-    #       map_workers=4,
-    #   )
+    Simple usage:
+    ```python
+    loader = SpiralDataLoader(scan, batch_size=32)
+    for batch in loader:
+        train_step(batch)
+    ```
+
+    With parallel transforms:
+    ```python
+    loader = SpiralDataLoader(
+        scan,
+        batch_size=32,
+        transform_fn=tokenize_batch,
+        map_workers=4,
+    )
+    ```
+    """
 
     def __init__(
         self,
@@ -220,16 +222,21 @@ class SpiralDataLoader:
 
         Returns:
             Dictionary containing samples_yielded, seed, and shards.
+
+        Example checkpoint:
+        ```python
+        loader = SpiralDataLoader(scan, batch_size=32, seed=42)
+        for i, batch in enumerate(loader):
+            if i == 10:
+                checkpoint = loader.state_dict()
+                break
+        ```
+
+        Example resume:
+        ```python
+        loader = SpiralDataLoader.from_state_dict(scan, checkpoint, batch_size=32)
+        ```
         """
-        # Example usage:
-        #   loader = SpiralDataLoader(scan, batch_size=32, seed=42)
-        #   for i, batch in enumerate(loader):
-        #       if i == 10:
-        #           checkpoint = loader.state_dict()
-        #           break
-        #
-        #   # Resume later with exact same shards
-        #   loader = SpiralDataLoader.from_state_dict(scan, checkpoint, batch_size=32)
         return {
             "samples_yielded": self._samples_yielded,
             "seed": self.seed,
@@ -257,20 +264,22 @@ class SpiralDataLoader:
 
         Returns:
             New SpiralDataLoader instance configured to resume from the checkpoint.
+
+        Save checkpoint during training:
+        ```python
+        loader = scan.to_distributed_data_loader(scan, batch_size=32, seed=42)
+        checkpoint = loader.state_dict()
+        ```
+
+        Resume later using the same shards from checkpoint:
+        ```python
+        resumed_loader = SpiralDataLoader.from_state_dict(
+            scan,
+            checkpoint,
+            batch_size=32,
+            transform_fn=my_transform,
+        )
         """
-        # Example usage:
-        #
-        # Save checkpoint during training:
-        #   loader = scan.to_distributed_data_loader(scan, batch_size=32, seed=42)
-        #   checkpoint = loader.state_dict()
-        #
-        # Resume later using the same shards from checkpoint:
-        #   resumed_loader = SpiralDataLoader.from_state_dict(
-        #       scan,
-        #       checkpoint,
-        #       batch_size=32,
-        #       transform_fn=my_transform,
-        #   )
 
         # Extract resume parameters from state
         seed = state.get("seed", 42)

spiral/expressions/__init__.py

@@ -8,31 +8,22 @@ import pyarrow as pa
 
 from spiral import _lib, arrow_
 
-from . import http as http
-from . import io as io
 from . import list_ as list
-from . import mp4 as mp4
-from . import png as png
-from . import qoi as qoi
-from . import refs as refs
 from . import str_ as str
 from . import struct as struct
 from . import text as text
-from . import tiff as tiff
 from .base import Expr, ExprLike, NativeExpr
+from .udf import UDF
 
 __all__ = [
     "Expr",
     "add",
     "and_",
-    "deref",
     "divide",
     "eq",
     "getitem",
     "gt",
     "gte",
-    "http",
-    "io",
     "is_not_null",
     "is_null",
     "lift",
@@ -48,19 +39,15 @@ __all__ = [
     "or_",
     "pack",
     "aux",
-    "ref",
-    "refs",
     "scalar",
     "select",
     "str",
     "struct",
     "subtract",
-    "tiff",
     "xor",
-    "png",
-    "qoi",
-    "mp4",
     "text",
+    "s3",
+    "UDF",
 ]
 
 # Inline some of the struct expressions since they're so common
@@ -68,8 +55,6 @@ getitem = struct.getitem
 merge = struct.merge
 pack = struct.pack
 select = struct.select
-ref = refs.ref
-deref = refs.deref
 
 
 def lift(expr: ExprLike) -> Expr:
@@ -127,9 +112,17 @@ def evaluate(expr: ExprLike) -> pa.RecordBatchReader:
         return pa.RecordBatchReader.from_batches(expr.schema, [expr])
     if isinstance(expr, pa.StructArray):
         return pa.Table.from_struct_array(expr).to_reader()
+
     if isinstance(expr, pa.ChunkedArray):
-        # TODO(marko): We shouldn't need to combine chunks here._
-        return evaluate(expr.combine_chunks())
+        if not pa.types.is_struct(expr.type):
+            raise ValueError("Arrow chunked array must be a struct type.")
+
+        def _iter_batches():
+            for chunk in expr.chunks:
+                yield pa.RecordBatch.from_struct_array(chunk)
+
+        return pa.RecordBatchReader.from_batches(pa.schema(expr.type.fields), _iter_batches())
+
     if isinstance(expr, pa.Array):
         raise ValueError("Arrow array must be a struct array.")
 

spiral/expressions/base.py

@@ -1,6 +1,5 @@
-import builtins
 import datetime
-from typing import TypeAlias
+from typing import TypeAlias, Union
 
 import pyarrow as pa
 
@@ -153,5 +152,11 @@ class Expr:
 
 
 ScalarLike: TypeAlias = bool | int | float | str | list["ScalarLike"] | datetime.datetime | None
-ArrowLike: TypeAlias = pa.Array | pa.ChunkedArray | pa.Scalar | pa.RecordBatch | pa.Table
-ExprLike: TypeAlias = Expr | dict[str, "ExprLike"] | builtins.list | ArrowLike | ScalarLike
+ArrowLike: TypeAlias = Union[
+    pa.RecordBatch,
+    "pa.Array[pa.Scalar[pa.DataType]]",
+    "pa.ChunkedArray[pa.Scalar[pa.DataType]]",
+    "pa.Scalar[pa.DataType]",
+    pa.Table,
+]
+ExprLike: TypeAlias = Expr | dict[str, "ExprLike"] | list["ExprLike"] | ArrowLike | ScalarLike

spiral/expressions/s3.py

@@ -0,0 +1,18 @@
+from spiral import _lib
+from spiral.expressions.base import Expr, ExprLike
+
+
+def get(expr: ExprLike) -> Expr:
+    """Read data from object storage by the object's URL.
+
+    Args:
+        expr: URLs of the data that needs to be read from object storage.
+    """
+    from spiral import expressions as se
+
+    expr = se.lift(expr)
+    return Expr(
+        _lib.expr.s3.get(
+            expr.__expr__,
+        )
+    )