pyspiral 0.7.4__cp312-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl → 0.7.6__cp312-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl

{pyspiral-0.7.4.dist-info → pyspiral-0.7.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyspiral
-Version: 0.7.4
+Version: 0.7.6
 Classifier: Intended Audience :: Science/Research
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python

{pyspiral-0.7.4.dist-info → pyspiral-0.7.6.dist-info}/RECORD

@@ -1,8 +1,8 @@
-pyspiral-0.7.4.dist-info/METADATA,sha256=dkmwzJEE-q1T5D2wSLTcYa2KdV64EVtMvJjGIpCp8vA,1874
-pyspiral-0.7.4.dist-info/WHEEL,sha256=0ecHyBdkJfSXYIVmWsPh7S-4h4fSrB4FlXhlnIu9c_A,130
-pyspiral-0.7.4.dist-info/entry_points.txt,sha256=R96Y3FpYX6XbQu9qMPfUTgiCcf4qM9OBQQZTDdBkZwA,74
+pyspiral-0.7.6.dist-info/METADATA,sha256=mBOLr8JlizyBr3w2oyvielfM-EdxU3vOtLG85rQpcHw,1874
+pyspiral-0.7.6.dist-info/WHEEL,sha256=0ecHyBdkJfSXYIVmWsPh7S-4h4fSrB4FlXhlnIu9c_A,130
+pyspiral-0.7.6.dist-info/entry_points.txt,sha256=R96Y3FpYX6XbQu9qMPfUTgiCcf4qM9OBQQZTDdBkZwA,74
 spiral/__init__.py,sha256=PwaYBWFBtB7cYi7peMmhk_Lm5XzjRoLwOtLbUhc1ZDo,1449
-spiral/_lib.abi3.so,sha256=ehNLyfXaddUympC2OxPfDQdEYYkKdeyHpApMIbcoszI,61374168
+spiral/_lib.abi3.so,sha256=dEIf28Sm6Ur-uWeMlqzGzPMTveg3BzBUl9Iz1aP6TsU,61499608
 spiral/adbc.py,sha256=7IxfWIeQN-fh0W5OdN_PP2x3pzQYg6ZUOLsHg3jktqw,14842
 spiral/api/__init__.py,sha256=ULBlVq3PnfNOO6T5naE_ULmmii-83--qTuN2PpAUQN0,2241
 spiral/api/admin.py,sha256=A1iVR1XYJSObZivPAD5UzmPuMgupXc9kaHNYYa_kwfs,585
@@ -13,7 +13,7 @@ spiral/api/organizations.py,sha256=B-8zZ7lFJANGK7dUNbo_aU-cgI959JBP9VcWb6wdgi0,1
 spiral/api/projects.py,sha256=1JC7VjqZJfwR6zfhBZr3OCwaf6zb-MXMOBTE_NztmcE,6356
 spiral/api/telemetry.py,sha256=tfdA3E_EWJwFVxkQfkm8tiYGRubnx2LuE5nbfsk1oG4,474
 spiral/api/text_indexes.py,sha256=_zVlGBytl-9-Unbu2POfZgLh40H1YRcagFtplgIG428,1828
-spiral/api/types.py,sha256=lGdiKViRgIEJXD2ubwnyEIEwHkfRumlZjVEaHMV3Tm8,682
+spiral/api/types.py,sha256=HpHsoBuf7IdlXb7Dw-BkBkEvxBVIhkI8JviqhuoP9pY,696
 spiral/api/workers.py,sha256=0wZNUHMioDT53P1OBJfpjyDfIodHwwT6858z2IlRIM4,636
 spiral/api/workloads.py,sha256=XAyXV7vgZcoyyoPoGvOT4jTpyFKFMvrrAfhL6d1h1kE,1748
 spiral/arrow_.py,sha256=fdSIfIs7UjDxXZlppvOW0zz86W_70Pa5pagJilH2kOE,7583
@@ -39,7 +39,7 @@ spiral/client.py,sha256=53dVv8wxYMmozUfR8MVcUufKGqdVIdb0yZ0gchczBoQ,6426
 spiral/core/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 spiral/core/_tools/__init__.pyi,sha256=b2KLfTOQ67pjfbYt07o0IGiTu5o2bZw69lllV8v0Dps,143
 spiral/core/authn/__init__.pyi,sha256=z_GWyIS62fuiYQrYO8hzw4W8oGaiciqS1u5qtAt54VY,769
-spiral/core/client/__init__.pyi,sha256=ajF8XaxThnTdsPpw1k3pPLEurIaDg9yeXqwIRJNlJTY,6665
+spiral/core/client/__init__.pyi,sha256=ZoeCQnEauDbaR0X7aMfHBJtLnoc7ib4JXvsmst6cAUM,6993
 spiral/core/expr/__init__.pyi,sha256=3HSKjkotiEkxBvGBALXEBIie0JiyI9bCpehwA3nMQkU,571
 spiral/core/expr/images/__init__.pyi,sha256=wnE_wZXq7a4iqTg3SVm-ssxGw1WQZyk5dGOPaP4Btko,73
 spiral/core/expr/list_/__init__.pyi,sha256=Q_9c87eIQfZbqlaw_rq3fvs93YEsW7K5VYk6VZ4g6mU,126
@@ -49,7 +49,7 @@ spiral/core/expr/struct_/__init__.pyi,sha256=MXckd98eV_x3X0RhEWvlkA3DcDXRtLs5pNn
 spiral/core/expr/text/__init__.pyi,sha256=ed83n1xcsGY7_QDhMmJGnSQ20UrJFXcdv1AveSEcS1c,175
 spiral/core/expr/udf/__init__.pyi,sha256=zsZs081KVhY3-1JidqTkWMW81Qd_ScoTGZvasIhIK-4,358
 spiral/core/expr/video/__init__.pyi,sha256=nQJEcSsigZuRpMjkI_O4EEtMK_n2zRvorcL_KEeD5vU,95
-spiral/core/table/__init__.pyi,sha256=szCtZqZ_L0vF_99x7d3olwazJdEn5LwkgRK-8QEOzrI,3914
+spiral/core/table/__init__.pyi,sha256=BkjzdcY0JL97nDRoY5s2gjfdIn1CZ7TR-uYp8f6cxEw,3940
 spiral/core/table/manifests/__init__.pyi,sha256=eVfDpmhYSjafIvvALqAkZe5baN3Y1HpKpxYEbjwd4gQ,1043
 spiral/core/table/metastore/__init__.pyi,sha256=rc3u9MwEKRvL2kxOc8lBorddFRnM8o_o1frqtae86a4,1697
 spiral/core/table/spec/__init__.pyi,sha256=fVuc2j3uoTdWfYNm720OfUIgrLYw9fRwj44maI5bgdY,5709
@@ -60,8 +60,8 @@ spiral/debug/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 spiral/debug/manifests.py,sha256=7f1O3ba9mrA5nXpOF9cEIQuUAteP5wiBkFy_diQJ7No,3216
 spiral/debug/metrics.py,sha256=XdRDcjggtsLNGCAjam6IxG9072pz_d2C8iLApNRFUtk,2044
 spiral/debug/scan.py,sha256=UEm_aRnql5pwDPTpZgakMLNjlzkKL4RurBFFqH_BLAQ,9526
-spiral/enrichment.py,sha256=iKZn4tLsRQZPtaY-WdJSqPZ3H5UMfVyavcwTKO_3aCw,6980
-spiral/expressions/__init__.py,sha256=vMNFeeozkWph3dBpEkHPThUhZdT9ZZzxHe71HnkWlDU,8020
+spiral/enrichment.py,sha256=OYOvFg98p-ti6HTw6tD6ThMGXUUKQ7JEUnqYKvMjOag,7046
+spiral/expressions/__init__.py,sha256=ZsD8g7vB0G7xy19GUiH4m79kw7KEkTQRwJl5Gn1cgtw,8049
 spiral/expressions/base.py,sha256=PvhJkcUSsPSIaxirHVzM9zlqyBXiaiia1HXohXdOmL4,5377
 spiral/expressions/file.py,sha256=7D9jIENJcoT0KFharBLkzK9dZgO4DYn5K_KCt0twefg,518
 spiral/expressions/http.py,sha256=OOHh0WBxg3vwza_m74-rkoQWSclRMI60aPAbQ6yKZi0,486
@@ -76,7 +76,7 @@ spiral/grpc_.py,sha256=f3czdP1Mxme42Y5--a5ogYq1TTiWn-J_MlGjwJ2mWwM,1015
 spiral/iceberg.py,sha256=JGq62Qnf296r9_hRAoH85GQq45-uSBjwXWw_CvPi6G4,930
 spiral/iterable_dataset.py,sha256=Eekg9ad8tcwXcloHWReBbvCSr5ZappRHn2ldKTvwqS0,4622
 spiral/key_space_index.py,sha256=NAB_nONEjpMYbse8suz42w7Qb5OPHuKN9h9CT2NJe08,1460
-spiral/project.py,sha256=VsokZgS0TqIel7UAXMyoBToxn-l_D3ivGwc41x7HLF0,7277
+spiral/project.py,sha256=dkYc5iWZzz_HMKcu1EXUNNsI7hnEyGy8VrnKdVmKjjE,8199
 spiral/protogen/_/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 spiral/protogen/_/arrow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 spiral/protogen/_/arrow/flight/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -104,6 +104,6 @@ spiral/streaming_/stream.py,sha256=DM1hBDHnWm1ZFKZ-hZ4zxeSXITcUI6kWzwdJZvywI8o,5
 spiral/substrait_.py,sha256=AKeOD4KIXvz2J4TYxnIneOiHddtBIyOhuNxVO_uH0eg,12592
 spiral/table.py,sha256=ep8ZYtl6POebkPViR2FrekhFazNmAbOAESoLUODlup8,12242
 spiral/text_index.py,sha256=FQ9rgIEGLSJryS9lFdMhKtPFey18BXoWbPXyvZPJJ04,442
-spiral/transaction.py,sha256=KQhx3DvQyxG2C8md-YGsF_PgBRfayI0r_7ebMItDHdI,3938
+spiral/transaction.py,sha256=bI5oqBAmPMSF0yOOYcPfGbV37Xc1-_V-wQNKw1xOlTA,4136
 spiral/types_.py,sha256=W_jyO7F6rpPiH69jhgSgV7OxQZbOlb1Ho3InpKUP6Eo,155
-pyspiral-0.7.4.dist-info/RECORD,,
+pyspiral-0.7.6.dist-info/RECORD,,

spiral/api/types.py

@@ -12,6 +12,7 @@ def _validate_root_uri(uri: str) -> str:
 UserId = str
 OrgId = str
 ProjectId = str
+TableId = str
 RoleId = str
 IndexId = str
 WorkerId = str

spiral/core/client/__init__.pyi

@@ -1,7 +1,7 @@
 from typing import Any, Literal
 
 import pyarrow as pa
-from spiral.api.types import DatasetName, IndexName, ProjectId, RootUri, TableName
+from spiral.api.types import DatasetName, IndexName, ProjectId, RootUri, TableId, TableName
 from spiral.core.authn import Authn
 from spiral.core.table import ColumnGroupState, KeyRange, KeySpaceState, Scan, ScanState, Snapshot, Table, Transaction
 from spiral.core.table.spec import ColumnGroup, Schema
@@ -69,6 +69,22 @@ class Spiral:
         """Create a new table in the specified project."""
         ...
 
+    def move_table(
+        self,
+        table_id: TableId,
+        new_dataset: DatasetName,
+    ):
+        """Move a table to a dataset in the same project."""
+        ...
+
+    def rename_table(
+        self,
+        table_id: TableId,
+        new_table: TableName,
+    ):
+        """Rename a table."""
+        ...
+
     def text_index(self, index_id: str) -> TextIndex:
         """Get a text index."""
         ...

spiral/core/table/__init__.pyi

@@ -115,7 +115,7 @@ class Transaction:
     def ops(self) -> list[Operation]: ...
     def take(self) -> list[Operation]: ...
     def include(self, ops: list[Operation]): ...
-    def commit(self): ...
+    def commit(self, *, compact: bool = False): ...
     def abort(self): ...
     def is_empty(self) -> bool: ...
     def metrics(self) -> dict[str, Any]: ...

spiral/enrichment.py

@@ -151,7 +151,8 @@ class Enrichment:
             logger.warning("Transaction not committed. No rows were read for enrichment.")
             return
 
-        tx.commit(tx_dump=tx_dump)
+        # Always compact in distributed enrichment.
+        tx.commit(compact=True, tx_dump=tx_dump)
 
 
 @dataclasses.dataclass

spiral/expressions/__init__.py

@@ -131,10 +131,10 @@ def evaluate(expr: ExprLike) -> pa.RecordBatchReader:
     if isinstance(expr, pa.Array):
         raise ValueError("Arrow array must be a struct array.")
 
-    if isinstance(expr, Expr):
-        raise NotImplementedError("cannot evaluate an Expr")
-    if isinstance(expr, NativeExpr):
-        raise NotImplementedError("cannot evaluate a NativeExpr")
+    if isinstance(expr, Expr) or isinstance(expr, NativeExpr):
+        raise NotImplementedError(
+            "Expr evaluation not supported yet. Use Arrow to write instead. Reach out if you require this feature."
+        )
 
     if isinstance(expr, dict):
         # NOTE: we assume this is a struct expression. We could be smarter and be context aware to determine if

spiral/project.py

@@ -89,6 +89,34 @@ class Project:
 
         return Table(self._spiral, core_table, identifier=f"{self._id}.{dataset}.{table}")
 
+    def move_table(self, identifier: str, new_dataset: str):
+        """Move a table to a new dataset in the project.
+
+        Args:
+            identifier: The table identifier, in the form `dataset.table` or `table`.
+            new_dataset: The dataset into which to move this table.
+        """
+        table = self.table(identifier)
+
+        self._spiral.core.move_table(
+            table_id=table.table_id,
+            new_dataset=new_dataset,
+        )
+
+    def rename_table(self, identifier: str, new_table: str):
+        """Move a table to a new dataset in the project.
+
+        Args:
+            identifier: The table identifier, in the form `dataset.table` or `table`.
+            new_dataset: The dataset into which to move this table.
+        """
+        table = self.table(identifier)
+
+        self._spiral.core.rename_table(
+            table_id=table.table_id,
+            new_table=new_table,
+        )
+
     def _parse_table_identifier(self, identifier: str) -> tuple[str, str]:
         parts = identifier.split(".")
         if len(parts) == 1:

spiral/transaction.py

@@ -1,4 +1,5 @@
 import logging
+from pathlib import Path
 
 from spiral.core.table import KeyRange
 from spiral.core.table import Transaction as CoreTransaction
@@ -95,18 +96,23 @@ class Transaction:
         """
         self._core.include(ops)
 
-    def commit(self, *, tx_dump: str | None = None):
+    def commit(self, *, compact: bool = False, tx_dump: str | None = None):
         """Commit the transaction."""
-        # TODO(marko): We can remove this when I have more trust in large tx commits.
         if tx_dump is not None:
             try:
-                with open(tx_dump, "w") as f:
+                # Create parent directories if they don't exist
+                dump_path = Path(tx_dump)
+                dump_path.parent.mkdir(parents=True, exist_ok=True)
+
+                # Write operations to file
+                with open(dump_path, "w") as f:
                     f.writelines([op.to_json() for op in self._core.ops()])
+
                 logger.info(f"Transaction dumped to {tx_dump}")
             except Exception as e:
                 logger.error(f"Failed to dump transaction to {tx_dump}: {e}")
 
-        self._core.commit()
+        self._core.commit(compact=compact)
 
     def abort(self):
         """Abort the transaction."""