pyspiral 0.6.14__cp312-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl → 0.6.15__cp312-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl

{pyspiral-0.6.14.dist-info → pyspiral-0.6.15.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyspiral
-Version: 0.6.14
+Version: 0.6.15
 Classifier: Intended Audience :: Science/Research
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python

{pyspiral-0.6.14.dist-info → pyspiral-0.6.15.dist-info}/RECORD

@@ -1,8 +1,8 @@
-pyspiral-0.6.14.dist-info/METADATA,sha256=WGdXph89n9RYh7SmWR_6GH9vAn-5suV4sAQAf1E8QCM,1875
-pyspiral-0.6.14.dist-info/WHEEL,sha256=0ecHyBdkJfSXYIVmWsPh7S-4h4fSrB4FlXhlnIu9c_A,130
-pyspiral-0.6.14.dist-info/entry_points.txt,sha256=uft7u-a6g40NLt4Q6BleWbK4NY0M8nZuYPpP8DV0EOk,45
+pyspiral-0.6.15.dist-info/METADATA,sha256=1wRDpjCt2Im2qJm0yh94g-_VYl-19528b6-Vc_ad0D4,1875
+pyspiral-0.6.15.dist-info/WHEEL,sha256=0ecHyBdkJfSXYIVmWsPh7S-4h4fSrB4FlXhlnIu9c_A,130
+pyspiral-0.6.15.dist-info/entry_points.txt,sha256=uft7u-a6g40NLt4Q6BleWbK4NY0M8nZuYPpP8DV0EOk,45
 spiral/__init__.py,sha256=gAysTwG_oEeKVMdCOfOzDhl0bM2miiK8Ds2vvUihBWw,1153
-spiral/_lib.abi3.so,sha256=SAIgrAqdywVARZifctTLKRxd3a9Iq_srL7UszH1rEsY,61219912
+spiral/_lib.abi3.so,sha256=5vNwYz5Yu337Ivah4ia1SwWQEgyfI0Ylxo6WthXIal8,61245800
 spiral/adbc.py,sha256=7IxfWIeQN-fh0W5OdN_PP2x3pzQYg6ZUOLsHg3jktqw,14842
 spiral/api/__init__.py,sha256=ULBlVq3PnfNOO6T5naE_ULmmii-83--qTuN2PpAUQN0,2241
 spiral/api/admin.py,sha256=A1iVR1XYJSObZivPAD5UzmPuMgupXc9kaHNYYa_kwfs,585
@@ -60,12 +60,13 @@ spiral/debug/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 spiral/debug/manifests.py,sha256=7f1O3ba9mrA5nXpOF9cEIQuUAteP5wiBkFy_diQJ7No,3216
 spiral/debug/metrics.py,sha256=XdRDcjggtsLNGCAjam6IxG9072pz_d2C8iLApNRFUtk,2044
 spiral/debug/scan.py,sha256=UEm_aRnql5pwDPTpZgakMLNjlzkKL4RurBFFqH_BLAQ,9526
-spiral/enrichment.py,sha256=e2yzNWTTG73uEkLTc4ccTNRQ94cBtM04eGzlJ2-kBOI,5851
-spiral/expressions/__init__.py,sha256=Fp7Xx3exh9KJad92tgd_TGGIpYLQTHqWjW-pexzQibU,7981
+spiral/enrichment.py,sha256=YDaXcJPtmJzpLrYmn2pdllVcRIkXlb578KKgkIb38Eo,6518
+spiral/expressions/__init__.py,sha256=vMNFeeozkWph3dBpEkHPThUhZdT9ZZzxHe71HnkWlDU,8020
 spiral/expressions/base.py,sha256=PvhJkcUSsPSIaxirHVzM9zlqyBXiaiia1HXohXdOmL4,5377
+spiral/expressions/file.py,sha256=HRzGjc3goIlUlKjysoirexDaflNdnj9OoZ6j2uTKZnA,388
 spiral/expressions/http.py,sha256=WfHVLqz_LjBr78mN3ARBRQqgBrkao7-S73JxjC4Xwvo,356
 spiral/expressions/list_.py,sha256=MMt5lf5H1M3O-x6N_PvqOLGq9NOk6Ukv0fPWwPC_uy4,1809
-spiral/expressions/s3.py,sha256=bkd0HANerNKlOblp2z7JJOSWjF9Bw9lZe1A-KTrUEgk,378
+spiral/expressions/s3.py,sha256=d6Z_nnBRXopoPWnwGxlgjwyeYlrIMmonX-83PkE8LPo,375
 spiral/expressions/str_.py,sha256=tY8RXW3JWvr1-bEfCZtk5FAf11wKJnXPuA9EoeJ9tA4,1265
 spiral/expressions/struct.py,sha256=pGAnCDh6AK0BK1XfZ1qG4ce4ranIQEE1HQsgmzBcfwQ,2038
 spiral/expressions/text.py,sha256=-02gBWYoyNQ3qQ1--9HTa8IryUDojYQVIp8C7rgnOWQ,1893
@@ -105,4 +106,4 @@ spiral/table.py,sha256=prjDBcm6Qerdq3ypXzfbXb7ngAcO0j-Z9aTeZvzKoqs,12209
 spiral/text_index.py,sha256=FQ9rgIEGLSJryS9lFdMhKtPFey18BXoWbPXyvZPJJ04,442
 spiral/transaction.py,sha256=hQm6DfCklMDpIYJ9qA2wR45cCuUPGCiJy1tHGE3AsEY,3418
 spiral/types_.py,sha256=W_jyO7F6rpPiH69jhgSgV7OxQZbOlb1Ho3InpKUP6Eo,155
-pyspiral-0.6.14.dist-info/RECORD,,
+pyspiral-0.6.15.dist-info/RECORD,,

spiral/enrichment.py

@@ -64,7 +64,12 @@ class Enrichment:
     # TODO(marko): Need to figure out this sharding with key space index in places.
     #   We could compute on-demand instead of requiring a resource.
     def apply_dask(
-        self, *, index: Optional["KeySpaceIndex"] = None, partition_size_bytes: int | None = None, **kwargs
+        self,
+        *,
+        index: Optional["KeySpaceIndex"] = None,
+        partition_size_bytes: int | None = None,
+        tx_dump: str | None = None,
+        **kwargs,
     ) -> None:
         """Use distributed Dask to apply the enrichment. Requires `dask[distributed]` to be installed.
 
@@ -79,6 +84,9 @@ class Enrichment:
         Args:
             index: Optional key space index to use for sharding the enrichment.
                 If not provided, the table's default sharding will be used.
+            partition_size_bytes: The maximum partition size in bytes.
+                If not provided, the default partition size is used.
+            tx_dump: Optional path to dump the transaction operations as a JSON file for debugging.
             **kwargs: Additional keyword arguments to pass to `dask.distributed.Client`
                 such as `address` to connect to an existing cluster.
         """
@@ -92,6 +100,7 @@ class Enrichment:
 
         # Start a transaction BEFORE the planning scan.
         tx = self._table.txn()
+        backup_ops = []
         plan_scan = self._table.spiral.scan(self._projection, where=self._where)
 
         # Determine the "tasks". Use the index if provided.
@@ -114,11 +123,18 @@ class Enrichment:
         for result in dask_client.gather(enrichments):
             result: EnrichmentTaskResult
             tx.include(result.ops)
+            backup_ops.extend(result.ops)
 
         if tx.is_empty():
             logger.warning("Transaction not committed. No rows were read for enrichment.")
             return
 
+        # TODO(marko): We can remove this when I have more trust in very large tx commits.
+        if tx_dump is not None:
+            with open(tx_dump, "w") as f:
+                f.writelines([op.to_json() for op in backup_ops])
+            logger.info(f"Transaction dumped to {tx_dump}")
+
         tx.commit()
 
 

spiral/expressions/__init__.py

@@ -8,6 +8,7 @@ import pyarrow as pa
 
 from spiral import _lib, arrow_
 
+from . import file as file
 from . import http as http
 from . import list_ as list
 from . import s3 as s3
@@ -50,6 +51,7 @@ __all__ = [
     "text",
     "s3",
     "http",
+    "file",
     "UDF",
 ]
 

spiral/expressions/file.py

@@ -0,0 +1,16 @@
+from spiral import _lib
+from spiral.expressions.base import Expr, ExprLike
+
+
+def get(expr: ExprLike) -> Expr:
+    """Read data from the local filesystem by the file:// URL.
+
+    Args:
+        expr: URLs of the data that needs to be read.
+    """
+    from spiral import expressions as se
+
+    expr = se.lift(expr)
+
+    # This just works :)
+    return Expr(_lib.expr.s3.get(expr.__expr__))

spiral/expressions/s3.py

@@ -3,7 +3,7 @@ from spiral.expressions.base import Expr, ExprLike
 
 
 def get(expr: ExprLike) -> Expr:
-    """Read data from object storage by the object's URL.
+    """Read data from object storage by the s3:// URL.
 
     Args:
         expr: URLs of the data that needs to be read from object storage.