pyspiral 0.6.1__cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl → 0.6.2__cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl

{pyspiral-0.6.1.dist-info → pyspiral-0.6.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyspiral
-Version: 0.6.1
+Version: 0.6.2
 Classifier: Intended Audience :: Science/Research
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python

{pyspiral-0.6.1.dist-info → pyspiral-0.6.2.dist-info}/RECORD

@@ -1,8 +1,8 @@
-pyspiral-0.6.1.dist-info/METADATA,sha256=dvU4zmSKjosVkzzsl8kYTMvaVneaplCKGJ0ml4SBRxo,1836
-pyspiral-0.6.1.dist-info/WHEEL,sha256=PxcKzGLVtZeSnGJDErQ-Emkn2AvBXbmzIogfnaf7-q8,130
-pyspiral-0.6.1.dist-info/entry_points.txt,sha256=uft7u-a6g40NLt4Q6BleWbK4NY0M8nZuYPpP8DV0EOk,45
+pyspiral-0.6.2.dist-info/METADATA,sha256=n0a5SuYMybj-eUwB9No9IFxSk9Cn_pDZfjpx0HuDxRw,1836
+pyspiral-0.6.2.dist-info/WHEEL,sha256=PxcKzGLVtZeSnGJDErQ-Emkn2AvBXbmzIogfnaf7-q8,130
+pyspiral-0.6.2.dist-info/entry_points.txt,sha256=uft7u-a6g40NLt4Q6BleWbK4NY0M8nZuYPpP8DV0EOk,45
 spiral/__init__.py,sha256=5c0faqg-kHZBDwriQ7LzLAMcFolIucp-IA1EzNvCZ3k,711
-spiral/_lib.abi3.so,sha256=2-DAyExXsya0cbr_NZlY2HaguPy17G7wFTfp3BS8YFE,54826728
+spiral/_lib.abi3.so,sha256=oOsM8f904rj2leVKR8cCOkLQ7uxcf36tKVIN0bCVOok,55184120
 spiral/adbc.py,sha256=7IxfWIeQN-fh0W5OdN_PP2x3pzQYg6ZUOLsHg3jktqw,14842
 spiral/api/__init__.py,sha256=ULBlVq3PnfNOO6T5naE_ULmmii-83--qTuN2PpAUQN0,2241
 spiral/api/admin.py,sha256=A1iVR1XYJSObZivPAD5UzmPuMgupXc9kaHNYYa_kwfs,585
@@ -39,8 +39,8 @@ spiral/client.py,sha256=Po9xgCH3NwVsCeRZMm3eJUPV77Rknyj-9MfCS1TbdTg,6623
 spiral/core/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 spiral/core/authn/__init__.pyi,sha256=Jw_8ywTMDTwgAtGxMtFED63rU0jOgrv-eZtaZ5sR5t4,757
 spiral/core/client/__init__.pyi,sha256=iEhZgbySG5LScfrtkiiHW1iHghgehsrVmPP-v5Pv_vk,5740
-spiral/core/table/__init__.pyi,sha256=_N9JyGoJXcbJWXjZal0HSSxNbbMptrqKZJHj5GA_mQg,2974
-spiral/core/table/manifests/__init__.pyi,sha256=3V59-K1qr1z2dGfgRKXaHSVheK8NNw8Q8PFhfbeQd_4,1065
+spiral/core/table/__init__.pyi,sha256=sjjShdgM_Uh8Roou1k02MnrqYpdAX4QuyRlIRlnyp1M,3073
+spiral/core/table/manifests/__init__.pyi,sha256=eVfDpmhYSjafIvvALqAkZe5baN3Y1HpKpxYEbjwd4gQ,1043
 spiral/core/table/metastore/__init__.pyi,sha256=rc3u9MwEKRvL2kxOc8lBorddFRnM8o_o1frqtae86a4,1697
 spiral/core/table/spec/__init__.pyi,sha256=0NyGeyEhV_ebwKWVU3sqSvdF2D9v8kEVwo6wYAHF99M,5579
 spiral/dataset.py,sha256=NNqG-oOrhbmNC2OMZ9AYAm4YkwwBozeRI6zXtz4cspA,8008
@@ -90,10 +90,10 @@ spiral/settings.py,sha256=Nap68xM-1ZvF3yDhkyRnNDIAVMIgxmIksglg_1iT0-0,3069
 spiral/snapshot.py,sha256=_l2wrqUXz2RARjIDxOWw4aQpegJohvggIoWuCllzStA,1506
 spiral/streaming_/__init__.py,sha256=s7MlW2ERsuZmZGExLFL6RcZon2e0tNBocBg5ANgki7k,61
 spiral/streaming_/reader.py,sha256=Kpqknv2jn12jUhHOEEDArj0JZwrWb8XjoOGs9HrdVyA,4047
-spiral/streaming_/stream.py,sha256=xFTtGB6CspEKstzBeyyaOeOR3KDiJc21m07ZpD1AXZQ,5669
+spiral/streaming_/stream.py,sha256=nxJEisPfZ2-Ebkm83hz_3v8NH27FxBku-1jw7UDlQuM,5881
 spiral/substrait_.py,sha256=AKeOD4KIXvz2J4TYxnIneOiHddtBIyOhuNxVO_uH0eg,12592
-spiral/table.py,sha256=knJjSCPrFLXmjCbUzslGZ7avH0V_Z0o-rsctWDzmGK4,9821
+spiral/table.py,sha256=ZQFq5tuovDjQcpi38b5FUMuHNGI5XV0MnZbC6vbza1o,10312
 spiral/text_index.py,sha256=FQ9rgIEGLSJryS9lFdMhKtPFey18BXoWbPXyvZPJJ04,442
-spiral/transaction.py,sha256=O3vSaTc7zpeC5qbqnj-VWKwK6rrp_mYV2JuPHp2ZJ80,1464
+spiral/transaction.py,sha256=nSykH4UGs9hGtWuSWK9YyT9jfEuvzfkKoUgMM5Xt4zU,1841
 spiral/types_.py,sha256=W_jyO7F6rpPiH69jhgSgV7OxQZbOlb1Ho3InpKUP6Eo,155
-pyspiral-0.6.1.dist-info/RECORD,,
+pyspiral-0.6.2.dist-info/RECORD,,

spiral/core/table/__init__.pyi

@@ -77,6 +77,7 @@ class Scan:
         output_path: str,
         key_range: KeyRange,
         expected_cardinality: int | None = None,
+        shard_row_block_size: int = 8192,
     ) -> None: ...
 
 class KeySpaceState:
@@ -94,6 +95,7 @@ class Transaction:
     status: str
 
     def write(self, expr: Expr, *, partition_size_bytes: int | None = None): ...
+    def drop_columns(self, column_paths: list[str]): ...
     def commit(self): ...
     def abort(self): ...
     def metrics(self) -> dict[str, Any]: ...

spiral/core/table/manifests/__init__.pyi

@@ -8,7 +8,7 @@ class FragmentManifest:
     def __getitem__(self, idx: int): ...
     def to_arrow(self) -> pa.RecordBatchReader: ...
     @staticmethod
-    def compute_schema(data_schema: pa.Schema) -> pa.Schema: ...
+    def compute_schema() -> pa.Schema: ...
     @staticmethod
     def from_fragment(fragment_file: FragmentFile) -> FragmentManifest: ...
     @staticmethod

spiral/streaming_/stream.py

@@ -24,10 +24,13 @@ class SpiralStream:
     Stream can be passed to MDS's StreamingDataset in `streams` argument.
     """
 
-    def __init__(self, scan: CoreScan, shards: list[Shard], cache_dir: str | None = None):
+    def __init__(
+        self, scan: CoreScan, shards: list[Shard], cache_dir: str | None = None, shard_row_block_size: int = 8192
+    ):
         self._scan = scan
         # TODO(marko): Read shards only on world.is_local_leader in `get_shards` and materialize on disk.
         self._shards = shards
+        self.shard_row_block_size = shard_row_block_size
 
         if cache_dir is not None:
             if not os.path.exists(cache_dir):
@@ -92,7 +95,12 @@ class SpiralStream:
             return 0
 
         # This method exists but it's hidden.
-        self._scan._prepare_shard(shard_path, shard.shard.key_range, expected_cardinality=shard.shard.cardinality)
+        self._scan._prepare_shard(
+            shard_path,
+            shard.shard.key_range,
+            expected_cardinality=shard.shard.cardinality,
+            shard_row_block_size=self.shard_row_block_size,
+        )
 
         # Get the size of the file on disk.
         stat = os.stat(shard_path)

spiral/table.py

@@ -109,6 +109,17 @@ class Table(Expr):
                 partition_size_bytes=partition_size_bytes,
             )
 
+    def drop_columns(self, column_paths: list[str]) -> None:
+        """
+        Drops the specified columns from the table.
+
+
+        :param column_paths: Fully qualified column names. (e.g., "column_name" or "nested.field").
+            All columns must exist, if a a column doesn't exist the function will return an error.
+        """
+        with self.txn() as txn:
+            txn.drop_columns(column_paths)
+
     def snapshot(self, asof: datetime | int | None = None) -> Snapshot:
         """Returns a snapshot of the table at the given timestamp."""
         if isinstance(asof, datetime):
@@ -185,6 +196,7 @@ class Table(Expr):
         batch_size: int | None = None,
         cache_dir: str | None = None,
         cache_limit: int | str | None = None,
+        predownload: int | None = None,
         sampling_method: str = "balanced",
         sampling_granularity: int = 1,
         partition_algo: str = "relaxed",
@@ -215,6 +227,7 @@ class Table(Expr):
             streams=[stream],
             batch_size=batch_size,
             cache_limit=cache_limit,
+            predownload=predownload,
             sampling_method=sampling_method,
             sampling_granularity=sampling_granularity,
             partition_algo=partition_algo,

spiral/transaction.py

@@ -39,6 +39,16 @@ class Transaction:
 
         self._core.write(expr.__expr__, partition_size_bytes=partition_size_bytes)
 
+    def drop_columns(self, column_paths: list[str]):
+        """
+        Drops the specified columns from the table.
+
+
+        :param column_paths: Fully qualified column names. (e.g., "column_name" or "nested.field").
+            All columns must exist, if a a column doesn't exist the function will return an error.
+        """
+        self._core.drop_columns(column_paths)
+
     def commit(self):
         """Commit the transaction."""
         self._core.commit()