pyspiral 0.6.1__cp310-abi3-macosx_11_0_arm64.whl → 0.6.3__cp310-abi3-macosx_11_0_arm64.whl

{pyspiral-0.6.1.dist-info → pyspiral-0.6.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyspiral
-Version: 0.6.1
+Version: 0.6.3
 Classifier: Intended Audience :: Science/Research
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python

{pyspiral-0.6.1.dist-info → pyspiral-0.6.3.dist-info}/RECORD

@@ -1,8 +1,8 @@
-pyspiral-0.6.1.dist-info/METADATA,sha256=dvU4zmSKjosVkzzsl8kYTMvaVneaplCKGJ0ml4SBRxo,1836
-pyspiral-0.6.1.dist-info/WHEEL,sha256=Eg6gwEJKNVa1g53Yg4W5oLzLA6e9MoXTlIiHdDxmtOw,103
-pyspiral-0.6.1.dist-info/entry_points.txt,sha256=uft7u-a6g40NLt4Q6BleWbK4NY0M8nZuYPpP8DV0EOk,45
+pyspiral-0.6.3.dist-info/METADATA,sha256=93TxmaIrXRNq5xlBQAx76ClIGjWdnh9bfL_f6KrB9K0,1836
+pyspiral-0.6.3.dist-info/WHEEL,sha256=Eg6gwEJKNVa1g53Yg4W5oLzLA6e9MoXTlIiHdDxmtOw,103
+pyspiral-0.6.3.dist-info/entry_points.txt,sha256=uft7u-a6g40NLt4Q6BleWbK4NY0M8nZuYPpP8DV0EOk,45
 spiral/__init__.py,sha256=5c0faqg-kHZBDwriQ7LzLAMcFolIucp-IA1EzNvCZ3k,711
-spiral/_lib.abi3.so,sha256=PBqFvphQEfKc4zSGihmAZIq_t1uO1EvVJYzZXz1P5dI,62642240
+spiral/_lib.abi3.so,sha256=mUXCHvNLIOssJZROGFsmyfHMqOeuLEmUXjgV1_8HORk,63655744
 spiral/adbc.py,sha256=7IxfWIeQN-fh0W5OdN_PP2x3pzQYg6ZUOLsHg3jktqw,14842
 spiral/api/__init__.py,sha256=ULBlVq3PnfNOO6T5naE_ULmmii-83--qTuN2PpAUQN0,2241
 spiral/api/admin.py,sha256=A1iVR1XYJSObZivPAD5UzmPuMgupXc9kaHNYYa_kwfs,585
@@ -39,8 +39,8 @@ spiral/client.py,sha256=Po9xgCH3NwVsCeRZMm3eJUPV77Rknyj-9MfCS1TbdTg,6623
 spiral/core/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 spiral/core/authn/__init__.pyi,sha256=Jw_8ywTMDTwgAtGxMtFED63rU0jOgrv-eZtaZ5sR5t4,757
 spiral/core/client/__init__.pyi,sha256=iEhZgbySG5LScfrtkiiHW1iHghgehsrVmPP-v5Pv_vk,5740
-spiral/core/table/__init__.pyi,sha256=_N9JyGoJXcbJWXjZal0HSSxNbbMptrqKZJHj5GA_mQg,2974
-spiral/core/table/manifests/__init__.pyi,sha256=3V59-K1qr1z2dGfgRKXaHSVheK8NNw8Q8PFhfbeQd_4,1065
+spiral/core/table/__init__.pyi,sha256=sjjShdgM_Uh8Roou1k02MnrqYpdAX4QuyRlIRlnyp1M,3073
+spiral/core/table/manifests/__init__.pyi,sha256=eVfDpmhYSjafIvvALqAkZe5baN3Y1HpKpxYEbjwd4gQ,1043
 spiral/core/table/metastore/__init__.pyi,sha256=rc3u9MwEKRvL2kxOc8lBorddFRnM8o_o1frqtae86a4,1697
 spiral/core/table/spec/__init__.pyi,sha256=0NyGeyEhV_ebwKWVU3sqSvdF2D9v8kEVwo6wYAHF99M,5579
 spiral/dataset.py,sha256=NNqG-oOrhbmNC2OMZ9AYAm4YkwwBozeRI6zXtz4cspA,8008
@@ -48,7 +48,7 @@ spiral/datetime_.py,sha256=1TA1RYIRU22qcUuipIjVhAtGnPDVn2z9WttuhkmfkwY,964
 spiral/debug/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 spiral/debug/manifests.py,sha256=oaPB4534pQdqvPXCZetVNSvvhpdXTrv_1pN-_bAkeAo,2893
 spiral/debug/metrics.py,sha256=XdRDcjggtsLNGCAjam6IxG9072pz_d2C8iLApNRFUtk,2044
-spiral/debug/scan.py,sha256=9bMmVQFs5M6Rldm0fmrmmvn9LbSSTKBV5tIu37mEn78,8938
+spiral/debug/scan.py,sha256=UEm_aRnql5pwDPTpZgakMLNjlzkKL4RurBFFqH_BLAQ,9526
 spiral/expressions/__init__.py,sha256=T8PIb0_UB9kynK0dpWbUD4No5lKRTG-wKnao8xOcXjY,6381
 spiral/expressions/base.py,sha256=OOUDrbkLBE0lSkAmM-6FP2F2N8zhN_in3S_UDrWLDeQ,4805
 spiral/expressions/http.py,sha256=begUydWoFHEqjeLkATvI_v66Ez6_rR-OQBWO5cHbb9c,2742
@@ -90,10 +90,10 @@ spiral/settings.py,sha256=Nap68xM-1ZvF3yDhkyRnNDIAVMIgxmIksglg_1iT0-0,3069
 spiral/snapshot.py,sha256=_l2wrqUXz2RARjIDxOWw4aQpegJohvggIoWuCllzStA,1506
 spiral/streaming_/__init__.py,sha256=s7MlW2ERsuZmZGExLFL6RcZon2e0tNBocBg5ANgki7k,61
 spiral/streaming_/reader.py,sha256=Kpqknv2jn12jUhHOEEDArj0JZwrWb8XjoOGs9HrdVyA,4047
-spiral/streaming_/stream.py,sha256=xFTtGB6CspEKstzBeyyaOeOR3KDiJc21m07ZpD1AXZQ,5669
+spiral/streaming_/stream.py,sha256=nxJEisPfZ2-Ebkm83hz_3v8NH27FxBku-1jw7UDlQuM,5881
 spiral/substrait_.py,sha256=AKeOD4KIXvz2J4TYxnIneOiHddtBIyOhuNxVO_uH0eg,12592
-spiral/table.py,sha256=knJjSCPrFLXmjCbUzslGZ7avH0V_Z0o-rsctWDzmGK4,9821
+spiral/table.py,sha256=ZQFq5tuovDjQcpi38b5FUMuHNGI5XV0MnZbC6vbza1o,10312
 spiral/text_index.py,sha256=FQ9rgIEGLSJryS9lFdMhKtPFey18BXoWbPXyvZPJJ04,442
-spiral/transaction.py,sha256=O3vSaTc7zpeC5qbqnj-VWKwK6rrp_mYV2JuPHp2ZJ80,1464
+spiral/transaction.py,sha256=nSykH4UGs9hGtWuSWK9YyT9jfEuvzfkKoUgMM5Xt4zU,1841
 spiral/types_.py,sha256=W_jyO7F6rpPiH69jhgSgV7OxQZbOlb1Ho3InpKUP6Eo,155
-pyspiral-0.6.1.dist-info/RECORD,,
+pyspiral-0.6.3.dist-info/RECORD,,

spiral/core/table/__init__.pyi

@@ -77,6 +77,7 @@ class Scan:
         output_path: str,
         key_range: KeyRange,
         expected_cardinality: int | None = None,
+        shard_row_block_size: int = 8192,
     ) -> None: ...
 
 class KeySpaceState:
@@ -94,6 +95,7 @@ class Transaction:
     status: str
 
     def write(self, expr: Expr, *, partition_size_bytes: int | None = None): ...
+    def drop_columns(self, column_paths: list[str]): ...
     def commit(self): ...
     def abort(self): ...
     def metrics(self) -> dict[str, Any]: ...

spiral/core/table/manifests/__init__.pyi

@@ -8,7 +8,7 @@ class FragmentManifest:
     def __getitem__(self, idx: int): ...
     def to_arrow(self) -> pa.RecordBatchReader: ...
     @staticmethod
-    def compute_schema(data_schema: pa.Schema) -> pa.Schema: ...
+    def compute_schema() -> pa.Schema: ...
     @staticmethod
     def from_fragment(fragment_file: FragmentFile) -> FragmentManifest: ...
     @staticmethod

spiral/debug/scan.py

@@ -146,14 +146,32 @@ def _get_fragment_color(manifest_file: FragmentFile, color_index, total_colors):
         return cm.viridis(color_index / total_colors)
 
 
+def _get_human_size(size_bytes: int) -> str:
+    # Convert bytes to a human-readable format
+    for unit in ["B", "KB", "MB", "GB", "TB"]:
+        if size_bytes < 1024:
+            return f"{size_bytes:.2f} {unit}"
+        size_bytes /= 1024
+    return f"{size_bytes:.2f} PB"
+
+
+def _maybe_truncate(text, max_length: int = 30) -> str:
+    text = str(text)
+    if len(text) <= max_length:
+        return text
+
+    half_length = (max_length - 3) // 2
+    return text[:half_length] + "..." + text[-half_length:]
+
+
 def _get_fragment_legend(manifest_file: FragmentFile):
     return "\n".join(
         [
             f"id: {manifest_file.id}",
-            f"size: {manifest_file.size_bytes:,} bytes",
+            f"size: {_get_human_size(manifest_file.size_bytes)} ({manifest_file.size_bytes} bytes)",
             f"key_span: {manifest_file.key_span}",
-            f"key_min: {manifest_file.key_extent.min}",
-            f"key_max: {manifest_file.key_extent.max}",
+            f"key_min: {_maybe_truncate(manifest_file.key_extent.min)}",
+            f"key_max: {_maybe_truncate(manifest_file.key_extent.max)}",
             f"format: {manifest_file.format}",
             f"level: {manifest_file.level}",
             f"committed_at: {_format_timestamp(manifest_file.committed_at)}",

spiral/streaming_/stream.py

@@ -24,10 +24,13 @@ class SpiralStream:
     Stream can be passed to MDS's StreamingDataset in `streams` argument.
     """
 
-    def __init__(self, scan: CoreScan, shards: list[Shard], cache_dir: str | None = None):
+    def __init__(
+        self, scan: CoreScan, shards: list[Shard], cache_dir: str | None = None, shard_row_block_size: int = 8192
+    ):
         self._scan = scan
         # TODO(marko): Read shards only on world.is_local_leader in `get_shards` and materialize on disk.
         self._shards = shards
+        self.shard_row_block_size = shard_row_block_size
 
         if cache_dir is not None:
             if not os.path.exists(cache_dir):
@@ -92,7 +95,12 @@ class SpiralStream:
             return 0
 
         # This method exists but it's hidden.
-        self._scan._prepare_shard(shard_path, shard.shard.key_range, expected_cardinality=shard.shard.cardinality)
+        self._scan._prepare_shard(
+            shard_path,
+            shard.shard.key_range,
+            expected_cardinality=shard.shard.cardinality,
+            shard_row_block_size=self.shard_row_block_size,
+        )
 
         # Get the size of the file on disk.
         stat = os.stat(shard_path)

spiral/table.py

@@ -109,6 +109,17 @@ class Table(Expr):
                 partition_size_bytes=partition_size_bytes,
             )
 
+    def drop_columns(self, column_paths: list[str]) -> None:
+        """
+        Drops the specified columns from the table.
+
+
+        :param column_paths: Fully qualified column names. (e.g., "column_name" or "nested.field").
+            All columns must exist, if a a column doesn't exist the function will return an error.
+        """
+        with self.txn() as txn:
+            txn.drop_columns(column_paths)
+
     def snapshot(self, asof: datetime | int | None = None) -> Snapshot:
         """Returns a snapshot of the table at the given timestamp."""
         if isinstance(asof, datetime):
@@ -185,6 +196,7 @@ class Table(Expr):
         batch_size: int | None = None,
         cache_dir: str | None = None,
         cache_limit: int | str | None = None,
+        predownload: int | None = None,
         sampling_method: str = "balanced",
         sampling_granularity: int = 1,
         partition_algo: str = "relaxed",
@@ -215,6 +227,7 @@ class Table(Expr):
             streams=[stream],
             batch_size=batch_size,
             cache_limit=cache_limit,
+            predownload=predownload,
             sampling_method=sampling_method,
             sampling_granularity=sampling_granularity,
             partition_algo=partition_algo,

spiral/transaction.py

@@ -39,6 +39,16 @@ class Transaction:
 
         self._core.write(expr.__expr__, partition_size_bytes=partition_size_bytes)
 
+    def drop_columns(self, column_paths: list[str]):
+        """
+        Drops the specified columns from the table.
+
+
+        :param column_paths: Fully qualified column names. (e.g., "column_name" or "nested.field").
+            All columns must exist, if a a column doesn't exist the function will return an error.
+        """
+        self._core.drop_columns(column_paths)
+
     def commit(self):
         """Commit the transaction."""
         self._core.commit()