shackleton 0.0.1__tar.gz → 0.0.2__tar.gz

{shackleton-0.0.1 → shackleton-0.0.2}/PKG-INFO

@@ -1,10 +1,11 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: shackleton
-Version: 0.0.1
+Version: 0.0.2
 Summary: Polars based database
 Author-email: Endre Márk Borza <endremborza@gmail.com>
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
+License-File: LICENSE
 Requires-Dist: polars
 Requires-Dist: branthebuilder[doc] ; extra == "doc"
 Requires-Dist: branthebuilder ; extra == "test"

{shackleton-0.0.1 → shackleton-0.0.2}/shackleton/__init__.py

@@ -1,4 +1,5 @@
 """Polars based database"""
+
 from .core import TableShack  # noqa: F401
 
-__version__ = "0.0.1"
+__version__ = "0.0.2"

{shackleton-0.0.1 → shackleton-0.0.2}/shackleton/core.py

@@ -27,7 +27,10 @@ class TableShack:
         return self._write_meta(df, self._replace_records)
 
     def lazy_read(self, path: Path) -> pl.LazyFrame:
-        o = (pl.scan_ipc if self.ipc else pl.scan_parquet)(path)
+        if self.ipc:
+            o = pl.scan_ipc(path)
+        else:
+            o = pl.scan_parquet(path, hive_partitioning=False)
         if self.id_col:
             return o.set_sorted(self.id_col)
         return o
@@ -77,6 +80,7 @@ class TableShack:
             return pl.concat([old_df, df])
 
     def _replace_records(self, df: pl.LazyFrame, old_df: pl.LazyFrame) -> pl.LazyFrame:
+        assert self.id_col is not None, "can only replace id'd records"
         return old_df.merge_sorted(df, key=self.id_col).unique(
             subset=self.id_col, keep="last"
         )
@@ -85,7 +89,7 @@ class TableShack:
         self,
         df: pl.DataFrame,
         fun: Callable[[pl.LazyFrame, pl.LazyFrame], pl.LazyFrame],
-        subdirs: tuple[str] = (),
+        subdirs: tuple[str, ...] = (),
         partitioned=False,
     ):
         if self.partition_cols and not partitioned:
@@ -103,7 +107,7 @@ class TableShack:
         self._add_compression(fun)(out, true_path)
 
     def _gb_handle(self, df: pl.DataFrame, fun):
-        for gid, gdf in df.groupby(self.partition_cols):
+        for gid, gdf in df.group_by(self.partition_cols):
             hive_names = [f"{k}={v}" for k, v in zip(self.partition_cols, gid)]
             self._write_meta(gdf.drop(self.partition_cols), fun, hive_names, True)
 

{shackleton-0.0.1 → shackleton-0.0.2}/shackleton/tests/test_core.py

@@ -37,4 +37,20 @@ def test_import(tmp_path, id_col, comp):
         shack.replace_records(df)
         assert shack.get_full_df().shape[0] == df.shape[0]
 
-    assert next(shack.dfs).shape[0]
+    some_dfs = False
+    for _df in shack.dfs:
+        assert _df.shape[0] > 0
+        some_dfs = True
+    assert some_dfs
+
+    n2 = 5678
+    df2 = pl.DataFrame(
+        {
+            "A": [rng.random() for _ in range(n2)],
+            "B": [rng.randint(0, 19) for _ in range(n2)],
+            "C": [rng.choice(byte_set) for _ in range(n2)],
+        }
+    )
+
+    shack.extend(df2)
+    assert shack.get_full_df().shape[0] == (n + n2)