shackleton 1.2.0__tar.gz → 2.0.1__tar.gz

shackleton-2.0.1/.github/workflows/release.yml

@@ -0,0 +1,27 @@
+name: Release
+
+on:
+  push:
+    tags:
+      - 'v*'
+
+jobs:
+  gate:
+    # compatibility_test.yml here also requires the OS matrix to pass
+    uses: ./.github/workflows/test.yml
+  release:
+    needs: gate
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+    - uses: actions/checkout@v7
+      with:
+        fetch-depth: 0
+    - uses: astral-sh/setup-uv@v10.3.0
+    - run: uv build && uv publish
+      env:
+        UV_PUBLISH_TOKEN: ${{ secrets.PYPI_TOKEN }}
+    - run: gh release create "$GITHUB_REF_NAME" dist/* --verify-tag --notes "$(git tag -l --format='%(contents)' "$GITHUB_REF_NAME")"
+      env:
+        GH_TOKEN: ${{ github.token }}

{shackleton-1.2.0 → shackleton-2.0.1}/.github/workflows/test.yml

@@ -7,13 +7,14 @@ on:
   push:
     branches:
       - main
+  workflow_call:
 
 jobs:
   test:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4
-    - uses: astral-sh/setup-uv@v5
+    - uses: actions/checkout@v7
+    - uses: astral-sh/setup-uv@v10.3.0
       with:
         python-version: '3.12'
         enable-cache: true

{shackleton-1.2.0 → shackleton-2.0.1}/.gitignore

@@ -68,19 +68,12 @@ instance/
 # Scrapy stuff:
 .scrapy
 
-# Sphinx documentation
-docs/_build/
+# Generated coverage HTML (report.md is committed, HTML is not)
+coverage/html/
 
 # PyBuilder
 target/
 
-# Jupyter Notebook
-.ipynb_checkpoints
-
-# IPython
-profile_default/
-ipython_config.py
-
 # pyenv
 .python-version
 
@@ -113,9 +106,6 @@ venv.bak/
 # Rope project settings
 .ropeproject
 
-# mkdocs documentation
-/site
-
 # mypy
 .mypy_cache/
 .dmypy.json

{shackleton-1.2.0 → shackleton-2.0.1}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: shackleton
-Version: 1.2.0
+Version: 2.0.1
 Summary: Polars-based persistent tabular store backed by Parquet or Arrow IPC files
 Project-URL: Homepage, https://github.com/endremborza/shackleton
 Author-email: Endre Márk Borza <endremborza@gmail.com>
@@ -13,7 +13,7 @@ License: Copyright 2023 Endre Márk Borza
         THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 License-File: LICENSE
 Requires-Python: >=3.12
-Requires-Dist: polars
+Requires-Dist: polars>=2
 Description-Content-Type: text/markdown
 
 # shackleton
@@ -22,7 +22,7 @@ Description-Content-Type: text/markdown
 
 Persistent tabular store backed by Parquet or Arrow IPC, built on [polars](https://pola.rs).
 
-- **Partitioned writes** — by column value or xxhash bucket
+- **Partitioned writes** — by column value or hash bucket
 - **Sorted merge** — maintain sorted order on extend via `id_col`
 - **Lazy reads** — returns `pl.LazyFrame` so polars can push down predicates
 - **Schema evolution** — diagonal concat handles mismatched columns transparently
@@ -91,7 +91,7 @@ df_eu = repo.get_partition_df({"region": "eu"})
 ## Hash partitioning
 
 When partition values have unbounded cardinality, hash into fixed buckets instead.
-Uses polars' native xxhash — vectorised, GIL-free.
+Uses polars' `Series.hash` — vectorised, GIL-free. That hash is not stable across polars versions while bucket assignment lives on disk, so before its first bucket-addressed operation (a write, `get_partition_*`, `purge_partition`) a repo samples stored rows and raises `HashDriftError` if they no longer hash to their bucket. `repo.rehash()` moves every row into its current bucket.
 
 ```python
 from shackleton import HashPartitioner
@@ -113,15 +113,15 @@ df = repo.get_full_df()
 ## Deduplication with `dedup_cols`
 
 When `dedup_cols` is set, each `extend` drops duplicate rows on those columns
-after merging with existing data — only the first occurrence is kept. Combine
-with `id_col` for sorted-merge dedup; use alone for unordered dedup.
+after merging with existing data. With `id_col` the merge is sorted and the
+existing (first) row wins; without `id_col` the surviving occurrence is arbitrary.
 Not supported with `max_records > 0` (raises at construction time).
 
 ```python
 repo = TableRepo(Path("/tmp/events"), dedup_cols=["event_id"])
 
 repo.extend(pl.DataFrame({"event_id": [1, 2, 3], "val": [10, 20, 30]}))
-# Re-ingest with overlapping IDs — existing rows are kept, new ones appended
+# Re-ingest with overlapping IDs — duplicates collapse to one row per event_id
 repo.extend(pl.DataFrame({"event_id": [2, 4], "val": [99, 40]}))
 
 repo.get_full_df().shape  # (4, 2) — event_id 2 kept once
@@ -131,9 +131,7 @@ repo.get_full_df().shape  # (4, 2) — event_id 2 kept once
 
 ## Sorted merge with `id_col`
 
-When `id_col` is set, each extend keeps the file sorted by that column
-(when `max_records == 0`) and `replace_records` does an upsert: matching IDs
-are updated, new IDs appended.
+When `id_col` is set, each extend keeps the file sorted by it (when `max_records == 0`) and `replace_records` does an upsert: matching IDs are updated, new IDs appended. `id_col` is a column or a list of columns, sorted in that order. On equal keys stored rows come before incoming ones, and within one `replace_records` batch the last row for a key wins.
 
 ```python
 repo = TableRepo(Path("/tmp/prices"), id_col="ticker")

{shackleton-1.2.0 → shackleton-2.0.1}/README.md

@@ -4,7 +4,7 @@
 
 Persistent tabular store backed by Parquet or Arrow IPC, built on [polars](https://pola.rs).
 
-- **Partitioned writes** — by column value or xxhash bucket
+- **Partitioned writes** — by column value or hash bucket
 - **Sorted merge** — maintain sorted order on extend via `id_col`
 - **Lazy reads** — returns `pl.LazyFrame` so polars can push down predicates
 - **Schema evolution** — diagonal concat handles mismatched columns transparently
@@ -73,7 +73,7 @@ df_eu = repo.get_partition_df({"region": "eu"})
 ## Hash partitioning
 
 When partition values have unbounded cardinality, hash into fixed buckets instead.
-Uses polars' native xxhash — vectorised, GIL-free.
+Uses polars' `Series.hash` — vectorised, GIL-free. That hash is not stable across polars versions while bucket assignment lives on disk, so before its first bucket-addressed operation (a write, `get_partition_*`, `purge_partition`) a repo samples stored rows and raises `HashDriftError` if they no longer hash to their bucket. `repo.rehash()` moves every row into its current bucket.
 
 ```python
 from shackleton import HashPartitioner
@@ -95,15 +95,15 @@ df = repo.get_full_df()
 ## Deduplication with `dedup_cols`
 
 When `dedup_cols` is set, each `extend` drops duplicate rows on those columns
-after merging with existing data — only the first occurrence is kept. Combine
-with `id_col` for sorted-merge dedup; use alone for unordered dedup.
+after merging with existing data. With `id_col` the merge is sorted and the
+existing (first) row wins; without `id_col` the surviving occurrence is arbitrary.
 Not supported with `max_records > 0` (raises at construction time).
 
 ```python
 repo = TableRepo(Path("/tmp/events"), dedup_cols=["event_id"])
 
 repo.extend(pl.DataFrame({"event_id": [1, 2, 3], "val": [10, 20, 30]}))
-# Re-ingest with overlapping IDs — existing rows are kept, new ones appended
+# Re-ingest with overlapping IDs — duplicates collapse to one row per event_id
 repo.extend(pl.DataFrame({"event_id": [2, 4], "val": [99, 40]}))
 
 repo.get_full_df().shape  # (4, 2) — event_id 2 kept once
@@ -113,9 +113,7 @@ repo.get_full_df().shape  # (4, 2) — event_id 2 kept once
 
 ## Sorted merge with `id_col`
 
-When `id_col` is set, each extend keeps the file sorted by that column
-(when `max_records == 0`) and `replace_records` does an upsert: matching IDs
-are updated, new IDs appended.
+When `id_col` is set, each extend keeps the file sorted by it (when `max_records == 0`) and `replace_records` does an upsert: matching IDs are updated, new IDs appended. `id_col` is a column or a list of columns, sorted in that order. On equal keys stored rows come before incoming ones, and within one `replace_records` batch the last row for a key wins.
 
 ```python
 repo = TableRepo(Path("/tmp/prices"), id_col="ticker")

shackleton-2.0.1/docs/release_notes.md

@@ -0,0 +1,29 @@
+## v2.0.1
+
+- Fixed composite-key merges to be order-exact: stored rows now reliably come before incoming ones on equal keys, where polars' streaming engine could previously reorder ties.
+- Fixed `replace_records` to work against a partition with no stored data yet.
+- Writes are now atomic: data is staged to a temp file and moved into place with `os.replace`, so a crash mid-merge can no longer leave a partially-written file.
+- Fixed partitions to be flagged sorted on read only when they are a single file, since appended parts are unsorted and were previously mis-flagged.
+
+---
+
+## v2.0.0
+
+- **Breaking:** requires polars ≥2; existing `HashPartitioner` stores built under older polars versions will raise `HashDriftError` on their next bucket-addressed operation until migrated.
+- Detects hash bucket drift before writes, `get_partition_*` and `purge_partition`, and adds `TableRepo.rehash()` to rebuild a store under the installed polars' hashing; full reads are unaffected and skip the check.
+- `id_col` now accepts a list of columns for composite-key merges and dedup.
+- Merges are now order-exact: on equal keys stored rows precede incoming ones, so `extend` with `dedup_cols` keeps the stored row and `replace_records` keeps the batch's last row instead of an arbitrary one.
+- Writes are now staged to a temp file and atomically renamed into place, and merges with existing data stream instead of collecting fully in memory.
+- `paths` are now returned sorted, so reads are deterministic.
+
+---
+
+## v1.2.0
+
+deduped
+
+---
+
+## v1.1.0
+
+realign, simplify

{shackleton-1.2.0 → shackleton-2.0.1}/pyproject.toml

@@ -6,7 +6,7 @@ authors = [{name = "Endre Márk Borza", email = "endremborza@gmail.com"}]
 license = {file = "LICENSE"}
 readme = "README.md"
 requires-python = ">=3.12"
-dependencies = ["polars"]
+dependencies = ["polars>=2"]
 
 [project.urls]
 Homepage = "https://github.com/endremborza/shackleton"

shackleton-2.0.1/shackleton/__init__.py

@@ -0,0 +1,13 @@
+"""Polars-based persistent tabular store backed by Parquet or Arrow IPC files."""
+
+from .core import (  # noqa: F401
+    DfBatchWriter,
+    HashDriftError,
+    HashPartitioner,
+    RecordWriter,
+    TableRepo,
+)
+
+TableShack = TableRepo  # backward-compat alias
+
+__version__ = "2.0.1"

{shackleton-1.2.0 → shackleton-2.0.1}/shackleton/core.py

@@ -1,8 +1,10 @@
 import contextlib
 import fcntl
+import os
+import shutil
 import threading
 from concurrent.futures import ThreadPoolExecutor
-from dataclasses import dataclass, field
+from dataclasses import dataclass, field, replace
 from functools import cached_property
 from pathlib import Path
 from typing import Any, Callable, Iterable, TypeVar
@@ -12,6 +14,13 @@ import polars as pl
 T = TypeVar("T")
 
 FILE_NAME = "data"
+HASH_CHECK_DIRS = 8
+HASH_CHECK_ROWS = 64
+
+
+class HashDriftError(Exception):
+    """Stored rows sit in buckets the installed polars no longer hashes them to."""
+
 
 _PATH_LOCKS: dict[str, threading.Lock] = {}
 _LOCKS_LOCK = threading.Lock()
@@ -44,7 +53,11 @@ def _dir_lock(path_str: str):
 class HashPartitioner:
     """Partition rows by hashing a column into `num_groups` zero-padded buckets.
 
-    Uses polars' native xxhash for vectorised, GIL-free partitioning.
+    Uses polars' ``Series.hash`` for vectorised, GIL-free partitioning. That hash
+    is not stable across polars versions while bucket assignment is persisted on
+    disk, so ``TableRepo`` samples stored rows before its first bucket-addressed
+    operation and raises ``HashDriftError`` if they no longer hash to their bucket;
+    ``TableRepo.rehash`` rewrites the layout.
     The computed column (``key``) is stripped before writing to files.
     """
 
@@ -75,13 +88,15 @@ class TableRepo:
 
     Args:
         root_path: Root directory for all data files.
-        id_col: Column used as a unique row identifier. Required for
-            ``replace_records``; enables sorted merge on ``extend`` when
-            ``max_records == 0`` (no effect on the per-file append path).
+        id_col: Column, or columns in sort order, identifying a row. Required
+            for ``replace_records``; enables sorted merge on ``extend`` when
+            ``max_records == 0`` (no effect on the per-file append path). On
+            equal keys, stored rows precede incoming ones and a batch keeps
+            its own order: ``replace_records`` keeps the batch's last row.
         dedup_cols: Columns forming a composite dedup key. When set, ``extend``
-            drops duplicates after merging with existing data. Only effective
-            when ``max_records == 0``; raises ``ValueError`` otherwise.
-            ``replace_records`` still requires a scalar ``id_col``.
+            drops duplicates after merging with existing data, keeping the
+            stored row when ``id_col`` is set. Only effective when
+            ``max_records == 0``; raises ``ValueError`` otherwise.
         partition_cols: Column name(s) to partition by, or a
             ``HashPartitioner`` for hash-based bucketing. Partition values
             are encoded as hive-style directories (``col=val/``).
@@ -96,13 +111,14 @@ class TableRepo:
     """
 
     root_path: Path
-    id_col: str | None = None
+    id_col: str | list[str] | None = None
     dedup_cols: list[str] | None = None
     partition_cols: list[str] | HashPartitioner = field(default_factory=list)
     max_records: int = 0
     ipc: bool = False
     compression: str | None = None
     compression_level: int | None = None
+    _buckets_verified: bool = field(default=False, init=False, repr=False)
 
     def __post_init__(self) -> None:
         if self.dedup_cols and self.max_records:
@@ -112,6 +128,13 @@ class TableRepo:
     def extension(self) -> str:
         return ".arrow" if self.ipc else ".parquet"
 
+    @cached_property
+    def id_cols(self) -> list[str]:
+        """``id_col`` as a list; empty when unset."""
+        if self.id_col is None:
+            return []
+        return [self.id_col] if isinstance(self.id_col, str) else list(self.id_col)
+
     # --- write ---
 
     def extend(self, df: pl.DataFrame) -> None:
@@ -124,7 +147,7 @@ class TableRepo:
         Requires ``id_col`` to be set. For multi-file partitions
         (``max_records > 0``) performs a full read-merge-rewrite.
         """
-        assert self.id_col is not None, "replace_records requires id_col"
+        assert self.id_cols, "replace_records requires id_col"
         self._for_each_partition(df, self._replace_dir)
 
     def replace_all(self, df: pl.DataFrame) -> None:
@@ -151,14 +174,31 @@ class TableRepo:
         for p in self.paths:
             p.unlink()
 
+    def rehash(self) -> None:
+        """Move every row of a ``HashPartitioner`` store into the bucket the
+        installed polars hashes it to — the fix for ``HashDriftError``.
+
+        Builds the new layout in a sibling ``<root>.rehash`` directory and swaps
+        it in. Holds the whole store in memory; assumes no concurrent writers.
+        """
+        if self._hasher is None:
+            raise ValueError("rehash requires a HashPartitioner")
+        staging = self.root_path.with_name(self.root_path.name + ".rehash")
+        if staging.exists():
+            raise FileExistsError(f"{staging} left by an interrupted rehash")
+        df = self.get_full_df()
+        if df.is_empty():
+            return
+        replace(self, root_path=staging).extend(df)
+        shutil.rmtree(self.root_path)
+        staging.rename(self.root_path)
+        self._buckets_verified = True
+
     # --- read ---
 
     def get_full_lf(self) -> pl.LazyFrame:
         """Lazy frame over all data in the store."""
-        parts = list(self.paths)
-        if not parts:
-            return pl.LazyFrame()
-        return pl.concat(map(self._lazy_read, parts), how="diagonal")
+        return self._concat_lf(self.paths)
 
     def get_full_df(self) -> pl.DataFrame:
         """Collect all data into a single DataFrame."""
@@ -171,10 +211,7 @@ class TableRepo:
         For ``HashPartitioner`` repos, use ``repo.partition_cols.key``
         as the dict key with the zero-padded bucket string as the value.
         """
-        paths = list(self._partition_paths(partitions))
-        if not paths:
-            return pl.LazyFrame()
-        return pl.concat(map(self._lazy_read, paths), how="diagonal")
+        return self._concat_lf(self._partition_paths(partitions))
 
     def get_partition_df(self, partitions: dict[str, Any]) -> pl.DataFrame:
         """Collect partitions matching ``partitions`` dict into a DataFrame."""
@@ -183,9 +220,9 @@ class TableRepo:
     # --- iteration ---
 
     @property
-    def paths(self) -> Iterable[Path]:
-        """All data file paths in the store."""
-        return self.root_path.glob("**/*" + self.extension)
+    def paths(self) -> list[Path]:
+        """All data file paths in the store, sorted so reads are deterministic."""
+        return sorted(self.root_path.glob("**/*" + self.extension))
 
     @property
     def n_files(self) -> int:
@@ -248,8 +285,7 @@ class TableRepo:
 
         def _apply(d: Path) -> T:
             files = sorted(d.glob("*" + self.extension))
-            lf = pl.concat([self._lazy_read(f) for f in files], how="diagonal")
-            return fn(lf.collect())
+            return fn(self._concat_lf(files).collect())
 
         if workers is None:
             return [_apply(d) for d in dirs]
@@ -269,9 +305,7 @@ class TableRepo:
                 files = sorted(d.glob("*" + self.extension))
                 if len(files) <= 1:
                     continue
-                df = pl.concat(
-                    [self._lazy_read(f) for f in files], how="diagonal"
-                ).collect()
+                df = self._concat_lf(files).collect()
                 for f in files:
                     f.unlink()
                 for idx, offset in enumerate(range(0, max(len(df), 1), target_rows)):
@@ -282,47 +316,73 @@ class TableRepo:
 
     # --- internals ---
 
+    def _concat_lf(self, paths: Iterable[Path]) -> pl.LazyFrame:
+        paths = list(paths)
+        if not paths:
+            return pl.LazyFrame()
+        return pl.concat(map(self._lazy_read, paths), how="diagonal")
+
     def _lazy_read(self, path: Path) -> pl.LazyFrame:
         lf = (
             pl.scan_ipc(path)
             if self.ipc
             else pl.scan_parquet(path, hive_partitioning=False)
         )
-        return lf.set_sorted(self.id_col) if self.id_col else lf
+        # only single-file partitions stay sorted; the append path does not sort
+        return (
+            lf.set_sorted(self.id_cols) if self.id_cols and not self.max_records else lf
+        )
 
-    def _write(self, df: pl.DataFrame, path: Path) -> None:
+    def _write(self, data: pl.DataFrame | pl.LazyFrame, path: Path) -> None:
+        """Write atomically; a LazyFrame (a merge with stored data) is streamed."""
         path.parent.mkdir(parents=True, exist_ok=True)
+        staged = path.with_name(path.name + ".tmp")
         kw: dict = {}
         if self.compression:
             kw["compression"] = self.compression
-        if self.ipc:
-            df.write_ipc(path, **kw)
+        if not self.ipc and self.compression_level is not None:
+            kw["compression_level"] = self.compression_level
+        if isinstance(data, pl.LazyFrame):
+            (data.sink_ipc if self.ipc else data.sink_parquet)(staged, **kw)
         else:
-            if self.compression_level is not None:
-                kw["compression_level"] = self.compression_level
-            df.write_parquet(path, **kw)
+            (data.write_ipc if self.ipc else data.write_parquet)(staged, **kw)
+        os.replace(staged, path)
+
+    def _sort(self, lf: pl.LazyFrame) -> pl.LazyFrame:
+        return lf.sort(self.id_cols, maintain_order=True)
 
     def _merge_extend(self, new: pl.LazyFrame, old: pl.LazyFrame) -> pl.LazyFrame:
-        if self.id_col:
-            out = old.merge_sorted(new.sort(self.id_col), key=self.id_col)
+        if self.id_cols:
+            out = old.merge_sorted(
+                self._sort(new), key=self.id_cols, maintain_order=True
+            )
         else:
             out = pl.concat([old, new], how="diagonal")
         if self.dedup_cols:
-            # when id_col is set, merge_sorted makes dupes adjacent; keep="first"
-            # with maintain_order=True then runs in O(n) and preserves sort
+            # an ordered merge puts stored rows before incoming ones on equal keys,
+            # so keep="first" keeps the stored row and maintain_order the sort
             out = out.unique(
                 subset=self.dedup_cols,
-                keep="first" if self.id_col else "any",
-                maintain_order=bool(self.id_col),
+                keep="first" if self.id_cols else "any",
+                maintain_order=bool(self.id_cols),
             )
         return out
 
-    def _merge_replace(self, new: pl.LazyFrame, old: pl.LazyFrame) -> pl.LazyFrame:
-        return (
-            old.merge_sorted(new.sort(self.id_col), key=self.id_col)
-            .unique(subset=self.id_col, keep="last")
-            .sort(self.id_col)
+    def _merge_replace(
+        self, new: pl.LazyFrame, old: pl.LazyFrame | None
+    ) -> pl.LazyFrame:
+        merged = (
+            self._sort(new)
+            if old is None
+            else old.merge_sorted(
+                self._sort(new), key=self.id_cols, maintain_order=True
+            )
         )
+        # sorted, so a key's rows are adjacent: keep the last row of each run
+        last_of_run = pl.any_horizontal(
+            pl.col(c).ne_missing(pl.col(c).shift(-1)) for c in self.id_cols
+        )
+        return merged.filter(last_of_run)
 
     def _for_each_partition(self, df: pl.DataFrame, handler: Callable) -> None:
         keys = self._part_keys
@@ -330,18 +390,37 @@ class TableRepo:
             handler(df, self.root_path)
             return
 
-        is_hash = isinstance(self.partition_cols, HashPartitioner)
-        if is_hash:
-            df = self.partition_cols.add_to_df(df)
+        hasher = self._hasher
+        if hasher is not None:
+            self._verify_buckets(hasher)
+            df = hasher.add_to_df(df)
 
         for group_vals, group_df in df.group_by(keys):
             group_vals = group_vals if isinstance(group_vals, tuple) else (group_vals,)
             hive_parts = [f"{k}={v}" for k, v in zip(keys, group_vals)]
             part_dir = Path(self.root_path, *hive_parts)
-            if is_hash:
+            if hasher is not None:
                 group_df = group_df.drop(keys)
             handler(group_df, part_dir)
 
+    def _verify_buckets(self, hasher: HashPartitioner) -> None:
+        if self._buckets_verified:
+            return
+        for d in self._partition_dirs[:HASH_CHECK_DIRS]:
+            bucket = d.name.split("=", 1)[1]
+            sample = (
+                self._lazy_read(next(d.glob("*" + self.extension)))
+                .select(hasher.col)
+                .head(HASH_CHECK_ROWS)
+                .collect()
+            )
+            if (hasher.add_to_df(sample)[hasher.key] != bucket).any():
+                raise HashDriftError(
+                    f"{self.root_path}: rows of bucket {bucket} hash elsewhere under "
+                    f"polars {pl.__version__}; rewrite with TableRepo.rehash()"
+                )
+        self._buckets_verified = True
+
     def _extend_dir(self, df: pl.DataFrame, root: Path) -> None:
         with _dir_lock(str(root)):
             if self.max_records > 0:
@@ -353,11 +432,13 @@ class TableRepo:
                     out = df
                     if self.dedup_cols:
                         out = out.unique(subset=self.dedup_cols, maintain_order=False)
-                    if self.id_col:
-                        out = out.sort(self.id_col)
+                    if self.id_cols:
+                        out = out.sort(self.id_cols, maintain_order=True)
+                    self._write(out, path)
                 else:
-                    out = self._merge_extend(df.lazy(), self._lazy_read(path)).collect()
-                self._write(out, path)
+                    self._write(
+                        self._merge_extend(df.lazy(), self._lazy_read(path)), path
+                    )
 
     def _purge_dir(self, root: Path) -> None:
         with _dir_lock(str(root)):
@@ -371,21 +452,19 @@ class TableRepo:
     def _replace_dir(self, df: pl.DataFrame, root: Path) -> None:
         with _dir_lock(str(root)):
             existing = sorted(root.glob("*" + self.extension))
-            if not existing:
-                self._write(df.sort(self.id_col), self._single_path(root))
-                return
+            old = None
+            if existing:
+                old = self._concat_lf(existing)
+                if self.max_records > 0:
+                    old = self._sort(old)
+            out = self._merge_replace(df.lazy(), old)
             if self.max_records > 0:
-                old = pl.concat(
-                    [self._lazy_read(p) for p in existing], how="diagonal"
-                ).collect()
-                out = self._merge_replace(df.lazy(), old.lazy()).collect()
+                out = out.collect()
                 for p in existing:
                     p.unlink()
                 self._append_parts(out, root, [])
             else:
-                path = existing[0]
-                out = self._merge_replace(df.lazy(), self._lazy_read(path)).collect()
-                self._write(out, path)
+                self._write(out, self._single_path(root))
 
     def _append_parts(self, df: pl.DataFrame, root: Path, existing: list[Path]) -> None:
         if existing:
@@ -406,6 +485,8 @@ class TableRepo:
         keys = self._part_keys
         if not keys:
             return []
+        if self._hasher is not None:
+            self._verify_buckets(self._hasher)
         g = "/".join(f"{c}={partitions.get(c, '*')}" for c in keys)
         return self.root_path.glob(f"{g}/*{self.extension}")
 
@@ -423,12 +504,17 @@ class TableRepo:
                 result.append(d)
         return result
 
+    @cached_property
+    def _hasher(self) -> HashPartitioner | None:
+        pc = self.partition_cols
+        return pc if isinstance(pc, HashPartitioner) else None
+
     @cached_property
     def _part_keys(self) -> list[str]:
         if not self.partition_cols:
             return []
-        if isinstance(self.partition_cols, HashPartitioner):
-            return [self.partition_cols.key]
+        if self._hasher is not None:
+            return [self._hasher.key]
         return list(self.partition_cols)
 
 

{shackleton-1.2.0 → shackleton-2.0.1}/shackleton/tests/test_core.py

@@ -5,7 +5,7 @@ from base64 import b16encode
 import polars as pl
 import pytest
 
-from shackleton import HashPartitioner, TableRepo
+from shackleton import HashDriftError, HashPartitioner, TableRepo
 
 
 def make_df(n: int, seed: int = 0) -> pl.DataFrame:
@@ -173,7 +173,16 @@ def test_replace_records_dedup(tmp_path):
     repo.replace_records(pl.DataFrame({"id": [2, 2], "val": [20, 21]}))
     result = repo.get_full_df().sort("id")
     assert result.shape[0] == 3
-    assert result.filter(pl.col("id") == 2)["val"][0] in (20, 21)
+    assert result.filter(pl.col("id") == 2)["val"][0] == 21  # batch's last row wins
+
+
+def test_replace_records_over_duplicate_ids(tmp_path):
+    repo = TableRepo(tmp_path / "data", id_col="id")
+    ids = list(range(10))
+    repo.extend(pl.DataFrame({"id": ids, "val": 1}))
+    repo.extend(pl.DataFrame({"id": ids, "val": 2}))
+    repo.replace_records(pl.DataFrame({"id": ids, "val": 3}))
+    assert repo.get_full_df()["val"].to_list() == [3] * 10
 
 
 def test_replace_records_with_max_records(tmp_path):
@@ -187,12 +196,44 @@ def test_replace_records_with_max_records(tmp_path):
     assert repo.n_files == 2  # 4 rows at max_records=2
 
 
+def test_replace_records_max_records_unsorted_parts(tmp_path):
+    repo = TableRepo(tmp_path / "data", id_col="id", max_records=2)
+    repo.extend(pl.DataFrame({"id": [3, 1, 2], "val": [30, 10, 20]}))
+    repo.replace_records(pl.DataFrame({"id": [1], "val": [100]}))
+    result = repo.get_full_df()
+    assert result["id"].to_list() == [1, 2, 3]
+    assert result["val"].to_list() == [100, 20, 30]
+
+
+def test_composite_id_col(tmp_path):
+    repo = TableRepo(tmp_path / "data", id_col=["a", "b"])
+    repo.extend(pl.DataFrame({"a": [2, 1, 1], "b": [1, 2, 1], "val": [21, 12, 11]}))
+    repo.extend(pl.DataFrame({"a": [1, 2], "b": [3, 0], "val": [13, 20]}))
+    assert repo.get_full_df()["val"].to_list() == [11, 12, 13, 20, 21]
+    repo.replace_records(pl.DataFrame({"a": [1, 3], "b": [2, 0], "val": [-12, 30]}))
+    assert repo.get_full_df()["val"].to_list() == [11, -12, 13, 20, 21, 30]
+
+
 def test_replace_records_empty_repo(tmp_path):
     repo = TableRepo(tmp_path / "data", id_col="id")
     repo.replace_records(pl.DataFrame({"id": [1, 2], "val": [1, 2]}))
     assert repo.get_full_df().shape[0] == 2
 
 
+def test_replace_records_empty_repo_max_records(tmp_path):
+    repo = TableRepo(tmp_path / "data", id_col="id", max_records=2)
+    repo.replace_records(
+        pl.DataFrame({"id": [1, 2, 3, 4, 5], "val": [10, 20, 30, 40, 50]})
+    )
+    assert repo.n_files == 3  # 5 rows at max_records=2 → (2, 2, 1)
+    assert all(p.name.startswith("file-") for p in repo.paths)
+    repo.extend(pl.DataFrame({"id": [6, 7], "val": [60, 70]}))
+    result = repo.get_full_df().sort("id")
+    assert result["id"].to_list() == [1, 2, 3, 4, 5, 6, 7]
+    assert all(p.name.startswith("file-") for p in repo.paths)
+    assert repo.n_files == 4  # 7 rows at max_records=2 → (2, 2, 2, 1)
+
+
 def test_id_col_sorted_merge_on_extend(tmp_path):
     repo = TableRepo(tmp_path / "data", id_col="id")
     repo.extend(pl.DataFrame({"id": [1, 3], "val": [10, 30]}))
@@ -243,6 +284,15 @@ def test_dedup_cols_with_id_col_old_wins(tmp_path):
     assert result.filter(pl.col("id") == 3)["val"][0] == 30  # new row inserted
 
 
+def test_dedup_cols_old_wins_within_id_col_runs(tmp_path):
+    repo = TableRepo(tmp_path / "data", id_col="g", dedup_cols=["g", "id"])
+    ids = list(range(10))
+    df = pl.DataFrame({"g": [i // 4 for i in ids], "id": ids})
+    repo.extend(df.with_columns(val=pl.lit("stored")))
+    repo.extend(df.with_columns(val=pl.lit("incoming")))
+    assert repo.get_full_df()["val"].to_list() == ["stored"] * 10
+
+
 # ---------------------------------------------------------------------------
 # IPC / Parquet format
 # ---------------------------------------------------------------------------
@@ -323,6 +373,26 @@ def test_hash_partitioner_extend_twice(tmp_path):
     assert repo.get_full_df().shape[0] == 60
 
 
+def test_hash_drift_raises_until_rehash(tmp_path, monkeypatch):
+    hp = HashPartitioner("id", num_groups=4)
+    df = pl.DataFrame({"id": range(100), "val": 0})
+    TableRepo(tmp_path / "data", id_col="id", partition_cols=hp).extend(df)
+
+    upgraded = pl.Expr.hash
+    monkeypatch.setattr(pl.Expr, "hash", lambda e, seed=0: upgraded(e, seed + 1))
+    repo = TableRepo(tmp_path / "data", id_col="id", partition_cols=hp)
+    assert repo.get_full_df().shape[0] == 100  # full reads are layout-agnostic
+    with pytest.raises(HashDriftError):
+        repo.replace_records(df.with_columns(val=pl.lit(1)))
+    with pytest.raises(HashDriftError):
+        repo.get_partition_df({hp.key: "0"})
+
+    repo.rehash()
+    fresh = TableRepo(tmp_path / "data", id_col="id", partition_cols=hp)
+    fresh.replace_records(df.with_columns(val=pl.lit(1)))
+    assert fresh.get_full_df().sort("id")["val"].to_list() == [1] * 100
+
+
 # ---------------------------------------------------------------------------
 # RecordWriter
 # ---------------------------------------------------------------------------

{shackleton-1.2.0 → shackleton-2.0.1}/uv.lock

@@ -40,30 +40,30 @@ wheels = [
 
 [[package]]
 name = "polars"
-version = "1.39.0"
+version = "2.0.0"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "polars-runtime-32" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/ab/b8/3a6a5b85e34af7936620f331f04f8bed235625439f5bd80832f968648618/polars-1.39.0.tar.gz", hash = "sha256:e63a25fb7682ae660e36067915a7c71a653b17f82308a8eb67a190a80daf0710", size = 728783, upload-time = "2026-03-12T14:24:47.876Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/8e/e9/001f371ec6a1bb54893f599ceebd56e6144fed4091f09f09fec0021a9276/polars-2.0.0.tar.gz", hash = "sha256:62da109e27a19a9d36657ee25dc035c9d3f87e7bd610526fe467dc37ea7dc115", size = 778215, upload-time = "2026-10-06T11:51:29.679Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/a7/f8/fad8470d9701c1b208cc24919a661efdf565373e77e7d06400642a759285/polars-1.39.0-py3-none-any.whl", hash = "sha256:4d1198b41bc47561673d9f54d0f595125202a3f53e3502821802958d3e60efe9", size = 823938, upload-time = "2026-03-12T14:22:37.78Z" },
+    { url = "https://files.pythonhosted.org/packages/ac/09/cc33bbd5463749c116b62c204d88bed6c02a6cb901eac7adab0d38651b07/polars-2.0.0-py3-none-any.whl", hash = "sha256:35d62f3541b7a6d4c360a2e2f07fccc0c2bcbd33b0ea51c83a25417a47a3f3ad", size = 876611, upload-time = "2026-10-06T11:44:04.327Z" },
 ]
 
 [[package]]
 name = "polars-runtime-32"
-version = "1.39.0"
+version = "2.0.0"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/8b/1e/fce83ad77bfed1bf4a83f74dde19e2572c32fc040e93bd98d161e3950eaf/polars_runtime_32-1.39.0.tar.gz", hash = "sha256:f5aabed8c7318fcad5173e83bee385445f54b5f8c83b1ec9eab78bdffa293141", size = 2870686, upload-time = "2026-03-12T14:24:49.41Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/34/ad/dbb6f6d7070867951532bcfe5e6a648d8777b416b18cddabc07030404e8c/polars_runtime_32-2.0.0.tar.gz", hash = "sha256:b5f9afcc742b4a67eabd2c680ff0f12eb02ede9b4bf807bffabd6dbb9a58d5c7", size = 3591339, upload-time = "2026-10-06T11:51:31.076Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/2b/6d/143b552baa9e859ae266f087f3ec0aeb29e5acc39e1f49c1a64023cee469/polars_runtime_32-1.39.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:4a4bc06ca97238d963979e3f888fbb500ee607f03cefe43a9062381e259503e2", size = 45299222, upload-time = "2026-03-12T14:22:40.821Z" },
-    { url = "https://files.pythonhosted.org/packages/97/ec/eb4e57eedfb97019f951b298fa4cd232a50db65aa6702c735b6f272a0fa0/polars_runtime_32-1.39.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:e9914b9e168634bc21d07ee03b8fa92d0aaa8ac7b2bb1c9e2f1f78622aa1b8f4", size = 40863978, upload-time = "2026-03-12T14:22:45.16Z" },
-    { url = "https://files.pythonhosted.org/packages/5f/b7/28fa0345586f7c449dd27d687c32a10dcea470ebc5a978d7fc47e463b298/polars_runtime_32-1.39.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8ded58f1c28e17ecbff8625cb1ad93016761260348acb79b1a4cd077970e89e5", size = 43231627, upload-time = "2026-03-12T14:22:49.464Z" },
-    { url = "https://files.pythonhosted.org/packages/cf/60/c0d0b6720437685223457242a79f6bba443485ca85928645786479ebed86/polars_runtime_32-1.39.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b82c872b25ef6628462f90f1b6b3950779aee36889e83b3693d0a69684d3d86a", size = 46899324, upload-time = "2026-03-12T14:22:54.364Z" },
-    { url = "https://files.pythonhosted.org/packages/73/98/53ad9c8a6f151e098e4f65c5146f9e538f1ba148feb5289fd2a4c5e2d764/polars_runtime_32-1.39.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:4a0e9d6b56362f3ba1a33d0538ae14c9b9a8e0fb835f86abfc82fa7b2c7d89c9", size = 43389283, upload-time = "2026-03-12T14:22:59.767Z" },
-    { url = "https://files.pythonhosted.org/packages/74/a2/21f77d6e588ee7c8e7f6232d135538690411de2ea6415d8bbe9b8d684f37/polars_runtime_32-1.39.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:0daea3919661ba672b00bd01b5547cd29bb6414732457abb72cbc75103cf3c90", size = 46509946, upload-time = "2026-03-12T14:23:05.215Z" },
-    { url = "https://files.pythonhosted.org/packages/24/a3/37a56ad2d931c857b892b22760b9bf9a53f681d9ccf27741cf6dd8489320/polars_runtime_32-1.39.0-cp310-abi3-win_amd64.whl", hash = "sha256:d6e9d1cf264aacfe5bf03241c04ef435d0f9cfec3fbe079acc3a7328a737961a", size = 47012669, upload-time = "2026-03-12T14:23:11.134Z" },
-    { url = "https://files.pythonhosted.org/packages/b3/eb/936f5eeae196e8c8aaabe5f7d98891be8a5bbc741d50ce5c60f55575ad29/polars_runtime_32-1.39.0-cp310-abi3-win_arm64.whl", hash = "sha256:d69abde5f148566860bbe910010847bd7791e72f7c8063a4d2c462246a33a72a", size = 41885761, upload-time = "2026-03-12T14:23:16.773Z" },
+    { url = "https://files.pythonhosted.org/packages/82/88/d35dec6c8928dfbaa1cccf9b626a1067da906e792c92d9f994ca825ab2b5/polars_runtime_32-2.0.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:ffb7ac6cf4e8c4a652df1951e3c3840c7c23a033603d5a9efd422fa8dd699d82", size = 52494314, upload-time = "2026-10-06T11:44:07.768Z" },
+    { url = "https://files.pythonhosted.org/packages/5f/fd/2237bf53ffaff47cdf1edc6c10587a7a6444d4951150eeb08d84f3493ff8/polars_runtime_32-2.0.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:7012d8a0201bd95638545ce8f256c0efe2c5cab0f806eb043021dddde5a9498b", size = 47930083, upload-time = "2026-10-06T11:44:11.592Z" },
+    { url = "https://files.pythonhosted.org/packages/0d/0d/85e3ed90417996fc09770be91b39979074fe2978fc15b431bf8a9459760d/polars_runtime_32-2.0.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b85bb42e6009acc9629afcc70a83473fd468694d6a30ffb0ab376c8dd1a0a17", size = 50417889, upload-time = "2026-10-06T11:50:20.774Z" },
+    { url = "https://files.pythonhosted.org/packages/83/88/e9fecfd49159da92f54ff2445883577a0f1bc195da53ecc9535c458d55dd/polars_runtime_32-2.0.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0d6ac584ea2b38913784db943879412380d92e28ab9cb88e20a77ba71ba3f911", size = 54475036, upload-time = "2026-10-06T11:50:24.411Z" },
+    { url = "https://files.pythonhosted.org/packages/48/ad/b2abf732697b21467aaaeaac0f3bf7eee0d89c59ce8125f1ed41b28a2d97/polars_runtime_32-2.0.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:a6bf5e260e0a6f00d0f9181438fe9e45776df8c66cee9cba16e3675cc3888488", size = 50579474, upload-time = "2026-10-06T11:50:28.377Z" },
+    { url = "https://files.pythonhosted.org/packages/7f/05/304deee59a95865e1b5e9ec7b066069b49093b81b768f473d9d3b165c686/polars_runtime_32-2.0.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:55c26eef325b6840584d91aac232e9cf3ac19e1b904594b9b54131be1edeab4d", size = 54413293, upload-time = "2026-10-06T11:50:31.828Z" },
+    { url = "https://files.pythonhosted.org/packages/61/59/8c9fd7199f7c4eb1b64e640306a946a2e4a46337b3bbb33b840972c7d84b/polars_runtime_32-2.0.0-cp310-abi3-win_amd64.whl", hash = "sha256:7da1caf3c7b4f397fb213c984013a0c755557619a2d511899a1ff74392484078", size = 54229989, upload-time = "2026-10-06T11:50:35.206Z" },
+    { url = "https://files.pythonhosted.org/packages/e2/93/43608026f38aa6ed4d22da8597706a61682ee403caef0021ce8e6dc73227/polars_runtime_32-2.0.0-cp310-abi3-win_arm64.whl", hash = "sha256:c30ba698c8904048df4a9bc3d6c5033cc2d0a7cbb0e13f4fd2de5a1947b61994", size = 48730655, upload-time = "2026-10-06T11:50:38.756Z" },
 ]
 
 [[package]]
@@ -104,7 +104,7 @@ dev = [
 ]
 
 [package.metadata]
-requires-dist = [{ name = "polars" }]
+requires-dist = [{ name = "polars", specifier = ">=2" }]
 
 [package.metadata.requires-dev]
 dev = [{ name = "pytest", specifier = ">=9.0.2" }]

shackleton-1.2.0/.github/workflows/twine_release.yml

@@ -1,15 +0,0 @@
-name: Python Package Release
-
-on:
-  release:
-    types: [published]
-
-jobs:
-  release:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v4
-    - uses: astral-sh/setup-uv@v5
-    - run: uv build && uv publish
-      env:
-        UV_PUBLISH_TOKEN: ${{ secrets.PYPI_TOKEN }}

shackleton-1.2.0/docs/release_notes.md

@@ -1,9 +0,0 @@
-## v1.2.0
-
-deduped
-
----
-
-## v1.1.0
-
-realign, simplify

shackleton-1.2.0/shackleton/__init__.py

@@ -1,7 +0,0 @@
-"""Polars-based persistent tabular store backed by Parquet or Arrow IPC files."""
-
-from .core import DfBatchWriter, HashPartitioner, RecordWriter, TableRepo  # noqa: F401
-
-TableShack = TableRepo  # backward-compat alias
-
-__version__ = "1.2.0"