supertable 2.3.6__tar.gz → 2.3.7__tar.gz

{supertable-2.3.6/supertable.egg-info → supertable-2.3.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: supertable
-Version: 2.3.6
+Version: 2.3.7
 Summary: SuperTable — versioned data lake library for SQL analytics on Parquet + Redis.
 Author: Levente Kupas
 Author-email: Levente Kupas <lkupas@kladnasoft.com>

{supertable-2.3.6 → supertable-2.3.7}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "supertable"
-version = "2.3.6"
+version = "2.3.7"
 description = "SuperTable — versioned data lake library for SQL analytics on Parquet + Redis."
 readme = "README.md"
 requires-python = ">=3.10"

{supertable-2.3.6 → supertable-2.3.7}/setup.py

@@ -19,7 +19,7 @@ long_description = readme.read_text(encoding="utf-8") if readme.exists() else ""
 
 setup(
     name="supertable",
-    version="2.3.6",
+    version="2.3.7",
     description="SuperTable — versioned data lake library for SQL analytics on Parquet + Redis.",
     long_description=long_description,
     long_description_content_type="text/markdown",

{supertable-2.3.6 → supertable-2.3.7}/supertable/__init__.py

@@ -25,7 +25,7 @@ See the ``supertable.demo`` package for runnable end-to-end demos and the
 project documentation for the full API surface.
 """
 
-__version__ = "2.3.6"
+__version__ = "2.3.7"
 
 # Re-export the core public surface so users can do ``from supertable import …``
 # instead of remembering submodule paths.

{supertable-2.3.6 → supertable-2.3.7}/supertable/config/settings.py

@@ -157,6 +157,13 @@ class Settings:
     SUPERTABLE_DUCKDB_MATERIALIZE: str = "view"    # SUPERTABLE_DUCKDB_MATERIALIZE
     SUPERTABLE_DUCKDB_PRESIGNED: bool = False      # SUPERTABLE_DUCKDB_PRESIGNED
     SUPERTABLE_DUCKDB_USE_HTTPFS: bool = False     # SUPERTABLE_DUCKDB_USE_HTTPFS
+    # Write-path overwrite/delete resolution via the DuckDB pushdown probe.
+    # Disabled by default: the polars fallback reads only the projected key
+    # columns through the storage SDK and needs no httpfs extension, so it works
+    # in environments without one (or without internet to install it).  Enable
+    # only where httpfs is available and the probe's row-group skipping is worth
+    # it (e.g. very wide tables / many overlapping files).
+    SUPERTABLE_DUCKDB_WRITE_PROBE: bool = False    # SUPERTABLE_DUCKDB_WRITE_PROBE
     # Deletion-vector (tombstone) table cache.  Each entry is a small
     # `DISTINCT __rowid__` table keyed by the stable tombstone path; the
     # tombstone view ANTI JOINs it instead of re-reading the parquet every
@@ -437,6 +444,7 @@ def _build_settings() -> Settings:
         SUPERTABLE_DUCKDB_MATERIALIZE=_env_str("SUPERTABLE_DUCKDB_MATERIALIZE", "view"),
         SUPERTABLE_DUCKDB_PRESIGNED=_env_bool("SUPERTABLE_DUCKDB_PRESIGNED", False),
         SUPERTABLE_DUCKDB_USE_HTTPFS=_env_bool("SUPERTABLE_DUCKDB_USE_HTTPFS", False),
+        SUPERTABLE_DUCKDB_WRITE_PROBE=_env_bool("SUPERTABLE_DUCKDB_WRITE_PROBE", False),
         SUPERTABLE_DUCKDB_TOMBSTONE_CACHE_MAX_PER_TABLE=_env_int("SUPERTABLE_DUCKDB_TOMBSTONE_CACHE_MAX_PER_TABLE", 8),
         SUPERTABLE_DUCKDB_TOMBSTONE_CACHE_TTL_SEC=_env_int("SUPERTABLE_DUCKDB_TOMBSTONE_CACHE_TTL_SEC", 300),
         SUPERTABLE_DEBUG_TIMINGS=_env_bool("SUPERTABLE_DEBUG_TIMINGS", False),

{supertable-2.3.6 → supertable-2.3.7}/supertable/processing.py

@@ -1316,11 +1316,13 @@ def resolve_overwrite_writes(
 ) -> Tuple[polars.DataFrame, List[Tuple[str, int]]]:
     """Single-pass overwrite resolution: stale filtering + delete-vector pairs.
 
-    Returns ``(filtered_incoming_df, delete_pairs)`` computed from ONE DuckDB
-    pushdown probe over the overlapping files.  Falls back to the original polars
+    Returns ``(filtered_incoming_df, delete_pairs)``.  When the DuckDB pushdown
+    probe is enabled (``SUPERTABLE_DUCKDB_WRITE_PROBE``) it is computed from ONE
+    probe over the overlapping files.  Falls back to the original polars
     full-read path (``filter_stale_incoming_rows`` + ``identify_deleted_rowids``)
-    when DuckDB is unavailable, the probe fails, or the file schema can't be
-    probed — semantics are identical on both paths.
+    when the probe is disabled (the default), DuckDB is unavailable, the probe
+    fails, or the file schema can't be probed — semantics are identical on both
+    paths.
 
     *newer_than_col* falsy ⇒ no stale filtering (delete/upsert without conflict
     resolution); the incoming df is returned unchanged and every overlapping row
@@ -1343,9 +1345,15 @@ def resolve_overwrite_writes(
         f"{incoming_keys.height} unique incoming key(s) on {overwrite_columns}, "
         f"newer_than={newer_than_col}"
     )
-    matched = _duckdb_probe_overlap_matches(
-        overlap_true, overwrite_columns, newer_than_col, incoming_keys, profiler=p,
-    )
+    # The DuckDB pushdown probe is opt-in (SUPERTABLE_DUCKDB_WRITE_PROBE).  When
+    # disabled (the default), skip it entirely and use the polars fallback below,
+    # which reads only the projected key columns via the storage SDK and needs no
+    # httpfs extension — the safe path for environments without one.
+    matched = None
+    if settings.SUPERTABLE_DUCKDB_WRITE_PROBE:
+        matched = _duckdb_probe_overlap_matches(
+            overlap_true, overwrite_columns, newer_than_col, incoming_keys, profiler=p,
+        )
     if matched is not None:
         try:
             return _derive_stale_and_deletes(

{supertable-2.3.6 → supertable-2.3.7}/supertable/tests/test_resolve_overwrite_writes.py

@@ -20,21 +20,37 @@ local files regardless of the ambient STORAGE_TYPE.
 """
 from __future__ import annotations
 
+import dataclasses
 from unittest.mock import patch
 
 import polars as pl
 import pyarrow.parquet as pq
 import pytest
 
+import supertable.processing as st_processing
 from supertable.processing import (
     resolve_overwrite_writes,
     filter_stale_incoming_rows,
     identify_deleted_rowids,
 )
+from supertable.config.settings import settings
 from supertable.storage.local_storage import LocalStorage
 from supertable.utils.profiler import Profiler
 
 
+@pytest.fixture(autouse=True)
+def _enable_write_probe(monkeypatch):
+    """These tests validate the DuckDB pushdown probe path, which is opt-in
+    (``SUPERTABLE_DUCKDB_WRITE_PROBE``, default off).  Force it on so the probe
+    is actually exercised; without this the gate in ``resolve_overwrite_writes``
+    would route every call to the polars fallback and the probe assertions
+    (``probe_files`` present, no ``overwrite_resolve_fallback``) would be vacuous."""
+    monkeypatch.setattr(
+        st_processing, "settings",
+        dataclasses.replace(settings, SUPERTABLE_DUCKDB_WRITE_PROBE=True),
+    )
+
+
 # ---------------------------------------------------------------------------
 # Helpers
 # ---------------------------------------------------------------------------

supertable-2.3.7/supertable/tests/test_write_probe_gate.py

@@ -0,0 +1,130 @@
+# supertable/tests/test_write_probe_gate.py
+"""Gate test for ``SUPERTABLE_DUCKDB_WRITE_PROBE``.
+
+The DuckDB pushdown probe in the overwrite/delete write path is opt-in and
+disabled by default.  Environments without the httpfs extension (or without
+internet to install it) must NOT stall on a DuckDB httpfs install; they use the
+polars fallback, which reads only the projected key columns through the storage
+SDK.  These tests pin the gate's contract:
+
+  * flag OFF (default) -> the probe is never called; resolution goes through the
+    polars fallback (profiler 'overwrite_resolve_fallback' set, no 'probe_files').
+  * flag ON            -> the probe IS called ('probe_files' set).
+  * both produce identical (filtered rows, delete pairs) -- the gate changes
+    only the mechanism, never the result (the fallback is the semantic oracle).
+"""
+from __future__ import annotations
+
+import dataclasses
+from unittest.mock import patch
+
+import polars as pl
+import pyarrow.parquet as pq
+import pytest
+
+import supertable.processing as st_processing
+from supertable.config.settings import settings
+from supertable.processing import resolve_overwrite_writes
+from supertable.storage.local_storage import LocalStorage
+from supertable.utils.profiler import Profiler
+
+
+@pytest.fixture(autouse=True)
+def _local_storage():
+    """Force both probe and fallback reads through a real LocalStorage so the
+    on-disk tmp parquet files are read identically regardless of STORAGE_TYPE."""
+    with patch("supertable.processing._get_storage", return_value=LocalStorage()):
+        yield
+
+
+def _write(d, name, df):
+    path = str(d / name)
+    pq.write_table(df.to_arrow(), path)
+    return (path, True, (d / name).stat().st_size)
+
+
+def _set_probe(monkeypatch, enabled: bool):
+    monkeypatch.setattr(
+        st_processing, "settings",
+        dataclasses.replace(settings, SUPERTABLE_DUCKDB_WRITE_PROBE=enabled),
+    )
+
+
+def _spy_probe(monkeypatch):
+    """Wrap the real probe to count calls without altering its behavior."""
+    calls = {"n": 0}
+    real = st_processing._duckdb_probe_overlap_matches
+
+    def _counting(*a, **k):
+        calls["n"] += 1
+        return real(*a, **k)
+
+    monkeypatch.setattr(st_processing, "_duckdb_probe_overlap_matches", _counting)
+    return calls
+
+
+def _resolve(incoming, files, keys, ntc, prof):
+    return resolve_overwrite_writes(
+        incoming_df=incoming, overlapping_files=files,
+        overwrite_columns=keys, newer_than_col=ntc, profiler=prof,
+    )
+
+
+def test_probe_disabled_by_default_uses_fallback(tmp_path, monkeypatch):
+    f = _write(tmp_path, "a.parquet", pl.DataFrame(
+        {"__rowid__": [1], "user_id": [5], "name": ["Alice"], "updated_at": [7]}))
+    incoming = pl.DataFrame({"user_id": [5], "name": ["Bob"], "updated_at": [9]})
+
+    _set_probe(monkeypatch, False)
+    calls = _spy_probe(monkeypatch)
+    prof = Profiler()
+    filt, pairs = _resolve(incoming, {f}, ["user_id"], "updated_at", prof)
+
+    counts = prof.emit_counts()
+    assert calls["n"] == 0, "probe must NOT be called when the flag is off"
+    assert "overwrite_resolve_fallback" in counts, f"fallback not taken; counts={counts}"
+    assert "probe_files" not in counts, f"probe ran despite flag off; counts={counts}"
+    # Correct result via the fallback: the newer incoming row survives and
+    # tombstones the existing row's __rowid__.
+    assert filt.height == 1
+    assert pairs == [(f[0], 1)]
+
+
+def test_probe_enabled_calls_probe(tmp_path, monkeypatch):
+    f = _write(tmp_path, "a.parquet", pl.DataFrame(
+        {"__rowid__": [1], "user_id": [5], "name": ["Alice"], "updated_at": [7]}))
+    incoming = pl.DataFrame({"user_id": [5], "name": ["Bob"], "updated_at": [9]})
+
+    _set_probe(monkeypatch, True)
+    calls = _spy_probe(monkeypatch)
+    prof = Profiler()
+    filt, pairs = _resolve(incoming, {f}, ["user_id"], "updated_at", prof)
+
+    counts = prof.emit_counts()
+    assert calls["n"] == 1, "probe must be called when the flag is on"
+    assert "probe_files" in counts, f"probe did not run; counts={counts}"
+    assert filt.height == 1
+    assert pairs == [(f[0], 1)]
+
+
+def test_gate_result_identical_on_and_off(tmp_path, monkeypatch):
+    """The flag changes only the mechanism: filtered rows + delete pairs match.
+
+    user_id=5 is newer (9 > 7) -> survives + tombstones rowid 1; user_id=6 is
+    stale (3 < 7) -> dropped, no tombstone.  Identical on both code paths.
+    """
+    f = _write(tmp_path, "d.parquet", pl.DataFrame(
+        {"__rowid__": [1, 2], "user_id": [5, 6], "name": ["A", "B"],
+         "updated_at": [7, 7]}))
+    incoming = pl.DataFrame(
+        {"user_id": [5, 6], "name": ["X", "Y"], "updated_at": [9, 3]})
+
+    def _run(enabled):
+        _set_probe(monkeypatch, enabled)
+        filt, pairs = _resolve(incoming, {f}, ["user_id"], "updated_at", Profiler())
+        rows = sorted(
+            filt.select(["user_id", "name", "updated_at"]).to_dicts(), key=repr
+        )
+        return rows, sorted(pairs)
+
+    assert _run(True) == _run(False)

{supertable-2.3.6 → supertable-2.3.7/supertable.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: supertable
-Version: 2.3.6
+Version: 2.3.7
 Summary: SuperTable — versioned data lake library for SQL analytics on Parquet + Redis.
 Author: Levente Kupas
 Author-email: Levente Kupas <lkupas@kladnasoft.com>

{supertable-2.3.6 → supertable-2.3.7}/supertable.egg-info/SOURCES.txt

@@ -176,6 +176,7 @@ supertable/tests/test_stats_schema_snapshot.py
 supertable/tests/test_super_table.py
 supertable/tests/test_supertable_all.py
 supertable/tests/test_system_query.py
+supertable/tests/test_write_probe_gate.py
 supertable/utils/__init__.py
 supertable/utils/helper.py
 supertable/utils/profiler.py