supertable 2.3.4__tar.gz → 2.3.6__tar.gz

{supertable-2.3.4/supertable.egg-info → supertable-2.3.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: supertable
-Version: 2.3.4
+Version: 2.3.6
 Summary: SuperTable — versioned data lake library for SQL analytics on Parquet + Redis.
 Author: Levente Kupas
 Author-email: Levente Kupas <lkupas@kladnasoft.com>

{supertable-2.3.4 → supertable-2.3.6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "supertable"
-version = "2.3.4"
+version = "2.3.6"
 description = "SuperTable — versioned data lake library for SQL analytics on Parquet + Redis."
 readme = "README.md"
 requires-python = ">=3.10"

{supertable-2.3.4 → supertable-2.3.6}/setup.py

@@ -19,7 +19,7 @@ long_description = readme.read_text(encoding="utf-8") if readme.exists() else ""
 
 setup(
     name="supertable",
-    version="2.3.4",
+    version="2.3.6",
     description="SuperTable — versioned data lake library for SQL analytics on Parquet + Redis.",
     long_description=long_description,
     long_description_content_type="text/markdown",

{supertable-2.3.4 → supertable-2.3.6}/supertable/__init__.py

@@ -25,7 +25,7 @@ See the ``supertable.demo`` package for runnable end-to-end demos and the
 project documentation for the full API surface.
 """
 
-__version__ = "2.3.4"
+__version__ = "2.3.6"
 
 # Re-export the core public surface so users can do ``from supertable import …``
 # instead of remembering submodule paths.

{supertable-2.3.4 → supertable-2.3.6}/supertable/config/defaults.py

@@ -17,6 +17,12 @@ handler.setFormatter(colorlog.ColoredFormatter(
 logging.basicConfig(level=logging.INFO, handlers=[handler])
 logger = logging.getLogger(__name__)
 
+# Quiet noisy third-party HTTP client loggers.  At DEBUG these emit one line
+# per request (connection setup + every HEAD/GET/PUT), which drowns out
+# SuperTable's own logs.  WARNING keeps genuine connection problems visible.
+for _noisy_logger in ("urllib3", "botocore", "boto3", "s3transfer", "boto"):
+    logging.getLogger(_noisy_logger).setLevel(logging.WARNING)
+
 _VALID_LOG_LEVELS = frozenset({"DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"})
 
 @dataclass(slots=True)

supertable-2.3.6/supertable/config/homedir.py

@@ -0,0 +1,96 @@
+import os
+import sys
+import tempfile
+
+from supertable.config.settings import settings
+from supertable.config.defaults import logger
+
+# If this file is located in a subdirectory, adjust the path logic as needed.
+# Currently appending ".." from __file__ to add the project root directory
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+
+# ---------- lazy home directory resolution ----------
+_resolved_home: str | None = None
+
+def _is_writable_dir(path: str) -> bool:
+    """Create *path* if needed and verify we can actually write a file in it.
+
+    ``os.access(..., W_OK)`` is unreliable under containers, ACLs and
+    root-squashed mounts, so probe with a real create+unlink: this is the
+    difference between a home that merely *resolves* and one DuckDB can root
+    its temp/spill, cache and extension dirs under.
+    """
+    try:
+        os.makedirs(path, exist_ok=True)
+        with tempfile.NamedTemporaryFile(dir=path):
+            pass
+        return True
+    except OSError:
+        return False
+
+def _resolve_app_home() -> str:
+    """
+    Resolve, expand, and normalise the application home directory once.
+
+    The home must be *writable*, not just resolvable: DuckDB roots its
+    temp/spill, external-cache and extension directories here, so a
+    non-writable home silently breaks every write (the probe fails with
+    ``errno 13`` and falls back to the slow full-read path).  We therefore
+    verify writability and, when the configured home is not usable, fall back
+    to ``<tempdir>/supertable`` with a loud warning rather than returning a
+    path that only looks valid.
+    """
+    global _resolved_home
+    if _resolved_home is not None:
+        return _resolved_home
+
+    raw = settings.SUPERTABLE_HOME
+    expanded = os.path.abspath(os.path.expanduser(raw))
+
+    if _is_writable_dir(expanded):
+        logger.debug(f"Ensured app home directory exists: {expanded}")
+        _resolved_home = expanded
+        return _resolved_home
+
+    fallback = os.path.join(tempfile.gettempdir(), "supertable")
+    if _is_writable_dir(fallback):
+        logger.warning(
+            f"SUPERTABLE_HOME={expanded!r} is not writable; falling back to "
+            f"{fallback!r}. Set SUPERTABLE_HOME to a writable directory to "
+            f"silence this — DuckDB temp/spill, cache and extensions live under it."
+        )
+        _resolved_home = fallback
+        return _resolved_home
+
+    raise RuntimeError(
+        f"No writable application home: tried SUPERTABLE_HOME={expanded!r} and "
+        f"fallback {fallback!r}. Set SUPERTABLE_HOME to a writable directory."
+    )
+
+def change_to_app_home(home_dir: str | None = None) -> None:
+    """
+    Attempts to change the current working directory to `home_dir`.
+    If home_dir is not provided, uses the resolved app home.
+    Logs the outcome.
+    """
+    target = home_dir if home_dir else _resolve_app_home()
+    expanded_dir = os.path.expanduser(target)
+    try:
+        os.chdir(expanded_dir)
+        logger.debug(f"Changed working directory to {expanded_dir}")
+    except Exception as e:
+        logger.error(f"Failed to change working directory to {expanded_dir}: {e}")
+
+# ---------- eager init (preserves original import-time behaviour) ----------
+_app_home = _resolve_app_home()
+change_to_app_home(_app_home)
+logger.debug(f"Current working directory: {os.getcwd()}")
+
+# ---------- public API ----------
+
+# Kept for backward compatibility; prefer get_app_home() for the expanded path.
+app_home = _app_home
+
+def get_app_home() -> str:
+    """Return the fully expanded, absolute application home directory."""
+    return _resolve_app_home()

{supertable-2.3.4 → supertable-2.3.6}/supertable/data_writer.py

@@ -5,6 +5,7 @@ import json
 import os
 import time
 import uuid
+from concurrent.futures import ThreadPoolExecutor
 from datetime import datetime, timezone
 import re
 
@@ -343,8 +344,18 @@ class DataWriter:
             # layout and tight row-group zonemaps). Together with __rowid__ it
             # is hidden from query output by the read view's
             # ``EXCLUDE (__rowid__, __timestamp__)`` projection.
+            #
+            # System-owned, exactly like __rowid__ above: ALWAYS overwrite any
+            # caller-supplied __timestamp__ instead of preserving it.  It is a
+            # reserved internal column that is both the dedup ORDER BY key (newest
+            # per key wins) and the source of the __p_year__/month/day partition
+            # derivation (processing.py); letting a caller inject an arbitrary value
+            # (wrong dtype, non-UTC, or chosen to game which row wins) would
+            # silently corrupt partitioning and dedup.  ``newer_than`` is the
+            # supported, explicit mechanism for caller-controlled conflict
+            # resolution.
             table_config = self._get_table_config(simple_name)
-            if not delete_only and "__timestamp__" not in dataframe.columns:
+            if not delete_only:
                 dataframe = dataframe.with_columns(
                     polars.lit(datetime.now(timezone.utc)).alias("__timestamp__")
                 )
@@ -371,6 +382,14 @@ class DataWriter:
                 profiler=profiler,
             )
             mark("overlap")
+            if overwrite_columns:
+                _snap_files = len(last_simple_table.get("resources") or [])
+                _cand = sum(1 for _, ov, _ in overlapping_files if ov)
+                logger.debug(lp(
+                    f"step[overlap]: {_cand}/{_snap_files} existing file(s) are overwrite "
+                    f"candidates on {overwrite_columns} "
+                    f"(snapshot has no per-file key stats → every file is suspect)"
+                ))
 
             # --- Stats-driven file pruning (consumer 5a) ----------------------
             # Narrow the overwrite/delete candidate set using the external stats
@@ -387,6 +406,10 @@ class DataWriter:
                     stored_stats_df = load_stats(stats_file, allow_cache=True, profiler=profiler)
                     if stored_stats_df is not None and stored_stats_df.height > 0:
                         probe = probe_ranges_from_df(dataframe, overwrite_columns)
+                        _probe_desc = {
+                            c: (f"{v[0]}[{v[1]}..{v[2]}]" if v else "unconstrained(null/unsupported)")
+                            for c, v in probe.items()
+                        }
                         before = len(overlapping_files)
                         overlapping_files = prune_overlapping_files_by_stats(
                             overlapping_files,
@@ -395,8 +418,21 @@ class DataWriter:
                             profiler=profiler,
                         )
                         pruned = before - len(overlapping_files)
+                        logger.debug(lp(
+                            f"step[stats-prune]: df-probe {_probe_desc} vs {stored_stats_df.height} "
+                            f"stored stat row(s) → kept {len(overlapping_files)}/{before}, "
+                            f"pruned {pruned} (no data file opened)"
+                        ))
                         if pruned > 0:
                             logger.info(lp(f"stats pruning: skipped {pruned}/{before} candidate files"))
+                    else:
+                        logger.debug(lp(
+                            "step[stats-prune]: stats artifact empty → no pruning, all candidates retained"
+                        ))
+                else:
+                    logger.debug(lp(
+                        "step[stats-prune]: snapshot has no stats_file → no pruning, all candidates retained"
+                    ))
                 mark("stats_prune")
 
             # File cache: used only by delete_only's identify_all_rowids below.
@@ -420,6 +456,16 @@ class DataWriter:
                     newer_than_col=newer_than,
                     profiler=profiler,
                 )
+                mark("resolve_overwrite")
+                _counts = profiler.counts
+                _fallback = bool(_counts.get("overwrite_resolve_fallback"))
+                logger.debug(lp(
+                    f"step[probe-resolve] via {'polars-fallback' if _fallback else 'duckdb-pushdown'}: "
+                    f"matched {_counts.get('probe_rows_matched', _counts.get('delete_rows_matched', 0))} "
+                    f"existing row(s) on {overwrite_columns} → "
+                    f"{len(resolved_delete_pairs or [])} (file,__rowid__) delete pair(s); "
+                    f"{dataframe.height}/{pre_filter_count} incoming row(s) survive"
+                ))
                 if newer_than:
                     skipped = pre_filter_count - dataframe.height
                     if skipped > 0:
@@ -476,12 +522,21 @@ class DataWriter:
                 # Load the current deletion-vector once: used both to exclude
                 # already-tombstoned rows from this write's deletes (below) and,
                 # via prev_df, to extend the vector without a second read.
+                # required=True: a DV that exists but cannot be read must abort
+                # the write, never be treated as empty — silently dropping the
+                # carried-forward vector would resurrect previously deleted rows.
                 prev_dv_df = (
-                    _read_parquet_safe(prev_tombstone_path, profiler=profiler)
+                    _read_parquet_safe(prev_tombstone_path, profiler=profiler, required=True)
                     if prev_tombstone_path else None
                 )
+                # The rowid set is consumed only by the idempotency filter below,
+                # which runs only when this write actually tombstones rows
+                # (overwrite or delete_only).  Pure appends tombstone nothing, so
+                # skip materialising the whole deletion-vector as a Python set —
+                # prev_dv_df is still carried forward into build_tombstone_file.
                 prev_dv_rowids = set()
-                if prev_dv_df is not None and "__rowid__" in prev_dv_df.columns:
+                if (overwrite_columns or delete_only) and prev_dv_df is not None \
+                        and "__rowid__" in prev_dv_df.columns:
                     prev_dv_rowids = set(prev_dv_df.get_column("__rowid__").to_list())
 
                 # 1. Identify which existing rows this write deletes/replaces.
@@ -515,34 +570,91 @@ class DataWriter:
                     ]
                 deleted = len(new_delete_pairs)
                 mark("identify_deletes")
+                logger.debug(lp(
+                    f"step[deletes]: tombstoning {deleted} live row(s) this write "
+                    f"(excluded {len(prev_dv_rowids)} row(s) already in the deletion-vector)"
+                ))
 
-                # 2. Write the incoming rows as a new file (insert/upsert side).
-                #    delete_only carries only predicate columns — nothing to insert.
-                if not delete_only and dataframe.height > 0:
+                # 2. + 3.  Write the incoming rows as a new data file (insert/
+                #    upsert side) AND carry-forward/extend the deletion-vector
+                #    tombstone file.  These two object-store PUTs are independent:
+                #    neither reads the other's output and they write to disjoint
+                #    dirs (data/ vs tombstone/), so they run concurrently to
+                #    overlap the two round-trips.  delete_only carries only
+                #    predicate columns → nothing to insert.  No new deletes →
+                #    build_tombstone reuses the previous file (combined_df=None).
+                #
+                #    Profiler is NOT thread-safe, so each branch records into its
+                #    own sub-profiler which the parent merges after the join;
+                #    each branch also measures its own wall time so the per-phase
+                #    monitoring timings stay meaningful despite the overlap.
+                #    Footers of files written via the write_bytes path are captured
+                #    in footer_md_cache so stats extraction (step 6) reuses them
+                #    instead of re-downloading each freshly-written file.
+                footer_md_cache = {}
+                tombstone_dir = os.path.join(simple_table.simple_dir, "tombstone")
+                do_insert = (not delete_only and dataframe.height > 0)
+
+                def _write_data_branch():
+                    sub = Profiler()
+                    t = time.perf_counter()
                     write_parquet_and_collect_resources(
                         write_df=dataframe,
                         overwrite_columns=[],
                         data_dir=simple_table.data_dir,
                         new_resources=new_resources,
                         compression_level=compression_level,
-                        profiler=profiler,
+                        profiler=sub,
+                        footer_md_out=footer_md_cache,
                     )
+                    return sub, time.perf_counter() - t
+
+                def _write_tombstone_branch():
+                    sub = Profiler()
+                    t = time.perf_counter()
+                    tp, cdf = build_tombstone_file(
+                        tombstone_dir=tombstone_dir,
+                        prev_tombstone_path=prev_tombstone_path,
+                        new_pairs=new_delete_pairs,
+                        compression_level=compression_level,
+                        profiler=sub,
+                        prev_df=prev_dv_df,
+                    )
+                    return tp, cdf, sub, time.perf_counter() - t
+
+                if do_insert:
+                    with ThreadPoolExecutor(max_workers=2) as _ex:
+                        _f_data = _ex.submit(_write_data_branch)
+                        _f_tomb = _ex.submit(_write_tombstone_branch)
+                        # .result() re-raises in the parent: a failure in either
+                        # PUT aborts the write before any snapshot commit, exactly
+                        # as the former sequential path did (an orphaned immutable
+                        # file no snapshot references is harmless garbage).
+                        data_sub, data_secs = _f_data.result()
+                        tombstone_path, combined_tombstone_df, tomb_sub, tomb_secs = (
+                            _f_tomb.result()
+                        )
+                    profiler.merge(data_sub)
+                    profiler.merge(tomb_sub)
                     inserted = dataframe.height
                 else:
+                    tombstone_path, combined_tombstone_df, tomb_sub, tomb_secs = (
+                        _write_tombstone_branch()
+                    )
+                    profiler.merge(tomb_sub)
+                    data_secs = 0.0
                     inserted = 0
-                mark("write_parquet")
 
-                # 3. Carry forward + extend the deletion-vector tombstone file.
-                #    No new deletes → reuse the previous file (combined_df=None).
-                tombstone_dir = os.path.join(simple_table.simple_dir, "tombstone")
-                tombstone_path, combined_tombstone_df = build_tombstone_file(
-                    tombstone_dir=tombstone_dir,
-                    prev_tombstone_path=prev_tombstone_path,
-                    new_pairs=new_delete_pairs,
-                    compression_level=compression_level,
-                    profiler=profiler,
-                    prev_df=prev_dv_df,
-                )
+                # Assign the two per-phase timings from each branch's own measured
+                # wall time (they overlapped, so the serial mark() deltas would
+                # misattribute the time), then advance the mark() baseline.
+                timings["write_parquet"] = data_secs
+                timings["build_tombstone"] = tomb_secs
+                t_last = time.time()
+                logger.debug(lp(
+                    f"step[write]: appended {inserted} incoming row(s) as {len(new_resources)} "
+                    f"new immutable file(s) (no existing data file rewritten)"
+                ))
 
                 # Track the live deletion-vector row count so meta reads can
                 # deduct dead rows from the physical resource row totals.
@@ -553,7 +665,10 @@ class DataWriter:
                     if combined_tombstone_df is not None
                     else int(last_simple_table.get("tombstone_rows", 0) or 0)
                 )
-                mark("build_tombstone")
+                logger.debug(lp(
+                    f"step[tombstone]: deletion-vector now {tombstone_rows} row(s) "
+                    f"({'rewritten' if combined_tombstone_df is not None else 'carried forward unchanged'})"
+                ))
 
                 # 3b. Eager reclamation of fully-dead files.  Any existing data
                 #     file whose every physical row is now tombstoned is 100%
@@ -698,7 +813,9 @@ class DataWriter:
                     r.get("file") for r in new_resources
                     if isinstance(r, dict) and r.get("file")
                 ]
-                new_stats_rows = extract_stats_rows(new_data_files, profiler=profiler)
+                new_stats_rows = extract_stats_rows(
+                    new_data_files, profiler=profiler, footer_md_cache=footer_md_cache
+                )
                 stats_path, combined_stats_df = build_stats_file(
                     stats_dir=stats_dir,
                     prev_stats_path=last_simple_table.get("stats_file"),
@@ -812,7 +929,7 @@ class DataWriter:
                         schema_json = "{}"
                     _org, _sup = self.super_table.organization, self.super_table.super_name
                     self.catalog.r.set(RK.schema(_org, _sup, simple_name), schema_json)
-                    self.catalog.r.sadd(RK.table_names(_org, _sup), simple_name)
+                    self.catalog.r.sadd(RK.meta_table_names(_org, _sup), simple_name)
                 except Exception as e:
                     logger.debug(f"[data-writer] schema/table_names Redis write failed: {e}")
 
@@ -862,7 +979,7 @@ class DataWriter:
                         f"total={total_duration:.3f} | "
                         f"convert={timings.get('convert', 0):.3f} | dedup_ts={timings.get('dedup_ts', 0):.3f} | validate={timings.get('validate', 0):.3f} | "
                         f"lock={timings.get('lock', 0):.3f} | snapshot={timings.get('snapshot', 0):.3f} | "
-                        f"overlap={timings.get('overlap', 0):.3f} | stats_prune={timings.get('stats_prune', 0):.3f} | newer_than={timings.get('newer_than', 0):.3f} | "
+                        f"overlap={timings.get('overlap', 0):.3f} | stats_prune={timings.get('stats_prune', 0):.3f} | resolve_overwrite={timings.get('resolve_overwrite', 0):.3f} | newer_than={timings.get('newer_than', 0):.3f} | "
                         f"identify_deletes={timings.get('identify_deletes', 0):.3f} | write_parquet={timings.get('write_parquet', 0):.3f} | "
                         f"build_tombstone={timings.get('build_tombstone', 0):.3f} | compact_tombstones={timings.get('compact_tombstones', 0):.3f} | compact_small={timings.get('compact_small', 0):.3f} | build_stats={timings.get('build_stats', 0):.3f} | "
                         f"update_simple={timings.get('update_simple', 0):.3f} | bump_root={timings.get('bump_root', 0):.3f} | "
@@ -1134,8 +1251,17 @@ class DataWriter:
             # the *write* path; compact() is explicit maintenance and always
             # consumes the vector.
             tombstone_path = last_simple_table.get("tombstone")
+            # required=True: a DV that exists but cannot be read must abort the
+            # compaction, never be treated as empty. A swallowed read here would
+            # set should_run_tombstones=False, skipping both Phase A and the
+            # pointer-clear below, so Phase B would carry the dead rows into the
+            # new file while the vector kept pointing at the sunset __file__ —
+            # leaving them permanently unreclaimable. Failing loud leaves the
+            # prior snapshot + vector intact for a retry, and matches the
+            # write-path carry-forward read (required=True) above.
             tombstone_df = (
-                _read_parquet_safe(tombstone_path) if tombstone_path else None
+                _read_parquet_safe(tombstone_path, required=True)
+                if tombstone_path else None
             )
             tombstone_rows = (
                 tombstone_df.height if tombstone_df is not None else 0
@@ -1199,6 +1325,24 @@ class DataWriter:
                 r for r in (list(tomb_new_resources) + list(small_new_resources))
                 if r.get("file") not in all_sunset
             ]
+            # ``all_new_resources`` is the full set of files written by THIS
+            # compaction; it feeds stats extraction, the schema model_df and the
+            # result metrics below, all of which need every new file.
+            #
+            # For ``simple_table.update`` it must NOT be reused verbatim, though:
+            # Phase A's outputs were already spliced into
+            # ``last_simple_table["resources"]`` (the in-memory baseline that
+            # ``update`` starts from) right after Phase A ran.  ``update`` does
+            # ``(baseline - sunset) + new_resources`` with no dedup, so any
+            # Phase-A output that Phase B did NOT consume (left un-sunset because
+            # it exceeded the ``small_only`` threshold, or its read failed) would
+            # be counted once from the baseline AND once from new_resources —
+            # i.e. the same file listed twice in the new snapshot.  Hand ``update``
+            # only Phase B's brand-new files, which are the only resources genuinely
+            # absent from that baseline.
+            update_new_resources = [
+                r for r in small_new_resources if r.get("file") not in all_sunset
+            ]
             result["files_compacted"] = considered
             result["new_resources"] = len(all_new_resources)
             result["sunset_files"] = len(all_sunset)
@@ -1291,7 +1435,7 @@ class DataWriter:
                 )
 
                 new_snapshot_dict, new_snapshot_path = simple_table.update(
-                    all_new_resources,
+                    update_new_resources,
                     all_sunset,
                     model_df,
                     last_snapshot=last_simple_table,

{supertable-2.3.4 → supertable-2.3.6}/supertable/demo/quickstart/defaults.py

@@ -14,7 +14,9 @@ from enum import Enum
 
 from supertable.config import defaults
 
-logging.getLogger("supertable").setLevel(logging.INFO)
+# Follow the configured SUPERTABLE_LOG_LEVEL (resolved in supertable.config.defaults)
+# instead of hard-pinning INFO, so DEBUG surfaces the detailed write step[...] logs.
+logging.getLogger("supertable").setLevel(defaults.default.LOG_LEVEL)
 
 defaults.default.IS_SHOW_TIMING = True