supertable 2.2.1__tar.gz → 2.2.3__tar.gz

{supertable-2.2.1/supertable.egg-info → supertable-2.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: supertable
-Version: 2.2.1
+Version: 2.2.3
 Summary: SuperTable — versioned data lake library for SQL analytics on Parquet + Redis.
 Author: Levente Kupas
 Author-email: Levente Kupas <lkupas@kladnasoft.com>

{supertable-2.2.1 → supertable-2.2.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "supertable"
-version = "2.2.1"
+version = "2.2.3"
 description = "SuperTable — versioned data lake library for SQL analytics on Parquet + Redis."
 readme = "README.md"
 requires-python = ">=3.10"

{supertable-2.2.1 → supertable-2.2.3}/setup.py

@@ -19,7 +19,7 @@ long_description = readme.read_text(encoding="utf-8") if readme.exists() else ""
 
 setup(
     name="supertable",
-    version="2.2.1",
+    version="2.2.3",
     description="SuperTable — versioned data lake library for SQL analytics on Parquet + Redis.",
     long_description=long_description,
     long_description_content_type="text/markdown",

{supertable-2.2.1 → supertable-2.2.3}/supertable/__init__.py

@@ -25,7 +25,7 @@ See the ``supertable.demo`` package for runnable end-to-end demos and the
 project documentation for the full API surface.
 """
 
-__version__ = "2.2.1"
+__version__ = "2.2.3"
 
 # Re-export the core public surface so users can do ``from supertable import …``
 # instead of remembering submodule paths.

{supertable-2.2.1 → supertable-2.2.3}/supertable/config/settings.py

@@ -282,21 +282,6 @@ class Settings:
     SUPERTABLE_SHARE_PRESIGN_TTL: int = 14400                # SUPERTABLE_SHARE_PRESIGN_TTL (seconds, default 4h)
     SUPERTABLE_SHARE_REFRESH_BUFFER: int = 600               # SUPERTABLE_SHARE_REFRESH_BUFFER (seconds, refresh 10min before expiry)
 
-    # ── Storage GC (deferred deletion of sunset parquets + old snapshot JSONs) ──
-    # Sunset parquet files and old snapshot JSONs are never deleted by the
-    # writer directly. When either flag is enabled, the writer XADDs paths
-    # to a per-table Redis STREAM after a successful leaf-CAS. A separate
-    # cleaner daemon (``python -m supertable.gc.cleaner``) drains entries
-    # older than ``SUPERTABLE_GC_DELAY_SEC`` and calls ``storage.delete()``.
-    # The delay window avoids the race where an in-flight reader's
-    # ``parquet_scan([...])`` resolves a file path right before the writer
-    # deletes it. Defaults below preserve today's behaviour (off, unlimited).
-    SUPERTABLE_SNAPSHOT_RETENTION: int = 0                   # SUPERTABLE_SNAPSHOT_RETENTION (0 = keep all snapshot JSONs)
-    SUPERTABLE_SUNSET_GC_ENABLED: bool = False               # SUPERTABLE_SUNSET_GC_ENABLED (delete sunset parquets via queue)
-    SUPERTABLE_GC_DELAY_SEC: int = 1800                      # SUPERTABLE_GC_DELAY_SEC (min age before cleaner deletes)
-    SUPERTABLE_GC_SLEEP_SEC: int = 60                        # SUPERTABLE_GC_SLEEP_SEC (cleaner loop sleep)
-    SUPERTABLE_GC_BATCH_SIZE: int = 500                      # SUPERTABLE_GC_BATCH_SIZE (max entries processed per table per tick)
-
 
     # ── Convenience properties ───────────────────────────────────────
 
@@ -543,13 +528,6 @@ def _build_settings() -> Settings:
         # Data Sharing
         SUPERTABLE_SHARE_PRESIGN_TTL=_env_int("SUPERTABLE_SHARE_PRESIGN_TTL", 14400),
         SUPERTABLE_SHARE_REFRESH_BUFFER=_env_int("SUPERTABLE_SHARE_REFRESH_BUFFER", 600),
-
-        # ── Storage GC ───────────────────────────────────────────────
-        SUPERTABLE_SNAPSHOT_RETENTION=_env_int("SUPERTABLE_SNAPSHOT_RETENTION", 0),
-        SUPERTABLE_SUNSET_GC_ENABLED=_env_bool("SUPERTABLE_SUNSET_GC_ENABLED", False),
-        SUPERTABLE_GC_DELAY_SEC=_env_int("SUPERTABLE_GC_DELAY_SEC", 1800),
-        SUPERTABLE_GC_SLEEP_SEC=_env_int("SUPERTABLE_GC_SLEEP_SEC", 60),
-        SUPERTABLE_GC_BATCH_SIZE=_env_int("SUPERTABLE_GC_BATCH_SIZE", 500),
     )
 
 

{supertable-2.2.1 → supertable-2.2.3}/supertable/data_writer.py

@@ -13,13 +13,13 @@ from polars import DataFrame
 from supertable.config.defaults import logger
 from supertable.config.settings import settings
 from supertable.errors import SuperTableNotFoundError, TableNotFoundError
-from supertable.gc.queue import collect_old_snapshot_paths, enqueue_deletions
 from supertable.monitoring.partitions import MONITORING_SINK_TABLES
 from supertable.monitoring_writer import MonitoringWriter  # async monitoring
 from supertable.super_table import SuperTable
 from supertable import redis_keys as RK
 from supertable.simple_table import SimpleTable
 from supertable.utils.timer import Timer
+from supertable.utils.profiler import Profiler
 from supertable.processing import (
     process_overlapping_files,
     process_delete_only,
@@ -277,7 +277,11 @@ class DataWriter:
 
         t0 = time.time()
         t_last = t0
-        timings = {}
+        profiler = Profiler()
+        # Convenience view: top-level stages still indexed by the original
+        # short names ("lock", "snapshot", "overlap", ...) so the existing
+        # log line keeps working unchanged.
+        timings = profiler.timings
 
         def mark(stage: str):
             nonlocal t_last
@@ -338,6 +342,7 @@ class DataWriter:
             overlapping_files = find_overlapping_files(
                 last_simple_table, dataframe, overwrite_columns, locking=None,
                 table_config=table_config,
+                profiler=profiler,
             )
             mark("overlap")
 
@@ -354,6 +359,7 @@ class DataWriter:
                     overwrite_columns=overwrite_columns,
                     newer_than_col=newer_than,
                     file_cache=file_cache,
+                    profiler=profiler,
                 )
                 skipped = pre_filter_count - dataframe.height
                 if skipped > 0:
@@ -385,6 +391,8 @@ class DataWriter:
                         "skipped_stale": skipped,
                         "lineage": _safe_json(lineage or {}),
                         "duration": round(time.time() - t0, 6),
+                        "timings": profiler.emit_timings(),
+                        "counts": profiler.emit_counts(),
                     }
                     # Don't return here — fall through to finally (lock release)
                     # and the post-finally monitoring block.  Returning inside the
@@ -431,6 +439,7 @@ class DataWriter:
                             data_dir=simple_table.data_dir,
                             compression_level=compression_level,
                             table_config=table_config,
+                            profiler=profiler,
                         )
                         new_resources = compact_resources
                         sunset_files = compact_sunset
@@ -474,6 +483,7 @@ class DataWriter:
                             simple_table.data_dir,
                             compression_level,
                             file_cache=file_cache,
+                            profiler=profiler,
                         )
                     else:
                         inserted, deleted, total_rows, total_columns, new_resources, sunset_files = process_overlapping_files(
@@ -484,6 +494,7 @@ class DataWriter:
                             compression_level,
                             file_cache=file_cache,
                             table_config=table_config,
+                            profiler=profiler,
                         )
                     mark("process")
 
@@ -514,6 +525,7 @@ class DataWriter:
                     last_snapshot=last_simple_table,
                     last_snapshot_path=last_simple_table_path,
                     lineage=eff_lineage,
+                    profiler=profiler,
                 )
                 mark("update_simple")
 
@@ -534,67 +546,29 @@ class DataWriter:
                     ],
                 }
 
-                try:
-                    self.catalog.set_leaf_payload_cas(
-                        self.super_table.organization,
-                        self.super_table.super_name,
-                        simple_name,
-                        payload,
-                        new_snapshot_path,
-                        now_ms=now_ms,
-                    )
-                except Exception:
-                    # Backward compatible fallback.
-                    self.catalog.set_leaf_path_cas(
-                        self.super_table.organization,
-                        self.super_table.super_name,
-                        simple_name,
-                        new_snapshot_path,
-                        now_ms=now_ms,
-                    )
-
-                self.catalog.bump_root(self.super_table.organization, self.super_table.super_name, now_ms=now_ms)
-                mark("bump_root")
-
-                # --- Enqueue deferred deletions (post-CAS, post-bump) ----------
-                # Strict ordering matters: leaf-CAS + root-bump have committed
-                # the new snapshot as the authoritative state. Only NOW is it
-                # safe to schedule physical deletion of files that the old
-                # snapshot still references — any reader that loaded the
-                # previous leaf payload will finish before the cleaner's delay
-                # window expires.
-                #
-                # Both blocks are best-effort: enqueue_deletions and
-                # collect_old_snapshot_paths swallow Redis/storage errors and
-                # log a warning. A GC failure must never fail the write.
-                try:
-                    _org = self.super_table.organization
-                    _sup = self.super_table.super_name
-                    if settings.SUPERTABLE_SUNSET_GC_ENABLED and sunset_files:
-                        enqueue_deletions(
-                            self.catalog,
-                            _org, _sup, simple_name,
-                            "parquet",
-                            list(sunset_files),
-                            write_id=qid,
+                with profiler.span("redis.set_leaf"):
+                    try:
+                        self.catalog.set_leaf_payload_cas(
+                            self.super_table.organization,
+                            self.super_table.super_name,
+                            simple_name,
+                            payload,
+                            new_snapshot_path,
+                            now_ms=now_ms,
                         )
-                    if settings.SUPERTABLE_SNAPSHOT_RETENTION > 0:
-                        old_paths = collect_old_snapshot_paths(
-                            new_snapshot_dict,
-                            self.super_table.storage,
-                            settings.SUPERTABLE_SNAPSHOT_RETENTION,
+                    except Exception:
+                        # Backward compatible fallback.
+                        self.catalog.set_leaf_path_cas(
+                            self.super_table.organization,
+                            self.super_table.super_name,
+                            simple_name,
+                            new_snapshot_path,
+                            now_ms=now_ms,
                         )
-                        if old_paths:
-                            enqueue_deletions(
-                                self.catalog,
-                                _org, _sup, simple_name,
-                                "snapshot",
-                                old_paths,
-                                write_id=qid,
-                            )
-                except Exception as e:
-                    logger.warning(lp(f"GC enqueue failed (write still succeeded): {e}"))
-                mark("gc_enqueue")
+
+                with profiler.span("redis.bump_root"):
+                    self.catalog.bump_root(self.super_table.organization, self.super_table.super_name, now_ms=now_ms)
+                mark("bump_root")
 
                 # --- Store schema + table name in Redis (permanent, not cache) ---
                 try:
@@ -649,6 +623,8 @@ class DataWriter:
                     "skipped_stale": 0,
                     "lineage": _safe_json(lineage or {}),
                     "duration": round(time.time() - t0, 6),
+                    "timings": profiler.emit_timings(),
+                    "counts": profiler.emit_counts(),
                 }
                 mark("prepare_monitor")
 
@@ -977,12 +953,10 @@ class DataWriter:
             # Critical filter: a file produced by Phase A (tombstone
             # compaction) can be picked up as a small-file candidate
             # by Phase B (compact_resources) and then sunset there.
-            # If we leave it in ``all_new_resources`` it lands in the
-            # new snapshot **and** in ``all_sunset`` (slated for GC
-            # deletion). 30 min later GC deletes the file while the
-            # snapshot still references it — readers fail with
-            # FileNotFoundError. Filter the file out of new_resources
-            # when it also appears in any sunset set.
+            # If we leave it in ``all_new_resources`` it would land in the
+            # new snapshot **and** in ``all_sunset`` at the same time.
+            # Filter the file out of new_resources when it also appears
+            # in any sunset set so the snapshot stays consistent.
             all_sunset = set(tomb_sunset) | set(small_sunset)
             all_new_resources = [
                 r for r in (list(tomb_new_resources) + list(small_new_resources))
@@ -994,7 +968,7 @@ class DataWriter:
             result["total_rows_written"] = small_total_rows
 
             # Short-circuit: nothing to do, skip the snapshot rewrite
-            # and the leaf-CAS / root-bump / GC / mirror path. BUT do
+            # and the leaf-CAS / root-bump / mirror path. BUT do
             # NOT return here — execution must fall through to the
             # ``finally`` (which releases the lock) AND the monitoring
             # + audit emission blocks after the try/finally so the
@@ -1098,36 +1072,6 @@ class DataWriter:
                 )
                 mark("bump_root")
 
-                # --- GC enqueue (same flags as write) ----------------------------
-                try:
-                    _org = self.super_table.organization
-                    _sup = self.super_table.super_name
-                    if settings.SUPERTABLE_SUNSET_GC_ENABLED and all_sunset:
-                        enqueue_deletions(
-                            self.catalog,
-                            _org, _sup, simple_name,
-                            "parquet",
-                            list(all_sunset),
-                            write_id=qid,
-                        )
-                    if settings.SUPERTABLE_SNAPSHOT_RETENTION > 0:
-                        old_paths = collect_old_snapshot_paths(
-                            new_snapshot_dict,
-                            self.super_table.storage,
-                            settings.SUPERTABLE_SNAPSHOT_RETENTION,
-                        )
-                        if old_paths:
-                            enqueue_deletions(
-                                self.catalog,
-                                _org, _sup, simple_name,
-                                "snapshot",
-                                old_paths,
-                                write_id=qid,
-                            )
-                except Exception as e:
-                    logger.warning(lp(f"GC enqueue failed (compact still succeeded): {e}"))
-                mark("gc_enqueue")
-
                 # --- Mirroring ---------------------------------------------------
                 try:
                     MirrorFormats.mirror_if_enabled(

{supertable-2.2.1 → supertable-2.2.3}/supertable/demo/webshop/defaults.py

@@ -10,7 +10,7 @@ Used by:
 
 # --- SuperTable connection settings ---
 organization: str = "kladna-soft"
-super_name: str = "example"
+super_name: str = "webshop"
 role_name: str = "superadmin"
 
 # --- Data generation / loading settings ---