supertable 2.2.2__tar.gz → 2.2.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {supertable-2.2.2/supertable.egg-info → supertable-2.2.4}/PKG-INFO +1 -1
- {supertable-2.2.2 → supertable-2.2.4}/pyproject.toml +1 -1
- {supertable-2.2.2 → supertable-2.2.4}/setup.py +1 -1
- {supertable-2.2.2 → supertable-2.2.4}/supertable/__init__.py +1 -1
- {supertable-2.2.2 → supertable-2.2.4}/supertable/config/settings.py +0 -22
- {supertable-2.2.2 → supertable-2.2.4}/supertable/data_writer.py +11 -79
- {supertable-2.2.2 → supertable-2.2.4}/supertable/redis_catalog.py +1 -6
- {supertable-2.2.2 → supertable-2.2.4}/supertable/redis_keys.py +0 -77
- {supertable-2.2.2 → supertable-2.2.4}/supertable/simple_table.py +50 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/tests/test_data_writer_compact.py +3 -82
- {supertable-2.2.2 → supertable-2.2.4}/supertable/tests/test_redis_key_prefix.py +0 -21
- {supertable-2.2.2 → supertable-2.2.4/supertable.egg-info}/PKG-INFO +1 -1
- {supertable-2.2.2 → supertable-2.2.4}/supertable.egg-info/SOURCES.txt +0 -7
- supertable-2.2.2/supertable/gc/__init__.py +0 -26
- supertable-2.2.2/supertable/gc/cleaner.py +0 -258
- supertable-2.2.2/supertable/gc/daemon.py +0 -284
- supertable-2.2.2/supertable/gc/queue.py +0 -221
- supertable-2.2.2/supertable/tests/test_gc_cleaner.py +0 -330
- supertable-2.2.2/supertable/tests/test_gc_daemon.py +0 -251
- supertable-2.2.2/supertable/tests/test_gc_queue.py +0 -331
- {supertable-2.2.2 → supertable-2.2.4}/LICENSE +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/README.md +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/requirements.txt +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/setup.cfg +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/audit/__init__.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/audit/admin.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/audit/chain.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/audit/consumers.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/audit/crypto.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/audit/events.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/audit/export.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/audit/logger.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/audit/middleware.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/audit/reader.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/audit/retention.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/audit/tests/__init__.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/audit/tests/test_chain.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/audit/tests/test_crypto.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/audit/tests/test_emit.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/audit/tests/test_events.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/audit/tests/test_retention.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/audit/writer_parquet.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/audit/writer_redis.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/config/__init__.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/config/defaults.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/config/homedir.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/config/tests/__init__.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/config/tests/test_defaults.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/config/tests/test_homedir.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/config/tests/test_settings.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/data_classes.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/data_reader.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/__init__.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/__init__.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/__main__.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/check_filter_builder.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/controller.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/data_writer_helpers.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/defaults.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/dummy_data.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/read_parquet_header.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/s01_01_01_create_super_table.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/s01_01_02_enable_mirroring_formats.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/s01_02_create_roles.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/s01_03_create_users.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/s02_01_write_dummy_data.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/s02_02_write_single_data.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/s02_03_01_write_staging.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/s02_03_02_create_pipe.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/s02_04_01_write_monitoring_simple.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/s02_04_02_write_monitoring_parallel.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/s02_05_write_tombstone.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/s03_01_read_data_error.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/s03_02_01_read_super_data_ok.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/s03_02_02_read_table_data_ok.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/s03_03_read_meta.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/s03_04_read_staging.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/s03_06_01_read_roles.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/s03_06_02_read_user.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/s03_07_01_estimate_read.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/s03_07_02_estimate_files.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/s03_08_read_snapshot_history.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/s04_01_03_delete_pipe.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/s05_01_delete_table.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/s05_02_delete_super_table.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/webshop/__init__.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/webshop/core.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/webshop/defaults.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/webshop/generate.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/webshop/load.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/webshop/topup.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/engine/__init__.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/engine/data_estimator.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/engine/duckdb_lite.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/engine/duckdb_pro.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/engine/engine_common.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/engine/engine_enum.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/engine/executor.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/engine/plan_stats.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/engine/spark_thrift.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/engine/tests/__init__.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/engine/tests/conftest.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/engine/tests/test_dedup_read.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/engine/tests/test_engine.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/errors.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/locking/__init__.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/locking/benchmarks/__init__.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/locking/benchmarks/benchmark_locking.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/locking/benchmarks/measure_lock_speed.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/locking/benchmarks/measure_lock_time.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/locking/file_lock.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/locking/redis_lock.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/locking/tests/__init__.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/locking/tests/test_file_lock.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/locking/tests/test_redis_lock.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/logging.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/meta_reader.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/mirroring/__init__.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/mirroring/mirror_delta.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/mirroring/mirror_formats.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/mirroring/mirror_iceberg.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/mirroring/mirror_parquet.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/monitoring/__init__.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/monitoring/partitions.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/monitoring_writer.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/plan_extender.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/processing.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/query_plan_manager.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/rbac/__init__.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/rbac/access_control.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/rbac/filter_builder.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/rbac/permissions.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/rbac/role_manager.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/rbac/row_column_security.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/rbac/tests/test_filter_builder.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/rbac/tests/test_rbac.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/rbac/tests/test_rbac_per_table.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/rbac/user_manager.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/redis_connector.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/redis_infra.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/staging_area.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/storage/__init__.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/storage/azure_storage.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/storage/gcp_storage.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/storage/local_storage.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/storage/minio_storage.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/storage/s3_storage.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/storage/storage_factory.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/storage/storage_interface.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/storage/tests/test_storage.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/super_pipe.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/super_table.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/tests/__init__.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/tests/test_align_to_schema_fix.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/tests/test_create_if_missing.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/tests/test_data_reader.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/tests/test_data_reader_preflight.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/tests/test_data_writer.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/tests/test_data_writer_comprehensive.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/tests/test_data_writer_tombstones.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/tests/test_dedup_on_read_write.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/tests/test_errors.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/tests/test_meta_reader.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/tests/test_monitoring_partitions.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/tests/test_monitoring_sink_guard.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/tests/test_newer_than.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/tests/test_process_delete_only.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/tests/test_processing.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/tests/test_processing_compact_resources.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/tests/test_query_sql.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/tests/test_simple_table.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/tests/test_small_file_compaction.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/tests/test_super_table.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/tests/test_supertable_all.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/utils/__init__.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/utils/helper.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/utils/profiler.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/utils/sql_parser.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/utils/tests/test_sql_parser_columns.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable/utils/timer.py +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable.egg-info/dependency_links.txt +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable.egg-info/entry_points.txt +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable.egg-info/requires.txt +0 -0
- {supertable-2.2.2 → supertable-2.2.4}/supertable.egg-info/top_level.txt +0 -0
|
@@ -19,7 +19,7 @@ long_description = readme.read_text(encoding="utf-8") if readme.exists() else ""
|
|
|
19
19
|
|
|
20
20
|
setup(
|
|
21
21
|
name="supertable",
|
|
22
|
-
version="2.2.
|
|
22
|
+
version="2.2.4",
|
|
23
23
|
description="SuperTable — versioned data lake library for SQL analytics on Parquet + Redis.",
|
|
24
24
|
long_description=long_description,
|
|
25
25
|
long_description_content_type="text/markdown",
|
|
@@ -25,7 +25,7 @@ See the ``supertable.demo`` package for runnable end-to-end demos and the
|
|
|
25
25
|
project documentation for the full API surface.
|
|
26
26
|
"""
|
|
27
27
|
|
|
28
|
-
__version__ = "2.2.
|
|
28
|
+
__version__ = "2.2.4"
|
|
29
29
|
|
|
30
30
|
# Re-export the core public surface so users can do ``from supertable import …``
|
|
31
31
|
# instead of remembering submodule paths.
|
|
@@ -282,21 +282,6 @@ class Settings:
|
|
|
282
282
|
SUPERTABLE_SHARE_PRESIGN_TTL: int = 14400 # SUPERTABLE_SHARE_PRESIGN_TTL (seconds, default 4h)
|
|
283
283
|
SUPERTABLE_SHARE_REFRESH_BUFFER: int = 600 # SUPERTABLE_SHARE_REFRESH_BUFFER (seconds, refresh 10min before expiry)
|
|
284
284
|
|
|
285
|
-
# ── Storage GC (deferred deletion of sunset parquets + old snapshot JSONs) ──
|
|
286
|
-
# Sunset parquet files and old snapshot JSONs are never deleted by the
|
|
287
|
-
# writer directly. When either flag is enabled, the writer XADDs paths
|
|
288
|
-
# to a per-table Redis STREAM after a successful leaf-CAS. A separate
|
|
289
|
-
# cleaner daemon (``python -m supertable.gc.cleaner``) drains entries
|
|
290
|
-
# older than ``SUPERTABLE_GC_DELAY_SEC`` and calls ``storage.delete()``.
|
|
291
|
-
# The delay window avoids the race where an in-flight reader's
|
|
292
|
-
# ``parquet_scan([...])`` resolves a file path right before the writer
|
|
293
|
-
# deletes it. Defaults below preserve today's behaviour (off, unlimited).
|
|
294
|
-
SUPERTABLE_SNAPSHOT_RETENTION: int = 0 # SUPERTABLE_SNAPSHOT_RETENTION (0 = keep all snapshot JSONs)
|
|
295
|
-
SUPERTABLE_SUNSET_GC_ENABLED: bool = False # SUPERTABLE_SUNSET_GC_ENABLED (delete sunset parquets via queue)
|
|
296
|
-
SUPERTABLE_GC_DELAY_SEC: int = 1800 # SUPERTABLE_GC_DELAY_SEC (min age before cleaner deletes)
|
|
297
|
-
SUPERTABLE_GC_SLEEP_SEC: int = 60 # SUPERTABLE_GC_SLEEP_SEC (cleaner loop sleep)
|
|
298
|
-
SUPERTABLE_GC_BATCH_SIZE: int = 500 # SUPERTABLE_GC_BATCH_SIZE (max entries processed per table per tick)
|
|
299
|
-
|
|
300
285
|
|
|
301
286
|
# ── Convenience properties ───────────────────────────────────────
|
|
302
287
|
|
|
@@ -543,13 +528,6 @@ def _build_settings() -> Settings:
|
|
|
543
528
|
# Data Sharing
|
|
544
529
|
SUPERTABLE_SHARE_PRESIGN_TTL=_env_int("SUPERTABLE_SHARE_PRESIGN_TTL", 14400),
|
|
545
530
|
SUPERTABLE_SHARE_REFRESH_BUFFER=_env_int("SUPERTABLE_SHARE_REFRESH_BUFFER", 600),
|
|
546
|
-
|
|
547
|
-
# ── Storage GC ───────────────────────────────────────────────
|
|
548
|
-
SUPERTABLE_SNAPSHOT_RETENTION=_env_int("SUPERTABLE_SNAPSHOT_RETENTION", 0),
|
|
549
|
-
SUPERTABLE_SUNSET_GC_ENABLED=_env_bool("SUPERTABLE_SUNSET_GC_ENABLED", False),
|
|
550
|
-
SUPERTABLE_GC_DELAY_SEC=_env_int("SUPERTABLE_GC_DELAY_SEC", 1800),
|
|
551
|
-
SUPERTABLE_GC_SLEEP_SEC=_env_int("SUPERTABLE_GC_SLEEP_SEC", 60),
|
|
552
|
-
SUPERTABLE_GC_BATCH_SIZE=_env_int("SUPERTABLE_GC_BATCH_SIZE", 500),
|
|
553
531
|
)
|
|
554
532
|
|
|
555
533
|
|
|
@@ -13,7 +13,6 @@ from polars import DataFrame
|
|
|
13
13
|
from supertable.config.defaults import logger
|
|
14
14
|
from supertable.config.settings import settings
|
|
15
15
|
from supertable.errors import SuperTableNotFoundError, TableNotFoundError
|
|
16
|
-
from supertable.gc.queue import collect_old_snapshot_paths, enqueue_deletions
|
|
17
16
|
from supertable.monitoring.partitions import MONITORING_SINK_TABLES
|
|
18
17
|
from supertable.monitoring_writer import MonitoringWriter # async monitoring
|
|
19
18
|
from supertable.super_table import SuperTable
|
|
@@ -571,46 +570,6 @@ class DataWriter:
|
|
|
571
570
|
self.catalog.bump_root(self.super_table.organization, self.super_table.super_name, now_ms=now_ms)
|
|
572
571
|
mark("bump_root")
|
|
573
572
|
|
|
574
|
-
# --- Enqueue deferred deletions (post-CAS, post-bump) ----------
|
|
575
|
-
# Strict ordering matters: leaf-CAS + root-bump have committed
|
|
576
|
-
# the new snapshot as the authoritative state. Only NOW is it
|
|
577
|
-
# safe to schedule physical deletion of files that the old
|
|
578
|
-
# snapshot still references — any reader that loaded the
|
|
579
|
-
# previous leaf payload will finish before the cleaner's delay
|
|
580
|
-
# window expires.
|
|
581
|
-
#
|
|
582
|
-
# Both blocks are best-effort: enqueue_deletions and
|
|
583
|
-
# collect_old_snapshot_paths swallow Redis/storage errors and
|
|
584
|
-
# log a warning. A GC failure must never fail the write.
|
|
585
|
-
try:
|
|
586
|
-
_org = self.super_table.organization
|
|
587
|
-
_sup = self.super_table.super_name
|
|
588
|
-
if settings.SUPERTABLE_SUNSET_GC_ENABLED and sunset_files:
|
|
589
|
-
enqueue_deletions(
|
|
590
|
-
self.catalog,
|
|
591
|
-
_org, _sup, simple_name,
|
|
592
|
-
"parquet",
|
|
593
|
-
list(sunset_files),
|
|
594
|
-
write_id=qid,
|
|
595
|
-
)
|
|
596
|
-
if settings.SUPERTABLE_SNAPSHOT_RETENTION > 0:
|
|
597
|
-
old_paths = collect_old_snapshot_paths(
|
|
598
|
-
new_snapshot_dict,
|
|
599
|
-
self.super_table.storage,
|
|
600
|
-
settings.SUPERTABLE_SNAPSHOT_RETENTION,
|
|
601
|
-
)
|
|
602
|
-
if old_paths:
|
|
603
|
-
enqueue_deletions(
|
|
604
|
-
self.catalog,
|
|
605
|
-
_org, _sup, simple_name,
|
|
606
|
-
"snapshot",
|
|
607
|
-
old_paths,
|
|
608
|
-
write_id=qid,
|
|
609
|
-
)
|
|
610
|
-
except Exception as e:
|
|
611
|
-
logger.warning(lp(f"GC enqueue failed (write still succeeded): {e}"))
|
|
612
|
-
mark("gc_enqueue")
|
|
613
|
-
|
|
614
573
|
# --- Store schema + table name in Redis (permanent, not cache) ---
|
|
615
574
|
try:
|
|
616
575
|
schema_raw = new_snapshot_dict.get("schema", {})
|
|
@@ -795,9 +754,14 @@ class DataWriter:
|
|
|
795
754
|
qualify — large files are left untouched. ``small_only=False``
|
|
796
755
|
rewrites every resource regardless of size.
|
|
797
756
|
5. Commit the new snapshot via ``simple_table.update``,
|
|
798
|
-
leaf-CAS, ``bump_root``, and the same
|
|
757
|
+
leaf-CAS, ``bump_root``, optional mirroring, and the same
|
|
799
758
|
monitoring + audit pipeline as ``write``.
|
|
800
759
|
|
|
760
|
+
Compaction does **not** garbage-collect: the small files it
|
|
761
|
+
merges away are left in storage (still referenced by older
|
|
762
|
+
snapshot versions for time-travel) for an external GC to
|
|
763
|
+
reclaim later.
|
|
764
|
+
|
|
801
765
|
Concurrency: compaction is serialised against writes through
|
|
802
766
|
the same per-simple lock. A concurrent ``write()`` either
|
|
803
767
|
runs first (compaction sees the updated snapshot) or waits.
|
|
@@ -994,12 +958,10 @@ class DataWriter:
|
|
|
994
958
|
# Critical filter: a file produced by Phase A (tombstone
|
|
995
959
|
# compaction) can be picked up as a small-file candidate
|
|
996
960
|
# by Phase B (compact_resources) and then sunset there.
|
|
997
|
-
# If we leave it in ``all_new_resources`` it
|
|
998
|
-
# new snapshot **and** in ``all_sunset``
|
|
999
|
-
#
|
|
1000
|
-
#
|
|
1001
|
-
# FileNotFoundError. Filter the file out of new_resources
|
|
1002
|
-
# when it also appears in any sunset set.
|
|
961
|
+
# If we leave it in ``all_new_resources`` it would land in the
|
|
962
|
+
# new snapshot **and** in ``all_sunset`` at the same time.
|
|
963
|
+
# Filter the file out of new_resources when it also appears
|
|
964
|
+
# in any sunset set so the snapshot stays consistent.
|
|
1003
965
|
all_sunset = set(tomb_sunset) | set(small_sunset)
|
|
1004
966
|
all_new_resources = [
|
|
1005
967
|
r for r in (list(tomb_new_resources) + list(small_new_resources))
|
|
@@ -1011,7 +973,7 @@ class DataWriter:
|
|
|
1011
973
|
result["total_rows_written"] = small_total_rows
|
|
1012
974
|
|
|
1013
975
|
# Short-circuit: nothing to do, skip the snapshot rewrite
|
|
1014
|
-
# and the leaf-CAS / root-bump /
|
|
976
|
+
# and the leaf-CAS / root-bump / mirror path. BUT do
|
|
1015
977
|
# NOT return here — execution must fall through to the
|
|
1016
978
|
# ``finally`` (which releases the lock) AND the monitoring
|
|
1017
979
|
# + audit emission blocks after the try/finally so the
|
|
@@ -1115,36 +1077,6 @@ class DataWriter:
|
|
|
1115
1077
|
)
|
|
1116
1078
|
mark("bump_root")
|
|
1117
1079
|
|
|
1118
|
-
# --- GC enqueue (same flags as write) ----------------------------
|
|
1119
|
-
try:
|
|
1120
|
-
_org = self.super_table.organization
|
|
1121
|
-
_sup = self.super_table.super_name
|
|
1122
|
-
if settings.SUPERTABLE_SUNSET_GC_ENABLED and all_sunset:
|
|
1123
|
-
enqueue_deletions(
|
|
1124
|
-
self.catalog,
|
|
1125
|
-
_org, _sup, simple_name,
|
|
1126
|
-
"parquet",
|
|
1127
|
-
list(all_sunset),
|
|
1128
|
-
write_id=qid,
|
|
1129
|
-
)
|
|
1130
|
-
if settings.SUPERTABLE_SNAPSHOT_RETENTION > 0:
|
|
1131
|
-
old_paths = collect_old_snapshot_paths(
|
|
1132
|
-
new_snapshot_dict,
|
|
1133
|
-
self.super_table.storage,
|
|
1134
|
-
settings.SUPERTABLE_SNAPSHOT_RETENTION,
|
|
1135
|
-
)
|
|
1136
|
-
if old_paths:
|
|
1137
|
-
enqueue_deletions(
|
|
1138
|
-
self.catalog,
|
|
1139
|
-
_org, _sup, simple_name,
|
|
1140
|
-
"snapshot",
|
|
1141
|
-
old_paths,
|
|
1142
|
-
write_id=qid,
|
|
1143
|
-
)
|
|
1144
|
-
except Exception as e:
|
|
1145
|
-
logger.warning(lp(f"GC enqueue failed (compact still succeeded): {e}"))
|
|
1146
|
-
mark("gc_enqueue")
|
|
1147
|
-
|
|
1148
1080
|
# --- Mirroring ---------------------------------------------------
|
|
1149
1081
|
try:
|
|
1150
1082
|
MirrorFormats.mirror_if_enabled(
|
|
@@ -1111,7 +1111,7 @@ return 1
|
|
|
1111
1111
|
# ------------- Deletions (dangerous) -------------
|
|
1112
1112
|
|
|
1113
1113
|
def delete_simple_table(self, org: str, sup: str, simple: str) -> bool:
|
|
1114
|
-
"""Delete a simple table's Redis meta (leaf pointer + lock
|
|
1114
|
+
"""Delete a simple table's Redis meta (leaf pointer + lock).
|
|
1115
1115
|
|
|
1116
1116
|
This does **not** delete storage. Callers should delete storage first, then call this.
|
|
1117
1117
|
"""
|
|
@@ -1120,11 +1120,6 @@ return 1
|
|
|
1120
1120
|
keys = [
|
|
1121
1121
|
RK.meta_leaf(org, sup, simple),
|
|
1122
1122
|
RK.lock_leaf(org, sup, simple),
|
|
1123
|
-
# Nuke the deferred-deletion queue too — once the table is gone
|
|
1124
|
-
# there is no point draining pending deletions through the
|
|
1125
|
-
# cleaner, and leaving the stream around makes
|
|
1126
|
-
# SimpleTable.delete observably non-idempotent in Redis.
|
|
1127
|
-
RK.gc_pending(org, sup, simple),
|
|
1128
1123
|
]
|
|
1129
1124
|
try:
|
|
1130
1125
|
# DEL returns number of keys removed
|
|
@@ -729,83 +729,6 @@ def linked_share_doc(org: str, sup: str, link_id: str) -> str:
|
|
|
729
729
|
)
|
|
730
730
|
|
|
731
731
|
|
|
732
|
-
# --- GC / Deferred deletion queue ------------------------------------------ #
|
|
733
|
-
#
|
|
734
|
-
# Per-table Redis STREAM holding paths scheduled for delayed physical
|
|
735
|
-
# deletion (sunset parquet files, pruned snapshot JSONs). Producers
|
|
736
|
-
# (``data_writer``) XADD entries after the leaf-CAS commit succeeds; the
|
|
737
|
-
# org-level ``gc.cleaner`` daemon XRANGEs entries older than
|
|
738
|
-
# ``SUPERTABLE_GC_DELAY_SEC`` and ``storage.delete()`` them, then XDELs.
|
|
739
|
-
#
|
|
740
|
-
# The stream lives under the lakes scope so a supertable-wide drop
|
|
741
|
-
# (``catalog.delete_super_table``) naturally nukes it via the standard
|
|
742
|
-
# ``super_table_pattern`` SCAN.
|
|
743
|
-
|
|
744
|
-
def gc_pending(org: str, sup: str, simple: str) -> str:
|
|
745
|
-
"""Per-table deferred-deletion stream (STREAM)."""
|
|
746
|
-
return (
|
|
747
|
-
f"{SUPERTABLE_PREFIX}:{_safe('org', org)}:{LAKES_SCOPE}"
|
|
748
|
-
f":{_safe('sup', sup)}:gc:pending:doc:{_safe('simple', simple)}"
|
|
749
|
-
)
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
def gc_pending_pattern(org: str, sup: str) -> str:
|
|
753
|
-
"""SCAN pattern matching every GC stream in one supertable."""
|
|
754
|
-
return (
|
|
755
|
-
f"{SUPERTABLE_PREFIX}:{_safe('org', org)}:{LAKES_SCOPE}"
|
|
756
|
-
f":{_safe('sup', sup)}:gc:pending:doc:*"
|
|
757
|
-
)
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
def gc_pending_pattern_for_org(org: str) -> str:
|
|
761
|
-
"""SCAN pattern matching every GC stream across every supertable in one org.
|
|
762
|
-
|
|
763
|
-
Used by the per-org cleaner daemon to discover all per-table streams
|
|
764
|
-
it must drain.
|
|
765
|
-
"""
|
|
766
|
-
return (
|
|
767
|
-
f"{SUPERTABLE_PREFIX}:{_safe('org', org)}:{LAKES_SCOPE}"
|
|
768
|
-
f":*:gc:pending:doc:*"
|
|
769
|
-
)
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
def gc_pending_pattern_all_orgs() -> str:
|
|
773
|
-
"""SCAN pattern matching every GC stream across every org.
|
|
774
|
-
|
|
775
|
-
Used by the ``--all-orgs`` cleaner mode to discover the full set of
|
|
776
|
-
orgs that have pending deletions.
|
|
777
|
-
"""
|
|
778
|
-
return f"{SUPERTABLE_PREFIX}:*:{LAKES_SCOPE}:*:gc:pending:doc:*"
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
def parse_gc_pending_key(key: str) -> Optional[Tuple[str, str, str]]:
|
|
782
|
-
"""Extract ``(org, sup, simple)`` from a GC pending stream key.
|
|
783
|
-
|
|
784
|
-
Returns ``None`` when the key does not match the
|
|
785
|
-
``supertable:{org}:lakes:{sup}:gc:pending:doc:{simple}`` shape.
|
|
786
|
-
This is the canonical way for the cleaner to walk SCAN output and
|
|
787
|
-
recover the per-table coordinates without re-implementing the parse
|
|
788
|
-
rules.
|
|
789
|
-
"""
|
|
790
|
-
if not isinstance(key, str):
|
|
791
|
-
return None
|
|
792
|
-
parts = key.split(":")
|
|
793
|
-
# supertable : {org} : lakes : {sup} : gc : pending : doc : {simple}
|
|
794
|
-
if (
|
|
795
|
-
len(parts) == 8
|
|
796
|
-
and parts[0] == SUPERTABLE_PREFIX
|
|
797
|
-
and parts[2] == LAKES_SCOPE
|
|
798
|
-
and parts[4] == "gc"
|
|
799
|
-
and parts[5] == "pending"
|
|
800
|
-
and parts[6] == "doc"
|
|
801
|
-
and parts[1]
|
|
802
|
-
and parts[3]
|
|
803
|
-
and parts[7]
|
|
804
|
-
):
|
|
805
|
-
return parts[1], parts[3], parts[7]
|
|
806
|
-
return None
|
|
807
|
-
|
|
808
|
-
|
|
809
732
|
# --- Data Quality (dataisland-core quality module owns its sub-structure) #
|
|
810
733
|
|
|
811
734
|
def quality_prefix(org: str, sup: str) -> str:
|
|
@@ -260,6 +260,56 @@ class SimpleTable:
|
|
|
260
260
|
data = self.storage.read_json(path)
|
|
261
261
|
return data, path
|
|
262
262
|
|
|
263
|
+
def export_to(self, target_dir: str, compression_level: int = 3, small_only: bool = False):
|
|
264
|
+
"""Write a standalone copy of the current data into ``target_dir``.
|
|
265
|
+
|
|
266
|
+
Reads the current snapshot's parquet resources and re-writes them
|
|
267
|
+
as memory-bounded parquet chunks (each ~``max_memory_chunk_size``,
|
|
268
|
+
from the per-table config or the global default) under
|
|
269
|
+
``target_dir``. This is a pure copy: it does NOT create a new
|
|
270
|
+
snapshot, advance the Redis leaf, or touch ``data/``/``snapshots/``.
|
|
271
|
+
|
|
272
|
+
Args:
|
|
273
|
+
target_dir: destination directory for the exported parquet
|
|
274
|
+
files (created if missing). Typically an
|
|
275
|
+
``export/<timestamp>/`` folder next to ``data/``.
|
|
276
|
+
compression_level: zstd level for the exported parquet.
|
|
277
|
+
small_only: when False (default) every resource is read and
|
|
278
|
+
re-chunked; when True only files smaller than
|
|
279
|
+
``max_memory_chunk_size`` are included.
|
|
280
|
+
|
|
281
|
+
Returns:
|
|
282
|
+
``dict`` with ``files`` (list of written paths),
|
|
283
|
+
``files_written``, ``total_rows`` and ``total_bytes``.
|
|
284
|
+
"""
|
|
285
|
+
from supertable.processing import compact_resources
|
|
286
|
+
|
|
287
|
+
snapshot, _path = self.get_simple_table_snapshot()
|
|
288
|
+
table_config = self.catalog.get_table_config(
|
|
289
|
+
self.super_table.organization,
|
|
290
|
+
self.super_table.super_name,
|
|
291
|
+
self.simple_name,
|
|
292
|
+
) or {}
|
|
293
|
+
|
|
294
|
+
_considered, total_rows, new_resources, _sunset = compact_resources(
|
|
295
|
+
snapshot=snapshot,
|
|
296
|
+
data_dir=target_dir,
|
|
297
|
+
compression_level=compression_level,
|
|
298
|
+
table_config=table_config,
|
|
299
|
+
small_only=small_only,
|
|
300
|
+
)
|
|
301
|
+
|
|
302
|
+
files = [r.get("file") for r in new_resources if isinstance(r, dict) and r.get("file")]
|
|
303
|
+
total_bytes = sum(
|
|
304
|
+
int(r.get("file_size") or 0) for r in new_resources if isinstance(r, dict)
|
|
305
|
+
)
|
|
306
|
+
return {
|
|
307
|
+
"files": files,
|
|
308
|
+
"files_written": len(files),
|
|
309
|
+
"total_rows": int(total_rows),
|
|
310
|
+
"total_bytes": int(total_bytes),
|
|
311
|
+
}
|
|
312
|
+
|
|
263
313
|
def update(self, new_resources, sunset_files, model_df, last_snapshot=None, last_snapshot_path=None, lineage=None, profiler: Optional[Profiler] = None):
|
|
264
314
|
"""
|
|
265
315
|
Build and write a new heavy snapshot on storage.
|
|
@@ -38,8 +38,6 @@ _P_COMPACT_TOMB = f"{_MOD}.compact_tombstones"
|
|
|
38
38
|
_P_TOMB_THRESH = f"{_MOD}._tombstone_threshold"
|
|
39
39
|
_P_MIRROR = f"{_MOD}.MirrorFormats"
|
|
40
40
|
_P_MON_WRITER = f"{_MOD}.MonitoringWriter"
|
|
41
|
-
_P_ENQUEUE = f"{_MOD}.enqueue_deletions"
|
|
42
|
-
_P_COLLECT_OLD = f"{_MOD}.collect_old_snapshot_paths"
|
|
43
41
|
_P_AUDIT = f"{_MOD}._audit_emit"
|
|
44
42
|
_P_SETTINGS = f"{_MOD}.settings"
|
|
45
43
|
|
|
@@ -85,12 +83,9 @@ def _mk_simple_mock(snap: dict, *, snap_path: str = "/snap/v1.json",
|
|
|
85
83
|
return mock
|
|
86
84
|
|
|
87
85
|
|
|
88
|
-
def _stub_settings(
|
|
89
|
-
"""Build a fake ``settings`` object
|
|
90
|
-
|
|
91
|
-
s.SUPERTABLE_SUNSET_GC_ENABLED = sunset
|
|
92
|
-
s.SUPERTABLE_SNAPSHOT_RETENTION = retention
|
|
93
|
-
return s
|
|
86
|
+
def _stub_settings():
|
|
87
|
+
"""Build a fake ``settings`` object for patching."""
|
|
88
|
+
return MagicMock()
|
|
94
89
|
|
|
95
90
|
|
|
96
91
|
def _build_writer():
|
|
@@ -604,80 +599,6 @@ class TestSnapshotCommit:
|
|
|
604
599
|
dw.catalog.bump_root.assert_called_once()
|
|
605
600
|
|
|
606
601
|
|
|
607
|
-
# ===========================================================================
|
|
608
|
-
# 6. GC enqueue gating (same flags as write)
|
|
609
|
-
# ===========================================================================
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
class TestGCEnqueue:
|
|
613
|
-
|
|
614
|
-
@patch(_P_AUDIT)
|
|
615
|
-
@patch(_P_MON_WRITER)
|
|
616
|
-
@patch(_P_MIRROR)
|
|
617
|
-
@patch(_P_ENQUEUE)
|
|
618
|
-
@patch(_P_COLLECT_OLD, return_value=[])
|
|
619
|
-
@patch(_P_COMPACT_RES)
|
|
620
|
-
@patch(_P_COMPACT_TOMB, return_value=(0, [], set()))
|
|
621
|
-
@patch(_P_SETTINGS)
|
|
622
|
-
@patch(_P_SIMPLE_TABLE)
|
|
623
|
-
@patch(_P_CHECK_WRITE)
|
|
624
|
-
def test_sunset_gc_enqueue_when_flag_on(
|
|
625
|
-
self, mock_check_write, MockSimple, mock_settings_obj,
|
|
626
|
-
mock_compact_tomb, mock_compact_res, mock_collect_old,
|
|
627
|
-
mock_enqueue, MockMirror, MockMW, mock_audit,
|
|
628
|
-
):
|
|
629
|
-
mock_settings_obj.SUPERTABLE_SUNSET_GC_ENABLED = True
|
|
630
|
-
mock_settings_obj.SUPERTABLE_SNAPSHOT_RETENTION = 0
|
|
631
|
-
|
|
632
|
-
dw = _build_writer()
|
|
633
|
-
snap = _snapshot([_resource("a"), _resource("b")])
|
|
634
|
-
mock_simple = _mk_simple_mock(snap)
|
|
635
|
-
MockSimple.return_value = mock_simple
|
|
636
|
-
|
|
637
|
-
new_res = [{"file": "c.parquet", "file_size": 5000, "columns": []}]
|
|
638
|
-
sunset = {"a", "b"}
|
|
639
|
-
mock_compact_res.return_value = (2, 200, new_res, sunset)
|
|
640
|
-
dw._get_table_config = MagicMock(return_value={})
|
|
641
|
-
|
|
642
|
-
dw.compact("admin", "tbl")
|
|
643
|
-
|
|
644
|
-
# parquet enqueue called with the sunset paths
|
|
645
|
-
parquet_calls = [
|
|
646
|
-
c for c in mock_enqueue.call_args_list
|
|
647
|
-
if c.args[4] == "parquet"
|
|
648
|
-
]
|
|
649
|
-
assert len(parquet_calls) == 1
|
|
650
|
-
assert set(parquet_calls[0].args[5]) == sunset
|
|
651
|
-
|
|
652
|
-
@patch(_P_AUDIT)
|
|
653
|
-
@patch(_P_MON_WRITER)
|
|
654
|
-
@patch(_P_MIRROR)
|
|
655
|
-
@patch(_P_ENQUEUE)
|
|
656
|
-
@patch(_P_COLLECT_OLD, return_value=[])
|
|
657
|
-
@patch(_P_COMPACT_RES)
|
|
658
|
-
@patch(_P_COMPACT_TOMB, return_value=(0, [], set()))
|
|
659
|
-
@patch(_P_SETTINGS)
|
|
660
|
-
@patch(_P_SIMPLE_TABLE)
|
|
661
|
-
@patch(_P_CHECK_WRITE)
|
|
662
|
-
def test_no_gc_enqueue_when_flags_off(
|
|
663
|
-
self, mock_check_write, MockSimple, mock_settings_obj,
|
|
664
|
-
mock_compact_tomb, mock_compact_res, mock_collect_old,
|
|
665
|
-
mock_enqueue, MockMirror, MockMW, mock_audit,
|
|
666
|
-
):
|
|
667
|
-
mock_settings_obj.SUPERTABLE_SUNSET_GC_ENABLED = False
|
|
668
|
-
mock_settings_obj.SUPERTABLE_SNAPSHOT_RETENTION = 0
|
|
669
|
-
|
|
670
|
-
dw = _build_writer()
|
|
671
|
-
snap = _snapshot([_resource("a")])
|
|
672
|
-
MockSimple.return_value = _mk_simple_mock(snap)
|
|
673
|
-
mock_compact_res.return_value = (1, 100, [{"file": "c", "file_size": 1, "columns": []}], {"a"})
|
|
674
|
-
dw._get_table_config = MagicMock(return_value={})
|
|
675
|
-
|
|
676
|
-
dw.compact("admin", "tbl")
|
|
677
|
-
|
|
678
|
-
mock_enqueue.assert_not_called()
|
|
679
|
-
|
|
680
|
-
|
|
681
602
|
# ===========================================================================
|
|
682
603
|
# 7. Monitoring lifecycle
|
|
683
604
|
# ===========================================================================
|
|
@@ -121,12 +121,6 @@ def _all_helpers() -> list[tuple[str, str, str]]:
|
|
|
121
121
|
# ---- Schema -----------------------------------------------------
|
|
122
122
|
("schema", RK.schema(ORG, SUP, SIMPLE), f"{lake_pre}:schema:doc:{SIMPLE}"),
|
|
123
123
|
|
|
124
|
-
# ---- GC pending (deferred-deletion stream) ---------------------
|
|
125
|
-
("gc_pending", RK.gc_pending(ORG, SUP, SIMPLE), f"{lake_pre}:gc:pending:doc:{SIMPLE}"),
|
|
126
|
-
("gc_pending_pattern", RK.gc_pending_pattern(ORG, SUP), f"{lake_pre}:gc:pending:doc:*"),
|
|
127
|
-
("gc_pending_pattern_for_org", RK.gc_pending_pattern_for_org(ORG), f"supertable:{ORG}:lakes:*:gc:pending:doc:*"),
|
|
128
|
-
("gc_pending_pattern_all_orgs", RK.gc_pending_pattern_all_orgs(), "supertable:*:lakes:*:gc:pending:doc:*"),
|
|
129
|
-
|
|
130
124
|
# ---- Linked shares ---------------------------------------------
|
|
131
125
|
("linked_share_index", RK.linked_share_index(ORG, SUP), f"{lake_pre}:linked_shares:index"),
|
|
132
126
|
("linked_share_doc", RK.linked_share_doc(ORG, SUP, LINK), f"{lake_pre}:linked_shares:doc:{LINK}"),
|
|
@@ -421,20 +415,6 @@ def test_parse_registry_key_returns_none_for_other_keys():
|
|
|
421
415
|
assert RK.parse_registry_key(RK.app_master_mcp("lighthouse")) is None
|
|
422
416
|
|
|
423
417
|
|
|
424
|
-
def test_parse_gc_pending_key_extracts_fields():
|
|
425
|
-
key = RK.gc_pending("acme", "demo", "orders")
|
|
426
|
-
assert RK.parse_gc_pending_key(key) == ("acme", "demo", "orders")
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
def test_parse_gc_pending_key_returns_none_for_other_keys():
|
|
430
|
-
assert RK.parse_gc_pending_key(RK.meta_root("acme", "demo")) is None
|
|
431
|
-
assert RK.parse_gc_pending_key(RK.meta_leaf("acme", "demo", "orders")) is None
|
|
432
|
-
assert RK.parse_gc_pending_key(RK.lock_leaf("acme", "demo", "orders")) is None
|
|
433
|
-
assert RK.parse_gc_pending_key("supertable:acme:lakes:demo:gc:pending:doc:") is None
|
|
434
|
-
assert RK.parse_gc_pending_key("") is None
|
|
435
|
-
assert RK.parse_gc_pending_key(None) is None # type: ignore[arg-type]
|
|
436
|
-
|
|
437
|
-
|
|
438
418
|
def test_parse_monitor_partition_key_extracts_fields():
|
|
439
419
|
key = RK.monitor_partition("acme", "writes", "2026-06-09")
|
|
440
420
|
assert RK.parse_monitor_partition_key(key) == ("acme", "writes", "2026-06-09")
|
|
@@ -442,7 +422,6 @@ def test_parse_monitor_partition_key_extracts_fields():
|
|
|
442
422
|
|
|
443
423
|
def test_parse_monitor_partition_key_returns_none_for_other_keys():
|
|
444
424
|
assert RK.parse_monitor_partition_key(RK.meta_root("acme", "demo")) is None
|
|
445
|
-
assert RK.parse_monitor_partition_key(RK.gc_pending("acme", "demo", "t")) is None
|
|
446
425
|
# Drain handle is rejected (7 segments, not 6)
|
|
447
426
|
assert RK.parse_monitor_partition_key(
|
|
448
427
|
RK.monitor_partition_drain("acme", "writes", "2026-06-09")
|
|
@@ -107,10 +107,6 @@ supertable/engine/tests/__init__.py
|
|
|
107
107
|
supertable/engine/tests/conftest.py
|
|
108
108
|
supertable/engine/tests/test_dedup_read.py
|
|
109
109
|
supertable/engine/tests/test_engine.py
|
|
110
|
-
supertable/gc/__init__.py
|
|
111
|
-
supertable/gc/cleaner.py
|
|
112
|
-
supertable/gc/daemon.py
|
|
113
|
-
supertable/gc/queue.py
|
|
114
110
|
supertable/locking/__init__.py
|
|
115
111
|
supertable/locking/file_lock.py
|
|
116
112
|
supertable/locking/redis_lock.py
|
|
@@ -158,9 +154,6 @@ supertable/tests/test_data_writer_comprehensive.py
|
|
|
158
154
|
supertable/tests/test_data_writer_tombstones.py
|
|
159
155
|
supertable/tests/test_dedup_on_read_write.py
|
|
160
156
|
supertable/tests/test_errors.py
|
|
161
|
-
supertable/tests/test_gc_cleaner.py
|
|
162
|
-
supertable/tests/test_gc_daemon.py
|
|
163
|
-
supertable/tests/test_gc_queue.py
|
|
164
157
|
supertable/tests/test_meta_reader.py
|
|
165
158
|
supertable/tests/test_monitoring_partitions.py
|
|
166
159
|
supertable/tests/test_monitoring_sink_guard.py
|
|
@@ -1,26 +0,0 @@
|
|
|
1
|
-
# route: supertable.gc
|
|
2
|
-
"""
|
|
3
|
-
Deferred-deletion garbage collection for SuperTable storage.
|
|
4
|
-
|
|
5
|
-
The data writer never deletes files inline. When the GC flags are enabled
|
|
6
|
-
(``SUPERTABLE_SUNSET_GC_ENABLED`` / ``SUPERTABLE_SNAPSHOT_RETENTION``) the
|
|
7
|
-
writer XADDs paths to a per-table Redis STREAM after a successful leaf-CAS.
|
|
8
|
-
A long-running ``GCCleaner`` daemon (one per org) drains entries older than
|
|
9
|
-
``SUPERTABLE_GC_DELAY_SEC`` and calls ``storage.delete()`` on each path.
|
|
10
|
-
|
|
11
|
-
The delay window is what makes deletion safe under concurrent reads: a
|
|
12
|
-
query that resolved the leaf payload right before the writer committed
|
|
13
|
-
finishes long before the cleaner touches the file.
|
|
14
|
-
"""
|
|
15
|
-
|
|
16
|
-
from supertable.gc.queue import (
|
|
17
|
-
collect_old_snapshot_paths,
|
|
18
|
-
enqueue_deletions,
|
|
19
|
-
nuke_stream,
|
|
20
|
-
)
|
|
21
|
-
|
|
22
|
-
__all__ = [
|
|
23
|
-
"collect_old_snapshot_paths",
|
|
24
|
-
"enqueue_deletions",
|
|
25
|
-
"nuke_stream",
|
|
26
|
-
]
|