supertable 2.2.2__tar.gz → 2.2.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {supertable-2.2.2/supertable.egg-info → supertable-2.2.3}/PKG-INFO +1 -1
- {supertable-2.2.2 → supertable-2.2.3}/pyproject.toml +1 -1
- {supertable-2.2.2 → supertable-2.2.3}/setup.py +1 -1
- {supertable-2.2.2 → supertable-2.2.3}/supertable/__init__.py +1 -1
- {supertable-2.2.2 → supertable-2.2.3}/supertable/config/settings.py +0 -22
- {supertable-2.2.2 → supertable-2.2.3}/supertable/data_writer.py +5 -78
- {supertable-2.2.2 → supertable-2.2.3}/supertable/redis_catalog.py +1 -6
- {supertable-2.2.2 → supertable-2.2.3}/supertable/redis_keys.py +0 -77
- {supertable-2.2.2 → supertable-2.2.3}/supertable/tests/test_data_writer_compact.py +3 -82
- {supertable-2.2.2 → supertable-2.2.3}/supertable/tests/test_redis_key_prefix.py +0 -21
- {supertable-2.2.2 → supertable-2.2.3/supertable.egg-info}/PKG-INFO +1 -1
- {supertable-2.2.2 → supertable-2.2.3}/supertable.egg-info/SOURCES.txt +0 -7
- supertable-2.2.2/supertable/gc/__init__.py +0 -26
- supertable-2.2.2/supertable/gc/cleaner.py +0 -258
- supertable-2.2.2/supertable/gc/daemon.py +0 -284
- supertable-2.2.2/supertable/gc/queue.py +0 -221
- supertable-2.2.2/supertable/tests/test_gc_cleaner.py +0 -330
- supertable-2.2.2/supertable/tests/test_gc_daemon.py +0 -251
- supertable-2.2.2/supertable/tests/test_gc_queue.py +0 -331
- {supertable-2.2.2 → supertable-2.2.3}/LICENSE +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/README.md +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/requirements.txt +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/setup.cfg +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/audit/__init__.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/audit/admin.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/audit/chain.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/audit/consumers.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/audit/crypto.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/audit/events.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/audit/export.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/audit/logger.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/audit/middleware.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/audit/reader.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/audit/retention.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/audit/tests/__init__.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/audit/tests/test_chain.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/audit/tests/test_crypto.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/audit/tests/test_emit.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/audit/tests/test_events.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/audit/tests/test_retention.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/audit/writer_parquet.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/audit/writer_redis.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/config/__init__.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/config/defaults.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/config/homedir.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/config/tests/__init__.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/config/tests/test_defaults.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/config/tests/test_homedir.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/config/tests/test_settings.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/data_classes.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/data_reader.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/demo/__init__.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/demo/quickstart/__init__.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/demo/quickstart/__main__.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/demo/quickstart/check_filter_builder.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/demo/quickstart/controller.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/demo/quickstart/data_writer_helpers.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/demo/quickstart/defaults.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/demo/quickstart/dummy_data.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/demo/quickstart/read_parquet_header.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/demo/quickstart/s01_01_01_create_super_table.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/demo/quickstart/s01_01_02_enable_mirroring_formats.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/demo/quickstart/s01_02_create_roles.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/demo/quickstart/s01_03_create_users.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/demo/quickstart/s02_01_write_dummy_data.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/demo/quickstart/s02_02_write_single_data.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/demo/quickstart/s02_03_01_write_staging.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/demo/quickstart/s02_03_02_create_pipe.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/demo/quickstart/s02_04_01_write_monitoring_simple.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/demo/quickstart/s02_04_02_write_monitoring_parallel.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/demo/quickstart/s02_05_write_tombstone.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/demo/quickstart/s03_01_read_data_error.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/demo/quickstart/s03_02_01_read_super_data_ok.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/demo/quickstart/s03_02_02_read_table_data_ok.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/demo/quickstart/s03_03_read_meta.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/demo/quickstart/s03_04_read_staging.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/demo/quickstart/s03_06_01_read_roles.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/demo/quickstart/s03_06_02_read_user.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/demo/quickstart/s03_07_01_estimate_read.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/demo/quickstart/s03_07_02_estimate_files.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/demo/quickstart/s03_08_read_snapshot_history.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/demo/quickstart/s04_01_03_delete_pipe.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/demo/quickstart/s05_01_delete_table.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/demo/quickstart/s05_02_delete_super_table.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/demo/webshop/__init__.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/demo/webshop/core.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/demo/webshop/defaults.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/demo/webshop/generate.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/demo/webshop/load.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/demo/webshop/topup.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/engine/__init__.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/engine/data_estimator.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/engine/duckdb_lite.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/engine/duckdb_pro.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/engine/engine_common.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/engine/engine_enum.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/engine/executor.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/engine/plan_stats.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/engine/spark_thrift.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/engine/tests/__init__.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/engine/tests/conftest.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/engine/tests/test_dedup_read.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/engine/tests/test_engine.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/errors.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/locking/__init__.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/locking/benchmarks/__init__.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/locking/benchmarks/benchmark_locking.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/locking/benchmarks/measure_lock_speed.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/locking/benchmarks/measure_lock_time.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/locking/file_lock.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/locking/redis_lock.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/locking/tests/__init__.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/locking/tests/test_file_lock.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/locking/tests/test_redis_lock.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/logging.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/meta_reader.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/mirroring/__init__.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/mirroring/mirror_delta.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/mirroring/mirror_formats.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/mirroring/mirror_iceberg.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/mirroring/mirror_parquet.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/monitoring/__init__.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/monitoring/partitions.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/monitoring_writer.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/plan_extender.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/processing.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/query_plan_manager.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/rbac/__init__.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/rbac/access_control.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/rbac/filter_builder.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/rbac/permissions.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/rbac/role_manager.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/rbac/row_column_security.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/rbac/tests/test_filter_builder.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/rbac/tests/test_rbac.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/rbac/tests/test_rbac_per_table.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/rbac/user_manager.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/redis_connector.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/redis_infra.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/simple_table.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/staging_area.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/storage/__init__.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/storage/azure_storage.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/storage/gcp_storage.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/storage/local_storage.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/storage/minio_storage.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/storage/s3_storage.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/storage/storage_factory.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/storage/storage_interface.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/storage/tests/test_storage.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/super_pipe.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/super_table.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/tests/__init__.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/tests/test_align_to_schema_fix.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/tests/test_create_if_missing.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/tests/test_data_reader.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/tests/test_data_reader_preflight.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/tests/test_data_writer.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/tests/test_data_writer_comprehensive.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/tests/test_data_writer_tombstones.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/tests/test_dedup_on_read_write.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/tests/test_errors.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/tests/test_meta_reader.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/tests/test_monitoring_partitions.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/tests/test_monitoring_sink_guard.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/tests/test_newer_than.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/tests/test_process_delete_only.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/tests/test_processing.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/tests/test_processing_compact_resources.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/tests/test_query_sql.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/tests/test_simple_table.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/tests/test_small_file_compaction.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/tests/test_super_table.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/tests/test_supertable_all.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/utils/__init__.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/utils/helper.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/utils/profiler.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/utils/sql_parser.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/utils/tests/test_sql_parser_columns.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable/utils/timer.py +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable.egg-info/dependency_links.txt +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable.egg-info/entry_points.txt +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable.egg-info/requires.txt +0 -0
- {supertable-2.2.2 → supertable-2.2.3}/supertable.egg-info/top_level.txt +0 -0
|
@@ -19,7 +19,7 @@ long_description = readme.read_text(encoding="utf-8") if readme.exists() else ""
|
|
|
19
19
|
|
|
20
20
|
setup(
|
|
21
21
|
name="supertable",
|
|
22
|
-
version="2.2.
|
|
22
|
+
version="2.2.3",
|
|
23
23
|
description="SuperTable — versioned data lake library for SQL analytics on Parquet + Redis.",
|
|
24
24
|
long_description=long_description,
|
|
25
25
|
long_description_content_type="text/markdown",
|
|
@@ -25,7 +25,7 @@ See the ``supertable.demo`` package for runnable end-to-end demos and the
|
|
|
25
25
|
project documentation for the full API surface.
|
|
26
26
|
"""
|
|
27
27
|
|
|
28
|
-
__version__ = "2.2.
|
|
28
|
+
__version__ = "2.2.3"
|
|
29
29
|
|
|
30
30
|
# Re-export the core public surface so users can do ``from supertable import …``
|
|
31
31
|
# instead of remembering submodule paths.
|
|
@@ -282,21 +282,6 @@ class Settings:
|
|
|
282
282
|
SUPERTABLE_SHARE_PRESIGN_TTL: int = 14400 # SUPERTABLE_SHARE_PRESIGN_TTL (seconds, default 4h)
|
|
283
283
|
SUPERTABLE_SHARE_REFRESH_BUFFER: int = 600 # SUPERTABLE_SHARE_REFRESH_BUFFER (seconds, refresh 10min before expiry)
|
|
284
284
|
|
|
285
|
-
# ── Storage GC (deferred deletion of sunset parquets + old snapshot JSONs) ──
|
|
286
|
-
# Sunset parquet files and old snapshot JSONs are never deleted by the
|
|
287
|
-
# writer directly. When either flag is enabled, the writer XADDs paths
|
|
288
|
-
# to a per-table Redis STREAM after a successful leaf-CAS. A separate
|
|
289
|
-
# cleaner daemon (``python -m supertable.gc.cleaner``) drains entries
|
|
290
|
-
# older than ``SUPERTABLE_GC_DELAY_SEC`` and calls ``storage.delete()``.
|
|
291
|
-
# The delay window avoids the race where an in-flight reader's
|
|
292
|
-
# ``parquet_scan([...])`` resolves a file path right before the writer
|
|
293
|
-
# deletes it. Defaults below preserve today's behaviour (off, unlimited).
|
|
294
|
-
SUPERTABLE_SNAPSHOT_RETENTION: int = 0 # SUPERTABLE_SNAPSHOT_RETENTION (0 = keep all snapshot JSONs)
|
|
295
|
-
SUPERTABLE_SUNSET_GC_ENABLED: bool = False # SUPERTABLE_SUNSET_GC_ENABLED (delete sunset parquets via queue)
|
|
296
|
-
SUPERTABLE_GC_DELAY_SEC: int = 1800 # SUPERTABLE_GC_DELAY_SEC (min age before cleaner deletes)
|
|
297
|
-
SUPERTABLE_GC_SLEEP_SEC: int = 60 # SUPERTABLE_GC_SLEEP_SEC (cleaner loop sleep)
|
|
298
|
-
SUPERTABLE_GC_BATCH_SIZE: int = 500 # SUPERTABLE_GC_BATCH_SIZE (max entries processed per table per tick)
|
|
299
|
-
|
|
300
285
|
|
|
301
286
|
# ── Convenience properties ───────────────────────────────────────
|
|
302
287
|
|
|
@@ -543,13 +528,6 @@ def _build_settings() -> Settings:
|
|
|
543
528
|
# Data Sharing
|
|
544
529
|
SUPERTABLE_SHARE_PRESIGN_TTL=_env_int("SUPERTABLE_SHARE_PRESIGN_TTL", 14400),
|
|
545
530
|
SUPERTABLE_SHARE_REFRESH_BUFFER=_env_int("SUPERTABLE_SHARE_REFRESH_BUFFER", 600),
|
|
546
|
-
|
|
547
|
-
# ── Storage GC ───────────────────────────────────────────────
|
|
548
|
-
SUPERTABLE_SNAPSHOT_RETENTION=_env_int("SUPERTABLE_SNAPSHOT_RETENTION", 0),
|
|
549
|
-
SUPERTABLE_SUNSET_GC_ENABLED=_env_bool("SUPERTABLE_SUNSET_GC_ENABLED", False),
|
|
550
|
-
SUPERTABLE_GC_DELAY_SEC=_env_int("SUPERTABLE_GC_DELAY_SEC", 1800),
|
|
551
|
-
SUPERTABLE_GC_SLEEP_SEC=_env_int("SUPERTABLE_GC_SLEEP_SEC", 60),
|
|
552
|
-
SUPERTABLE_GC_BATCH_SIZE=_env_int("SUPERTABLE_GC_BATCH_SIZE", 500),
|
|
553
531
|
)
|
|
554
532
|
|
|
555
533
|
|
|
@@ -13,7 +13,6 @@ from polars import DataFrame
|
|
|
13
13
|
from supertable.config.defaults import logger
|
|
14
14
|
from supertable.config.settings import settings
|
|
15
15
|
from supertable.errors import SuperTableNotFoundError, TableNotFoundError
|
|
16
|
-
from supertable.gc.queue import collect_old_snapshot_paths, enqueue_deletions
|
|
17
16
|
from supertable.monitoring.partitions import MONITORING_SINK_TABLES
|
|
18
17
|
from supertable.monitoring_writer import MonitoringWriter # async monitoring
|
|
19
18
|
from supertable.super_table import SuperTable
|
|
@@ -571,46 +570,6 @@ class DataWriter:
|
|
|
571
570
|
self.catalog.bump_root(self.super_table.organization, self.super_table.super_name, now_ms=now_ms)
|
|
572
571
|
mark("bump_root")
|
|
573
572
|
|
|
574
|
-
# --- Enqueue deferred deletions (post-CAS, post-bump) ----------
|
|
575
|
-
# Strict ordering matters: leaf-CAS + root-bump have committed
|
|
576
|
-
# the new snapshot as the authoritative state. Only NOW is it
|
|
577
|
-
# safe to schedule physical deletion of files that the old
|
|
578
|
-
# snapshot still references — any reader that loaded the
|
|
579
|
-
# previous leaf payload will finish before the cleaner's delay
|
|
580
|
-
# window expires.
|
|
581
|
-
#
|
|
582
|
-
# Both blocks are best-effort: enqueue_deletions and
|
|
583
|
-
# collect_old_snapshot_paths swallow Redis/storage errors and
|
|
584
|
-
# log a warning. A GC failure must never fail the write.
|
|
585
|
-
try:
|
|
586
|
-
_org = self.super_table.organization
|
|
587
|
-
_sup = self.super_table.super_name
|
|
588
|
-
if settings.SUPERTABLE_SUNSET_GC_ENABLED and sunset_files:
|
|
589
|
-
enqueue_deletions(
|
|
590
|
-
self.catalog,
|
|
591
|
-
_org, _sup, simple_name,
|
|
592
|
-
"parquet",
|
|
593
|
-
list(sunset_files),
|
|
594
|
-
write_id=qid,
|
|
595
|
-
)
|
|
596
|
-
if settings.SUPERTABLE_SNAPSHOT_RETENTION > 0:
|
|
597
|
-
old_paths = collect_old_snapshot_paths(
|
|
598
|
-
new_snapshot_dict,
|
|
599
|
-
self.super_table.storage,
|
|
600
|
-
settings.SUPERTABLE_SNAPSHOT_RETENTION,
|
|
601
|
-
)
|
|
602
|
-
if old_paths:
|
|
603
|
-
enqueue_deletions(
|
|
604
|
-
self.catalog,
|
|
605
|
-
_org, _sup, simple_name,
|
|
606
|
-
"snapshot",
|
|
607
|
-
old_paths,
|
|
608
|
-
write_id=qid,
|
|
609
|
-
)
|
|
610
|
-
except Exception as e:
|
|
611
|
-
logger.warning(lp(f"GC enqueue failed (write still succeeded): {e}"))
|
|
612
|
-
mark("gc_enqueue")
|
|
613
|
-
|
|
614
573
|
# --- Store schema + table name in Redis (permanent, not cache) ---
|
|
615
574
|
try:
|
|
616
575
|
schema_raw = new_snapshot_dict.get("schema", {})
|
|
@@ -994,12 +953,10 @@ class DataWriter:
|
|
|
994
953
|
# Critical filter: a file produced by Phase A (tombstone
|
|
995
954
|
# compaction) can be picked up as a small-file candidate
|
|
996
955
|
# by Phase B (compact_resources) and then sunset there.
|
|
997
|
-
# If we leave it in ``all_new_resources`` it
|
|
998
|
-
# new snapshot **and** in ``all_sunset``
|
|
999
|
-
#
|
|
1000
|
-
#
|
|
1001
|
-
# FileNotFoundError. Filter the file out of new_resources
|
|
1002
|
-
# when it also appears in any sunset set.
|
|
956
|
+
# If we leave it in ``all_new_resources`` it would land in the
|
|
957
|
+
# new snapshot **and** in ``all_sunset`` at the same time.
|
|
958
|
+
# Filter the file out of new_resources when it also appears
|
|
959
|
+
# in any sunset set so the snapshot stays consistent.
|
|
1003
960
|
all_sunset = set(tomb_sunset) | set(small_sunset)
|
|
1004
961
|
all_new_resources = [
|
|
1005
962
|
r for r in (list(tomb_new_resources) + list(small_new_resources))
|
|
@@ -1011,7 +968,7 @@ class DataWriter:
|
|
|
1011
968
|
result["total_rows_written"] = small_total_rows
|
|
1012
969
|
|
|
1013
970
|
# Short-circuit: nothing to do, skip the snapshot rewrite
|
|
1014
|
-
# and the leaf-CAS / root-bump /
|
|
971
|
+
# and the leaf-CAS / root-bump / mirror path. BUT do
|
|
1015
972
|
# NOT return here — execution must fall through to the
|
|
1016
973
|
# ``finally`` (which releases the lock) AND the monitoring
|
|
1017
974
|
# + audit emission blocks after the try/finally so the
|
|
@@ -1115,36 +1072,6 @@ class DataWriter:
|
|
|
1115
1072
|
)
|
|
1116
1073
|
mark("bump_root")
|
|
1117
1074
|
|
|
1118
|
-
# --- GC enqueue (same flags as write) ----------------------------
|
|
1119
|
-
try:
|
|
1120
|
-
_org = self.super_table.organization
|
|
1121
|
-
_sup = self.super_table.super_name
|
|
1122
|
-
if settings.SUPERTABLE_SUNSET_GC_ENABLED and all_sunset:
|
|
1123
|
-
enqueue_deletions(
|
|
1124
|
-
self.catalog,
|
|
1125
|
-
_org, _sup, simple_name,
|
|
1126
|
-
"parquet",
|
|
1127
|
-
list(all_sunset),
|
|
1128
|
-
write_id=qid,
|
|
1129
|
-
)
|
|
1130
|
-
if settings.SUPERTABLE_SNAPSHOT_RETENTION > 0:
|
|
1131
|
-
old_paths = collect_old_snapshot_paths(
|
|
1132
|
-
new_snapshot_dict,
|
|
1133
|
-
self.super_table.storage,
|
|
1134
|
-
settings.SUPERTABLE_SNAPSHOT_RETENTION,
|
|
1135
|
-
)
|
|
1136
|
-
if old_paths:
|
|
1137
|
-
enqueue_deletions(
|
|
1138
|
-
self.catalog,
|
|
1139
|
-
_org, _sup, simple_name,
|
|
1140
|
-
"snapshot",
|
|
1141
|
-
old_paths,
|
|
1142
|
-
write_id=qid,
|
|
1143
|
-
)
|
|
1144
|
-
except Exception as e:
|
|
1145
|
-
logger.warning(lp(f"GC enqueue failed (compact still succeeded): {e}"))
|
|
1146
|
-
mark("gc_enqueue")
|
|
1147
|
-
|
|
1148
1075
|
# --- Mirroring ---------------------------------------------------
|
|
1149
1076
|
try:
|
|
1150
1077
|
MirrorFormats.mirror_if_enabled(
|
|
@@ -1111,7 +1111,7 @@ return 1
|
|
|
1111
1111
|
# ------------- Deletions (dangerous) -------------
|
|
1112
1112
|
|
|
1113
1113
|
def delete_simple_table(self, org: str, sup: str, simple: str) -> bool:
|
|
1114
|
-
"""Delete a simple table's Redis meta (leaf pointer + lock
|
|
1114
|
+
"""Delete a simple table's Redis meta (leaf pointer + lock).
|
|
1115
1115
|
|
|
1116
1116
|
This does **not** delete storage. Callers should delete storage first, then call this.
|
|
1117
1117
|
"""
|
|
@@ -1120,11 +1120,6 @@ return 1
|
|
|
1120
1120
|
keys = [
|
|
1121
1121
|
RK.meta_leaf(org, sup, simple),
|
|
1122
1122
|
RK.lock_leaf(org, sup, simple),
|
|
1123
|
-
# Nuke the deferred-deletion queue too — once the table is gone
|
|
1124
|
-
# there is no point draining pending deletions through the
|
|
1125
|
-
# cleaner, and leaving the stream around makes
|
|
1126
|
-
# SimpleTable.delete observably non-idempotent in Redis.
|
|
1127
|
-
RK.gc_pending(org, sup, simple),
|
|
1128
1123
|
]
|
|
1129
1124
|
try:
|
|
1130
1125
|
# DEL returns number of keys removed
|
|
@@ -729,83 +729,6 @@ def linked_share_doc(org: str, sup: str, link_id: str) -> str:
|
|
|
729
729
|
)
|
|
730
730
|
|
|
731
731
|
|
|
732
|
-
# --- GC / Deferred deletion queue ------------------------------------------ #
|
|
733
|
-
#
|
|
734
|
-
# Per-table Redis STREAM holding paths scheduled for delayed physical
|
|
735
|
-
# deletion (sunset parquet files, pruned snapshot JSONs). Producers
|
|
736
|
-
# (``data_writer``) XADD entries after the leaf-CAS commit succeeds; the
|
|
737
|
-
# org-level ``gc.cleaner`` daemon XRANGEs entries older than
|
|
738
|
-
# ``SUPERTABLE_GC_DELAY_SEC`` and ``storage.delete()`` them, then XDELs.
|
|
739
|
-
#
|
|
740
|
-
# The stream lives under the lakes scope so a supertable-wide drop
|
|
741
|
-
# (``catalog.delete_super_table``) naturally nukes it via the standard
|
|
742
|
-
# ``super_table_pattern`` SCAN.
|
|
743
|
-
|
|
744
|
-
def gc_pending(org: str, sup: str, simple: str) -> str:
|
|
745
|
-
"""Per-table deferred-deletion stream (STREAM)."""
|
|
746
|
-
return (
|
|
747
|
-
f"{SUPERTABLE_PREFIX}:{_safe('org', org)}:{LAKES_SCOPE}"
|
|
748
|
-
f":{_safe('sup', sup)}:gc:pending:doc:{_safe('simple', simple)}"
|
|
749
|
-
)
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
def gc_pending_pattern(org: str, sup: str) -> str:
|
|
753
|
-
"""SCAN pattern matching every GC stream in one supertable."""
|
|
754
|
-
return (
|
|
755
|
-
f"{SUPERTABLE_PREFIX}:{_safe('org', org)}:{LAKES_SCOPE}"
|
|
756
|
-
f":{_safe('sup', sup)}:gc:pending:doc:*"
|
|
757
|
-
)
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
def gc_pending_pattern_for_org(org: str) -> str:
|
|
761
|
-
"""SCAN pattern matching every GC stream across every supertable in one org.
|
|
762
|
-
|
|
763
|
-
Used by the per-org cleaner daemon to discover all per-table streams
|
|
764
|
-
it must drain.
|
|
765
|
-
"""
|
|
766
|
-
return (
|
|
767
|
-
f"{SUPERTABLE_PREFIX}:{_safe('org', org)}:{LAKES_SCOPE}"
|
|
768
|
-
f":*:gc:pending:doc:*"
|
|
769
|
-
)
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
def gc_pending_pattern_all_orgs() -> str:
|
|
773
|
-
"""SCAN pattern matching every GC stream across every org.
|
|
774
|
-
|
|
775
|
-
Used by the ``--all-orgs`` cleaner mode to discover the full set of
|
|
776
|
-
orgs that have pending deletions.
|
|
777
|
-
"""
|
|
778
|
-
return f"{SUPERTABLE_PREFIX}:*:{LAKES_SCOPE}:*:gc:pending:doc:*"
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
def parse_gc_pending_key(key: str) -> Optional[Tuple[str, str, str]]:
|
|
782
|
-
"""Extract ``(org, sup, simple)`` from a GC pending stream key.
|
|
783
|
-
|
|
784
|
-
Returns ``None`` when the key does not match the
|
|
785
|
-
``supertable:{org}:lakes:{sup}:gc:pending:doc:{simple}`` shape.
|
|
786
|
-
This is the canonical way for the cleaner to walk SCAN output and
|
|
787
|
-
recover the per-table coordinates without re-implementing the parse
|
|
788
|
-
rules.
|
|
789
|
-
"""
|
|
790
|
-
if not isinstance(key, str):
|
|
791
|
-
return None
|
|
792
|
-
parts = key.split(":")
|
|
793
|
-
# supertable : {org} : lakes : {sup} : gc : pending : doc : {simple}
|
|
794
|
-
if (
|
|
795
|
-
len(parts) == 8
|
|
796
|
-
and parts[0] == SUPERTABLE_PREFIX
|
|
797
|
-
and parts[2] == LAKES_SCOPE
|
|
798
|
-
and parts[4] == "gc"
|
|
799
|
-
and parts[5] == "pending"
|
|
800
|
-
and parts[6] == "doc"
|
|
801
|
-
and parts[1]
|
|
802
|
-
and parts[3]
|
|
803
|
-
and parts[7]
|
|
804
|
-
):
|
|
805
|
-
return parts[1], parts[3], parts[7]
|
|
806
|
-
return None
|
|
807
|
-
|
|
808
|
-
|
|
809
732
|
# --- Data Quality (dataisland-core quality module owns its sub-structure) #
|
|
810
733
|
|
|
811
734
|
def quality_prefix(org: str, sup: str) -> str:
|
|
@@ -38,8 +38,6 @@ _P_COMPACT_TOMB = f"{_MOD}.compact_tombstones"
|
|
|
38
38
|
_P_TOMB_THRESH = f"{_MOD}._tombstone_threshold"
|
|
39
39
|
_P_MIRROR = f"{_MOD}.MirrorFormats"
|
|
40
40
|
_P_MON_WRITER = f"{_MOD}.MonitoringWriter"
|
|
41
|
-
_P_ENQUEUE = f"{_MOD}.enqueue_deletions"
|
|
42
|
-
_P_COLLECT_OLD = f"{_MOD}.collect_old_snapshot_paths"
|
|
43
41
|
_P_AUDIT = f"{_MOD}._audit_emit"
|
|
44
42
|
_P_SETTINGS = f"{_MOD}.settings"
|
|
45
43
|
|
|
@@ -85,12 +83,9 @@ def _mk_simple_mock(snap: dict, *, snap_path: str = "/snap/v1.json",
|
|
|
85
83
|
return mock
|
|
86
84
|
|
|
87
85
|
|
|
88
|
-
def _stub_settings(
|
|
89
|
-
"""Build a fake ``settings`` object
|
|
90
|
-
|
|
91
|
-
s.SUPERTABLE_SUNSET_GC_ENABLED = sunset
|
|
92
|
-
s.SUPERTABLE_SNAPSHOT_RETENTION = retention
|
|
93
|
-
return s
|
|
86
|
+
def _stub_settings():
|
|
87
|
+
"""Build a fake ``settings`` object for patching."""
|
|
88
|
+
return MagicMock()
|
|
94
89
|
|
|
95
90
|
|
|
96
91
|
def _build_writer():
|
|
@@ -604,80 +599,6 @@ class TestSnapshotCommit:
|
|
|
604
599
|
dw.catalog.bump_root.assert_called_once()
|
|
605
600
|
|
|
606
601
|
|
|
607
|
-
# ===========================================================================
|
|
608
|
-
# 6. GC enqueue gating (same flags as write)
|
|
609
|
-
# ===========================================================================
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
class TestGCEnqueue:
|
|
613
|
-
|
|
614
|
-
@patch(_P_AUDIT)
|
|
615
|
-
@patch(_P_MON_WRITER)
|
|
616
|
-
@patch(_P_MIRROR)
|
|
617
|
-
@patch(_P_ENQUEUE)
|
|
618
|
-
@patch(_P_COLLECT_OLD, return_value=[])
|
|
619
|
-
@patch(_P_COMPACT_RES)
|
|
620
|
-
@patch(_P_COMPACT_TOMB, return_value=(0, [], set()))
|
|
621
|
-
@patch(_P_SETTINGS)
|
|
622
|
-
@patch(_P_SIMPLE_TABLE)
|
|
623
|
-
@patch(_P_CHECK_WRITE)
|
|
624
|
-
def test_sunset_gc_enqueue_when_flag_on(
|
|
625
|
-
self, mock_check_write, MockSimple, mock_settings_obj,
|
|
626
|
-
mock_compact_tomb, mock_compact_res, mock_collect_old,
|
|
627
|
-
mock_enqueue, MockMirror, MockMW, mock_audit,
|
|
628
|
-
):
|
|
629
|
-
mock_settings_obj.SUPERTABLE_SUNSET_GC_ENABLED = True
|
|
630
|
-
mock_settings_obj.SUPERTABLE_SNAPSHOT_RETENTION = 0
|
|
631
|
-
|
|
632
|
-
dw = _build_writer()
|
|
633
|
-
snap = _snapshot([_resource("a"), _resource("b")])
|
|
634
|
-
mock_simple = _mk_simple_mock(snap)
|
|
635
|
-
MockSimple.return_value = mock_simple
|
|
636
|
-
|
|
637
|
-
new_res = [{"file": "c.parquet", "file_size": 5000, "columns": []}]
|
|
638
|
-
sunset = {"a", "b"}
|
|
639
|
-
mock_compact_res.return_value = (2, 200, new_res, sunset)
|
|
640
|
-
dw._get_table_config = MagicMock(return_value={})
|
|
641
|
-
|
|
642
|
-
dw.compact("admin", "tbl")
|
|
643
|
-
|
|
644
|
-
# parquet enqueue called with the sunset paths
|
|
645
|
-
parquet_calls = [
|
|
646
|
-
c for c in mock_enqueue.call_args_list
|
|
647
|
-
if c.args[4] == "parquet"
|
|
648
|
-
]
|
|
649
|
-
assert len(parquet_calls) == 1
|
|
650
|
-
assert set(parquet_calls[0].args[5]) == sunset
|
|
651
|
-
|
|
652
|
-
@patch(_P_AUDIT)
|
|
653
|
-
@patch(_P_MON_WRITER)
|
|
654
|
-
@patch(_P_MIRROR)
|
|
655
|
-
@patch(_P_ENQUEUE)
|
|
656
|
-
@patch(_P_COLLECT_OLD, return_value=[])
|
|
657
|
-
@patch(_P_COMPACT_RES)
|
|
658
|
-
@patch(_P_COMPACT_TOMB, return_value=(0, [], set()))
|
|
659
|
-
@patch(_P_SETTINGS)
|
|
660
|
-
@patch(_P_SIMPLE_TABLE)
|
|
661
|
-
@patch(_P_CHECK_WRITE)
|
|
662
|
-
def test_no_gc_enqueue_when_flags_off(
|
|
663
|
-
self, mock_check_write, MockSimple, mock_settings_obj,
|
|
664
|
-
mock_compact_tomb, mock_compact_res, mock_collect_old,
|
|
665
|
-
mock_enqueue, MockMirror, MockMW, mock_audit,
|
|
666
|
-
):
|
|
667
|
-
mock_settings_obj.SUPERTABLE_SUNSET_GC_ENABLED = False
|
|
668
|
-
mock_settings_obj.SUPERTABLE_SNAPSHOT_RETENTION = 0
|
|
669
|
-
|
|
670
|
-
dw = _build_writer()
|
|
671
|
-
snap = _snapshot([_resource("a")])
|
|
672
|
-
MockSimple.return_value = _mk_simple_mock(snap)
|
|
673
|
-
mock_compact_res.return_value = (1, 100, [{"file": "c", "file_size": 1, "columns": []}], {"a"})
|
|
674
|
-
dw._get_table_config = MagicMock(return_value={})
|
|
675
|
-
|
|
676
|
-
dw.compact("admin", "tbl")
|
|
677
|
-
|
|
678
|
-
mock_enqueue.assert_not_called()
|
|
679
|
-
|
|
680
|
-
|
|
681
602
|
# ===========================================================================
|
|
682
603
|
# 7. Monitoring lifecycle
|
|
683
604
|
# ===========================================================================
|
|
@@ -121,12 +121,6 @@ def _all_helpers() -> list[tuple[str, str, str]]:
|
|
|
121
121
|
# ---- Schema -----------------------------------------------------
|
|
122
122
|
("schema", RK.schema(ORG, SUP, SIMPLE), f"{lake_pre}:schema:doc:{SIMPLE}"),
|
|
123
123
|
|
|
124
|
-
# ---- GC pending (deferred-deletion stream) ---------------------
|
|
125
|
-
("gc_pending", RK.gc_pending(ORG, SUP, SIMPLE), f"{lake_pre}:gc:pending:doc:{SIMPLE}"),
|
|
126
|
-
("gc_pending_pattern", RK.gc_pending_pattern(ORG, SUP), f"{lake_pre}:gc:pending:doc:*"),
|
|
127
|
-
("gc_pending_pattern_for_org", RK.gc_pending_pattern_for_org(ORG), f"supertable:{ORG}:lakes:*:gc:pending:doc:*"),
|
|
128
|
-
("gc_pending_pattern_all_orgs", RK.gc_pending_pattern_all_orgs(), "supertable:*:lakes:*:gc:pending:doc:*"),
|
|
129
|
-
|
|
130
124
|
# ---- Linked shares ---------------------------------------------
|
|
131
125
|
("linked_share_index", RK.linked_share_index(ORG, SUP), f"{lake_pre}:linked_shares:index"),
|
|
132
126
|
("linked_share_doc", RK.linked_share_doc(ORG, SUP, LINK), f"{lake_pre}:linked_shares:doc:{LINK}"),
|
|
@@ -421,20 +415,6 @@ def test_parse_registry_key_returns_none_for_other_keys():
|
|
|
421
415
|
assert RK.parse_registry_key(RK.app_master_mcp("lighthouse")) is None
|
|
422
416
|
|
|
423
417
|
|
|
424
|
-
def test_parse_gc_pending_key_extracts_fields():
|
|
425
|
-
key = RK.gc_pending("acme", "demo", "orders")
|
|
426
|
-
assert RK.parse_gc_pending_key(key) == ("acme", "demo", "orders")
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
def test_parse_gc_pending_key_returns_none_for_other_keys():
|
|
430
|
-
assert RK.parse_gc_pending_key(RK.meta_root("acme", "demo")) is None
|
|
431
|
-
assert RK.parse_gc_pending_key(RK.meta_leaf("acme", "demo", "orders")) is None
|
|
432
|
-
assert RK.parse_gc_pending_key(RK.lock_leaf("acme", "demo", "orders")) is None
|
|
433
|
-
assert RK.parse_gc_pending_key("supertable:acme:lakes:demo:gc:pending:doc:") is None
|
|
434
|
-
assert RK.parse_gc_pending_key("") is None
|
|
435
|
-
assert RK.parse_gc_pending_key(None) is None # type: ignore[arg-type]
|
|
436
|
-
|
|
437
|
-
|
|
438
418
|
def test_parse_monitor_partition_key_extracts_fields():
|
|
439
419
|
key = RK.monitor_partition("acme", "writes", "2026-06-09")
|
|
440
420
|
assert RK.parse_monitor_partition_key(key) == ("acme", "writes", "2026-06-09")
|
|
@@ -442,7 +422,6 @@ def test_parse_monitor_partition_key_extracts_fields():
|
|
|
442
422
|
|
|
443
423
|
def test_parse_monitor_partition_key_returns_none_for_other_keys():
|
|
444
424
|
assert RK.parse_monitor_partition_key(RK.meta_root("acme", "demo")) is None
|
|
445
|
-
assert RK.parse_monitor_partition_key(RK.gc_pending("acme", "demo", "t")) is None
|
|
446
425
|
# Drain handle is rejected (7 segments, not 6)
|
|
447
426
|
assert RK.parse_monitor_partition_key(
|
|
448
427
|
RK.monitor_partition_drain("acme", "writes", "2026-06-09")
|
|
@@ -107,10 +107,6 @@ supertable/engine/tests/__init__.py
|
|
|
107
107
|
supertable/engine/tests/conftest.py
|
|
108
108
|
supertable/engine/tests/test_dedup_read.py
|
|
109
109
|
supertable/engine/tests/test_engine.py
|
|
110
|
-
supertable/gc/__init__.py
|
|
111
|
-
supertable/gc/cleaner.py
|
|
112
|
-
supertable/gc/daemon.py
|
|
113
|
-
supertable/gc/queue.py
|
|
114
110
|
supertable/locking/__init__.py
|
|
115
111
|
supertable/locking/file_lock.py
|
|
116
112
|
supertable/locking/redis_lock.py
|
|
@@ -158,9 +154,6 @@ supertable/tests/test_data_writer_comprehensive.py
|
|
|
158
154
|
supertable/tests/test_data_writer_tombstones.py
|
|
159
155
|
supertable/tests/test_dedup_on_read_write.py
|
|
160
156
|
supertable/tests/test_errors.py
|
|
161
|
-
supertable/tests/test_gc_cleaner.py
|
|
162
|
-
supertable/tests/test_gc_daemon.py
|
|
163
|
-
supertable/tests/test_gc_queue.py
|
|
164
157
|
supertable/tests/test_meta_reader.py
|
|
165
158
|
supertable/tests/test_monitoring_partitions.py
|
|
166
159
|
supertable/tests/test_monitoring_sink_guard.py
|
|
@@ -1,26 +0,0 @@
|
|
|
1
|
-
# route: supertable.gc
|
|
2
|
-
"""
|
|
3
|
-
Deferred-deletion garbage collection for SuperTable storage.
|
|
4
|
-
|
|
5
|
-
The data writer never deletes files inline. When the GC flags are enabled
|
|
6
|
-
(``SUPERTABLE_SUNSET_GC_ENABLED`` / ``SUPERTABLE_SNAPSHOT_RETENTION``) the
|
|
7
|
-
writer XADDs paths to a per-table Redis STREAM after a successful leaf-CAS.
|
|
8
|
-
A long-running ``GCCleaner`` daemon (one per org) drains entries older than
|
|
9
|
-
``SUPERTABLE_GC_DELAY_SEC`` and calls ``storage.delete()`` on each path.
|
|
10
|
-
|
|
11
|
-
The delay window is what makes deletion safe under concurrent reads: a
|
|
12
|
-
query that resolved the leaf payload right before the writer committed
|
|
13
|
-
finishes long before the cleaner touches the file.
|
|
14
|
-
"""
|
|
15
|
-
|
|
16
|
-
from supertable.gc.queue import (
|
|
17
|
-
collect_old_snapshot_paths,
|
|
18
|
-
enqueue_deletions,
|
|
19
|
-
nuke_stream,
|
|
20
|
-
)
|
|
21
|
-
|
|
22
|
-
__all__ = [
|
|
23
|
-
"collect_old_snapshot_paths",
|
|
24
|
-
"enqueue_deletions",
|
|
25
|
-
"nuke_stream",
|
|
26
|
-
]
|