supertable 2.2.2__tar.gz → 2.2.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (184) hide show
  1. {supertable-2.2.2/supertable.egg-info → supertable-2.2.4}/PKG-INFO +1 -1
  2. {supertable-2.2.2 → supertable-2.2.4}/pyproject.toml +1 -1
  3. {supertable-2.2.2 → supertable-2.2.4}/setup.py +1 -1
  4. {supertable-2.2.2 → supertable-2.2.4}/supertable/__init__.py +1 -1
  5. {supertable-2.2.2 → supertable-2.2.4}/supertable/config/settings.py +0 -22
  6. {supertable-2.2.2 → supertable-2.2.4}/supertable/data_writer.py +11 -79
  7. {supertable-2.2.2 → supertable-2.2.4}/supertable/redis_catalog.py +1 -6
  8. {supertable-2.2.2 → supertable-2.2.4}/supertable/redis_keys.py +0 -77
  9. {supertable-2.2.2 → supertable-2.2.4}/supertable/simple_table.py +50 -0
  10. {supertable-2.2.2 → supertable-2.2.4}/supertable/tests/test_data_writer_compact.py +3 -82
  11. {supertable-2.2.2 → supertable-2.2.4}/supertable/tests/test_redis_key_prefix.py +0 -21
  12. {supertable-2.2.2 → supertable-2.2.4/supertable.egg-info}/PKG-INFO +1 -1
  13. {supertable-2.2.2 → supertable-2.2.4}/supertable.egg-info/SOURCES.txt +0 -7
  14. supertable-2.2.2/supertable/gc/__init__.py +0 -26
  15. supertable-2.2.2/supertable/gc/cleaner.py +0 -258
  16. supertable-2.2.2/supertable/gc/daemon.py +0 -284
  17. supertable-2.2.2/supertable/gc/queue.py +0 -221
  18. supertable-2.2.2/supertable/tests/test_gc_cleaner.py +0 -330
  19. supertable-2.2.2/supertable/tests/test_gc_daemon.py +0 -251
  20. supertable-2.2.2/supertable/tests/test_gc_queue.py +0 -331
  21. {supertable-2.2.2 → supertable-2.2.4}/LICENSE +0 -0
  22. {supertable-2.2.2 → supertable-2.2.4}/README.md +0 -0
  23. {supertable-2.2.2 → supertable-2.2.4}/requirements.txt +0 -0
  24. {supertable-2.2.2 → supertable-2.2.4}/setup.cfg +0 -0
  25. {supertable-2.2.2 → supertable-2.2.4}/supertable/audit/__init__.py +0 -0
  26. {supertable-2.2.2 → supertable-2.2.4}/supertable/audit/admin.py +0 -0
  27. {supertable-2.2.2 → supertable-2.2.4}/supertable/audit/chain.py +0 -0
  28. {supertable-2.2.2 → supertable-2.2.4}/supertable/audit/consumers.py +0 -0
  29. {supertable-2.2.2 → supertable-2.2.4}/supertable/audit/crypto.py +0 -0
  30. {supertable-2.2.2 → supertable-2.2.4}/supertable/audit/events.py +0 -0
  31. {supertable-2.2.2 → supertable-2.2.4}/supertable/audit/export.py +0 -0
  32. {supertable-2.2.2 → supertable-2.2.4}/supertable/audit/logger.py +0 -0
  33. {supertable-2.2.2 → supertable-2.2.4}/supertable/audit/middleware.py +0 -0
  34. {supertable-2.2.2 → supertable-2.2.4}/supertable/audit/reader.py +0 -0
  35. {supertable-2.2.2 → supertable-2.2.4}/supertable/audit/retention.py +0 -0
  36. {supertable-2.2.2 → supertable-2.2.4}/supertable/audit/tests/__init__.py +0 -0
  37. {supertable-2.2.2 → supertable-2.2.4}/supertable/audit/tests/test_chain.py +0 -0
  38. {supertable-2.2.2 → supertable-2.2.4}/supertable/audit/tests/test_crypto.py +0 -0
  39. {supertable-2.2.2 → supertable-2.2.4}/supertable/audit/tests/test_emit.py +0 -0
  40. {supertable-2.2.2 → supertable-2.2.4}/supertable/audit/tests/test_events.py +0 -0
  41. {supertable-2.2.2 → supertable-2.2.4}/supertable/audit/tests/test_retention.py +0 -0
  42. {supertable-2.2.2 → supertable-2.2.4}/supertable/audit/writer_parquet.py +0 -0
  43. {supertable-2.2.2 → supertable-2.2.4}/supertable/audit/writer_redis.py +0 -0
  44. {supertable-2.2.2 → supertable-2.2.4}/supertable/config/__init__.py +0 -0
  45. {supertable-2.2.2 → supertable-2.2.4}/supertable/config/defaults.py +0 -0
  46. {supertable-2.2.2 → supertable-2.2.4}/supertable/config/homedir.py +0 -0
  47. {supertable-2.2.2 → supertable-2.2.4}/supertable/config/tests/__init__.py +0 -0
  48. {supertable-2.2.2 → supertable-2.2.4}/supertable/config/tests/test_defaults.py +0 -0
  49. {supertable-2.2.2 → supertable-2.2.4}/supertable/config/tests/test_homedir.py +0 -0
  50. {supertable-2.2.2 → supertable-2.2.4}/supertable/config/tests/test_settings.py +0 -0
  51. {supertable-2.2.2 → supertable-2.2.4}/supertable/data_classes.py +0 -0
  52. {supertable-2.2.2 → supertable-2.2.4}/supertable/data_reader.py +0 -0
  53. {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/__init__.py +0 -0
  54. {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/__init__.py +0 -0
  55. {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/__main__.py +0 -0
  56. {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/check_filter_builder.py +0 -0
  57. {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/controller.py +0 -0
  58. {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/data_writer_helpers.py +0 -0
  59. {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/defaults.py +0 -0
  60. {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/dummy_data.py +0 -0
  61. {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/read_parquet_header.py +0 -0
  62. {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/s01_01_01_create_super_table.py +0 -0
  63. {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/s01_01_02_enable_mirroring_formats.py +0 -0
  64. {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/s01_02_create_roles.py +0 -0
  65. {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/s01_03_create_users.py +0 -0
  66. {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/s02_01_write_dummy_data.py +0 -0
  67. {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/s02_02_write_single_data.py +0 -0
  68. {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/s02_03_01_write_staging.py +0 -0
  69. {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/s02_03_02_create_pipe.py +0 -0
  70. {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/s02_04_01_write_monitoring_simple.py +0 -0
  71. {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/s02_04_02_write_monitoring_parallel.py +0 -0
  72. {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/s02_05_write_tombstone.py +0 -0
  73. {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/s03_01_read_data_error.py +0 -0
  74. {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/s03_02_01_read_super_data_ok.py +0 -0
  75. {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/s03_02_02_read_table_data_ok.py +0 -0
  76. {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/s03_03_read_meta.py +0 -0
  77. {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/s03_04_read_staging.py +0 -0
  78. {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/s03_06_01_read_roles.py +0 -0
  79. {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/s03_06_02_read_user.py +0 -0
  80. {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/s03_07_01_estimate_read.py +0 -0
  81. {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/s03_07_02_estimate_files.py +0 -0
  82. {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/s03_08_read_snapshot_history.py +0 -0
  83. {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/s04_01_03_delete_pipe.py +0 -0
  84. {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/s05_01_delete_table.py +0 -0
  85. {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/quickstart/s05_02_delete_super_table.py +0 -0
  86. {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/webshop/__init__.py +0 -0
  87. {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/webshop/core.py +0 -0
  88. {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/webshop/defaults.py +0 -0
  89. {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/webshop/generate.py +0 -0
  90. {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/webshop/load.py +0 -0
  91. {supertable-2.2.2 → supertable-2.2.4}/supertable/demo/webshop/topup.py +0 -0
  92. {supertable-2.2.2 → supertable-2.2.4}/supertable/engine/__init__.py +0 -0
  93. {supertable-2.2.2 → supertable-2.2.4}/supertable/engine/data_estimator.py +0 -0
  94. {supertable-2.2.2 → supertable-2.2.4}/supertable/engine/duckdb_lite.py +0 -0
  95. {supertable-2.2.2 → supertable-2.2.4}/supertable/engine/duckdb_pro.py +0 -0
  96. {supertable-2.2.2 → supertable-2.2.4}/supertable/engine/engine_common.py +0 -0
  97. {supertable-2.2.2 → supertable-2.2.4}/supertable/engine/engine_enum.py +0 -0
  98. {supertable-2.2.2 → supertable-2.2.4}/supertable/engine/executor.py +0 -0
  99. {supertable-2.2.2 → supertable-2.2.4}/supertable/engine/plan_stats.py +0 -0
  100. {supertable-2.2.2 → supertable-2.2.4}/supertable/engine/spark_thrift.py +0 -0
  101. {supertable-2.2.2 → supertable-2.2.4}/supertable/engine/tests/__init__.py +0 -0
  102. {supertable-2.2.2 → supertable-2.2.4}/supertable/engine/tests/conftest.py +0 -0
  103. {supertable-2.2.2 → supertable-2.2.4}/supertable/engine/tests/test_dedup_read.py +0 -0
  104. {supertable-2.2.2 → supertable-2.2.4}/supertable/engine/tests/test_engine.py +0 -0
  105. {supertable-2.2.2 → supertable-2.2.4}/supertable/errors.py +0 -0
  106. {supertable-2.2.2 → supertable-2.2.4}/supertable/locking/__init__.py +0 -0
  107. {supertable-2.2.2 → supertable-2.2.4}/supertable/locking/benchmarks/__init__.py +0 -0
  108. {supertable-2.2.2 → supertable-2.2.4}/supertable/locking/benchmarks/benchmark_locking.py +0 -0
  109. {supertable-2.2.2 → supertable-2.2.4}/supertable/locking/benchmarks/measure_lock_speed.py +0 -0
  110. {supertable-2.2.2 → supertable-2.2.4}/supertable/locking/benchmarks/measure_lock_time.py +0 -0
  111. {supertable-2.2.2 → supertable-2.2.4}/supertable/locking/file_lock.py +0 -0
  112. {supertable-2.2.2 → supertable-2.2.4}/supertable/locking/redis_lock.py +0 -0
  113. {supertable-2.2.2 → supertable-2.2.4}/supertable/locking/tests/__init__.py +0 -0
  114. {supertable-2.2.2 → supertable-2.2.4}/supertable/locking/tests/test_file_lock.py +0 -0
  115. {supertable-2.2.2 → supertable-2.2.4}/supertable/locking/tests/test_redis_lock.py +0 -0
  116. {supertable-2.2.2 → supertable-2.2.4}/supertable/logging.py +0 -0
  117. {supertable-2.2.2 → supertable-2.2.4}/supertable/meta_reader.py +0 -0
  118. {supertable-2.2.2 → supertable-2.2.4}/supertable/mirroring/__init__.py +0 -0
  119. {supertable-2.2.2 → supertable-2.2.4}/supertable/mirroring/mirror_delta.py +0 -0
  120. {supertable-2.2.2 → supertable-2.2.4}/supertable/mirroring/mirror_formats.py +0 -0
  121. {supertable-2.2.2 → supertable-2.2.4}/supertable/mirroring/mirror_iceberg.py +0 -0
  122. {supertable-2.2.2 → supertable-2.2.4}/supertable/mirroring/mirror_parquet.py +0 -0
  123. {supertable-2.2.2 → supertable-2.2.4}/supertable/monitoring/__init__.py +0 -0
  124. {supertable-2.2.2 → supertable-2.2.4}/supertable/monitoring/partitions.py +0 -0
  125. {supertable-2.2.2 → supertable-2.2.4}/supertable/monitoring_writer.py +0 -0
  126. {supertable-2.2.2 → supertable-2.2.4}/supertable/plan_extender.py +0 -0
  127. {supertable-2.2.2 → supertable-2.2.4}/supertable/processing.py +0 -0
  128. {supertable-2.2.2 → supertable-2.2.4}/supertable/query_plan_manager.py +0 -0
  129. {supertable-2.2.2 → supertable-2.2.4}/supertable/rbac/__init__.py +0 -0
  130. {supertable-2.2.2 → supertable-2.2.4}/supertable/rbac/access_control.py +0 -0
  131. {supertable-2.2.2 → supertable-2.2.4}/supertable/rbac/filter_builder.py +0 -0
  132. {supertable-2.2.2 → supertable-2.2.4}/supertable/rbac/permissions.py +0 -0
  133. {supertable-2.2.2 → supertable-2.2.4}/supertable/rbac/role_manager.py +0 -0
  134. {supertable-2.2.2 → supertable-2.2.4}/supertable/rbac/row_column_security.py +0 -0
  135. {supertable-2.2.2 → supertable-2.2.4}/supertable/rbac/tests/test_filter_builder.py +0 -0
  136. {supertable-2.2.2 → supertable-2.2.4}/supertable/rbac/tests/test_rbac.py +0 -0
  137. {supertable-2.2.2 → supertable-2.2.4}/supertable/rbac/tests/test_rbac_per_table.py +0 -0
  138. {supertable-2.2.2 → supertable-2.2.4}/supertable/rbac/user_manager.py +0 -0
  139. {supertable-2.2.2 → supertable-2.2.4}/supertable/redis_connector.py +0 -0
  140. {supertable-2.2.2 → supertable-2.2.4}/supertable/redis_infra.py +0 -0
  141. {supertable-2.2.2 → supertable-2.2.4}/supertable/staging_area.py +0 -0
  142. {supertable-2.2.2 → supertable-2.2.4}/supertable/storage/__init__.py +0 -0
  143. {supertable-2.2.2 → supertable-2.2.4}/supertable/storage/azure_storage.py +0 -0
  144. {supertable-2.2.2 → supertable-2.2.4}/supertable/storage/gcp_storage.py +0 -0
  145. {supertable-2.2.2 → supertable-2.2.4}/supertable/storage/local_storage.py +0 -0
  146. {supertable-2.2.2 → supertable-2.2.4}/supertable/storage/minio_storage.py +0 -0
  147. {supertable-2.2.2 → supertable-2.2.4}/supertable/storage/s3_storage.py +0 -0
  148. {supertable-2.2.2 → supertable-2.2.4}/supertable/storage/storage_factory.py +0 -0
  149. {supertable-2.2.2 → supertable-2.2.4}/supertable/storage/storage_interface.py +0 -0
  150. {supertable-2.2.2 → supertable-2.2.4}/supertable/storage/tests/test_storage.py +0 -0
  151. {supertable-2.2.2 → supertable-2.2.4}/supertable/super_pipe.py +0 -0
  152. {supertable-2.2.2 → supertable-2.2.4}/supertable/super_table.py +0 -0
  153. {supertable-2.2.2 → supertable-2.2.4}/supertable/tests/__init__.py +0 -0
  154. {supertable-2.2.2 → supertable-2.2.4}/supertable/tests/test_align_to_schema_fix.py +0 -0
  155. {supertable-2.2.2 → supertable-2.2.4}/supertable/tests/test_create_if_missing.py +0 -0
  156. {supertable-2.2.2 → supertable-2.2.4}/supertable/tests/test_data_reader.py +0 -0
  157. {supertable-2.2.2 → supertable-2.2.4}/supertable/tests/test_data_reader_preflight.py +0 -0
  158. {supertable-2.2.2 → supertable-2.2.4}/supertable/tests/test_data_writer.py +0 -0
  159. {supertable-2.2.2 → supertable-2.2.4}/supertable/tests/test_data_writer_comprehensive.py +0 -0
  160. {supertable-2.2.2 → supertable-2.2.4}/supertable/tests/test_data_writer_tombstones.py +0 -0
  161. {supertable-2.2.2 → supertable-2.2.4}/supertable/tests/test_dedup_on_read_write.py +0 -0
  162. {supertable-2.2.2 → supertable-2.2.4}/supertable/tests/test_errors.py +0 -0
  163. {supertable-2.2.2 → supertable-2.2.4}/supertable/tests/test_meta_reader.py +0 -0
  164. {supertable-2.2.2 → supertable-2.2.4}/supertable/tests/test_monitoring_partitions.py +0 -0
  165. {supertable-2.2.2 → supertable-2.2.4}/supertable/tests/test_monitoring_sink_guard.py +0 -0
  166. {supertable-2.2.2 → supertable-2.2.4}/supertable/tests/test_newer_than.py +0 -0
  167. {supertable-2.2.2 → supertable-2.2.4}/supertable/tests/test_process_delete_only.py +0 -0
  168. {supertable-2.2.2 → supertable-2.2.4}/supertable/tests/test_processing.py +0 -0
  169. {supertable-2.2.2 → supertable-2.2.4}/supertable/tests/test_processing_compact_resources.py +0 -0
  170. {supertable-2.2.2 → supertable-2.2.4}/supertable/tests/test_query_sql.py +0 -0
  171. {supertable-2.2.2 → supertable-2.2.4}/supertable/tests/test_simple_table.py +0 -0
  172. {supertable-2.2.2 → supertable-2.2.4}/supertable/tests/test_small_file_compaction.py +0 -0
  173. {supertable-2.2.2 → supertable-2.2.4}/supertable/tests/test_super_table.py +0 -0
  174. {supertable-2.2.2 → supertable-2.2.4}/supertable/tests/test_supertable_all.py +0 -0
  175. {supertable-2.2.2 → supertable-2.2.4}/supertable/utils/__init__.py +0 -0
  176. {supertable-2.2.2 → supertable-2.2.4}/supertable/utils/helper.py +0 -0
  177. {supertable-2.2.2 → supertable-2.2.4}/supertable/utils/profiler.py +0 -0
  178. {supertable-2.2.2 → supertable-2.2.4}/supertable/utils/sql_parser.py +0 -0
  179. {supertable-2.2.2 → supertable-2.2.4}/supertable/utils/tests/test_sql_parser_columns.py +0 -0
  180. {supertable-2.2.2 → supertable-2.2.4}/supertable/utils/timer.py +0 -0
  181. {supertable-2.2.2 → supertable-2.2.4}/supertable.egg-info/dependency_links.txt +0 -0
  182. {supertable-2.2.2 → supertable-2.2.4}/supertable.egg-info/entry_points.txt +0 -0
  183. {supertable-2.2.2 → supertable-2.2.4}/supertable.egg-info/requires.txt +0 -0
  184. {supertable-2.2.2 → supertable-2.2.4}/supertable.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: supertable
3
- Version: 2.2.2
3
+ Version: 2.2.4
4
4
  Summary: SuperTable — versioned data lake library for SQL analytics on Parquet + Redis.
5
5
  Author: Levente Kupas
6
6
  Author-email: Levente Kupas <lkupas@kladnasoft.com>
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "supertable"
7
- version = "2.2.2"
7
+ version = "2.2.4"
8
8
  description = "SuperTable — versioned data lake library for SQL analytics on Parquet + Redis."
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.10"
@@ -19,7 +19,7 @@ long_description = readme.read_text(encoding="utf-8") if readme.exists() else ""
19
19
 
20
20
  setup(
21
21
  name="supertable",
22
- version="2.2.2",
22
+ version="2.2.4",
23
23
  description="SuperTable — versioned data lake library for SQL analytics on Parquet + Redis.",
24
24
  long_description=long_description,
25
25
  long_description_content_type="text/markdown",
@@ -25,7 +25,7 @@ See the ``supertable.demo`` package for runnable end-to-end demos and the
25
25
  project documentation for the full API surface.
26
26
  """
27
27
 
28
- __version__ = "2.2.2"
28
+ __version__ = "2.2.4"
29
29
 
30
30
  # Re-export the core public surface so users can do ``from supertable import …``
31
31
  # instead of remembering submodule paths.
@@ -282,21 +282,6 @@ class Settings:
282
282
  SUPERTABLE_SHARE_PRESIGN_TTL: int = 14400 # SUPERTABLE_SHARE_PRESIGN_TTL (seconds, default 4h)
283
283
  SUPERTABLE_SHARE_REFRESH_BUFFER: int = 600 # SUPERTABLE_SHARE_REFRESH_BUFFER (seconds, refresh 10min before expiry)
284
284
 
285
- # ── Storage GC (deferred deletion of sunset parquets + old snapshot JSONs) ──
286
- # Sunset parquet files and old snapshot JSONs are never deleted by the
287
- # writer directly. When either flag is enabled, the writer XADDs paths
288
- # to a per-table Redis STREAM after a successful leaf-CAS. A separate
289
- # cleaner daemon (``python -m supertable.gc.cleaner``) drains entries
290
- # older than ``SUPERTABLE_GC_DELAY_SEC`` and calls ``storage.delete()``.
291
- # The delay window avoids the race where an in-flight reader's
292
- # ``parquet_scan([...])`` resolves a file path right before the writer
293
- # deletes it. Defaults below preserve today's behaviour (off, unlimited).
294
- SUPERTABLE_SNAPSHOT_RETENTION: int = 0 # SUPERTABLE_SNAPSHOT_RETENTION (0 = keep all snapshot JSONs)
295
- SUPERTABLE_SUNSET_GC_ENABLED: bool = False # SUPERTABLE_SUNSET_GC_ENABLED (delete sunset parquets via queue)
296
- SUPERTABLE_GC_DELAY_SEC: int = 1800 # SUPERTABLE_GC_DELAY_SEC (min age before cleaner deletes)
297
- SUPERTABLE_GC_SLEEP_SEC: int = 60 # SUPERTABLE_GC_SLEEP_SEC (cleaner loop sleep)
298
- SUPERTABLE_GC_BATCH_SIZE: int = 500 # SUPERTABLE_GC_BATCH_SIZE (max entries processed per table per tick)
299
-
300
285
 
301
286
  # ── Convenience properties ───────────────────────────────────────
302
287
 
@@ -543,13 +528,6 @@ def _build_settings() -> Settings:
543
528
  # Data Sharing
544
529
  SUPERTABLE_SHARE_PRESIGN_TTL=_env_int("SUPERTABLE_SHARE_PRESIGN_TTL", 14400),
545
530
  SUPERTABLE_SHARE_REFRESH_BUFFER=_env_int("SUPERTABLE_SHARE_REFRESH_BUFFER", 600),
546
-
547
- # ── Storage GC ───────────────────────────────────────────────
548
- SUPERTABLE_SNAPSHOT_RETENTION=_env_int("SUPERTABLE_SNAPSHOT_RETENTION", 0),
549
- SUPERTABLE_SUNSET_GC_ENABLED=_env_bool("SUPERTABLE_SUNSET_GC_ENABLED", False),
550
- SUPERTABLE_GC_DELAY_SEC=_env_int("SUPERTABLE_GC_DELAY_SEC", 1800),
551
- SUPERTABLE_GC_SLEEP_SEC=_env_int("SUPERTABLE_GC_SLEEP_SEC", 60),
552
- SUPERTABLE_GC_BATCH_SIZE=_env_int("SUPERTABLE_GC_BATCH_SIZE", 500),
553
531
  )
554
532
 
555
533
 
@@ -13,7 +13,6 @@ from polars import DataFrame
13
13
  from supertable.config.defaults import logger
14
14
  from supertable.config.settings import settings
15
15
  from supertable.errors import SuperTableNotFoundError, TableNotFoundError
16
- from supertable.gc.queue import collect_old_snapshot_paths, enqueue_deletions
17
16
  from supertable.monitoring.partitions import MONITORING_SINK_TABLES
18
17
  from supertable.monitoring_writer import MonitoringWriter # async monitoring
19
18
  from supertable.super_table import SuperTable
@@ -571,46 +570,6 @@ class DataWriter:
571
570
  self.catalog.bump_root(self.super_table.organization, self.super_table.super_name, now_ms=now_ms)
572
571
  mark("bump_root")
573
572
 
574
- # --- Enqueue deferred deletions (post-CAS, post-bump) ----------
575
- # Strict ordering matters: leaf-CAS + root-bump have committed
576
- # the new snapshot as the authoritative state. Only NOW is it
577
- # safe to schedule physical deletion of files that the old
578
- # snapshot still references — any reader that loaded the
579
- # previous leaf payload will finish before the cleaner's delay
580
- # window expires.
581
- #
582
- # Both blocks are best-effort: enqueue_deletions and
583
- # collect_old_snapshot_paths swallow Redis/storage errors and
584
- # log a warning. A GC failure must never fail the write.
585
- try:
586
- _org = self.super_table.organization
587
- _sup = self.super_table.super_name
588
- if settings.SUPERTABLE_SUNSET_GC_ENABLED and sunset_files:
589
- enqueue_deletions(
590
- self.catalog,
591
- _org, _sup, simple_name,
592
- "parquet",
593
- list(sunset_files),
594
- write_id=qid,
595
- )
596
- if settings.SUPERTABLE_SNAPSHOT_RETENTION > 0:
597
- old_paths = collect_old_snapshot_paths(
598
- new_snapshot_dict,
599
- self.super_table.storage,
600
- settings.SUPERTABLE_SNAPSHOT_RETENTION,
601
- )
602
- if old_paths:
603
- enqueue_deletions(
604
- self.catalog,
605
- _org, _sup, simple_name,
606
- "snapshot",
607
- old_paths,
608
- write_id=qid,
609
- )
610
- except Exception as e:
611
- logger.warning(lp(f"GC enqueue failed (write still succeeded): {e}"))
612
- mark("gc_enqueue")
613
-
614
573
  # --- Store schema + table name in Redis (permanent, not cache) ---
615
574
  try:
616
575
  schema_raw = new_snapshot_dict.get("schema", {})
@@ -795,9 +754,14 @@ class DataWriter:
795
754
  qualify — large files are left untouched. ``small_only=False``
796
755
  rewrites every resource regardless of size.
797
756
  5. Commit the new snapshot via ``simple_table.update``,
798
- leaf-CAS, ``bump_root``, and the same GC enqueue +
757
+ leaf-CAS, ``bump_root``, optional mirroring, and the same
799
758
  monitoring + audit pipeline as ``write``.
800
759
 
760
+ Compaction does **not** garbage-collect: the small files it
761
+ merges away are left in storage (still referenced by older
762
+ snapshot versions for time-travel) for an external GC to
763
+ reclaim later.
764
+
801
765
  Concurrency: compaction is serialised against writes through
802
766
  the same per-simple lock. A concurrent ``write()`` either
803
767
  runs first (compaction sees the updated snapshot) or waits.
@@ -994,12 +958,10 @@ class DataWriter:
994
958
  # Critical filter: a file produced by Phase A (tombstone
995
959
  # compaction) can be picked up as a small-file candidate
996
960
  # by Phase B (compact_resources) and then sunset there.
997
- # If we leave it in ``all_new_resources`` it lands in the
998
- # new snapshot **and** in ``all_sunset`` (slated for GC
999
- # deletion). 30 min later GC deletes the file while the
1000
- # snapshot still references it readers fail with
1001
- # FileNotFoundError. Filter the file out of new_resources
1002
- # when it also appears in any sunset set.
961
+ # If we leave it in ``all_new_resources`` it would land in the
962
+ # new snapshot **and** in ``all_sunset`` at the same time.
963
+ # Filter the file out of new_resources when it also appears
964
+ # in any sunset set so the snapshot stays consistent.
1003
965
  all_sunset = set(tomb_sunset) | set(small_sunset)
1004
966
  all_new_resources = [
1005
967
  r for r in (list(tomb_new_resources) + list(small_new_resources))
@@ -1011,7 +973,7 @@ class DataWriter:
1011
973
  result["total_rows_written"] = small_total_rows
1012
974
 
1013
975
  # Short-circuit: nothing to do, skip the snapshot rewrite
1014
- # and the leaf-CAS / root-bump / GC / mirror path. BUT do
976
+ # and the leaf-CAS / root-bump / mirror path. BUT do
1015
977
  # NOT return here — execution must fall through to the
1016
978
  # ``finally`` (which releases the lock) AND the monitoring
1017
979
  # + audit emission blocks after the try/finally so the
@@ -1115,36 +1077,6 @@ class DataWriter:
1115
1077
  )
1116
1078
  mark("bump_root")
1117
1079
 
1118
- # --- GC enqueue (same flags as write) ----------------------------
1119
- try:
1120
- _org = self.super_table.organization
1121
- _sup = self.super_table.super_name
1122
- if settings.SUPERTABLE_SUNSET_GC_ENABLED and all_sunset:
1123
- enqueue_deletions(
1124
- self.catalog,
1125
- _org, _sup, simple_name,
1126
- "parquet",
1127
- list(all_sunset),
1128
- write_id=qid,
1129
- )
1130
- if settings.SUPERTABLE_SNAPSHOT_RETENTION > 0:
1131
- old_paths = collect_old_snapshot_paths(
1132
- new_snapshot_dict,
1133
- self.super_table.storage,
1134
- settings.SUPERTABLE_SNAPSHOT_RETENTION,
1135
- )
1136
- if old_paths:
1137
- enqueue_deletions(
1138
- self.catalog,
1139
- _org, _sup, simple_name,
1140
- "snapshot",
1141
- old_paths,
1142
- write_id=qid,
1143
- )
1144
- except Exception as e:
1145
- logger.warning(lp(f"GC enqueue failed (compact still succeeded): {e}"))
1146
- mark("gc_enqueue")
1147
-
1148
1080
  # --- Mirroring ---------------------------------------------------
1149
1081
  try:
1150
1082
  MirrorFormats.mirror_if_enabled(
@@ -1111,7 +1111,7 @@ return 1
1111
1111
  # ------------- Deletions (dangerous) -------------
1112
1112
 
1113
1113
  def delete_simple_table(self, org: str, sup: str, simple: str) -> bool:
1114
- """Delete a simple table's Redis meta (leaf pointer + lock + GC stream).
1114
+ """Delete a simple table's Redis meta (leaf pointer + lock).
1115
1115
 
1116
1116
  This does **not** delete storage. Callers should delete storage first, then call this.
1117
1117
  """
@@ -1120,11 +1120,6 @@ return 1
1120
1120
  keys = [
1121
1121
  RK.meta_leaf(org, sup, simple),
1122
1122
  RK.lock_leaf(org, sup, simple),
1123
- # Nuke the deferred-deletion queue too — once the table is gone
1124
- # there is no point draining pending deletions through the
1125
- # cleaner, and leaving the stream around makes
1126
- # SimpleTable.delete observably non-idempotent in Redis.
1127
- RK.gc_pending(org, sup, simple),
1128
1123
  ]
1129
1124
  try:
1130
1125
  # DEL returns number of keys removed
@@ -729,83 +729,6 @@ def linked_share_doc(org: str, sup: str, link_id: str) -> str:
729
729
  )
730
730
 
731
731
 
732
- # --- GC / Deferred deletion queue ------------------------------------------ #
733
- #
734
- # Per-table Redis STREAM holding paths scheduled for delayed physical
735
- # deletion (sunset parquet files, pruned snapshot JSONs). Producers
736
- # (``data_writer``) XADD entries after the leaf-CAS commit succeeds; the
737
- # org-level ``gc.cleaner`` daemon XRANGEs entries older than
738
- # ``SUPERTABLE_GC_DELAY_SEC`` and ``storage.delete()`` them, then XDELs.
739
- #
740
- # The stream lives under the lakes scope so a supertable-wide drop
741
- # (``catalog.delete_super_table``) naturally nukes it via the standard
742
- # ``super_table_pattern`` SCAN.
743
-
744
- def gc_pending(org: str, sup: str, simple: str) -> str:
745
- """Per-table deferred-deletion stream (STREAM)."""
746
- return (
747
- f"{SUPERTABLE_PREFIX}:{_safe('org', org)}:{LAKES_SCOPE}"
748
- f":{_safe('sup', sup)}:gc:pending:doc:{_safe('simple', simple)}"
749
- )
750
-
751
-
752
- def gc_pending_pattern(org: str, sup: str) -> str:
753
- """SCAN pattern matching every GC stream in one supertable."""
754
- return (
755
- f"{SUPERTABLE_PREFIX}:{_safe('org', org)}:{LAKES_SCOPE}"
756
- f":{_safe('sup', sup)}:gc:pending:doc:*"
757
- )
758
-
759
-
760
- def gc_pending_pattern_for_org(org: str) -> str:
761
- """SCAN pattern matching every GC stream across every supertable in one org.
762
-
763
- Used by the per-org cleaner daemon to discover all per-table streams
764
- it must drain.
765
- """
766
- return (
767
- f"{SUPERTABLE_PREFIX}:{_safe('org', org)}:{LAKES_SCOPE}"
768
- f":*:gc:pending:doc:*"
769
- )
770
-
771
-
772
- def gc_pending_pattern_all_orgs() -> str:
773
- """SCAN pattern matching every GC stream across every org.
774
-
775
- Used by the ``--all-orgs`` cleaner mode to discover the full set of
776
- orgs that have pending deletions.
777
- """
778
- return f"{SUPERTABLE_PREFIX}:*:{LAKES_SCOPE}:*:gc:pending:doc:*"
779
-
780
-
781
- def parse_gc_pending_key(key: str) -> Optional[Tuple[str, str, str]]:
782
- """Extract ``(org, sup, simple)`` from a GC pending stream key.
783
-
784
- Returns ``None`` when the key does not match the
785
- ``supertable:{org}:lakes:{sup}:gc:pending:doc:{simple}`` shape.
786
- This is the canonical way for the cleaner to walk SCAN output and
787
- recover the per-table coordinates without re-implementing the parse
788
- rules.
789
- """
790
- if not isinstance(key, str):
791
- return None
792
- parts = key.split(":")
793
- # supertable : {org} : lakes : {sup} : gc : pending : doc : {simple}
794
- if (
795
- len(parts) == 8
796
- and parts[0] == SUPERTABLE_PREFIX
797
- and parts[2] == LAKES_SCOPE
798
- and parts[4] == "gc"
799
- and parts[5] == "pending"
800
- and parts[6] == "doc"
801
- and parts[1]
802
- and parts[3]
803
- and parts[7]
804
- ):
805
- return parts[1], parts[3], parts[7]
806
- return None
807
-
808
-
809
732
  # --- Data Quality (dataisland-core quality module owns its sub-structure) #
810
733
 
811
734
  def quality_prefix(org: str, sup: str) -> str:
@@ -260,6 +260,56 @@ class SimpleTable:
260
260
  data = self.storage.read_json(path)
261
261
  return data, path
262
262
 
263
+ def export_to(self, target_dir: str, compression_level: int = 3, small_only: bool = False):
264
+ """Write a standalone copy of the current data into ``target_dir``.
265
+
266
+ Reads the current snapshot's parquet resources and re-writes them
267
+ as memory-bounded parquet chunks (each ~``max_memory_chunk_size``,
268
+ from the per-table config or the global default) under
269
+ ``target_dir``. This is a pure copy: it does NOT create a new
270
+ snapshot, advance the Redis leaf, or touch ``data/``/``snapshots/``.
271
+
272
+ Args:
273
+ target_dir: destination directory for the exported parquet
274
+ files (created if missing). Typically an
275
+ ``export/<timestamp>/`` folder next to ``data/``.
276
+ compression_level: zstd level for the exported parquet.
277
+ small_only: when False (default) every resource is read and
278
+ re-chunked; when True only files smaller than
279
+ ``max_memory_chunk_size`` are included.
280
+
281
+ Returns:
282
+ ``dict`` with ``files`` (list of written paths),
283
+ ``files_written``, ``total_rows`` and ``total_bytes``.
284
+ """
285
+ from supertable.processing import compact_resources
286
+
287
+ snapshot, _path = self.get_simple_table_snapshot()
288
+ table_config = self.catalog.get_table_config(
289
+ self.super_table.organization,
290
+ self.super_table.super_name,
291
+ self.simple_name,
292
+ ) or {}
293
+
294
+ _considered, total_rows, new_resources, _sunset = compact_resources(
295
+ snapshot=snapshot,
296
+ data_dir=target_dir,
297
+ compression_level=compression_level,
298
+ table_config=table_config,
299
+ small_only=small_only,
300
+ )
301
+
302
+ files = [r.get("file") for r in new_resources if isinstance(r, dict) and r.get("file")]
303
+ total_bytes = sum(
304
+ int(r.get("file_size") or 0) for r in new_resources if isinstance(r, dict)
305
+ )
306
+ return {
307
+ "files": files,
308
+ "files_written": len(files),
309
+ "total_rows": int(total_rows),
310
+ "total_bytes": int(total_bytes),
311
+ }
312
+
263
313
  def update(self, new_resources, sunset_files, model_df, last_snapshot=None, last_snapshot_path=None, lineage=None, profiler: Optional[Profiler] = None):
264
314
  """
265
315
  Build and write a new heavy snapshot on storage.
@@ -38,8 +38,6 @@ _P_COMPACT_TOMB = f"{_MOD}.compact_tombstones"
38
38
  _P_TOMB_THRESH = f"{_MOD}._tombstone_threshold"
39
39
  _P_MIRROR = f"{_MOD}.MirrorFormats"
40
40
  _P_MON_WRITER = f"{_MOD}.MonitoringWriter"
41
- _P_ENQUEUE = f"{_MOD}.enqueue_deletions"
42
- _P_COLLECT_OLD = f"{_MOD}.collect_old_snapshot_paths"
43
41
  _P_AUDIT = f"{_MOD}._audit_emit"
44
42
  _P_SETTINGS = f"{_MOD}.settings"
45
43
 
@@ -85,12 +83,9 @@ def _mk_simple_mock(snap: dict, *, snap_path: str = "/snap/v1.json",
85
83
  return mock
86
84
 
87
85
 
88
- def _stub_settings(*, sunset=False, retention=0):
89
- """Build a fake ``settings`` object with GC flags."""
90
- s = MagicMock()
91
- s.SUPERTABLE_SUNSET_GC_ENABLED = sunset
92
- s.SUPERTABLE_SNAPSHOT_RETENTION = retention
93
- return s
86
+ def _stub_settings():
87
+ """Build a fake ``settings`` object for patching."""
88
+ return MagicMock()
94
89
 
95
90
 
96
91
  def _build_writer():
@@ -604,80 +599,6 @@ class TestSnapshotCommit:
604
599
  dw.catalog.bump_root.assert_called_once()
605
600
 
606
601
 
607
- # ===========================================================================
608
- # 6. GC enqueue gating (same flags as write)
609
- # ===========================================================================
610
-
611
-
612
- class TestGCEnqueue:
613
-
614
- @patch(_P_AUDIT)
615
- @patch(_P_MON_WRITER)
616
- @patch(_P_MIRROR)
617
- @patch(_P_ENQUEUE)
618
- @patch(_P_COLLECT_OLD, return_value=[])
619
- @patch(_P_COMPACT_RES)
620
- @patch(_P_COMPACT_TOMB, return_value=(0, [], set()))
621
- @patch(_P_SETTINGS)
622
- @patch(_P_SIMPLE_TABLE)
623
- @patch(_P_CHECK_WRITE)
624
- def test_sunset_gc_enqueue_when_flag_on(
625
- self, mock_check_write, MockSimple, mock_settings_obj,
626
- mock_compact_tomb, mock_compact_res, mock_collect_old,
627
- mock_enqueue, MockMirror, MockMW, mock_audit,
628
- ):
629
- mock_settings_obj.SUPERTABLE_SUNSET_GC_ENABLED = True
630
- mock_settings_obj.SUPERTABLE_SNAPSHOT_RETENTION = 0
631
-
632
- dw = _build_writer()
633
- snap = _snapshot([_resource("a"), _resource("b")])
634
- mock_simple = _mk_simple_mock(snap)
635
- MockSimple.return_value = mock_simple
636
-
637
- new_res = [{"file": "c.parquet", "file_size": 5000, "columns": []}]
638
- sunset = {"a", "b"}
639
- mock_compact_res.return_value = (2, 200, new_res, sunset)
640
- dw._get_table_config = MagicMock(return_value={})
641
-
642
- dw.compact("admin", "tbl")
643
-
644
- # parquet enqueue called with the sunset paths
645
- parquet_calls = [
646
- c for c in mock_enqueue.call_args_list
647
- if c.args[4] == "parquet"
648
- ]
649
- assert len(parquet_calls) == 1
650
- assert set(parquet_calls[0].args[5]) == sunset
651
-
652
- @patch(_P_AUDIT)
653
- @patch(_P_MON_WRITER)
654
- @patch(_P_MIRROR)
655
- @patch(_P_ENQUEUE)
656
- @patch(_P_COLLECT_OLD, return_value=[])
657
- @patch(_P_COMPACT_RES)
658
- @patch(_P_COMPACT_TOMB, return_value=(0, [], set()))
659
- @patch(_P_SETTINGS)
660
- @patch(_P_SIMPLE_TABLE)
661
- @patch(_P_CHECK_WRITE)
662
- def test_no_gc_enqueue_when_flags_off(
663
- self, mock_check_write, MockSimple, mock_settings_obj,
664
- mock_compact_tomb, mock_compact_res, mock_collect_old,
665
- mock_enqueue, MockMirror, MockMW, mock_audit,
666
- ):
667
- mock_settings_obj.SUPERTABLE_SUNSET_GC_ENABLED = False
668
- mock_settings_obj.SUPERTABLE_SNAPSHOT_RETENTION = 0
669
-
670
- dw = _build_writer()
671
- snap = _snapshot([_resource("a")])
672
- MockSimple.return_value = _mk_simple_mock(snap)
673
- mock_compact_res.return_value = (1, 100, [{"file": "c", "file_size": 1, "columns": []}], {"a"})
674
- dw._get_table_config = MagicMock(return_value={})
675
-
676
- dw.compact("admin", "tbl")
677
-
678
- mock_enqueue.assert_not_called()
679
-
680
-
681
602
  # ===========================================================================
682
603
  # 7. Monitoring lifecycle
683
604
  # ===========================================================================
@@ -121,12 +121,6 @@ def _all_helpers() -> list[tuple[str, str, str]]:
121
121
  # ---- Schema -----------------------------------------------------
122
122
  ("schema", RK.schema(ORG, SUP, SIMPLE), f"{lake_pre}:schema:doc:{SIMPLE}"),
123
123
 
124
- # ---- GC pending (deferred-deletion stream) ---------------------
125
- ("gc_pending", RK.gc_pending(ORG, SUP, SIMPLE), f"{lake_pre}:gc:pending:doc:{SIMPLE}"),
126
- ("gc_pending_pattern", RK.gc_pending_pattern(ORG, SUP), f"{lake_pre}:gc:pending:doc:*"),
127
- ("gc_pending_pattern_for_org", RK.gc_pending_pattern_for_org(ORG), f"supertable:{ORG}:lakes:*:gc:pending:doc:*"),
128
- ("gc_pending_pattern_all_orgs", RK.gc_pending_pattern_all_orgs(), "supertable:*:lakes:*:gc:pending:doc:*"),
129
-
130
124
  # ---- Linked shares ---------------------------------------------
131
125
  ("linked_share_index", RK.linked_share_index(ORG, SUP), f"{lake_pre}:linked_shares:index"),
132
126
  ("linked_share_doc", RK.linked_share_doc(ORG, SUP, LINK), f"{lake_pre}:linked_shares:doc:{LINK}"),
@@ -421,20 +415,6 @@ def test_parse_registry_key_returns_none_for_other_keys():
421
415
  assert RK.parse_registry_key(RK.app_master_mcp("lighthouse")) is None
422
416
 
423
417
 
424
- def test_parse_gc_pending_key_extracts_fields():
425
- key = RK.gc_pending("acme", "demo", "orders")
426
- assert RK.parse_gc_pending_key(key) == ("acme", "demo", "orders")
427
-
428
-
429
- def test_parse_gc_pending_key_returns_none_for_other_keys():
430
- assert RK.parse_gc_pending_key(RK.meta_root("acme", "demo")) is None
431
- assert RK.parse_gc_pending_key(RK.meta_leaf("acme", "demo", "orders")) is None
432
- assert RK.parse_gc_pending_key(RK.lock_leaf("acme", "demo", "orders")) is None
433
- assert RK.parse_gc_pending_key("supertable:acme:lakes:demo:gc:pending:doc:") is None
434
- assert RK.parse_gc_pending_key("") is None
435
- assert RK.parse_gc_pending_key(None) is None # type: ignore[arg-type]
436
-
437
-
438
418
  def test_parse_monitor_partition_key_extracts_fields():
439
419
  key = RK.monitor_partition("acme", "writes", "2026-06-09")
440
420
  assert RK.parse_monitor_partition_key(key) == ("acme", "writes", "2026-06-09")
@@ -442,7 +422,6 @@ def test_parse_monitor_partition_key_extracts_fields():
442
422
 
443
423
  def test_parse_monitor_partition_key_returns_none_for_other_keys():
444
424
  assert RK.parse_monitor_partition_key(RK.meta_root("acme", "demo")) is None
445
- assert RK.parse_monitor_partition_key(RK.gc_pending("acme", "demo", "t")) is None
446
425
  # Drain handle is rejected (7 segments, not 6)
447
426
  assert RK.parse_monitor_partition_key(
448
427
  RK.monitor_partition_drain("acme", "writes", "2026-06-09")
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: supertable
3
- Version: 2.2.2
3
+ Version: 2.2.4
4
4
  Summary: SuperTable — versioned data lake library for SQL analytics on Parquet + Redis.
5
5
  Author: Levente Kupas
6
6
  Author-email: Levente Kupas <lkupas@kladnasoft.com>
@@ -107,10 +107,6 @@ supertable/engine/tests/__init__.py
107
107
  supertable/engine/tests/conftest.py
108
108
  supertable/engine/tests/test_dedup_read.py
109
109
  supertable/engine/tests/test_engine.py
110
- supertable/gc/__init__.py
111
- supertable/gc/cleaner.py
112
- supertable/gc/daemon.py
113
- supertable/gc/queue.py
114
110
  supertable/locking/__init__.py
115
111
  supertable/locking/file_lock.py
116
112
  supertable/locking/redis_lock.py
@@ -158,9 +154,6 @@ supertable/tests/test_data_writer_comprehensive.py
158
154
  supertable/tests/test_data_writer_tombstones.py
159
155
  supertable/tests/test_dedup_on_read_write.py
160
156
  supertable/tests/test_errors.py
161
- supertable/tests/test_gc_cleaner.py
162
- supertable/tests/test_gc_daemon.py
163
- supertable/tests/test_gc_queue.py
164
157
  supertable/tests/test_meta_reader.py
165
158
  supertable/tests/test_monitoring_partitions.py
166
159
  supertable/tests/test_monitoring_sink_guard.py
@@ -1,26 +0,0 @@
1
- # route: supertable.gc
2
- """
3
- Deferred-deletion garbage collection for SuperTable storage.
4
-
5
- The data writer never deletes files inline. When the GC flags are enabled
6
- (``SUPERTABLE_SUNSET_GC_ENABLED`` / ``SUPERTABLE_SNAPSHOT_RETENTION``) the
7
- writer XADDs paths to a per-table Redis STREAM after a successful leaf-CAS.
8
- A long-running ``GCCleaner`` daemon (one per org) drains entries older than
9
- ``SUPERTABLE_GC_DELAY_SEC`` and calls ``storage.delete()`` on each path.
10
-
11
- The delay window is what makes deletion safe under concurrent reads: a
12
- query that resolved the leaf payload right before the writer committed
13
- finishes long before the cleaner touches the file.
14
- """
15
-
16
- from supertable.gc.queue import (
17
- collect_old_snapshot_paths,
18
- enqueue_deletions,
19
- nuke_stream,
20
- )
21
-
22
- __all__ = [
23
- "collect_old_snapshot_paths",
24
- "enqueue_deletions",
25
- "nuke_stream",
26
- ]