supertable 2.3.2__tar.gz → 2.3.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (186) hide show
  1. {supertable-2.3.2/supertable.egg-info → supertable-2.3.4}/PKG-INFO +1 -1
  2. {supertable-2.3.2 → supertable-2.3.4}/pyproject.toml +1 -1
  3. {supertable-2.3.2 → supertable-2.3.4}/setup.py +1 -1
  4. {supertable-2.3.2 → supertable-2.3.4}/supertable/__init__.py +1 -1
  5. {supertable-2.3.2 → supertable-2.3.4}/supertable/config/settings.py +21 -0
  6. {supertable-2.3.2 → supertable-2.3.4}/supertable/data_classes.py +9 -1
  7. {supertable-2.3.2 → supertable-2.3.4}/supertable/data_reader.py +3 -0
  8. {supertable-2.3.2 → supertable-2.3.4}/supertable/data_writer.py +166 -22
  9. {supertable-2.3.2 → supertable-2.3.4}/supertable/engine/duckdb_lite.py +29 -1
  10. {supertable-2.3.2 → supertable-2.3.4}/supertable/engine/duckdb_pro.py +34 -1
  11. {supertable-2.3.2 → supertable-2.3.4}/supertable/engine/engine_common.py +289 -7
  12. {supertable-2.3.2 → supertable-2.3.4}/supertable/engine/tests/test_engine.py +548 -0
  13. {supertable-2.3.2 → supertable-2.3.4}/supertable/processing.py +182 -21
  14. {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/test_data_reader.py +5 -0
  15. {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/test_data_writer.py +284 -0
  16. {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/test_processing_compact_resources.py +93 -0
  17. {supertable-2.3.2 → supertable-2.3.4/supertable.egg-info}/PKG-INFO +1 -1
  18. {supertable-2.3.2 → supertable-2.3.4}/LICENSE +0 -0
  19. {supertable-2.3.2 → supertable-2.3.4}/README.md +0 -0
  20. {supertable-2.3.2 → supertable-2.3.4}/requirements.txt +0 -0
  21. {supertable-2.3.2 → supertable-2.3.4}/setup.cfg +0 -0
  22. {supertable-2.3.2 → supertable-2.3.4}/supertable/audit/__init__.py +0 -0
  23. {supertable-2.3.2 → supertable-2.3.4}/supertable/audit/admin.py +0 -0
  24. {supertable-2.3.2 → supertable-2.3.4}/supertable/audit/chain.py +0 -0
  25. {supertable-2.3.2 → supertable-2.3.4}/supertable/audit/consumers.py +0 -0
  26. {supertable-2.3.2 → supertable-2.3.4}/supertable/audit/crypto.py +0 -0
  27. {supertable-2.3.2 → supertable-2.3.4}/supertable/audit/events.py +0 -0
  28. {supertable-2.3.2 → supertable-2.3.4}/supertable/audit/export.py +0 -0
  29. {supertable-2.3.2 → supertable-2.3.4}/supertable/audit/logger.py +0 -0
  30. {supertable-2.3.2 → supertable-2.3.4}/supertable/audit/middleware.py +0 -0
  31. {supertable-2.3.2 → supertable-2.3.4}/supertable/audit/reader.py +0 -0
  32. {supertable-2.3.2 → supertable-2.3.4}/supertable/audit/retention.py +0 -0
  33. {supertable-2.3.2 → supertable-2.3.4}/supertable/audit/tests/__init__.py +0 -0
  34. {supertable-2.3.2 → supertable-2.3.4}/supertable/audit/tests/test_chain.py +0 -0
  35. {supertable-2.3.2 → supertable-2.3.4}/supertable/audit/tests/test_crypto.py +0 -0
  36. {supertable-2.3.2 → supertable-2.3.4}/supertable/audit/tests/test_emit.py +0 -0
  37. {supertable-2.3.2 → supertable-2.3.4}/supertable/audit/tests/test_events.py +0 -0
  38. {supertable-2.3.2 → supertable-2.3.4}/supertable/audit/tests/test_retention.py +0 -0
  39. {supertable-2.3.2 → supertable-2.3.4}/supertable/audit/writer_parquet.py +0 -0
  40. {supertable-2.3.2 → supertable-2.3.4}/supertable/audit/writer_redis.py +0 -0
  41. {supertable-2.3.2 → supertable-2.3.4}/supertable/config/__init__.py +0 -0
  42. {supertable-2.3.2 → supertable-2.3.4}/supertable/config/defaults.py +0 -0
  43. {supertable-2.3.2 → supertable-2.3.4}/supertable/config/homedir.py +0 -0
  44. {supertable-2.3.2 → supertable-2.3.4}/supertable/config/tests/__init__.py +0 -0
  45. {supertable-2.3.2 → supertable-2.3.4}/supertable/config/tests/test_defaults.py +0 -0
  46. {supertable-2.3.2 → supertable-2.3.4}/supertable/config/tests/test_homedir.py +0 -0
  47. {supertable-2.3.2 → supertable-2.3.4}/supertable/config/tests/test_settings.py +0 -0
  48. {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/__init__.py +0 -0
  49. {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/__init__.py +0 -0
  50. {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/__main__.py +0 -0
  51. {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/check_filter_builder.py +0 -0
  52. {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/controller.py +0 -0
  53. {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/data_writer_helpers.py +0 -0
  54. {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/defaults.py +0 -0
  55. {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/dummy_data.py +0 -0
  56. {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/read_parquet_header.py +0 -0
  57. {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/s01_01_01_create_super_table.py +0 -0
  58. {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/s01_01_02_enable_mirroring_formats.py +0 -0
  59. {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/s01_02_create_roles.py +0 -0
  60. {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/s01_03_create_users.py +0 -0
  61. {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/s02_01_write_dummy_data.py +0 -0
  62. {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/s02_02_write_single_data.py +0 -0
  63. {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/s02_03_01_write_staging.py +0 -0
  64. {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/s02_03_02_create_pipe.py +0 -0
  65. {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/s02_04_01_write_monitoring_simple.py +0 -0
  66. {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/s02_04_02_write_monitoring_parallel.py +0 -0
  67. {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/s02_05_write_tombstone.py +0 -0
  68. {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/s03_01_read_data_error.py +0 -0
  69. {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/s03_02_01_read_super_data_ok.py +0 -0
  70. {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/s03_02_02_read_table_data_ok.py +0 -0
  71. {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/s03_03_read_meta.py +0 -0
  72. {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/s03_04_read_staging.py +0 -0
  73. {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/s03_06_01_read_roles.py +0 -0
  74. {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/s03_06_02_read_user.py +0 -0
  75. {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/s03_07_01_estimate_read.py +0 -0
  76. {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/s03_07_02_estimate_files.py +0 -0
  77. {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/s03_08_read_snapshot_history.py +0 -0
  78. {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/s04_01_03_delete_pipe.py +0 -0
  79. {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/s05_01_delete_table.py +0 -0
  80. {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/s05_02_delete_super_table.py +0 -0
  81. {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/webshop/__init__.py +0 -0
  82. {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/webshop/core.py +0 -0
  83. {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/webshop/defaults.py +0 -0
  84. {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/webshop/generate.py +0 -0
  85. {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/webshop/load.py +0 -0
  86. {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/webshop/topup.py +0 -0
  87. {supertable-2.3.2 → supertable-2.3.4}/supertable/engine/__init__.py +0 -0
  88. {supertable-2.3.2 → supertable-2.3.4}/supertable/engine/data_estimator.py +0 -0
  89. {supertable-2.3.2 → supertable-2.3.4}/supertable/engine/engine_config.py +0 -0
  90. {supertable-2.3.2 → supertable-2.3.4}/supertable/engine/engine_enum.py +0 -0
  91. {supertable-2.3.2 → supertable-2.3.4}/supertable/engine/executor.py +0 -0
  92. {supertable-2.3.2 → supertable-2.3.4}/supertable/engine/plan_stats.py +0 -0
  93. {supertable-2.3.2 → supertable-2.3.4}/supertable/engine/spark_thrift.py +0 -0
  94. {supertable-2.3.2 → supertable-2.3.4}/supertable/engine/tests/__init__.py +0 -0
  95. {supertable-2.3.2 → supertable-2.3.4}/supertable/engine/tests/conftest.py +0 -0
  96. {supertable-2.3.2 → supertable-2.3.4}/supertable/engine/tests/test_engine_config.py +0 -0
  97. {supertable-2.3.2 → supertable-2.3.4}/supertable/engine/tests/test_engine_routing.py +0 -0
  98. {supertable-2.3.2 → supertable-2.3.4}/supertable/engine/tests/test_engine_spill.py +0 -0
  99. {supertable-2.3.2 → supertable-2.3.4}/supertable/errors.py +0 -0
  100. {supertable-2.3.2 → supertable-2.3.4}/supertable/locking/__init__.py +0 -0
  101. {supertable-2.3.2 → supertable-2.3.4}/supertable/locking/benchmarks/__init__.py +0 -0
  102. {supertable-2.3.2 → supertable-2.3.4}/supertable/locking/benchmarks/benchmark_locking.py +0 -0
  103. {supertable-2.3.2 → supertable-2.3.4}/supertable/locking/benchmarks/measure_lock_speed.py +0 -0
  104. {supertable-2.3.2 → supertable-2.3.4}/supertable/locking/benchmarks/measure_lock_time.py +0 -0
  105. {supertable-2.3.2 → supertable-2.3.4}/supertable/locking/file_lock.py +0 -0
  106. {supertable-2.3.2 → supertable-2.3.4}/supertable/locking/redis_lock.py +0 -0
  107. {supertable-2.3.2 → supertable-2.3.4}/supertable/locking/tests/__init__.py +0 -0
  108. {supertable-2.3.2 → supertable-2.3.4}/supertable/locking/tests/test_file_lock.py +0 -0
  109. {supertable-2.3.2 → supertable-2.3.4}/supertable/locking/tests/test_redis_lock.py +0 -0
  110. {supertable-2.3.2 → supertable-2.3.4}/supertable/logging.py +0 -0
  111. {supertable-2.3.2 → supertable-2.3.4}/supertable/meta_reader.py +0 -0
  112. {supertable-2.3.2 → supertable-2.3.4}/supertable/mirroring/__init__.py +0 -0
  113. {supertable-2.3.2 → supertable-2.3.4}/supertable/mirroring/mirror_delta.py +0 -0
  114. {supertable-2.3.2 → supertable-2.3.4}/supertable/mirroring/mirror_formats.py +0 -0
  115. {supertable-2.3.2 → supertable-2.3.4}/supertable/mirroring/mirror_iceberg.py +0 -0
  116. {supertable-2.3.2 → supertable-2.3.4}/supertable/mirroring/mirror_parquet.py +0 -0
  117. {supertable-2.3.2 → supertable-2.3.4}/supertable/monitoring/__init__.py +0 -0
  118. {supertable-2.3.2 → supertable-2.3.4}/supertable/monitoring/partitions.py +0 -0
  119. {supertable-2.3.2 → supertable-2.3.4}/supertable/monitoring_writer.py +0 -0
  120. {supertable-2.3.2 → supertable-2.3.4}/supertable/plan_extender.py +0 -0
  121. {supertable-2.3.2 → supertable-2.3.4}/supertable/query_plan_manager.py +0 -0
  122. {supertable-2.3.2 → supertable-2.3.4}/supertable/rbac/__init__.py +0 -0
  123. {supertable-2.3.2 → supertable-2.3.4}/supertable/rbac/access_control.py +0 -0
  124. {supertable-2.3.2 → supertable-2.3.4}/supertable/rbac/filter_builder.py +0 -0
  125. {supertable-2.3.2 → supertable-2.3.4}/supertable/rbac/permissions.py +0 -0
  126. {supertable-2.3.2 → supertable-2.3.4}/supertable/rbac/role_manager.py +0 -0
  127. {supertable-2.3.2 → supertable-2.3.4}/supertable/rbac/row_column_security.py +0 -0
  128. {supertable-2.3.2 → supertable-2.3.4}/supertable/rbac/tests/test_filter_builder.py +0 -0
  129. {supertable-2.3.2 → supertable-2.3.4}/supertable/rbac/tests/test_rbac.py +0 -0
  130. {supertable-2.3.2 → supertable-2.3.4}/supertable/rbac/tests/test_rbac_per_table.py +0 -0
  131. {supertable-2.3.2 → supertable-2.3.4}/supertable/rbac/user_manager.py +0 -0
  132. {supertable-2.3.2 → supertable-2.3.4}/supertable/redis_catalog.py +0 -0
  133. {supertable-2.3.2 → supertable-2.3.4}/supertable/redis_connector.py +0 -0
  134. {supertable-2.3.2 → supertable-2.3.4}/supertable/redis_infra.py +0 -0
  135. {supertable-2.3.2 → supertable-2.3.4}/supertable/redis_keys.py +0 -0
  136. {supertable-2.3.2 → supertable-2.3.4}/supertable/simple_table.py +0 -0
  137. {supertable-2.3.2 → supertable-2.3.4}/supertable/staging_area.py +0 -0
  138. {supertable-2.3.2 → supertable-2.3.4}/supertable/storage/__init__.py +0 -0
  139. {supertable-2.3.2 → supertable-2.3.4}/supertable/storage/azure_storage.py +0 -0
  140. {supertable-2.3.2 → supertable-2.3.4}/supertable/storage/gcp_storage.py +0 -0
  141. {supertable-2.3.2 → supertable-2.3.4}/supertable/storage/local_storage.py +0 -0
  142. {supertable-2.3.2 → supertable-2.3.4}/supertable/storage/minio_storage.py +0 -0
  143. {supertable-2.3.2 → supertable-2.3.4}/supertable/storage/s3_storage.py +0 -0
  144. {supertable-2.3.2 → supertable-2.3.4}/supertable/storage/storage_factory.py +0 -0
  145. {supertable-2.3.2 → supertable-2.3.4}/supertable/storage/storage_interface.py +0 -0
  146. {supertable-2.3.2 → supertable-2.3.4}/supertable/storage/tests/test_storage.py +0 -0
  147. {supertable-2.3.2 → supertable-2.3.4}/supertable/super_pipe.py +0 -0
  148. {supertable-2.3.2 → supertable-2.3.4}/supertable/super_table.py +0 -0
  149. {supertable-2.3.2 → supertable-2.3.4}/supertable/system_query.py +0 -0
  150. {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/__init__.py +0 -0
  151. {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/test_align_to_schema_fix.py +0 -0
  152. {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/test_create_if_missing.py +0 -0
  153. {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/test_data_reader_preflight.py +0 -0
  154. {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/test_data_writer_compact.py +0 -0
  155. {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/test_data_writer_comprehensive.py +0 -0
  156. {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/test_errors.py +0 -0
  157. {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/test_meta_reader.py +0 -0
  158. {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/test_monitoring_partitions.py +0 -0
  159. {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/test_monitoring_sink_guard.py +0 -0
  160. {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/test_newer_than.py +0 -0
  161. {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/test_parquet_statistics.py +0 -0
  162. {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/test_processing.py +0 -0
  163. {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/test_processing_stats.py +0 -0
  164. {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/test_query_sql.py +0 -0
  165. {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/test_read_pruning_differential.py +0 -0
  166. {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/test_read_pruning_integration.py +0 -0
  167. {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/test_redis_key_prefix.py +0 -0
  168. {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/test_resolve_overwrite_writes.py +0 -0
  169. {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/test_simple_table.py +0 -0
  170. {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/test_stats_cache.py +0 -0
  171. {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/test_stats_pruning.py +0 -0
  172. {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/test_stats_schema_snapshot.py +0 -0
  173. {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/test_super_table.py +0 -0
  174. {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/test_supertable_all.py +0 -0
  175. {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/test_system_query.py +0 -0
  176. {supertable-2.3.2 → supertable-2.3.4}/supertable/utils/__init__.py +0 -0
  177. {supertable-2.3.2 → supertable-2.3.4}/supertable/utils/helper.py +0 -0
  178. {supertable-2.3.2 → supertable-2.3.4}/supertable/utils/profiler.py +0 -0
  179. {supertable-2.3.2 → supertable-2.3.4}/supertable/utils/sql_parser.py +0 -0
  180. {supertable-2.3.2 → supertable-2.3.4}/supertable/utils/tests/test_sql_parser_columns.py +0 -0
  181. {supertable-2.3.2 → supertable-2.3.4}/supertable/utils/timer.py +0 -0
  182. {supertable-2.3.2 → supertable-2.3.4}/supertable.egg-info/SOURCES.txt +0 -0
  183. {supertable-2.3.2 → supertable-2.3.4}/supertable.egg-info/dependency_links.txt +0 -0
  184. {supertable-2.3.2 → supertable-2.3.4}/supertable.egg-info/entry_points.txt +0 -0
  185. {supertable-2.3.2 → supertable-2.3.4}/supertable.egg-info/requires.txt +0 -0
  186. {supertable-2.3.2 → supertable-2.3.4}/supertable.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: supertable
3
- Version: 2.3.2
3
+ Version: 2.3.4
4
4
  Summary: SuperTable — versioned data lake library for SQL analytics on Parquet + Redis.
5
5
  Author: Levente Kupas
6
6
  Author-email: Levente Kupas <lkupas@kladnasoft.com>
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "supertable"
7
- version = "2.3.2"
7
+ version = "2.3.4"
8
8
  description = "SuperTable — versioned data lake library for SQL analytics on Parquet + Redis."
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.10"
@@ -19,7 +19,7 @@ long_description = readme.read_text(encoding="utf-8") if readme.exists() else ""
19
19
 
20
20
  setup(
21
21
  name="supertable",
22
- version="2.3.2",
22
+ version="2.3.4",
23
23
  description="SuperTable — versioned data lake library for SQL analytics on Parquet + Redis.",
24
24
  long_description=long_description,
25
25
  long_description_content_type="text/markdown",
@@ -25,7 +25,7 @@ See the ``supertable.demo`` package for runnable end-to-end demos and the
25
25
  project documentation for the full API surface.
26
26
  """
27
27
 
28
- __version__ = "2.3.2"
28
+ __version__ = "2.3.4"
29
29
 
30
30
  # Re-export the core public surface so users can do ``from supertable import …``
31
31
  # instead of remembering submodule paths.
@@ -157,6 +157,25 @@ class Settings:
157
157
  SUPERTABLE_DUCKDB_MATERIALIZE: str = "view" # SUPERTABLE_DUCKDB_MATERIALIZE
158
158
  SUPERTABLE_DUCKDB_PRESIGNED: bool = False # SUPERTABLE_DUCKDB_PRESIGNED
159
159
  SUPERTABLE_DUCKDB_USE_HTTPFS: bool = False # SUPERTABLE_DUCKDB_USE_HTTPFS
160
+ # Deletion-vector (tombstone) table cache. Each entry is a small
161
+ # `DISTINCT __rowid__` table keyed by the stable tombstone path; the
162
+ # tombstone view ANTI JOINs it instead of re-reading the parquet every
163
+ # query. Eviction is purely per-table — a churny table can never evict a
164
+ # slow table's cached deletion-vector:
165
+ # * Idle TTL (below): every entry, including a table's latest, is dropped
166
+ # once it goes unqueried for the TTL window.
167
+ # * Per-table cap (this knob): at most N most-recently-used versions are
168
+ # kept per table, so a burst of rewrites (e.g. 1000 updates in 5 min)
169
+ # retains only the last N rather than all of them.
170
+ # <= 0 disables the cache entirely (inline read_parquet fallback).
171
+ SUPERTABLE_DUCKDB_TOMBSTONE_CACHE_MAX_PER_TABLE: int = 8 # SUPERTABLE_DUCKDB_TOMBSTONE_CACHE_MAX_PER_TABLE
172
+ # Idle TTL (seconds): a cached deletion-vector is dropped once it has gone
173
+ # unqueried for this long; every query that uses it refreshes the timer.
174
+ # Applies to every entry (a table's latest included), so an abandoned table
175
+ # reclaims its cache instead of lingering until the connection resets.
176
+ # <= 0 keeps an entry only while a query references it (no persistence).
177
+ # Defaults to SUPERTABLE_ENGINE_FRESHNESS_SEC (300 s / 5 min).
178
+ SUPERTABLE_DUCKDB_TOMBSTONE_CACHE_TTL_SEC: int = 300 # SUPERTABLE_DUCKDB_TOMBSTONE_CACHE_TTL_SEC
160
179
  SUPERTABLE_DEBUG_TIMINGS: bool = False # SUPERTABLE_DEBUG_TIMINGS
161
180
 
162
181
  # ── Engine Routing / Executor ────────────────────────────────────
@@ -418,6 +437,8 @@ def _build_settings() -> Settings:
418
437
  SUPERTABLE_DUCKDB_MATERIALIZE=_env_str("SUPERTABLE_DUCKDB_MATERIALIZE", "view"),
419
438
  SUPERTABLE_DUCKDB_PRESIGNED=_env_bool("SUPERTABLE_DUCKDB_PRESIGNED", False),
420
439
  SUPERTABLE_DUCKDB_USE_HTTPFS=_env_bool("SUPERTABLE_DUCKDB_USE_HTTPFS", False),
440
+ SUPERTABLE_DUCKDB_TOMBSTONE_CACHE_MAX_PER_TABLE=_env_int("SUPERTABLE_DUCKDB_TOMBSTONE_CACHE_MAX_PER_TABLE", 8),
441
+ SUPERTABLE_DUCKDB_TOMBSTONE_CACHE_TTL_SEC=_env_int("SUPERTABLE_DUCKDB_TOMBSTONE_CACHE_TTL_SEC", 300),
421
442
  SUPERTABLE_DEBUG_TIMINGS=_env_bool("SUPERTABLE_DEBUG_TIMINGS", False),
422
443
 
423
444
  # ── Engine Routing ───────────────────────────────────────────
@@ -61,9 +61,17 @@ class TombstoneDef:
61
61
 
62
62
  - tombstone_path: storage path of the deletion-vector parquet
63
63
  (columns ``__file__`` + ``__rowid__``). ``None`` means no
64
- tombstone exists, so no anti-join is applied.
64
+ tombstone exists, so no anti-join is applied. This may be a
65
+ presigned/object-store URL that rotates per request, so it is
66
+ *not* stable enough to use as a cache key.
67
+ - cache_key: the bare, stable storage key of the same deletion-vector
68
+ parquet (no presign). Stable across pure appends (carry-forward
69
+ returns the previous tombstone), so DuckDB engines use it to key the
70
+ materialised deletion-vector table cache. ``None`` disables caching
71
+ for this alias (falls back to inline ``read_parquet``).
65
72
  """
66
73
  tombstone_path: Optional[str] = None
74
+ cache_key: Optional[str] = None
67
75
 
68
76
 
69
77
  @dataclass
@@ -322,6 +322,9 @@ class DataReader:
322
322
  # storage returns the key unchanged.
323
323
  reflection.tombstone_views[td.alias] = TombstoneDef(
324
324
  tombstone_path=estimator._to_duckdb_path(tomb_path),
325
+ # Bare key (pre-presign) is stable across
326
+ # appends → safe deletion-vector cache key.
327
+ cache_key=tomb_path,
325
328
  )
326
329
  except Exception as te:
327
330
  logger.debug(self._lp(f"[tombstone] leaf lookup failed for {td.alias}: {te}"))
@@ -26,6 +26,7 @@ from supertable.processing import (
26
26
  resolve_overwrite_writes,
27
27
  identify_all_rowids,
28
28
  build_tombstone_file,
29
+ reclaim_fully_dead_files,
29
30
  build_stats_file,
30
31
  extract_stats_rows,
31
32
  probe_ranges_from_df,
@@ -35,6 +36,7 @@ from supertable.processing import (
35
36
  write_parquet_and_collect_resources,
36
37
  compact_resources,
37
38
  compact_tombstones,
39
+ should_compact_small_files,
38
40
  _max_tombstone_rows,
39
41
  _read_parquet_safe,
40
42
  )
@@ -471,6 +473,17 @@ class DataWriter:
471
473
  new_resources = []
472
474
  sunset_files = set()
473
475
 
476
+ # Load the current deletion-vector once: used both to exclude
477
+ # already-tombstoned rows from this write's deletes (below) and,
478
+ # via prev_df, to extend the vector without a second read.
479
+ prev_dv_df = (
480
+ _read_parquet_safe(prev_tombstone_path, profiler=profiler)
481
+ if prev_tombstone_path else None
482
+ )
483
+ prev_dv_rowids = set()
484
+ if prev_dv_df is not None and "__rowid__" in prev_dv_df.columns:
485
+ prev_dv_rowids = set(prev_dv_df.get_column("__rowid__").to_list())
486
+
474
487
  # 1. Identify which existing rows this write deletes/replaces.
475
488
  # overwrite_columns drives the anti-join key (delete + upsert);
476
489
  # pure appends (no overwrite_columns) tombstone nothing. The
@@ -486,6 +499,20 @@ class DataWriter:
486
499
  file_cache=file_cache,
487
500
  profiler=profiler,
488
501
  )
502
+
503
+ # Never re-tombstone rows already in the deletion-vector. The
504
+ # overlap probe (and identify_all_rowids) scan the *physical*
505
+ # files, which still hold logically-deleted rows until
506
+ # compaction; without this filter every write re-counts those
507
+ # already-dead rows — inflating ``deleted`` and forcing a
508
+ # needless tombstone rewrite even when nothing live was removed.
509
+ # Excluding them makes ``deleted`` the true count of live rows
510
+ # removed and lets unchanged writes carry the vector forward.
511
+ if new_delete_pairs and prev_dv_rowids:
512
+ new_delete_pairs = [
513
+ (f, rid) for (f, rid) in new_delete_pairs
514
+ if rid not in prev_dv_rowids
515
+ ]
489
516
  deleted = len(new_delete_pairs)
490
517
  mark("identify_deletes")
491
518
 
@@ -514,6 +541,7 @@ class DataWriter:
514
541
  new_pairs=new_delete_pairs,
515
542
  compression_level=compression_level,
516
543
  profiler=profiler,
544
+ prev_df=prev_dv_df,
517
545
  )
518
546
 
519
547
  # Track the live deletion-vector row count so meta reads can
@@ -527,28 +555,94 @@ class DataWriter:
527
555
  )
528
556
  mark("build_tombstone")
529
557
 
530
- # 4. Threshold compaction: physically drop dead rows once the
531
- # deletion-vector grows past max_tombstone_rows, then clear it.
532
- if (
558
+ # 3b. Eager reclamation of fully-dead files. Any existing data
559
+ # file whose every physical row is now tombstoned is 100%
560
+ # dead: drop it from the snapshot for free (no rewrite) and
561
+ # remove its rowids from the vector. Without this, fully
562
+ # deleted files linger until the compaction threshold,
563
+ # bloating the snapshot and getting re-scanned by every later
564
+ # overwrite probe. Only runs when the vector changed this
565
+ # write (combined_tombstone_df is not None) — a carry-forward
566
+ # can create no newly-dead file.
567
+ if combined_tombstone_df is not None:
568
+ reclaimed_files, reclaimed_tomb_path, reclaimed_dv = (
569
+ reclaim_fully_dead_files(
570
+ resources=last_simple_table.get("resources") or [],
571
+ combined_dv=combined_tombstone_df,
572
+ tombstone_dir=tombstone_dir,
573
+ compression_level=compression_level,
574
+ profiler=profiler,
575
+ )
576
+ )
577
+ if reclaimed_files:
578
+ sunset_files |= reclaimed_files
579
+ tombstone_path = reclaimed_tomb_path
580
+ combined_tombstone_df = reclaimed_dv
581
+ tombstone_rows = (
582
+ reclaimed_dv.height if reclaimed_dv is not None else 0
583
+ )
584
+ logger.info(lp(
585
+ f"reclaimed {len(reclaimed_files)} fully-deleted "
586
+ f"file(s); deletion-vector now {tombstone_rows} rows"
587
+ ))
588
+ mark("reclaim_dead_files")
589
+
590
+ # 4. Threshold compaction (two triggers, same physical step):
591
+ # (a) the deletion-vector grew past max_tombstone_rows, or
592
+ # (b) the small files tripped the auto-compaction gate.
593
+ # Both must FIRST physically drop tombstoned rows (Phase A)
594
+ # and only THEN merge small files (Phase B): compact_resources
595
+ # rewrites data files WITHOUT consulting the deletion-vector,
596
+ # so sunsetting a vector-referenced file would orphan its dead
597
+ # rows (hidden on read, never reclaimable). Draining first
598
+ # guarantees Phase B only ever sees vector-free survivors.
599
+ post_write_resources = (
600
+ [r for r in (last_simple_table.get("resources") or [])
601
+ if r.get("file") not in sunset_files]
602
+ + new_resources
603
+ )
604
+ compaction_gate = should_compact_small_files(
605
+ post_write_resources, table_config
606
+ )
607
+ tombstone_threshold_hit = (
533
608
  combined_tombstone_df is not None
534
609
  and combined_tombstone_df.height >= _max_tombstone_rows(table_config)
535
- ):
536
- removed, compact_new, compact_sunset = compact_tombstones(
537
- snapshot=last_simple_table,
538
- tombstone_df=combined_tombstone_df,
539
- data_dir=simple_table.data_dir,
540
- compression_level=compression_level,
541
- table_config=table_config,
542
- profiler=profiler,
543
- )
544
- new_resources.extend(compact_new)
545
- sunset_files |= compact_sunset
546
- tombstone_path = None # deletion-vector fully consumed
547
- tombstone_rows = 0
548
- logger.info(lp(
549
- f"tombstone compaction removed {removed} rows "
550
- f"from {len(compact_sunset)} files"
551
- ))
610
+ )
611
+
612
+ # Phase A — drain the deletion-vector when either trigger fires
613
+ # and a vector is actually live (freshly built this write OR
614
+ # carried forward from a prior one).
615
+ if tombstone_threshold_hit or compaction_gate:
616
+ dv_to_drain = combined_tombstone_df
617
+ if dv_to_drain is None and tombstone_path:
618
+ # Pure carry-forward: the pointer is unchanged, so the
619
+ # live vector is exactly the one already loaded at the
620
+ # top of this block — reuse it instead of a second
621
+ # storage read (fall back to a read only if it wasn't
622
+ # loaded, which shouldn't happen when tombstone_path is
623
+ # set, but stays correct if it ever does).
624
+ dv_to_drain = (
625
+ prev_dv_df
626
+ if prev_dv_df is not None
627
+ else _read_parquet_safe(tombstone_path, profiler=profiler)
628
+ )
629
+ if dv_to_drain is not None and dv_to_drain.height > 0:
630
+ removed, tomb_new, tomb_sunset = compact_tombstones(
631
+ snapshot=last_simple_table,
632
+ tombstone_df=dv_to_drain,
633
+ data_dir=simple_table.data_dir,
634
+ compression_level=compression_level,
635
+ table_config=table_config,
636
+ profiler=profiler,
637
+ )
638
+ new_resources.extend(tomb_new)
639
+ sunset_files |= tomb_sunset
640
+ tombstone_path = None # deletion-vector fully consumed
641
+ tombstone_rows = 0
642
+ logger.info(lp(
643
+ f"tombstone compaction removed {removed} rows "
644
+ f"from {len(tomb_sunset)} files"
645
+ ))
552
646
 
553
647
  # 5. Pin the (carried-forward / new / cleared) tombstone pointer
554
648
  # and its row count.
@@ -556,6 +650,45 @@ class DataWriter:
556
650
  last_simple_table["tombstone_rows"] = tombstone_rows
557
651
  mark("compact_tombstones")
558
652
 
653
+ # Phase B — auto small-file compaction. Merge the accumulated
654
+ # small files (existing survivors + the file just written) once
655
+ # the gate is open so the file count stays bounded. The vector
656
+ # was drained above, so every surviving file is safe to sunset.
657
+ # Result folds into the SAME snapshot commit below (new_resources
658
+ # / sunset_files feed build_stats and simple_table.update).
659
+ compaction_ran = False
660
+ if compaction_gate:
661
+ live_resources = [
662
+ r for r in (last_simple_table.get("resources") or [])
663
+ if r.get("file") not in sunset_files
664
+ ]
665
+ live_resources += [
666
+ r for r in new_resources if r.get("file") not in sunset_files
667
+ ]
668
+ considered, comp_rows, comp_new, comp_sunset = compact_resources(
669
+ snapshot={"resources": live_resources},
670
+ data_dir=simple_table.data_dir,
671
+ compression_level=compression_level,
672
+ table_config=table_config,
673
+ small_only=True,
674
+ )
675
+ if comp_new or comp_sunset:
676
+ sunset_files |= comp_sunset
677
+ # A file written above (incoming or tombstone survivor)
678
+ # may have been re-merged here; drop any new_resources
679
+ # entry that is now sunset so the snapshot never lists a
680
+ # file as both live and gone.
681
+ new_resources = [
682
+ r for r in (new_resources + comp_new)
683
+ if r.get("file") not in sunset_files
684
+ ]
685
+ compaction_ran = True
686
+ logger.info(lp(
687
+ f"auto-compaction merged {considered} small files "
688
+ f"into {len(comp_new)} file(s) ({comp_rows} rows)"
689
+ ))
690
+ mark("compact_small")
691
+
559
692
  # 6. Carry forward + extend the external column-statistics parquet.
560
693
  # Read the footers of the newly written data files, drop the
561
694
  # rows of any sunset file, and append the new ones. No new
@@ -614,7 +747,18 @@ class DataWriter:
614
747
  # model_df would shrink schema / schemaString to that partial
615
748
  # shape even though all parquet files still have full schema.
616
749
  # See docs/03_data_model.md "Schema Field Semantics".
617
- schema_model_df = None if delete_only else dataframe
750
+ #
751
+ # When auto-compaction merged files this write, derive the
752
+ # schema from the compacted output instead: a merged file may
753
+ # union in columns from older files that the incoming frame
754
+ # lacks (schema-evolving tables), so `dataframe` would narrow
755
+ # the metadata even though the Parquet is wider.
756
+ if compaction_ran:
757
+ schema_model_df = self._build_compact_model_df(
758
+ new_resources, last_simple_table
759
+ )
760
+ else:
761
+ schema_model_df = None if delete_only else dataframe
618
762
  new_snapshot_dict, new_snapshot_path = simple_table.update(
619
763
  new_resources, sunset_files, schema_model_df,
620
764
  last_snapshot=last_simple_table,
@@ -720,7 +864,7 @@ class DataWriter:
720
864
  f"lock={timings.get('lock', 0):.3f} | snapshot={timings.get('snapshot', 0):.3f} | "
721
865
  f"overlap={timings.get('overlap', 0):.3f} | stats_prune={timings.get('stats_prune', 0):.3f} | newer_than={timings.get('newer_than', 0):.3f} | "
722
866
  f"identify_deletes={timings.get('identify_deletes', 0):.3f} | write_parquet={timings.get('write_parquet', 0):.3f} | "
723
- f"build_tombstone={timings.get('build_tombstone', 0):.3f} | compact_tombstones={timings.get('compact_tombstones', 0):.3f} | build_stats={timings.get('build_stats', 0):.3f} | "
867
+ f"build_tombstone={timings.get('build_tombstone', 0):.3f} | compact_tombstones={timings.get('compact_tombstones', 0):.3f} | compact_small={timings.get('compact_small', 0):.3f} | build_stats={timings.get('build_stats', 0):.3f} | "
724
868
  f"update_simple={timings.get('update_simple', 0):.3f} | bump_root={timings.get('bump_root', 0):.3f} | "
725
869
  f"mirror={timings.get('mirror', 0):.3f} | prepare_monitor={timings.get('prepare_monitor', 0):.3f}"
726
870
  )
@@ -10,6 +10,7 @@ import duckdb
10
10
  import pandas as pd
11
11
 
12
12
  from supertable.config.defaults import logger
13
+ from supertable.config.settings import settings
13
14
  from supertable.query_plan_manager import QueryPlanManager
14
15
  from supertable.utils.sql_parser import SQLParser
15
16
  from supertable.data_classes import Reflection
@@ -24,6 +25,7 @@ from supertable.engine.engine_common import (
24
25
  apply_runtime_pragmas,
25
26
  create_rbac_view,
26
27
  create_tombstone_view,
28
+ TombstoneCache,
27
29
  )
28
30
 
29
31
 
@@ -56,6 +58,13 @@ class DuckDBLite:
56
58
  self._lock = threading.Lock()
57
59
  self._con: Optional[duckdb.DuckDBPyConnection] = None
58
60
  self._httpfs_configured = False
61
+ # Shared deletion-vector table cache: per-table eviction (idle TTL +
62
+ # per-table version cap), bounded by config. Tables live on the
63
+ # persistent connection and are forgotten when it resets.
64
+ self._tombstone_cache = TombstoneCache(
65
+ settings.SUPERTABLE_DUCKDB_TOMBSTONE_CACHE_MAX_PER_TABLE,
66
+ settings.SUPERTABLE_DUCKDB_TOMBSTONE_CACHE_TTL_SEC,
67
+ )
59
68
 
60
69
  # ------------------------------------------------------------------
61
70
  # Connection lifecycle
@@ -92,6 +101,8 @@ class DuckDBLite:
92
101
  pass
93
102
  self._con = None
94
103
  self._httpfs_configured = False
104
+ # Tables died with the connection — just forget the registry.
105
+ self._tombstone_cache.clear_registry()
95
106
  logger.warning("[duckdb.lite] connection reset")
96
107
 
97
108
  # ------------------------------------------------------------------
@@ -163,6 +174,8 @@ class DuckDBLite:
163
174
 
164
175
  # Create per-query VIEWs. Dropped in finally regardless of outcome.
165
176
  created_views: List[str] = []
177
+ # Deletion-vector cache keys acquired this query — released in finally.
178
+ acquired_dv_keys: List[str] = []
166
179
  try:
167
180
  for alias, table_name in alias_to_table_name.items():
168
181
  files = alias_to_files[alias]
@@ -193,7 +206,15 @@ class DuckDBLite:
193
206
  source = query_alias_to_name[alias]
194
207
  tomb_def = tombstone_views.get(alias)
195
208
  view = f"tomb_{source}_{query_suffix}"
196
- create_tombstone_view(con, source, view, tomb_def)
209
+ # Reuse a materialised deletion-vector table when the cache is
210
+ # enabled and the alias has a stable key; otherwise the call
211
+ # falls back to the inline read_parquet path (dv_table=None).
212
+ cache_key = getattr(tomb_def, "cache_key", None) if tomb_def else None
213
+ tomb_path = getattr(tomb_def, "tombstone_path", None) if tomb_def else None
214
+ dv_table = self._tombstone_cache.acquire(con, cache_key, tomb_path)
215
+ if dv_table:
216
+ acquired_dv_keys.append(cache_key)
217
+ create_tombstone_view(con, source, view, tomb_def, dv_table=dv_table)
197
218
  created_views.append(view)
198
219
  query_alias_to_name[alias] = view
199
220
 
@@ -260,4 +281,11 @@ class DuckDBLite:
260
281
  con.execute(f"DROP VIEW IF EXISTS {view};")
261
282
  except Exception:
262
283
  pass
284
+ # Release deletion-vector refs now the views referencing them are
285
+ # gone; this may evict + DROP unreferenced DV tables over capacity.
286
+ for cache_key in acquired_dv_keys:
287
+ try:
288
+ self._tombstone_cache.release(con, cache_key)
289
+ except Exception:
290
+ pass
263
291
 
@@ -29,6 +29,7 @@ from supertable.engine.engine_common import (
29
29
  create_rbac_view,
30
30
  create_tombstone_view,
31
31
  rbac_view_name,
32
+ TombstoneCache,
32
33
  )
33
34
 
34
35
 
@@ -80,6 +81,14 @@ class DuckDBPro:
80
81
  # Multiple entries per key when old version still has in-flight queries.
81
82
  self._registry: Dict[Tuple[str, str], List[_ProCacheEntry]] = {}
82
83
 
84
+ # Shared deletion-vector table cache: per-table eviction (idle TTL +
85
+ # per-table version cap), bounded by config. Tables live on the
86
+ # persistent connection and are forgotten when it resets.
87
+ self._tombstone_cache = TombstoneCache(
88
+ settings.SUPERTABLE_DUCKDB_TOMBSTONE_CACHE_MAX_PER_TABLE,
89
+ settings.SUPERTABLE_DUCKDB_TOMBSTONE_CACHE_TTL_SEC,
90
+ )
91
+
83
92
  # Temp dir for spill — set on first query
84
93
  self._temp_dir: Optional[str] = None
85
94
 
@@ -123,6 +132,8 @@ class DuckDBPro:
123
132
  self._con = None
124
133
  self._httpfs_configured = False
125
134
  self._registry.clear()
135
+ # DV tables died with the connection — just forget the registry.
136
+ self._tombstone_cache.clear_registry()
126
137
  logger.warning("[duckdb.pro] connection reset — all cached views lost")
127
138
 
128
139
  # ---------------------------------------------------------
@@ -313,6 +324,8 @@ class DuckDBPro:
313
324
  # assignments are reached (which would cause a NameError otherwise).
314
325
  rbac_view_names: List[str] = []
315
326
  tombstone_view_names: List[str] = []
327
+ # Deletion-vector cache keys acquired this query — released in finally.
328
+ acquired_dv_keys: List[str] = []
316
329
  try:
317
330
  query_alias_to_name = dict(alias_to_table_name)
318
331
  # Per-query suffix so concurrent queries never collide on a shared
@@ -328,8 +341,18 @@ class DuckDBPro:
328
341
  source = query_alias_to_name[alias]
329
342
  tomb_def = tombstone_views.get(alias)
330
343
  view = f"tomb_{source}_{query_suffix}"
344
+ # Reuse a materialised deletion-vector table when the cache is
345
+ # enabled and the alias has a stable key; otherwise fall back to
346
+ # the inline read_parquet path (dv_table=None). All DDL — the DV
347
+ # CREATE TABLE inside acquire() and the view creation — runs
348
+ # under the connection lock, matching Pro's serialised model.
349
+ cache_key = getattr(tomb_def, "cache_key", None) if tomb_def else None
350
+ tomb_path = getattr(tomb_def, "tombstone_path", None) if tomb_def else None
331
351
  with self._lock:
332
- create_tombstone_view(con, source, view, tomb_def)
352
+ dv_table = self._tombstone_cache.acquire(con, cache_key, tomb_path)
353
+ if dv_table:
354
+ acquired_dv_keys.append(cache_key)
355
+ create_tombstone_view(con, source, view, tomb_def, dv_table=dv_table)
333
356
  tombstone_view_names.append(view)
334
357
  query_alias_to_name[alias] = view
335
358
 
@@ -395,6 +418,16 @@ class DuckDBPro:
395
418
  except Exception:
396
419
  pass
397
420
 
421
+ # Release deletion-vector refs now their views are gone; this may
422
+ # evict + DROP unreferenced DV tables over capacity.
423
+ if acquired_dv_keys:
424
+ with self._lock:
425
+ for cache_key in acquired_dv_keys:
426
+ try:
427
+ self._tombstone_cache.release(con, cache_key)
428
+ except Exception:
429
+ pass
430
+
398
431
  # Release refs and drop stale tables
399
432
  with self._lock:
400
433
  self._release_refs(tables_used)