supertable 2.3.2__tar.gz → 2.3.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {supertable-2.3.2/supertable.egg-info → supertable-2.3.4}/PKG-INFO +1 -1
- {supertable-2.3.2 → supertable-2.3.4}/pyproject.toml +1 -1
- {supertable-2.3.2 → supertable-2.3.4}/setup.py +1 -1
- {supertable-2.3.2 → supertable-2.3.4}/supertable/__init__.py +1 -1
- {supertable-2.3.2 → supertable-2.3.4}/supertable/config/settings.py +21 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/data_classes.py +9 -1
- {supertable-2.3.2 → supertable-2.3.4}/supertable/data_reader.py +3 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/data_writer.py +166 -22
- {supertable-2.3.2 → supertable-2.3.4}/supertable/engine/duckdb_lite.py +29 -1
- {supertable-2.3.2 → supertable-2.3.4}/supertable/engine/duckdb_pro.py +34 -1
- {supertable-2.3.2 → supertable-2.3.4}/supertable/engine/engine_common.py +289 -7
- {supertable-2.3.2 → supertable-2.3.4}/supertable/engine/tests/test_engine.py +548 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/processing.py +182 -21
- {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/test_data_reader.py +5 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/test_data_writer.py +284 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/test_processing_compact_resources.py +93 -0
- {supertable-2.3.2 → supertable-2.3.4/supertable.egg-info}/PKG-INFO +1 -1
- {supertable-2.3.2 → supertable-2.3.4}/LICENSE +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/README.md +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/requirements.txt +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/setup.cfg +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/audit/__init__.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/audit/admin.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/audit/chain.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/audit/consumers.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/audit/crypto.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/audit/events.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/audit/export.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/audit/logger.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/audit/middleware.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/audit/reader.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/audit/retention.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/audit/tests/__init__.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/audit/tests/test_chain.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/audit/tests/test_crypto.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/audit/tests/test_emit.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/audit/tests/test_events.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/audit/tests/test_retention.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/audit/writer_parquet.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/audit/writer_redis.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/config/__init__.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/config/defaults.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/config/homedir.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/config/tests/__init__.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/config/tests/test_defaults.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/config/tests/test_homedir.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/config/tests/test_settings.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/__init__.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/__init__.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/__main__.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/check_filter_builder.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/controller.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/data_writer_helpers.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/defaults.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/dummy_data.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/read_parquet_header.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/s01_01_01_create_super_table.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/s01_01_02_enable_mirroring_formats.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/s01_02_create_roles.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/s01_03_create_users.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/s02_01_write_dummy_data.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/s02_02_write_single_data.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/s02_03_01_write_staging.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/s02_03_02_create_pipe.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/s02_04_01_write_monitoring_simple.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/s02_04_02_write_monitoring_parallel.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/s02_05_write_tombstone.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/s03_01_read_data_error.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/s03_02_01_read_super_data_ok.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/s03_02_02_read_table_data_ok.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/s03_03_read_meta.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/s03_04_read_staging.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/s03_06_01_read_roles.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/s03_06_02_read_user.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/s03_07_01_estimate_read.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/s03_07_02_estimate_files.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/s03_08_read_snapshot_history.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/s04_01_03_delete_pipe.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/s05_01_delete_table.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/quickstart/s05_02_delete_super_table.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/webshop/__init__.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/webshop/core.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/webshop/defaults.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/webshop/generate.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/webshop/load.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/demo/webshop/topup.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/engine/__init__.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/engine/data_estimator.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/engine/engine_config.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/engine/engine_enum.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/engine/executor.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/engine/plan_stats.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/engine/spark_thrift.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/engine/tests/__init__.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/engine/tests/conftest.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/engine/tests/test_engine_config.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/engine/tests/test_engine_routing.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/engine/tests/test_engine_spill.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/errors.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/locking/__init__.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/locking/benchmarks/__init__.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/locking/benchmarks/benchmark_locking.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/locking/benchmarks/measure_lock_speed.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/locking/benchmarks/measure_lock_time.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/locking/file_lock.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/locking/redis_lock.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/locking/tests/__init__.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/locking/tests/test_file_lock.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/locking/tests/test_redis_lock.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/logging.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/meta_reader.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/mirroring/__init__.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/mirroring/mirror_delta.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/mirroring/mirror_formats.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/mirroring/mirror_iceberg.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/mirroring/mirror_parquet.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/monitoring/__init__.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/monitoring/partitions.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/monitoring_writer.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/plan_extender.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/query_plan_manager.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/rbac/__init__.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/rbac/access_control.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/rbac/filter_builder.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/rbac/permissions.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/rbac/role_manager.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/rbac/row_column_security.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/rbac/tests/test_filter_builder.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/rbac/tests/test_rbac.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/rbac/tests/test_rbac_per_table.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/rbac/user_manager.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/redis_catalog.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/redis_connector.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/redis_infra.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/redis_keys.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/simple_table.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/staging_area.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/storage/__init__.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/storage/azure_storage.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/storage/gcp_storage.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/storage/local_storage.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/storage/minio_storage.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/storage/s3_storage.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/storage/storage_factory.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/storage/storage_interface.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/storage/tests/test_storage.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/super_pipe.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/super_table.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/system_query.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/__init__.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/test_align_to_schema_fix.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/test_create_if_missing.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/test_data_reader_preflight.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/test_data_writer_compact.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/test_data_writer_comprehensive.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/test_errors.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/test_meta_reader.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/test_monitoring_partitions.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/test_monitoring_sink_guard.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/test_newer_than.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/test_parquet_statistics.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/test_processing.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/test_processing_stats.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/test_query_sql.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/test_read_pruning_differential.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/test_read_pruning_integration.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/test_redis_key_prefix.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/test_resolve_overwrite_writes.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/test_simple_table.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/test_stats_cache.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/test_stats_pruning.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/test_stats_schema_snapshot.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/test_super_table.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/test_supertable_all.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/tests/test_system_query.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/utils/__init__.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/utils/helper.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/utils/profiler.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/utils/sql_parser.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/utils/tests/test_sql_parser_columns.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable/utils/timer.py +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable.egg-info/SOURCES.txt +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable.egg-info/dependency_links.txt +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable.egg-info/entry_points.txt +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable.egg-info/requires.txt +0 -0
- {supertable-2.3.2 → supertable-2.3.4}/supertable.egg-info/top_level.txt +0 -0
|
@@ -19,7 +19,7 @@ long_description = readme.read_text(encoding="utf-8") if readme.exists() else ""
|
|
|
19
19
|
|
|
20
20
|
setup(
|
|
21
21
|
name="supertable",
|
|
22
|
-
version="2.3.
|
|
22
|
+
version="2.3.4",
|
|
23
23
|
description="SuperTable — versioned data lake library for SQL analytics on Parquet + Redis.",
|
|
24
24
|
long_description=long_description,
|
|
25
25
|
long_description_content_type="text/markdown",
|
|
@@ -25,7 +25,7 @@ See the ``supertable.demo`` package for runnable end-to-end demos and the
|
|
|
25
25
|
project documentation for the full API surface.
|
|
26
26
|
"""
|
|
27
27
|
|
|
28
|
-
__version__ = "2.3.
|
|
28
|
+
__version__ = "2.3.4"
|
|
29
29
|
|
|
30
30
|
# Re-export the core public surface so users can do ``from supertable import …``
|
|
31
31
|
# instead of remembering submodule paths.
|
|
@@ -157,6 +157,25 @@ class Settings:
|
|
|
157
157
|
SUPERTABLE_DUCKDB_MATERIALIZE: str = "view" # SUPERTABLE_DUCKDB_MATERIALIZE
|
|
158
158
|
SUPERTABLE_DUCKDB_PRESIGNED: bool = False # SUPERTABLE_DUCKDB_PRESIGNED
|
|
159
159
|
SUPERTABLE_DUCKDB_USE_HTTPFS: bool = False # SUPERTABLE_DUCKDB_USE_HTTPFS
|
|
160
|
+
# Deletion-vector (tombstone) table cache. Each entry is a small
|
|
161
|
+
# `DISTINCT __rowid__` table keyed by the stable tombstone path; the
|
|
162
|
+
# tombstone view ANTI JOINs it instead of re-reading the parquet every
|
|
163
|
+
# query. Eviction is purely per-table — a churny table can never evict a
|
|
164
|
+
# slow table's cached deletion-vector:
|
|
165
|
+
# * Idle TTL (below): every entry, including a table's latest, is dropped
|
|
166
|
+
# once it goes unqueried for the TTL window.
|
|
167
|
+
# * Per-table cap (this knob): at most N most-recently-used versions are
|
|
168
|
+
# kept per table, so a burst of rewrites (e.g. 1000 updates in 5 min)
|
|
169
|
+
# retains only the last N rather than all of them.
|
|
170
|
+
# <= 0 disables the cache entirely (inline read_parquet fallback).
|
|
171
|
+
SUPERTABLE_DUCKDB_TOMBSTONE_CACHE_MAX_PER_TABLE: int = 8 # SUPERTABLE_DUCKDB_TOMBSTONE_CACHE_MAX_PER_TABLE
|
|
172
|
+
# Idle TTL (seconds): a cached deletion-vector is dropped once it has gone
|
|
173
|
+
# unqueried for this long; every query that uses it refreshes the timer.
|
|
174
|
+
# Applies to every entry (a table's latest included), so an abandoned table
|
|
175
|
+
# reclaims its cache instead of lingering until the connection resets.
|
|
176
|
+
# <= 0 keeps an entry only while a query references it (no persistence).
|
|
177
|
+
# Defaults to SUPERTABLE_ENGINE_FRESHNESS_SEC (300 s / 5 min).
|
|
178
|
+
SUPERTABLE_DUCKDB_TOMBSTONE_CACHE_TTL_SEC: int = 300 # SUPERTABLE_DUCKDB_TOMBSTONE_CACHE_TTL_SEC
|
|
160
179
|
SUPERTABLE_DEBUG_TIMINGS: bool = False # SUPERTABLE_DEBUG_TIMINGS
|
|
161
180
|
|
|
162
181
|
# ── Engine Routing / Executor ────────────────────────────────────
|
|
@@ -418,6 +437,8 @@ def _build_settings() -> Settings:
|
|
|
418
437
|
SUPERTABLE_DUCKDB_MATERIALIZE=_env_str("SUPERTABLE_DUCKDB_MATERIALIZE", "view"),
|
|
419
438
|
SUPERTABLE_DUCKDB_PRESIGNED=_env_bool("SUPERTABLE_DUCKDB_PRESIGNED", False),
|
|
420
439
|
SUPERTABLE_DUCKDB_USE_HTTPFS=_env_bool("SUPERTABLE_DUCKDB_USE_HTTPFS", False),
|
|
440
|
+
SUPERTABLE_DUCKDB_TOMBSTONE_CACHE_MAX_PER_TABLE=_env_int("SUPERTABLE_DUCKDB_TOMBSTONE_CACHE_MAX_PER_TABLE", 8),
|
|
441
|
+
SUPERTABLE_DUCKDB_TOMBSTONE_CACHE_TTL_SEC=_env_int("SUPERTABLE_DUCKDB_TOMBSTONE_CACHE_TTL_SEC", 300),
|
|
421
442
|
SUPERTABLE_DEBUG_TIMINGS=_env_bool("SUPERTABLE_DEBUG_TIMINGS", False),
|
|
422
443
|
|
|
423
444
|
# ── Engine Routing ───────────────────────────────────────────
|
|
@@ -61,9 +61,17 @@ class TombstoneDef:
|
|
|
61
61
|
|
|
62
62
|
- tombstone_path: storage path of the deletion-vector parquet
|
|
63
63
|
(columns ``__file__`` + ``__rowid__``). ``None`` means no
|
|
64
|
-
tombstone exists, so no anti-join is applied.
|
|
64
|
+
tombstone exists, so no anti-join is applied. This may be a
|
|
65
|
+
presigned/object-store URL that rotates per request, so it is
|
|
66
|
+
*not* stable enough to use as a cache key.
|
|
67
|
+
- cache_key: the bare, stable storage key of the same deletion-vector
|
|
68
|
+
parquet (no presign). Stable across pure appends (carry-forward
|
|
69
|
+
returns the previous tombstone), so DuckDB engines use it to key the
|
|
70
|
+
materialised deletion-vector table cache. ``None`` disables caching
|
|
71
|
+
for this alias (falls back to inline ``read_parquet``).
|
|
65
72
|
"""
|
|
66
73
|
tombstone_path: Optional[str] = None
|
|
74
|
+
cache_key: Optional[str] = None
|
|
67
75
|
|
|
68
76
|
|
|
69
77
|
@dataclass
|
|
@@ -322,6 +322,9 @@ class DataReader:
|
|
|
322
322
|
# storage returns the key unchanged.
|
|
323
323
|
reflection.tombstone_views[td.alias] = TombstoneDef(
|
|
324
324
|
tombstone_path=estimator._to_duckdb_path(tomb_path),
|
|
325
|
+
# Bare key (pre-presign) is stable across
|
|
326
|
+
# appends → safe deletion-vector cache key.
|
|
327
|
+
cache_key=tomb_path,
|
|
325
328
|
)
|
|
326
329
|
except Exception as te:
|
|
327
330
|
logger.debug(self._lp(f"[tombstone] leaf lookup failed for {td.alias}: {te}"))
|
|
@@ -26,6 +26,7 @@ from supertable.processing import (
|
|
|
26
26
|
resolve_overwrite_writes,
|
|
27
27
|
identify_all_rowids,
|
|
28
28
|
build_tombstone_file,
|
|
29
|
+
reclaim_fully_dead_files,
|
|
29
30
|
build_stats_file,
|
|
30
31
|
extract_stats_rows,
|
|
31
32
|
probe_ranges_from_df,
|
|
@@ -35,6 +36,7 @@ from supertable.processing import (
|
|
|
35
36
|
write_parquet_and_collect_resources,
|
|
36
37
|
compact_resources,
|
|
37
38
|
compact_tombstones,
|
|
39
|
+
should_compact_small_files,
|
|
38
40
|
_max_tombstone_rows,
|
|
39
41
|
_read_parquet_safe,
|
|
40
42
|
)
|
|
@@ -471,6 +473,17 @@ class DataWriter:
|
|
|
471
473
|
new_resources = []
|
|
472
474
|
sunset_files = set()
|
|
473
475
|
|
|
476
|
+
# Load the current deletion-vector once: used both to exclude
|
|
477
|
+
# already-tombstoned rows from this write's deletes (below) and,
|
|
478
|
+
# via prev_df, to extend the vector without a second read.
|
|
479
|
+
prev_dv_df = (
|
|
480
|
+
_read_parquet_safe(prev_tombstone_path, profiler=profiler)
|
|
481
|
+
if prev_tombstone_path else None
|
|
482
|
+
)
|
|
483
|
+
prev_dv_rowids = set()
|
|
484
|
+
if prev_dv_df is not None and "__rowid__" in prev_dv_df.columns:
|
|
485
|
+
prev_dv_rowids = set(prev_dv_df.get_column("__rowid__").to_list())
|
|
486
|
+
|
|
474
487
|
# 1. Identify which existing rows this write deletes/replaces.
|
|
475
488
|
# overwrite_columns drives the anti-join key (delete + upsert);
|
|
476
489
|
# pure appends (no overwrite_columns) tombstone nothing. The
|
|
@@ -486,6 +499,20 @@ class DataWriter:
|
|
|
486
499
|
file_cache=file_cache,
|
|
487
500
|
profiler=profiler,
|
|
488
501
|
)
|
|
502
|
+
|
|
503
|
+
# Never re-tombstone rows already in the deletion-vector. The
|
|
504
|
+
# overlap probe (and identify_all_rowids) scan the *physical*
|
|
505
|
+
# files, which still hold logically-deleted rows until
|
|
506
|
+
# compaction; without this filter every write re-counts those
|
|
507
|
+
# already-dead rows — inflating ``deleted`` and forcing a
|
|
508
|
+
# needless tombstone rewrite even when nothing live was removed.
|
|
509
|
+
# Excluding them makes ``deleted`` the true count of live rows
|
|
510
|
+
# removed and lets unchanged writes carry the vector forward.
|
|
511
|
+
if new_delete_pairs and prev_dv_rowids:
|
|
512
|
+
new_delete_pairs = [
|
|
513
|
+
(f, rid) for (f, rid) in new_delete_pairs
|
|
514
|
+
if rid not in prev_dv_rowids
|
|
515
|
+
]
|
|
489
516
|
deleted = len(new_delete_pairs)
|
|
490
517
|
mark("identify_deletes")
|
|
491
518
|
|
|
@@ -514,6 +541,7 @@ class DataWriter:
|
|
|
514
541
|
new_pairs=new_delete_pairs,
|
|
515
542
|
compression_level=compression_level,
|
|
516
543
|
profiler=profiler,
|
|
544
|
+
prev_df=prev_dv_df,
|
|
517
545
|
)
|
|
518
546
|
|
|
519
547
|
# Track the live deletion-vector row count so meta reads can
|
|
@@ -527,28 +555,94 @@ class DataWriter:
|
|
|
527
555
|
)
|
|
528
556
|
mark("build_tombstone")
|
|
529
557
|
|
|
530
|
-
#
|
|
531
|
-
#
|
|
532
|
-
|
|
558
|
+
# 3b. Eager reclamation of fully-dead files. Any existing data
|
|
559
|
+
# file whose every physical row is now tombstoned is 100%
|
|
560
|
+
# dead: drop it from the snapshot for free (no rewrite) and
|
|
561
|
+
# remove its rowids from the vector. Without this, fully
|
|
562
|
+
# deleted files linger until the compaction threshold,
|
|
563
|
+
# bloating the snapshot and getting re-scanned by every later
|
|
564
|
+
# overwrite probe. Only runs when the vector changed this
|
|
565
|
+
# write (combined_tombstone_df is not None) — a carry-forward
|
|
566
|
+
# can create no newly-dead file.
|
|
567
|
+
if combined_tombstone_df is not None:
|
|
568
|
+
reclaimed_files, reclaimed_tomb_path, reclaimed_dv = (
|
|
569
|
+
reclaim_fully_dead_files(
|
|
570
|
+
resources=last_simple_table.get("resources") or [],
|
|
571
|
+
combined_dv=combined_tombstone_df,
|
|
572
|
+
tombstone_dir=tombstone_dir,
|
|
573
|
+
compression_level=compression_level,
|
|
574
|
+
profiler=profiler,
|
|
575
|
+
)
|
|
576
|
+
)
|
|
577
|
+
if reclaimed_files:
|
|
578
|
+
sunset_files |= reclaimed_files
|
|
579
|
+
tombstone_path = reclaimed_tomb_path
|
|
580
|
+
combined_tombstone_df = reclaimed_dv
|
|
581
|
+
tombstone_rows = (
|
|
582
|
+
reclaimed_dv.height if reclaimed_dv is not None else 0
|
|
583
|
+
)
|
|
584
|
+
logger.info(lp(
|
|
585
|
+
f"reclaimed {len(reclaimed_files)} fully-deleted "
|
|
586
|
+
f"file(s); deletion-vector now {tombstone_rows} rows"
|
|
587
|
+
))
|
|
588
|
+
mark("reclaim_dead_files")
|
|
589
|
+
|
|
590
|
+
# 4. Threshold compaction (two triggers, same physical step):
|
|
591
|
+
# (a) the deletion-vector grew past max_tombstone_rows, or
|
|
592
|
+
# (b) the small files tripped the auto-compaction gate.
|
|
593
|
+
# Both must FIRST physically drop tombstoned rows (Phase A)
|
|
594
|
+
# and only THEN merge small files (Phase B): compact_resources
|
|
595
|
+
# rewrites data files WITHOUT consulting the deletion-vector,
|
|
596
|
+
# so sunsetting a vector-referenced file would orphan its dead
|
|
597
|
+
# rows (hidden on read, never reclaimable). Draining first
|
|
598
|
+
# guarantees Phase B only ever sees vector-free survivors.
|
|
599
|
+
post_write_resources = (
|
|
600
|
+
[r for r in (last_simple_table.get("resources") or [])
|
|
601
|
+
if r.get("file") not in sunset_files]
|
|
602
|
+
+ new_resources
|
|
603
|
+
)
|
|
604
|
+
compaction_gate = should_compact_small_files(
|
|
605
|
+
post_write_resources, table_config
|
|
606
|
+
)
|
|
607
|
+
tombstone_threshold_hit = (
|
|
533
608
|
combined_tombstone_df is not None
|
|
534
609
|
and combined_tombstone_df.height >= _max_tombstone_rows(table_config)
|
|
535
|
-
)
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
610
|
+
)
|
|
611
|
+
|
|
612
|
+
# Phase A — drain the deletion-vector when either trigger fires
|
|
613
|
+
# and a vector is actually live (freshly built this write OR
|
|
614
|
+
# carried forward from a prior one).
|
|
615
|
+
if tombstone_threshold_hit or compaction_gate:
|
|
616
|
+
dv_to_drain = combined_tombstone_df
|
|
617
|
+
if dv_to_drain is None and tombstone_path:
|
|
618
|
+
# Pure carry-forward: the pointer is unchanged, so the
|
|
619
|
+
# live vector is exactly the one already loaded at the
|
|
620
|
+
# top of this block — reuse it instead of a second
|
|
621
|
+
# storage read (fall back to a read only if it wasn't
|
|
622
|
+
# loaded, which shouldn't happen when tombstone_path is
|
|
623
|
+
# set, but stays correct if it ever does).
|
|
624
|
+
dv_to_drain = (
|
|
625
|
+
prev_dv_df
|
|
626
|
+
if prev_dv_df is not None
|
|
627
|
+
else _read_parquet_safe(tombstone_path, profiler=profiler)
|
|
628
|
+
)
|
|
629
|
+
if dv_to_drain is not None and dv_to_drain.height > 0:
|
|
630
|
+
removed, tomb_new, tomb_sunset = compact_tombstones(
|
|
631
|
+
snapshot=last_simple_table,
|
|
632
|
+
tombstone_df=dv_to_drain,
|
|
633
|
+
data_dir=simple_table.data_dir,
|
|
634
|
+
compression_level=compression_level,
|
|
635
|
+
table_config=table_config,
|
|
636
|
+
profiler=profiler,
|
|
637
|
+
)
|
|
638
|
+
new_resources.extend(tomb_new)
|
|
639
|
+
sunset_files |= tomb_sunset
|
|
640
|
+
tombstone_path = None # deletion-vector fully consumed
|
|
641
|
+
tombstone_rows = 0
|
|
642
|
+
logger.info(lp(
|
|
643
|
+
f"tombstone compaction removed {removed} rows "
|
|
644
|
+
f"from {len(tomb_sunset)} files"
|
|
645
|
+
))
|
|
552
646
|
|
|
553
647
|
# 5. Pin the (carried-forward / new / cleared) tombstone pointer
|
|
554
648
|
# and its row count.
|
|
@@ -556,6 +650,45 @@ class DataWriter:
|
|
|
556
650
|
last_simple_table["tombstone_rows"] = tombstone_rows
|
|
557
651
|
mark("compact_tombstones")
|
|
558
652
|
|
|
653
|
+
# Phase B — auto small-file compaction. Merge the accumulated
|
|
654
|
+
# small files (existing survivors + the file just written) once
|
|
655
|
+
# the gate is open so the file count stays bounded. The vector
|
|
656
|
+
# was drained above, so every surviving file is safe to sunset.
|
|
657
|
+
# Result folds into the SAME snapshot commit below (new_resources
|
|
658
|
+
# / sunset_files feed build_stats and simple_table.update).
|
|
659
|
+
compaction_ran = False
|
|
660
|
+
if compaction_gate:
|
|
661
|
+
live_resources = [
|
|
662
|
+
r for r in (last_simple_table.get("resources") or [])
|
|
663
|
+
if r.get("file") not in sunset_files
|
|
664
|
+
]
|
|
665
|
+
live_resources += [
|
|
666
|
+
r for r in new_resources if r.get("file") not in sunset_files
|
|
667
|
+
]
|
|
668
|
+
considered, comp_rows, comp_new, comp_sunset = compact_resources(
|
|
669
|
+
snapshot={"resources": live_resources},
|
|
670
|
+
data_dir=simple_table.data_dir,
|
|
671
|
+
compression_level=compression_level,
|
|
672
|
+
table_config=table_config,
|
|
673
|
+
small_only=True,
|
|
674
|
+
)
|
|
675
|
+
if comp_new or comp_sunset:
|
|
676
|
+
sunset_files |= comp_sunset
|
|
677
|
+
# A file written above (incoming or tombstone survivor)
|
|
678
|
+
# may have been re-merged here; drop any new_resources
|
|
679
|
+
# entry that is now sunset so the snapshot never lists a
|
|
680
|
+
# file as both live and gone.
|
|
681
|
+
new_resources = [
|
|
682
|
+
r for r in (new_resources + comp_new)
|
|
683
|
+
if r.get("file") not in sunset_files
|
|
684
|
+
]
|
|
685
|
+
compaction_ran = True
|
|
686
|
+
logger.info(lp(
|
|
687
|
+
f"auto-compaction merged {considered} small files "
|
|
688
|
+
f"into {len(comp_new)} file(s) ({comp_rows} rows)"
|
|
689
|
+
))
|
|
690
|
+
mark("compact_small")
|
|
691
|
+
|
|
559
692
|
# 6. Carry forward + extend the external column-statistics parquet.
|
|
560
693
|
# Read the footers of the newly written data files, drop the
|
|
561
694
|
# rows of any sunset file, and append the new ones. No new
|
|
@@ -614,7 +747,18 @@ class DataWriter:
|
|
|
614
747
|
# model_df would shrink schema / schemaString to that partial
|
|
615
748
|
# shape even though all parquet files still have full schema.
|
|
616
749
|
# See docs/03_data_model.md "Schema Field Semantics".
|
|
617
|
-
|
|
750
|
+
#
|
|
751
|
+
# When auto-compaction merged files this write, derive the
|
|
752
|
+
# schema from the compacted output instead: a merged file may
|
|
753
|
+
# union in columns from older files that the incoming frame
|
|
754
|
+
# lacks (schema-evolving tables), so `dataframe` would narrow
|
|
755
|
+
# the metadata even though the Parquet is wider.
|
|
756
|
+
if compaction_ran:
|
|
757
|
+
schema_model_df = self._build_compact_model_df(
|
|
758
|
+
new_resources, last_simple_table
|
|
759
|
+
)
|
|
760
|
+
else:
|
|
761
|
+
schema_model_df = None if delete_only else dataframe
|
|
618
762
|
new_snapshot_dict, new_snapshot_path = simple_table.update(
|
|
619
763
|
new_resources, sunset_files, schema_model_df,
|
|
620
764
|
last_snapshot=last_simple_table,
|
|
@@ -720,7 +864,7 @@ class DataWriter:
|
|
|
720
864
|
f"lock={timings.get('lock', 0):.3f} | snapshot={timings.get('snapshot', 0):.3f} | "
|
|
721
865
|
f"overlap={timings.get('overlap', 0):.3f} | stats_prune={timings.get('stats_prune', 0):.3f} | newer_than={timings.get('newer_than', 0):.3f} | "
|
|
722
866
|
f"identify_deletes={timings.get('identify_deletes', 0):.3f} | write_parquet={timings.get('write_parquet', 0):.3f} | "
|
|
723
|
-
f"build_tombstone={timings.get('build_tombstone', 0):.3f} | compact_tombstones={timings.get('compact_tombstones', 0):.3f} | build_stats={timings.get('build_stats', 0):.3f} | "
|
|
867
|
+
f"build_tombstone={timings.get('build_tombstone', 0):.3f} | compact_tombstones={timings.get('compact_tombstones', 0):.3f} | compact_small={timings.get('compact_small', 0):.3f} | build_stats={timings.get('build_stats', 0):.3f} | "
|
|
724
868
|
f"update_simple={timings.get('update_simple', 0):.3f} | bump_root={timings.get('bump_root', 0):.3f} | "
|
|
725
869
|
f"mirror={timings.get('mirror', 0):.3f} | prepare_monitor={timings.get('prepare_monitor', 0):.3f}"
|
|
726
870
|
)
|
|
@@ -10,6 +10,7 @@ import duckdb
|
|
|
10
10
|
import pandas as pd
|
|
11
11
|
|
|
12
12
|
from supertable.config.defaults import logger
|
|
13
|
+
from supertable.config.settings import settings
|
|
13
14
|
from supertable.query_plan_manager import QueryPlanManager
|
|
14
15
|
from supertable.utils.sql_parser import SQLParser
|
|
15
16
|
from supertable.data_classes import Reflection
|
|
@@ -24,6 +25,7 @@ from supertable.engine.engine_common import (
|
|
|
24
25
|
apply_runtime_pragmas,
|
|
25
26
|
create_rbac_view,
|
|
26
27
|
create_tombstone_view,
|
|
28
|
+
TombstoneCache,
|
|
27
29
|
)
|
|
28
30
|
|
|
29
31
|
|
|
@@ -56,6 +58,13 @@ class DuckDBLite:
|
|
|
56
58
|
self._lock = threading.Lock()
|
|
57
59
|
self._con: Optional[duckdb.DuckDBPyConnection] = None
|
|
58
60
|
self._httpfs_configured = False
|
|
61
|
+
# Shared deletion-vector table cache: per-table eviction (idle TTL +
|
|
62
|
+
# per-table version cap), bounded by config. Tables live on the
|
|
63
|
+
# persistent connection and are forgotten when it resets.
|
|
64
|
+
self._tombstone_cache = TombstoneCache(
|
|
65
|
+
settings.SUPERTABLE_DUCKDB_TOMBSTONE_CACHE_MAX_PER_TABLE,
|
|
66
|
+
settings.SUPERTABLE_DUCKDB_TOMBSTONE_CACHE_TTL_SEC,
|
|
67
|
+
)
|
|
59
68
|
|
|
60
69
|
# ------------------------------------------------------------------
|
|
61
70
|
# Connection lifecycle
|
|
@@ -92,6 +101,8 @@ class DuckDBLite:
|
|
|
92
101
|
pass
|
|
93
102
|
self._con = None
|
|
94
103
|
self._httpfs_configured = False
|
|
104
|
+
# Tables died with the connection — just forget the registry.
|
|
105
|
+
self._tombstone_cache.clear_registry()
|
|
95
106
|
logger.warning("[duckdb.lite] connection reset")
|
|
96
107
|
|
|
97
108
|
# ------------------------------------------------------------------
|
|
@@ -163,6 +174,8 @@ class DuckDBLite:
|
|
|
163
174
|
|
|
164
175
|
# Create per-query VIEWs. Dropped in finally regardless of outcome.
|
|
165
176
|
created_views: List[str] = []
|
|
177
|
+
# Deletion-vector cache keys acquired this query — released in finally.
|
|
178
|
+
acquired_dv_keys: List[str] = []
|
|
166
179
|
try:
|
|
167
180
|
for alias, table_name in alias_to_table_name.items():
|
|
168
181
|
files = alias_to_files[alias]
|
|
@@ -193,7 +206,15 @@ class DuckDBLite:
|
|
|
193
206
|
source = query_alias_to_name[alias]
|
|
194
207
|
tomb_def = tombstone_views.get(alias)
|
|
195
208
|
view = f"tomb_{source}_{query_suffix}"
|
|
196
|
-
|
|
209
|
+
# Reuse a materialised deletion-vector table when the cache is
|
|
210
|
+
# enabled and the alias has a stable key; otherwise the call
|
|
211
|
+
# falls back to the inline read_parquet path (dv_table=None).
|
|
212
|
+
cache_key = getattr(tomb_def, "cache_key", None) if tomb_def else None
|
|
213
|
+
tomb_path = getattr(tomb_def, "tombstone_path", None) if tomb_def else None
|
|
214
|
+
dv_table = self._tombstone_cache.acquire(con, cache_key, tomb_path)
|
|
215
|
+
if dv_table:
|
|
216
|
+
acquired_dv_keys.append(cache_key)
|
|
217
|
+
create_tombstone_view(con, source, view, tomb_def, dv_table=dv_table)
|
|
197
218
|
created_views.append(view)
|
|
198
219
|
query_alias_to_name[alias] = view
|
|
199
220
|
|
|
@@ -260,4 +281,11 @@ class DuckDBLite:
|
|
|
260
281
|
con.execute(f"DROP VIEW IF EXISTS {view};")
|
|
261
282
|
except Exception:
|
|
262
283
|
pass
|
|
284
|
+
# Release deletion-vector refs now the views referencing them are
|
|
285
|
+
# gone; this may evict + DROP unreferenced DV tables over capacity.
|
|
286
|
+
for cache_key in acquired_dv_keys:
|
|
287
|
+
try:
|
|
288
|
+
self._tombstone_cache.release(con, cache_key)
|
|
289
|
+
except Exception:
|
|
290
|
+
pass
|
|
263
291
|
|
|
@@ -29,6 +29,7 @@ from supertable.engine.engine_common import (
|
|
|
29
29
|
create_rbac_view,
|
|
30
30
|
create_tombstone_view,
|
|
31
31
|
rbac_view_name,
|
|
32
|
+
TombstoneCache,
|
|
32
33
|
)
|
|
33
34
|
|
|
34
35
|
|
|
@@ -80,6 +81,14 @@ class DuckDBPro:
|
|
|
80
81
|
# Multiple entries per key when old version still has in-flight queries.
|
|
81
82
|
self._registry: Dict[Tuple[str, str], List[_ProCacheEntry]] = {}
|
|
82
83
|
|
|
84
|
+
# Shared deletion-vector table cache: per-table eviction (idle TTL +
|
|
85
|
+
# per-table version cap), bounded by config. Tables live on the
|
|
86
|
+
# persistent connection and are forgotten when it resets.
|
|
87
|
+
self._tombstone_cache = TombstoneCache(
|
|
88
|
+
settings.SUPERTABLE_DUCKDB_TOMBSTONE_CACHE_MAX_PER_TABLE,
|
|
89
|
+
settings.SUPERTABLE_DUCKDB_TOMBSTONE_CACHE_TTL_SEC,
|
|
90
|
+
)
|
|
91
|
+
|
|
83
92
|
# Temp dir for spill — set on first query
|
|
84
93
|
self._temp_dir: Optional[str] = None
|
|
85
94
|
|
|
@@ -123,6 +132,8 @@ class DuckDBPro:
|
|
|
123
132
|
self._con = None
|
|
124
133
|
self._httpfs_configured = False
|
|
125
134
|
self._registry.clear()
|
|
135
|
+
# DV tables died with the connection — just forget the registry.
|
|
136
|
+
self._tombstone_cache.clear_registry()
|
|
126
137
|
logger.warning("[duckdb.pro] connection reset — all cached views lost")
|
|
127
138
|
|
|
128
139
|
# ---------------------------------------------------------
|
|
@@ -313,6 +324,8 @@ class DuckDBPro:
|
|
|
313
324
|
# assignments are reached (which would cause a NameError otherwise).
|
|
314
325
|
rbac_view_names: List[str] = []
|
|
315
326
|
tombstone_view_names: List[str] = []
|
|
327
|
+
# Deletion-vector cache keys acquired this query — released in finally.
|
|
328
|
+
acquired_dv_keys: List[str] = []
|
|
316
329
|
try:
|
|
317
330
|
query_alias_to_name = dict(alias_to_table_name)
|
|
318
331
|
# Per-query suffix so concurrent queries never collide on a shared
|
|
@@ -328,8 +341,18 @@ class DuckDBPro:
|
|
|
328
341
|
source = query_alias_to_name[alias]
|
|
329
342
|
tomb_def = tombstone_views.get(alias)
|
|
330
343
|
view = f"tomb_{source}_{query_suffix}"
|
|
344
|
+
# Reuse a materialised deletion-vector table when the cache is
|
|
345
|
+
# enabled and the alias has a stable key; otherwise fall back to
|
|
346
|
+
# the inline read_parquet path (dv_table=None). All DDL — the DV
|
|
347
|
+
# CREATE TABLE inside acquire() and the view creation — runs
|
|
348
|
+
# under the connection lock, matching Pro's serialised model.
|
|
349
|
+
cache_key = getattr(tomb_def, "cache_key", None) if tomb_def else None
|
|
350
|
+
tomb_path = getattr(tomb_def, "tombstone_path", None) if tomb_def else None
|
|
331
351
|
with self._lock:
|
|
332
|
-
|
|
352
|
+
dv_table = self._tombstone_cache.acquire(con, cache_key, tomb_path)
|
|
353
|
+
if dv_table:
|
|
354
|
+
acquired_dv_keys.append(cache_key)
|
|
355
|
+
create_tombstone_view(con, source, view, tomb_def, dv_table=dv_table)
|
|
333
356
|
tombstone_view_names.append(view)
|
|
334
357
|
query_alias_to_name[alias] = view
|
|
335
358
|
|
|
@@ -395,6 +418,16 @@ class DuckDBPro:
|
|
|
395
418
|
except Exception:
|
|
396
419
|
pass
|
|
397
420
|
|
|
421
|
+
# Release deletion-vector refs now their views are gone; this may
|
|
422
|
+
# evict + DROP unreferenced DV tables over capacity.
|
|
423
|
+
if acquired_dv_keys:
|
|
424
|
+
with self._lock:
|
|
425
|
+
for cache_key in acquired_dv_keys:
|
|
426
|
+
try:
|
|
427
|
+
self._tombstone_cache.release(con, cache_key)
|
|
428
|
+
except Exception:
|
|
429
|
+
pass
|
|
430
|
+
|
|
398
431
|
# Release refs and drop stale tables
|
|
399
432
|
with self._lock:
|
|
400
433
|
self._release_refs(tables_used)
|