supertable 2.5.4__tar.gz → 2.5.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {supertable-2.5.4/supertable.egg-info → supertable-2.5.5}/PKG-INFO +1 -1
- {supertable-2.5.4 → supertable-2.5.5}/pyproject.toml +1 -1
- {supertable-2.5.4 → supertable-2.5.5}/supertable/__init__.py +41 -2
- {supertable-2.5.4 → supertable-2.5.5}/supertable/config/settings.py +12 -10
- {supertable-2.5.4 → supertable-2.5.5}/supertable/data_reader.py +90 -14
- {supertable-2.5.4 → supertable-2.5.5}/supertable/engine/adaptive_router.py +16 -1
- {supertable-2.5.4 → supertable-2.5.5}/supertable/engine/data_estimator.py +14 -7
- supertable-2.5.5/supertable/engine/disk_admission.py +168 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/engine/duckdb_engine.py +79 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/engine/engine_config.py +99 -17
- {supertable-2.5.4 → supertable-2.5.5}/supertable/engine/executor.py +260 -24
- {supertable-2.5.4 → supertable-2.5.5}/supertable/engine/file_cache.py +170 -12
- {supertable-2.5.4 → supertable-2.5.5}/supertable/engine/island_resources.py +488 -49
- {supertable-2.5.4 → supertable-2.5.5}/supertable/engine/islanddb.py +1588 -143
- {supertable-2.5.4 → supertable-2.5.5}/supertable/engine/query_observations.py +1 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/engine/range_cache.py +29 -9
- {supertable-2.5.4 → supertable-2.5.5}/supertable/processing.py +19 -5
- {supertable-2.5.4 → supertable-2.5.5}/supertable/redis_catalog.py +34 -3
- {supertable-2.5.4 → supertable-2.5.5}/supertable/super_table.py +10 -8
- {supertable-2.5.4 → supertable-2.5.5/supertable.egg-info}/PKG-INFO +1 -1
- {supertable-2.5.4 → supertable-2.5.5}/supertable.egg-info/SOURCES.txt +1 -0
- {supertable-2.5.4 → supertable-2.5.5}/LICENSE +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/MANIFEST.in +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/README.md +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/requirements.txt +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/setup.cfg +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/setup.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/audit/__init__.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/audit/admin.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/audit/chain.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/audit/consumers.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/audit/crypto.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/audit/diagnostics.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/audit/durable_journal.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/audit/events.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/audit/export.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/audit/logger.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/audit/middleware.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/audit/privileged.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/audit/privileged_outbox.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/audit/privileged_worker.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/audit/reader.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/audit/retention.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/audit/writer_parquet.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/audit/writer_redis.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/config/__init__.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/config/defaults.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/config/homedir.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/data_classes.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/data_writer.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/README.md +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/__init__.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/medcenter/__init__.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/medcenter/annotations.json +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/medcenter/core.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/medcenter/defaults.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/medcenter/demo_queries.md +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/medcenter/export_accounting.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/medcenter/finance_sources.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/medcenter/generate.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/medcenter/helpers.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/medcenter/load.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/medcenter/quality.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/medcenter/run.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/medcenter/transform.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/medcenter/validation.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/__init__.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/__main__.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/check_filter_builder.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/controller.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/data_writer_helpers.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/defaults.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/dummy_data.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/read_parquet_header.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/s01_01_01_create_super_table.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/s01_01_02_enable_mirroring_formats.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/s01_02_create_roles.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/s01_03_create_users.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/s02_01_write_dummy_data.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/s02_02_write_single_data.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/s02_03_01_write_staging.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/s02_03_02_create_pipe.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/s02_04_01_write_monitoring_simple.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/s02_04_02_write_monitoring_parallel.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/s02_05_write_tombstone.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/s03_01_read_data_error.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/s03_02_01_read_super_data_ok.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/s03_02_02_read_table_data_ok.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/s03_03_read_meta.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/s03_04_read_staging.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/s03_06_01_read_roles.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/s03_06_02_read_user.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/s03_07_01_estimate_read.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/s03_07_02_estimate_files.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/s03_08_read_snapshot_history.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/s04_01_03_delete_pipe.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/s05_01_delete_table.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/s05_02_delete_super_table.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/webshop/__init__.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/webshop/core.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/webshop/defaults.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/webshop/generate.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/webshop/load.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/webshop/stream_demo.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/webshop/topup.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/engine/__init__.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/engine/engine_common.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/engine/engine_enum.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/engine/island_spill.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/engine/join_pruner.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/engine/plan_stats.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/engine/remote_paths.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/engine/spark_thrift.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/engine/stable_http_relay.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/errors.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/locking/__init__.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/locking/file_lock.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/locking/redis_lock.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/logging.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/meta_reader.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/mirroring/__init__.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/mirroring/failure_safety.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/mirroring/mirror_delta.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/mirroring/mirror_formats.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/mirroring/mirror_iceberg.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/mirroring/mirror_parquet.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/monitoring/__init__.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/monitoring/partitions.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/monitoring_writer.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/odata_continuation.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/plan_extender.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/quality/__init__.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/quality/anomaly.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/quality/checker.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/quality/config.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/quality/cron.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/quality/execution.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/quality/history.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/quality/scheduler.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/quality/serialization.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/query_plan_manager.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/rbac/__init__.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/rbac/access_control.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/rbac/filter_builder.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/rbac/permissions.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/rbac/role_manager.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/rbac/row_column_security.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/rbac/user_manager.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/recovery/__init__.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/recovery/redis_rebuild.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/redis_connector.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/redis_infra.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/redis_keys.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/row_identity.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/simple_table.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/staging_area.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/storage/__init__.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/storage/azure_storage.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/storage/gcp_storage.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/storage/local_storage.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/storage/minio_storage.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/storage/s3_storage.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/storage/storage_factory.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/storage/storage_interface.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/super_pipe.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/system_query.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/tombstone_manifest_v2.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/utils/__init__.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/utils/diagnostic_redaction.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/utils/helper.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/utils/profiler.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/utils/snapshot.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/utils/spark_security.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/utils/sql_parser.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable/utils/timer.py +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable.egg-info/dependency_links.txt +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable.egg-info/entry_points.txt +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable.egg-info/requires.txt +0 -0
- {supertable-2.5.4 → supertable-2.5.5}/supertable.egg-info/top_level.txt +0 -0
|
@@ -25,11 +25,48 @@ See the ``supertable.demo`` package for runnable end-to-end demos and the
|
|
|
25
25
|
project documentation for the full API surface.
|
|
26
26
|
"""
|
|
27
27
|
|
|
28
|
-
__version__ = "2.5.
|
|
28
|
+
__version__ = "2.5.5"
|
|
29
29
|
|
|
30
|
+
import os
|
|
30
31
|
from importlib import import_module
|
|
31
32
|
from typing import TYPE_CHECKING, Any
|
|
32
33
|
|
|
34
|
+
|
|
35
|
+
def _bootstrap_island_polars_cpu_limit() -> None:
|
|
36
|
+
"""Apply an explicit Island CPU ceiling before Polars creates Rayon."""
|
|
37
|
+
raw = os.environ.get("SUPERTABLE_ISLAND_CPU_MAX")
|
|
38
|
+
if raw is None or not raw.strip():
|
|
39
|
+
return
|
|
40
|
+
try:
|
|
41
|
+
limit = int(raw.strip())
|
|
42
|
+
except ValueError:
|
|
43
|
+
raise RuntimeError(
|
|
44
|
+
"SUPERTABLE_ISLAND_CPU_MAX must be a non-negative integer"
|
|
45
|
+
) from None
|
|
46
|
+
if limit < 0:
|
|
47
|
+
raise RuntimeError(
|
|
48
|
+
"SUPERTABLE_ISLAND_CPU_MAX must be a non-negative integer"
|
|
49
|
+
)
|
|
50
|
+
if limit == 0:
|
|
51
|
+
return
|
|
52
|
+
existing_raw = os.environ.get("POLARS_MAX_THREADS")
|
|
53
|
+
if existing_raw is not None and existing_raw.strip():
|
|
54
|
+
try:
|
|
55
|
+
existing = int(existing_raw.strip())
|
|
56
|
+
except ValueError:
|
|
57
|
+
raise RuntimeError(
|
|
58
|
+
"POLARS_MAX_THREADS must be a positive integer"
|
|
59
|
+
) from None
|
|
60
|
+
if existing <= 0:
|
|
61
|
+
raise RuntimeError(
|
|
62
|
+
"POLARS_MAX_THREADS must be a positive integer"
|
|
63
|
+
)
|
|
64
|
+
limit = min(limit, existing)
|
|
65
|
+
os.environ["POLARS_MAX_THREADS"] = str(limit)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
_bootstrap_island_polars_cpu_limit()
|
|
69
|
+
|
|
33
70
|
# This public alias predates the ``supertable.engine`` implementation package.
|
|
34
71
|
# Importing the tiny enum after the now-lazy package is initialized ensures
|
|
35
72
|
# later imports of engine submodules cannot replace ``supertable.engine`` with
|
|
@@ -37,7 +74,9 @@ from typing import TYPE_CHECKING, Any
|
|
|
37
74
|
from supertable.engine.engine_enum import Engine as engine
|
|
38
75
|
|
|
39
76
|
|
|
40
|
-
# Importing the package is intentionally cheap and
|
|
77
|
+
# Importing the package is intentionally cheap and does not import native data
|
|
78
|
+
# engines. The only bootstrap side effect is the explicit Polars CPU ceiling
|
|
79
|
+
# above, which must exist before Rayon is initialized. The
|
|
41
80
|
# public convenience exports are loaded on first use (PEP 562) so importing
|
|
42
81
|
# ``supertable`` does not import PyArrow/Polars, connect to Redis, probe the
|
|
43
82
|
# filesystem, or initialise an application home.
|
|
@@ -263,7 +263,8 @@ class Settings:
|
|
|
263
263
|
# Parquet files; selective remote Island scans use the range cache below.
|
|
264
264
|
SUPERTABLE_ISLAND_CACHE_ENABLED: bool = True # SUPERTABLE_ISLAND_CACHE_ENABLED
|
|
265
265
|
SUPERTABLE_ISLAND_CACHE_DIR: str = "" # SUPERTABLE_ISLAND_CACHE_DIR
|
|
266
|
-
|
|
266
|
+
# Zero derives a safe share of currently free cache-filesystem capacity.
|
|
267
|
+
SUPERTABLE_ISLAND_CACHE_MAX_BYTES: int = 0 # SUPERTABLE_ISLAND_CACHE_MAX_BYTES
|
|
267
268
|
# Published Parquet objects are immutable/version-addressed. Capacity and
|
|
268
269
|
# integrity, rather than age, rotate the local object cache by default.
|
|
269
270
|
# Operators may still opt into an idle TTL with an explicitly positive
|
|
@@ -277,7 +278,7 @@ class Settings:
|
|
|
277
278
|
# for. Every cached interval is sealed to the provider version/ETag.
|
|
278
279
|
SUPERTABLE_ISLAND_RANGE_CACHE_ENABLED: bool = True # SUPERTABLE_ISLAND_RANGE_CACHE_ENABLED
|
|
279
280
|
SUPERTABLE_ISLAND_RANGE_CACHE_DIR: str = "" # SUPERTABLE_ISLAND_RANGE_CACHE_DIR
|
|
280
|
-
SUPERTABLE_ISLAND_RANGE_CACHE_MAX_BYTES: int =
|
|
281
|
+
SUPERTABLE_ISLAND_RANGE_CACHE_MAX_BYTES: int = 0 # SUPERTABLE_ISLAND_RANGE_CACHE_MAX_BYTES
|
|
281
282
|
# Immutable, version-sealed ranges rotate on capacity/corruption by default.
|
|
282
283
|
# A positive value explicitly restores idle-TTL eviction.
|
|
283
284
|
SUPERTABLE_ISLAND_RANGE_CACHE_TTL_SEC: int = 0 # SUPERTABLE_ISLAND_RANGE_CACHE_TTL_SEC
|
|
@@ -287,10 +288,11 @@ class Settings:
|
|
|
287
288
|
# quotas are hard; Polars/Arrow scan pools are process-global, so decoded
|
|
288
289
|
# scan-memory estimates are conservative routing/admission guards rather
|
|
289
290
|
# than an in-process per-query allocator cap.
|
|
290
|
-
SUPERTABLE_ISLAND_MEMORY_FRACTION: float = 0
|
|
291
|
-
SUPERTABLE_ISLAND_GLOBAL_MEMORY_FRACTION: float = 0
|
|
291
|
+
SUPERTABLE_ISLAND_MEMORY_FRACTION: float = 1.0 # SUPERTABLE_ISLAND_MEMORY_FRACTION
|
|
292
|
+
SUPERTABLE_ISLAND_GLOBAL_MEMORY_FRACTION: float = 1.0 # SUPERTABLE_ISLAND_GLOBAL_MEMORY_FRACTION
|
|
292
293
|
SUPERTABLE_ISLAND_MAX_MEMORY_BYTES: int = 0 # SUPERTABLE_ISLAND_MAX_MEMORY_BYTES
|
|
293
|
-
|
|
294
|
+
# Zero derives the effective container memory available to this process.
|
|
295
|
+
SUPERTABLE_ISLAND_MAX_RESULT_BYTES: int = 0 # SUPERTABLE_ISLAND_MAX_RESULT_BYTES
|
|
294
296
|
SUPERTABLE_ISLAND_CPU_MAX: int = 0 # SUPERTABLE_ISLAND_CPU_MAX
|
|
295
297
|
SUPERTABLE_ISLAND_IO_WORKERS_MAX: int = 16 # SUPERTABLE_ISLAND_IO_WORKERS_MAX
|
|
296
298
|
# Hard wall-clock bound for one admitted IslandDB execution. The bound is
|
|
@@ -637,7 +639,7 @@ def _build_settings() -> Settings:
|
|
|
637
639
|
SUPERTABLE_ISLAND_CACHE_ENABLED=_env_bool("SUPERTABLE_ISLAND_CACHE_ENABLED", True),
|
|
638
640
|
SUPERTABLE_ISLAND_CACHE_DIR=_env_str("SUPERTABLE_ISLAND_CACHE_DIR"),
|
|
639
641
|
SUPERTABLE_ISLAND_CACHE_MAX_BYTES=_env_int(
|
|
640
|
-
"SUPERTABLE_ISLAND_CACHE_MAX_BYTES",
|
|
642
|
+
"SUPERTABLE_ISLAND_CACHE_MAX_BYTES", 0,
|
|
641
643
|
),
|
|
642
644
|
SUPERTABLE_ISLAND_CACHE_TTL_SEC=_env_int(
|
|
643
645
|
"SUPERTABLE_ISLAND_CACHE_TTL_SEC", 0,
|
|
@@ -655,22 +657,22 @@ def _build_settings() -> Settings:
|
|
|
655
657
|
"SUPERTABLE_ISLAND_RANGE_CACHE_DIR",
|
|
656
658
|
),
|
|
657
659
|
SUPERTABLE_ISLAND_RANGE_CACHE_MAX_BYTES=_env_int(
|
|
658
|
-
"SUPERTABLE_ISLAND_RANGE_CACHE_MAX_BYTES",
|
|
660
|
+
"SUPERTABLE_ISLAND_RANGE_CACHE_MAX_BYTES", 0,
|
|
659
661
|
),
|
|
660
662
|
SUPERTABLE_ISLAND_RANGE_CACHE_TTL_SEC=_env_int(
|
|
661
663
|
"SUPERTABLE_ISLAND_RANGE_CACHE_TTL_SEC", 0,
|
|
662
664
|
),
|
|
663
665
|
SUPERTABLE_ISLAND_MEMORY_FRACTION=_env_float(
|
|
664
|
-
"SUPERTABLE_ISLAND_MEMORY_FRACTION", 0
|
|
666
|
+
"SUPERTABLE_ISLAND_MEMORY_FRACTION", 1.0,
|
|
665
667
|
),
|
|
666
668
|
SUPERTABLE_ISLAND_GLOBAL_MEMORY_FRACTION=_env_float(
|
|
667
|
-
"SUPERTABLE_ISLAND_GLOBAL_MEMORY_FRACTION", 0
|
|
669
|
+
"SUPERTABLE_ISLAND_GLOBAL_MEMORY_FRACTION", 1.0,
|
|
668
670
|
),
|
|
669
671
|
SUPERTABLE_ISLAND_MAX_MEMORY_BYTES=_env_int(
|
|
670
672
|
"SUPERTABLE_ISLAND_MAX_MEMORY_BYTES", 0,
|
|
671
673
|
),
|
|
672
674
|
SUPERTABLE_ISLAND_MAX_RESULT_BYTES=_env_int(
|
|
673
|
-
"SUPERTABLE_ISLAND_MAX_RESULT_BYTES",
|
|
675
|
+
"SUPERTABLE_ISLAND_MAX_RESULT_BYTES", 0,
|
|
674
676
|
),
|
|
675
677
|
SUPERTABLE_ISLAND_CPU_MAX=_env_int(
|
|
676
678
|
"SUPERTABLE_ISLAND_CPU_MAX", 0,
|
|
@@ -191,6 +191,9 @@ class _MonitoredResultStream:
|
|
|
191
191
|
self._finalize_complete = False
|
|
192
192
|
self._finalizing_thread: Optional[int] = None
|
|
193
193
|
self._finalize_error: BaseException | None = None
|
|
194
|
+
add_terminal_callback = getattr(inner, "add_terminal_callback", None)
|
|
195
|
+
if callable(add_terminal_callback):
|
|
196
|
+
add_terminal_callback(self._inner_terminal)
|
|
194
197
|
|
|
195
198
|
def __iter__(self):
|
|
196
199
|
return self
|
|
@@ -251,6 +254,29 @@ class _MonitoredResultStream:
|
|
|
251
254
|
with self._finalize_condition:
|
|
252
255
|
return self._finalized
|
|
253
256
|
|
|
257
|
+
def _inner_terminal(self, kind: str) -> None:
|
|
258
|
+
if kind == "completed":
|
|
259
|
+
status, message = Status.OK.value, None
|
|
260
|
+
elif kind == "timed_out":
|
|
261
|
+
status, message = Status.ERROR.value, "result stream timed out"
|
|
262
|
+
elif kind == "cancelled":
|
|
263
|
+
status, message = Status.ERROR.value, "result stream cancelled"
|
|
264
|
+
elif kind == "failed":
|
|
265
|
+
status, message = Status.ERROR.value, _QUERY_STREAM_ERROR
|
|
266
|
+
else:
|
|
267
|
+
status = Status.ERROR.value
|
|
268
|
+
message = "result stream closed before exhaustion"
|
|
269
|
+
try:
|
|
270
|
+
self._finish(status, message)
|
|
271
|
+
except BaseException as exc:
|
|
272
|
+
# A watcher-thread callback cannot raise into the request thread.
|
|
273
|
+
# _finish retains the error so any later stream operation observes
|
|
274
|
+
# it; protected failure audit emission has already been attempted.
|
|
275
|
+
logger.error(
|
|
276
|
+
"stream terminal monitoring failed; error_type=%s",
|
|
277
|
+
safe_exception_type(exc),
|
|
278
|
+
)
|
|
279
|
+
|
|
254
280
|
def __next__(self):
|
|
255
281
|
try:
|
|
256
282
|
batch = next(self._inner)
|
|
@@ -909,6 +935,9 @@ class DataReader:
|
|
|
909
935
|
actual_engine = str(
|
|
910
936
|
getattr(engine_used, "value", engine_used) or ""
|
|
911
937
|
)[:64]
|
|
938
|
+
audit_outcome = (
|
|
939
|
+
Outcome.SUCCESS if outcome == "success" else Outcome.FAILURE
|
|
940
|
+
)
|
|
912
941
|
emit(
|
|
913
942
|
category=EventCategory.DATA_ACCESS,
|
|
914
943
|
action=Actions.QUERY_EXECUTE,
|
|
@@ -930,7 +959,7 @@ class DataReader:
|
|
|
930
959
|
"column_count": max(0, int(result_columns)),
|
|
931
960
|
"outcome": outcome,
|
|
932
961
|
},
|
|
933
|
-
outcome=
|
|
962
|
+
outcome=audit_outcome,
|
|
934
963
|
)
|
|
935
964
|
|
|
936
965
|
def _assert_targets_exist(self, physical_tables) -> None:
|
|
@@ -2113,6 +2142,7 @@ class DataReader:
|
|
|
2113
2142
|
})
|
|
2114
2143
|
stream_timer.capture_and_reset_timing(event="EXECUTING_QUERY")
|
|
2115
2144
|
stream_timer.capture_duration(event="TOTAL_EXECUTE")
|
|
2145
|
+
monitoring_error: Optional[MonitoringPostExecutionError] = None
|
|
2116
2146
|
try:
|
|
2117
2147
|
extend_execution_plan(
|
|
2118
2148
|
query_plan_manager=stream_qpm,
|
|
@@ -2130,22 +2160,46 @@ class DataReader:
|
|
|
2130
2160
|
safe_cause = MonitoringDurabilityError(
|
|
2131
2161
|
"monitoring durability failure"
|
|
2132
2162
|
)
|
|
2133
|
-
|
|
2163
|
+
monitoring_error = MonitoringPostExecutionError(
|
|
2134
2164
|
organization=stream_qpm.organization,
|
|
2135
2165
|
super_name=stream_qpm.super_name,
|
|
2136
2166
|
query_id=str(getattr(stream_qpm, "query_id", "")),
|
|
2137
2167
|
status=final_status,
|
|
2138
2168
|
cause=safe_cause,
|
|
2139
|
-
)
|
|
2169
|
+
)
|
|
2140
2170
|
finally:
|
|
2141
2171
|
stream_timer.capture_and_reset_timing(event="EXTENDING_PLAN")
|
|
2142
|
-
|
|
2143
|
-
|
|
2144
|
-
|
|
2145
|
-
|
|
2146
|
-
|
|
2147
|
-
|
|
2172
|
+
audit_engine = _actual_engine_from_plan_stats(
|
|
2173
|
+
stream_plan_stats,
|
|
2174
|
+
fallback=_engine_used,
|
|
2175
|
+
)
|
|
2176
|
+
try:
|
|
2177
|
+
if final_status == Status.OK.value:
|
|
2178
|
+
self._emit_successful_query_audit(
|
|
2179
|
+
role_name=role_name,
|
|
2180
|
+
engine_used=audit_engine,
|
|
2181
|
+
result_rows=result_rows,
|
|
2182
|
+
result_columns=result_columns,
|
|
2183
|
+
)
|
|
2184
|
+
else:
|
|
2185
|
+
self._emit_successful_query_audit(
|
|
2186
|
+
role_name=role_name,
|
|
2187
|
+
engine_used=audit_engine,
|
|
2188
|
+
result_rows=result_rows,
|
|
2189
|
+
result_columns=result_columns,
|
|
2190
|
+
outcome="failure",
|
|
2191
|
+
)
|
|
2192
|
+
except Exception:
|
|
2193
|
+
if final_status == Status.OK.value and monitoring_error is None:
|
|
2194
|
+
raise
|
|
2195
|
+
# Preserve the producer/cancellation or monitoring error
|
|
2196
|
+
# just as the materialized failure path does.
|
|
2197
|
+
logger.debug(
|
|
2198
|
+
self._lp("failed to emit query failure audit"),
|
|
2199
|
+
exc_info=True,
|
|
2148
2200
|
)
|
|
2201
|
+
if monitoring_error is not None:
|
|
2202
|
+
raise monitoring_error
|
|
2149
2203
|
|
|
2150
2204
|
return (
|
|
2151
2205
|
_MonitoredResultStream(result_value, _finalize_stream_monitoring),
|
|
@@ -2771,6 +2825,32 @@ def _engine_name(value: Any) -> str:
|
|
|
2771
2825
|
return str(raw or "").strip().casefold()
|
|
2772
2826
|
|
|
2773
2827
|
|
|
2828
|
+
def _actual_engine_from_plan_stats(plan_stats: Any, *, fallback: Any) -> str:
|
|
2829
|
+
"""Resolve a stream's engine only after deferred delivery has finished."""
|
|
2830
|
+
try:
|
|
2831
|
+
stats = tuple(
|
|
2832
|
+
entry for entry in (getattr(plan_stats, "stats", ()) or ())
|
|
2833
|
+
if isinstance(entry, dict)
|
|
2834
|
+
)
|
|
2835
|
+
outcome = _latest_plan_stat(stats, "AUTO_ROUTING_OUTCOME")
|
|
2836
|
+
if isinstance(outcome, dict):
|
|
2837
|
+
actual = _engine_name(outcome.get("actual_engine"))
|
|
2838
|
+
if actual:
|
|
2839
|
+
return actual
|
|
2840
|
+
recorded = _engine_name(_latest_plan_stat(stats, "ENGINE"))
|
|
2841
|
+
if recorded:
|
|
2842
|
+
return recorded
|
|
2843
|
+
for entry in reversed(stats):
|
|
2844
|
+
attempt = entry.get("ENGINE_ATTEMPT")
|
|
2845
|
+
if isinstance(attempt, dict):
|
|
2846
|
+
attempted = _engine_name(attempt.get("engine"))
|
|
2847
|
+
if attempted:
|
|
2848
|
+
return attempted
|
|
2849
|
+
except Exception:
|
|
2850
|
+
pass
|
|
2851
|
+
return _engine_name(fallback)
|
|
2852
|
+
|
|
2853
|
+
|
|
2774
2854
|
def _update_query_out(
|
|
2775
2855
|
reader: DataReader,
|
|
2776
2856
|
out: Optional[Dict[str, Any]],
|
|
@@ -2837,14 +2917,10 @@ def _update_query_out(
|
|
|
2837
2917
|
if not selected and isinstance(outcome, dict):
|
|
2838
2918
|
selected = _engine_name(outcome.get("selected_engine"))
|
|
2839
2919
|
|
|
2840
|
-
actual = ""
|
|
2920
|
+
actual = _actual_engine_from_plan_stats(plan_stats, fallback="")
|
|
2841
2921
|
fallback = False
|
|
2842
2922
|
if isinstance(outcome, dict):
|
|
2843
|
-
actual = _engine_name(outcome.get("actual_engine"))
|
|
2844
2923
|
fallback = bool(outcome.get("fallback", False))
|
|
2845
|
-
actual = actual or _engine_name(recorded_engine)
|
|
2846
|
-
if not actual and engine_attempts:
|
|
2847
|
-
actual = _engine_name(engine_attempts[-1].get("engine"))
|
|
2848
2924
|
selected = selected or actual or requested
|
|
2849
2925
|
if actual and selected and not isinstance(outcome, dict):
|
|
2850
2926
|
fallback = actual != selected
|
|
@@ -138,6 +138,7 @@ class RoutingFeatures:
|
|
|
138
138
|
freshness_age_seconds: int = -1
|
|
139
139
|
has_active_tombstone: bool = False
|
|
140
140
|
streaming_result: bool = False
|
|
141
|
+
island_plan_evaluated: bool = False
|
|
141
142
|
island_advice: str = ""
|
|
142
143
|
island_cpu_workers: int = 0
|
|
143
144
|
island_io_workers: int = 0
|
|
@@ -193,6 +194,7 @@ class RoutingFeatures:
|
|
|
193
194
|
"stream" if self.streaming_result else "materialized",
|
|
194
195
|
"tombstone" if self.has_active_tombstone else "clean",
|
|
195
196
|
"fresh" if self.data_is_fresh else "stable",
|
|
197
|
+
"island_plan" if self.island_plan_evaluated else "no_island_plan",
|
|
196
198
|
str(self.island_advice or "none"),
|
|
197
199
|
str(self._magnitude_bucket(self.island_cpu_workers)),
|
|
198
200
|
str(self._magnitude_bucket(self.island_io_workers)),
|
|
@@ -502,10 +504,23 @@ class AdaptiveEngineRouter:
|
|
|
502
504
|
)
|
|
503
505
|
if not features.source_bytes_complete:
|
|
504
506
|
rejections[Engine.ISLANDDB].append("source byte estimate incomplete")
|
|
505
|
-
if
|
|
507
|
+
if (
|
|
508
|
+
features.island_plan_evaluated
|
|
509
|
+
and features.island_advice not in {
|
|
510
|
+
"island_in_memory", "island_spill",
|
|
511
|
+
}
|
|
512
|
+
):
|
|
506
513
|
rejections[Engine.ISLANDDB].append(
|
|
507
514
|
"IslandDB resource plan is not executable"
|
|
508
515
|
)
|
|
516
|
+
elif (
|
|
517
|
+
availability.island_enabled
|
|
518
|
+
and availability.island_supported
|
|
519
|
+
and not features.island_plan_evaluated
|
|
520
|
+
):
|
|
521
|
+
rejections[Engine.ISLANDDB].append(
|
|
522
|
+
"IslandDB resource plan is unavailable"
|
|
523
|
+
)
|
|
509
524
|
|
|
510
525
|
if not availability.spark_available:
|
|
511
526
|
rejections[Engine.SPARK_SQL].append("no safe fitting Spark cluster")
|
|
@@ -1684,11 +1684,13 @@ class DataEstimator:
|
|
|
1684
1684
|
return 0, False
|
|
1685
1685
|
widths: List[int] = []
|
|
1686
1686
|
for name in requested:
|
|
1687
|
-
|
|
1687
|
+
normalized_type = str(
|
|
1688
|
+
normalized_types.get(name) or ""
|
|
1689
|
+
).strip().casefold()
|
|
1690
|
+
width = self._decoded_fixed_width(normalized_type)
|
|
1688
1691
|
if (
|
|
1689
1692
|
width is None
|
|
1690
|
-
and
|
|
1691
|
-
== "binary"
|
|
1693
|
+
and normalized_type in {"binary", "string", "utf8"}
|
|
1692
1694
|
):
|
|
1693
1695
|
bound = (max_value_bytes or {}).get(name)
|
|
1694
1696
|
if (
|
|
@@ -1696,9 +1698,12 @@ class DataEstimator:
|
|
|
1696
1698
|
and not isinstance(bound, bool)
|
|
1697
1699
|
and bound >= 0
|
|
1698
1700
|
):
|
|
1699
|
-
#
|
|
1700
|
-
#
|
|
1701
|
-
|
|
1701
|
+
# Binary owns a 32-bit offset; Polars' native UTF-8 value
|
|
1702
|
+
# representation can retain a 16-byte string view. The
|
|
1703
|
+
# caller adds validity/alignment slack below.
|
|
1704
|
+
width = int(bound) + (
|
|
1705
|
+
4 if normalized_type == "binary" else 16
|
|
1706
|
+
)
|
|
1702
1707
|
if width is None:
|
|
1703
1708
|
return 0, False
|
|
1704
1709
|
widths.append(width)
|
|
@@ -2511,7 +2516,9 @@ class DataEstimator:
|
|
|
2511
2516
|
for file_key in survivors
|
|
2512
2517
|
)
|
|
2513
2518
|
for column_name, type_name in schema_types.items()
|
|
2514
|
-
if str(type_name).strip().casefold()
|
|
2519
|
+
if str(type_name).strip().casefold() in {
|
|
2520
|
+
"binary", "string", "utf8",
|
|
2521
|
+
}
|
|
2515
2522
|
and survivors
|
|
2516
2523
|
and all(
|
|
2517
2524
|
isinstance(resource_value_bounds.get(file_key), dict)
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
"""Cross-process disk admission for caches and IslandDB spill files."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import fcntl
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
import shutil
|
|
9
|
+
import time
|
|
10
|
+
import uuid
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
from supertable.config.homedir import get_app_home
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
_STALE_SECONDS = 24 * 60 * 60
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class DiskAdmissionUnavailable(RuntimeError):
|
|
20
|
+
"""The shared filesystem reservation cannot be admitted."""
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _existing_ancestor(path: str | os.PathLike[str]) -> Path:
|
|
24
|
+
candidate = Path(path).expanduser().resolve()
|
|
25
|
+
while not candidate.exists() and candidate != candidate.parent:
|
|
26
|
+
candidate = candidate.parent
|
|
27
|
+
return candidate
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _reservation_root(path: str | os.PathLike[str]) -> Path:
|
|
31
|
+
probe = _existing_ancestor(path)
|
|
32
|
+
try:
|
|
33
|
+
device = os.stat(probe).st_dev
|
|
34
|
+
except OSError as exc:
|
|
35
|
+
raise DiskAdmissionUnavailable("cannot identify cache filesystem") from exc
|
|
36
|
+
base = Path(get_app_home()) / ".supertable-disk-admission"
|
|
37
|
+
try:
|
|
38
|
+
if os.stat(_existing_ancestor(base)).st_dev != device:
|
|
39
|
+
base = probe / ".supertable-disk-admission"
|
|
40
|
+
except OSError:
|
|
41
|
+
base = probe / ".supertable-disk-admission"
|
|
42
|
+
return base / str(device)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _read_size(path: Path) -> int:
|
|
46
|
+
try:
|
|
47
|
+
with path.open("r", encoding="utf-8") as handle:
|
|
48
|
+
value = json.load(handle).get("size", 0)
|
|
49
|
+
return max(0, int(value))
|
|
50
|
+
except (OSError, ValueError, TypeError, json.JSONDecodeError):
|
|
51
|
+
return 0
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _clean_stale(root: Path) -> None:
|
|
55
|
+
now = time.time()
|
|
56
|
+
try:
|
|
57
|
+
entries = tuple(root.iterdir())
|
|
58
|
+
except OSError:
|
|
59
|
+
return
|
|
60
|
+
for entry in entries:
|
|
61
|
+
if not entry.name.endswith(".json"):
|
|
62
|
+
continue
|
|
63
|
+
try:
|
|
64
|
+
if now - entry.stat().st_mtime > _STALE_SECONDS:
|
|
65
|
+
entry.unlink(missing_ok=True)
|
|
66
|
+
continue
|
|
67
|
+
with entry.open("r", encoding="utf-8") as handle:
|
|
68
|
+
payload = json.load(handle)
|
|
69
|
+
pid = payload.get("pid")
|
|
70
|
+
if isinstance(pid, int) and pid > 0 and pid != os.getpid():
|
|
71
|
+
try:
|
|
72
|
+
os.kill(pid, 0)
|
|
73
|
+
except ProcessLookupError:
|
|
74
|
+
entry.unlink(missing_ok=True)
|
|
75
|
+
except PermissionError:
|
|
76
|
+
pass
|
|
77
|
+
except (OSError, ValueError, TypeError, json.JSONDecodeError):
|
|
78
|
+
try:
|
|
79
|
+
entry.unlink(missing_ok=True)
|
|
80
|
+
except OSError:
|
|
81
|
+
pass
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
class DiskReservation:
|
|
85
|
+
"""An idempotent shared filesystem reservation."""
|
|
86
|
+
|
|
87
|
+
def __init__(self, path: Path):
|
|
88
|
+
self.path = path
|
|
89
|
+
self._released = False
|
|
90
|
+
|
|
91
|
+
def release(self) -> None:
|
|
92
|
+
if self._released:
|
|
93
|
+
return
|
|
94
|
+
self._released = True
|
|
95
|
+
try:
|
|
96
|
+
self.path.unlink(missing_ok=True)
|
|
97
|
+
except OSError:
|
|
98
|
+
pass
|
|
99
|
+
|
|
100
|
+
def __enter__(self) -> "DiskReservation":
|
|
101
|
+
return self
|
|
102
|
+
|
|
103
|
+
def __exit__(self, exc_type, exc, tb) -> None:
|
|
104
|
+
self.release()
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def reserve_disk(
|
|
108
|
+
path: str | os.PathLike[str],
|
|
109
|
+
size: int,
|
|
110
|
+
*,
|
|
111
|
+
min_free_bytes: int = 0,
|
|
112
|
+
) -> DiskReservation | None:
|
|
113
|
+
"""Atomically reserve bytes shared by cache and spill users."""
|
|
114
|
+
size = max(0, int(size))
|
|
115
|
+
if size == 0:
|
|
116
|
+
return None
|
|
117
|
+
root = _reservation_root(path)
|
|
118
|
+
root.mkdir(parents=True, exist_ok=True)
|
|
119
|
+
lock_path = root / ".lock"
|
|
120
|
+
with lock_path.open("a+") as lock:
|
|
121
|
+
fcntl.flock(lock.fileno(), fcntl.LOCK_EX)
|
|
122
|
+
_clean_stale(root)
|
|
123
|
+
try:
|
|
124
|
+
free = max(0, int(shutil.disk_usage(_existing_ancestor(path)).free))
|
|
125
|
+
except (OSError, ValueError, TypeError):
|
|
126
|
+
raise DiskAdmissionUnavailable("cannot inspect free disk space") from None
|
|
127
|
+
reserved = sum(
|
|
128
|
+
_read_size(entry)
|
|
129
|
+
for entry in root.iterdir()
|
|
130
|
+
if entry.name.endswith(".json")
|
|
131
|
+
)
|
|
132
|
+
if free - reserved - size < max(0, int(min_free_bytes)):
|
|
133
|
+
raise DiskAdmissionUnavailable(
|
|
134
|
+
"shared filesystem reservation would exhaust the disk reserve"
|
|
135
|
+
)
|
|
136
|
+
token = root / f"{os.getpid()}-{uuid.uuid4().hex}.json"
|
|
137
|
+
temporary = token.with_suffix(".tmp")
|
|
138
|
+
payload = {
|
|
139
|
+
"size": size,
|
|
140
|
+
"pid": os.getpid(),
|
|
141
|
+
"created_ns": time.time_ns(),
|
|
142
|
+
}
|
|
143
|
+
with temporary.open("x", encoding="utf-8") as handle:
|
|
144
|
+
json.dump(payload, handle, separators=(",", ":"))
|
|
145
|
+
handle.flush()
|
|
146
|
+
os.fsync(handle.fileno())
|
|
147
|
+
os.replace(temporary, token)
|
|
148
|
+
fcntl.flock(lock.fileno(), fcntl.LOCK_UN)
|
|
149
|
+
return DiskReservation(token)
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def release_local_reservation(local_path: str | os.PathLike[str]) -> None:
|
|
153
|
+
"""Release a shared token embedded in a cache reservation record."""
|
|
154
|
+
try:
|
|
155
|
+
with Path(local_path).open("r", encoding="utf-8") as handle:
|
|
156
|
+
token = json.load(handle).get("shared_token")
|
|
157
|
+
if token:
|
|
158
|
+
DiskReservation(Path(str(token))).release()
|
|
159
|
+
except (OSError, ValueError, TypeError, json.JSONDecodeError):
|
|
160
|
+
pass
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
__all__ = [
|
|
164
|
+
"DiskAdmissionUnavailable",
|
|
165
|
+
"DiskReservation",
|
|
166
|
+
"release_local_reservation",
|
|
167
|
+
"reserve_disk",
|
|
168
|
+
]
|