supertable 2.5.4__tar.gz → 2.5.5__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (179) hide show
  1. {supertable-2.5.4/supertable.egg-info → supertable-2.5.5}/PKG-INFO +1 -1
  2. {supertable-2.5.4 → supertable-2.5.5}/pyproject.toml +1 -1
  3. {supertable-2.5.4 → supertable-2.5.5}/supertable/__init__.py +41 -2
  4. {supertable-2.5.4 → supertable-2.5.5}/supertable/config/settings.py +12 -10
  5. {supertable-2.5.4 → supertable-2.5.5}/supertable/data_reader.py +90 -14
  6. {supertable-2.5.4 → supertable-2.5.5}/supertable/engine/adaptive_router.py +16 -1
  7. {supertable-2.5.4 → supertable-2.5.5}/supertable/engine/data_estimator.py +14 -7
  8. supertable-2.5.5/supertable/engine/disk_admission.py +168 -0
  9. {supertable-2.5.4 → supertable-2.5.5}/supertable/engine/duckdb_engine.py +79 -0
  10. {supertable-2.5.4 → supertable-2.5.5}/supertable/engine/engine_config.py +99 -17
  11. {supertable-2.5.4 → supertable-2.5.5}/supertable/engine/executor.py +260 -24
  12. {supertable-2.5.4 → supertable-2.5.5}/supertable/engine/file_cache.py +170 -12
  13. {supertable-2.5.4 → supertable-2.5.5}/supertable/engine/island_resources.py +488 -49
  14. {supertable-2.5.4 → supertable-2.5.5}/supertable/engine/islanddb.py +1588 -143
  15. {supertable-2.5.4 → supertable-2.5.5}/supertable/engine/query_observations.py +1 -0
  16. {supertable-2.5.4 → supertable-2.5.5}/supertable/engine/range_cache.py +29 -9
  17. {supertable-2.5.4 → supertable-2.5.5}/supertable/processing.py +19 -5
  18. {supertable-2.5.4 → supertable-2.5.5}/supertable/redis_catalog.py +34 -3
  19. {supertable-2.5.4 → supertable-2.5.5}/supertable/super_table.py +10 -8
  20. {supertable-2.5.4 → supertable-2.5.5/supertable.egg-info}/PKG-INFO +1 -1
  21. {supertable-2.5.4 → supertable-2.5.5}/supertable.egg-info/SOURCES.txt +1 -0
  22. {supertable-2.5.4 → supertable-2.5.5}/LICENSE +0 -0
  23. {supertable-2.5.4 → supertable-2.5.5}/MANIFEST.in +0 -0
  24. {supertable-2.5.4 → supertable-2.5.5}/README.md +0 -0
  25. {supertable-2.5.4 → supertable-2.5.5}/requirements.txt +0 -0
  26. {supertable-2.5.4 → supertable-2.5.5}/setup.cfg +0 -0
  27. {supertable-2.5.4 → supertable-2.5.5}/setup.py +0 -0
  28. {supertable-2.5.4 → supertable-2.5.5}/supertable/audit/__init__.py +0 -0
  29. {supertable-2.5.4 → supertable-2.5.5}/supertable/audit/admin.py +0 -0
  30. {supertable-2.5.4 → supertable-2.5.5}/supertable/audit/chain.py +0 -0
  31. {supertable-2.5.4 → supertable-2.5.5}/supertable/audit/consumers.py +0 -0
  32. {supertable-2.5.4 → supertable-2.5.5}/supertable/audit/crypto.py +0 -0
  33. {supertable-2.5.4 → supertable-2.5.5}/supertable/audit/diagnostics.py +0 -0
  34. {supertable-2.5.4 → supertable-2.5.5}/supertable/audit/durable_journal.py +0 -0
  35. {supertable-2.5.4 → supertable-2.5.5}/supertable/audit/events.py +0 -0
  36. {supertable-2.5.4 → supertable-2.5.5}/supertable/audit/export.py +0 -0
  37. {supertable-2.5.4 → supertable-2.5.5}/supertable/audit/logger.py +0 -0
  38. {supertable-2.5.4 → supertable-2.5.5}/supertable/audit/middleware.py +0 -0
  39. {supertable-2.5.4 → supertable-2.5.5}/supertable/audit/privileged.py +0 -0
  40. {supertable-2.5.4 → supertable-2.5.5}/supertable/audit/privileged_outbox.py +0 -0
  41. {supertable-2.5.4 → supertable-2.5.5}/supertable/audit/privileged_worker.py +0 -0
  42. {supertable-2.5.4 → supertable-2.5.5}/supertable/audit/reader.py +0 -0
  43. {supertable-2.5.4 → supertable-2.5.5}/supertable/audit/retention.py +0 -0
  44. {supertable-2.5.4 → supertable-2.5.5}/supertable/audit/writer_parquet.py +0 -0
  45. {supertable-2.5.4 → supertable-2.5.5}/supertable/audit/writer_redis.py +0 -0
  46. {supertable-2.5.4 → supertable-2.5.5}/supertable/config/__init__.py +0 -0
  47. {supertable-2.5.4 → supertable-2.5.5}/supertable/config/defaults.py +0 -0
  48. {supertable-2.5.4 → supertable-2.5.5}/supertable/config/homedir.py +0 -0
  49. {supertable-2.5.4 → supertable-2.5.5}/supertable/data_classes.py +0 -0
  50. {supertable-2.5.4 → supertable-2.5.5}/supertable/data_writer.py +0 -0
  51. {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/README.md +0 -0
  52. {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/__init__.py +0 -0
  53. {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/medcenter/__init__.py +0 -0
  54. {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/medcenter/annotations.json +0 -0
  55. {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/medcenter/core.py +0 -0
  56. {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/medcenter/defaults.py +0 -0
  57. {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/medcenter/demo_queries.md +0 -0
  58. {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/medcenter/export_accounting.py +0 -0
  59. {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/medcenter/finance_sources.py +0 -0
  60. {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/medcenter/generate.py +0 -0
  61. {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/medcenter/helpers.py +0 -0
  62. {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/medcenter/load.py +0 -0
  63. {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/medcenter/quality.py +0 -0
  64. {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/medcenter/run.py +0 -0
  65. {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/medcenter/transform.py +0 -0
  66. {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/medcenter/validation.py +0 -0
  67. {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/__init__.py +0 -0
  68. {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/__main__.py +0 -0
  69. {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/check_filter_builder.py +0 -0
  70. {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/controller.py +0 -0
  71. {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/data_writer_helpers.py +0 -0
  72. {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/defaults.py +0 -0
  73. {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/dummy_data.py +0 -0
  74. {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/read_parquet_header.py +0 -0
  75. {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/s01_01_01_create_super_table.py +0 -0
  76. {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/s01_01_02_enable_mirroring_formats.py +0 -0
  77. {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/s01_02_create_roles.py +0 -0
  78. {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/s01_03_create_users.py +0 -0
  79. {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/s02_01_write_dummy_data.py +0 -0
  80. {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/s02_02_write_single_data.py +0 -0
  81. {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/s02_03_01_write_staging.py +0 -0
  82. {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/s02_03_02_create_pipe.py +0 -0
  83. {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/s02_04_01_write_monitoring_simple.py +0 -0
  84. {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/s02_04_02_write_monitoring_parallel.py +0 -0
  85. {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/s02_05_write_tombstone.py +0 -0
  86. {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/s03_01_read_data_error.py +0 -0
  87. {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/s03_02_01_read_super_data_ok.py +0 -0
  88. {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/s03_02_02_read_table_data_ok.py +0 -0
  89. {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/s03_03_read_meta.py +0 -0
  90. {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/s03_04_read_staging.py +0 -0
  91. {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/s03_06_01_read_roles.py +0 -0
  92. {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/s03_06_02_read_user.py +0 -0
  93. {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/s03_07_01_estimate_read.py +0 -0
  94. {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/s03_07_02_estimate_files.py +0 -0
  95. {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/s03_08_read_snapshot_history.py +0 -0
  96. {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/s04_01_03_delete_pipe.py +0 -0
  97. {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/s05_01_delete_table.py +0 -0
  98. {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/quickstart/s05_02_delete_super_table.py +0 -0
  99. {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/webshop/__init__.py +0 -0
  100. {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/webshop/core.py +0 -0
  101. {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/webshop/defaults.py +0 -0
  102. {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/webshop/generate.py +0 -0
  103. {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/webshop/load.py +0 -0
  104. {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/webshop/stream_demo.py +0 -0
  105. {supertable-2.5.4 → supertable-2.5.5}/supertable/demo/webshop/topup.py +0 -0
  106. {supertable-2.5.4 → supertable-2.5.5}/supertable/engine/__init__.py +0 -0
  107. {supertable-2.5.4 → supertable-2.5.5}/supertable/engine/engine_common.py +0 -0
  108. {supertable-2.5.4 → supertable-2.5.5}/supertable/engine/engine_enum.py +0 -0
  109. {supertable-2.5.4 → supertable-2.5.5}/supertable/engine/island_spill.py +0 -0
  110. {supertable-2.5.4 → supertable-2.5.5}/supertable/engine/join_pruner.py +0 -0
  111. {supertable-2.5.4 → supertable-2.5.5}/supertable/engine/plan_stats.py +0 -0
  112. {supertable-2.5.4 → supertable-2.5.5}/supertable/engine/remote_paths.py +0 -0
  113. {supertable-2.5.4 → supertable-2.5.5}/supertable/engine/spark_thrift.py +0 -0
  114. {supertable-2.5.4 → supertable-2.5.5}/supertable/engine/stable_http_relay.py +0 -0
  115. {supertable-2.5.4 → supertable-2.5.5}/supertable/errors.py +0 -0
  116. {supertable-2.5.4 → supertable-2.5.5}/supertable/locking/__init__.py +0 -0
  117. {supertable-2.5.4 → supertable-2.5.5}/supertable/locking/file_lock.py +0 -0
  118. {supertable-2.5.4 → supertable-2.5.5}/supertable/locking/redis_lock.py +0 -0
  119. {supertable-2.5.4 → supertable-2.5.5}/supertable/logging.py +0 -0
  120. {supertable-2.5.4 → supertable-2.5.5}/supertable/meta_reader.py +0 -0
  121. {supertable-2.5.4 → supertable-2.5.5}/supertable/mirroring/__init__.py +0 -0
  122. {supertable-2.5.4 → supertable-2.5.5}/supertable/mirroring/failure_safety.py +0 -0
  123. {supertable-2.5.4 → supertable-2.5.5}/supertable/mirroring/mirror_delta.py +0 -0
  124. {supertable-2.5.4 → supertable-2.5.5}/supertable/mirroring/mirror_formats.py +0 -0
  125. {supertable-2.5.4 → supertable-2.5.5}/supertable/mirroring/mirror_iceberg.py +0 -0
  126. {supertable-2.5.4 → supertable-2.5.5}/supertable/mirroring/mirror_parquet.py +0 -0
  127. {supertable-2.5.4 → supertable-2.5.5}/supertable/monitoring/__init__.py +0 -0
  128. {supertable-2.5.4 → supertable-2.5.5}/supertable/monitoring/partitions.py +0 -0
  129. {supertable-2.5.4 → supertable-2.5.5}/supertable/monitoring_writer.py +0 -0
  130. {supertable-2.5.4 → supertable-2.5.5}/supertable/odata_continuation.py +0 -0
  131. {supertable-2.5.4 → supertable-2.5.5}/supertable/plan_extender.py +0 -0
  132. {supertable-2.5.4 → supertable-2.5.5}/supertable/quality/__init__.py +0 -0
  133. {supertable-2.5.4 → supertable-2.5.5}/supertable/quality/anomaly.py +0 -0
  134. {supertable-2.5.4 → supertable-2.5.5}/supertable/quality/checker.py +0 -0
  135. {supertable-2.5.4 → supertable-2.5.5}/supertable/quality/config.py +0 -0
  136. {supertable-2.5.4 → supertable-2.5.5}/supertable/quality/cron.py +0 -0
  137. {supertable-2.5.4 → supertable-2.5.5}/supertable/quality/execution.py +0 -0
  138. {supertable-2.5.4 → supertable-2.5.5}/supertable/quality/history.py +0 -0
  139. {supertable-2.5.4 → supertable-2.5.5}/supertable/quality/scheduler.py +0 -0
  140. {supertable-2.5.4 → supertable-2.5.5}/supertable/quality/serialization.py +0 -0
  141. {supertable-2.5.4 → supertable-2.5.5}/supertable/query_plan_manager.py +0 -0
  142. {supertable-2.5.4 → supertable-2.5.5}/supertable/rbac/__init__.py +0 -0
  143. {supertable-2.5.4 → supertable-2.5.5}/supertable/rbac/access_control.py +0 -0
  144. {supertable-2.5.4 → supertable-2.5.5}/supertable/rbac/filter_builder.py +0 -0
  145. {supertable-2.5.4 → supertable-2.5.5}/supertable/rbac/permissions.py +0 -0
  146. {supertable-2.5.4 → supertable-2.5.5}/supertable/rbac/role_manager.py +0 -0
  147. {supertable-2.5.4 → supertable-2.5.5}/supertable/rbac/row_column_security.py +0 -0
  148. {supertable-2.5.4 → supertable-2.5.5}/supertable/rbac/user_manager.py +0 -0
  149. {supertable-2.5.4 → supertable-2.5.5}/supertable/recovery/__init__.py +0 -0
  150. {supertable-2.5.4 → supertable-2.5.5}/supertable/recovery/redis_rebuild.py +0 -0
  151. {supertable-2.5.4 → supertable-2.5.5}/supertable/redis_connector.py +0 -0
  152. {supertable-2.5.4 → supertable-2.5.5}/supertable/redis_infra.py +0 -0
  153. {supertable-2.5.4 → supertable-2.5.5}/supertable/redis_keys.py +0 -0
  154. {supertable-2.5.4 → supertable-2.5.5}/supertable/row_identity.py +0 -0
  155. {supertable-2.5.4 → supertable-2.5.5}/supertable/simple_table.py +0 -0
  156. {supertable-2.5.4 → supertable-2.5.5}/supertable/staging_area.py +0 -0
  157. {supertable-2.5.4 → supertable-2.5.5}/supertable/storage/__init__.py +0 -0
  158. {supertable-2.5.4 → supertable-2.5.5}/supertable/storage/azure_storage.py +0 -0
  159. {supertable-2.5.4 → supertable-2.5.5}/supertable/storage/gcp_storage.py +0 -0
  160. {supertable-2.5.4 → supertable-2.5.5}/supertable/storage/local_storage.py +0 -0
  161. {supertable-2.5.4 → supertable-2.5.5}/supertable/storage/minio_storage.py +0 -0
  162. {supertable-2.5.4 → supertable-2.5.5}/supertable/storage/s3_storage.py +0 -0
  163. {supertable-2.5.4 → supertable-2.5.5}/supertable/storage/storage_factory.py +0 -0
  164. {supertable-2.5.4 → supertable-2.5.5}/supertable/storage/storage_interface.py +0 -0
  165. {supertable-2.5.4 → supertable-2.5.5}/supertable/super_pipe.py +0 -0
  166. {supertable-2.5.4 → supertable-2.5.5}/supertable/system_query.py +0 -0
  167. {supertable-2.5.4 → supertable-2.5.5}/supertable/tombstone_manifest_v2.py +0 -0
  168. {supertable-2.5.4 → supertable-2.5.5}/supertable/utils/__init__.py +0 -0
  169. {supertable-2.5.4 → supertable-2.5.5}/supertable/utils/diagnostic_redaction.py +0 -0
  170. {supertable-2.5.4 → supertable-2.5.5}/supertable/utils/helper.py +0 -0
  171. {supertable-2.5.4 → supertable-2.5.5}/supertable/utils/profiler.py +0 -0
  172. {supertable-2.5.4 → supertable-2.5.5}/supertable/utils/snapshot.py +0 -0
  173. {supertable-2.5.4 → supertable-2.5.5}/supertable/utils/spark_security.py +0 -0
  174. {supertable-2.5.4 → supertable-2.5.5}/supertable/utils/sql_parser.py +0 -0
  175. {supertable-2.5.4 → supertable-2.5.5}/supertable/utils/timer.py +0 -0
  176. {supertable-2.5.4 → supertable-2.5.5}/supertable.egg-info/dependency_links.txt +0 -0
  177. {supertable-2.5.4 → supertable-2.5.5}/supertable.egg-info/entry_points.txt +0 -0
  178. {supertable-2.5.4 → supertable-2.5.5}/supertable.egg-info/requires.txt +0 -0
  179. {supertable-2.5.4 → supertable-2.5.5}/supertable.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: supertable
3
- Version: 2.5.4
3
+ Version: 2.5.5
4
4
  Summary: SuperTable — versioned data lake library for SQL analytics on Parquet + Redis.
5
5
  Author-email: Levente Kupas <lkupas@kladnasoft.com>
6
6
  License-Expression: LicenseRef-FSL-1.1-ALv2
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "supertable"
7
- version = "2.5.4"
7
+ version = "2.5.5"
8
8
  description = "SuperTable — versioned data lake library for SQL analytics on Parquet + Redis."
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.10"
@@ -25,11 +25,48 @@ See the ``supertable.demo`` package for runnable end-to-end demos and the
25
25
  project documentation for the full API surface.
26
26
  """
27
27
 
28
- __version__ = "2.5.4"
28
+ __version__ = "2.5.5"
29
29
 
30
+ import os
30
31
  from importlib import import_module
31
32
  from typing import TYPE_CHECKING, Any
32
33
 
34
+
35
+ def _bootstrap_island_polars_cpu_limit() -> None:
36
+ """Apply an explicit Island CPU ceiling before Polars creates Rayon."""
37
+ raw = os.environ.get("SUPERTABLE_ISLAND_CPU_MAX")
38
+ if raw is None or not raw.strip():
39
+ return
40
+ try:
41
+ limit = int(raw.strip())
42
+ except ValueError:
43
+ raise RuntimeError(
44
+ "SUPERTABLE_ISLAND_CPU_MAX must be a non-negative integer"
45
+ ) from None
46
+ if limit < 0:
47
+ raise RuntimeError(
48
+ "SUPERTABLE_ISLAND_CPU_MAX must be a non-negative integer"
49
+ )
50
+ if limit == 0:
51
+ return
52
+ existing_raw = os.environ.get("POLARS_MAX_THREADS")
53
+ if existing_raw is not None and existing_raw.strip():
54
+ try:
55
+ existing = int(existing_raw.strip())
56
+ except ValueError:
57
+ raise RuntimeError(
58
+ "POLARS_MAX_THREADS must be a positive integer"
59
+ ) from None
60
+ if existing <= 0:
61
+ raise RuntimeError(
62
+ "POLARS_MAX_THREADS must be a positive integer"
63
+ )
64
+ limit = min(limit, existing)
65
+ os.environ["POLARS_MAX_THREADS"] = str(limit)
66
+
67
+
68
+ _bootstrap_island_polars_cpu_limit()
69
+
33
70
  # This public alias predates the ``supertable.engine`` implementation package.
34
71
  # Importing the tiny enum after the now-lazy package is initialized ensures
35
72
  # later imports of engine submodules cannot replace ``supertable.engine`` with
@@ -37,7 +74,9 @@ from typing import TYPE_CHECKING, Any
37
74
  from supertable.engine.engine_enum import Engine as engine
38
75
 
39
76
 
40
- # Importing the package is intentionally cheap and side-effect-free. The
77
+ # Importing the package is intentionally cheap and does not import native data
78
+ # engines. The only bootstrap side effect is the explicit Polars CPU ceiling
79
+ # above, which must exist before Rayon is initialized. The
41
80
  # public convenience exports are loaded on first use (PEP 562) so importing
42
81
  # ``supertable`` does not import PyArrow/Polars, connect to Redis, probe the
43
82
  # filesystem, or initialise an application home.
@@ -263,7 +263,8 @@ class Settings:
263
263
  # Parquet files; selective remote Island scans use the range cache below.
264
264
  SUPERTABLE_ISLAND_CACHE_ENABLED: bool = True # SUPERTABLE_ISLAND_CACHE_ENABLED
265
265
  SUPERTABLE_ISLAND_CACHE_DIR: str = "" # SUPERTABLE_ISLAND_CACHE_DIR
266
- SUPERTABLE_ISLAND_CACHE_MAX_BYTES: int = 20 * 1024 * 1024 * 1024 # SUPERTABLE_ISLAND_CACHE_MAX_BYTES
266
+ # Zero derives a safe share of currently free cache-filesystem capacity.
267
+ SUPERTABLE_ISLAND_CACHE_MAX_BYTES: int = 0 # SUPERTABLE_ISLAND_CACHE_MAX_BYTES
267
268
  # Published Parquet objects are immutable/version-addressed. Capacity and
268
269
  # integrity, rather than age, rotate the local object cache by default.
269
270
  # Operators may still opt into an idle TTL with an explicitly positive
@@ -277,7 +278,7 @@ class Settings:
277
278
  # for. Every cached interval is sealed to the provider version/ETag.
278
279
  SUPERTABLE_ISLAND_RANGE_CACHE_ENABLED: bool = True # SUPERTABLE_ISLAND_RANGE_CACHE_ENABLED
279
280
  SUPERTABLE_ISLAND_RANGE_CACHE_DIR: str = "" # SUPERTABLE_ISLAND_RANGE_CACHE_DIR
280
- SUPERTABLE_ISLAND_RANGE_CACHE_MAX_BYTES: int = 20 * 1024 * 1024 * 1024 # SUPERTABLE_ISLAND_RANGE_CACHE_MAX_BYTES
281
+ SUPERTABLE_ISLAND_RANGE_CACHE_MAX_BYTES: int = 0 # SUPERTABLE_ISLAND_RANGE_CACHE_MAX_BYTES
281
282
  # Immutable, version-sealed ranges rotate on capacity/corruption by default.
282
283
  # A positive value explicitly restores idle-TTL eviction.
283
284
  SUPERTABLE_ISLAND_RANGE_CACHE_TTL_SEC: int = 0 # SUPERTABLE_ISLAND_RANGE_CACHE_TTL_SEC
@@ -287,10 +288,11 @@ class Settings:
287
288
  # quotas are hard; Polars/Arrow scan pools are process-global, so decoded
288
289
  # scan-memory estimates are conservative routing/admission guards rather
289
290
  # than an in-process per-query allocator cap.
290
- SUPERTABLE_ISLAND_MEMORY_FRACTION: float = 0.60 # SUPERTABLE_ISLAND_MEMORY_FRACTION
291
- SUPERTABLE_ISLAND_GLOBAL_MEMORY_FRACTION: float = 0.80 # SUPERTABLE_ISLAND_GLOBAL_MEMORY_FRACTION
291
+ SUPERTABLE_ISLAND_MEMORY_FRACTION: float = 1.0 # SUPERTABLE_ISLAND_MEMORY_FRACTION
292
+ SUPERTABLE_ISLAND_GLOBAL_MEMORY_FRACTION: float = 1.0 # SUPERTABLE_ISLAND_GLOBAL_MEMORY_FRACTION
292
293
  SUPERTABLE_ISLAND_MAX_MEMORY_BYTES: int = 0 # SUPERTABLE_ISLAND_MAX_MEMORY_BYTES
293
- SUPERTABLE_ISLAND_MAX_RESULT_BYTES: int = 512 * 1024 * 1024 # SUPERTABLE_ISLAND_MAX_RESULT_BYTES
294
+ # Zero derives the effective container memory available to this process.
295
+ SUPERTABLE_ISLAND_MAX_RESULT_BYTES: int = 0 # SUPERTABLE_ISLAND_MAX_RESULT_BYTES
294
296
  SUPERTABLE_ISLAND_CPU_MAX: int = 0 # SUPERTABLE_ISLAND_CPU_MAX
295
297
  SUPERTABLE_ISLAND_IO_WORKERS_MAX: int = 16 # SUPERTABLE_ISLAND_IO_WORKERS_MAX
296
298
  # Hard wall-clock bound for one admitted IslandDB execution. The bound is
@@ -637,7 +639,7 @@ def _build_settings() -> Settings:
637
639
  SUPERTABLE_ISLAND_CACHE_ENABLED=_env_bool("SUPERTABLE_ISLAND_CACHE_ENABLED", True),
638
640
  SUPERTABLE_ISLAND_CACHE_DIR=_env_str("SUPERTABLE_ISLAND_CACHE_DIR"),
639
641
  SUPERTABLE_ISLAND_CACHE_MAX_BYTES=_env_int(
640
- "SUPERTABLE_ISLAND_CACHE_MAX_BYTES", 20 * 1024 * 1024 * 1024,
642
+ "SUPERTABLE_ISLAND_CACHE_MAX_BYTES", 0,
641
643
  ),
642
644
  SUPERTABLE_ISLAND_CACHE_TTL_SEC=_env_int(
643
645
  "SUPERTABLE_ISLAND_CACHE_TTL_SEC", 0,
@@ -655,22 +657,22 @@ def _build_settings() -> Settings:
655
657
  "SUPERTABLE_ISLAND_RANGE_CACHE_DIR",
656
658
  ),
657
659
  SUPERTABLE_ISLAND_RANGE_CACHE_MAX_BYTES=_env_int(
658
- "SUPERTABLE_ISLAND_RANGE_CACHE_MAX_BYTES", 20 * 1024 * 1024 * 1024,
660
+ "SUPERTABLE_ISLAND_RANGE_CACHE_MAX_BYTES", 0,
659
661
  ),
660
662
  SUPERTABLE_ISLAND_RANGE_CACHE_TTL_SEC=_env_int(
661
663
  "SUPERTABLE_ISLAND_RANGE_CACHE_TTL_SEC", 0,
662
664
  ),
663
665
  SUPERTABLE_ISLAND_MEMORY_FRACTION=_env_float(
664
- "SUPERTABLE_ISLAND_MEMORY_FRACTION", 0.60,
666
+ "SUPERTABLE_ISLAND_MEMORY_FRACTION", 1.0,
665
667
  ),
666
668
  SUPERTABLE_ISLAND_GLOBAL_MEMORY_FRACTION=_env_float(
667
- "SUPERTABLE_ISLAND_GLOBAL_MEMORY_FRACTION", 0.80,
669
+ "SUPERTABLE_ISLAND_GLOBAL_MEMORY_FRACTION", 1.0,
668
670
  ),
669
671
  SUPERTABLE_ISLAND_MAX_MEMORY_BYTES=_env_int(
670
672
  "SUPERTABLE_ISLAND_MAX_MEMORY_BYTES", 0,
671
673
  ),
672
674
  SUPERTABLE_ISLAND_MAX_RESULT_BYTES=_env_int(
673
- "SUPERTABLE_ISLAND_MAX_RESULT_BYTES", 512 * 1024 * 1024,
675
+ "SUPERTABLE_ISLAND_MAX_RESULT_BYTES", 0,
674
676
  ),
675
677
  SUPERTABLE_ISLAND_CPU_MAX=_env_int(
676
678
  "SUPERTABLE_ISLAND_CPU_MAX", 0,
@@ -191,6 +191,9 @@ class _MonitoredResultStream:
191
191
  self._finalize_complete = False
192
192
  self._finalizing_thread: Optional[int] = None
193
193
  self._finalize_error: BaseException | None = None
194
+ add_terminal_callback = getattr(inner, "add_terminal_callback", None)
195
+ if callable(add_terminal_callback):
196
+ add_terminal_callback(self._inner_terminal)
194
197
 
195
198
  def __iter__(self):
196
199
  return self
@@ -251,6 +254,29 @@ class _MonitoredResultStream:
251
254
  with self._finalize_condition:
252
255
  return self._finalized
253
256
 
257
+ def _inner_terminal(self, kind: str) -> None:
258
+ if kind == "completed":
259
+ status, message = Status.OK.value, None
260
+ elif kind == "timed_out":
261
+ status, message = Status.ERROR.value, "result stream timed out"
262
+ elif kind == "cancelled":
263
+ status, message = Status.ERROR.value, "result stream cancelled"
264
+ elif kind == "failed":
265
+ status, message = Status.ERROR.value, _QUERY_STREAM_ERROR
266
+ else:
267
+ status = Status.ERROR.value
268
+ message = "result stream closed before exhaustion"
269
+ try:
270
+ self._finish(status, message)
271
+ except BaseException as exc:
272
+ # A watcher-thread callback cannot raise into the request thread.
273
+ # _finish retains the error so any later stream operation observes
274
+ # it; protected failure audit emission has already been attempted.
275
+ logger.error(
276
+ "stream terminal monitoring failed; error_type=%s",
277
+ safe_exception_type(exc),
278
+ )
279
+
254
280
  def __next__(self):
255
281
  try:
256
282
  batch = next(self._inner)
@@ -909,6 +935,9 @@ class DataReader:
909
935
  actual_engine = str(
910
936
  getattr(engine_used, "value", engine_used) or ""
911
937
  )[:64]
938
+ audit_outcome = (
939
+ Outcome.SUCCESS if outcome == "success" else Outcome.FAILURE
940
+ )
912
941
  emit(
913
942
  category=EventCategory.DATA_ACCESS,
914
943
  action=Actions.QUERY_EXECUTE,
@@ -930,7 +959,7 @@ class DataReader:
930
959
  "column_count": max(0, int(result_columns)),
931
960
  "outcome": outcome,
932
961
  },
933
- outcome=Outcome.SUCCESS,
962
+ outcome=audit_outcome,
934
963
  )
935
964
 
936
965
  def _assert_targets_exist(self, physical_tables) -> None:
@@ -2113,6 +2142,7 @@ class DataReader:
2113
2142
  })
2114
2143
  stream_timer.capture_and_reset_timing(event="EXECUTING_QUERY")
2115
2144
  stream_timer.capture_duration(event="TOTAL_EXECUTE")
2145
+ monitoring_error: Optional[MonitoringPostExecutionError] = None
2116
2146
  try:
2117
2147
  extend_execution_plan(
2118
2148
  query_plan_manager=stream_qpm,
@@ -2130,22 +2160,46 @@ class DataReader:
2130
2160
  safe_cause = MonitoringDurabilityError(
2131
2161
  "monitoring durability failure"
2132
2162
  )
2133
- raise MonitoringPostExecutionError(
2163
+ monitoring_error = MonitoringPostExecutionError(
2134
2164
  organization=stream_qpm.organization,
2135
2165
  super_name=stream_qpm.super_name,
2136
2166
  query_id=str(getattr(stream_qpm, "query_id", "")),
2137
2167
  status=final_status,
2138
2168
  cause=safe_cause,
2139
- ) from None
2169
+ )
2140
2170
  finally:
2141
2171
  stream_timer.capture_and_reset_timing(event="EXTENDING_PLAN")
2142
- if final_status == Status.OK.value:
2143
- self._emit_successful_query_audit(
2144
- role_name=role_name,
2145
- engine_used=_engine_used,
2146
- result_rows=result_rows,
2147
- result_columns=result_columns,
2172
+ audit_engine = _actual_engine_from_plan_stats(
2173
+ stream_plan_stats,
2174
+ fallback=_engine_used,
2175
+ )
2176
+ try:
2177
+ if final_status == Status.OK.value:
2178
+ self._emit_successful_query_audit(
2179
+ role_name=role_name,
2180
+ engine_used=audit_engine,
2181
+ result_rows=result_rows,
2182
+ result_columns=result_columns,
2183
+ )
2184
+ else:
2185
+ self._emit_successful_query_audit(
2186
+ role_name=role_name,
2187
+ engine_used=audit_engine,
2188
+ result_rows=result_rows,
2189
+ result_columns=result_columns,
2190
+ outcome="failure",
2191
+ )
2192
+ except Exception:
2193
+ if final_status == Status.OK.value and monitoring_error is None:
2194
+ raise
2195
+ # Preserve the producer/cancellation or monitoring error
2196
+ # just as the materialized failure path does.
2197
+ logger.debug(
2198
+ self._lp("failed to emit query failure audit"),
2199
+ exc_info=True,
2148
2200
  )
2201
+ if monitoring_error is not None:
2202
+ raise monitoring_error
2149
2203
 
2150
2204
  return (
2151
2205
  _MonitoredResultStream(result_value, _finalize_stream_monitoring),
@@ -2771,6 +2825,32 @@ def _engine_name(value: Any) -> str:
2771
2825
  return str(raw or "").strip().casefold()
2772
2826
 
2773
2827
 
2828
+ def _actual_engine_from_plan_stats(plan_stats: Any, *, fallback: Any) -> str:
2829
+ """Resolve a stream's engine only after deferred delivery has finished."""
2830
+ try:
2831
+ stats = tuple(
2832
+ entry for entry in (getattr(plan_stats, "stats", ()) or ())
2833
+ if isinstance(entry, dict)
2834
+ )
2835
+ outcome = _latest_plan_stat(stats, "AUTO_ROUTING_OUTCOME")
2836
+ if isinstance(outcome, dict):
2837
+ actual = _engine_name(outcome.get("actual_engine"))
2838
+ if actual:
2839
+ return actual
2840
+ recorded = _engine_name(_latest_plan_stat(stats, "ENGINE"))
2841
+ if recorded:
2842
+ return recorded
2843
+ for entry in reversed(stats):
2844
+ attempt = entry.get("ENGINE_ATTEMPT")
2845
+ if isinstance(attempt, dict):
2846
+ attempted = _engine_name(attempt.get("engine"))
2847
+ if attempted:
2848
+ return attempted
2849
+ except Exception:
2850
+ pass
2851
+ return _engine_name(fallback)
2852
+
2853
+
2774
2854
  def _update_query_out(
2775
2855
  reader: DataReader,
2776
2856
  out: Optional[Dict[str, Any]],
@@ -2837,14 +2917,10 @@ def _update_query_out(
2837
2917
  if not selected and isinstance(outcome, dict):
2838
2918
  selected = _engine_name(outcome.get("selected_engine"))
2839
2919
 
2840
- actual = ""
2920
+ actual = _actual_engine_from_plan_stats(plan_stats, fallback="")
2841
2921
  fallback = False
2842
2922
  if isinstance(outcome, dict):
2843
- actual = _engine_name(outcome.get("actual_engine"))
2844
2923
  fallback = bool(outcome.get("fallback", False))
2845
- actual = actual or _engine_name(recorded_engine)
2846
- if not actual and engine_attempts:
2847
- actual = _engine_name(engine_attempts[-1].get("engine"))
2848
2924
  selected = selected or actual or requested
2849
2925
  if actual and selected and not isinstance(outcome, dict):
2850
2926
  fallback = actual != selected
@@ -138,6 +138,7 @@ class RoutingFeatures:
138
138
  freshness_age_seconds: int = -1
139
139
  has_active_tombstone: bool = False
140
140
  streaming_result: bool = False
141
+ island_plan_evaluated: bool = False
141
142
  island_advice: str = ""
142
143
  island_cpu_workers: int = 0
143
144
  island_io_workers: int = 0
@@ -193,6 +194,7 @@ class RoutingFeatures:
193
194
  "stream" if self.streaming_result else "materialized",
194
195
  "tombstone" if self.has_active_tombstone else "clean",
195
196
  "fresh" if self.data_is_fresh else "stable",
197
+ "island_plan" if self.island_plan_evaluated else "no_island_plan",
196
198
  str(self.island_advice or "none"),
197
199
  str(self._magnitude_bucket(self.island_cpu_workers)),
198
200
  str(self._magnitude_bucket(self.island_io_workers)),
@@ -502,10 +504,23 @@ class AdaptiveEngineRouter:
502
504
  )
503
505
  if not features.source_bytes_complete:
504
506
  rejections[Engine.ISLANDDB].append("source byte estimate incomplete")
505
- if features.island_advice not in {"island_in_memory", "island_spill"}:
507
+ if (
508
+ features.island_plan_evaluated
509
+ and features.island_advice not in {
510
+ "island_in_memory", "island_spill",
511
+ }
512
+ ):
506
513
  rejections[Engine.ISLANDDB].append(
507
514
  "IslandDB resource plan is not executable"
508
515
  )
516
+ elif (
517
+ availability.island_enabled
518
+ and availability.island_supported
519
+ and not features.island_plan_evaluated
520
+ ):
521
+ rejections[Engine.ISLANDDB].append(
522
+ "IslandDB resource plan is unavailable"
523
+ )
509
524
 
510
525
  if not availability.spark_available:
511
526
  rejections[Engine.SPARK_SQL].append("no safe fitting Spark cluster")
@@ -1684,11 +1684,13 @@ class DataEstimator:
1684
1684
  return 0, False
1685
1685
  widths: List[int] = []
1686
1686
  for name in requested:
1687
- width = self._decoded_fixed_width(normalized_types.get(name))
1687
+ normalized_type = str(
1688
+ normalized_types.get(name) or ""
1689
+ ).strip().casefold()
1690
+ width = self._decoded_fixed_width(normalized_type)
1688
1691
  if (
1689
1692
  width is None
1690
- and str(normalized_types.get(name) or "").strip().casefold()
1691
- == "binary"
1693
+ and normalized_type in {"binary", "string", "utf8"}
1692
1694
  ):
1693
1695
  bound = (max_value_bytes or {}).get(name)
1694
1696
  if (
@@ -1696,9 +1698,12 @@ class DataEstimator:
1696
1698
  and not isinstance(bound, bool)
1697
1699
  and bound >= 0
1698
1700
  ):
1699
- # Variable Binary owns a 32-bit offset per value in
1700
- # addition to its exactly sealed maximum payload width.
1701
- width = int(bound) + 4
1701
+ # Binary owns a 32-bit offset; Polars' native UTF-8 value
1702
+ # representation can retain a 16-byte string view. The
1703
+ # caller adds validity/alignment slack below.
1704
+ width = int(bound) + (
1705
+ 4 if normalized_type == "binary" else 16
1706
+ )
1702
1707
  if width is None:
1703
1708
  return 0, False
1704
1709
  widths.append(width)
@@ -2511,7 +2516,9 @@ class DataEstimator:
2511
2516
  for file_key in survivors
2512
2517
  )
2513
2518
  for column_name, type_name in schema_types.items()
2514
- if str(type_name).strip().casefold() == "binary"
2519
+ if str(type_name).strip().casefold() in {
2520
+ "binary", "string", "utf8",
2521
+ }
2515
2522
  and survivors
2516
2523
  and all(
2517
2524
  isinstance(resource_value_bounds.get(file_key), dict)
@@ -0,0 +1,168 @@
1
+ """Cross-process disk admission for caches and IslandDB spill files."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import fcntl
6
+ import json
7
+ import os
8
+ import shutil
9
+ import time
10
+ import uuid
11
+ from pathlib import Path
12
+
13
+ from supertable.config.homedir import get_app_home
14
+
15
+
16
+ _STALE_SECONDS = 24 * 60 * 60
17
+
18
+
19
+ class DiskAdmissionUnavailable(RuntimeError):
20
+ """The shared filesystem reservation cannot be admitted."""
21
+
22
+
23
+ def _existing_ancestor(path: str | os.PathLike[str]) -> Path:
24
+ candidate = Path(path).expanduser().resolve()
25
+ while not candidate.exists() and candidate != candidate.parent:
26
+ candidate = candidate.parent
27
+ return candidate
28
+
29
+
30
+ def _reservation_root(path: str | os.PathLike[str]) -> Path:
31
+ probe = _existing_ancestor(path)
32
+ try:
33
+ device = os.stat(probe).st_dev
34
+ except OSError as exc:
35
+ raise DiskAdmissionUnavailable("cannot identify cache filesystem") from exc
36
+ base = Path(get_app_home()) / ".supertable-disk-admission"
37
+ try:
38
+ if os.stat(_existing_ancestor(base)).st_dev != device:
39
+ base = probe / ".supertable-disk-admission"
40
+ except OSError:
41
+ base = probe / ".supertable-disk-admission"
42
+ return base / str(device)
43
+
44
+
45
+ def _read_size(path: Path) -> int:
46
+ try:
47
+ with path.open("r", encoding="utf-8") as handle:
48
+ value = json.load(handle).get("size", 0)
49
+ return max(0, int(value))
50
+ except (OSError, ValueError, TypeError, json.JSONDecodeError):
51
+ return 0
52
+
53
+
54
+ def _clean_stale(root: Path) -> None:
55
+ now = time.time()
56
+ try:
57
+ entries = tuple(root.iterdir())
58
+ except OSError:
59
+ return
60
+ for entry in entries:
61
+ if not entry.name.endswith(".json"):
62
+ continue
63
+ try:
64
+ if now - entry.stat().st_mtime > _STALE_SECONDS:
65
+ entry.unlink(missing_ok=True)
66
+ continue
67
+ with entry.open("r", encoding="utf-8") as handle:
68
+ payload = json.load(handle)
69
+ pid = payload.get("pid")
70
+ if isinstance(pid, int) and pid > 0 and pid != os.getpid():
71
+ try:
72
+ os.kill(pid, 0)
73
+ except ProcessLookupError:
74
+ entry.unlink(missing_ok=True)
75
+ except PermissionError:
76
+ pass
77
+ except (OSError, ValueError, TypeError, json.JSONDecodeError):
78
+ try:
79
+ entry.unlink(missing_ok=True)
80
+ except OSError:
81
+ pass
82
+
83
+
84
+ class DiskReservation:
85
+ """An idempotent shared filesystem reservation."""
86
+
87
+ def __init__(self, path: Path):
88
+ self.path = path
89
+ self._released = False
90
+
91
+ def release(self) -> None:
92
+ if self._released:
93
+ return
94
+ self._released = True
95
+ try:
96
+ self.path.unlink(missing_ok=True)
97
+ except OSError:
98
+ pass
99
+
100
+ def __enter__(self) -> "DiskReservation":
101
+ return self
102
+
103
+ def __exit__(self, exc_type, exc, tb) -> None:
104
+ self.release()
105
+
106
+
107
+ def reserve_disk(
108
+ path: str | os.PathLike[str],
109
+ size: int,
110
+ *,
111
+ min_free_bytes: int = 0,
112
+ ) -> DiskReservation | None:
113
+ """Atomically reserve bytes shared by cache and spill users."""
114
+ size = max(0, int(size))
115
+ if size == 0:
116
+ return None
117
+ root = _reservation_root(path)
118
+ root.mkdir(parents=True, exist_ok=True)
119
+ lock_path = root / ".lock"
120
+ with lock_path.open("a+") as lock:
121
+ fcntl.flock(lock.fileno(), fcntl.LOCK_EX)
122
+ _clean_stale(root)
123
+ try:
124
+ free = max(0, int(shutil.disk_usage(_existing_ancestor(path)).free))
125
+ except (OSError, ValueError, TypeError):
126
+ raise DiskAdmissionUnavailable("cannot inspect free disk space") from None
127
+ reserved = sum(
128
+ _read_size(entry)
129
+ for entry in root.iterdir()
130
+ if entry.name.endswith(".json")
131
+ )
132
+ if free - reserved - size < max(0, int(min_free_bytes)):
133
+ raise DiskAdmissionUnavailable(
134
+ "shared filesystem reservation would exhaust the disk reserve"
135
+ )
136
+ token = root / f"{os.getpid()}-{uuid.uuid4().hex}.json"
137
+ temporary = token.with_suffix(".tmp")
138
+ payload = {
139
+ "size": size,
140
+ "pid": os.getpid(),
141
+ "created_ns": time.time_ns(),
142
+ }
143
+ with temporary.open("x", encoding="utf-8") as handle:
144
+ json.dump(payload, handle, separators=(",", ":"))
145
+ handle.flush()
146
+ os.fsync(handle.fileno())
147
+ os.replace(temporary, token)
148
+ fcntl.flock(lock.fileno(), fcntl.LOCK_UN)
149
+ return DiskReservation(token)
150
+
151
+
152
+ def release_local_reservation(local_path: str | os.PathLike[str]) -> None:
153
+ """Release a shared token embedded in a cache reservation record."""
154
+ try:
155
+ with Path(local_path).open("r", encoding="utf-8") as handle:
156
+ token = json.load(handle).get("shared_token")
157
+ if token:
158
+ DiskReservation(Path(str(token))).release()
159
+ except (OSError, ValueError, TypeError, json.JSONDecodeError):
160
+ pass
161
+
162
+
163
+ __all__ = [
164
+ "DiskAdmissionUnavailable",
165
+ "DiskReservation",
166
+ "release_local_reservation",
167
+ "reserve_disk",
168
+ ]