deltacat 1.1.3__tar.gz → 1.1.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (219) hide show
  1. {deltacat-1.1.3 → deltacat-1.1.4}/PKG-INFO +1 -1
  2. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/__init__.py +1 -1
  3. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/aws/constants.py +0 -4
  4. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/aws/s3u.py +26 -5
  5. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/utils/round_completion_file.py +3 -0
  6. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor_v2/compaction_session.py +2 -0
  7. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor_v2/constants.py +0 -3
  8. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor_v2/utils/merge.py +0 -3
  9. deltacat-1.1.4/deltacat/logs.py +255 -0
  10. deltacat-1.1.4/deltacat/tests/aws/test_s3u.py +12 -0
  11. deltacat-1.1.4/deltacat/tests/test_logs.py +127 -0
  12. deltacat-1.1.4/deltacat/tests/utils/test_placement.py +25 -0
  13. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/utils/placement.py +14 -7
  14. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat.egg-info/PKG-INFO +1 -1
  15. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat.egg-info/SOURCES.txt +3 -0
  16. deltacat-1.1.3/deltacat/logs.py +0 -198
  17. {deltacat-1.1.3 → deltacat-1.1.4}/MANIFEST.in +0 -0
  18. {deltacat-1.1.3 → deltacat-1.1.4}/README.md +0 -0
  19. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/aws/__init__.py +0 -0
  20. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/aws/clients.py +0 -0
  21. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/aws/redshift/__init__.py +0 -0
  22. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/aws/redshift/model/__init__.py +0 -0
  23. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/aws/redshift/model/manifest.py +0 -0
  24. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/benchmarking/__init__.py +0 -0
  25. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/benchmarking/benchmark_parquet_reads.py +0 -0
  26. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/benchmarking/conftest.py +0 -0
  27. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/catalog/__init__.py +0 -0
  28. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/catalog/default_catalog_impl/__init__.py +0 -0
  29. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/catalog/delegate.py +0 -0
  30. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/catalog/interface.py +0 -0
  31. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/catalog/model/__init__.py +0 -0
  32. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/catalog/model/catalog.py +0 -0
  33. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/catalog/model/table_definition.py +0 -0
  34. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/__init__.py +0 -0
  35. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/__init__.py +0 -0
  36. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/compaction_session.py +0 -0
  37. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/model/__init__.py +0 -0
  38. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/model/compact_partition_params.py +0 -0
  39. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/model/compaction_session_audit_info.py +0 -0
  40. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/model/compactor_version.py +0 -0
  41. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/model/dedupe_result.py +0 -0
  42. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/model/delta_annotated.py +0 -0
  43. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/model/delta_file_envelope.py +0 -0
  44. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/model/delta_file_locator.py +0 -0
  45. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/model/hash_bucket_result.py +0 -0
  46. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/model/materialize_result.py +0 -0
  47. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/model/primary_key_index.py +0 -0
  48. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/model/pyarrow_write_result.py +0 -0
  49. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/model/repartition_result.py +0 -0
  50. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/model/round_completion_info.py +0 -0
  51. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/model/table_object_store.py +0 -0
  52. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/repartition_session.py +0 -0
  53. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/steps/__init__.py +0 -0
  54. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/steps/dedupe.py +0 -0
  55. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/steps/hash_bucket.py +0 -0
  56. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/steps/materialize.py +0 -0
  57. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/steps/repartition.py +0 -0
  58. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/utils/__init__.py +0 -0
  59. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/utils/io.py +0 -0
  60. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/utils/primary_key_index.py +0 -0
  61. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/utils/sort_key.py +0 -0
  62. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/utils/system_columns.py +0 -0
  63. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor_v2/__init__.py +0 -0
  64. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor_v2/deletes/__init__.py +0 -0
  65. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor_v2/deletes/delete_file_envelope.py +0 -0
  66. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor_v2/deletes/delete_strategy.py +0 -0
  67. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor_v2/deletes/delete_strategy_equality_delete.py +0 -0
  68. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor_v2/deletes/model.py +0 -0
  69. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor_v2/deletes/utils.py +0 -0
  70. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor_v2/model/__init__.py +0 -0
  71. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor_v2/model/hash_bucket_input.py +0 -0
  72. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor_v2/model/hash_bucket_result.py +0 -0
  73. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor_v2/model/merge_file_group.py +0 -0
  74. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor_v2/model/merge_input.py +0 -0
  75. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor_v2/model/merge_result.py +0 -0
  76. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor_v2/steps/__init__.py +0 -0
  77. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor_v2/steps/hash_bucket.py +0 -0
  78. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor_v2/steps/merge.py +0 -0
  79. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor_v2/utils/__init__.py +0 -0
  80. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor_v2/utils/content_type_params.py +0 -0
  81. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor_v2/utils/dedupe.py +0 -0
  82. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor_v2/utils/delta.py +0 -0
  83. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor_v2/utils/io.py +0 -0
  84. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor_v2/utils/primary_key_index.py +0 -0
  85. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor_v2/utils/task_options.py +0 -0
  86. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/merge_on_read/__init__.py +0 -0
  87. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/merge_on_read/daft.py +0 -0
  88. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/merge_on_read/model/__init__.py +0 -0
  89. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/merge_on_read/model/merge_on_read_params.py +0 -0
  90. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/merge_on_read/utils/__init__.py +0 -0
  91. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/merge_on_read/utils/delta.py +0 -0
  92. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/metastats/__init__.py +0 -0
  93. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/metastats/config/__init__.py +0 -0
  94. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/metastats/meta_stats.py +0 -0
  95. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/metastats/model/__init__.py +0 -0
  96. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/metastats/model/partition_stats_dict.py +0 -0
  97. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/metastats/model/stats_cluster_size_estimator.py +0 -0
  98. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/metastats/stats.py +0 -0
  99. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/metastats/utils/__init__.py +0 -0
  100. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/metastats/utils/constants.py +0 -0
  101. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/metastats/utils/io.py +0 -0
  102. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/metastats/utils/pyarrow_memory_estimation_function.py +0 -0
  103. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/metastats/utils/ray_utils.py +0 -0
  104. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/stats/__init__.py +0 -0
  105. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/stats/basic.py +0 -0
  106. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/stats/models/__init__.py +0 -0
  107. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/stats/models/delta_column_stats.py +0 -0
  108. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/stats/models/delta_stats.py +0 -0
  109. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/stats/models/delta_stats_cache_result.py +0 -0
  110. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/stats/models/manifest_entry_stats.py +0 -0
  111. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/stats/models/stats_result.py +0 -0
  112. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/stats/types.py +0 -0
  113. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/stats/utils/__init__.py +0 -0
  114. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/stats/utils/intervals.py +0 -0
  115. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/stats/utils/io.py +0 -0
  116. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/stats/utils/manifest_stats_file.py +0 -0
  117. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/constants.py +0 -0
  118. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/exceptions.py +0 -0
  119. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/io/__init__.py +0 -0
  120. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/io/aws/__init__.py +0 -0
  121. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/io/aws/redshift/__init__.py +0 -0
  122. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/io/aws/redshift/redshift_datasource.py +0 -0
  123. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/io/dataset.py +0 -0
  124. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/io/file_object_store.py +0 -0
  125. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/io/memcached_object_store.py +0 -0
  126. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/io/object_store.py +0 -0
  127. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/io/ray_plasma_object_store.py +0 -0
  128. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/io/read_api.py +0 -0
  129. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/io/redis_object_store.py +0 -0
  130. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/io/s3_object_store.py +0 -0
  131. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/storage/__init__.py +0 -0
  132. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/storage/interface.py +0 -0
  133. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/storage/model/__init__.py +0 -0
  134. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/storage/model/delete_parameters.py +0 -0
  135. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/storage/model/delta.py +0 -0
  136. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/storage/model/list_result.py +0 -0
  137. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/storage/model/locator.py +0 -0
  138. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/storage/model/namespace.py +0 -0
  139. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/storage/model/partition.py +0 -0
  140. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/storage/model/sort_key.py +0 -0
  141. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/storage/model/stream.py +0 -0
  142. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/storage/model/table.py +0 -0
  143. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/storage/model/table_version.py +0 -0
  144. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/storage/model/types.py +0 -0
  145. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/__init__.py +0 -0
  146. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/aws/__init__.py +0 -0
  147. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/aws/test_clients.py +0 -0
  148. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/catalog/__init__.py +0 -0
  149. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/catalog/test_default_catalog_impl.py +0 -0
  150. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/compute/__init__.py +0 -0
  151. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/compute/compact_partition_rebase_then_incremental_test_cases.py +0 -0
  152. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/compute/compact_partition_test_cases.py +0 -0
  153. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/compute/compactor/__init__.py +0 -0
  154. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/compute/compactor/steps/__init__.py +0 -0
  155. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/compute/compactor/steps/test_repartition.py +0 -0
  156. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/compute/compactor/utils/__init__.py +0 -0
  157. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/compute/compactor/utils/test_io.py +0 -0
  158. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/compute/compactor_v2/__init__.py +0 -0
  159. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/compute/compactor_v2/test_compaction_session.py +0 -0
  160. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/compute/compactor_v2/test_hashlib.py +0 -0
  161. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/compute/compactor_v2/utils/__init__.py +0 -0
  162. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/compute/compactor_v2/utils/test_task_options.py +0 -0
  163. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/compute/test_compact_partition_incremental.py +0 -0
  164. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/compute/test_compact_partition_params.py +0 -0
  165. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/compute/test_compact_partition_rebase_then_incremental.py +0 -0
  166. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/compute/test_util_common.py +0 -0
  167. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/compute/test_util_constant.py +0 -0
  168. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/compute/test_util_create_table_deltas_repo.py +0 -0
  169. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/io/__init__.py +0 -0
  170. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/io/test_cloudpickle_bug_fix.py +0 -0
  171. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/io/test_file_object_store.py +0 -0
  172. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/io/test_memcached_object_store.py +0 -0
  173. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/io/test_ray_plasma_object_store.py +0 -0
  174. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/io/test_redis_object_store.py +0 -0
  175. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/io/test_s3_object_store.py +0 -0
  176. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/local_deltacat_storage/__init__.py +0 -0
  177. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/stats/__init__.py +0 -0
  178. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/stats/test_intervals.py +0 -0
  179. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/test_utils/__init__.py +0 -0
  180. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/test_utils/constants.py +0 -0
  181. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/test_utils/pyarrow.py +0 -0
  182. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/test_utils/storage.py +0 -0
  183. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/test_utils/utils.py +0 -0
  184. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/utils/__init__.py +0 -0
  185. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/utils/data/__init__.py +0 -0
  186. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/utils/test_cloudpickle.py +0 -0
  187. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/utils/test_daft.py +0 -0
  188. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/utils/test_metrics.py +0 -0
  189. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/utils/test_pyarrow.py +0 -0
  190. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/utils/test_record_batch_tables.py +0 -0
  191. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/utils/test_resources.py +0 -0
  192. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/types/__init__.py +0 -0
  193. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/types/media.py +0 -0
  194. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/types/partial_download.py +0 -0
  195. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/types/tables.py +0 -0
  196. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/utils/__init__.py +0 -0
  197. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/utils/arguments.py +0 -0
  198. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/utils/cloudpickle.py +0 -0
  199. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/utils/common.py +0 -0
  200. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/utils/daft.py +0 -0
  201. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/utils/metrics.py +0 -0
  202. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/utils/numpy.py +0 -0
  203. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/utils/pandas.py +0 -0
  204. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/utils/performance.py +0 -0
  205. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/utils/pyarrow.py +0 -0
  206. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/utils/ray_utils/__init__.py +0 -0
  207. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/utils/ray_utils/collections.py +0 -0
  208. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/utils/ray_utils/concurrency.py +0 -0
  209. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/utils/ray_utils/dataset.py +0 -0
  210. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/utils/ray_utils/performance.py +0 -0
  211. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/utils/ray_utils/runtime.py +0 -0
  212. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/utils/resources.py +0 -0
  213. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/utils/s3fs.py +0 -0
  214. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/utils/schema.py +0 -0
  215. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat.egg-info/dependency_links.txt +0 -0
  216. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat.egg-info/requires.txt +0 -0
  217. {deltacat-1.1.3 → deltacat-1.1.4}/deltacat.egg-info/top_level.txt +0 -0
  218. {deltacat-1.1.3 → deltacat-1.1.4}/setup.cfg +0 -0
  219. {deltacat-1.1.3 → deltacat-1.1.4}/setup.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: deltacat
3
- Version: 1.1.3
3
+ Version: 1.1.4
4
4
  Summary: A scalable, fast, ACID-compliant Data Catalog powered by Ray.
5
5
  Home-page: https://github.com/ray-project/deltacat
6
6
  Author: Ray Team
@@ -44,7 +44,7 @@ from deltacat.types.tables import TableWriteMode
44
44
 
45
45
  deltacat.logs.configure_deltacat_logger(logging.getLogger(__name__))
46
46
 
47
- __version__ = "1.1.3"
47
+ __version__ = "1.1.4"
48
48
 
49
49
 
50
50
  __all__ = [
@@ -6,7 +6,3 @@ DAFT_MAX_S3_CONNECTIONS_PER_FILE = env_integer("DAFT_MAX_S3_CONNECTIONS_PER_FILE
6
6
  BOTO_MAX_RETRIES = env_integer("BOTO_MAX_RETRIES", 5)
7
7
  TIMEOUT_ERROR_CODES: List[str] = ["ReadTimeoutError", "ConnectTimeoutError"]
8
8
  AWS_REGION = env_string("AWS_REGION", "us-east-1")
9
-
10
- # Metric Names
11
- DOWNLOAD_MANIFEST_ENTRY_METRIC_PREFIX = "download_manifest_entry"
12
- UPLOAD_SLICED_TABLE_METRIC_PREFIX = "upload_sliced_table"
@@ -27,8 +27,6 @@ import deltacat.aws.clients as aws_utils
27
27
  from deltacat import logs
28
28
  from deltacat.aws.constants import (
29
29
  TIMEOUT_ERROR_CODES,
30
- DOWNLOAD_MANIFEST_ENTRY_METRIC_PREFIX,
31
- UPLOAD_SLICED_TABLE_METRIC_PREFIX,
32
30
  )
33
31
  from deltacat.exceptions import NonRetryableError, RetryableError
34
32
  from deltacat.storage import (
@@ -54,7 +52,6 @@ from deltacat.types.tables import (
54
52
  )
55
53
  from deltacat.types.partial_download import PartialFileDownloadParams
56
54
  from deltacat.utils.common import ReadKwargsProvider
57
- from deltacat.utils.metrics import metrics
58
55
 
59
56
  logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
60
57
 
@@ -121,6 +118,32 @@ class UuidBlockWritePathProvider(BlockWritePathProvider):
121
118
  self.block_refs.append(block)
122
119
  return write_path
123
120
 
121
+ def __call__(
122
+ self,
123
+ base_path: str,
124
+ *,
125
+ filesystem: Optional[pa.filesystem.FileSystem] = None,
126
+ dataset_uuid: Optional[str] = None,
127
+ block: Optional[ObjectRef[Block]] = None,
128
+ block_index: Optional[int] = None,
129
+ file_format: Optional[str] = None,
130
+ ) -> str:
131
+ """
132
+ TODO: BlockWritePathProvider is deprecated as of Ray version 2.20.0. Please use FilenameProvider.
133
+ See: https://docs.ray.io/en/master/data/api/doc/ray.data.datasource.FilenameProvider.html
134
+ Also See: https://github.com/ray-project/deltacat/issues/299
135
+
136
+ Hence, this class only works with Ray version 2.20.0 or lower when used in Ray Dataset.
137
+ """
138
+ return self._get_write_path_for_block(
139
+ base_path,
140
+ filesystem=filesystem,
141
+ dataset_uuid=dataset_uuid,
142
+ block=block,
143
+ block_index=block_index,
144
+ file_format=file_format,
145
+ )
146
+
124
147
 
125
148
  class S3Url:
126
149
  def __init__(self, url: str):
@@ -243,7 +266,6 @@ def read_file(
243
266
  raise e
244
267
 
245
268
 
246
- @metrics(prefix=UPLOAD_SLICED_TABLE_METRIC_PREFIX)
247
269
  def upload_sliced_table(
248
270
  table: Union[LocalTable, DistributedDataset],
249
271
  s3_url_prefix: str,
@@ -352,7 +374,6 @@ def upload_table(
352
374
  return manifest_entries
353
375
 
354
376
 
355
- @metrics(prefix=DOWNLOAD_MANIFEST_ENTRY_METRIC_PREFIX)
356
377
  def download_manifest_entry(
357
378
  manifest_entry: ManifestEntry,
358
379
  token_holder: Optional[Dict[str, Any]] = None,
@@ -6,6 +6,7 @@ from deltacat.compute.compactor import RoundCompletionInfo
6
6
  from deltacat.storage import PartitionLocator
7
7
  from deltacat.aws import s3u as s3_utils
8
8
  from typing import Optional
9
+ from deltacat.utils.metrics import metrics
9
10
 
10
11
  logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
11
12
 
@@ -18,6 +19,7 @@ def get_round_completion_file_s3_url(
18
19
  return f"{base_url}.json"
19
20
 
20
21
 
22
+ @metrics
21
23
  def read_round_completion_file(
22
24
  bucket: str,
23
25
  source_partition_locator: PartitionLocator,
@@ -38,6 +40,7 @@ def read_round_completion_file(
38
40
  return round_completion_info
39
41
 
40
42
 
43
+ @metrics
41
44
  def write_round_completion_file(
42
45
  bucket: Optional[str],
43
46
  source_partition_locator: Optional[PartitionLocator],
@@ -50,6 +50,7 @@ from deltacat.compute.compactor_v2.steps import merge as mg
50
50
  from deltacat.compute.compactor_v2.steps import hash_bucket as hb
51
51
  from deltacat.compute.compactor_v2.utils import io
52
52
  from deltacat.compute.compactor.utils import round_completion_file as rcf
53
+ from deltacat.utils.metrics import metrics
53
54
 
54
55
  from typing import List, Optional, Tuple
55
56
  from collections import defaultdict
@@ -73,6 +74,7 @@ if importlib.util.find_spec("memray"):
73
74
  logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
74
75
 
75
76
 
77
+ @metrics
76
78
  def compact_partition(params: CompactPartitionParams, **kwargs) -> Optional[str]:
77
79
 
78
80
  assert (
@@ -65,6 +65,3 @@ DISCOVER_DELTAS_METRIC_PREFIX = "discover_deltas"
65
65
 
66
66
  # Metric prefix for prepare deletes
67
67
  PREPARE_DELETES_METRIC_PREFIX = "prepare_deletes"
68
-
69
- # Metric prefix for materialize
70
- MATERIALIZE_METRIC_PREFIX = "delta_materialize"
@@ -31,14 +31,11 @@ from deltacat.compute.compactor_v2.deletes.delete_strategy import (
31
31
  from deltacat.compute.compactor_v2.deletes.delete_file_envelope import (
32
32
  DeleteFileEnvelope,
33
33
  )
34
- from deltacat.utils.metrics import metrics
35
- from deltacat.compute.compactor_v2.constants import MATERIALIZE_METRIC_PREFIX
36
34
 
37
35
 
38
36
  logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
39
37
 
40
38
 
41
- @metrics(prefix=MATERIALIZE_METRIC_PREFIX)
42
39
  def materialize(
43
40
  input: MergeInput,
44
41
  task_index: int,
@@ -0,0 +1,255 @@
1
+ import logging
2
+ import os
3
+ import json
4
+ import pathlib
5
+ from logging import FileHandler, Handler, Logger, LoggerAdapter, handlers
6
+ from typing import Any, Dict, Optional, Union
7
+
8
+ import ray
9
+ from ray.runtime_context import RuntimeContext
10
+
11
+ from deltacat.constants import (
12
+ DELTACAT_APP_LOG_LEVEL,
13
+ DELTACAT_SYS_LOG_LEVEL,
14
+ DELTACAT_APP_LOG_DIR,
15
+ DELTACAT_SYS_LOG_DIR,
16
+ DELTACAT_APP_INFO_LOG_BASE_FILE_NAME,
17
+ DELTACAT_SYS_INFO_LOG_BASE_FILE_NAME,
18
+ DELTACAT_APP_DEBUG_LOG_BASE_FILE_NAME,
19
+ DELTACAT_SYS_DEBUG_LOG_BASE_FILE_NAME,
20
+ )
21
+
22
+ DEFAULT_LOG_LEVEL = "INFO"
23
+ DEFAULT_LOG_FORMAT = {
24
+ "level": "levelname",
25
+ "message": "message",
26
+ "loggerName": "name",
27
+ "processName": "processName",
28
+ "processID": "process",
29
+ "threadName": "threadName",
30
+ "timestamp": "asctime",
31
+ "filename": "filename",
32
+ "lineno": "lineno",
33
+ }
34
+ DEFAULT_MAX_BYTES_PER_LOG = 2 ^ 20 * 256 # 256 MiB
35
+ DEFAULT_BACKUP_COUNT = 0
36
+
37
+
38
+ class JsonFormatter(logging.Formatter):
39
+ """
40
+ Formatter that outputs JSON strings after parsing the LogRecord.
41
+
42
+ @param dict fmt_dict: Key: logging format attribute pairs. Defaults to {"message": "message"}.
43
+ @param str time_format: time.strftime() format string. Default: "%Y-%m-%dT%H:%M:%S"
44
+ @param str msec_format: Microsecond formatting. Appended at the end. Default: "%s.%03dZ"
45
+ """
46
+
47
+ def __init__(
48
+ self,
49
+ fmt_dict: dict = None,
50
+ time_format: str = "%Y-%m-%dT%H:%M:%S",
51
+ msec_format: str = "%s.%03dZ",
52
+ ):
53
+ self.fmt_dict = fmt_dict if fmt_dict is not None else {"message": "message"}
54
+ self.default_time_format = time_format
55
+ self.default_msec_format = msec_format
56
+ self.datefmt = None
57
+ if ray.is_initialized():
58
+ self.ray_runtime_ctx: RuntimeContext = ray.get_runtime_context()
59
+ self.context = {}
60
+ self.context["worker_id"] = self.ray_runtime_ctx.get_worker_id()
61
+ self.context["node_id"] = self.ray_runtime_ctx.get_node_id()
62
+ self.context["job_id"] = self.ray_runtime_ctx.get_job_id()
63
+ else:
64
+ self.ray_runtime_ctx = None
65
+ self.context = {}
66
+
67
+ def usesTime(self) -> bool:
68
+ """
69
+ Overwritten to look for the attribute in the format dict values instead of the fmt string.
70
+ """
71
+ return "asctime" in self.fmt_dict.values()
72
+
73
+ def formatMessage(self, record) -> dict:
74
+ """
75
+ Overwritten to return a dictionary of the relevant LogRecord attributes instead of a string.
76
+ KeyError is raised if an unknown attribute is provided in the fmt_dict.
77
+ """
78
+ return {
79
+ fmt_key: record.__dict__[fmt_val]
80
+ for fmt_key, fmt_val in self.fmt_dict.items()
81
+ }
82
+
83
+ def format(self, record) -> str:
84
+ """
85
+ Mostly the same as the parent's class method, the difference being that a dict is manipulated and dumped as JSON
86
+ instead of a string.
87
+ """
88
+ record.message = record.getMessage()
89
+
90
+ if self.usesTime():
91
+ record.asctime = self.formatTime(record, self.datefmt)
92
+
93
+ message_dict = self.formatMessage(record)
94
+
95
+ if record.exc_info:
96
+ # Cache the traceback text to avoid converting it multiple times
97
+ # (it's constant anyway)
98
+ if not record.exc_text:
99
+ record.exc_text = self.formatException(record.exc_info)
100
+
101
+ if record.exc_text:
102
+ message_dict["exc_info"] = record.exc_text
103
+
104
+ if record.stack_info:
105
+ message_dict["stack_info"] = self.formatStack(record.stack_info)
106
+
107
+ if self.ray_runtime_ctx:
108
+ # only workers will have task ID
109
+ if (
110
+ self.ray_runtime_ctx.worker
111
+ and self.ray_runtime_ctx.worker.mode == ray._private.worker.WORKER_MODE
112
+ ):
113
+ self.context["task_id"] = self.ray_runtime_ctx.get_task_id()
114
+ self.context[
115
+ "assigned_resources"
116
+ ] = self.ray_runtime_ctx.get_assigned_resources()
117
+
118
+ message_dict["ray_runtime_context"] = self.context
119
+
120
+ return json.dumps(message_dict, default=str)
121
+
122
+
123
+ class DeltaCATLoggerAdapter(logging.LoggerAdapter):
124
+ """
125
+ Logger Adapter class with additional functionality
126
+ """
127
+
128
+ def __init__(self, logger: Logger, extra: Optional[Dict[str, Any]] = {}):
129
+ super().__init__(logger, extra)
130
+
131
+ def debug_conditional(self, msg, do_print: bool, *args, **kwargs):
132
+ if do_print:
133
+ self.debug(msg, *args, **kwargs)
134
+
135
+ def info_conditional(self, msg, do_print: bool, *args, **kwargs):
136
+ if do_print:
137
+ self.info(msg, *args, **kwargs)
138
+
139
+ def warning_conditional(self, msg, do_print: bool, *args, **kwargs):
140
+ if do_print:
141
+ self.warning(msg, *args, **kwargs)
142
+
143
+ def error_conditional(self, msg, do_print: bool, *args, **kwargs):
144
+ if do_print:
145
+ self.error(msg, *args, **kwargs)
146
+
147
+
148
+ def _add_logger_handler(logger: Logger, handler: Handler) -> Logger:
149
+
150
+ logger.setLevel(logging.getLevelName("DEBUG"))
151
+ logger.addHandler(handler)
152
+ return logger
153
+
154
+
155
+ def _create_rotating_file_handler(
156
+ log_directory: str,
157
+ log_base_file_name: str,
158
+ logging_level: Union[str, int] = DEFAULT_LOG_LEVEL,
159
+ max_bytes_per_log_file: int = DEFAULT_MAX_BYTES_PER_LOG,
160
+ backup_count: int = DEFAULT_BACKUP_COUNT,
161
+ logging_format: Union[str, dict] = DEFAULT_LOG_FORMAT,
162
+ ) -> FileHandler:
163
+
164
+ if type(logging_level) is str:
165
+ logging_level = logging.getLevelName(logging_level.upper())
166
+ assert log_base_file_name, "log file name is required"
167
+ assert log_directory, "log directory is required"
168
+ log_dir_path = pathlib.Path(log_directory)
169
+ log_dir_path.mkdir(parents=True, exist_ok=True)
170
+ handler = handlers.RotatingFileHandler(
171
+ os.path.join(log_directory, log_base_file_name),
172
+ maxBytes=max_bytes_per_log_file,
173
+ backupCount=backup_count,
174
+ )
175
+
176
+ if type(logging_format) is str:
177
+ handler.setFormatter(logging.Formatter(logging_format))
178
+ else:
179
+ handler.setFormatter(JsonFormatter(logging_format))
180
+
181
+ handler.setLevel(logging_level)
182
+ return handler
183
+
184
+
185
+ def _file_handler_exists(logger: Logger, log_dir: str, log_base_file_name: str) -> bool:
186
+
187
+ handler_exists = False
188
+ base_file_path = os.path.join(log_dir, log_base_file_name)
189
+
190
+ if logger.handlers:
191
+ norm_base_file_path = os.path.normpath(base_file_path)
192
+ handler_exists = any(
193
+ [
194
+ isinstance(handler, logging.FileHandler)
195
+ and os.path.normpath(handler.baseFilename) == norm_base_file_path
196
+ for handler in logger.handlers
197
+ ]
198
+ )
199
+ return handler_exists
200
+
201
+
202
+ def _configure_logger(
203
+ logger: Logger,
204
+ log_level: int,
205
+ log_dir: str,
206
+ log_base_file_name: str,
207
+ debug_log_base_file_name: str,
208
+ ) -> Union[Logger, LoggerAdapter]:
209
+ # This maintains log level of rotating file handlers
210
+ primary_log_level = log_level
211
+ logger.propagate = False
212
+ if log_level <= logging.getLevelName("DEBUG"):
213
+ if not _file_handler_exists(logger, log_dir, debug_log_base_file_name):
214
+ handler = _create_rotating_file_handler(
215
+ log_dir, debug_log_base_file_name, "DEBUG"
216
+ )
217
+ _add_logger_handler(logger, handler)
218
+ primary_log_level = logging.getLevelName("INFO")
219
+ if not _file_handler_exists(logger, log_dir, log_base_file_name):
220
+ handler = _create_rotating_file_handler(
221
+ log_dir, log_base_file_name, primary_log_level
222
+ )
223
+ _add_logger_handler(logger, handler)
224
+
225
+ return DeltaCATLoggerAdapter(logger)
226
+
227
+
228
+ def configure_deltacat_logger(
229
+ logger: Logger, level: int = None
230
+ ) -> Union[Logger, LoggerAdapter]:
231
+ if level is None:
232
+ level = logging.getLevelName(DELTACAT_SYS_LOG_LEVEL)
233
+
234
+ return _configure_logger(
235
+ logger,
236
+ level,
237
+ DELTACAT_SYS_LOG_DIR,
238
+ DELTACAT_SYS_INFO_LOG_BASE_FILE_NAME,
239
+ DELTACAT_SYS_DEBUG_LOG_BASE_FILE_NAME,
240
+ )
241
+
242
+
243
+ def configure_application_logger(
244
+ logger: Logger, level: int = None
245
+ ) -> Union[Logger, LoggerAdapter]:
246
+ if level is None:
247
+ level = logging.getLevelName(DELTACAT_APP_LOG_LEVEL)
248
+
249
+ return _configure_logger(
250
+ logger,
251
+ level,
252
+ DELTACAT_APP_LOG_DIR,
253
+ DELTACAT_APP_INFO_LOG_BASE_FILE_NAME,
254
+ DELTACAT_APP_DEBUG_LOG_BASE_FILE_NAME,
255
+ )
@@ -0,0 +1,12 @@
1
+ import unittest
2
+ from deltacat.aws.s3u import UuidBlockWritePathProvider, CapturedBlockWritePaths
3
+
4
+
5
+ class TestUuidBlockWritePathProvider(unittest.TestCase):
6
+ def test_uuid_block_write_provider_sanity(self):
7
+ capture_object = CapturedBlockWritePaths()
8
+ provider = UuidBlockWritePathProvider(capture_object=capture_object)
9
+
10
+ result = provider("base_path")
11
+
12
+ self.assertRegex(result, r"^base_path/[\w-]{36}$")
@@ -0,0 +1,127 @@
1
+ import unittest
2
+ import json
3
+ import ray
4
+ from logging import LogRecord
5
+ from deltacat.logs import JsonFormatter
6
+
7
+
8
+ class TestJsonFormatter(unittest.TestCase):
9
+ def test_usesTime_sanity(self):
10
+
11
+ formatter = JsonFormatter()
12
+
13
+ self.assertFalse(formatter.usesTime())
14
+
15
+ def test_usesTime_success_case(self):
16
+
17
+ formatter = JsonFormatter(fmt_dict={"asctime": "asctime"})
18
+
19
+ self.assertTrue(formatter.usesTime())
20
+
21
+ def test_formatMessage_sanity(self):
22
+
23
+ formatter = JsonFormatter({"message": "msg"})
24
+
25
+ record = LogRecord(
26
+ level="INFO",
27
+ name="test",
28
+ pathname="test",
29
+ lineno=0,
30
+ message="test_message",
31
+ msg="test_message",
32
+ args=None,
33
+ exc_info=None,
34
+ )
35
+
36
+ result = formatter.formatMessage(record)
37
+
38
+ self.assertEqual({"message": "test_message"}, result)
39
+
40
+ def test_format_sanity(self):
41
+ formatter = JsonFormatter({"message": "msg"})
42
+
43
+ record = LogRecord(
44
+ level="INFO",
45
+ name="test",
46
+ pathname="test",
47
+ lineno=0,
48
+ message="test_message",
49
+ msg="test_message",
50
+ args=None,
51
+ exc_info=None,
52
+ )
53
+
54
+ result = formatter.format(record)
55
+
56
+ self.assertEqual({"message": "test_message"}, json.loads(result))
57
+ self.assertFalse(ray.is_initialized())
58
+ self.assertNotIn("ray_runtime_context", json.loads(result))
59
+
60
+ def test_format_when_ray_initialized(self):
61
+ ray.init(local_mode=True, ignore_reinit_error=True)
62
+
63
+ formatter = JsonFormatter({"message": "msg"})
64
+
65
+ record = LogRecord(
66
+ level="INFO",
67
+ name="test",
68
+ pathname="test",
69
+ lineno=0,
70
+ message="test_message",
71
+ msg="test_message",
72
+ args=None,
73
+ exc_info=None,
74
+ )
75
+
76
+ result = formatter.format(record)
77
+ result = json.loads(result)
78
+
79
+ self.assertEqual("test_message", result["message"])
80
+ self.assertTrue(ray.is_initialized())
81
+ self.assertIn("ray_runtime_context", result)
82
+ self.assertIn("job_id", result["ray_runtime_context"])
83
+ self.assertIn("node_id", result["ray_runtime_context"])
84
+ self.assertIn("worker_id", result["ray_runtime_context"])
85
+ self.assertNotIn(
86
+ "task_id",
87
+ result["ray_runtime_context"],
88
+ "We expect task ID not be present outside a remote task",
89
+ )
90
+ ray.shutdown()
91
+
92
+ def test_format_when_ray_initialized_in_task(self):
93
+ # worker mode is only true when local_mode is False
94
+ ray.init(local_mode=False, ignore_reinit_error=True)
95
+
96
+ @ray.remote
97
+ def ray_remote_task():
98
+ formatter = JsonFormatter({"message": "msg"})
99
+
100
+ record = LogRecord(
101
+ level="INFO",
102
+ name="test",
103
+ pathname="test",
104
+ lineno=0,
105
+ message="test_message",
106
+ msg="test_message",
107
+ args=None,
108
+ exc_info=None,
109
+ )
110
+
111
+ result = formatter.format(record)
112
+ result = json.loads(result)
113
+ return result
114
+
115
+ result = ray.get(ray_remote_task.remote())
116
+ self.assertEqual("test_message", result["message"])
117
+ self.assertTrue(ray.is_initialized())
118
+ self.assertIn("ray_runtime_context", result)
119
+ self.assertIn("job_id", result["ray_runtime_context"])
120
+ self.assertIn("node_id", result["ray_runtime_context"])
121
+ self.assertIn("worker_id", result["ray_runtime_context"])
122
+ self.assertIn(
123
+ "task_id",
124
+ result["ray_runtime_context"],
125
+ "We expect task ID to be present inside a remote task",
126
+ )
127
+ ray.shutdown()
@@ -0,0 +1,25 @@
1
+ import unittest
2
+ import ray
3
+ from deltacat.utils.placement import (
4
+ PlacementGroupManager,
5
+ _get_available_resources_per_node,
6
+ )
7
+
8
+
9
+ class TestPlacementGroupManager(unittest.TestCase):
10
+ @classmethod
11
+ def setUpClass(cls):
12
+ super().setUpClass()
13
+ ray.init(local_mode=True, ignore_reinit_error=True)
14
+
15
+ def test_placement_group_manager_sanity(self):
16
+
17
+ pgm = PlacementGroupManager(1, 1, 1)
18
+
19
+ self.assertIsNotNone(pgm)
20
+
21
+ def test_ray_state_api_returns_correctly(self):
22
+
23
+ result = _get_available_resources_per_node()
24
+
25
+ self.assertIsNotNone(result)
@@ -2,6 +2,7 @@ import logging
2
2
  import re
3
3
  import time
4
4
  from dataclasses import dataclass
5
+ from packaging.version import Version
5
6
  from typing import Any, Dict, List, Optional, Tuple, Union
6
7
 
7
8
  import ray
@@ -19,6 +20,16 @@ logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
19
20
  # Issue: https://github.com/ray-project/ray/issues/29959
20
21
 
21
22
 
23
+ def _get_available_resources_per_node():
24
+ # This API changed after this commit
25
+ # https://github.com/ray-project/ray/pull/43252
26
+ # TODO: Use this method from a durable State API once it's available
27
+ if Version(ray.__version__) >= Version("2.10.0"):
28
+ return ray._private.state.available_resources_per_node()
29
+ else:
30
+ return ray._private.state.state._available_resources_per_node()
31
+
32
+
22
33
  @dataclass
23
34
  class PlacementGroupConfig:
24
35
  def __init__(self, opts, resource, node_ips):
@@ -90,9 +101,7 @@ class NodeGroupManager:
90
101
  Returns:
91
102
  group_res: a dict of resources, e.g., {'CPU':0,'memory':0,'object_store_memory':0}
92
103
  """
93
- all_available_resources = (
94
- ray._private.state.state._available_resources_per_node()
95
- )
104
+ all_available_resources = _get_available_resources_per_node()
96
105
  group_keys = [x[0] for x in self.init_groups]
97
106
  group_res = {}
98
107
  for k in group_keys:
@@ -127,9 +136,7 @@ class NodeGroupManager:
127
136
  Returns:
128
137
  group_res: dict of updated resource(cpu, memory, object store memory) for a given group
129
138
  """
130
- all_available_resources = (
131
- ray._private.state.state._available_resources_per_node()
132
- )
139
+ all_available_resources = _get_available_resources_per_node()
133
140
  group_res = {"CPU": 0, "memory": 0, "object_store_memory": 0, "node_id": []}
134
141
  for v in all_available_resources.values():
135
142
  keys = v.keys()
@@ -285,7 +292,7 @@ def _config(
285
292
  for bd in bundles:
286
293
  node_ids.append(bd["node_id"])
287
294
  # query available resources given list of node id
288
- all_nodes_available_res = ray._private.state.state._available_resources_per_node()
295
+ all_nodes_available_res = _get_available_resources_per_node()
289
296
  pg_res = {"CPU": 0, "memory": 0, "object_store_memory": 0}
290
297
  node_ips = []
291
298
  for node_id in node_ids:
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: deltacat
3
- Version: 1.1.3
3
+ Version: 1.1.4
4
4
  Summary: A scalable, fast, ACID-compliant Data Catalog powered by Ray.
5
5
  Home-page: https://github.com/ray-project/deltacat
6
6
  Author: Ray Team
@@ -141,8 +141,10 @@ deltacat/storage/model/table.py
141
141
  deltacat/storage/model/table_version.py
142
142
  deltacat/storage/model/types.py
143
143
  deltacat/tests/__init__.py
144
+ deltacat/tests/test_logs.py
144
145
  deltacat/tests/aws/__init__.py
145
146
  deltacat/tests/aws/test_clients.py
147
+ deltacat/tests/aws/test_s3u.py
146
148
  deltacat/tests/catalog/__init__.py
147
149
  deltacat/tests/catalog/test_default_catalog_impl.py
148
150
  deltacat/tests/compute/__init__.py
@@ -183,6 +185,7 @@ deltacat/tests/utils/__init__.py
183
185
  deltacat/tests/utils/test_cloudpickle.py
184
186
  deltacat/tests/utils/test_daft.py
185
187
  deltacat/tests/utils/test_metrics.py
188
+ deltacat/tests/utils/test_placement.py
186
189
  deltacat/tests/utils/test_pyarrow.py
187
190
  deltacat/tests/utils/test_record_batch_tables.py
188
191
  deltacat/tests/utils/test_resources.py