deltacat 1.1.3__tar.gz → 1.1.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {deltacat-1.1.3 → deltacat-1.1.4}/PKG-INFO +1 -1
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/__init__.py +1 -1
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/aws/constants.py +0 -4
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/aws/s3u.py +26 -5
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/utils/round_completion_file.py +3 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor_v2/compaction_session.py +2 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor_v2/constants.py +0 -3
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor_v2/utils/merge.py +0 -3
- deltacat-1.1.4/deltacat/logs.py +255 -0
- deltacat-1.1.4/deltacat/tests/aws/test_s3u.py +12 -0
- deltacat-1.1.4/deltacat/tests/test_logs.py +127 -0
- deltacat-1.1.4/deltacat/tests/utils/test_placement.py +25 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/utils/placement.py +14 -7
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat.egg-info/PKG-INFO +1 -1
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat.egg-info/SOURCES.txt +3 -0
- deltacat-1.1.3/deltacat/logs.py +0 -198
- {deltacat-1.1.3 → deltacat-1.1.4}/MANIFEST.in +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/README.md +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/aws/__init__.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/aws/clients.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/aws/redshift/__init__.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/aws/redshift/model/__init__.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/aws/redshift/model/manifest.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/benchmarking/__init__.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/benchmarking/benchmark_parquet_reads.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/benchmarking/conftest.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/catalog/__init__.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/catalog/default_catalog_impl/__init__.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/catalog/delegate.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/catalog/interface.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/catalog/model/__init__.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/catalog/model/catalog.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/catalog/model/table_definition.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/__init__.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/__init__.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/compaction_session.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/model/__init__.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/model/compact_partition_params.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/model/compaction_session_audit_info.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/model/compactor_version.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/model/dedupe_result.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/model/delta_annotated.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/model/delta_file_envelope.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/model/delta_file_locator.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/model/hash_bucket_result.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/model/materialize_result.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/model/primary_key_index.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/model/pyarrow_write_result.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/model/repartition_result.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/model/round_completion_info.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/model/table_object_store.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/repartition_session.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/steps/__init__.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/steps/dedupe.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/steps/hash_bucket.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/steps/materialize.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/steps/repartition.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/utils/__init__.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/utils/io.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/utils/primary_key_index.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/utils/sort_key.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor/utils/system_columns.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor_v2/__init__.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor_v2/deletes/__init__.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor_v2/deletes/delete_file_envelope.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor_v2/deletes/delete_strategy.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor_v2/deletes/delete_strategy_equality_delete.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor_v2/deletes/model.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor_v2/deletes/utils.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor_v2/model/__init__.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor_v2/model/hash_bucket_input.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor_v2/model/hash_bucket_result.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor_v2/model/merge_file_group.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor_v2/model/merge_input.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor_v2/model/merge_result.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor_v2/steps/__init__.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor_v2/steps/hash_bucket.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor_v2/steps/merge.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor_v2/utils/__init__.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor_v2/utils/content_type_params.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor_v2/utils/dedupe.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor_v2/utils/delta.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor_v2/utils/io.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor_v2/utils/primary_key_index.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/compactor_v2/utils/task_options.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/merge_on_read/__init__.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/merge_on_read/daft.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/merge_on_read/model/__init__.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/merge_on_read/model/merge_on_read_params.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/merge_on_read/utils/__init__.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/merge_on_read/utils/delta.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/metastats/__init__.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/metastats/config/__init__.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/metastats/meta_stats.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/metastats/model/__init__.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/metastats/model/partition_stats_dict.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/metastats/model/stats_cluster_size_estimator.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/metastats/stats.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/metastats/utils/__init__.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/metastats/utils/constants.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/metastats/utils/io.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/metastats/utils/pyarrow_memory_estimation_function.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/metastats/utils/ray_utils.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/stats/__init__.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/stats/basic.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/stats/models/__init__.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/stats/models/delta_column_stats.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/stats/models/delta_stats.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/stats/models/delta_stats_cache_result.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/stats/models/manifest_entry_stats.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/stats/models/stats_result.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/stats/types.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/stats/utils/__init__.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/stats/utils/intervals.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/stats/utils/io.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/compute/stats/utils/manifest_stats_file.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/constants.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/exceptions.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/io/__init__.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/io/aws/__init__.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/io/aws/redshift/__init__.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/io/aws/redshift/redshift_datasource.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/io/dataset.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/io/file_object_store.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/io/memcached_object_store.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/io/object_store.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/io/ray_plasma_object_store.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/io/read_api.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/io/redis_object_store.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/io/s3_object_store.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/storage/__init__.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/storage/interface.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/storage/model/__init__.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/storage/model/delete_parameters.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/storage/model/delta.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/storage/model/list_result.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/storage/model/locator.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/storage/model/namespace.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/storage/model/partition.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/storage/model/sort_key.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/storage/model/stream.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/storage/model/table.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/storage/model/table_version.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/storage/model/types.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/__init__.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/aws/__init__.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/aws/test_clients.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/catalog/__init__.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/catalog/test_default_catalog_impl.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/compute/__init__.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/compute/compact_partition_rebase_then_incremental_test_cases.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/compute/compact_partition_test_cases.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/compute/compactor/__init__.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/compute/compactor/steps/__init__.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/compute/compactor/steps/test_repartition.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/compute/compactor/utils/__init__.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/compute/compactor/utils/test_io.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/compute/compactor_v2/__init__.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/compute/compactor_v2/test_compaction_session.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/compute/compactor_v2/test_hashlib.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/compute/compactor_v2/utils/__init__.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/compute/compactor_v2/utils/test_task_options.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/compute/test_compact_partition_incremental.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/compute/test_compact_partition_params.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/compute/test_compact_partition_rebase_then_incremental.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/compute/test_util_common.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/compute/test_util_constant.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/compute/test_util_create_table_deltas_repo.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/io/__init__.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/io/test_cloudpickle_bug_fix.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/io/test_file_object_store.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/io/test_memcached_object_store.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/io/test_ray_plasma_object_store.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/io/test_redis_object_store.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/io/test_s3_object_store.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/local_deltacat_storage/__init__.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/stats/__init__.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/stats/test_intervals.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/test_utils/__init__.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/test_utils/constants.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/test_utils/pyarrow.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/test_utils/storage.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/test_utils/utils.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/utils/__init__.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/utils/data/__init__.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/utils/test_cloudpickle.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/utils/test_daft.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/utils/test_metrics.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/utils/test_pyarrow.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/utils/test_record_batch_tables.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/tests/utils/test_resources.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/types/__init__.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/types/media.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/types/partial_download.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/types/tables.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/utils/__init__.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/utils/arguments.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/utils/cloudpickle.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/utils/common.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/utils/daft.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/utils/metrics.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/utils/numpy.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/utils/pandas.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/utils/performance.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/utils/pyarrow.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/utils/ray_utils/__init__.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/utils/ray_utils/collections.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/utils/ray_utils/concurrency.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/utils/ray_utils/dataset.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/utils/ray_utils/performance.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/utils/ray_utils/runtime.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/utils/resources.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/utils/s3fs.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat/utils/schema.py +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat.egg-info/dependency_links.txt +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat.egg-info/requires.txt +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/deltacat.egg-info/top_level.txt +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/setup.cfg +0 -0
- {deltacat-1.1.3 → deltacat-1.1.4}/setup.py +0 -0
@@ -6,7 +6,3 @@ DAFT_MAX_S3_CONNECTIONS_PER_FILE = env_integer("DAFT_MAX_S3_CONNECTIONS_PER_FILE
|
|
6
6
|
BOTO_MAX_RETRIES = env_integer("BOTO_MAX_RETRIES", 5)
|
7
7
|
TIMEOUT_ERROR_CODES: List[str] = ["ReadTimeoutError", "ConnectTimeoutError"]
|
8
8
|
AWS_REGION = env_string("AWS_REGION", "us-east-1")
|
9
|
-
|
10
|
-
# Metric Names
|
11
|
-
DOWNLOAD_MANIFEST_ENTRY_METRIC_PREFIX = "download_manifest_entry"
|
12
|
-
UPLOAD_SLICED_TABLE_METRIC_PREFIX = "upload_sliced_table"
|
@@ -27,8 +27,6 @@ import deltacat.aws.clients as aws_utils
|
|
27
27
|
from deltacat import logs
|
28
28
|
from deltacat.aws.constants import (
|
29
29
|
TIMEOUT_ERROR_CODES,
|
30
|
-
DOWNLOAD_MANIFEST_ENTRY_METRIC_PREFIX,
|
31
|
-
UPLOAD_SLICED_TABLE_METRIC_PREFIX,
|
32
30
|
)
|
33
31
|
from deltacat.exceptions import NonRetryableError, RetryableError
|
34
32
|
from deltacat.storage import (
|
@@ -54,7 +52,6 @@ from deltacat.types.tables import (
|
|
54
52
|
)
|
55
53
|
from deltacat.types.partial_download import PartialFileDownloadParams
|
56
54
|
from deltacat.utils.common import ReadKwargsProvider
|
57
|
-
from deltacat.utils.metrics import metrics
|
58
55
|
|
59
56
|
logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
|
60
57
|
|
@@ -121,6 +118,32 @@ class UuidBlockWritePathProvider(BlockWritePathProvider):
|
|
121
118
|
self.block_refs.append(block)
|
122
119
|
return write_path
|
123
120
|
|
121
|
+
def __call__(
|
122
|
+
self,
|
123
|
+
base_path: str,
|
124
|
+
*,
|
125
|
+
filesystem: Optional[pa.filesystem.FileSystem] = None,
|
126
|
+
dataset_uuid: Optional[str] = None,
|
127
|
+
block: Optional[ObjectRef[Block]] = None,
|
128
|
+
block_index: Optional[int] = None,
|
129
|
+
file_format: Optional[str] = None,
|
130
|
+
) -> str:
|
131
|
+
"""
|
132
|
+
TODO: BlockWritePathProvider is deprecated as of Ray version 2.20.0. Please use FilenameProvider.
|
133
|
+
See: https://docs.ray.io/en/master/data/api/doc/ray.data.datasource.FilenameProvider.html
|
134
|
+
Also See: https://github.com/ray-project/deltacat/issues/299
|
135
|
+
|
136
|
+
Hence, this class only works with Ray version 2.20.0 or lower when used in Ray Dataset.
|
137
|
+
"""
|
138
|
+
return self._get_write_path_for_block(
|
139
|
+
base_path,
|
140
|
+
filesystem=filesystem,
|
141
|
+
dataset_uuid=dataset_uuid,
|
142
|
+
block=block,
|
143
|
+
block_index=block_index,
|
144
|
+
file_format=file_format,
|
145
|
+
)
|
146
|
+
|
124
147
|
|
125
148
|
class S3Url:
|
126
149
|
def __init__(self, url: str):
|
@@ -243,7 +266,6 @@ def read_file(
|
|
243
266
|
raise e
|
244
267
|
|
245
268
|
|
246
|
-
@metrics(prefix=UPLOAD_SLICED_TABLE_METRIC_PREFIX)
|
247
269
|
def upload_sliced_table(
|
248
270
|
table: Union[LocalTable, DistributedDataset],
|
249
271
|
s3_url_prefix: str,
|
@@ -352,7 +374,6 @@ def upload_table(
|
|
352
374
|
return manifest_entries
|
353
375
|
|
354
376
|
|
355
|
-
@metrics(prefix=DOWNLOAD_MANIFEST_ENTRY_METRIC_PREFIX)
|
356
377
|
def download_manifest_entry(
|
357
378
|
manifest_entry: ManifestEntry,
|
358
379
|
token_holder: Optional[Dict[str, Any]] = None,
|
@@ -6,6 +6,7 @@ from deltacat.compute.compactor import RoundCompletionInfo
|
|
6
6
|
from deltacat.storage import PartitionLocator
|
7
7
|
from deltacat.aws import s3u as s3_utils
|
8
8
|
from typing import Optional
|
9
|
+
from deltacat.utils.metrics import metrics
|
9
10
|
|
10
11
|
logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
|
11
12
|
|
@@ -18,6 +19,7 @@ def get_round_completion_file_s3_url(
|
|
18
19
|
return f"{base_url}.json"
|
19
20
|
|
20
21
|
|
22
|
+
@metrics
|
21
23
|
def read_round_completion_file(
|
22
24
|
bucket: str,
|
23
25
|
source_partition_locator: PartitionLocator,
|
@@ -38,6 +40,7 @@ def read_round_completion_file(
|
|
38
40
|
return round_completion_info
|
39
41
|
|
40
42
|
|
43
|
+
@metrics
|
41
44
|
def write_round_completion_file(
|
42
45
|
bucket: Optional[str],
|
43
46
|
source_partition_locator: Optional[PartitionLocator],
|
@@ -50,6 +50,7 @@ from deltacat.compute.compactor_v2.steps import merge as mg
|
|
50
50
|
from deltacat.compute.compactor_v2.steps import hash_bucket as hb
|
51
51
|
from deltacat.compute.compactor_v2.utils import io
|
52
52
|
from deltacat.compute.compactor.utils import round_completion_file as rcf
|
53
|
+
from deltacat.utils.metrics import metrics
|
53
54
|
|
54
55
|
from typing import List, Optional, Tuple
|
55
56
|
from collections import defaultdict
|
@@ -73,6 +74,7 @@ if importlib.util.find_spec("memray"):
|
|
73
74
|
logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
|
74
75
|
|
75
76
|
|
77
|
+
@metrics
|
76
78
|
def compact_partition(params: CompactPartitionParams, **kwargs) -> Optional[str]:
|
77
79
|
|
78
80
|
assert (
|
@@ -31,14 +31,11 @@ from deltacat.compute.compactor_v2.deletes.delete_strategy import (
|
|
31
31
|
from deltacat.compute.compactor_v2.deletes.delete_file_envelope import (
|
32
32
|
DeleteFileEnvelope,
|
33
33
|
)
|
34
|
-
from deltacat.utils.metrics import metrics
|
35
|
-
from deltacat.compute.compactor_v2.constants import MATERIALIZE_METRIC_PREFIX
|
36
34
|
|
37
35
|
|
38
36
|
logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
|
39
37
|
|
40
38
|
|
41
|
-
@metrics(prefix=MATERIALIZE_METRIC_PREFIX)
|
42
39
|
def materialize(
|
43
40
|
input: MergeInput,
|
44
41
|
task_index: int,
|
@@ -0,0 +1,255 @@
|
|
1
|
+
import logging
|
2
|
+
import os
|
3
|
+
import json
|
4
|
+
import pathlib
|
5
|
+
from logging import FileHandler, Handler, Logger, LoggerAdapter, handlers
|
6
|
+
from typing import Any, Dict, Optional, Union
|
7
|
+
|
8
|
+
import ray
|
9
|
+
from ray.runtime_context import RuntimeContext
|
10
|
+
|
11
|
+
from deltacat.constants import (
|
12
|
+
DELTACAT_APP_LOG_LEVEL,
|
13
|
+
DELTACAT_SYS_LOG_LEVEL,
|
14
|
+
DELTACAT_APP_LOG_DIR,
|
15
|
+
DELTACAT_SYS_LOG_DIR,
|
16
|
+
DELTACAT_APP_INFO_LOG_BASE_FILE_NAME,
|
17
|
+
DELTACAT_SYS_INFO_LOG_BASE_FILE_NAME,
|
18
|
+
DELTACAT_APP_DEBUG_LOG_BASE_FILE_NAME,
|
19
|
+
DELTACAT_SYS_DEBUG_LOG_BASE_FILE_NAME,
|
20
|
+
)
|
21
|
+
|
22
|
+
DEFAULT_LOG_LEVEL = "INFO"
|
23
|
+
DEFAULT_LOG_FORMAT = {
|
24
|
+
"level": "levelname",
|
25
|
+
"message": "message",
|
26
|
+
"loggerName": "name",
|
27
|
+
"processName": "processName",
|
28
|
+
"processID": "process",
|
29
|
+
"threadName": "threadName",
|
30
|
+
"timestamp": "asctime",
|
31
|
+
"filename": "filename",
|
32
|
+
"lineno": "lineno",
|
33
|
+
}
|
34
|
+
DEFAULT_MAX_BYTES_PER_LOG = 2 ^ 20 * 256 # 256 MiB
|
35
|
+
DEFAULT_BACKUP_COUNT = 0
|
36
|
+
|
37
|
+
|
38
|
+
class JsonFormatter(logging.Formatter):
|
39
|
+
"""
|
40
|
+
Formatter that outputs JSON strings after parsing the LogRecord.
|
41
|
+
|
42
|
+
@param dict fmt_dict: Key: logging format attribute pairs. Defaults to {"message": "message"}.
|
43
|
+
@param str time_format: time.strftime() format string. Default: "%Y-%m-%dT%H:%M:%S"
|
44
|
+
@param str msec_format: Microsecond formatting. Appended at the end. Default: "%s.%03dZ"
|
45
|
+
"""
|
46
|
+
|
47
|
+
def __init__(
|
48
|
+
self,
|
49
|
+
fmt_dict: dict = None,
|
50
|
+
time_format: str = "%Y-%m-%dT%H:%M:%S",
|
51
|
+
msec_format: str = "%s.%03dZ",
|
52
|
+
):
|
53
|
+
self.fmt_dict = fmt_dict if fmt_dict is not None else {"message": "message"}
|
54
|
+
self.default_time_format = time_format
|
55
|
+
self.default_msec_format = msec_format
|
56
|
+
self.datefmt = None
|
57
|
+
if ray.is_initialized():
|
58
|
+
self.ray_runtime_ctx: RuntimeContext = ray.get_runtime_context()
|
59
|
+
self.context = {}
|
60
|
+
self.context["worker_id"] = self.ray_runtime_ctx.get_worker_id()
|
61
|
+
self.context["node_id"] = self.ray_runtime_ctx.get_node_id()
|
62
|
+
self.context["job_id"] = self.ray_runtime_ctx.get_job_id()
|
63
|
+
else:
|
64
|
+
self.ray_runtime_ctx = None
|
65
|
+
self.context = {}
|
66
|
+
|
67
|
+
def usesTime(self) -> bool:
|
68
|
+
"""
|
69
|
+
Overwritten to look for the attribute in the format dict values instead of the fmt string.
|
70
|
+
"""
|
71
|
+
return "asctime" in self.fmt_dict.values()
|
72
|
+
|
73
|
+
def formatMessage(self, record) -> dict:
|
74
|
+
"""
|
75
|
+
Overwritten to return a dictionary of the relevant LogRecord attributes instead of a string.
|
76
|
+
KeyError is raised if an unknown attribute is provided in the fmt_dict.
|
77
|
+
"""
|
78
|
+
return {
|
79
|
+
fmt_key: record.__dict__[fmt_val]
|
80
|
+
for fmt_key, fmt_val in self.fmt_dict.items()
|
81
|
+
}
|
82
|
+
|
83
|
+
def format(self, record) -> str:
|
84
|
+
"""
|
85
|
+
Mostly the same as the parent's class method, the difference being that a dict is manipulated and dumped as JSON
|
86
|
+
instead of a string.
|
87
|
+
"""
|
88
|
+
record.message = record.getMessage()
|
89
|
+
|
90
|
+
if self.usesTime():
|
91
|
+
record.asctime = self.formatTime(record, self.datefmt)
|
92
|
+
|
93
|
+
message_dict = self.formatMessage(record)
|
94
|
+
|
95
|
+
if record.exc_info:
|
96
|
+
# Cache the traceback text to avoid converting it multiple times
|
97
|
+
# (it's constant anyway)
|
98
|
+
if not record.exc_text:
|
99
|
+
record.exc_text = self.formatException(record.exc_info)
|
100
|
+
|
101
|
+
if record.exc_text:
|
102
|
+
message_dict["exc_info"] = record.exc_text
|
103
|
+
|
104
|
+
if record.stack_info:
|
105
|
+
message_dict["stack_info"] = self.formatStack(record.stack_info)
|
106
|
+
|
107
|
+
if self.ray_runtime_ctx:
|
108
|
+
# only workers will have task ID
|
109
|
+
if (
|
110
|
+
self.ray_runtime_ctx.worker
|
111
|
+
and self.ray_runtime_ctx.worker.mode == ray._private.worker.WORKER_MODE
|
112
|
+
):
|
113
|
+
self.context["task_id"] = self.ray_runtime_ctx.get_task_id()
|
114
|
+
self.context[
|
115
|
+
"assigned_resources"
|
116
|
+
] = self.ray_runtime_ctx.get_assigned_resources()
|
117
|
+
|
118
|
+
message_dict["ray_runtime_context"] = self.context
|
119
|
+
|
120
|
+
return json.dumps(message_dict, default=str)
|
121
|
+
|
122
|
+
|
123
|
+
class DeltaCATLoggerAdapter(logging.LoggerAdapter):
|
124
|
+
"""
|
125
|
+
Logger Adapter class with additional functionality
|
126
|
+
"""
|
127
|
+
|
128
|
+
def __init__(self, logger: Logger, extra: Optional[Dict[str, Any]] = {}):
|
129
|
+
super().__init__(logger, extra)
|
130
|
+
|
131
|
+
def debug_conditional(self, msg, do_print: bool, *args, **kwargs):
|
132
|
+
if do_print:
|
133
|
+
self.debug(msg, *args, **kwargs)
|
134
|
+
|
135
|
+
def info_conditional(self, msg, do_print: bool, *args, **kwargs):
|
136
|
+
if do_print:
|
137
|
+
self.info(msg, *args, **kwargs)
|
138
|
+
|
139
|
+
def warning_conditional(self, msg, do_print: bool, *args, **kwargs):
|
140
|
+
if do_print:
|
141
|
+
self.warning(msg, *args, **kwargs)
|
142
|
+
|
143
|
+
def error_conditional(self, msg, do_print: bool, *args, **kwargs):
|
144
|
+
if do_print:
|
145
|
+
self.error(msg, *args, **kwargs)
|
146
|
+
|
147
|
+
|
148
|
+
def _add_logger_handler(logger: Logger, handler: Handler) -> Logger:
|
149
|
+
|
150
|
+
logger.setLevel(logging.getLevelName("DEBUG"))
|
151
|
+
logger.addHandler(handler)
|
152
|
+
return logger
|
153
|
+
|
154
|
+
|
155
|
+
def _create_rotating_file_handler(
|
156
|
+
log_directory: str,
|
157
|
+
log_base_file_name: str,
|
158
|
+
logging_level: Union[str, int] = DEFAULT_LOG_LEVEL,
|
159
|
+
max_bytes_per_log_file: int = DEFAULT_MAX_BYTES_PER_LOG,
|
160
|
+
backup_count: int = DEFAULT_BACKUP_COUNT,
|
161
|
+
logging_format: Union[str, dict] = DEFAULT_LOG_FORMAT,
|
162
|
+
) -> FileHandler:
|
163
|
+
|
164
|
+
if type(logging_level) is str:
|
165
|
+
logging_level = logging.getLevelName(logging_level.upper())
|
166
|
+
assert log_base_file_name, "log file name is required"
|
167
|
+
assert log_directory, "log directory is required"
|
168
|
+
log_dir_path = pathlib.Path(log_directory)
|
169
|
+
log_dir_path.mkdir(parents=True, exist_ok=True)
|
170
|
+
handler = handlers.RotatingFileHandler(
|
171
|
+
os.path.join(log_directory, log_base_file_name),
|
172
|
+
maxBytes=max_bytes_per_log_file,
|
173
|
+
backupCount=backup_count,
|
174
|
+
)
|
175
|
+
|
176
|
+
if type(logging_format) is str:
|
177
|
+
handler.setFormatter(logging.Formatter(logging_format))
|
178
|
+
else:
|
179
|
+
handler.setFormatter(JsonFormatter(logging_format))
|
180
|
+
|
181
|
+
handler.setLevel(logging_level)
|
182
|
+
return handler
|
183
|
+
|
184
|
+
|
185
|
+
def _file_handler_exists(logger: Logger, log_dir: str, log_base_file_name: str) -> bool:
|
186
|
+
|
187
|
+
handler_exists = False
|
188
|
+
base_file_path = os.path.join(log_dir, log_base_file_name)
|
189
|
+
|
190
|
+
if logger.handlers:
|
191
|
+
norm_base_file_path = os.path.normpath(base_file_path)
|
192
|
+
handler_exists = any(
|
193
|
+
[
|
194
|
+
isinstance(handler, logging.FileHandler)
|
195
|
+
and os.path.normpath(handler.baseFilename) == norm_base_file_path
|
196
|
+
for handler in logger.handlers
|
197
|
+
]
|
198
|
+
)
|
199
|
+
return handler_exists
|
200
|
+
|
201
|
+
|
202
|
+
def _configure_logger(
|
203
|
+
logger: Logger,
|
204
|
+
log_level: int,
|
205
|
+
log_dir: str,
|
206
|
+
log_base_file_name: str,
|
207
|
+
debug_log_base_file_name: str,
|
208
|
+
) -> Union[Logger, LoggerAdapter]:
|
209
|
+
# This maintains log level of rotating file handlers
|
210
|
+
primary_log_level = log_level
|
211
|
+
logger.propagate = False
|
212
|
+
if log_level <= logging.getLevelName("DEBUG"):
|
213
|
+
if not _file_handler_exists(logger, log_dir, debug_log_base_file_name):
|
214
|
+
handler = _create_rotating_file_handler(
|
215
|
+
log_dir, debug_log_base_file_name, "DEBUG"
|
216
|
+
)
|
217
|
+
_add_logger_handler(logger, handler)
|
218
|
+
primary_log_level = logging.getLevelName("INFO")
|
219
|
+
if not _file_handler_exists(logger, log_dir, log_base_file_name):
|
220
|
+
handler = _create_rotating_file_handler(
|
221
|
+
log_dir, log_base_file_name, primary_log_level
|
222
|
+
)
|
223
|
+
_add_logger_handler(logger, handler)
|
224
|
+
|
225
|
+
return DeltaCATLoggerAdapter(logger)
|
226
|
+
|
227
|
+
|
228
|
+
def configure_deltacat_logger(
|
229
|
+
logger: Logger, level: int = None
|
230
|
+
) -> Union[Logger, LoggerAdapter]:
|
231
|
+
if level is None:
|
232
|
+
level = logging.getLevelName(DELTACAT_SYS_LOG_LEVEL)
|
233
|
+
|
234
|
+
return _configure_logger(
|
235
|
+
logger,
|
236
|
+
level,
|
237
|
+
DELTACAT_SYS_LOG_DIR,
|
238
|
+
DELTACAT_SYS_INFO_LOG_BASE_FILE_NAME,
|
239
|
+
DELTACAT_SYS_DEBUG_LOG_BASE_FILE_NAME,
|
240
|
+
)
|
241
|
+
|
242
|
+
|
243
|
+
def configure_application_logger(
|
244
|
+
logger: Logger, level: int = None
|
245
|
+
) -> Union[Logger, LoggerAdapter]:
|
246
|
+
if level is None:
|
247
|
+
level = logging.getLevelName(DELTACAT_APP_LOG_LEVEL)
|
248
|
+
|
249
|
+
return _configure_logger(
|
250
|
+
logger,
|
251
|
+
level,
|
252
|
+
DELTACAT_APP_LOG_DIR,
|
253
|
+
DELTACAT_APP_INFO_LOG_BASE_FILE_NAME,
|
254
|
+
DELTACAT_APP_DEBUG_LOG_BASE_FILE_NAME,
|
255
|
+
)
|
@@ -0,0 +1,12 @@
|
|
1
|
+
import unittest
|
2
|
+
from deltacat.aws.s3u import UuidBlockWritePathProvider, CapturedBlockWritePaths
|
3
|
+
|
4
|
+
|
5
|
+
class TestUuidBlockWritePathProvider(unittest.TestCase):
|
6
|
+
def test_uuid_block_write_provider_sanity(self):
|
7
|
+
capture_object = CapturedBlockWritePaths()
|
8
|
+
provider = UuidBlockWritePathProvider(capture_object=capture_object)
|
9
|
+
|
10
|
+
result = provider("base_path")
|
11
|
+
|
12
|
+
self.assertRegex(result, r"^base_path/[\w-]{36}$")
|
@@ -0,0 +1,127 @@
|
|
1
|
+
import unittest
|
2
|
+
import json
|
3
|
+
import ray
|
4
|
+
from logging import LogRecord
|
5
|
+
from deltacat.logs import JsonFormatter
|
6
|
+
|
7
|
+
|
8
|
+
class TestJsonFormatter(unittest.TestCase):
|
9
|
+
def test_usesTime_sanity(self):
|
10
|
+
|
11
|
+
formatter = JsonFormatter()
|
12
|
+
|
13
|
+
self.assertFalse(formatter.usesTime())
|
14
|
+
|
15
|
+
def test_usesTime_success_case(self):
|
16
|
+
|
17
|
+
formatter = JsonFormatter(fmt_dict={"asctime": "asctime"})
|
18
|
+
|
19
|
+
self.assertTrue(formatter.usesTime())
|
20
|
+
|
21
|
+
def test_formatMessage_sanity(self):
|
22
|
+
|
23
|
+
formatter = JsonFormatter({"message": "msg"})
|
24
|
+
|
25
|
+
record = LogRecord(
|
26
|
+
level="INFO",
|
27
|
+
name="test",
|
28
|
+
pathname="test",
|
29
|
+
lineno=0,
|
30
|
+
message="test_message",
|
31
|
+
msg="test_message",
|
32
|
+
args=None,
|
33
|
+
exc_info=None,
|
34
|
+
)
|
35
|
+
|
36
|
+
result = formatter.formatMessage(record)
|
37
|
+
|
38
|
+
self.assertEqual({"message": "test_message"}, result)
|
39
|
+
|
40
|
+
def test_format_sanity(self):
|
41
|
+
formatter = JsonFormatter({"message": "msg"})
|
42
|
+
|
43
|
+
record = LogRecord(
|
44
|
+
level="INFO",
|
45
|
+
name="test",
|
46
|
+
pathname="test",
|
47
|
+
lineno=0,
|
48
|
+
message="test_message",
|
49
|
+
msg="test_message",
|
50
|
+
args=None,
|
51
|
+
exc_info=None,
|
52
|
+
)
|
53
|
+
|
54
|
+
result = formatter.format(record)
|
55
|
+
|
56
|
+
self.assertEqual({"message": "test_message"}, json.loads(result))
|
57
|
+
self.assertFalse(ray.is_initialized())
|
58
|
+
self.assertNotIn("ray_runtime_context", json.loads(result))
|
59
|
+
|
60
|
+
def test_format_when_ray_initialized(self):
|
61
|
+
ray.init(local_mode=True, ignore_reinit_error=True)
|
62
|
+
|
63
|
+
formatter = JsonFormatter({"message": "msg"})
|
64
|
+
|
65
|
+
record = LogRecord(
|
66
|
+
level="INFO",
|
67
|
+
name="test",
|
68
|
+
pathname="test",
|
69
|
+
lineno=0,
|
70
|
+
message="test_message",
|
71
|
+
msg="test_message",
|
72
|
+
args=None,
|
73
|
+
exc_info=None,
|
74
|
+
)
|
75
|
+
|
76
|
+
result = formatter.format(record)
|
77
|
+
result = json.loads(result)
|
78
|
+
|
79
|
+
self.assertEqual("test_message", result["message"])
|
80
|
+
self.assertTrue(ray.is_initialized())
|
81
|
+
self.assertIn("ray_runtime_context", result)
|
82
|
+
self.assertIn("job_id", result["ray_runtime_context"])
|
83
|
+
self.assertIn("node_id", result["ray_runtime_context"])
|
84
|
+
self.assertIn("worker_id", result["ray_runtime_context"])
|
85
|
+
self.assertNotIn(
|
86
|
+
"task_id",
|
87
|
+
result["ray_runtime_context"],
|
88
|
+
"We expect task ID not be present outside a remote task",
|
89
|
+
)
|
90
|
+
ray.shutdown()
|
91
|
+
|
92
|
+
def test_format_when_ray_initialized_in_task(self):
|
93
|
+
# worker mode is only true when local_mode is False
|
94
|
+
ray.init(local_mode=False, ignore_reinit_error=True)
|
95
|
+
|
96
|
+
@ray.remote
|
97
|
+
def ray_remote_task():
|
98
|
+
formatter = JsonFormatter({"message": "msg"})
|
99
|
+
|
100
|
+
record = LogRecord(
|
101
|
+
level="INFO",
|
102
|
+
name="test",
|
103
|
+
pathname="test",
|
104
|
+
lineno=0,
|
105
|
+
message="test_message",
|
106
|
+
msg="test_message",
|
107
|
+
args=None,
|
108
|
+
exc_info=None,
|
109
|
+
)
|
110
|
+
|
111
|
+
result = formatter.format(record)
|
112
|
+
result = json.loads(result)
|
113
|
+
return result
|
114
|
+
|
115
|
+
result = ray.get(ray_remote_task.remote())
|
116
|
+
self.assertEqual("test_message", result["message"])
|
117
|
+
self.assertTrue(ray.is_initialized())
|
118
|
+
self.assertIn("ray_runtime_context", result)
|
119
|
+
self.assertIn("job_id", result["ray_runtime_context"])
|
120
|
+
self.assertIn("node_id", result["ray_runtime_context"])
|
121
|
+
self.assertIn("worker_id", result["ray_runtime_context"])
|
122
|
+
self.assertIn(
|
123
|
+
"task_id",
|
124
|
+
result["ray_runtime_context"],
|
125
|
+
"We expect task ID to be present inside a remote task",
|
126
|
+
)
|
127
|
+
ray.shutdown()
|
@@ -0,0 +1,25 @@
|
|
1
|
+
import unittest
|
2
|
+
import ray
|
3
|
+
from deltacat.utils.placement import (
|
4
|
+
PlacementGroupManager,
|
5
|
+
_get_available_resources_per_node,
|
6
|
+
)
|
7
|
+
|
8
|
+
|
9
|
+
class TestPlacementGroupManager(unittest.TestCase):
|
10
|
+
@classmethod
|
11
|
+
def setUpClass(cls):
|
12
|
+
super().setUpClass()
|
13
|
+
ray.init(local_mode=True, ignore_reinit_error=True)
|
14
|
+
|
15
|
+
def test_placement_group_manager_sanity(self):
|
16
|
+
|
17
|
+
pgm = PlacementGroupManager(1, 1, 1)
|
18
|
+
|
19
|
+
self.assertIsNotNone(pgm)
|
20
|
+
|
21
|
+
def test_ray_state_api_returns_correctly(self):
|
22
|
+
|
23
|
+
result = _get_available_resources_per_node()
|
24
|
+
|
25
|
+
self.assertIsNotNone(result)
|
@@ -2,6 +2,7 @@ import logging
|
|
2
2
|
import re
|
3
3
|
import time
|
4
4
|
from dataclasses import dataclass
|
5
|
+
from packaging.version import Version
|
5
6
|
from typing import Any, Dict, List, Optional, Tuple, Union
|
6
7
|
|
7
8
|
import ray
|
@@ -19,6 +20,16 @@ logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
|
|
19
20
|
# Issue: https://github.com/ray-project/ray/issues/29959
|
20
21
|
|
21
22
|
|
23
|
+
def _get_available_resources_per_node():
|
24
|
+
# This API changed after this commit
|
25
|
+
# https://github.com/ray-project/ray/pull/43252
|
26
|
+
# TODO: Use this method from a durable State API once it's available
|
27
|
+
if Version(ray.__version__) >= Version("2.10.0"):
|
28
|
+
return ray._private.state.available_resources_per_node()
|
29
|
+
else:
|
30
|
+
return ray._private.state.state._available_resources_per_node()
|
31
|
+
|
32
|
+
|
22
33
|
@dataclass
|
23
34
|
class PlacementGroupConfig:
|
24
35
|
def __init__(self, opts, resource, node_ips):
|
@@ -90,9 +101,7 @@ class NodeGroupManager:
|
|
90
101
|
Returns:
|
91
102
|
group_res: a dict of resources, e.g., {'CPU':0,'memory':0,'object_store_memory':0}
|
92
103
|
"""
|
93
|
-
all_available_resources = (
|
94
|
-
ray._private.state.state._available_resources_per_node()
|
95
|
-
)
|
104
|
+
all_available_resources = _get_available_resources_per_node()
|
96
105
|
group_keys = [x[0] for x in self.init_groups]
|
97
106
|
group_res = {}
|
98
107
|
for k in group_keys:
|
@@ -127,9 +136,7 @@ class NodeGroupManager:
|
|
127
136
|
Returns:
|
128
137
|
group_res: dict of updated resource(cpu, memory, object store memory) for a given group
|
129
138
|
"""
|
130
|
-
all_available_resources = (
|
131
|
-
ray._private.state.state._available_resources_per_node()
|
132
|
-
)
|
139
|
+
all_available_resources = _get_available_resources_per_node()
|
133
140
|
group_res = {"CPU": 0, "memory": 0, "object_store_memory": 0, "node_id": []}
|
134
141
|
for v in all_available_resources.values():
|
135
142
|
keys = v.keys()
|
@@ -285,7 +292,7 @@ def _config(
|
|
285
292
|
for bd in bundles:
|
286
293
|
node_ids.append(bd["node_id"])
|
287
294
|
# query available resources given list of node id
|
288
|
-
all_nodes_available_res =
|
295
|
+
all_nodes_available_res = _get_available_resources_per_node()
|
289
296
|
pg_res = {"CPU": 0, "memory": 0, "object_store_memory": 0}
|
290
297
|
node_ips = []
|
291
298
|
for node_id in node_ids:
|
@@ -141,8 +141,10 @@ deltacat/storage/model/table.py
|
|
141
141
|
deltacat/storage/model/table_version.py
|
142
142
|
deltacat/storage/model/types.py
|
143
143
|
deltacat/tests/__init__.py
|
144
|
+
deltacat/tests/test_logs.py
|
144
145
|
deltacat/tests/aws/__init__.py
|
145
146
|
deltacat/tests/aws/test_clients.py
|
147
|
+
deltacat/tests/aws/test_s3u.py
|
146
148
|
deltacat/tests/catalog/__init__.py
|
147
149
|
deltacat/tests/catalog/test_default_catalog_impl.py
|
148
150
|
deltacat/tests/compute/__init__.py
|
@@ -183,6 +185,7 @@ deltacat/tests/utils/__init__.py
|
|
183
185
|
deltacat/tests/utils/test_cloudpickle.py
|
184
186
|
deltacat/tests/utils/test_daft.py
|
185
187
|
deltacat/tests/utils/test_metrics.py
|
188
|
+
deltacat/tests/utils/test_placement.py
|
186
189
|
deltacat/tests/utils/test_pyarrow.py
|
187
190
|
deltacat/tests/utils/test_record_batch_tables.py
|
188
191
|
deltacat/tests/utils/test_resources.py
|