qwak-core 0.4.251__tar.gz → 0.4.253__tar.gz

{qwak_core-0.4.251 → qwak_core-0.4.253}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: qwak-core
-Version: 0.4.251
+Version: 0.4.253
 Summary: Qwak Core contains the necessary objects and communication tools for using the Qwak Platform
 License: Apache-2.0
 Keywords: mlops,ml,deployment,serving,model
@@ -18,6 +18,7 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Provides-Extra: feature-store
+Provides-Extra: local-core-dependencies
 Requires-Dist: PyYAML
 Requires-Dist: cachetools
 Requires-Dist: chevron (==0.14.0)
@@ -27,8 +28,11 @@ Requires-Dist: dependency-injector (>=4.0)
 Requires-Dist: filelock
 Requires-Dist: frogml-storage (>=0.11.2)
 Requires-Dist: grpcio (>=1.57.0)
+Requires-Dist: grpcio-tools (>=1.47.0) ; (python_full_version >= "3.7.1" and python_version < "3.10") and (extra == "local-core-dependencies")
+Requires-Dist: grpcio-tools (>=1.56.2) ; (python_version >= "3.10") and (extra == "local-core-dependencies")
 Requires-Dist: joblib (>=1.3.2,<2.0.0)
 Requires-Dist: marshmallow-dataclass (>=8.5.8,<9.0.0)
+Requires-Dist: mypy-protobuf (>=3.0.0,<4.0.0) ; extra == "local-core-dependencies"
 Requires-Dist: protobuf (>=3.10,<4) ; python_full_version >= "3.7.1" and python_version < "3.10"
 Requires-Dist: protobuf (>=4.21.6) ; python_version >= "3.10"
 Requires-Dist: pyarrow (>=6.0.0) ; extra == "feature-store"

{qwak_core-0.4.251 → qwak_core-0.4.253}/_qwak_proto/qwak/kube_deployment_captain/batch_job_pb2.py

@@ -15,7 +15,7 @@ _sym_db = _symbol_database.Default()
 
 
 
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n,qwak/kube_deployment_captain/batch_job.proto\x12\x1cqwak.kube.deployment.captain\"\xd2\x03\n\x1cListInferenceJobFilesRequest\x12\x0e\n\x06\x62ucket\x18\x01 \x01(\t\x12\x16\n\x0e\x64irectory_path\x18\x02 \x01(\t\x12\x18\n\x0ctoken_secret\x18\x03 \x01(\tB\x02\x18\x01\x12\x19\n\rsecret_secret\x18\x04 \x01(\tB\x02\x18\x01\x12\x1a\n\x12secret_service_url\x18\x05 \x01(\t\x12\x14\n\x08role_arn\x18\x06 \x01(\tB\x02\x18\x01\x12G\n\x0f\x61ws_credentials\x18\x07 \x01(\x0b\x32,.qwak.kube.deployment.captain.AwsCredentialsH\x00\x12G\n\x0fgcp_credentials\x18\x08 \x01(\x0b\x32,.qwak.kube.deployment.captain.GcpCredentialsH\x00\x12P\n\x14grouped_task_details\x18\t \x01(\x0b\x32\x30.qwak.kube.deployment.captain.GroupedTaskDetailsH\x01\x12\x16\n\x0e\x65nvironment_id\x18\n \x01(\tB\x1a\n\x18\x63loud_client_credentialsB\x0b\n\ttask_flow\"c\n\x12GroupedTaskDetails\x12\x17\n\x0f\x63oncurrentTasks\x18\x01 \x01(\x05\x12\x18\n\x10\x64\x65stination_path\x18\x02 \x01(\t\x12\x1a\n\x12\x64\x65stination_bucket\x18\x03 \x01(\t\"\xb4\x02\n)CleanStorageObjectFromCloudStorageRequest\x12\x0e\n\x06\x62ucket\x18\x01 \x01(\t\x12\x15\n\robject_prefix\x18\x02 \x03(\t\x12\x1a\n\x12secret_service_url\x18\x03 \x01(\t\x12G\n\x0f\x61ws_credentials\x18\x04 \x01(\x0b\x32,.qwak.kube.deployment.captain.AwsCredentialsH\x00\x12G\n\x0fgcp_credentials\x18\x05 \x01(\x0b\x32,.qwak.kube.deployment.captain.GcpCredentialsH\x00\x12\x16\n\x0e\x65nvironment_id\x18\x06 \x01(\tB\x1a\n\x18\x63loud_client_credentials\"O\n\x0e\x41wsCredentials\x12\x10\n\x08role_arn\x18\x01 \x01(\t\x12\x14\n\x0ctoken_secret\x18\x02 \x01(\t\x12\x15\n\rsecret_secret\x18\x04 \x01(\t\"9\n\x0eGcpCredentials\x12\'\n\x1fservice_account_json_key_secret\x18\x01 \x01(\t\"\xa6\x01\n\x1dListInferenceJobFilesResponse\x12\x12\n\nfile_names\x18\x01 \x03(\t\x12\x0f\n\x07success\x18\x02 \x01(\x08\x12\x16\n\x0e\x66\x61ilure_reason\x18\x03 \x01(\t\x12H\n\x11list_file_details\x18\x04 \x03(\x0b\x32-.qwak.kube.deployment.captain.ListFileDetails\"W\n\x0fListFileDetails\x12\x15\n\ris_point_file\x18\x01 \x01(\x08\x12\x17\n\x0fpoint_file_path\x18\x02 \x01(\t\x12\x14\n\x0cobject_paths\x18\x03 \x03(\t\"\xf0\x01\n\"CreateInferenceTaskExecutorRequest\x12\x63\n\x1btask_executor_configuration\x18\x01 \x01(\x0b\x32>.qwak.kube.deployment.captain.TaskExecutorConfigurationMessage\x12\x65\n\x1cinference_task_configuration\x18\x02 \x01(\x0b\x32?.qwak.kube.deployment.captain.InferenceTaskConfigurationMessage\"\x89\x01\n\x1aPrepareInferenceJobRequest\x12k\n#inference_job_configuration_message\x18\x01 \x01(\x0b\x32>.qwak.kube.deployment.captain.InferenceJobConfigurationMessage\"F\n\x1bPrepareInferenceJobResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x16\n\x0e\x66\x61ilure_reason\x18\x02 \x01(\t\"\xcd\x04\n TaskExecutorConfigurationMessage\x12\x18\n\x10inference_job_id\x18\x01 \x01(\t\x12\x19\n\x11inference_task_id\x18\x02 \x01(\t\x12G\n\x10model_identifier\x18\x05 \x01(\x0b\x32-.qwak.kube.deployment.captain.ModelIdentifier\x12\x11\n\timage_url\x18\x06 \x01(\t\x12\x15\n\rbackoff_limit\x18\x07 \x01(\x05\x12\x0f\n\x03\x63pu\x18\x08 \x01(\x02\x42\x02\x18\x01\x12\x19\n\rmemory_amount\x18\t \x01(\x05\x42\x02\x18\x01\x12\x45\n\x0cmemory_units\x18\n \x01(\x0e\x32+.qwak.kube.deployment.captain.MemoryUnitApiB\x02\x18\x01\x12\x16\n\x0e\x65nvironment_id\x18\x0b \x01(\t\x12\x1b\n\x13\x63ustom_iam_role_arn\x18\x0c \x01(\t\x12\x41\n\x08job_size\x18\r \x01(\x0b\x32/.qwak.kube.deployment.captain.BatchJobResources\x12\x17\n\x0fpurchase_option\x18\x0e \x01(\t\x12\x19\n\x11image_pull_secret\x18\x0f \x01(\t\x12%\n\x1d\x63ustom_service_account_secret\x18\x10 \x01(\t\x12#\n\x1bjfrog_token_api_secret_name\x18\x11 \x01(\t\x12\x16\n\x0ejfrog_base_url\x18\x12 \x01(\t\"q\n InferenceJobConfigurationMessage\x12\x18\n\x10inference_job_id\x18\x01 \x01(\t\x12\x16\n\x0e\x65nvironment_id\x18\x02 \x01(\t\x12\x1b\n\x13\x63ustom_iam_role_arn\x18\x03 \x01(\t\"`\n\x0fModelIdentifier\x12\x10\n\x08model_id\x18\x01 \x01(\t\x12\x10\n\x08\x62uild_id\x18\x02 \x01(\t\x12\x15\n\tbranch_id\x18\x03 \x01(\tB\x02\x18\x01\x12\x12\n\nmodel_uuid\x18\x04 \x01(\t\"\x8c\x04\n!InferenceTaskConfigurationMessage\x12\x15\n\rsource_bucket\x18\x01 \x01(\t\x12\x1a\n\x12\x64\x65stination_bucket\x18\x02 \x01(\t\x12\x10\n\x08\x66ilepath\x18\x03 \x01(\t\x12\x18\n\x10\x64\x65stination_path\x18\x04 \x01(\t\x12\x44\n\x0finput_file_type\x18\x05 \x01(\x0e\x32+.qwak.kube.deployment.captain.InputFileType\x12\x46\n\x10output_file_type\x18\x06 \x01(\x0e\x32,.qwak.kube.deployment.captain.OutputFileType\x12\x14\n\x0ctoken_secret\x18\x07 \x01(\t\x12\x15\n\rsecret_secret\x18\x08 \x01(\t\x12\x43\n\nparameters\x18\t \x03(\x0b\x32/.qwak.kube.deployment.captain.BatchJobParameter\x12\x43\n\x0e\x63loud_provider\x18\n \x01(\x0e\x32+.qwak.kube.deployment.captain.CloudProvider\x12\'\n\x1fservice_account_json_key_secret\x18\x0b \x01(\t\x12\x1a\n\x12is_point_file_path\x18\x0c \x01(\x08\"/\n\x11\x42\x61tchJobParameter\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"?\n#CleanInferenceTasksExecutorsRequest\x12\x18\n\x10inference_job_id\x18\x01 \x01(\t\"O\n$CleanInferenceTasksExecutorsResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x16\n\x0e\x66\x61ilure_reason\x18\x02 \x01(\t\"X\n!CleanInferenceTaskExecutorRequest\x12\x18\n\x10inference_job_id\x18\x01 \x01(\t\x12\x19\n\x11inference_task_id\x18\x02 \x01(\t\"M\n\"CleanInferenceTaskExecutorResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x16\n\x0e\x66\x61ilure_reason\x18\x02 \x01(\t\"N\n#CreateInferenceTaskExecutorResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x16\n\x0e\x66\x61ilure_reason\x18\x02 \x01(\t\"\xf7\x02\n\x1eStartInferenceJobWarmupRequest\x12\x10\n\x08model_id\x18\x01 \x01(\t\x12\x15\n\tbranch_id\x18\x02 \x01(\tB\x02\x18\x01\x12\x10\n\x08\x62uild_id\x18\x03 \x01(\t\x12\x11\n\timage_url\x18\x04 \x01(\t\x12\x0f\n\x03\x63pu\x18\x05 \x01(\x02\x42\x02\x18\x01\x12\x19\n\rmemory_amount\x18\x06 \x01(\x05\x42\x02\x18\x01\x12\x45\n\x0cmemory_units\x18\x07 \x01(\x0e\x32+.qwak.kube.deployment.captain.MemoryUnitApiB\x02\x18\x01\x12\x11\n\texecutors\x18\x08 \x01(\x05\x12\x0f\n\x07timeout\x18\t \x01(\x05\x12\x41\n\x08job_size\x18\n \x01(\x0b\x32/.qwak.kube.deployment.captain.BatchJobResources\x12\x12\n\nmodel_uuid\x18\x0b \x01(\t\x12\x19\n\x11image_pull_secret\x18\x0c \x01(\t\"!\n\x1fStartInferenceJobWarmupResponse\"p\n\x1f\x43\x61ncelInferenceJobWarmupRequest\x12\x10\n\x08model_id\x18\x01 \x01(\t\x12\x15\n\tbranch_id\x18\x02 \x01(\tB\x02\x18\x01\x12\x10\n\x08\x62uild_id\x18\x03 \x01(\t\x12\x12\n\nmodel_uuid\x18\x04 \x01(\t\"\"\n CancelInferenceJobWarmupResponse\"\xd5\x01\n\x11\x42\x61tchJobResources\x12\x16\n\x0enumber_of_pods\x18\x01 \x01(\x05\x12\x0b\n\x03\x63pu\x18\x02 \x01(\x02\x12\x15\n\rmemory_amount\x18\x03 \x01(\x05\x12\x41\n\x0cmemory_units\x18\x04 \x01(\x0e\x32+.qwak.kube.deployment.captain.MemoryUnitApi\x12\x41\n\rgpu_resources\x18\x05 \x01(\x0b\x32*.qwak.kube.deployment.captain.GpuResources\"[\n\x0cGpuResources\x12\x37\n\x08gpu_type\x18\x01 \x01(\x0e\x32%.qwak.kube.deployment.captain.GpuType\x12\x12\n\ngpu_amount\x18\x02 \x01(\x05*:\n\rMemoryUnitApi\x12\x17\n\x13UNKNOWN_MEMORY_UNIT\x10\x00\x12\x07\n\x03MIB\x10\x01\x12\x07\n\x03GIB\x10\x02*\x81\x01\n\rInputFileType\x12\x1d\n\x19UNDEFINED_INPUT_FILE_TYPE\x10\x00\x12\x17\n\x13\x43SV_INPUT_FILE_TYPE\x10\x01\x12\x1b\n\x17\x46\x45\x41THER_INPUT_FILE_TYPE\x10\x02\x12\x1b\n\x17PARQUET_INPUT_FILE_TYPE\x10\x03*\x86\x01\n\x0eOutputFileType\x12\x1e\n\x1aUNDEFINED_OUTPUT_FILE_TYPE\x10\x00\x12\x18\n\x14\x43SV_OUTPUT_FILE_TYPE\x10\x01\x12\x1c\n\x18\x46\x45\x41THER_OUTPUT_FILE_TYPE\x10\x02\x12\x1c\n\x18PARQUET_OUTPUT_FILE_TYPE\x10\x03*\x90\x02\n\x07GpuType\x12\x0f\n\x0bINVALID_GPU\x10\x00\x12\x0e\n\nNVIDIA_K80\x10\x01\x12\x0f\n\x0bNVIDIA_V100\x10\x02\x12\x0f\n\x0bNVIDIA_A100\x10\x03\x12\r\n\tNVIDIA_T4\x10\x04\x12\x0f\n\x0bNVIDIA_A10G\x10\x05\x12\r\n\tNVIDIA_L4\x10\x06\x12\x14\n\x10NVIDIA_T4_1_4_15\x10\x07\x12\x14\n\x10NVIDIA_T4_1_8_30\x10\x08\x12\x15\n\x11NVIDIA_T4_1_16_60\x10\t\x12\x1e\n\x1aNVIDIA_A100_80GB_8_96_1360\x10\n\x12\x16\n\x12NVIDIA_V100_1_8_52\x10\x0b\x12\x18\n\x14NVIDIA_V100_4_32_208\x10\x0c*=\n\rCloudProvider\x12\x1a\n\x16UNKNOWN_CLOUD_PROVIDER\x10\x00\x12\x07\n\x03\x41WS\x10\x01\x12\x07\n\x03GCP\x10\x02\x42+\n\'com.qwak.ai.kube.deployment.captain.apiP\x01\x62\x06proto3')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n,qwak/kube_deployment_captain/batch_job.proto\x12\x1cqwak.kube.deployment.captain\"\xd2\x03\n\x1cListInferenceJobFilesRequest\x12\x0e\n\x06\x62ucket\x18\x01 \x01(\t\x12\x16\n\x0e\x64irectory_path\x18\x02 \x01(\t\x12\x18\n\x0ctoken_secret\x18\x03 \x01(\tB\x02\x18\x01\x12\x19\n\rsecret_secret\x18\x04 \x01(\tB\x02\x18\x01\x12\x1a\n\x12secret_service_url\x18\x05 \x01(\t\x12\x14\n\x08role_arn\x18\x06 \x01(\tB\x02\x18\x01\x12G\n\x0f\x61ws_credentials\x18\x07 \x01(\x0b\x32,.qwak.kube.deployment.captain.AwsCredentialsH\x00\x12G\n\x0fgcp_credentials\x18\x08 \x01(\x0b\x32,.qwak.kube.deployment.captain.GcpCredentialsH\x00\x12P\n\x14grouped_task_details\x18\t \x01(\x0b\x32\x30.qwak.kube.deployment.captain.GroupedTaskDetailsH\x01\x12\x16\n\x0e\x65nvironment_id\x18\n \x01(\tB\x1a\n\x18\x63loud_client_credentialsB\x0b\n\ttask_flow\"c\n\x12GroupedTaskDetails\x12\x17\n\x0f\x63oncurrentTasks\x18\x01 \x01(\x05\x12\x18\n\x10\x64\x65stination_path\x18\x02 \x01(\t\x12\x1a\n\x12\x64\x65stination_bucket\x18\x03 \x01(\t\"\xb4\x02\n)CleanStorageObjectFromCloudStorageRequest\x12\x0e\n\x06\x62ucket\x18\x01 \x01(\t\x12\x15\n\robject_prefix\x18\x02 \x03(\t\x12\x1a\n\x12secret_service_url\x18\x03 \x01(\t\x12G\n\x0f\x61ws_credentials\x18\x04 \x01(\x0b\x32,.qwak.kube.deployment.captain.AwsCredentialsH\x00\x12G\n\x0fgcp_credentials\x18\x05 \x01(\x0b\x32,.qwak.kube.deployment.captain.GcpCredentialsH\x00\x12\x16\n\x0e\x65nvironment_id\x18\x06 \x01(\tB\x1a\n\x18\x63loud_client_credentials\"O\n\x0e\x41wsCredentials\x12\x10\n\x08role_arn\x18\x01 \x01(\t\x12\x14\n\x0ctoken_secret\x18\x02 \x01(\t\x12\x15\n\rsecret_secret\x18\x04 \x01(\t\"9\n\x0eGcpCredentials\x12\'\n\x1fservice_account_json_key_secret\x18\x01 \x01(\t\"\xa6\x01\n\x1dListInferenceJobFilesResponse\x12\x12\n\nfile_names\x18\x01 \x03(\t\x12\x0f\n\x07success\x18\x02 \x01(\x08\x12\x16\n\x0e\x66\x61ilure_reason\x18\x03 \x01(\t\x12H\n\x11list_file_details\x18\x04 \x03(\x0b\x32-.qwak.kube.deployment.captain.ListFileDetails\"W\n\x0fListFileDetails\x12\x15\n\ris_point_file\x18\x01 \x01(\x08\x12\x17\n\x0fpoint_file_path\x18\x02 \x01(\t\x12\x14\n\x0cobject_paths\x18\x03 \x03(\t\"\xf0\x01\n\"CreateInferenceTaskExecutorRequest\x12\x63\n\x1btask_executor_configuration\x18\x01 \x01(\x0b\x32>.qwak.kube.deployment.captain.TaskExecutorConfigurationMessage\x12\x65\n\x1cinference_task_configuration\x18\x02 \x01(\x0b\x32?.qwak.kube.deployment.captain.InferenceTaskConfigurationMessage\"\x89\x01\n\x1aPrepareInferenceJobRequest\x12k\n#inference_job_configuration_message\x18\x01 \x01(\x0b\x32>.qwak.kube.deployment.captain.InferenceJobConfigurationMessage\"F\n\x1bPrepareInferenceJobResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x16\n\x0e\x66\x61ilure_reason\x18\x02 \x01(\t\"\xcd\x04\n TaskExecutorConfigurationMessage\x12\x18\n\x10inference_job_id\x18\x01 \x01(\t\x12\x19\n\x11inference_task_id\x18\x02 \x01(\t\x12G\n\x10model_identifier\x18\x05 \x01(\x0b\x32-.qwak.kube.deployment.captain.ModelIdentifier\x12\x11\n\timage_url\x18\x06 \x01(\t\x12\x15\n\rbackoff_limit\x18\x07 \x01(\x05\x12\x0f\n\x03\x63pu\x18\x08 \x01(\x02\x42\x02\x18\x01\x12\x19\n\rmemory_amount\x18\t \x01(\x05\x42\x02\x18\x01\x12\x45\n\x0cmemory_units\x18\n \x01(\x0e\x32+.qwak.kube.deployment.captain.MemoryUnitApiB\x02\x18\x01\x12\x16\n\x0e\x65nvironment_id\x18\x0b \x01(\t\x12\x1b\n\x13\x63ustom_iam_role_arn\x18\x0c \x01(\t\x12\x41\n\x08job_size\x18\r \x01(\x0b\x32/.qwak.kube.deployment.captain.BatchJobResources\x12\x17\n\x0fpurchase_option\x18\x0e \x01(\t\x12\x19\n\x11image_pull_secret\x18\x0f \x01(\t\x12%\n\x1d\x63ustom_service_account_secret\x18\x10 \x01(\t\x12#\n\x1bjfrog_token_api_secret_name\x18\x11 \x01(\t\x12\x16\n\x0ejfrog_base_url\x18\x12 \x01(\t\"q\n InferenceJobConfigurationMessage\x12\x18\n\x10inference_job_id\x18\x01 \x01(\t\x12\x16\n\x0e\x65nvironment_id\x18\x02 \x01(\t\x12\x1b\n\x13\x63ustom_iam_role_arn\x18\x03 \x01(\t\"`\n\x0fModelIdentifier\x12\x10\n\x08model_id\x18\x01 \x01(\t\x12\x10\n\x08\x62uild_id\x18\x02 \x01(\t\x12\x15\n\tbranch_id\x18\x03 \x01(\tB\x02\x18\x01\x12\x12\n\nmodel_uuid\x18\x04 \x01(\t\"\x85\x01\n\x1dListFilesPreStepConfiguration\x12\x13\n\x0bsource_path\x18\x01 \x01(\t\x12\x1c\n\x14list_files_file_name\x18\x02 \x01(\t\x12\x1d\n\x15total_number_of_tasks\x18\x03 \x01(\x05\x12\x12\n\ntask_index\x18\x04 \x01(\x05\"\xf4\x04\n!InferenceTaskConfigurationMessage\x12\x15\n\rsource_bucket\x18\x01 \x01(\t\x12\x1a\n\x12\x64\x65stination_bucket\x18\x02 \x01(\t\x12\x10\n\x08\x66ilepath\x18\x03 \x01(\t\x12\x18\n\x10\x64\x65stination_path\x18\x04 \x01(\t\x12\x44\n\x0finput_file_type\x18\x05 \x01(\x0e\x32+.qwak.kube.deployment.captain.InputFileType\x12\x46\n\x10output_file_type\x18\x06 \x01(\x0e\x32,.qwak.kube.deployment.captain.OutputFileType\x12\x14\n\x0ctoken_secret\x18\x07 \x01(\t\x12\x15\n\rsecret_secret\x18\x08 \x01(\t\x12\x43\n\nparameters\x18\t \x03(\x0b\x32/.qwak.kube.deployment.captain.BatchJobParameter\x12\x43\n\x0e\x63loud_provider\x18\n \x01(\x0e\x32+.qwak.kube.deployment.captain.CloudProvider\x12\'\n\x1fservice_account_json_key_secret\x18\x0b \x01(\t\x12\x1a\n\x12is_point_file_path\x18\x0c \x01(\x08\x12\x66\n!list_files_pre_step_configuration\x18\r \x01(\x0b\x32;.qwak.kube.deployment.captain.ListFilesPreStepConfiguration\"/\n\x11\x42\x61tchJobParameter\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"?\n#CleanInferenceTasksExecutorsRequest\x12\x18\n\x10inference_job_id\x18\x01 \x01(\t\"O\n$CleanInferenceTasksExecutorsResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x16\n\x0e\x66\x61ilure_reason\x18\x02 \x01(\t\"X\n!CleanInferenceTaskExecutorRequest\x12\x18\n\x10inference_job_id\x18\x01 \x01(\t\x12\x19\n\x11inference_task_id\x18\x02 \x01(\t\"M\n\"CleanInferenceTaskExecutorResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x16\n\x0e\x66\x61ilure_reason\x18\x02 \x01(\t\"N\n#CreateInferenceTaskExecutorResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x16\n\x0e\x66\x61ilure_reason\x18\x02 \x01(\t\"\xf7\x02\n\x1eStartInferenceJobWarmupRequest\x12\x10\n\x08model_id\x18\x01 \x01(\t\x12\x15\n\tbranch_id\x18\x02 \x01(\tB\x02\x18\x01\x12\x10\n\x08\x62uild_id\x18\x03 \x01(\t\x12\x11\n\timage_url\x18\x04 \x01(\t\x12\x0f\n\x03\x63pu\x18\x05 \x01(\x02\x42\x02\x18\x01\x12\x19\n\rmemory_amount\x18\x06 \x01(\x05\x42\x02\x18\x01\x12\x45\n\x0cmemory_units\x18\x07 \x01(\x0e\x32+.qwak.kube.deployment.captain.MemoryUnitApiB\x02\x18\x01\x12\x11\n\texecutors\x18\x08 \x01(\x05\x12\x0f\n\x07timeout\x18\t \x01(\x05\x12\x41\n\x08job_size\x18\n \x01(\x0b\x32/.qwak.kube.deployment.captain.BatchJobResources\x12\x12\n\nmodel_uuid\x18\x0b \x01(\t\x12\x19\n\x11image_pull_secret\x18\x0c \x01(\t\"!\n\x1fStartInferenceJobWarmupResponse\"p\n\x1f\x43\x61ncelInferenceJobWarmupRequest\x12\x10\n\x08model_id\x18\x01 \x01(\t\x12\x15\n\tbranch_id\x18\x02 \x01(\tB\x02\x18\x01\x12\x10\n\x08\x62uild_id\x18\x03 \x01(\t\x12\x12\n\nmodel_uuid\x18\x04 \x01(\t\"\"\n CancelInferenceJobWarmupResponse\"\xd5\x01\n\x11\x42\x61tchJobResources\x12\x16\n\x0enumber_of_pods\x18\x01 \x01(\x05\x12\x0b\n\x03\x63pu\x18\x02 \x01(\x02\x12\x15\n\rmemory_amount\x18\x03 \x01(\x05\x12\x41\n\x0cmemory_units\x18\x04 \x01(\x0e\x32+.qwak.kube.deployment.captain.MemoryUnitApi\x12\x41\n\rgpu_resources\x18\x05 \x01(\x0b\x32*.qwak.kube.deployment.captain.GpuResources\"[\n\x0cGpuResources\x12\x37\n\x08gpu_type\x18\x01 \x01(\x0e\x32%.qwak.kube.deployment.captain.GpuType\x12\x12\n\ngpu_amount\x18\x02 \x01(\x05*:\n\rMemoryUnitApi\x12\x17\n\x13UNKNOWN_MEMORY_UNIT\x10\x00\x12\x07\n\x03MIB\x10\x01\x12\x07\n\x03GIB\x10\x02*\x81\x01\n\rInputFileType\x12\x1d\n\x19UNDEFINED_INPUT_FILE_TYPE\x10\x00\x12\x17\n\x13\x43SV_INPUT_FILE_TYPE\x10\x01\x12\x1b\n\x17\x46\x45\x41THER_INPUT_FILE_TYPE\x10\x02\x12\x1b\n\x17PARQUET_INPUT_FILE_TYPE\x10\x03*\x86\x01\n\x0eOutputFileType\x12\x1e\n\x1aUNDEFINED_OUTPUT_FILE_TYPE\x10\x00\x12\x18\n\x14\x43SV_OUTPUT_FILE_TYPE\x10\x01\x12\x1c\n\x18\x46\x45\x41THER_OUTPUT_FILE_TYPE\x10\x02\x12\x1c\n\x18PARQUET_OUTPUT_FILE_TYPE\x10\x03*\x90\x02\n\x07GpuType\x12\x0f\n\x0bINVALID_GPU\x10\x00\x12\x0e\n\nNVIDIA_K80\x10\x01\x12\x0f\n\x0bNVIDIA_V100\x10\x02\x12\x0f\n\x0bNVIDIA_A100\x10\x03\x12\r\n\tNVIDIA_T4\x10\x04\x12\x0f\n\x0bNVIDIA_A10G\x10\x05\x12\r\n\tNVIDIA_L4\x10\x06\x12\x14\n\x10NVIDIA_T4_1_4_15\x10\x07\x12\x14\n\x10NVIDIA_T4_1_8_30\x10\x08\x12\x15\n\x11NVIDIA_T4_1_16_60\x10\t\x12\x1e\n\x1aNVIDIA_A100_80GB_8_96_1360\x10\n\x12\x16\n\x12NVIDIA_V100_1_8_52\x10\x0b\x12\x18\n\x14NVIDIA_V100_4_32_208\x10\x0c*=\n\rCloudProvider\x12\x1a\n\x16UNKNOWN_CLOUD_PROVIDER\x10\x00\x12\x07\n\x03\x41WS\x10\x01\x12\x07\n\x03GCP\x10\x02\x42+\n\'com.qwak.ai.kube.deployment.captain.apiP\x01\x62\x06proto3')
 
 _MEMORYUNITAPI = DESCRIPTOR.enum_types_by_name['MemoryUnitApi']
 MemoryUnitApi = enum_type_wrapper.EnumTypeWrapper(_MEMORYUNITAPI)
@@ -69,6 +69,7 @@ _PREPAREINFERENCEJOBRESPONSE = DESCRIPTOR.message_types_by_name['PrepareInferenc
 _TASKEXECUTORCONFIGURATIONMESSAGE = DESCRIPTOR.message_types_by_name['TaskExecutorConfigurationMessage']
 _INFERENCEJOBCONFIGURATIONMESSAGE = DESCRIPTOR.message_types_by_name['InferenceJobConfigurationMessage']
 _MODELIDENTIFIER = DESCRIPTOR.message_types_by_name['ModelIdentifier']
+_LISTFILESPRESTEPCONFIGURATION = DESCRIPTOR.message_types_by_name['ListFilesPreStepConfiguration']
 _INFERENCETASKCONFIGURATIONMESSAGE = DESCRIPTOR.message_types_by_name['InferenceTaskConfigurationMessage']
 _BATCHJOBPARAMETER = DESCRIPTOR.message_types_by_name['BatchJobParameter']
 _CLEANINFERENCETASKSEXECUTORSREQUEST = DESCRIPTOR.message_types_by_name['CleanInferenceTasksExecutorsRequest']
@@ -173,6 +174,13 @@ ModelIdentifier = _reflection.GeneratedProtocolMessageType('ModelIdentifier', (_
   })
 _sym_db.RegisterMessage(ModelIdentifier)
 
+ListFilesPreStepConfiguration = _reflection.GeneratedProtocolMessageType('ListFilesPreStepConfiguration', (_message.Message,), {
+  'DESCRIPTOR' : _LISTFILESPRESTEPCONFIGURATION,
+  '__module__' : 'qwak.kube_deployment_captain.batch_job_pb2'
+  # @@protoc_insertion_point(class_scope:qwak.kube.deployment.captain.ListFilesPreStepConfiguration)
+  })
+_sym_db.RegisterMessage(ListFilesPreStepConfiguration)
+
 InferenceTaskConfigurationMessage = _reflection.GeneratedProtocolMessageType('InferenceTaskConfigurationMessage', (_message.Message,), {
   'DESCRIPTOR' : _INFERENCETASKCONFIGURATIONMESSAGE,
   '__module__' : 'qwak.kube_deployment_captain.batch_job_pb2'
@@ -292,16 +300,16 @@ if _descriptor._USE_C_DESCRIPTORS == False:
   _STARTINFERENCEJOBWARMUPREQUEST.fields_by_name['memory_units']._serialized_options = b'\030\001'
   _CANCELINFERENCEJOBWARMUPREQUEST.fields_by_name['branch_id']._options = None
   _CANCELINFERENCEJOBWARMUPREQUEST.fields_by_name['branch_id']._serialized_options = b'\030\001'
-  _MEMORYUNITAPI._serialized_start=4460
-  _MEMORYUNITAPI._serialized_end=4518
-  _INPUTFILETYPE._serialized_start=4521
-  _INPUTFILETYPE._serialized_end=4650
-  _OUTPUTFILETYPE._serialized_start=4653
-  _OUTPUTFILETYPE._serialized_end=4787
-  _GPUTYPE._serialized_start=4790
-  _GPUTYPE._serialized_end=5062
-  _CLOUDPROVIDER._serialized_start=5064
-  _CLOUDPROVIDER._serialized_end=5125
+  _MEMORYUNITAPI._serialized_start=4700
+  _MEMORYUNITAPI._serialized_end=4758
+  _INPUTFILETYPE._serialized_start=4761
+  _INPUTFILETYPE._serialized_end=4890
+  _OUTPUTFILETYPE._serialized_start=4893
+  _OUTPUTFILETYPE._serialized_end=5027
+  _GPUTYPE._serialized_start=5030
+  _GPUTYPE._serialized_end=5302
+  _CLOUDPROVIDER._serialized_start=5304
+  _CLOUDPROVIDER._serialized_end=5365
   _LISTINFERENCEJOBFILESREQUEST._serialized_start=79
   _LISTINFERENCEJOBFILESREQUEST._serialized_end=545
   _GROUPEDTASKDETAILS._serialized_start=547
@@ -328,30 +336,32 @@ if _descriptor._USE_C_DESCRIPTORS == False:
   _INFERENCEJOBCONFIGURATIONMESSAGE._serialized_end=2517
   _MODELIDENTIFIER._serialized_start=2519
   _MODELIDENTIFIER._serialized_end=2615
-  _INFERENCETASKCONFIGURATIONMESSAGE._serialized_start=2618
-  _INFERENCETASKCONFIGURATIONMESSAGE._serialized_end=3142
-  _BATCHJOBPARAMETER._serialized_start=3144
-  _BATCHJOBPARAMETER._serialized_end=3191
-  _CLEANINFERENCETASKSEXECUTORSREQUEST._serialized_start=3193
-  _CLEANINFERENCETASKSEXECUTORSREQUEST._serialized_end=3256
-  _CLEANINFERENCETASKSEXECUTORSRESPONSE._serialized_start=3258
-  _CLEANINFERENCETASKSEXECUTORSRESPONSE._serialized_end=3337
-  _CLEANINFERENCETASKEXECUTORREQUEST._serialized_start=3339
-  _CLEANINFERENCETASKEXECUTORREQUEST._serialized_end=3427
-  _CLEANINFERENCETASKEXECUTORRESPONSE._serialized_start=3429
-  _CLEANINFERENCETASKEXECUTORRESPONSE._serialized_end=3506
-  _CREATEINFERENCETASKEXECUTORRESPONSE._serialized_start=3508
-  _CREATEINFERENCETASKEXECUTORRESPONSE._serialized_end=3586
-  _STARTINFERENCEJOBWARMUPREQUEST._serialized_start=3589
-  _STARTINFERENCEJOBWARMUPREQUEST._serialized_end=3964
-  _STARTINFERENCEJOBWARMUPRESPONSE._serialized_start=3966
-  _STARTINFERENCEJOBWARMUPRESPONSE._serialized_end=3999
-  _CANCELINFERENCEJOBWARMUPREQUEST._serialized_start=4001
-  _CANCELINFERENCEJOBWARMUPREQUEST._serialized_end=4113
-  _CANCELINFERENCEJOBWARMUPRESPONSE._serialized_start=4115
-  _CANCELINFERENCEJOBWARMUPRESPONSE._serialized_end=4149
-  _BATCHJOBRESOURCES._serialized_start=4152
-  _BATCHJOBRESOURCES._serialized_end=4365
-  _GPURESOURCES._serialized_start=4367
-  _GPURESOURCES._serialized_end=4458
+  _LISTFILESPRESTEPCONFIGURATION._serialized_start=2618
+  _LISTFILESPRESTEPCONFIGURATION._serialized_end=2751
+  _INFERENCETASKCONFIGURATIONMESSAGE._serialized_start=2754
+  _INFERENCETASKCONFIGURATIONMESSAGE._serialized_end=3382
+  _BATCHJOBPARAMETER._serialized_start=3384
+  _BATCHJOBPARAMETER._serialized_end=3431
+  _CLEANINFERENCETASKSEXECUTORSREQUEST._serialized_start=3433
+  _CLEANINFERENCETASKSEXECUTORSREQUEST._serialized_end=3496
+  _CLEANINFERENCETASKSEXECUTORSRESPONSE._serialized_start=3498
+  _CLEANINFERENCETASKSEXECUTORSRESPONSE._serialized_end=3577
+  _CLEANINFERENCETASKEXECUTORREQUEST._serialized_start=3579
+  _CLEANINFERENCETASKEXECUTORREQUEST._serialized_end=3667
+  _CLEANINFERENCETASKEXECUTORRESPONSE._serialized_start=3669
+  _CLEANINFERENCETASKEXECUTORRESPONSE._serialized_end=3746
+  _CREATEINFERENCETASKEXECUTORRESPONSE._serialized_start=3748
+  _CREATEINFERENCETASKEXECUTORRESPONSE._serialized_end=3826
+  _STARTINFERENCEJOBWARMUPREQUEST._serialized_start=3829
+  _STARTINFERENCEJOBWARMUPREQUEST._serialized_end=4204
+  _STARTINFERENCEJOBWARMUPRESPONSE._serialized_start=4206
+  _STARTINFERENCEJOBWARMUPRESPONSE._serialized_end=4239
+  _CANCELINFERENCEJOBWARMUPREQUEST._serialized_start=4241
+  _CANCELINFERENCEJOBWARMUPREQUEST._serialized_end=4353
+  _CANCELINFERENCEJOBWARMUPRESPONSE._serialized_start=4355
+  _CANCELINFERENCEJOBWARMUPRESPONSE._serialized_end=4389
+  _BATCHJOBRESOURCES._serialized_start=4392
+  _BATCHJOBRESOURCES._serialized_end=4605
+  _GPURESOURCES._serialized_start=4607
+  _GPURESOURCES._serialized_end=4698
 # @@protoc_insertion_point(module_scope)

{qwak_core-0.4.251 → qwak_core-0.4.253}/_qwak_proto/qwak/kube_deployment_captain/batch_job_pb2.pyi

@@ -524,6 +524,33 @@ class ModelIdentifier(google.protobuf.message.Message):
 
 global___ModelIdentifier = ModelIdentifier
 
+class ListFilesPreStepConfiguration(google.protobuf.message.Message):
+    DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+    SOURCE_PATH_FIELD_NUMBER: builtins.int
+    LIST_FILES_FILE_NAME_FIELD_NUMBER: builtins.int
+    TOTAL_NUMBER_OF_TASKS_FIELD_NUMBER: builtins.int
+    TASK_INDEX_FIELD_NUMBER: builtins.int
+    source_path: builtins.str
+    """Directory path in source bucket of data files to run inference on"""
+    list_files_file_name: builtins.str
+    """Name for the file to save the list of files in the source path"""
+    total_number_of_tasks: builtins.int
+    """Total number of inference tasks, used to divide the files list into similarly sized groups"""
+    task_index: builtins.int
+    """Index of the specific task, used to determine on which files group to run inference on"""
+    def __init__(
+        self,
+        *,
+        source_path: builtins.str = ...,
+        list_files_file_name: builtins.str = ...,
+        total_number_of_tasks: builtins.int = ...,
+        task_index: builtins.int = ...,
+    ) -> None: ...
+    def ClearField(self, field_name: typing_extensions.Literal["list_files_file_name", b"list_files_file_name", "source_path", b"source_path", "task_index", b"task_index", "total_number_of_tasks", b"total_number_of_tasks"]) -> None: ...
+
+global___ListFilesPreStepConfiguration = ListFilesPreStepConfiguration
+
 class InferenceTaskConfigurationMessage(google.protobuf.message.Message):
     DESCRIPTOR: google.protobuf.descriptor.Descriptor
 
@@ -539,6 +566,7 @@ class InferenceTaskConfigurationMessage(google.protobuf.message.Message):
     CLOUD_PROVIDER_FIELD_NUMBER: builtins.int
     SERVICE_ACCOUNT_JSON_KEY_SECRET_FIELD_NUMBER: builtins.int
     IS_POINT_FILE_PATH_FIELD_NUMBER: builtins.int
+    LIST_FILES_PRE_STEP_CONFIGURATION_FIELD_NUMBER: builtins.int
     source_bucket: builtins.str
     """The source bucket in cloud storage"""
     destination_bucket: builtins.str
@@ -564,6 +592,9 @@ class InferenceTaskConfigurationMessage(google.protobuf.message.Message):
     """The service account json key secret"""
     is_point_file_path: builtins.bool
     """Is the file contains points to inference files"""
+    @property
+    def list_files_pre_step_configuration(self) -> global___ListFilesPreStepConfiguration:
+        """Optional pre step configuration, used to list files in the source directory and run inference on subset of files"""
     def __init__(
         self,
         *,
@@ -579,8 +610,10 @@ class InferenceTaskConfigurationMessage(google.protobuf.message.Message):
         cloud_provider: global___CloudProvider.ValueType = ...,
         service_account_json_key_secret: builtins.str = ...,
         is_point_file_path: builtins.bool = ...,
+        list_files_pre_step_configuration: global___ListFilesPreStepConfiguration | None = ...,
     ) -> None: ...
-    def ClearField(self, field_name: typing_extensions.Literal["cloud_provider", b"cloud_provider", "destination_bucket", b"destination_bucket", "destination_path", b"destination_path", "filepath", b"filepath", "input_file_type", b"input_file_type", "is_point_file_path", b"is_point_file_path", "output_file_type", b"output_file_type", "parameters", b"parameters", "secret_secret", b"secret_secret", "service_account_json_key_secret", b"service_account_json_key_secret", "source_bucket", b"source_bucket", "token_secret", b"token_secret"]) -> None: ...
+    def HasField(self, field_name: typing_extensions.Literal["list_files_pre_step_configuration", b"list_files_pre_step_configuration"]) -> builtins.bool: ...
+    def ClearField(self, field_name: typing_extensions.Literal["cloud_provider", b"cloud_provider", "destination_bucket", b"destination_bucket", "destination_path", b"destination_path", "filepath", b"filepath", "input_file_type", b"input_file_type", "is_point_file_path", b"is_point_file_path", "list_files_pre_step_configuration", b"list_files_pre_step_configuration", "output_file_type", b"output_file_type", "parameters", b"parameters", "secret_secret", b"secret_secret", "service_account_json_key_secret", b"service_account_json_key_secret", "source_bucket", b"source_bucket", "token_secret", b"token_secret"]) -> None: ...
 
 global___InferenceTaskConfigurationMessage = InferenceTaskConfigurationMessage
 

{qwak_core-0.4.251 → qwak_core-0.4.253}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "qwak-core"
-version = "0.4.251"
+version = "0.4.253"
 description = "Qwak Core contains the necessary objects and communication tools for using the Qwak Platform"
 authors = ["Qwak <info@qwak.com>"]
 readme = "README.md"
@@ -55,6 +55,11 @@ chevron = "==0.14.0"
 dacite = "==1.8.1"
 cachetools = "*"
 frogml-storage = ">=0.11.2"
+grpcio-tools = [
+    { version = ">=1.47.0", python = ">=3.7.1,<3.10", optional = true },
+    { version = ">=1.56.2", python = ">=3.10", optional = true },
+]
+mypy-protobuf = { version = "^3.0.0", optional = true }
 
 [tool.poetry.group.dev.dependencies]
 pyarrow = { version = ">=6.0.0" }
@@ -104,6 +109,7 @@ croniter = "==1.4.1"
 
 [tool.poetry.extras]
 feature-store = ["pyarrow", "pyathena", "cloudpickle", "pyspark" ,"pandas"]
+local-core-dependencies = ["grpcio-tools", "mypy-protobuf"]
 
 [tool.black]
 extend-exclude = '''

{qwak_core-0.4.251 → qwak_core-0.4.253}/qwak/__init__.py

@@ -1,7 +1,7 @@
 """Top-level package for qwak-core."""
 
 __author__ = "Qwak.ai"
-__version__ = "0.4.251"
+__version__ = "0.4.253"
 
 from qwak.inner.di_configuration import wire_dependencies
 from qwak.model.experiment_tracking import log_metric, log_param

{qwak_core-0.4.251 → qwak_core-0.4.253}/qwak/model_loggers/artifact_logger.py

@@ -1,9 +1,12 @@
-import urllib.request
+from datetime import timedelta
 from typing import Optional
+import requests
+from requests import Response
 
 from _qwak_proto.qwak.builds.build_url_pb2 import BuildVersioningTagsType
 from _qwak_proto.qwak.builds.builds_orchestrator_service_pb2 import (
     GetBuildVersioningUploadURLResponse,
+    GetBuildVersioningDownloadURLResponse,
 )
 from qwak.clients.build_orchestrator.client import BuildOrchestratorClient
 from qwak.clients.file_versioning.client import FileVersioningManagementClient
@@ -15,6 +18,8 @@ from qwak.inner.model_loggers_utils import (
     validate_tag,
 )
 
+MAX_CHUNK_SIZE = 8 * 1024
+
 
 def log_file(
     from_path: str,
@@ -80,21 +85,40 @@ def load_file(
     Returns:
         the path to the newly created data file
     """
+    print(f"Loading file to {to_path}")
     if not validate_tag(tag):
         raise QwakException(
             "Tag should contain only letters, numbers, underscore or hyphen"
         )
 
     model_id = validate_model(model_id)
-    download_url_response = BuildOrchestratorClient().get_build_versioning_download_url(
-        build_id=build_id, model_id=model_id, tag=tag
+    download_url_response: GetBuildVersioningDownloadURLResponse = (
+        BuildOrchestratorClient().get_build_versioning_download_url(
+            build_id=build_id,
+            model_id=model_id,
+            tag=tag,
+            tag_type=BuildVersioningTagsType.FILE_TAG_TYPE,
+        )
     )
 
     try:
-        filename, headers = urllib.request.urlretrieve(  # nosec B310
-            url=download_url_response.download_url, filename=to_path
+        response: Response = requests.get(
+            download_url_response.download_url,
+            headers=download_url_response.headers,
+            stream=True,
+            timeout=(
+                timedelta(seconds=10).total_seconds(),  # timeout to connect
+                timedelta(minutes=20).total_seconds(),  # timeout to read
+            ),
         )
+        print(f"Downloading file finished with status {response.status_code}")
+        response.raise_for_status()
+
+        with open(to_path, "wb") as f:
+            for chunk in response.iter_content(chunk_size=MAX_CHUNK_SIZE):
+                if chunk:
+                    f.write(chunk)
 
-        return filename
+        return to_path
     except Exception as error:
         raise QwakException(f"Unable to load save artifact locally: {str(error)}")