qwak-core 0.4.251__py3-none-any.whl → 0.4.253__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _qwak_proto/qwak/kube_deployment_captain/batch_job_pb2.py +47 -37
- _qwak_proto/qwak/kube_deployment_captain/batch_job_pb2.pyi +34 -1
- qwak/__init__.py +1 -1
- qwak/model_loggers/artifact_logger.py +30 -6
- {qwak_core-0.4.251.dist-info → qwak_core-0.4.253.dist-info}/METADATA +5 -1
- {qwak_core-0.4.251.dist-info → qwak_core-0.4.253.dist-info}/RECORD +7 -7
- {qwak_core-0.4.251.dist-info → qwak_core-0.4.253.dist-info}/WHEEL +0 -0
@@ -15,7 +15,7 @@ _sym_db = _symbol_database.Default()
|
|
15
15
|
|
16
16
|
|
17
17
|
|
18
|
-
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n,qwak/kube_deployment_captain/batch_job.proto\x12\x1cqwak.kube.deployment.captain\"\xd2\x03\n\x1cListInferenceJobFilesRequest\x12\x0e\n\x06\x62ucket\x18\x01 \x01(\t\x12\x16\n\x0e\x64irectory_path\x18\x02 \x01(\t\x12\x18\n\x0ctoken_secret\x18\x03 \x01(\tB\x02\x18\x01\x12\x19\n\rsecret_secret\x18\x04 \x01(\tB\x02\x18\x01\x12\x1a\n\x12secret_service_url\x18\x05 \x01(\t\x12\x14\n\x08role_arn\x18\x06 \x01(\tB\x02\x18\x01\x12G\n\x0f\x61ws_credentials\x18\x07 \x01(\x0b\x32,.qwak.kube.deployment.captain.AwsCredentialsH\x00\x12G\n\x0fgcp_credentials\x18\x08 \x01(\x0b\x32,.qwak.kube.deployment.captain.GcpCredentialsH\x00\x12P\n\x14grouped_task_details\x18\t \x01(\x0b\x32\x30.qwak.kube.deployment.captain.GroupedTaskDetailsH\x01\x12\x16\n\x0e\x65nvironment_id\x18\n \x01(\tB\x1a\n\x18\x63loud_client_credentialsB\x0b\n\ttask_flow\"c\n\x12GroupedTaskDetails\x12\x17\n\x0f\x63oncurrentTasks\x18\x01 \x01(\x05\x12\x18\n\x10\x64\x65stination_path\x18\x02 \x01(\t\x12\x1a\n\x12\x64\x65stination_bucket\x18\x03 \x01(\t\"\xb4\x02\n)CleanStorageObjectFromCloudStorageRequest\x12\x0e\n\x06\x62ucket\x18\x01 \x01(\t\x12\x15\n\robject_prefix\x18\x02 \x03(\t\x12\x1a\n\x12secret_service_url\x18\x03 \x01(\t\x12G\n\x0f\x61ws_credentials\x18\x04 \x01(\x0b\x32,.qwak.kube.deployment.captain.AwsCredentialsH\x00\x12G\n\x0fgcp_credentials\x18\x05 \x01(\x0b\x32,.qwak.kube.deployment.captain.GcpCredentialsH\x00\x12\x16\n\x0e\x65nvironment_id\x18\x06 \x01(\tB\x1a\n\x18\x63loud_client_credentials\"O\n\x0e\x41wsCredentials\x12\x10\n\x08role_arn\x18\x01 \x01(\t\x12\x14\n\x0ctoken_secret\x18\x02 \x01(\t\x12\x15\n\rsecret_secret\x18\x04 \x01(\t\"9\n\x0eGcpCredentials\x12\'\n\x1fservice_account_json_key_secret\x18\x01 \x01(\t\"\xa6\x01\n\x1dListInferenceJobFilesResponse\x12\x12\n\nfile_names\x18\x01 \x03(\t\x12\x0f\n\x07success\x18\x02 \x01(\x08\x12\x16\n\x0e\x66\x61ilure_reason\x18\x03 \x01(\t\x12H\n\x11list_file_details\x18\x04 \x03(\x0b\x32-.qwak.kube.deployment.captain.ListFileDetails\"W\n\x0fListFileDetails\x12\x15\n\ris_point_file\x18\x01 \x01(\x08\x12\x17\n\x0fpoint_file_path\x18\x02 \x01(\t\x12\x14\n\x0cobject_paths\x18\x03 \x03(\t\"\xf0\x01\n\"CreateInferenceTaskExecutorRequest\x12\x63\n\x1btask_executor_configuration\x18\x01 \x01(\x0b\x32>.qwak.kube.deployment.captain.TaskExecutorConfigurationMessage\x12\x65\n\x1cinference_task_configuration\x18\x02 \x01(\x0b\x32?.qwak.kube.deployment.captain.InferenceTaskConfigurationMessage\"\x89\x01\n\x1aPrepareInferenceJobRequest\x12k\n#inference_job_configuration_message\x18\x01 \x01(\x0b\x32>.qwak.kube.deployment.captain.InferenceJobConfigurationMessage\"F\n\x1bPrepareInferenceJobResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x16\n\x0e\x66\x61ilure_reason\x18\x02 \x01(\t\"\xcd\x04\n TaskExecutorConfigurationMessage\x12\x18\n\x10inference_job_id\x18\x01 \x01(\t\x12\x19\n\x11inference_task_id\x18\x02 \x01(\t\x12G\n\x10model_identifier\x18\x05 \x01(\x0b\x32-.qwak.kube.deployment.captain.ModelIdentifier\x12\x11\n\timage_url\x18\x06 \x01(\t\x12\x15\n\rbackoff_limit\x18\x07 \x01(\x05\x12\x0f\n\x03\x63pu\x18\x08 \x01(\x02\x42\x02\x18\x01\x12\x19\n\rmemory_amount\x18\t \x01(\x05\x42\x02\x18\x01\x12\x45\n\x0cmemory_units\x18\n \x01(\x0e\x32+.qwak.kube.deployment.captain.MemoryUnitApiB\x02\x18\x01\x12\x16\n\x0e\x65nvironment_id\x18\x0b \x01(\t\x12\x1b\n\x13\x63ustom_iam_role_arn\x18\x0c \x01(\t\x12\x41\n\x08job_size\x18\r \x01(\x0b\x32/.qwak.kube.deployment.captain.BatchJobResources\x12\x17\n\x0fpurchase_option\x18\x0e \x01(\t\x12\x19\n\x11image_pull_secret\x18\x0f \x01(\t\x12%\n\x1d\x63ustom_service_account_secret\x18\x10 \x01(\t\x12#\n\x1bjfrog_token_api_secret_name\x18\x11 \x01(\t\x12\x16\n\x0ejfrog_base_url\x18\x12 \x01(\t\"q\n InferenceJobConfigurationMessage\x12\x18\n\x10inference_job_id\x18\x01 \x01(\t\x12\x16\n\x0e\x65nvironment_id\x18\x02 \x01(\t\x12\x1b\n\x13\x63ustom_iam_role_arn\x18\x03 \x01(\t\"`\n\x0fModelIdentifier\x12\x10\n\x08model_id\x18\x01 \x01(\t\x12\x10\n\x08\x62uild_id\x18\x02 \x01(\t\x12\x15\n\tbranch_id\x18\x03 \x01(\tB\x02\x18\x01\x12\x12\n\nmodel_uuid\x18\x04 \x01(\t\"\
|
18
|
+
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n,qwak/kube_deployment_captain/batch_job.proto\x12\x1cqwak.kube.deployment.captain\"\xd2\x03\n\x1cListInferenceJobFilesRequest\x12\x0e\n\x06\x62ucket\x18\x01 \x01(\t\x12\x16\n\x0e\x64irectory_path\x18\x02 \x01(\t\x12\x18\n\x0ctoken_secret\x18\x03 \x01(\tB\x02\x18\x01\x12\x19\n\rsecret_secret\x18\x04 \x01(\tB\x02\x18\x01\x12\x1a\n\x12secret_service_url\x18\x05 \x01(\t\x12\x14\n\x08role_arn\x18\x06 \x01(\tB\x02\x18\x01\x12G\n\x0f\x61ws_credentials\x18\x07 \x01(\x0b\x32,.qwak.kube.deployment.captain.AwsCredentialsH\x00\x12G\n\x0fgcp_credentials\x18\x08 \x01(\x0b\x32,.qwak.kube.deployment.captain.GcpCredentialsH\x00\x12P\n\x14grouped_task_details\x18\t \x01(\x0b\x32\x30.qwak.kube.deployment.captain.GroupedTaskDetailsH\x01\x12\x16\n\x0e\x65nvironment_id\x18\n \x01(\tB\x1a\n\x18\x63loud_client_credentialsB\x0b\n\ttask_flow\"c\n\x12GroupedTaskDetails\x12\x17\n\x0f\x63oncurrentTasks\x18\x01 \x01(\x05\x12\x18\n\x10\x64\x65stination_path\x18\x02 \x01(\t\x12\x1a\n\x12\x64\x65stination_bucket\x18\x03 \x01(\t\"\xb4\x02\n)CleanStorageObjectFromCloudStorageRequest\x12\x0e\n\x06\x62ucket\x18\x01 \x01(\t\x12\x15\n\robject_prefix\x18\x02 \x03(\t\x12\x1a\n\x12secret_service_url\x18\x03 \x01(\t\x12G\n\x0f\x61ws_credentials\x18\x04 \x01(\x0b\x32,.qwak.kube.deployment.captain.AwsCredentialsH\x00\x12G\n\x0fgcp_credentials\x18\x05 \x01(\x0b\x32,.qwak.kube.deployment.captain.GcpCredentialsH\x00\x12\x16\n\x0e\x65nvironment_id\x18\x06 \x01(\tB\x1a\n\x18\x63loud_client_credentials\"O\n\x0e\x41wsCredentials\x12\x10\n\x08role_arn\x18\x01 \x01(\t\x12\x14\n\x0ctoken_secret\x18\x02 \x01(\t\x12\x15\n\rsecret_secret\x18\x04 \x01(\t\"9\n\x0eGcpCredentials\x12\'\n\x1fservice_account_json_key_secret\x18\x01 \x01(\t\"\xa6\x01\n\x1dListInferenceJobFilesResponse\x12\x12\n\nfile_names\x18\x01 \x03(\t\x12\x0f\n\x07success\x18\x02 \x01(\x08\x12\x16\n\x0e\x66\x61ilure_reason\x18\x03 \x01(\t\x12H\n\x11list_file_details\x18\x04 \x03(\x0b\x32-.qwak.kube.deployment.captain.ListFileDetails\"W\n\x0fListFileDetails\x12\x15\n\ris_point_file\x18\x01 \x01(\x08\x12\x17\n\x0fpoint_file_path\x18\x02 \x01(\t\x12\x14\n\x0cobject_paths\x18\x03 \x03(\t\"\xf0\x01\n\"CreateInferenceTaskExecutorRequest\x12\x63\n\x1btask_executor_configuration\x18\x01 \x01(\x0b\x32>.qwak.kube.deployment.captain.TaskExecutorConfigurationMessage\x12\x65\n\x1cinference_task_configuration\x18\x02 \x01(\x0b\x32?.qwak.kube.deployment.captain.InferenceTaskConfigurationMessage\"\x89\x01\n\x1aPrepareInferenceJobRequest\x12k\n#inference_job_configuration_message\x18\x01 \x01(\x0b\x32>.qwak.kube.deployment.captain.InferenceJobConfigurationMessage\"F\n\x1bPrepareInferenceJobResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x16\n\x0e\x66\x61ilure_reason\x18\x02 \x01(\t\"\xcd\x04\n TaskExecutorConfigurationMessage\x12\x18\n\x10inference_job_id\x18\x01 \x01(\t\x12\x19\n\x11inference_task_id\x18\x02 \x01(\t\x12G\n\x10model_identifier\x18\x05 \x01(\x0b\x32-.qwak.kube.deployment.captain.ModelIdentifier\x12\x11\n\timage_url\x18\x06 \x01(\t\x12\x15\n\rbackoff_limit\x18\x07 \x01(\x05\x12\x0f\n\x03\x63pu\x18\x08 \x01(\x02\x42\x02\x18\x01\x12\x19\n\rmemory_amount\x18\t \x01(\x05\x42\x02\x18\x01\x12\x45\n\x0cmemory_units\x18\n \x01(\x0e\x32+.qwak.kube.deployment.captain.MemoryUnitApiB\x02\x18\x01\x12\x16\n\x0e\x65nvironment_id\x18\x0b \x01(\t\x12\x1b\n\x13\x63ustom_iam_role_arn\x18\x0c \x01(\t\x12\x41\n\x08job_size\x18\r \x01(\x0b\x32/.qwak.kube.deployment.captain.BatchJobResources\x12\x17\n\x0fpurchase_option\x18\x0e \x01(\t\x12\x19\n\x11image_pull_secret\x18\x0f \x01(\t\x12%\n\x1d\x63ustom_service_account_secret\x18\x10 \x01(\t\x12#\n\x1bjfrog_token_api_secret_name\x18\x11 \x01(\t\x12\x16\n\x0ejfrog_base_url\x18\x12 \x01(\t\"q\n InferenceJobConfigurationMessage\x12\x18\n\x10inference_job_id\x18\x01 \x01(\t\x12\x16\n\x0e\x65nvironment_id\x18\x02 \x01(\t\x12\x1b\n\x13\x63ustom_iam_role_arn\x18\x03 \x01(\t\"`\n\x0fModelIdentifier\x12\x10\n\x08model_id\x18\x01 \x01(\t\x12\x10\n\x08\x62uild_id\x18\x02 \x01(\t\x12\x15\n\tbranch_id\x18\x03 \x01(\tB\x02\x18\x01\x12\x12\n\nmodel_uuid\x18\x04 \x01(\t\"\x85\x01\n\x1dListFilesPreStepConfiguration\x12\x13\n\x0bsource_path\x18\x01 \x01(\t\x12\x1c\n\x14list_files_file_name\x18\x02 \x01(\t\x12\x1d\n\x15total_number_of_tasks\x18\x03 \x01(\x05\x12\x12\n\ntask_index\x18\x04 \x01(\x05\"\xf4\x04\n!InferenceTaskConfigurationMessage\x12\x15\n\rsource_bucket\x18\x01 \x01(\t\x12\x1a\n\x12\x64\x65stination_bucket\x18\x02 \x01(\t\x12\x10\n\x08\x66ilepath\x18\x03 \x01(\t\x12\x18\n\x10\x64\x65stination_path\x18\x04 \x01(\t\x12\x44\n\x0finput_file_type\x18\x05 \x01(\x0e\x32+.qwak.kube.deployment.captain.InputFileType\x12\x46\n\x10output_file_type\x18\x06 \x01(\x0e\x32,.qwak.kube.deployment.captain.OutputFileType\x12\x14\n\x0ctoken_secret\x18\x07 \x01(\t\x12\x15\n\rsecret_secret\x18\x08 \x01(\t\x12\x43\n\nparameters\x18\t \x03(\x0b\x32/.qwak.kube.deployment.captain.BatchJobParameter\x12\x43\n\x0e\x63loud_provider\x18\n \x01(\x0e\x32+.qwak.kube.deployment.captain.CloudProvider\x12\'\n\x1fservice_account_json_key_secret\x18\x0b \x01(\t\x12\x1a\n\x12is_point_file_path\x18\x0c \x01(\x08\x12\x66\n!list_files_pre_step_configuration\x18\r \x01(\x0b\x32;.qwak.kube.deployment.captain.ListFilesPreStepConfiguration\"/\n\x11\x42\x61tchJobParameter\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"?\n#CleanInferenceTasksExecutorsRequest\x12\x18\n\x10inference_job_id\x18\x01 \x01(\t\"O\n$CleanInferenceTasksExecutorsResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x16\n\x0e\x66\x61ilure_reason\x18\x02 \x01(\t\"X\n!CleanInferenceTaskExecutorRequest\x12\x18\n\x10inference_job_id\x18\x01 \x01(\t\x12\x19\n\x11inference_task_id\x18\x02 \x01(\t\"M\n\"CleanInferenceTaskExecutorResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x16\n\x0e\x66\x61ilure_reason\x18\x02 \x01(\t\"N\n#CreateInferenceTaskExecutorResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x16\n\x0e\x66\x61ilure_reason\x18\x02 \x01(\t\"\xf7\x02\n\x1eStartInferenceJobWarmupRequest\x12\x10\n\x08model_id\x18\x01 \x01(\t\x12\x15\n\tbranch_id\x18\x02 \x01(\tB\x02\x18\x01\x12\x10\n\x08\x62uild_id\x18\x03 \x01(\t\x12\x11\n\timage_url\x18\x04 \x01(\t\x12\x0f\n\x03\x63pu\x18\x05 \x01(\x02\x42\x02\x18\x01\x12\x19\n\rmemory_amount\x18\x06 \x01(\x05\x42\x02\x18\x01\x12\x45\n\x0cmemory_units\x18\x07 \x01(\x0e\x32+.qwak.kube.deployment.captain.MemoryUnitApiB\x02\x18\x01\x12\x11\n\texecutors\x18\x08 \x01(\x05\x12\x0f\n\x07timeout\x18\t \x01(\x05\x12\x41\n\x08job_size\x18\n \x01(\x0b\x32/.qwak.kube.deployment.captain.BatchJobResources\x12\x12\n\nmodel_uuid\x18\x0b \x01(\t\x12\x19\n\x11image_pull_secret\x18\x0c \x01(\t\"!\n\x1fStartInferenceJobWarmupResponse\"p\n\x1f\x43\x61ncelInferenceJobWarmupRequest\x12\x10\n\x08model_id\x18\x01 \x01(\t\x12\x15\n\tbranch_id\x18\x02 \x01(\tB\x02\x18\x01\x12\x10\n\x08\x62uild_id\x18\x03 \x01(\t\x12\x12\n\nmodel_uuid\x18\x04 \x01(\t\"\"\n CancelInferenceJobWarmupResponse\"\xd5\x01\n\x11\x42\x61tchJobResources\x12\x16\n\x0enumber_of_pods\x18\x01 \x01(\x05\x12\x0b\n\x03\x63pu\x18\x02 \x01(\x02\x12\x15\n\rmemory_amount\x18\x03 \x01(\x05\x12\x41\n\x0cmemory_units\x18\x04 \x01(\x0e\x32+.qwak.kube.deployment.captain.MemoryUnitApi\x12\x41\n\rgpu_resources\x18\x05 \x01(\x0b\x32*.qwak.kube.deployment.captain.GpuResources\"[\n\x0cGpuResources\x12\x37\n\x08gpu_type\x18\x01 \x01(\x0e\x32%.qwak.kube.deployment.captain.GpuType\x12\x12\n\ngpu_amount\x18\x02 \x01(\x05*:\n\rMemoryUnitApi\x12\x17\n\x13UNKNOWN_MEMORY_UNIT\x10\x00\x12\x07\n\x03MIB\x10\x01\x12\x07\n\x03GIB\x10\x02*\x81\x01\n\rInputFileType\x12\x1d\n\x19UNDEFINED_INPUT_FILE_TYPE\x10\x00\x12\x17\n\x13\x43SV_INPUT_FILE_TYPE\x10\x01\x12\x1b\n\x17\x46\x45\x41THER_INPUT_FILE_TYPE\x10\x02\x12\x1b\n\x17PARQUET_INPUT_FILE_TYPE\x10\x03*\x86\x01\n\x0eOutputFileType\x12\x1e\n\x1aUNDEFINED_OUTPUT_FILE_TYPE\x10\x00\x12\x18\n\x14\x43SV_OUTPUT_FILE_TYPE\x10\x01\x12\x1c\n\x18\x46\x45\x41THER_OUTPUT_FILE_TYPE\x10\x02\x12\x1c\n\x18PARQUET_OUTPUT_FILE_TYPE\x10\x03*\x90\x02\n\x07GpuType\x12\x0f\n\x0bINVALID_GPU\x10\x00\x12\x0e\n\nNVIDIA_K80\x10\x01\x12\x0f\n\x0bNVIDIA_V100\x10\x02\x12\x0f\n\x0bNVIDIA_A100\x10\x03\x12\r\n\tNVIDIA_T4\x10\x04\x12\x0f\n\x0bNVIDIA_A10G\x10\x05\x12\r\n\tNVIDIA_L4\x10\x06\x12\x14\n\x10NVIDIA_T4_1_4_15\x10\x07\x12\x14\n\x10NVIDIA_T4_1_8_30\x10\x08\x12\x15\n\x11NVIDIA_T4_1_16_60\x10\t\x12\x1e\n\x1aNVIDIA_A100_80GB_8_96_1360\x10\n\x12\x16\n\x12NVIDIA_V100_1_8_52\x10\x0b\x12\x18\n\x14NVIDIA_V100_4_32_208\x10\x0c*=\n\rCloudProvider\x12\x1a\n\x16UNKNOWN_CLOUD_PROVIDER\x10\x00\x12\x07\n\x03\x41WS\x10\x01\x12\x07\n\x03GCP\x10\x02\x42+\n\'com.qwak.ai.kube.deployment.captain.apiP\x01\x62\x06proto3')
|
19
19
|
|
20
20
|
_MEMORYUNITAPI = DESCRIPTOR.enum_types_by_name['MemoryUnitApi']
|
21
21
|
MemoryUnitApi = enum_type_wrapper.EnumTypeWrapper(_MEMORYUNITAPI)
|
@@ -69,6 +69,7 @@ _PREPAREINFERENCEJOBRESPONSE = DESCRIPTOR.message_types_by_name['PrepareInferenc
|
|
69
69
|
_TASKEXECUTORCONFIGURATIONMESSAGE = DESCRIPTOR.message_types_by_name['TaskExecutorConfigurationMessage']
|
70
70
|
_INFERENCEJOBCONFIGURATIONMESSAGE = DESCRIPTOR.message_types_by_name['InferenceJobConfigurationMessage']
|
71
71
|
_MODELIDENTIFIER = DESCRIPTOR.message_types_by_name['ModelIdentifier']
|
72
|
+
_LISTFILESPRESTEPCONFIGURATION = DESCRIPTOR.message_types_by_name['ListFilesPreStepConfiguration']
|
72
73
|
_INFERENCETASKCONFIGURATIONMESSAGE = DESCRIPTOR.message_types_by_name['InferenceTaskConfigurationMessage']
|
73
74
|
_BATCHJOBPARAMETER = DESCRIPTOR.message_types_by_name['BatchJobParameter']
|
74
75
|
_CLEANINFERENCETASKSEXECUTORSREQUEST = DESCRIPTOR.message_types_by_name['CleanInferenceTasksExecutorsRequest']
|
@@ -173,6 +174,13 @@ ModelIdentifier = _reflection.GeneratedProtocolMessageType('ModelIdentifier', (_
|
|
173
174
|
})
|
174
175
|
_sym_db.RegisterMessage(ModelIdentifier)
|
175
176
|
|
177
|
+
ListFilesPreStepConfiguration = _reflection.GeneratedProtocolMessageType('ListFilesPreStepConfiguration', (_message.Message,), {
|
178
|
+
'DESCRIPTOR' : _LISTFILESPRESTEPCONFIGURATION,
|
179
|
+
'__module__' : 'qwak.kube_deployment_captain.batch_job_pb2'
|
180
|
+
# @@protoc_insertion_point(class_scope:qwak.kube.deployment.captain.ListFilesPreStepConfiguration)
|
181
|
+
})
|
182
|
+
_sym_db.RegisterMessage(ListFilesPreStepConfiguration)
|
183
|
+
|
176
184
|
InferenceTaskConfigurationMessage = _reflection.GeneratedProtocolMessageType('InferenceTaskConfigurationMessage', (_message.Message,), {
|
177
185
|
'DESCRIPTOR' : _INFERENCETASKCONFIGURATIONMESSAGE,
|
178
186
|
'__module__' : 'qwak.kube_deployment_captain.batch_job_pb2'
|
@@ -292,16 +300,16 @@ if _descriptor._USE_C_DESCRIPTORS == False:
|
|
292
300
|
_STARTINFERENCEJOBWARMUPREQUEST.fields_by_name['memory_units']._serialized_options = b'\030\001'
|
293
301
|
_CANCELINFERENCEJOBWARMUPREQUEST.fields_by_name['branch_id']._options = None
|
294
302
|
_CANCELINFERENCEJOBWARMUPREQUEST.fields_by_name['branch_id']._serialized_options = b'\030\001'
|
295
|
-
_MEMORYUNITAPI._serialized_start=
|
296
|
-
_MEMORYUNITAPI._serialized_end=
|
297
|
-
_INPUTFILETYPE._serialized_start=
|
298
|
-
_INPUTFILETYPE._serialized_end=
|
299
|
-
_OUTPUTFILETYPE._serialized_start=
|
300
|
-
_OUTPUTFILETYPE._serialized_end=
|
301
|
-
_GPUTYPE._serialized_start=
|
302
|
-
_GPUTYPE._serialized_end=
|
303
|
-
_CLOUDPROVIDER._serialized_start=
|
304
|
-
_CLOUDPROVIDER._serialized_end=
|
303
|
+
_MEMORYUNITAPI._serialized_start=4700
|
304
|
+
_MEMORYUNITAPI._serialized_end=4758
|
305
|
+
_INPUTFILETYPE._serialized_start=4761
|
306
|
+
_INPUTFILETYPE._serialized_end=4890
|
307
|
+
_OUTPUTFILETYPE._serialized_start=4893
|
308
|
+
_OUTPUTFILETYPE._serialized_end=5027
|
309
|
+
_GPUTYPE._serialized_start=5030
|
310
|
+
_GPUTYPE._serialized_end=5302
|
311
|
+
_CLOUDPROVIDER._serialized_start=5304
|
312
|
+
_CLOUDPROVIDER._serialized_end=5365
|
305
313
|
_LISTINFERENCEJOBFILESREQUEST._serialized_start=79
|
306
314
|
_LISTINFERENCEJOBFILESREQUEST._serialized_end=545
|
307
315
|
_GROUPEDTASKDETAILS._serialized_start=547
|
@@ -328,30 +336,32 @@ if _descriptor._USE_C_DESCRIPTORS == False:
|
|
328
336
|
_INFERENCEJOBCONFIGURATIONMESSAGE._serialized_end=2517
|
329
337
|
_MODELIDENTIFIER._serialized_start=2519
|
330
338
|
_MODELIDENTIFIER._serialized_end=2615
|
331
|
-
|
332
|
-
|
333
|
-
|
334
|
-
|
335
|
-
|
336
|
-
|
337
|
-
|
338
|
-
|
339
|
-
|
340
|
-
|
341
|
-
|
342
|
-
|
343
|
-
|
344
|
-
|
345
|
-
|
346
|
-
|
347
|
-
|
348
|
-
|
349
|
-
|
350
|
-
|
351
|
-
|
352
|
-
|
353
|
-
|
354
|
-
|
355
|
-
|
356
|
-
|
339
|
+
_LISTFILESPRESTEPCONFIGURATION._serialized_start=2618
|
340
|
+
_LISTFILESPRESTEPCONFIGURATION._serialized_end=2751
|
341
|
+
_INFERENCETASKCONFIGURATIONMESSAGE._serialized_start=2754
|
342
|
+
_INFERENCETASKCONFIGURATIONMESSAGE._serialized_end=3382
|
343
|
+
_BATCHJOBPARAMETER._serialized_start=3384
|
344
|
+
_BATCHJOBPARAMETER._serialized_end=3431
|
345
|
+
_CLEANINFERENCETASKSEXECUTORSREQUEST._serialized_start=3433
|
346
|
+
_CLEANINFERENCETASKSEXECUTORSREQUEST._serialized_end=3496
|
347
|
+
_CLEANINFERENCETASKSEXECUTORSRESPONSE._serialized_start=3498
|
348
|
+
_CLEANINFERENCETASKSEXECUTORSRESPONSE._serialized_end=3577
|
349
|
+
_CLEANINFERENCETASKEXECUTORREQUEST._serialized_start=3579
|
350
|
+
_CLEANINFERENCETASKEXECUTORREQUEST._serialized_end=3667
|
351
|
+
_CLEANINFERENCETASKEXECUTORRESPONSE._serialized_start=3669
|
352
|
+
_CLEANINFERENCETASKEXECUTORRESPONSE._serialized_end=3746
|
353
|
+
_CREATEINFERENCETASKEXECUTORRESPONSE._serialized_start=3748
|
354
|
+
_CREATEINFERENCETASKEXECUTORRESPONSE._serialized_end=3826
|
355
|
+
_STARTINFERENCEJOBWARMUPREQUEST._serialized_start=3829
|
356
|
+
_STARTINFERENCEJOBWARMUPREQUEST._serialized_end=4204
|
357
|
+
_STARTINFERENCEJOBWARMUPRESPONSE._serialized_start=4206
|
358
|
+
_STARTINFERENCEJOBWARMUPRESPONSE._serialized_end=4239
|
359
|
+
_CANCELINFERENCEJOBWARMUPREQUEST._serialized_start=4241
|
360
|
+
_CANCELINFERENCEJOBWARMUPREQUEST._serialized_end=4353
|
361
|
+
_CANCELINFERENCEJOBWARMUPRESPONSE._serialized_start=4355
|
362
|
+
_CANCELINFERENCEJOBWARMUPRESPONSE._serialized_end=4389
|
363
|
+
_BATCHJOBRESOURCES._serialized_start=4392
|
364
|
+
_BATCHJOBRESOURCES._serialized_end=4605
|
365
|
+
_GPURESOURCES._serialized_start=4607
|
366
|
+
_GPURESOURCES._serialized_end=4698
|
357
367
|
# @@protoc_insertion_point(module_scope)
|
@@ -524,6 +524,33 @@ class ModelIdentifier(google.protobuf.message.Message):
|
|
524
524
|
|
525
525
|
global___ModelIdentifier = ModelIdentifier
|
526
526
|
|
527
|
+
class ListFilesPreStepConfiguration(google.protobuf.message.Message):
|
528
|
+
DESCRIPTOR: google.protobuf.descriptor.Descriptor
|
529
|
+
|
530
|
+
SOURCE_PATH_FIELD_NUMBER: builtins.int
|
531
|
+
LIST_FILES_FILE_NAME_FIELD_NUMBER: builtins.int
|
532
|
+
TOTAL_NUMBER_OF_TASKS_FIELD_NUMBER: builtins.int
|
533
|
+
TASK_INDEX_FIELD_NUMBER: builtins.int
|
534
|
+
source_path: builtins.str
|
535
|
+
"""Directory path in source bucket of data files to run inference on"""
|
536
|
+
list_files_file_name: builtins.str
|
537
|
+
"""Name for the file to save the list of files in the source path"""
|
538
|
+
total_number_of_tasks: builtins.int
|
539
|
+
"""Total number of inference tasks, used to divide the files list into similarly sized groups"""
|
540
|
+
task_index: builtins.int
|
541
|
+
"""Index of the specific task, used to determine on which files group to run inference on"""
|
542
|
+
def __init__(
|
543
|
+
self,
|
544
|
+
*,
|
545
|
+
source_path: builtins.str = ...,
|
546
|
+
list_files_file_name: builtins.str = ...,
|
547
|
+
total_number_of_tasks: builtins.int = ...,
|
548
|
+
task_index: builtins.int = ...,
|
549
|
+
) -> None: ...
|
550
|
+
def ClearField(self, field_name: typing_extensions.Literal["list_files_file_name", b"list_files_file_name", "source_path", b"source_path", "task_index", b"task_index", "total_number_of_tasks", b"total_number_of_tasks"]) -> None: ...
|
551
|
+
|
552
|
+
global___ListFilesPreStepConfiguration = ListFilesPreStepConfiguration
|
553
|
+
|
527
554
|
class InferenceTaskConfigurationMessage(google.protobuf.message.Message):
|
528
555
|
DESCRIPTOR: google.protobuf.descriptor.Descriptor
|
529
556
|
|
@@ -539,6 +566,7 @@ class InferenceTaskConfigurationMessage(google.protobuf.message.Message):
|
|
539
566
|
CLOUD_PROVIDER_FIELD_NUMBER: builtins.int
|
540
567
|
SERVICE_ACCOUNT_JSON_KEY_SECRET_FIELD_NUMBER: builtins.int
|
541
568
|
IS_POINT_FILE_PATH_FIELD_NUMBER: builtins.int
|
569
|
+
LIST_FILES_PRE_STEP_CONFIGURATION_FIELD_NUMBER: builtins.int
|
542
570
|
source_bucket: builtins.str
|
543
571
|
"""The source bucket in cloud storage"""
|
544
572
|
destination_bucket: builtins.str
|
@@ -564,6 +592,9 @@ class InferenceTaskConfigurationMessage(google.protobuf.message.Message):
|
|
564
592
|
"""The service account json key secret"""
|
565
593
|
is_point_file_path: builtins.bool
|
566
594
|
"""Is the file contains points to inference files"""
|
595
|
+
@property
|
596
|
+
def list_files_pre_step_configuration(self) -> global___ListFilesPreStepConfiguration:
|
597
|
+
"""Optional pre step configuration, used to list files in the source directory and run inference on subset of files"""
|
567
598
|
def __init__(
|
568
599
|
self,
|
569
600
|
*,
|
@@ -579,8 +610,10 @@ class InferenceTaskConfigurationMessage(google.protobuf.message.Message):
|
|
579
610
|
cloud_provider: global___CloudProvider.ValueType = ...,
|
580
611
|
service_account_json_key_secret: builtins.str = ...,
|
581
612
|
is_point_file_path: builtins.bool = ...,
|
613
|
+
list_files_pre_step_configuration: global___ListFilesPreStepConfiguration | None = ...,
|
582
614
|
) -> None: ...
|
583
|
-
def
|
615
|
+
def HasField(self, field_name: typing_extensions.Literal["list_files_pre_step_configuration", b"list_files_pre_step_configuration"]) -> builtins.bool: ...
|
616
|
+
def ClearField(self, field_name: typing_extensions.Literal["cloud_provider", b"cloud_provider", "destination_bucket", b"destination_bucket", "destination_path", b"destination_path", "filepath", b"filepath", "input_file_type", b"input_file_type", "is_point_file_path", b"is_point_file_path", "list_files_pre_step_configuration", b"list_files_pre_step_configuration", "output_file_type", b"output_file_type", "parameters", b"parameters", "secret_secret", b"secret_secret", "service_account_json_key_secret", b"service_account_json_key_secret", "source_bucket", b"source_bucket", "token_secret", b"token_secret"]) -> None: ...
|
584
617
|
|
585
618
|
global___InferenceTaskConfigurationMessage = InferenceTaskConfigurationMessage
|
586
619
|
|
qwak/__init__.py
CHANGED
@@ -1,9 +1,12 @@
|
|
1
|
-
import
|
1
|
+
from datetime import timedelta
|
2
2
|
from typing import Optional
|
3
|
+
import requests
|
4
|
+
from requests import Response
|
3
5
|
|
4
6
|
from _qwak_proto.qwak.builds.build_url_pb2 import BuildVersioningTagsType
|
5
7
|
from _qwak_proto.qwak.builds.builds_orchestrator_service_pb2 import (
|
6
8
|
GetBuildVersioningUploadURLResponse,
|
9
|
+
GetBuildVersioningDownloadURLResponse,
|
7
10
|
)
|
8
11
|
from qwak.clients.build_orchestrator.client import BuildOrchestratorClient
|
9
12
|
from qwak.clients.file_versioning.client import FileVersioningManagementClient
|
@@ -15,6 +18,8 @@ from qwak.inner.model_loggers_utils import (
|
|
15
18
|
validate_tag,
|
16
19
|
)
|
17
20
|
|
21
|
+
MAX_CHUNK_SIZE = 8 * 1024
|
22
|
+
|
18
23
|
|
19
24
|
def log_file(
|
20
25
|
from_path: str,
|
@@ -80,21 +85,40 @@ def load_file(
|
|
80
85
|
Returns:
|
81
86
|
the path to the newly created data file
|
82
87
|
"""
|
88
|
+
print(f"Loading file to {to_path}")
|
83
89
|
if not validate_tag(tag):
|
84
90
|
raise QwakException(
|
85
91
|
"Tag should contain only letters, numbers, underscore or hyphen"
|
86
92
|
)
|
87
93
|
|
88
94
|
model_id = validate_model(model_id)
|
89
|
-
download_url_response =
|
90
|
-
|
95
|
+
download_url_response: GetBuildVersioningDownloadURLResponse = (
|
96
|
+
BuildOrchestratorClient().get_build_versioning_download_url(
|
97
|
+
build_id=build_id,
|
98
|
+
model_id=model_id,
|
99
|
+
tag=tag,
|
100
|
+
tag_type=BuildVersioningTagsType.FILE_TAG_TYPE,
|
101
|
+
)
|
91
102
|
)
|
92
103
|
|
93
104
|
try:
|
94
|
-
|
95
|
-
|
105
|
+
response: Response = requests.get(
|
106
|
+
download_url_response.download_url,
|
107
|
+
headers=download_url_response.headers,
|
108
|
+
stream=True,
|
109
|
+
timeout=(
|
110
|
+
timedelta(seconds=10).total_seconds(), # timeout to connect
|
111
|
+
timedelta(minutes=20).total_seconds(), # timeout to read
|
112
|
+
),
|
96
113
|
)
|
114
|
+
print(f"Downloading file finished with status {response.status_code}")
|
115
|
+
response.raise_for_status()
|
116
|
+
|
117
|
+
with open(to_path, "wb") as f:
|
118
|
+
for chunk in response.iter_content(chunk_size=MAX_CHUNK_SIZE):
|
119
|
+
if chunk:
|
120
|
+
f.write(chunk)
|
97
121
|
|
98
|
-
return
|
122
|
+
return to_path
|
99
123
|
except Exception as error:
|
100
124
|
raise QwakException(f"Unable to load save artifact locally: {str(error)}")
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.3
|
2
2
|
Name: qwak-core
|
3
|
-
Version: 0.4.
|
3
|
+
Version: 0.4.253
|
4
4
|
Summary: Qwak Core contains the necessary objects and communication tools for using the Qwak Platform
|
5
5
|
License: Apache-2.0
|
6
6
|
Keywords: mlops,ml,deployment,serving,model
|
@@ -18,6 +18,7 @@ Classifier: Programming Language :: Python :: 3.8
|
|
18
18
|
Classifier: Programming Language :: Python :: Implementation :: CPython
|
19
19
|
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
20
20
|
Provides-Extra: feature-store
|
21
|
+
Provides-Extra: local-core-dependencies
|
21
22
|
Requires-Dist: PyYAML
|
22
23
|
Requires-Dist: cachetools
|
23
24
|
Requires-Dist: chevron (==0.14.0)
|
@@ -27,8 +28,11 @@ Requires-Dist: dependency-injector (>=4.0)
|
|
27
28
|
Requires-Dist: filelock
|
28
29
|
Requires-Dist: frogml-storage (>=0.11.2)
|
29
30
|
Requires-Dist: grpcio (>=1.57.0)
|
31
|
+
Requires-Dist: grpcio-tools (>=1.47.0) ; (python_full_version >= "3.7.1" and python_version < "3.10") and (extra == "local-core-dependencies")
|
32
|
+
Requires-Dist: grpcio-tools (>=1.56.2) ; (python_version >= "3.10") and (extra == "local-core-dependencies")
|
30
33
|
Requires-Dist: joblib (>=1.3.2,<2.0.0)
|
31
34
|
Requires-Dist: marshmallow-dataclass (>=8.5.8,<9.0.0)
|
35
|
+
Requires-Dist: mypy-protobuf (>=3.0.0,<4.0.0) ; extra == "local-core-dependencies"
|
32
36
|
Requires-Dist: protobuf (>=3.10,<4) ; python_full_version >= "3.7.1" and python_version < "3.10"
|
33
37
|
Requires-Dist: protobuf (>=4.21.6) ; python_version >= "3.10"
|
34
38
|
Requires-Dist: pyarrow (>=6.0.0) ; extra == "feature-store"
|
@@ -447,8 +447,8 @@ _qwak_proto/qwak/kube_deployment_captain/alert_pb2_grpc.py,sha256=1oboBPFxaTEXt9
|
|
447
447
|
_qwak_proto/qwak/kube_deployment_captain/alerting_pb2.py,sha256=KKieMFIpkudqk_PC0Y7dcSivMh_YEHQnsSbnEs2FywE,3704
|
448
448
|
_qwak_proto/qwak/kube_deployment_captain/alerting_pb2.pyi,sha256=AGXtot0uXW-mGd2xuixCCJB6eTt0UuF3b2oAhb4IsNc,3759
|
449
449
|
_qwak_proto/qwak/kube_deployment_captain/alerting_pb2_grpc.py,sha256=1oboBPFxaTEXt9Aw7EAj8gXHDCNMhZD2VXqocC9l_gk,159
|
450
|
-
_qwak_proto/qwak/kube_deployment_captain/batch_job_pb2.py,sha256=
|
451
|
-
_qwak_proto/qwak/kube_deployment_captain/batch_job_pb2.pyi,sha256=
|
450
|
+
_qwak_proto/qwak/kube_deployment_captain/batch_job_pb2.py,sha256=FM6yH3yQykNWXJnshZhnM57-YIoHrzRyyvofwlp_neE,29123
|
451
|
+
_qwak_proto/qwak/kube_deployment_captain/batch_job_pb2.pyi,sha256=4u4U_F76gCFfsLb9ivpZKtLuzR7zssncYrlHNIj3CK0,39265
|
452
452
|
_qwak_proto/qwak/kube_deployment_captain/batch_job_pb2_grpc.py,sha256=1oboBPFxaTEXt9Aw7EAj8gXHDCNMhZD2VXqocC9l_gk,159
|
453
453
|
_qwak_proto/qwak/kube_deployment_captain/deployment_pb2.py,sha256=9YrlZLZQk38TUz0_yGgRmz3ljqqgnynLQc18oc6Q3dk,14037
|
454
454
|
_qwak_proto/qwak/kube_deployment_captain/deployment_pb2.pyi,sha256=Sz11iTrL3BlUEl1jXNf0Q1SDe1SX_MjjyB9bMrtsOVg,24090
|
@@ -582,7 +582,7 @@ _qwak_proto/qwak/workspace/workspace_pb2_grpc.py,sha256=1oboBPFxaTEXt9Aw7EAj8gXH
|
|
582
582
|
_qwak_proto/qwak/workspace/workspace_service_pb2.py,sha256=AB3C9S_AbOD7Nx1Ni4j1rW6PNtYTV1zjiqFQk-goQ74,21429
|
583
583
|
_qwak_proto/qwak/workspace/workspace_service_pb2.pyi,sha256=nKKCHwnovZhsy8TSVmdz-Vtl0nviOOoX56HD-41Xo08,13726
|
584
584
|
_qwak_proto/qwak/workspace/workspace_service_pb2_grpc.py,sha256=yKGuexxTBza99Ihe0DSTniV2ZSd_AG47inHenqfi890,27193
|
585
|
-
qwak/__init__.py,sha256=
|
585
|
+
qwak/__init__.py,sha256=h1h_J8C7TMOXTantCGO5DqTlP3H6fb0xw0z8optQEUw,587
|
586
586
|
qwak/automations/__init__.py,sha256=qFZRvCxUUn8gcxkJR0v19ulHW2oJ0x6-Rif7HiheDP4,1522
|
587
587
|
qwak/automations/automation_executions.py,sha256=5MeH_epYYWb8NKXgAozwT_jPyyUDednBHG7izloi7RY,3228
|
588
588
|
qwak/automations/automations.py,sha256=3yx8e2v0uSKDnXbqyknasyEoQ5vxGni6K40Hbi1_zkk,12599
|
@@ -961,7 +961,7 @@ qwak/model/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,
|
|
961
961
|
qwak/model/utils/extract_wrapped_function.py,sha256=uIle1zL8vbmeS3PGAuaNFLIUQAsvpuzk3LlH-Teba94,320
|
962
962
|
qwak/model/utils/feature_utils.py,sha256=jB4pfmhqN2_B4THR7YADlCWbymU3cm_8u2RzTegrv_w,2389
|
963
963
|
qwak/model_loggers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
964
|
-
qwak/model_loggers/artifact_logger.py,sha256=
|
964
|
+
qwak/model_loggers/artifact_logger.py,sha256=Emv8eSpgxJ_5_uCiNLUloTNdXPCtWC0pJDcWMcc6A78,3909
|
965
965
|
qwak/model_loggers/data_logger.py,sha256=aTRnRwKmKUFZq9SRg4sY-b1yGDY2XB-pExlQ0WlcH0k,5629
|
966
966
|
qwak/model_loggers/model_logger.py,sha256=WxKLEpfzDaRn3kanrI0kj4zqqKQrwT2sin1IS1WNEA8,852
|
967
967
|
qwak/qwak_client/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
@@ -1056,6 +1056,6 @@ qwak_services_mock/mocks/workspace_manager_service_mock.py,sha256=O9ZSwln4T4kHVk
|
|
1056
1056
|
qwak_services_mock/services_mock.py,sha256=zXtHcX8a_acz7ynxuCBxxVpHpde7aAGjIn6Uw52LY1s,19593
|
1057
1057
|
qwak_services_mock/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
1058
1058
|
qwak_services_mock/utils/service_utils.py,sha256=ZlB0CnB1J6oBn6_m7fQO2U8tKoboHdUa6ljjkRMYNXU,265
|
1059
|
-
qwak_core-0.4.
|
1060
|
-
qwak_core-0.4.
|
1061
|
-
qwak_core-0.4.
|
1059
|
+
qwak_core-0.4.253.dist-info/METADATA,sha256=MvUHjty_FOLEHMbX845cc0gPFneR3LxXdK0UOwKCLqA,2549
|
1060
|
+
qwak_core-0.4.253.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
|
1061
|
+
qwak_core-0.4.253.dist-info/RECORD,,
|
File without changes
|