oracle-ads 2.12.9__py3-none-any.whl → 2.12.10__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. ads/aqua/__init__.py +4 -3
  2. ads/aqua/app.py +28 -16
  3. ads/aqua/client/__init__.py +3 -0
  4. ads/aqua/client/client.py +799 -0
  5. ads/aqua/common/enums.py +3 -0
  6. ads/aqua/common/utils.py +62 -2
  7. ads/aqua/data.py +2 -19
  8. ads/aqua/evaluation/evaluation.py +20 -12
  9. ads/aqua/extension/aqua_ws_msg_handler.py +14 -7
  10. ads/aqua/extension/base_handler.py +12 -9
  11. ads/aqua/extension/finetune_handler.py +8 -14
  12. ads/aqua/extension/model_handler.py +24 -2
  13. ads/aqua/finetuning/constants.py +5 -2
  14. ads/aqua/finetuning/entities.py +67 -17
  15. ads/aqua/finetuning/finetuning.py +69 -54
  16. ads/aqua/model/entities.py +3 -1
  17. ads/aqua/model/model.py +196 -98
  18. ads/aqua/modeldeployment/deployment.py +22 -10
  19. ads/cli.py +16 -8
  20. ads/common/auth.py +9 -9
  21. ads/llm/autogen/__init__.py +2 -0
  22. ads/llm/autogen/constants.py +15 -0
  23. ads/llm/autogen/reports/__init__.py +2 -0
  24. ads/llm/autogen/reports/base.py +67 -0
  25. ads/llm/autogen/reports/data.py +103 -0
  26. ads/llm/autogen/reports/session.py +526 -0
  27. ads/llm/autogen/reports/templates/chat_box.html +13 -0
  28. ads/llm/autogen/reports/templates/chat_box_lt.html +5 -0
  29. ads/llm/autogen/reports/templates/chat_box_rt.html +6 -0
  30. ads/llm/autogen/reports/utils.py +56 -0
  31. ads/llm/autogen/v02/__init__.py +4 -0
  32. ads/llm/autogen/{client_v02.py → v02/client.py} +23 -10
  33. ads/llm/autogen/v02/log_handlers/__init__.py +2 -0
  34. ads/llm/autogen/v02/log_handlers/oci_file_handler.py +83 -0
  35. ads/llm/autogen/v02/loggers/__init__.py +6 -0
  36. ads/llm/autogen/v02/loggers/metric_logger.py +320 -0
  37. ads/llm/autogen/v02/loggers/session_logger.py +580 -0
  38. ads/llm/autogen/v02/loggers/utils.py +86 -0
  39. ads/llm/autogen/v02/runtime_logging.py +163 -0
  40. ads/llm/langchain/plugins/chat_models/oci_data_science.py +12 -11
  41. ads/model/__init__.py +11 -13
  42. ads/model/artifact.py +47 -8
  43. ads/model/extractor/embedding_onnx_extractor.py +80 -0
  44. ads/model/framework/embedding_onnx_model.py +438 -0
  45. ads/model/generic_model.py +26 -24
  46. ads/model/model_metadata.py +8 -7
  47. ads/opctl/config/merger.py +13 -14
  48. ads/opctl/operator/common/operator_config.py +4 -4
  49. ads/opctl/operator/lowcode/common/transformations.py +50 -8
  50. ads/opctl/operator/lowcode/common/utils.py +22 -6
  51. ads/opctl/operator/lowcode/forecast/__main__.py +10 -0
  52. ads/opctl/operator/lowcode/forecast/const.py +2 -0
  53. ads/opctl/operator/lowcode/forecast/model/arima.py +19 -13
  54. ads/opctl/operator/lowcode/forecast/model/automlx.py +129 -36
  55. ads/opctl/operator/lowcode/forecast/model/autots.py +1 -0
  56. ads/opctl/operator/lowcode/forecast/model/base_model.py +61 -14
  57. ads/opctl/operator/lowcode/forecast/model/forecast_datasets.py +1 -1
  58. ads/opctl/operator/lowcode/forecast/model/neuralprophet.py +10 -3
  59. ads/opctl/operator/lowcode/forecast/model/prophet.py +25 -18
  60. ads/opctl/operator/lowcode/forecast/operator_config.py +31 -0
  61. ads/opctl/operator/lowcode/forecast/schema.yaml +76 -0
  62. ads/opctl/operator/lowcode/forecast/utils.py +4 -3
  63. ads/opctl/operator/lowcode/forecast/whatifserve/__init__.py +7 -0
  64. ads/opctl/operator/lowcode/forecast/whatifserve/deployment_manager.py +233 -0
  65. ads/opctl/operator/lowcode/forecast/whatifserve/score.py +238 -0
  66. ads/telemetry/base.py +18 -11
  67. ads/telemetry/client.py +33 -13
  68. ads/templates/schemas/openapi.json +1740 -0
  69. ads/templates/score_embedding_onnx.jinja2 +202 -0
  70. {oracle_ads-2.12.9.dist-info → oracle_ads-2.12.10.dist-info}/METADATA +9 -8
  71. {oracle_ads-2.12.9.dist-info → oracle_ads-2.12.10.dist-info}/RECORD +74 -48
  72. {oracle_ads-2.12.9.dist-info → oracle_ads-2.12.10.dist-info}/LICENSE.txt +0 -0
  73. {oracle_ads-2.12.9.dist-info → oracle_ads-2.12.10.dist-info}/WHEEL +0 -0
  74. {oracle_ads-2.12.9.dist-info → oracle_ads-2.12.10.dist-info}/entry_points.txt +0 -0
ads/aqua/model/model.py CHANGED
@@ -1,5 +1,5 @@
1
1
  #!/usr/bin/env python
2
- # Copyright (c) 2024 Oracle and/or its affiliates.
2
+ # Copyright (c) 2024, 2025 Oracle and/or its affiliates.
3
3
  # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
4
4
  import os
5
5
  import pathlib
@@ -15,15 +15,20 @@ from oci.data_science.models import JobRun, Metadata, Model, UpdateModelDetails
15
15
  from ads.aqua import ODSC_MODEL_COMPARTMENT_OCID, logger
16
16
  from ads.aqua.app import AquaApp
17
17
  from ads.aqua.common.enums import (
18
+ CustomInferenceContainerTypeFamily,
18
19
  FineTuningContainerTypeFamily,
19
20
  InferenceContainerTypeFamily,
20
21
  Tags,
21
22
  )
22
- from ads.aqua.common.errors import AquaRuntimeError, AquaValueError
23
+ from ads.aqua.common.errors import (
24
+ AquaFileNotFoundError,
25
+ AquaRuntimeError,
26
+ AquaValueError,
27
+ )
23
28
  from ads.aqua.common.utils import (
24
29
  LifecycleStatus,
25
30
  _build_resource_identifier,
26
- copy_model_config,
31
+ cleanup_local_hf_model_artifact,
27
32
  create_word_icon,
28
33
  generate_tei_cmd_var,
29
34
  get_artifact_path,
@@ -160,7 +165,7 @@ class AquaModelApp(AquaApp):
160
165
  target_compartment = compartment_id or COMPARTMENT_OCID
161
166
 
162
167
  if service_model.compartment_id != ODSC_MODEL_COMPARTMENT_OCID:
163
- logger.debug(
168
+ logger.info(
164
169
  f"Aqua Model {model_id} already exists in user's compartment."
165
170
  "Skipped copying."
166
171
  )
@@ -191,8 +196,8 @@ class AquaModelApp(AquaApp):
191
196
  # TODO: decide what kwargs will be needed.
192
197
  .create(model_by_reference=True, **kwargs)
193
198
  )
194
- logger.debug(
195
- f"Aqua Model {custom_model.id} created with the service model {model_id}"
199
+ logger.info(
200
+ f"Aqua Model {custom_model.id} created with the service model {model_id}."
196
201
  )
197
202
 
198
203
  # tracks unique models that were created in the user compartment
@@ -223,11 +228,16 @@ class AquaModelApp(AquaApp):
223
228
 
224
229
  cached_item = self._service_model_details_cache.get(model_id)
225
230
  if cached_item:
231
+ logger.info(f"Fetching model details for model {model_id} from cache.")
226
232
  return cached_item
227
233
 
234
+ logger.info(f"Fetching model details for model {model_id}.")
228
235
  ds_model = DataScienceModel.from_id(model_id)
229
236
  if not self._if_show(ds_model):
230
- raise AquaRuntimeError(f"Target model `{ds_model.id} `is not Aqua model.")
237
+ raise AquaRuntimeError(
238
+ f"Target model `{ds_model.id} `is not an Aqua model as it does not contain "
239
+ f"{Tags.AQUA_TAG} tag."
240
+ )
231
241
 
232
242
  is_fine_tuned_model = bool(
233
243
  ds_model.freeform_tags
@@ -246,16 +256,21 @@ class AquaModelApp(AquaApp):
246
256
  ds_model.custom_metadata_list._to_oci_metadata()
247
257
  )
248
258
  if artifact_path != UNKNOWN:
259
+ model_card_path = (
260
+ f"{artifact_path.rstrip('/')}/config/{README}"
261
+ if is_verified_type
262
+ else f"{artifact_path.rstrip('/')}/{README}"
263
+ )
249
264
  model_card = str(
250
265
  read_file(
251
- file_path=(
252
- f"{artifact_path.rstrip('/')}/config/{README}"
253
- if is_verified_type
254
- else f"{artifact_path.rstrip('/')}/{README}"
255
- ),
266
+ file_path=model_card_path,
256
267
  auth=default_signer(),
257
268
  )
258
269
  )
270
+ if not model_card:
271
+ logger.warn(
272
+ f"Model card for {model_id} is empty or could not be loaded from {model_card_path}."
273
+ )
259
274
 
260
275
  inference_container = ds_model.custom_metadata_list.get(
261
276
  ModelCustomMetadataFields.DEPLOYMENT_CONTAINER,
@@ -301,9 +316,10 @@ class AquaModelApp(AquaApp):
301
316
  try:
302
317
  jobrun_ocid = ds_model.provenance_metadata.training_id
303
318
  jobrun = self.ds_client.get_job_run(jobrun_ocid).data
304
- except Exception:
319
+ except Exception as e:
305
320
  logger.debug(
306
321
  f"Missing jobrun information in the provenance metadata of the given model {model_id}."
322
+ f"\nError: {str(e)}"
307
323
  )
308
324
  jobrun = None
309
325
 
@@ -312,7 +328,10 @@ class AquaModelApp(AquaApp):
312
328
  FineTuningCustomMetadata.FT_SOURCE
313
329
  ).value
314
330
  except ValueError as e:
315
- logger.debug(str(e))
331
+ logger.debug(
332
+ f"Custom metadata is missing {FineTuningCustomMetadata.FT_SOURCE} key for "
333
+ f"model {model_id}.\nError: {str(e)}"
334
+ )
316
335
  source_id = UNKNOWN
317
336
 
318
337
  try:
@@ -320,7 +339,10 @@ class AquaModelApp(AquaApp):
320
339
  FineTuningCustomMetadata.FT_SOURCE_NAME
321
340
  ).value
322
341
  except ValueError as e:
323
- logger.debug(str(e))
342
+ logger.debug(
343
+ f"Custom metadata is missing {FineTuningCustomMetadata.FT_SOURCE_NAME} key for "
344
+ f"model {model_id}.\nError: {str(e)}"
345
+ )
324
346
  source_name = UNKNOWN
325
347
 
326
348
  source_identifier = _build_resource_identifier(
@@ -370,14 +392,17 @@ class AquaModelApp(AquaApp):
370
392
  Tags.AQUA_FINE_TUNED_MODEL_TAG, None
371
393
  )
372
394
  if is_registered_model or is_fine_tuned_model:
395
+ logger.info(f"Deleting model {model_id}.")
373
396
  return ds_model.delete()
374
397
  else:
375
398
  raise AquaRuntimeError(
376
399
  f"Failed to delete model:{model_id}. Only registered models or finetuned model can be deleted."
377
400
  )
378
401
 
379
- @telemetry(entry_point="plugin=model&action=delete", name="aqua")
380
- def edit_registered_model(self, id, inference_container, enable_finetuning, task):
402
+ @telemetry(entry_point="plugin=model&action=edit", name="aqua")
403
+ def edit_registered_model(
404
+ self, id, inference_container, inference_container_uri, enable_finetuning, task
405
+ ):
381
406
  """Edits the default config of unverified registered model.
382
407
 
383
408
  Parameters
@@ -386,6 +411,8 @@ class AquaModelApp(AquaApp):
386
411
  The model OCID.
387
412
  inference_container: str.
388
413
  The inference container family name
414
+ inference_container_uri: str
415
+ The inference container uri for embedding models
389
416
  enable_finetuning: str
390
417
  Flag to enable or disable finetuning over the model. Defaults to None
391
418
  task:
@@ -401,19 +428,44 @@ class AquaModelApp(AquaApp):
401
428
  if ds_model.freeform_tags.get(Tags.BASE_MODEL_CUSTOM, None):
402
429
  if ds_model.freeform_tags.get(Tags.AQUA_SERVICE_MODEL_TAG, None):
403
430
  raise AquaRuntimeError(
404
- f"Failed to edit model:{id}. Only registered unverified models can be edited."
431
+ "Only registered unverified models can be edited."
405
432
  )
406
433
  else:
407
434
  custom_metadata_list = ds_model.custom_metadata_list
408
435
  freeform_tags = ds_model.freeform_tags
409
436
  if inference_container:
410
- custom_metadata_list.add(
411
- key=ModelCustomMetadataFields.DEPLOYMENT_CONTAINER,
412
- value=inference_container,
413
- category=MetadataCustomCategory.OTHER,
414
- description="Deployment container mapping for SMC",
415
- replace=True,
416
- )
437
+ if (
438
+ inference_container in CustomInferenceContainerTypeFamily
439
+ and inference_container_uri is None
440
+ ):
441
+ raise AquaRuntimeError(
442
+ "Inference container URI must be provided."
443
+ )
444
+ else:
445
+ custom_metadata_list.add(
446
+ key=ModelCustomMetadataFields.DEPLOYMENT_CONTAINER,
447
+ value=inference_container,
448
+ category=MetadataCustomCategory.OTHER,
449
+ description="Deployment container mapping for SMC",
450
+ replace=True,
451
+ )
452
+ if inference_container_uri:
453
+ if (
454
+ inference_container in CustomInferenceContainerTypeFamily
455
+ or inference_container is None
456
+ ):
457
+ custom_metadata_list.add(
458
+ key=ModelCustomMetadataFields.DEPLOYMENT_CONTAINER_URI,
459
+ value=inference_container_uri,
460
+ category=MetadataCustomCategory.OTHER,
461
+ description=f"Inference container URI for {ds_model.display_name}",
462
+ replace=True,
463
+ )
464
+ else:
465
+ raise AquaRuntimeError(
466
+ f"Inference container URI can be edited only with container values: {CustomInferenceContainerTypeFamily.values()}"
467
+ )
468
+
417
469
  if enable_finetuning is not None:
418
470
  if enable_finetuning.lower() == "true":
419
471
  custom_metadata_list.add(
@@ -447,10 +499,9 @@ class AquaModelApp(AquaApp):
447
499
  freeform_tags=freeform_tags,
448
500
  )
449
501
  AquaApp().update_model(id, update_model_details)
502
+ logger.info(f"Updated model details for the model {id}.")
450
503
  else:
451
- raise AquaRuntimeError(
452
- f"Failed to edit model:{id}. Only registered unverified models can be edited."
453
- )
504
+ raise AquaRuntimeError("Only registered unverified models can be edited.")
454
505
 
455
506
  def _fetch_metric_from_metadata(
456
507
  self,
@@ -706,7 +757,7 @@ class AquaModelApp(AquaApp):
706
757
  )
707
758
 
708
759
  logger.info(
709
- f"Fetch {len(models)} model in compartment_id={compartment_id or ODSC_MODEL_COMPARTMENT_OCID}."
760
+ f"Fetched {len(models)} model in compartment_id={compartment_id or ODSC_MODEL_COMPARTMENT_OCID}."
710
761
  )
711
762
 
712
763
  aqua_models = []
@@ -736,10 +787,12 @@ class AquaModelApp(AquaApp):
736
787
  dict with the key used, and True if cache has the key that needs to be deleted.
737
788
  """
738
789
  res = {}
739
- logger.info("Clearing _service_models_cache")
740
790
  with self._cache_lock:
741
791
  if ODSC_MODEL_COMPARTMENT_OCID in self._service_models_cache:
742
792
  self._service_models_cache.pop(key=ODSC_MODEL_COMPARTMENT_OCID)
793
+ logger.info(
794
+ f"Cleared models cache for service compartment {ODSC_MODEL_COMPARTMENT_OCID}."
795
+ )
743
796
  res = {
744
797
  "key": {
745
798
  "compartment_id": ODSC_MODEL_COMPARTMENT_OCID,
@@ -756,10 +809,10 @@ class AquaModelApp(AquaApp):
756
809
  dict with the key used, and True if cache has the key that needs to be deleted.
757
810
  """
758
811
  res = {}
759
- logger.info(f"Clearing _service_model_details_cache for {model_id}")
760
812
  with self._cache_lock:
761
813
  if model_id in self._service_model_details_cache:
762
814
  self._service_model_details_cache.pop(key=model_id)
815
+ logger.info(f"Clearing model details cache for model {model_id}.")
763
816
  res = {"key": {"model_id": model_id}, "cache_deleted": True}
764
817
 
765
818
  return res
@@ -844,7 +897,8 @@ class AquaModelApp(AquaApp):
844
897
  metadata = ModelCustomMetadata()
845
898
  if not inference_container:
846
899
  raise AquaRuntimeError(
847
- f"Require Inference container information. Model: {model_name} does not have associated inference container defaults. Check docs for more information on how to pass inference container."
900
+ f"Require Inference container information. Model: {model_name} does not have associated inference "
901
+ f"container defaults. Check docs for more information on how to pass inference container."
848
902
  )
849
903
  metadata.add(
850
904
  key=AQUA_DEPLOYMENT_CONTAINER_METADATA_NAME,
@@ -869,8 +923,7 @@ class AquaModelApp(AquaApp):
869
923
  # only add cmd vars if inference container is not an SMC
870
924
  if (
871
925
  inference_container not in smc_container_set
872
- and inference_container
873
- == InferenceContainerTypeFamily.AQUA_TEI_CONTAINER_FAMILY
926
+ and inference_container in CustomInferenceContainerTypeFamily.values()
874
927
  ):
875
928
  cmd_vars = generate_tei_cmd_var(os_path)
876
929
  metadata.add(
@@ -915,24 +968,6 @@ class AquaModelApp(AquaApp):
915
968
  )
916
969
  tags[Tags.LICENSE] = validation_result.tags.get(Tags.LICENSE, UNKNOWN)
917
970
 
918
- try:
919
- # If verified model already has a artifact json, use that.
920
- artifact_path = metadata.get(MODEL_BY_REFERENCE_OSS_PATH_KEY).value
921
- logger.info(
922
- f"Found model artifact in the service bucket. "
923
- f"Using artifact from service bucket instead of {os_path}"
924
- )
925
-
926
- # todo: implement generic copy_folder method
927
- # copy model config from artifact path to user bucket
928
- copy_model_config(
929
- artifact_path=artifact_path, os_path=os_path, auth=default_signer()
930
- )
931
- except Exception:
932
- logger.debug(
933
- f"Proceeding with model registration without copying model config files at {os_path}. "
934
- f"Default configuration will be used for deployment and fine-tuning."
935
- )
936
971
  # Set artifact location to user bucket, and replace existing key if present.
937
972
  metadata.add(
938
973
  key=MODEL_BY_REFERENCE_OSS_PATH_KEY,
@@ -952,7 +987,7 @@ class AquaModelApp(AquaApp):
952
987
  .with_freeform_tags(**tags)
953
988
  .with_defined_tags(**(defined_tags or {}))
954
989
  ).create(model_by_reference=True)
955
- logger.debug(model)
990
+ logger.debug(f"Created model catalog entry for the model:\n{model}")
956
991
  return model
957
992
 
958
993
  @staticmethod
@@ -972,13 +1007,23 @@ class AquaModelApp(AquaApp):
972
1007
  # todo: revisit this logic to account for .bin files. In the current state, .bin and .safetensor models
973
1008
  # are grouped in one category and validation checks for config.json files only.
974
1009
  if model_format == ModelFormat.SAFETENSORS:
1010
+ model_files.extend(
1011
+ list_os_files_with_extension(oss_path=os_path, extension=".safetensors")
1012
+ )
975
1013
  try:
976
1014
  load_config(
977
1015
  file_path=os_path,
978
1016
  config_file_name=AQUA_MODEL_ARTIFACT_CONFIG,
979
1017
  )
980
- except Exception:
981
- pass
1018
+ except Exception as ex:
1019
+ message = (
1020
+ f"The model path {os_path} does not contain the file config.json. "
1021
+ f"Please check if the path is correct or the model artifacts are available at this location."
1022
+ )
1023
+ logger.warning(
1024
+ f"{message}\n"
1025
+ f"Details: {ex.reason if isinstance(ex, AquaFileNotFoundError) else str(ex)}\n"
1026
+ )
982
1027
  else:
983
1028
  model_files.append(AQUA_MODEL_ARTIFACT_CONFIG)
984
1029
 
@@ -986,6 +1031,9 @@ class AquaModelApp(AquaApp):
986
1031
  model_files.extend(
987
1032
  list_os_files_with_extension(oss_path=os_path, extension=".gguf")
988
1033
  )
1034
+ logger.debug(
1035
+ f"Fetched {len(model_files)} model files from {os_path} for model format {model_format}."
1036
+ )
989
1037
  return model_files
990
1038
 
991
1039
  @staticmethod
@@ -1022,12 +1070,17 @@ class AquaModelApp(AquaApp):
1022
1070
 
1023
1071
  for model_sibling in model_siblings:
1024
1072
  extension = pathlib.Path(model_sibling.rfilename).suffix[1:].upper()
1025
- if model_format == ModelFormat.SAFETENSORS:
1026
- if model_sibling.rfilename == AQUA_MODEL_ARTIFACT_CONFIG:
1027
- model_files.append(model_sibling.rfilename)
1028
- elif extension == model_format.value:
1073
+ if (
1074
+ model_format == ModelFormat.SAFETENSORS
1075
+ and model_sibling.rfilename == AQUA_MODEL_ARTIFACT_CONFIG
1076
+ ):
1077
+ model_files.append(model_sibling.rfilename)
1078
+ if extension == model_format.value:
1029
1079
  model_files.append(model_sibling.rfilename)
1030
1080
 
1081
+ logger.debug(
1082
+ f"Fetched {len(model_files)} model files for the model {model_name} for model format {model_format}."
1083
+ )
1031
1084
  return model_files
1032
1085
 
1033
1086
  def _validate_model(
@@ -1061,7 +1114,10 @@ class AquaModelApp(AquaApp):
1061
1114
  safetensors_model_files = self.get_hf_model_files(
1062
1115
  model_name, ModelFormat.SAFETENSORS
1063
1116
  )
1064
- if safetensors_model_files:
1117
+ if (
1118
+ safetensors_model_files
1119
+ and AQUA_MODEL_ARTIFACT_CONFIG in safetensors_model_files
1120
+ ):
1065
1121
  hf_download_config_present = True
1066
1122
  gguf_model_files = self.get_hf_model_files(model_name, ModelFormat.GGUF)
1067
1123
  else:
@@ -1117,8 +1173,11 @@ class AquaModelApp(AquaApp):
1117
1173
  Tags.LICENSE: license_value,
1118
1174
  }
1119
1175
  validation_result.tags = hf_tags
1120
- except Exception:
1121
- pass
1176
+ except Exception as ex:
1177
+ logger.debug(
1178
+ f"An error occurred while getting tag information for model {model_name}. "
1179
+ f"Error: {str(ex)}"
1180
+ )
1122
1181
 
1123
1182
  validation_result.model_formats = model_formats
1124
1183
 
@@ -1173,40 +1232,55 @@ class AquaModelApp(AquaApp):
1173
1232
  model_name: str = None,
1174
1233
  ):
1175
1234
  if import_model_details.download_from_hf:
1176
- # validates config.json exists for safetensors model from hugginface
1177
- if not hf_download_config_present:
1235
+ # validates config.json exists for safetensors model from huggingface
1236
+ if not (
1237
+ hf_download_config_present
1238
+ or import_model_details.ignore_model_artifact_check
1239
+ ):
1178
1240
  raise AquaRuntimeError(
1179
1241
  f"The model {model_name} does not contain {AQUA_MODEL_ARTIFACT_CONFIG} file as required "
1180
1242
  f"by {ModelFormat.SAFETENSORS.value} format model."
1181
1243
  f" Please check if the model name is correct in Hugging Face repository."
1182
1244
  )
1245
+ validation_result.telemetry_model_name = model_name
1183
1246
  else:
1247
+ # validate if config.json is available from object storage, and get model name for telemetry
1248
+ model_config = None
1184
1249
  try:
1185
1250
  model_config = load_config(
1186
1251
  file_path=import_model_details.os_path,
1187
1252
  config_file_name=AQUA_MODEL_ARTIFACT_CONFIG,
1188
1253
  )
1189
1254
  except Exception as ex:
1190
- logger.error(
1191
- f"Exception occurred while loading config file from {import_model_details.os_path}"
1192
- f"Exception message: {ex}"
1193
- )
1194
- raise AquaRuntimeError(
1255
+ message = (
1195
1256
  f"The model path {import_model_details.os_path} does not contain the file config.json. "
1196
1257
  f"Please check if the path is correct or the model artifacts are available at this location."
1197
- ) from ex
1198
- else:
1258
+ )
1259
+ if not import_model_details.ignore_model_artifact_check:
1260
+ logger.error(
1261
+ f"{message}\n"
1262
+ f"Details: {ex.reason if isinstance(ex, AquaFileNotFoundError) else str(ex)}"
1263
+ )
1264
+ raise AquaRuntimeError(message) from ex
1265
+ else:
1266
+ logger.warning(
1267
+ f"{message}\n"
1268
+ f"Proceeding with model registration as ignore_model_artifact_check field is set."
1269
+ )
1270
+
1271
+ if verified_model:
1272
+ # model_type validation, log message if metadata field doesn't match.
1199
1273
  try:
1200
1274
  metadata_model_type = verified_model.custom_metadata_list.get(
1201
1275
  AQUA_MODEL_ARTIFACT_CONFIG_MODEL_TYPE
1202
1276
  ).value
1203
- if metadata_model_type:
1277
+ if metadata_model_type and model_config is not None:
1204
1278
  if AQUA_MODEL_ARTIFACT_CONFIG_MODEL_TYPE in model_config:
1205
1279
  if (
1206
1280
  model_config[AQUA_MODEL_ARTIFACT_CONFIG_MODEL_TYPE]
1207
1281
  != metadata_model_type
1208
1282
  ):
1209
- raise AquaRuntimeError(
1283
+ logger.debug(
1210
1284
  f"The {AQUA_MODEL_ARTIFACT_CONFIG_MODEL_TYPE} attribute in {AQUA_MODEL_ARTIFACT_CONFIG}"
1211
1285
  f" at {import_model_details.os_path} is invalid, expected {metadata_model_type} for "
1212
1286
  f"the model {model_name}. Please check if the path is correct or "
@@ -1218,22 +1292,26 @@ class AquaModelApp(AquaApp):
1218
1292
  f"Could not find {AQUA_MODEL_ARTIFACT_CONFIG_MODEL_TYPE} attribute in "
1219
1293
  f"{AQUA_MODEL_ARTIFACT_CONFIG}. Proceeding with model registration."
1220
1294
  )
1221
- except Exception:
1222
- pass
1223
- if verified_model:
1224
- validation_result.telemetry_model_name = verified_model.display_name
1225
- elif (
1226
- model_config is not None
1227
- and AQUA_MODEL_ARTIFACT_CONFIG_MODEL_NAME in model_config
1228
- ):
1229
- validation_result.telemetry_model_name = f"{AQUA_MODEL_TYPE_CUSTOM}_{model_config[AQUA_MODEL_ARTIFACT_CONFIG_MODEL_NAME]}"
1230
- elif (
1231
- model_config is not None
1232
- and AQUA_MODEL_ARTIFACT_CONFIG_MODEL_TYPE in model_config
1233
- ):
1234
- validation_result.telemetry_model_name = f"{AQUA_MODEL_TYPE_CUSTOM}_{model_config[AQUA_MODEL_ARTIFACT_CONFIG_MODEL_TYPE]}"
1235
- else:
1236
- validation_result.telemetry_model_name = AQUA_MODEL_TYPE_CUSTOM
1295
+ except Exception as ex:
1296
+ # todo: raise exception if model_type doesn't match. Currently log message and pass since service
1297
+ # models do not have this metadata.
1298
+ logger.debug(
1299
+ f"Error occurred while processing metadata for model {model_name}. "
1300
+ f"Exception: {str(ex)}"
1301
+ )
1302
+ validation_result.telemetry_model_name = verified_model.display_name
1303
+ elif (
1304
+ model_config is not None
1305
+ and AQUA_MODEL_ARTIFACT_CONFIG_MODEL_NAME in model_config
1306
+ ):
1307
+ validation_result.telemetry_model_name = f"{AQUA_MODEL_TYPE_CUSTOM}_{model_config[AQUA_MODEL_ARTIFACT_CONFIG_MODEL_NAME]}"
1308
+ elif (
1309
+ model_config is not None
1310
+ and AQUA_MODEL_ARTIFACT_CONFIG_MODEL_TYPE in model_config
1311
+ ):
1312
+ validation_result.telemetry_model_name = f"{AQUA_MODEL_TYPE_CUSTOM}_{model_config[AQUA_MODEL_ARTIFACT_CONFIG_MODEL_TYPE]}"
1313
+ else:
1314
+ validation_result.telemetry_model_name = AQUA_MODEL_TYPE_CUSTOM
1237
1315
 
1238
1316
  @staticmethod
1239
1317
  def _validate_gguf_format(
@@ -1322,20 +1400,24 @@ class AquaModelApp(AquaApp):
1322
1400
  Returns
1323
1401
  -------
1324
1402
  model_artifact_path (str): Location where the model artifacts are downloaded.
1325
-
1326
1403
  """
1327
1404
  # Download the model from hub
1328
- if not local_dir:
1329
- local_dir = os.path.join(os.path.expanduser("~"), "cached-model")
1330
- local_dir = os.path.join(local_dir, model_name)
1331
- os.makedirs(local_dir, exist_ok=True)
1332
- snapshot_download(
1405
+ if local_dir:
1406
+ local_dir = os.path.join(local_dir, model_name)
1407
+ os.makedirs(local_dir, exist_ok=True)
1408
+
1409
+ # if local_dir is not set, the return value points to the cached data folder
1410
+ local_dir = snapshot_download(
1333
1411
  repo_id=model_name,
1334
1412
  local_dir=local_dir,
1335
1413
  allow_patterns=allow_patterns,
1336
1414
  ignore_patterns=ignore_patterns,
1337
1415
  )
1338
1416
  # Upload to object storage and skip .cache/huggingface/ folder
1417
+ logger.debug(
1418
+ f"Uploading local artifacts from local directory {local_dir} to {os_path}."
1419
+ )
1420
+ # Upload to object storage
1339
1421
  model_artifact_path = upload_folder(
1340
1422
  os_path=os_path,
1341
1423
  local_dir=local_dir,
@@ -1365,6 +1447,8 @@ class AquaModelApp(AquaApp):
1365
1447
  ignore_patterns (list): Model files matching any of the patterns are not downloaded.
1366
1448
  Example: ["*.json"] will ignore all .json files. ["folder/*"] will ignore all files under `folder`.
1367
1449
  Patterns are Standard Wildcards (globbing patterns) and rules can be found here: https://docs.python.org/3/library/fnmatch.html
1450
+ cleanup_model_cache (bool): Deletes downloaded files from local machine after model is successfully
1451
+ registered. Set to True by default.
1368
1452
 
1369
1453
  Returns:
1370
1454
  AquaModel:
@@ -1379,6 +1463,7 @@ class AquaModelApp(AquaApp):
1379
1463
  import_model_details.model.startswith("ocid")
1380
1464
  and "datasciencemodel" in import_model_details.model
1381
1465
  ):
1466
+ logger.info(f"Fetching details for model {import_model_details.model}.")
1382
1467
  verified_model = DataScienceModel.from_id(import_model_details.model)
1383
1468
  else:
1384
1469
  # If users passes model name, check if there is model with the same name in the service model catalog. If it is there, then use that model
@@ -1416,7 +1501,6 @@ class AquaModelApp(AquaApp):
1416
1501
  ).rstrip("/")
1417
1502
  else:
1418
1503
  artifact_path = import_model_details.os_path.rstrip("/")
1419
-
1420
1504
  # Create Model catalog entry with pass by reference
1421
1505
  ds_model = self._create_model_catalog_entry(
1422
1506
  os_path=artifact_path,
@@ -1474,6 +1558,14 @@ class AquaModelApp(AquaApp):
1474
1558
  detail=validation_result.telemetry_model_name,
1475
1559
  )
1476
1560
 
1561
+ if (
1562
+ import_model_details.download_from_hf
1563
+ and import_model_details.cleanup_model_cache
1564
+ ):
1565
+ cleanup_local_hf_model_artifact(
1566
+ model_name=model_name, local_dir=import_model_details.local_dir
1567
+ )
1568
+
1477
1569
  return AquaModel(**aqua_model_attributes)
1478
1570
 
1479
1571
  def _if_show(self, model: DataScienceModel) -> bool:
@@ -1501,7 +1593,7 @@ class AquaModelApp(AquaApp):
1501
1593
  elif model_type == ModelType.BASE:
1502
1594
  filter_tag = Tags.BASE_MODEL_CUSTOM
1503
1595
  else:
1504
- raise ValueError(
1596
+ raise AquaValueError(
1505
1597
  f"Model of type {model_type} is unknown. The values should be in {ModelType.values()}"
1506
1598
  )
1507
1599
 
@@ -1541,7 +1633,10 @@ class AquaModelApp(AquaApp):
1541
1633
  oci_model = self.ds_client.get_model(model_id).data
1542
1634
  artifact_path = get_artifact_path(oci_model.custom_metadata_list)
1543
1635
  if not artifact_path:
1544
- raise AquaRuntimeError("Failed to get artifact path from custom metadata.")
1636
+ raise AquaRuntimeError(
1637
+ f"License could not be loaded. Failed to get artifact path from custom metadata for"
1638
+ f"the model {model_id}."
1639
+ )
1545
1640
 
1546
1641
  content = str(
1547
1642
  read_file(
@@ -1572,6 +1667,9 @@ class AquaModelApp(AquaApp):
1572
1667
 
1573
1668
  for aqua_model_summary in aqua_model_list:
1574
1669
  if aqua_model_summary.name.lower() == model_id_lower:
1670
+ logger.info(
1671
+ f"Found matching verified model id {aqua_model_summary.id} for the model {model_id}"
1672
+ )
1575
1673
  return aqua_model_summary.id
1576
1674
 
1577
1675
  return None