oracle-ads 2.10.1__py3-none-any.whl → 2.11.0__py3-none-any.whl

ads/model/deployment/model_deployment.py

@@ -1761,7 +1761,11 @@ class ModelDeployment(Builder):
             }
 
         logs = {}
-        if self.infrastructure.access_log:
+        if (
+            self.infrastructure.access_log and 
+            self.infrastructure.access_log.get(self.infrastructure.CONST_LOG_GROUP_ID, None)
+            and self.infrastructure.access_log.get(self.infrastructure.CONST_LOG_ID, None)
+        ):
             logs[self.infrastructure.CONST_ACCESS] = {
                 self.infrastructure.CONST_LOG_GROUP_ID: self.infrastructure.access_log.get(
                     "logGroupId", None
@@ -1770,7 +1774,11 @@ class ModelDeployment(Builder):
                     "logId", None
                 ),
             }
-        if self.infrastructure.predict_log:
+        if (
+            self.infrastructure.predict_log and 
+            self.infrastructure.predict_log.get(self.infrastructure.CONST_LOG_GROUP_ID, None)
+            and self.infrastructure.predict_log.get(self.infrastructure.CONST_LOG_ID, None)
+        ):
             logs[self.infrastructure.CONST_PREDICT] = {
                 self.infrastructure.CONST_LOG_GROUP_ID: self.infrastructure.predict_log.get(
                     "logGroupId", None

ads/model/generic_model.py

@@ -2054,6 +2054,7 @@ class GenericModel(MetadataMixin, Introspectable, EvaluatorMixin):
         remove_existing_artifact: Optional[bool] = True,
         reload: Optional[bool] = True,
         version_label: Optional[str] = None,
+        model_by_reference: Optional[bool] = False,
         **kwargs,
     ) -> str:
         """Saves model artifacts to the model catalog.
@@ -2091,6 +2092,8 @@ class GenericModel(MetadataMixin, Introspectable, EvaluatorMixin):
             The number of worker processes to use in parallel for uploading individual parts of a multipart upload.
         reload: (bool, optional)
             Whether to reload to check if `load_model()` works in `score.py`. Default to `True`.
+        model_by_reference: (bool, optional)
+            Whether model artifact is made available to Model Store by reference.
         kwargs:
             project_id: (str, optional).
                 Project OCID. If not specified, the value will be taken either
@@ -2220,6 +2223,7 @@ class GenericModel(MetadataMixin, Introspectable, EvaluatorMixin):
             overwrite_existing_artifact=overwrite_existing_artifact,
             remove_existing_artifact=remove_existing_artifact,
             parallel_process_count=parallel_process_count,
+            model_by_reference=model_by_reference,
             **kwargs,
         )
 
@@ -2620,6 +2624,7 @@ class GenericModel(MetadataMixin, Introspectable, EvaluatorMixin):
         remove_existing_artifact: Optional[bool] = True,
         model_version_set: Optional[Union[str, ModelVersionSet]] = None,
         version_label: Optional[str] = None,
+        model_by_reference: Optional[bool] = False,
         **kwargs: Dict,
     ) -> "ModelDeployment":
         """Shortcut for prepare, save and deploy steps.
@@ -2724,6 +2729,8 @@ class GenericModel(MetadataMixin, Introspectable, EvaluatorMixin):
             The Model version set OCID, or name, or `ModelVersionSet` instance.
         version_label: (str, optional). Defaults to None.
             The model version lebel.
+        model_by_reference: (bool, optional)
+            Whether model artifact is made available to Model Store by reference.
         kwargs:
             impute_values: (dict, optional).
                 The dictionary where the key is the column index(or names is accepted
@@ -2827,6 +2834,7 @@ class GenericModel(MetadataMixin, Introspectable, EvaluatorMixin):
             model_version_set=model_version_set,
             version_label=version_label,
             region=kwargs.pop("region", None),
+            model_by_reference=model_by_reference,
         )
         # Set default deployment_display_name if not specified - randomly generated easy to remember name generated
         if not deployment_display_name:

ads/model/model_file_description_schema.json

@@ -0,0 +1,68 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "properties": {
+    "models": {
+      "items": {
+        "properties": {
+          "bucketName": {
+            "type": "string"
+          },
+          "namespace": {
+            "type": "string"
+          },
+          "objects": {
+            "items": {
+              "properties": {
+                "name": {
+                  "type": "string"
+                },
+                "sizeInBytes": {
+                  "minimum": 0,
+                  "type": "integer"
+                },
+                "version": {
+                  "type": "string"
+                }
+              },
+              "required": [
+                "name",
+                "version",
+                "sizeInBytes"
+              ],
+              "type": "object"
+            },
+            "minItems": 1,
+            "type": "array"
+          },
+          "prefix": {
+            "type": "string"
+          }
+        },
+        "required": [
+          "namespace",
+          "bucketName",
+          "prefix",
+          "objects"
+        ],
+        "type": "object"
+      },
+      "minItems": 1,
+      "type": "array"
+    },
+    "type": {
+      "enum": [
+        "modelOSSReferenceDescription"
+      ],
+      "type": "string"
+    },
+    "version": {
+      "type": "string"
+    }
+  },
+  "required": [
+    "version",
+    "type",
+    "models"
+  ],
+  "type": "object"
+}

ads/model/model_metadata.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*--
 
-# Copyright (c) 2021, 2023 Oracle and/or its affiliates.
+# Copyright (c) 2021, 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 import json

ads/model/service/oci_datascience_model.py

@@ -38,6 +38,8 @@ MODEL_NEEDS_TO_BE_SAVED = (
     "Model needs to be saved to the Model Catalog before it can be accessed."
 )
 
+MODEL_BY_REFERENCE_DESC = "modelDescription"
+
 
 class ModelProvenanceNotFoundError(Exception):  # pragma: no cover
     pass
@@ -304,18 +306,25 @@ class OCIDataScienceModel(
     @check_for_model_id(
         msg="Model needs to be saved to the Model Catalog before the artifact can be created."
     )
-    def create_model_artifact(self, bytes_content: BytesIO) -> None:
+    def create_model_artifact(
+        self,
+        bytes_content: BytesIO,
+        extension: str = None,
+    ) -> None:
         """Creates model artifact for specified model.
 
         Parameters
         ----------
         bytes_content: BytesIO
             Model artifacts to upload.
+        extension: str
+            File extension, defaults to zip
         """
+        ext = ".json" if extension and extension.lower() == ".json" else ".zip"
         self.client.create_model_artifact(
             self.id,
             bytes_content,
-            content_disposition=f'attachment; filename="{self.id}.zip"',
+            content_disposition=f'attachment; filename="{self.id}{ext}"',
         )
 
     @check_for_model_id(
@@ -423,10 +432,14 @@ class OCIDataScienceModel(
         OCIDataScienceModel
             The `OCIDataScienceModel` instance (self).
         """
+
+        model_details = self.to_oci_model(UpdateModelDetails)
+
+        # Clean up the model version set, otherwise it throws an error that model is already
+        # associated with the model version set.
+        model_details.model_version_set_id = None
         return self.update_from_oci_model(
-            self.client.update_model(
-                self.id, self.to_oci_model(UpdateModelDetails)
-            ).data
+            self.client.update_model(self.id, model_details).data
         )
 
     @check_for_model_id(
@@ -539,3 +552,19 @@ class OCIDataScienceModel(
         if not ocid:
             raise ValueError("Model OCID not provided.")
         return super().from_ocid(ocid)
+
+    def is_model_by_reference(self):
+        """Checks if model is created by reference
+        Returns
+        -------
+            bool flag denoting whether model was created by reference.
+
+        """
+        if self.custom_metadata_list:
+            for metadata in self.custom_metadata_list:
+                if (
+                    metadata.key == MODEL_BY_REFERENCE_DESC
+                    and metadata.value.lower() == "true"
+                ):
+                    return True
+        return False

ads/opctl/operator/lowcode/anomaly/README.md

@@ -37,7 +37,8 @@ To run anomaly detection locally, create and activate a new conda environment (`
 ```yaml
 - datapane
 - cerberus
-- oracle-automlx==23.2.3
+- oracle-automlx==23.4.1
+- oracle-automlx[classic]==23.4.1
 - "git+https://github.com/oracle/accelerated-data-science.git@feature/anomaly#egg=oracle-ads"
 ```
 

ads/opctl/operator/lowcode/anomaly/__main__.py

@@ -36,11 +36,17 @@ def operate(operator_config: AnomalyOperatorConfig) -> None:
             operator_config.spec.model = "autots"
             operator_config.spec.model_kwargs = dict()
             datasets = AnomalyDatasets(operator_config.spec)
-            AnomalyOperatorModelFactory.get_model(
-                operator_config, datasets
-            ).generate_report()
+            try:
+                AnomalyOperatorModelFactory.get_model(
+                    operator_config, datasets
+                ).generate_report()
+            except Exception as e2:
+                logger.debug(
+                    f"Failed to backup forecast with error {e2.args}. Raising original error."
+                )
+            raise e
         else:
-            raise
+            raise e
 
 
 def verify(spec: Dict, **kwargs: Dict) -> bool:

ads/opctl/operator/lowcode/anomaly/environment.yaml

@@ -7,5 +7,6 @@ dependencies:
   - pip:
       - datapane
       - cerberus
-      - oracle-automlx==23.2.3
+      - oracle-automlx==23.4.1
+      - oracle-automlx[classic]==23.4.1
       - "oracle-ads[anomaly]"

ads/opctl/operator/lowcode/anomaly/model/automlx.py

@@ -17,20 +17,26 @@ class AutoMLXOperatorModel(AnomalyOperatorBaseModel):
     """Class representing AutoMLX operator model."""
 
     @runtime_dependency(
-        module="automl",
+        module="automlx",
         err_msg=(
-            "Please run `pip3 install oracle-automlx==23.2.3` to "
-            "install the required dependencies for automlx."
+            "Please run `pip3 install oracle-automlx==23.4.1` and "
+            "`pip3 install oracle-automlx[classic]==23.4.1` "
+            "to install the required dependencies for automlx."
         ),
     )
     def _build_model(self) -> pd.DataFrame:
+        from automlx import init
+        try:
+            init(engine="ray", engine_opts={"ray_setup": {"_temp_dir": "/tmp/ray-temp"}})
+        except Exception as e:
+            logger.info("Ray already initialized")
         date_column = self.spec.datetime_column.name
         anomaly_output = AnomalyOutput(date_column=date_column)
+        time_budget = self.spec.model_kwargs.pop("time_budget", -1)
 
-        time_budget = self.spec.model_kwargs.pop("time_budget", None)
         # Iterate over the full_data_dict items
         for target, df in self.datasets.full_data_dict.items():
-            est = automl.Pipeline(task="anomaly_detection", **self.spec.model_kwargs)
+            est = automlx.Pipeline(task="anomaly_detection", **self.spec.model_kwargs)
             est.fit(
                 X=df,
                 X_valid=self.X_valid_dict[target]
@@ -39,10 +45,10 @@ class AutoMLXOperatorModel(AnomalyOperatorBaseModel):
                 y_valid=self.y_valid_dict[target]
                 if self.y_valid_dict is not None
                 else None,
-                time_budget=time_budget,
                 contamination=self.spec.contamination
                 if self.y_valid_dict is not None
                 else None,
+                time_budget=time_budget,
             )
             y_pred = est.predict(df)
             scores = est.predict_proba(df)

ads/opctl/operator/lowcode/forecast/README.md

@@ -38,8 +38,9 @@ To run forecasting locally, create and activate a new conda environment (`ads-fo
 - datapane
 - cerberus
 - sktime
-- optuna==2.9.0
-- oracle-automlx==23.2.3
+- optuna==3.1.0
+- oracle-automlx==23.4.1
+- oracle-automlx[forecasting]==23.4.1
 - oracle-ads>=2.9.0
 ```
 

ads/opctl/operator/lowcode/forecast/environment.yaml

@@ -15,5 +15,6 @@ dependencies:
       - sktime
       - shap
       - autots[additional]
-      - optuna==2.9.0
-      - oracle-automlx==23.2.3
+      - optuna==3.1.0
+      - oracle-automlx==23.4.1
+      - oracle-automlx[forecasting]==23.4.1

ads/opctl/operator/lowcode/forecast/model/automlx.py

@@ -45,7 +45,7 @@ class AutoMLXOperatorModel(ForecastOperatorBaseModel):
             model_kwargs_cleaned.get("score_metric", AUTOMLX_DEFAULT_SCORE_METRIC),
         )
         model_kwargs_cleaned.pop("task", None)
-        time_budget = model_kwargs_cleaned.pop("time_budget", None)
+        time_budget = model_kwargs_cleaned.pop("time_budget", -1)
         model_kwargs_cleaned[
             "preprocessing"
         ] = self.spec.preprocessing or model_kwargs_cleaned.get("preprocessing", True)
@@ -55,9 +55,11 @@ class AutoMLXOperatorModel(ForecastOperatorBaseModel):
         return data.set_index(self.spec.datetime_column.name)
 
     @runtime_dependency(
-        module="automl",
+        module="automlx",
         err_msg=(
-            "Please run `pip3 install oracle-automlx==23.2.3` to install the required dependencies for automlx."
+            "Please run `pip3 install oracle-automlx==23.4.1` and "
+            "`pip3 install oracle-automlx[forecasting]==23.4.1` "
+            "to install the required dependencies for automlx."
         ),
     )
     @runtime_dependency(
@@ -67,15 +69,13 @@ class AutoMLXOperatorModel(ForecastOperatorBaseModel):
         ),
     )
     def _build_model(self) -> pd.DataFrame:
-        from automl import init
+        from automlx import init
         from sktime.forecasting.model_selection import temporal_train_test_split
+        try:
+            init(engine="ray", engine_opts={"ray_setup": {"_temp_dir": "/tmp/ray-temp"}})
+        except Exception as e:
+            logger.info("Ray already initialized")
 
-        init(
-            engine="local",
-            engine_opts={"n_jobs": -1, "model_n_jobs": -1},
-            check_deprecation_warnings=False,
-            logger=50,
-        )
 
         full_data_dict = self.datasets.get_data_by_series()
 
@@ -95,7 +95,7 @@ class AutoMLXOperatorModel(ForecastOperatorBaseModel):
 
         for i, (s_id, df) in enumerate(full_data_dict.items()):
             try:
-                logger.debug(f"Running automl on series {s_id}")
+                logger.debug(f"Running automlx on series {s_id}")
                 model_kwargs = model_kwargs_cleaned.copy()
                 target = self.original_target_column
                 self.forecast_output.init_series_output(
@@ -110,7 +110,7 @@ class AutoMLXOperatorModel(ForecastOperatorBaseModel):
                 if self.loaded_models is not None:
                     model = self.loaded_models[s_id]
                 else:
-                    model = automl.Pipeline(
+                    model = automlx.Pipeline(
                         task="forecasting",
                         **model_kwargs,
                     )
@@ -149,18 +149,7 @@ class AutoMLXOperatorModel(ForecastOperatorBaseModel):
 
                 self.model_parameters[s_id] = {
                     "framework": SupportedModels.AutoMLX,
-                    "score_metric": model.score_metric,
-                    "random_state": model.random_state,
-                    "model_list": model.model_list,
-                    "n_algos_tuned": model.n_algos_tuned,
-                    "adaptive_sampling": model.adaptive_sampling,
-                    "min_features": model.min_features,
-                    "optimization": model.optimization,
-                    "preprocessing": model.preprocessing,
-                    "search_space": model.search_space,
                     "time_series_period": model.time_series_period,
-                    "min_class_instances": model.min_class_instances,
-                    "max_tuning_trials": model.max_tuning_trials,
                     "selected_model": model.selected_model_,
                     "selected_model_params": model.selected_model_params_,
                 }

ads/telemetry/base.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (c) 2024 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import logging
+
+from ads import set_auth
+from ads.common import oci_client as oc
+from ads.common.auth import default_signer
+from ads.config import OCI_RESOURCE_PRINCIPAL_VERSION
+
+
+logger = logging.getLogger(__name__)
+class TelemetryBase:
+    """Base class for Telemetry Client."""
+
+    def __init__(self, bucket: str, namespace: str = None) -> None:
+        """Initializes the telemetry client.
+
+        Parameters
+        ----------
+        bucket : str
+            OCI object storage bucket name storing the telemetry objects.
+        namespace : str, optional
+            Namespace of the OCI object storage bucket, by default None.
+        """
+        if OCI_RESOURCE_PRINCIPAL_VERSION:
+            set_auth("resource_principal")
+        self._auth = default_signer()
+        self.os_client = oc.OCIClientFactory(**self._auth).object_storage
+        self.bucket = bucket
+        self._namespace = namespace
+        self._service_endpoint = None
+        logger.debug(f"Initialized Telemetry. Namespace: {self.namespace}, Bucket: {self.bucket}")
+
+
+    @property
+    def namespace(self) -> str:
+        """Gets the namespace of the object storage from the tenancy.
+
+        Returns
+        -------
+        str
+            The namespace of the tenancy.
+        """
+        if not self._namespace:
+            self._namespace = self.os_client.get_namespace().data
+        return self._namespace
+
+    @property
+    def service_endpoint(self):
+        """Gets the tenancy-specific endpoint.
+
+        Returns
+        -------
+        str
+            Tenancy-specific endpoint.
+        """
+        if not self._service_endpoint:
+            self._service_endpoint = self.os_client.base_client.endpoint
+        return self._service_endpoint

ads/telemetry/client.py

@@ -0,0 +1,105 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (c) 2024 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+
+import logging
+import threading
+import urllib.parse
+import requests
+from requests import Response
+from .base import TelemetryBase
+from ads.config import DEBUG_TELEMETRY
+
+
+logger = logging.getLogger(__name__)
+
+
+class TelemetryClient(TelemetryBase):
+    """Represents a telemetry python client providing functions to record an event.
+
+    Methods
+    -------
+    record_event(category: str = None, action: str = None, path: str = None, **kwargs) -> None
+        Send a head request to generate an event record.
+    record_event_async(category: str = None, action: str = None, path: str = None,  **kwargs)
+        Starts thread to send a head request to generate an event record.
+
+    Examples
+    --------
+    >>> import os
+    >>> import traceback
+    >>> from ads.telemetry.client import TelemetryClient
+    >>> AQUA_BUCKET = os.environ.get("AQUA_BUCKET", "service-managed-models")
+    >>> AQUA_BUCKET_NS = os.environ.get("AQUA_BUCKET_NS", "ociodscdev")
+    >>> telemetry = TelemetryClient(bucket=AQUA_BUCKET, namespace=AQUA_BUCKET_NS)
+    >>> telemetry.record_event_async(category="aqua/service/model", action="create") # records create action
+    >>> telemetry.record_event_async(category="aqua/service/model/create", action="shape", detail="VM.GPU.A10.1")
+    """
+
+    @staticmethod
+    def _encode_user_agent(**kwargs):
+        message = urllib.parse.urlencode(kwargs)
+        return message
+
+    def record_event(
+        self, category: str = None, action: str = None, detail: str = None, **kwargs
+    ) -> Response:
+        """Send a head request to generate an event record.
+
+        Parameters
+        ----------
+        category: (str)
+            Category of the event, which is also the path to the directory containing the object representing the event.
+        action: (str)
+            Filename of the object representing the event.
+        detail: (str)
+            Can be used to pass additional values, if required. When set, detail is converted to an action,
+            category and action are grouped together for telemetry parsing in the backend.
+        **kwargs:
+            Can be used to pass additional attributes like value that will be passed in the headers of the request.
+
+        Returns
+        -------
+        Response
+        """
+        try:
+            if not category or not action:
+                raise ValueError("Please specify the category and the action.")
+            if detail:
+                category, action = f"{category}/{action}", detail
+            endpoint = f"{self.service_endpoint}/n/{self.namespace}/b/{self.bucket}/o/telemetry/{category}/{action}"
+            headers = {"User-Agent": self._encode_user_agent(**kwargs)}
+            logger.debug(f"Sending telemetry to endpoint: {endpoint}")
+            signer = self._auth["signer"]
+            response = requests.head(endpoint, auth=signer, headers=headers)
+            logger.debug(f"Telemetry status code: {response.status_code}")
+            return response
+        except Exception as e:
+            if DEBUG_TELEMETRY:
+                logger.error(f"There is an error recording telemetry: {e}")
+
+    def record_event_async(
+        self, category: str = None, action: str = None, detail: str = None, **kwargs
+    ):
+        """Send a head request to generate an event record.
+
+        Parameters
+        ----------
+        category (str)
+            Category of the event, which is also the path to the directory containing the object representing the event.
+        action (str)
+            Filename of the object representing the event.
+
+        Returns
+        -------
+        Thread
+            A started thread to send a head request to generate an event record.
+        """
+        thread = threading.Thread(
+            target=self.record_event, args=(category, action, detail), kwargs=kwargs
+        )
+        thread.daemon = True
+        thread.start()
+        return thread

ads/telemetry/telemetry.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*--
 
-# Copyright (c) 2022, 2023 Oracle and/or its affiliates.
+# Copyright (c) 2022, 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 import os
@@ -102,7 +102,10 @@ def telemetry(
                 entry_point
             )
             try:
-                return func(*args, **{**kwargs, **{TELEMETRY_ARGUMENT_NAME: telemetry}})
+                # todo: inject telemetry arg and later update all functions that use the @telemetry
+                #   decorator to accept **kwargs. Comment the below line as some aqua apis don't support kwargs.
+                # return func(*args, **{**kwargs, **{TELEMETRY_ARGUMENT_NAME: telemetry}})
+                return func(*args, **kwargs)
             except:
                 raise
             finally:
@@ -178,7 +181,7 @@ class Telemetry:
         self: Telemetry
             An instance of the Telemetry.
         """
-        os.environ[self.environ_variable] = self._original_value
+        os.environ[self.environ_variable] = self._original_value or ""
         return self
 
     def clean(self) -> "Telemetry":