oracle-ads 2.10.0__py3-none-any.whl → 2.11.0__py3-none-any.whl

ads/model/deployment/model_deployment.py

@@ -1761,7 +1761,11 @@ class ModelDeployment(Builder):
             }
 
         logs = {}
-        if self.infrastructure.access_log:
+        if (
+            self.infrastructure.access_log and 
+            self.infrastructure.access_log.get(self.infrastructure.CONST_LOG_GROUP_ID, None)
+            and self.infrastructure.access_log.get(self.infrastructure.CONST_LOG_ID, None)
+        ):
             logs[self.infrastructure.CONST_ACCESS] = {
                 self.infrastructure.CONST_LOG_GROUP_ID: self.infrastructure.access_log.get(
                     "logGroupId", None
@@ -1770,7 +1774,11 @@ class ModelDeployment(Builder):
                     "logId", None
                 ),
             }
-        if self.infrastructure.predict_log:
+        if (
+            self.infrastructure.predict_log and 
+            self.infrastructure.predict_log.get(self.infrastructure.CONST_LOG_GROUP_ID, None)
+            and self.infrastructure.predict_log.get(self.infrastructure.CONST_LOG_ID, None)
+        ):
             logs[self.infrastructure.CONST_PREDICT] = {
                 self.infrastructure.CONST_LOG_GROUP_ID: self.infrastructure.predict_log.get(
                     "logGroupId", None

ads/model/generic_model.py

@@ -2054,6 +2054,7 @@ class GenericModel(MetadataMixin, Introspectable, EvaluatorMixin):
         remove_existing_artifact: Optional[bool] = True,
         reload: Optional[bool] = True,
         version_label: Optional[str] = None,
+        model_by_reference: Optional[bool] = False,
         **kwargs,
     ) -> str:
         """Saves model artifacts to the model catalog.
@@ -2091,6 +2092,8 @@ class GenericModel(MetadataMixin, Introspectable, EvaluatorMixin):
             The number of worker processes to use in parallel for uploading individual parts of a multipart upload.
         reload: (bool, optional)
             Whether to reload to check if `load_model()` works in `score.py`. Default to `True`.
+        model_by_reference: (bool, optional)
+            Whether model artifact is made available to Model Store by reference.
         kwargs:
             project_id: (str, optional).
                 Project OCID. If not specified, the value will be taken either
@@ -2220,6 +2223,7 @@ class GenericModel(MetadataMixin, Introspectable, EvaluatorMixin):
             overwrite_existing_artifact=overwrite_existing_artifact,
             remove_existing_artifact=remove_existing_artifact,
             parallel_process_count=parallel_process_count,
+            model_by_reference=model_by_reference,
             **kwargs,
         )
 
@@ -2620,6 +2624,7 @@ class GenericModel(MetadataMixin, Introspectable, EvaluatorMixin):
         remove_existing_artifact: Optional[bool] = True,
         model_version_set: Optional[Union[str, ModelVersionSet]] = None,
         version_label: Optional[str] = None,
+        model_by_reference: Optional[bool] = False,
         **kwargs: Dict,
     ) -> "ModelDeployment":
         """Shortcut for prepare, save and deploy steps.
@@ -2724,6 +2729,8 @@ class GenericModel(MetadataMixin, Introspectable, EvaluatorMixin):
             The Model version set OCID, or name, or `ModelVersionSet` instance.
         version_label: (str, optional). Defaults to None.
             The model version lebel.
+        model_by_reference: (bool, optional)
+            Whether model artifact is made available to Model Store by reference.
         kwargs:
             impute_values: (dict, optional).
                 The dictionary where the key is the column index(or names is accepted
@@ -2827,6 +2834,7 @@ class GenericModel(MetadataMixin, Introspectable, EvaluatorMixin):
             model_version_set=model_version_set,
             version_label=version_label,
             region=kwargs.pop("region", None),
+            model_by_reference=model_by_reference,
         )
         # Set default deployment_display_name if not specified - randomly generated easy to remember name generated
         if not deployment_display_name:

ads/model/model_file_description_schema.json

@@ -0,0 +1,68 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "properties": {
+    "models": {
+      "items": {
+        "properties": {
+          "bucketName": {
+            "type": "string"
+          },
+          "namespace": {
+            "type": "string"
+          },
+          "objects": {
+            "items": {
+              "properties": {
+                "name": {
+                  "type": "string"
+                },
+                "sizeInBytes": {
+                  "minimum": 0,
+                  "type": "integer"
+                },
+                "version": {
+                  "type": "string"
+                }
+              },
+              "required": [
+                "name",
+                "version",
+                "sizeInBytes"
+              ],
+              "type": "object"
+            },
+            "minItems": 1,
+            "type": "array"
+          },
+          "prefix": {
+            "type": "string"
+          }
+        },
+        "required": [
+          "namespace",
+          "bucketName",
+          "prefix",
+          "objects"
+        ],
+        "type": "object"
+      },
+      "minItems": 1,
+      "type": "array"
+    },
+    "type": {
+      "enum": [
+        "modelOSSReferenceDescription"
+      ],
+      "type": "string"
+    },
+    "version": {
+      "type": "string"
+    }
+  },
+  "required": [
+    "version",
+    "type",
+    "models"
+  ],
+  "type": "object"
+}

ads/model/model_metadata.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*--
 
-# Copyright (c) 2021, 2023 Oracle and/or its affiliates.
+# Copyright (c) 2021, 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 import json

ads/model/service/oci_datascience_model.py

@@ -38,6 +38,8 @@ MODEL_NEEDS_TO_BE_SAVED = (
     "Model needs to be saved to the Model Catalog before it can be accessed."
 )
 
+MODEL_BY_REFERENCE_DESC = "modelDescription"
+
 
 class ModelProvenanceNotFoundError(Exception):  # pragma: no cover
     pass
@@ -304,18 +306,25 @@ class OCIDataScienceModel(
     @check_for_model_id(
         msg="Model needs to be saved to the Model Catalog before the artifact can be created."
     )
-    def create_model_artifact(self, bytes_content: BytesIO) -> None:
+    def create_model_artifact(
+        self,
+        bytes_content: BytesIO,
+        extension: str = None,
+    ) -> None:
         """Creates model artifact for specified model.
 
         Parameters
         ----------
         bytes_content: BytesIO
             Model artifacts to upload.
+        extension: str
+            File extension, defaults to zip
         """
+        ext = ".json" if extension and extension.lower() == ".json" else ".zip"
         self.client.create_model_artifact(
             self.id,
             bytes_content,
-            content_disposition=f'attachment; filename="{self.id}.zip"',
+            content_disposition=f'attachment; filename="{self.id}{ext}"',
         )
 
     @check_for_model_id(
@@ -423,10 +432,14 @@ class OCIDataScienceModel(
         OCIDataScienceModel
             The `OCIDataScienceModel` instance (self).
         """
+
+        model_details = self.to_oci_model(UpdateModelDetails)
+
+        # Clean up the model version set, otherwise it throws an error that model is already
+        # associated with the model version set.
+        model_details.model_version_set_id = None
         return self.update_from_oci_model(
-            self.client.update_model(
-                self.id, self.to_oci_model(UpdateModelDetails)
-            ).data
+            self.client.update_model(self.id, model_details).data
         )
 
     @check_for_model_id(
@@ -539,3 +552,19 @@ class OCIDataScienceModel(
         if not ocid:
             raise ValueError("Model OCID not provided.")
         return super().from_ocid(ocid)
+
+    def is_model_by_reference(self):
+        """Checks if model is created by reference
+        Returns
+        -------
+            bool flag denoting whether model was created by reference.
+
+        """
+        if self.custom_metadata_list:
+            for metadata in self.custom_metadata_list:
+                if (
+                    metadata.key == MODEL_BY_REFERENCE_DESC
+                    and metadata.value.lower() == "true"
+                ):
+                    return True
+        return False

ads/opctl/config/merger.py

@@ -11,7 +11,7 @@ import json
 
 import yaml
 
-from ads.common.auth import AuthType
+from ads.common.auth import AuthType, ResourcePrincipal
 from ads.opctl import logger
 from ads.opctl.config.base import ConfigProcessor
 from ads.opctl.config.utils import read_from_ini, _DefaultNoneDict
@@ -115,7 +115,7 @@ class ConfigMerger(ConfigProcessor):
         )
         # set default auth
         if not self.config["execution"].get("auth", None):
-            if is_in_notebook_session():
+            if ResourcePrincipal.supported():
                 self.config["execution"]["auth"] = (
                     exec_config.get("auth") or AuthType.RESOURCE_PRINCIPAL
                 )

ads/opctl/operator/__init__.py

@@ -14,7 +14,9 @@ def __registered_operators():
     return [
         f
         for f in os.listdir(target_dir)
-        if os.path.isdir(os.path.join(target_dir, f)) and not f.startswith("__")
+        if os.path.isdir(os.path.join(target_dir, f))
+        and not f.startswith("__")
+        and f != "common"
     ]
 
 

ads/opctl/operator/cli.py

@@ -9,12 +9,14 @@ from typing import Any, Dict
 import click
 import fsspec
 import yaml
+import logging
 from ads.opctl.operator.common.utils import default_signer
 from ads.common.auth import AuthType
 from ads.common.object_storage_details import ObjectStorageDetails
 from ads.opctl.constants import BACKEND_NAME, RUNTIME_TYPE
 from ads.opctl.decorator.common import click_options, with_auth, with_click_unknown_args
 from ads.opctl.utils import suppress_traceback
+from ads.opctl import logger
 
 from .__init__ import __operators__
 from .cmd import run as cmd_run
@@ -311,10 +313,14 @@ def publish_conda(debug: bool, **kwargs: Dict[str, Any]) -> None:
 @click.pass_context
 @with_click_unknown_args
 @with_auth
-def run(ctx: click.core.Context, debug: bool, **kwargs: Dict[str, Any]) -> None:
+def run(ctx: click.core.Context, debug: bool = False, **kwargs: Dict[str, Any]) -> None:
     """
     Runs the operator with the given specification on the targeted backend.
     """
+    if debug:
+        logger.setLevel(logging.DEBUG)
+    else:
+        logger.setLevel(logging.CRITICAL)
     operator_spec = {}
     backend = kwargs.pop("backend")
 

ads/opctl/operator/cmd.py

@@ -48,7 +48,7 @@ from .common.backend_factory import BackendFactory
 from .common.errors import (
     OperatorCondaNotFoundError,
     OperatorImageNotFoundError,
-    OperatorSchemaYamlError,
+    InvalidParameterError,
 )
 from .common.operator_loader import _operator_info_list
 
@@ -167,7 +167,7 @@ def init(
             )
     else:
         overwrite = True
-        output = os.path.join(tempfile.TemporaryDirectory().name, "")
+        output = operator_utils.create_output_folder(name=type + "/")
 
     # generating operator specification
     operator_config = {}
@@ -422,7 +422,7 @@ def verify(
             run_name="verify",
         )
         operator_module.get("verify")(config, **kwargs)
-    except OperatorSchemaYamlError as ex:
+    except InvalidParameterError as ex:
         logger.debug(ex)
         raise ValueError(
             f"The operator's specification is not valid for the `{operator_info.type}` operator. "

ads/opctl/operator/common/errors.py

@@ -7,8 +7,9 @@
 from ads.opctl.operator import __operators__
 
 
-class OperatorSchemaYamlError(Exception):
+class InvalidParameterError(Exception):
     """Exception raised when there is an issue with the schema."""
+
     def __init__(self, error: str):
         super().__init__(
             "Invalid operator specification. Check the YAML structure and ensure it "

ads/opctl/operator/common/operator_config.py

@@ -8,12 +8,31 @@
 import json
 from abc import abstractmethod
 from dataclasses import dataclass
-from typing import Any, Dict
+from typing import Any, Dict, List
 
 from ads.common.serializer import DataClassSerializable
 
 from ads.opctl.operator.common.utils import OperatorValidator
-from ads.opctl.operator.common.errors import OperatorSchemaYamlError
+from ads.opctl.operator.common.errors import InvalidParameterError
+
+@dataclass(repr=True)
+class InputData(DataClassSerializable):
+    """Class representing operator specification input data details."""
+
+    connect_args: Dict = None
+    format: str = None
+    columns: List[str] = None
+    url: str = None
+    filters: List[str] = None
+    options: Dict = None
+    limit: int = None
+    sql: str = None
+    table_name: str = None
+
+
+@dataclass(repr=True)
+class OutputDirectory(InputData):
+    """Class representing operator specification output directory details."""
 
 
 @dataclass(repr=True)
@@ -65,7 +84,7 @@ class OperatorConfig(DataClassSerializable):
         result = validator.validate(obj_dict)
 
         if not result:
-            raise OperatorSchemaYamlError(json.dumps(validator.errors, indent=2))
+            raise InvalidParameterError(json.dumps(validator.errors, indent=2))
         return True
 
     @classmethod

ads/opctl/operator/common/utils.py

@@ -29,6 +29,22 @@ class OperatorValidator(Validator):
     pass
 
 
+def create_output_folder(name):
+    output_folder = name
+    protocol = fsspec.utils.get_protocol(output_folder)
+    storage_options = {}
+    if protocol != "file":
+        storage_options = auth or default_signer()
+
+    fs = fsspec.filesystem(protocol, **storage_options)
+    name_suffix = 1
+    while fs.exists(output_folder):
+        name_suffix = name_suffix + 1
+        output_folder = f"{name}_{name_suffix}"
+    fs.mkdirs(output_folder)
+    return output_folder
+
+
 def _build_image(
     dockerfile: str,
     image_name: str,

ads/opctl/operator/lowcode/anomaly/MLoperator

@@ -0,0 +1,15 @@
+type: anomaly
+version: v1
+conda_type: service
+name: Anomaly Detection Operator
+gpu: no
+keywords:
+  - Anomaly Detection
+backends:
+  - job
+  - operator.local
+description: |
+  Anomaly Detection is the identification of rare items, events, or observations in data that
+  differ significantly from the expectation. This can be used for several scenarios like asset
+  monitoring, maintenance and prognostic surveillance in industries such as utility,
+  aviation and manufacturing.

ads/opctl/operator/lowcode/anomaly/README.md

@@ -0,0 +1,209 @@
+# Anomaly Detection Operator
+
+Anomaly Detection is the identification of rare items, events, or observations in data that differ significantly from the expectation. This can be used for several scenarios like asset monitoring, maintenance and prognostic surveillance in industries such as utility, aviation and manufacturing.
+
+Below are the steps to configure and run the Anomaly Detection Operator on different resources.
+
+## 1. Prerequisites
+
+Follow the [CLI Configuration](https://accelerated-data-science.readthedocs.io/en/latest/user_guide/cli/opctl/configure.html) steps from the ADS documentation. This step is mandatory as it sets up default values for different options while running the Anomaly Detection Operator on OCI Data Science jobs or OCI Data Flow applications. If you have previously done this and used a flexible shape, make sure to adjust `ml_job_config.ini` with shape config details and `docker_registry` information.
+
+- ocpus = 1
+- memory_in_gbs = 16
+- docker_registry = `<iad.ocir.io/namespace/>`
+
+## 2. Generating configs
+
+To generate starter configs, run the command below. This will create a list of YAML configs and place them in the `output` folder.
+
+```bash
+ads operator init -t anomaly --overwrite --output ~/anomaly/
+```
+
+The most important files expected to be generated are:
+
+- `anomaly.yaml`: Contains anomaly detection related configuration.
+- `backend_operator_local_python_config.yaml`: This includes a local backend configuration for running anomaly detection in a local environment. The environment should be set up manually before running the operator.
+- `backend_operator_local_container_config.yaml`: This includes a local backend configuration for running anomaly detection within a local container. The container should be built before running the operator. Please refer to the instructions below for details on how to accomplish this.
+- `backend_job_container_config.yaml`: Contains Data Science job-related config to run anomaly detection in a Data Science job within a container (BYOC) runtime. The container should be built and published before running the operator. Please refer to the instructions below for details on how to accomplish this.
+- `backend_job_python_config.yaml`: Contains Data Science job-related config to run anomaly detection in a Data Science job within a conda runtime. The conda should be built and published before running the operator.
+
+All generated configurations should be ready to use without the need for any additional adjustments. However, they are provided as starter kit configurations that can be customized as needed.
+
+## 3. Running anomaly detection on the local conda environment
+
+To run anomaly detection locally, create and activate a new conda environment (`ads-anomaly`). Install all the required libraries listed in the `environment.yaml` file.
+
+```yaml
+- datapane
+- cerberus
+- oracle-automlx==23.4.1
+- oracle-automlx[classic]==23.4.1
+- "git+https://github.com/oracle/accelerated-data-science.git@feature/anomaly#egg=oracle-ads"
+```
+
+Please review the previously generated `anomaly.yaml` file using the `init` command, and make any necessary adjustments to the input and output file locations. By default, it assumes that the files should be located in the same folder from which the `init` command was executed.
+
+Use the command below to verify the anomaly detection config.
+
+```bash
+ads operator verify -f ~/anomaly/anomaly.yaml
+```
+
+Use the following command to run the anomaly detection within the `ads-anomaly` conda environment.
+
+```bash
+ads operator run -f ~/anomaly/anomaly.yaml -b local
+```
+
+The operator will run in your local environment without requiring any additional modifications.
+
+## 4. Running anomaly detection on the local container
+
+To run the anomaly detection detection operator within a local container, follow these steps:
+
+Use the command below to build the anomaly detection container.
+
+```bash
+ads operator build-image -t anomaly
+```
+
+This will create a new `anomaly:v1` image, with `/etc/operator` as the designated working directory within the container.
+
+
+Check the `backend_operator_local_container_config.yaml` config file. By default, it should have a `volume` section with the `.oci` configs folder mounted.
+
+```yaml
+volume:
+  - "/Users/<user>/.oci:/root/.oci"
+```
+
+Mounting the OCI configs folder is only required if an OCI Object Storage bucket will be used to store the input anomaly detection data or output anomaly detection result. The input/output folders can also be mounted to the container.
+
+```yaml
+volume:
+  - /Users/<user>/.oci:/root/.oci
+  - /Users/<user>/anomaly/data:/etc/operator/data
+  - /Users/<user>/anomaly/result:/etc/operator/result
+```
+
+The full config can look like:
+```yaml
+kind: operator.local
+spec:
+  image: anomaly:v1
+  volume:
+  - /Users/<user>/.oci:/root/.oci
+  - /Users/<user>/anomaly/data:/etc/operator/data
+  - /Users/<user>/anomaly/result:/etc/operator/result
+type: container
+version: v1
+```
+
+Run the anomaly detection within a container using the command below:
+
+```bash
+ads operator run -f ~/anomaly/anomaly.yaml --backend-config ~/anomaly/backend_operator_local_container_config.yaml
+```
+
+## 5. Running anomaly detection in the Data Science job within container runtime
+
+To execute the anomaly detection detection operator within a Data Science job using container runtime, please follow the steps outlined below:
+
+You can use the following command to build the anomaly detection container. This step can be skipped if you have already done this for running the operator within a local container.
+
+```bash
+ads operator build-image -t anomaly
+```
+
+This will create a new `anomaly:v1` image, with `/etc/operator` as the designated working directory within the container.
+
+Publish the `anomaly:v1` container to the [Oracle Container Registry](https://docs.public.oneportal.content.oci.oraclecloud.com/en-us/iaas/Content/Registry/home.htm). To become familiar with OCI, read the documentation links posted below.
+
+- [Access Container Registry](https://docs.public.oneportal.content.oci.oraclecloud.com/en-us/iaas/Content/Registry/Concepts/registryoverview.htm#access)
+- [Create repositories](https://docs.public.oneportal.content.oci.oraclecloud.com/en-us/iaas/Content/Registry/Tasks/registrycreatingarepository.htm#top)
+- [Push images](https://docs.public.oneportal.content.oci.oraclecloud.com/en-us/iaas/Content/Registry/Tasks/registrypushingimagesusingthedockercli.htm#Pushing_Images_Using_the_Docker_CLI)
+
+To publish `anomaly:v1` to OCR, use the command posted below:
+
+```bash
+ads operator publish-image anomaly:v1 --registry <iad.ocir.io/tenancy/>
+```
+
+After the container is published to OCR, it can be used within Data Science jobs service. Check the `backend_job_container_config.yaml` config file. It should contain pre-populated infrastructure and runtime sections. The runtime section should contain an image property, something like `image: iad.ocir.io/<tenancy>/anomaly:v1`. More details about supported options can be found in the ADS Jobs documentation - [Run a Container](https://accelerated-data-science.readthedocs.io/en/latest/user_guide/jobs/run_container.html).
+
+Adjust the `anomaly.yaml` config with proper input/output folders. When the anomaly detection is run in the Data Science job, it will not have access to local folders. Therefore, input data and output folders should be placed in the Object Storage bucket. Open the `anomaly.yaml` and adjust the following fields:
+
+```yaml
+input_data:
+  url: oci://bucket@namespace/anomaly/input_data/data.csv
+output_directory:
+  url: oci://bucket@namespace/anomaly/result/
+test_data:
+  url: oci://bucket@namespace/anomaly/input_data/test.csv
+```
+
+Run the anomaly detection on the Data Science jobs using the command posted below:
+
+```bash
+ads operator run -f ~/anomaly/anomaly.yaml --backend-config ~/anomaly/backend_job_container_config.yaml
+```
+
+The logs can be monitored using the `ads opctl watch` command.
+
+```bash
+ads opctl watch <OCID>
+```
+
+## 6. Running anomaly detection in the Data Science job within conda runtime
+
+To execute the anomaly detection detection operator within a Data Science job using conda runtime, please follow the steps outlined below:
+
+You can use the following command to build the anomaly detection conda environment.
+
+```bash
+ads operator build-conda -t anomaly
+```
+
+This will create a new `anomaly_v1` conda environment and place it in the folder specified within `ads opctl configure` command.
+
+Use the command below to Publish the `anomaly_v1` conda environment to the Object Storage bucket.
+
+```bash
+ads operator publish-conda -t anomaly
+```
+More details about configuring CLI can be found here - [Configuring CLI](https://accelerated-data-science.readthedocs.io/en/latest/user_guide/cli/opctl/configure.html)
+
+
+After the conda environment is published to Object Storage, it can be used within Data Science jobs service. Check the `backend_job_python_config.yaml` config file. It should contain pre-populated infrastructure and runtime sections. The runtime section should contain a `conda` section.
+
+```yaml
+conda:
+  type: published
+  uri: oci://bucket@namespace/conda_environments/cpu/anomaly/1/anomaly_v1
+```
+
+More details about supported options can be found in the ADS Jobs documentation - [Run a Python Workload](https://accelerated-data-science.readthedocs.io/en/latest/user_guide/jobs/run_python.html).
+
+Adjust the `anomaly.yaml` config with proper input/output folders. When the anomaly detection is run in the Data Science job, it will not have access to local folders. Therefore, input data and output folders should be placed in the Object Storage bucket. Open the `anomaly.yaml` and adjust the following fields:
+
+```yaml
+input_data:
+  url: oci://bucket@namespace/anomaly/input_data/data.csv
+output_directory:
+  url: oci://bucket@namespace/anomaly/result/
+test_data:
+  url: oci://bucket@namespace/anomaly/input_data/test.csv
+```
+
+Run the anomaly detection on the Data Science jobs using the command posted below:
+
+```bash
+ads operator run -f ~/anomaly/anomaly.yaml --backend-config ~/anomaly/backend_job_python_config.yaml
+```
+
+The logs can be monitored using the `ads opctl watch` command.
+
+```bash
+ads opctl watch <OCID>
+```

ads/opctl/operator/lowcode/anomaly/__init__.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2023 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/