oracle-ads 2.10.1__py3-none-any.whl → 2.11.0__py3-none-any.whl

ads/cli.py

@@ -1,13 +1,15 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*--
 
-# Copyright (c) 2021, 2023 Oracle and/or its affiliates.
+# Copyright (c) 2021, 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 import traceback
 import sys
 
+import fire
 from ads.common import logger
+from ads.aqua.cli import AquaCommand
 
 try:
     import click
@@ -37,14 +39,43 @@ ADS_VERSION = metadata.version("oracle_ads")
 @click.group()
 @click.version_option(version=ADS_VERSION, prog_name="ads")
 @click.help_option("--help", "-h")
-def cli():
+def click_cli():
     pass
 
 
-cli.add_command(ads.opctl.cli.commands)
-cli.add_command(ads.jobs.cli.commands)
-cli.add_command(ads.pipeline.cli.commands)
-cli.add_command(ads.opctl.operator.cli.commands)
+@click.command
+def aqua_cli():
+    """CLI for AQUA."""
+    # This is a dummy entry for click.
+    # The `ads aqua` commands are handled by AquaCommand
+
+
+click_cli.add_command(ads.opctl.cli.commands)
+click_cli.add_command(ads.jobs.cli.commands)
+click_cli.add_command(ads.pipeline.cli.commands)
+click_cli.add_command(ads.opctl.operator.cli.commands)
+click_cli.add_command(aqua_cli, name="aqua")
+
+
+# fix for fire issue with --help
+# https://github.com/google/python-fire/issues/258
+def _SeparateFlagArgs(args):
+    try:
+        index = args.index("--help")
+        args = args[:index]
+        return args, ["--help"]
+    except ValueError:
+        return args, []
+
+
+fire.core.parser.SeparateFlagArgs = _SeparateFlagArgs
+
+
+def cli():
+    if len(sys.argv) > 1 and sys.argv[1] == "aqua":
+        fire.Fire(AquaCommand, command=sys.argv[2:], name="ads aqua")
+    else:
+        click_cli()
 
 
 if __name__ == "__main__":

ads/common/decorator/__init__.py

@@ -1,5 +1,9 @@
 #!/usr/bin/env python
-# -*- coding: utf-8; -*-
-
-# Copyright (c) 2020, 2022 Oracle and its affiliates.
+# -*- coding: utf-8 -*--
+# Copyright (c) 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+from ads.common.decorator.require_nonempty_arg import require_nonempty_arg
+from ads.common.decorator.argument_to_case import argument_to_case
+from ads.common.decorator.deprecate import deprecated
+from ads.common.decorator.runtime_dependency import runtime_dependency

ads/common/decorator/require_nonempty_arg.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+# Copyright (c) 2024 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import inspect
+from functools import wraps
+from typing import Any, Callable, List, Union
+
+from ads.common.decorator.utils import _get_original_func
+
+
+def require_nonempty_arg(
+    arg_name: Union[str, List[str]], error_msg: str = "A required argument is empty"
+) -> Callable:
+    """
+    A decorator to ensure that a specific argument of a function is not empty.
+
+    Parameters
+    ----------
+    arg_name (Union[str, List[str]]): The name of the argument or the list of the arguments to check.
+    error_msg (str, optional)
+        The error message to raise if the check fails.
+
+    Returns
+    -------
+    Callable
+        A wrapped function that includes the check.
+
+    Raises
+    ------
+    ValueError
+        If the specified argument is empty.
+    """
+
+    def decorator(func: Callable) -> Callable:
+        @wraps(func)
+        def wrapper(*args: Any, **kwargs: Any) -> Any:
+            # Retrieving the original function from the decorated one.
+            # This is necessary when the chain of decorators is used.
+            # The only original function arguments should be processed.
+            original_func = _get_original_func(func)
+
+            # Get the signature of the function
+            sig = inspect.signature(original_func)
+            bound_args = sig.bind(*args, **kwargs)
+            bound_args.apply_defaults()
+
+            # Check if the argument is present and not empty
+            if isinstance(arg_name, str):
+                arguments_to_check = [arg_name]
+            else:
+                arguments_to_check = arg_name
+
+            if not any(
+                check_name in bound_args.arguments and bound_args.arguments[check_name]
+                for check_name in arguments_to_check
+            ):
+                raise ValueError(error_msg)
+
+            return func(*args, **kwargs)
+
+        return wrapper
+
+    return decorator

ads/common/object_storage_details.py

@@ -1,18 +1,24 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*--
 
-# Copyright (c) 2021, 2023 Oracle and/or its affiliates.
+# Copyright (c) 2021, 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 import json
 import os
 import re
 from dataclasses import dataclass
-from typing import Dict
+from typing import Dict, List
 from urllib.parse import urlparse
 
+
+import oci
 from ads.common import auth as authutil
 from ads.common import oci_client
+from ads.dataset.progress import TqdmProgressBar
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+THREAD_POOL_MAX_WORKERS = 10
 
 
 class InvalidObjectStoragePath(Exception):  # pragma: no cover
@@ -33,6 +39,8 @@ class ObjectStorageDetails:
         The Object Storage namespace. Will be extracted automatically if not provided.
     filepath: (str, optional). Defaults to empty string.
         The path to the object.
+    version: (str, optional). Defaults to None.
+        The version of the object.
     auth: (Dict, optional). Defaults to None.
         ADS auth dictionary for OCI authentication.
         This can be generated by calling ads.common.auth.api_keys() or ads.common.auth.resource_principal()
@@ -42,6 +50,7 @@ class ObjectStorageDetails:
     bucket: str
     namespace: str = ""
     filepath: str = ""
+    version: str = None
     auth: Dict = None
 
     def __post_init__(self):
@@ -49,8 +58,16 @@ class ObjectStorageDetails:
             self.auth = authutil.default_signer()
         # Extract OS namespace if not provided.
         if not self.namespace:
-            os_client = oci_client.OCIClientFactory(**self.auth).object_storage
-            self.namespace = os_client.get_namespace().data
+            self.namespace = self.os_client.get_namespace().data
+
+    @property
+    def os_client(self):
+        if not hasattr(self, "__client"):
+            self.__client = self.create_os_client()
+        return self.__client
+
+    def create_os_client(self):
+        return oci_client.OCIClientFactory(**self.auth).object_storage
 
     def __repr__(self):
         return self.path
@@ -86,7 +103,7 @@ class ObjectStorageDetails:
             url_parse = urlparse(env_path)
             bucket_name = url_parse.username
             namespace = url_parse.hostname
-            object_name = url_parse.path.strip("/")
+            object_name = url_parse.path.lstrip("/")
             return cls(bucket=bucket_name, namespace=namespace, filepath=object_name)
         except:
             raise Exception(
@@ -106,8 +123,7 @@ class ObjectStorageDetails:
         Dict
             The metadata in dictionary format.
         """
-        os_client = oci_client.OCIClientFactory(**self.auth).object_storage
-        res = os_client.get_object(self.namespace, self.bucket, self.filepath)
+        res = self.os_client.get_object(self.namespace, self.bucket, self.filepath)
         metadata = res.data.headers["opc-meta-manifest"]
         metadata_json = json.loads(metadata)
         return metadata_json
@@ -138,3 +154,146 @@ class ObjectStorageDetails:
         if not uri:
             return False
         return uri.lower().startswith("oci://")
+
+    def is_bucket_versioned(self) -> bool:
+        """Check if the given bucket is versioned.
+        Returns
+        -------
+        bool: return True if the bucket is versioned.
+
+        """
+        res = self.os_client.get_bucket(
+            namespace_name=self.namespace, bucket_name=self.bucket
+        ).data
+        return res.versioning == "Enabled"
+
+    def list_objects(self, **kwargs):
+        """Lists objects in a given oss path
+
+        Parameters
+        -------
+        **kwargs:
+            namespace, bucket, filepath are set by the class. By default, fields gets all values. For other supported
+            parameters, check https://docs.oracle.com/iaas/api/#/en/objectstorage/20160918/Object/ListObjects
+
+        Returns
+        -------
+            Object of type oci.object_storage.models.ListObjects
+        """
+        fields = kwargs.pop(
+            "fields",
+            "name,etag,size,timeCreated,md5,timeModified,storageTier,archivalState",
+        )
+
+        objects = oci.pagination.list_call_get_all_results(
+            self.os_client.list_objects,
+            namespace_name=self.namespace,
+            bucket_name=self.bucket,
+            prefix=self.filepath,
+            fields=fields,
+            **kwargs,
+        ).data
+        return objects
+
+    def list_object_versions(
+        self,
+        **kwargs,
+    ):
+        """Lists object versions in a given oss path
+
+        Parameters
+        -------
+        **kwargs:
+            namespace, bucket, filepath are set by the class. By default, fields gets all values. For other supported
+            parameters, check https://docs.oracle.com/iaas/api/#/en/objectstorage/20160918/Object/ListObjectVersions
+
+        Returns
+        -------
+            Object of type oci.object_storage.models.ObjectVersionCollection
+        """
+        fields = kwargs.pop(
+            "fields",
+            "name,etag,size,timeCreated,md5,timeModified,storageTier,archivalState",
+        )
+
+        objects = oci.pagination.list_call_get_all_results(
+            self.os_client.list_object_versions,
+            namespace_name=self.namespace,
+            bucket_name=self.bucket,
+            prefix=self.filepath,
+            fields=fields,
+            **kwargs,
+        ).data
+        return objects
+
+    def download_from_object_storage(
+        self,
+        path: "ObjectStorageDetails",
+        target_dir: str,
+        progress_bar: TqdmProgressBar = None,
+    ):
+        """Downloads the files with object versions set in the paths dict.
+
+        Parameters
+        ----------
+        path:
+            OSS path along with a value of file version id.
+            If version_id is not available, download the latest version.
+        target_dir:
+            Local directory to save the files
+        progress_bar:
+            an instance of the TqdmProgressBar, can update description in the calling progress bar
+
+        Returns
+        -------
+            None
+        """
+        if progress_bar:
+            progress_bar.update(
+                description=f"Copying model artifacts by reference from {path.path} to {target_dir}",
+                n=0,
+            )
+        res = self.os_client.get_object(
+            namespace_name=path.namespace,
+            bucket_name=path.bucket,
+            object_name=path.filepath,
+            version_id=path.version,
+        )
+        local_filepath = os.path.join(target_dir, path.bucket, path.filepath)
+        os.makedirs(os.path.dirname(local_filepath), exist_ok=True)
+
+        with open(local_filepath, "wb") as _file:
+            for chunk in res.data.iter_content(chunk_size=4096):
+                _file.write(chunk)
+
+    def bulk_download_from_object_storage(
+        self,
+        paths: List["ObjectStorageDetails"],
+        target_dir: str,
+        progress_bar: TqdmProgressBar = None,
+    ):
+        """Downloads the files with object versions set in the paths dict parallely.
+
+        Parameters
+        ----------
+        paths:
+            Contains a list of OSS paths along with a value of file version id.
+            If version_id is not available, download the latest version.
+        target_dir:
+            Local directory to save the files
+        progress_bar:
+            an instance of the TqdmProgressBar, can update description in the calling progress bar
+
+        Returns
+        -------
+            None
+        """
+        with ThreadPoolExecutor(max_workers=THREAD_POOL_MAX_WORKERS) as pool:
+            futures = {
+                pool.submit(
+                    self.download_from_object_storage, path, target_dir, progress_bar
+                ): path
+                for path in paths
+            }
+            for future in as_completed(futures):
+                future.result()

ads/common/oci_client.py

@@ -20,12 +20,14 @@ from oci.object_storage import ObjectStorageClient
 from oci.resource_search import ResourceSearchClient
 from oci.secrets import SecretsClient
 from oci.vault import VaultsClient
+from oci.logging import LoggingManagementClient
+from oci.core import VirtualNetworkClient
+from oci.limits import LimitsClient
 
 logger = logging.getLogger(__name__)
 
 
 class OCIClientFactory:
-
     """
     A factory class to create OCI client objects. The constructor takes in config, signer and client_kwargs. `client_kwargs` is passed
     to the client constructor as key word arguments.
@@ -72,6 +74,9 @@ class OCIClientFactory:
             "data_labeling_cp": DataLabelingManagementClient,
             "resource_search": ResourceSearchClient,
             "data_catalog": DataCatalogClient,
+            "logging_management": LoggingManagementClient,
+            "virtual_network": VirtualNetworkClient,
+            "limits": LimitsClient,
             "marketplace": MarketplaceClient,
             "artifacts": ArtifactsClient,
         }
@@ -149,6 +154,18 @@ class OCIClientFactory:
     def data_catalog(self):
         return self.create_client("data_catalog")
 
+    @property
+    def logging_management(self):
+        return self.create_client("logging_management")
+
+    @property
+    def virtual_network(self):
+        return self.create_client("virtual_network")
+
+    @property
+    def limits(self):
+        return self.create_client("limits")
+
     @property
     def marketplace(self):
         return self.create_client("marketplace")

ads/common/oci_logging.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8; -*-
 
-# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
+# Copyright (c) 2021, 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 import datetime
@@ -384,7 +384,7 @@ class OCILog(OCILoggingModelMixin, oci.logging.models.Log):
                 records = []
                 result_too_large = True
             else:
-                raise oci.exceptions.ServiceError from ex
+                raise ex
         if result_too_large or len(records) >= LIMIT_PER_REQUEST:
             mid = time_start + (time_end - time_start) / 2
             # The log search API used RFC3339 time format.

ads/common/oci_mixin.py

@@ -798,14 +798,13 @@ class OCIModelMixin(OCISerializableMixin):
                         logger.error(
                             "Failed to synchronize the properties of %s due to service error:\n%s",
                             self.__class__,
-                            str(ex),
+                            traceback.format_exc(),
                         )
-                except Exception as ex:
+                except Exception:
                     logger.error(
-                        "Failed to synchronize the properties of %s: %s\n%s",
+                        "Failed to synchronize the properties of %s.\n%s",
                         self.__class__,
-                        type(ex),
-                        str(ex),
+                        traceback.format_exc(),
                     )
         return super().__getattribute__(name)
 

ads/common/serializer.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8; -*-
 
-# Copyright (c) 2021, 2023 Oracle and/or its affiliates.
+# Copyright (c) 2021, 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 """
@@ -13,6 +13,7 @@ writing serialized objects to and from files.
 import dataclasses
 import json
 from abc import ABC, abstractmethod
+from datetime import datetime
 from enum import Enum
 from typing import Dict, Optional, Union
 from urllib.parse import urlparse
@@ -93,6 +94,13 @@ class Serializable(ABC):
         """
         pass
 
+    @staticmethod
+    def serialize(obj):
+        """JSON serializer for objects not serializable by default json code."""
+        if isinstance(obj, datetime):
+            return obj.isoformat()
+        raise TypeError(f"Type {type(obj)} not serializable.")
+
     @staticmethod
     def _write_to_file(s: str, uri: str, **kwargs) -> None:
         """Write string s into location specified by uri.
@@ -153,7 +161,11 @@ class Serializable(ABC):
             return f.read()
 
     def to_json(
-        self, uri: str = None, encoder: callable = json.JSONEncoder, **kwargs
+        self,
+        uri: str = None,
+        encoder: callable = json.JSONEncoder,
+        default: callable = None,
+        **kwargs,
     ) -> str:
         """Returns object serialized as a JSON string
 
@@ -163,6 +175,9 @@ class Serializable(ABC):
             URI location to save the JSON string. Defaults to None.
         encoder: (callable, optional)
             Encoder for custom data structures. Defaults to JSONEncoder.
+        default: (callable, optional)
+            A function that gets called for objects that can't otherwise be serialized.
+            It should return JSON-serializable version of the object or original object.
 
         kwargs
         ------
@@ -179,7 +194,9 @@ class Serializable(ABC):
             Serialized version of object.
             `None` in case when `uri` provided.
         """
-        json_string = json.dumps(self.to_dict(**kwargs), cls=encoder)
+        json_string = json.dumps(
+            self.to_dict(**kwargs), cls=encoder, default=default or self.serialize
+        )
         if uri:
             self._write_to_file(s=json_string, uri=uri, **kwargs)
             return None
@@ -337,13 +354,23 @@ class Serializable(ABC):
     def __repr__(self):
         """Returns printable version of object.
 
-        Parameters
+        Returns
         ----------
         string
             Serialized version of object as a YAML string
         """
         return self.to_yaml()
 
+    def __str__(self):
+        """Returns printable version of object.
+
+        Returns
+        ----------
+        string
+            Serialized version of object as a YAML string
+        """
+        return self.to_json()
+
 
 class DataClassSerializable(Serializable):
     """Wrapper class that inherit from Serializable class.
@@ -436,7 +463,9 @@ class DataClassSerializable(Serializable):
                 "These fields will be ignored."
             )
 
-        obj = cls(**{key: obj_dict.get(key) for key in allowed_fields})
+        obj = cls(
+            **{key: obj_dict.get(key) for key in allowed_fields if key in obj_dict}
+        )
 
         for key, value in obj_dict.items():
             if (

ads/common/utils.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8; -*-
 
-# Copyright (c) 2020, 2023 Oracle and/or its affiliates.
+# Copyright (c) 2020, 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 from __future__ import absolute_import, print_function
@@ -33,29 +33,25 @@ import fsspec
 import matplotlib as mpl
 import numpy as np
 import pandas as pd
-from ads.common import logger
-from ads.common.decorator.deprecate import deprecated
-from ads.common.word_lists import adjectives, animals
-from ads.dataset.progress import TqdmProgressBar
 from cycler import cycler
+from oci import object_storage
 from pandas.core.dtypes.common import is_datetime64_dtype, is_numeric_dtype
 from sklearn.model_selection import train_test_split
 from tqdm import tqdm
 
+from ads import config
 from ads.common import logger
 from ads.common.decorator.deprecate import deprecated
 from ads.common.decorator.runtime_dependency import (
     OptionalDependency,
     runtime_dependency,
 )
+from ads.common.object_storage_details import ObjectStorageDetails
+from ads.common.oci_client import OCIClientFactory
 from ads.common.word_lists import adjectives, animals
-from ads import config
-from ads.dataset.progress import DummyProgressBar, TqdmProgressBar
+from ads.dataset.progress import TqdmProgressBar
 
 from . import auth as authutil
-from oci import object_storage
-from ads.common.oci_client import OCIClientFactory
-from ads.common.object_storage_details import ObjectStorageDetails
 
 # For Model / Model Artifact libraries
 lib_translator = {"sklearn": "scikit-learn"}
@@ -1717,3 +1713,72 @@ def upload_to_os(
         )
 
     return response
+
+
+def get_console_link(
+    resource: str,
+    ocid: str,
+    region: str,
+) -> str:
+    """
+    This method returns the web console link for the given resource.
+    Parameters
+    ----------
+    resource: str
+        identify the type of OCI resource. {model, model-deployments, notebook-sessions, jobs} is supported.
+    ocid: str
+        OCID of the resource
+    region: str
+        The Region Identifier that the client should connect to.
+
+    Returns
+    -------
+    console_link_url: str
+        a valid link to the console for the given resource
+    """
+    console_link_url = (
+        f"https://cloud.oracle.com/data-science/{resource}/{ocid}?region={region}"
+    )
+    return console_link_url
+
+
+def get_log_links(
+    region: str,
+    log_group_id: str,
+    compartment_id: str = None,
+    log_id: str = None,
+    source_id: str = None,
+) -> str:
+    """
+    This method returns the web console link for the given log ids.
+    
+    Parameters
+    ----------
+    log_group_id: str, required
+        OCID of the resource
+    log_id: str, optional
+        OCID of the resource
+    region: str
+        The Region Identifier that the client should connect to.
+    compartment_id: str, optional
+        The compartment OCID of the resource.
+    source_id: str, optional
+        The OCID of the resource.
+
+    Returns
+    -------
+    console_link_url: str
+        a valid link to the console for the given resource.
+    """
+    console_link_url = ""
+    if log_group_id and log_id:
+        # format: https://cloud.oracle.com/logging/search?searchQuery=search "<compartment>/<log_group>/<log>" | source='<source>' | sort by datetime desc&regions=<region>
+        query_range = f'''search "{compartment_id}/{log_group_id}/{log_id}"'''
+        query_source = f"source='{source_id}'"
+        sort_condition = f"sort by datetime desc&regions={region}"
+        search_query = f"search?searchQuery={query_range} | {query_source} | {sort_condition}"
+        console_link_url = f"https://cloud.oracle.com/logging/{search_query}"
+    elif log_group_id:
+        console_link_url = f"https://cloud.oracle.com/logging/log-groups/{log_group_id}?region={region}"
+
+    return console_link_url