oracle-ads 2.10.1__tar.gz → 2.11.1__tar.gz

{oracle_ads-2.10.1 → oracle_ads-2.11.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: oracle_ads
-Version: 2.10.1
+Version: 2.11.1
 Summary: Oracle Accelerated Data Science SDK
 Keywords: Oracle Cloud Infrastructure,OCI,Machine Learning,ML,Artificial Intelligence,AI,Data Science,Cloud,Oracle
 Author: Oracle Data Science
@@ -24,7 +24,8 @@ Requires-Dist: matplotlib>=3.1.3
 Requires-Dist: numpy>=1.19.2
 Requires-Dist: oci>=2.113.0
 Requires-Dist: ocifs>=1.1.3
-Requires-Dist: pandas>1.2.1,<2.1
+Requires-Dist: pandas>1.2.1; python_version<'3.9'
+Requires-Dist: pandas>=2.2.0; python_version>='3.9'
 Requires-Dist: psutil>=5.7.2
 Requires-Dist: python_jsonschema_objects>=0.3.13
 Requires-Dist: requests
@@ -33,9 +34,11 @@ Requires-Dist: tabulate>=0.8.9
 Requires-Dist: tqdm>=4.59.0
 Requires-Dist: oracle_ads[opctl] ; extra == "anomaly"
 Requires-Dist: autots ; extra == "anomaly"
-Requires-Dist: oracle-automlx[anomaly]==23.2.3 ; extra == "anomaly"
+Requires-Dist: oracle-automlx[forecasting]==23.4.1 ; extra == "anomaly"
+Requires-Dist: oracle-automlx[classic]==23.4.1 ; extra == "anomaly"
 Requires-Dist: oracledb ; extra == "anomaly"
 Requires-Dist: datapane ; extra == "anomaly"
+Requires-Dist: jupyter_server ; extra == "aqua"
 Requires-Dist: hdfs[kerberos] ; extra == "bds"
 Requires-Dist: ibis-framework[impala] ; extra == "bds"
 Requires-Dist: sqlalchemy ; extra == "bds"
@@ -63,9 +66,10 @@ Requires-Dist: holidays==0.21.13 ; extra == "forecast"
 Requires-Dist: neuralprophet ; extra == "forecast"
 Requires-Dist: numpy ; extra == "forecast"
 Requires-Dist: oci-cli ; extra == "forecast"
-Requires-Dist: optuna==2.9.0 ; extra == "forecast"
+Requires-Dist: optuna==3.1.0 ; extra == "forecast"
 Requires-Dist: oracle-ads ; extra == "forecast"
-Requires-Dist: oracle-automlx[forecasting]==23.2.3 ; extra == "forecast"
+Requires-Dist: oracle-automlx[forecasting]==23.4.1 ; extra == "forecast"
+Requires-Dist: oracle-automlx[classic]==23.4.1 ; extra == "forecast"
 Requires-Dist: pmdarima ; extra == "forecast"
 Requires-Dist: prophet ; extra == "forecast"
 Requires-Dist: shap ; extra == "forecast"
@@ -76,7 +80,7 @@ Requires-Dist: oracledb ; extra == "forecast"
 Requires-Dist: geopandas ; extra == "geo"
 Requires-Dist: oracle_ads[viz] ; extra == "geo"
 Requires-Dist: transformers ; extra == "huggingface"
-Requires-Dist: langchain>=0.0.295 ; extra == "llm"
+Requires-Dist: langchain>=0.1.10 ; extra == "llm"
 Requires-Dist: evaluate>=0.4.0 ; extra == "llm"
 Requires-Dist: ipython>=7.23.1, <8.0 ; extra == "notebook"
 Requires-Dist: ipywidgets~=7.6.3 ; extra == "notebook"
@@ -97,6 +101,8 @@ Requires-Dist: nbformat ; extra == "opctl"
 Requires-Dist: oci-cli ; extra == "opctl"
 Requires-Dist: py-cpuinfo ; extra == "opctl"
 Requires-Dist: rich ; extra == "opctl"
+Requires-Dist: fire ; extra == "opctl"
+Requires-Dist: cachetools ; extra == "opctl"
 Requires-Dist: optuna==2.9.0 ; extra == "optuna"
 Requires-Dist: oracle_ads[viz] ; extra == "optuna"
 Requires-Dist: aiohttp ; extra == "pii"
@@ -112,6 +118,28 @@ Requires-Dist: spacy==3.6.1 ; extra == "pii"
 Requires-Dist: pyspark>=3.0.0 ; extra == "spark"
 Requires-Dist: oracle_ads[viz] ; extra == "tensorflow"
 Requires-Dist: tensorflow ; extra == "tensorflow"
+Requires-Dist: dask==2023.5.0 ; extra == "testsuite" and ( python_version=='3.8')
+Requires-Dist: dask==2023.10.1 ; extra == "testsuite" and ( python_version>='3.9')
+Requires-Dist: arff ; extra == "testsuite"
+Requires-Dist: category_encoders==2.6.3 ; extra == "testsuite"
+Requires-Dist: cohere==4.53 ; extra == "testsuite"
+Requires-Dist: dask==2023.10.1 ; extra == "testsuite" and ( python_version>='3.9')
+Requires-Dist: dask==2023.5.0 ; extra == "testsuite" and ( python_version=='3.8')
+Requires-Dist: faiss-cpu ; extra == "testsuite"
+Requires-Dist: fastparquet==2024.2.0 ; extra == "testsuite"
+Requires-Dist: imbalanced-learn ; extra == "testsuite"
+Requires-Dist: lxml ; extra == "testsuite"
+Requires-Dist: mysql-connector-python ; extra == "testsuite"
+Requires-Dist: nltk ; extra == "testsuite"
+Requires-Dist: notebook==6.4.12 ; extra == "testsuite"
+Requires-Dist: opensearch-py ; extra == "testsuite"
+Requires-Dist: pdfplumber ; extra == "testsuite"
+Requires-Dist: py4j ; extra == "testsuite"
+Requires-Dist: pyarrow ; extra == "testsuite"
+Requires-Dist: statsmodels ; extra == "testsuite" and ( python_version=='3.8')
+Requires-Dist: statsmodels>=0.14.1 ; extra == "testsuite" and ( python_version>='3.9')
+Requires-Dist: tables ; extra == "testsuite"
+Requires-Dist: xlrd>=1.2.0 ; extra == "testsuite"
 Requires-Dist: spacy ; extra == "text"
 Requires-Dist: wordcloud>=1.8.1 ; extra == "text"
 Requires-Dist: oracle_ads[viz] ; extra == "torch"
@@ -125,6 +153,7 @@ Requires-Dist: seaborn>=0.11.0 ; extra == "viz"
 Project-URL: Documentation, https://accelerated-data-science.readthedocs.io/en/latest/index.html
 Project-URL: Github, https://github.com/oracle/accelerated-data-science
 Provides-Extra: anomaly
+Provides-Extra: aqua
 Provides-Extra: bds
 Provides-Extra: boosted
 Provides-Extra: data
@@ -140,6 +169,7 @@ Provides-Extra: optuna
 Provides-Extra: pii
 Provides-Extra: spark
 Provides-Extra: tensorflow
+Provides-Extra: testsuite
 Provides-Extra: text
 Provides-Extra: torch
 Provides-Extra: viz
@@ -178,7 +208,7 @@ You have various options when installing ADS.
 To use the AI Forecast Operator, install the "forecast" dependencies using the following command:
 
 ```bash
-  python3 -m pip install 'oracle_ads[forecast]==2.9.0'
+  python3 -m pip install 'oracle_ads[forecast]>=2.9.0'
 ```
 
 ### Installing extras libraries

{oracle_ads-2.10.1 → oracle_ads-2.11.1}/README.md

@@ -32,7 +32,7 @@ You have various options when installing ADS.
 To use the AI Forecast Operator, install the "forecast" dependencies using the following command:
 
 ```bash
-  python3 -m pip install 'oracle_ads[forecast]==2.9.0'
+  python3 -m pip install 'oracle_ads[forecast]>=2.9.0'
 ```
 
 ### Installing extras libraries

{oracle_ads-2.10.1 → oracle_ads-2.11.1}/THIRD_PARTY_LICENSES.txt

@@ -405,6 +405,12 @@ delta
 * Source code: https://github.com/delta-io/delta/
 * Project home: https://delta.io/
 
+cachetools
+* Copyright (c) 2014-2024 Thomas Kemmer
+* License: The MIT License (MIT)
+* Source code: https://github.com/tkem/cachetools/
+* Project home: https://cachetools.readthedocs.io/
+
 =============================== Licenses ===============================
 ------------------------------------------------------------------------
 

oracle_ads-2.11.1/ads/aqua/__init__.py

@@ -0,0 +1,12 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (c) 2024 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+
+import logging
+import sys
+
+logger = logging.getLogger(__name__)
+handler = logging.StreamHandler(sys.stdout)
+logger.setLevel(logging.INFO)

oracle_ads-2.11.1/ads/aqua/base.py

@@ -0,0 +1,324 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (c) 2024 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import os
+from typing import Dict, Union
+
+import oci
+from oci.data_science.models import UpdateModelDetails, UpdateModelProvenanceDetails
+
+from ads import set_auth
+from ads.aqua import logger
+from ads.aqua.data import Tags
+from ads.aqua.exception import AquaRuntimeError, AquaValueError
+from ads.aqua.utils import (
+    UNKNOWN,
+    _is_valid_mvs,
+    get_artifact_path,
+    get_base_model_from_tags,
+    is_valid_ocid,
+    load_config,
+    logger,
+)
+from ads.common import oci_client as oc
+from ads.common.auth import default_signer
+from ads.common.utils import extract_region
+from ads.config import (
+    AQUA_TELEMETRY_BUCKET,
+    AQUA_TELEMETRY_BUCKET_NS,
+    OCI_ODSC_SERVICE_ENDPOINT,
+    OCI_RESOURCE_PRINCIPAL_VERSION,
+)
+from ads.model.datascience_model import DataScienceModel
+from ads.model.deployment.model_deployment import ModelDeployment
+from ads.model.model_metadata import (
+    ModelCustomMetadata,
+    ModelProvenanceMetadata,
+    ModelTaxonomyMetadata,
+)
+from ads.model.model_version_set import ModelVersionSet
+from ads.telemetry import telemetry
+from ads.telemetry.client import TelemetryClient
+
+
+class AquaApp:
+    """Base Aqua App to contain common components."""
+
+    @telemetry(name="aqua")
+    def __init__(self) -> None:
+        if OCI_RESOURCE_PRINCIPAL_VERSION:
+            set_auth("resource_principal")
+        self._auth = default_signer({"service_endpoint": OCI_ODSC_SERVICE_ENDPOINT})
+        self.ds_client = oc.OCIClientFactory(**self._auth).data_science
+        self.logging_client = oc.OCIClientFactory(**default_signer()).logging_management
+        self.identity_client = oc.OCIClientFactory(**default_signer()).identity
+        self.region = extract_region(self._auth)
+        self._telemetry = None
+
+    def list_resource(
+        self,
+        list_func_ref,
+        **kwargs,
+    ) -> list:
+        """Generic method to list OCI Data Science resources.
+
+        Parameters
+        ----------
+        list_func_ref : function
+            A reference to the list operation which will be called.
+        **kwargs :
+            Additional keyword arguments to filter the resource.
+            The kwargs are passed into OCI API.
+
+        Returns
+        -------
+        list
+            A list of OCI Data Science resources.
+        """
+        return oci.pagination.list_call_get_all_results(
+            list_func_ref,
+            **kwargs,
+        ).data
+
+    def update_model(self, model_id: str, update_model_details: UpdateModelDetails):
+        """Updates model details.
+
+        Parameters
+        ----------
+        model_id : str
+            The id of target model.
+        update_model_details: UpdateModelDetails
+            The model details to be updated.
+        """
+        self.ds_client.update_model(
+            model_id=model_id, update_model_details=update_model_details
+        )
+
+    def update_model_provenance(
+        self,
+        model_id: str,
+        update_model_provenance_details: UpdateModelProvenanceDetails,
+    ):
+        """Updates model provenance details.
+
+        Parameters
+        ----------
+        model_id : str
+            The id of target model.
+        update_model_provenance_details: UpdateModelProvenanceDetails
+            The model provenance details to be updated.
+        """
+        self.ds_client.update_model_provenance(
+            model_id=model_id,
+            update_model_provenance_details=update_model_provenance_details,
+        )
+
+    # TODO: refactor model evaluation implementation to use it.
+    @staticmethod
+    def get_source(source_id: str) -> Union[ModelDeployment, DataScienceModel]:
+        if is_valid_ocid(source_id):
+            if "datasciencemodeldeployment" in source_id:
+                return ModelDeployment.from_id(source_id)
+            elif "datasciencemodel" in source_id:
+                return DataScienceModel.from_id(source_id)
+
+        raise AquaValueError(
+            f"Invalid source {source_id}. "
+            "Specify either a model or model deployment id."
+        )
+
+    # TODO: refactor model evaluation implementation to use it.
+    @staticmethod
+    def create_model_version_set(
+        model_version_set_id: str = None,
+        model_version_set_name: str = None,
+        description: str = None,
+        compartment_id: str = None,
+        project_id: str = None,
+        **kwargs,
+    ) -> tuple:
+        """Creates ModelVersionSet from given ID or Name.
+
+        Parameters
+        ----------
+        model_version_set_id: (str, optional):
+            ModelVersionSet OCID.
+        model_version_set_name: (str, optional):
+            ModelVersionSet Name.
+        description: (str, optional):
+            TBD
+        compartment_id: (str, optional):
+            Compartment OCID.
+        project_id: (str, optional):
+            Project OCID.
+        tag: (str, optional)
+            calling tag, can be Tags.AQUA_FINE_TUNING or Tags.AQUA_EVALUATION
+
+        Returns
+        -------
+        tuple: (model_version_set_id, model_version_set_name)
+        """
+        # TODO: tag should be selected based on which operation (eval/FT) invoke this method
+        #   currently only used by fine-tuning flow.
+        tag = Tags.AQUA_FINE_TUNING.value
+
+        if not model_version_set_id:
+            tag = Tags.AQUA_FINE_TUNING.value # TODO: Fix this
+            try:
+                model_version_set = ModelVersionSet.from_name(
+                    name=model_version_set_name,
+                    compartment_id=compartment_id,
+                )
+
+                if not _is_valid_mvs(model_version_set, tag):
+                    raise AquaValueError(
+                        f"Invalid model version set name. Please provide a model version set with `{tag}` in tags."
+                    )
+
+            except:
+                logger.debug(
+                    f"Model version set {model_version_set_name} doesn't exist. "
+                    "Creating new model version set."
+                )
+                mvs_freeform_tags = {
+                    tag: tag,
+                }
+                model_version_set = (
+                    ModelVersionSet()
+                    .with_compartment_id(compartment_id)
+                    .with_project_id(project_id)
+                    .with_name(model_version_set_name)
+                    .with_description(description)
+                    .with_freeform_tags(**mvs_freeform_tags)
+                    # TODO: decide what parameters will be needed
+                    # when refactor eval to use this method, we need to pass tag here.
+                    .create(**kwargs)
+                )
+                logger.debug(
+                    f"Successfully created model version set {model_version_set_name} with id {model_version_set.id}."
+                )
+            return (model_version_set.id, model_version_set_name)
+        else:
+            model_version_set = ModelVersionSet.from_id(model_version_set_id)
+            # TODO: tag should be selected based on which operation (eval/FT) invoke this method
+            if not _is_valid_mvs(model_version_set, tag):
+                raise AquaValueError(
+                    f"Invalid model version set id. Please provide a model version set with `{tag}` in tags."
+                )
+            return (model_version_set_id, model_version_set.name)
+
+    # TODO: refactor model evaluation implementation to use it.
+    @staticmethod
+    def create_model_catalog(
+        display_name: str,
+        description: str,
+        model_version_set_id: str,
+        model_custom_metadata: Union[ModelCustomMetadata, Dict],
+        model_taxonomy_metadata: Union[ModelTaxonomyMetadata, Dict],
+        compartment_id: str,
+        project_id: str,
+        **kwargs,
+    ) -> DataScienceModel:
+        model = (
+            DataScienceModel()
+            .with_compartment_id(compartment_id)
+            .with_project_id(project_id)
+            .with_display_name(display_name)
+            .with_description(description)
+            .with_model_version_set_id(model_version_set_id)
+            .with_custom_metadata_list(model_custom_metadata)
+            .with_defined_metadata_list(model_taxonomy_metadata)
+            .with_provenance_metadata(ModelProvenanceMetadata(training_id=UNKNOWN))
+            # TODO: decide what parameters will be needed
+            .create(
+                **kwargs,
+            )
+        )
+        return model
+
+    def if_artifact_exist(self, model_id: str, **kwargs) -> bool:
+        """Checks if the artifact exists.
+
+        Parameters
+        ----------
+        model_id : str
+            The model OCID.
+        **kwargs :
+            Additional keyword arguments passed in head_model_artifact.
+
+        Returns
+        -------
+        bool
+            Whether the artifact exists.
+        """
+
+        try:
+            response = self.ds_client.head_model_artifact(model_id=model_id, **kwargs)
+            return True if response.status == 200 else False
+        except oci.exceptions.ServiceError as ex:
+            if ex.status == 404:
+                logger.info(f"Artifact not found in model {model_id}.")
+                return False
+
+    def get_config(self, model_id: str, config_file_name: str) -> Dict:
+        """Gets the config for the given Aqua model.
+
+        Parameters
+        ----------
+        model_id: str
+            The OCID of the Aqua model.
+        config_file_name: str
+            name of the config file
+
+        Returns
+        -------
+        Dict:
+            A dict of allowed configs.
+        """
+        oci_model = self.ds_client.get_model(model_id).data
+        oci_aqua = (
+            (
+                Tags.AQUA_TAG.value in oci_model.freeform_tags
+                or Tags.AQUA_TAG.value.lower() in oci_model.freeform_tags
+            )
+            if oci_model.freeform_tags
+            else False
+        )
+
+        if not oci_aqua:
+            raise AquaRuntimeError(f"Target model {oci_model.id} is not Aqua model.")
+
+        config = {}
+        artifact_path = get_artifact_path(oci_model.custom_metadata_list)
+        if not artifact_path:
+            logger.error(
+                f"Failed to get artifact path from custom metadata for the model: {model_id}"
+            )
+            return config
+
+        try:
+            config_path = f"{os.path.dirname(artifact_path)}/config/"
+            config = load_config(
+                config_path,
+                config_file_name=config_file_name,
+            )
+        except:
+            pass
+
+        if not config:
+            logger.error(
+                f"{config_file_name} is not available for the model: {model_id}. Check if the custom metadata has the artifact path set."
+            )
+            return config
+
+        return config
+
+    @property
+    def telemetry(self):
+        if not self._telemetry:
+            self._telemetry = TelemetryClient(
+                bucket=AQUA_TELEMETRY_BUCKET, namespace=AQUA_TELEMETRY_BUCKET_NS
+            )
+        return self._telemetry

oracle_ads-2.11.1/ads/aqua/cli.py

@@ -0,0 +1,19 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2024 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+from ads.aqua.deployment import AquaDeploymentApp
+from ads.aqua.finetune import AquaFineTuningApp
+from ads.aqua.model import AquaModelApp
+from ads.aqua.evaluation import AquaEvaluationApp
+
+
+class AquaCommand:
+    """Contains the command groups for project Aqua."""
+
+    model = AquaModelApp
+    fine_tuning = AquaFineTuningApp
+    deployment = AquaDeploymentApp
+    evaluation = AquaEvaluationApp

oracle_ads-2.11.1/ads/aqua/config/deployment_config_defaults.json

@@ -0,0 +1,9 @@
+{
+  "shape": [
+    "VM.GPU.A10.1",
+    "VM.GPU.A10.2",
+    "BM.GPU.A10.4",
+    "BM.GPU4.8",
+    "BM.GPU.A100-v2.8"
+  ]
+}

oracle_ads-2.11.1/ads/aqua/config/resource_limit_names.json

@@ -0,0 +1,7 @@
+{
+  "BM.GPU.A10.4": "ds-gpu-a10-count",
+  "BM.GPU.A100-v2.8": "ds-gpu-a100-v2-count",
+  "BM.GPU4.8": "ds-gpu4-count",
+  "VM.GPU.A10.1": "ds-gpu-a10-count",
+  "VM.GPU.A10.2": "ds-gpu-a10-count"
+}

oracle_ads-2.11.1/ads/aqua/constants.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (c) 2024 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+"""This module defines constants used in ads.aqua module."""
+from enum import Enum
+
+UNKNOWN_VALUE = ""
+
+
+class RqsAdditionalDetails:
+    METADATA = "metadata"
+    CREATED_BY = "createdBy"
+    DESCRIPTION = "description"
+    MODEL_VERSION_SET_ID = "modelVersionSetId"
+    MODEL_VERSION_SET_NAME = "modelVersionSetName"
+    PROJECT_ID = "projectId"
+    VERSION_LABEL = "versionLabel"
+
+
+class FineTuningDefinedMetadata(Enum):
+    """Represents the defined metadata keys used in Fine Tuning."""
+
+    VAL_SET_SIZE = "val_set_size"
+    TRAINING_DATA = "training_data"
+
+
+class FineTuningCustomMetadata(Enum):
+    """Represents the custom metadata keys used in Fine Tuning."""
+
+    FT_SOURCE = "fine_tune_source"
+    FT_SOURCE_NAME = "fine_tune_source_name"
+    FT_OUTPUT_PATH = "fine_tune_output_path"
+    FT_JOB_ID = "fine_tune_job_id"
+    FT_JOB_RUN_ID = "fine_tune_jobrun_id"
+    TRAINING_METRICS_FINAL = "train_metrics_final"
+    VALIDATION_METRICS_FINAL = "val_metrics_final"
+    TRAINING_METRICS_EPOCH = "train_metrics_epoch"
+    VALIDATION_METRICS_EPOCH = "val_metrics_epoch"
+
+
+TRAINING_METRICS_FINAL = "training_metrics_final"
+VALIDATION_METRICS_FINAL = "validation_metrics_final"
+TRINING_METRICS = "training_metrics"
+VALIDATION_METRICS = "validation_metrics"

oracle_ads-2.11.1/ads/aqua/data.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (c) 2024 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+from dataclasses import dataclass
+from enum import Enum
+
+from ads.common.serializer import DataClassSerializable
+
+
+@dataclass(repr=False)
+class AquaResourceIdentifier(DataClassSerializable):
+    id: str = ""
+    name: str = ""
+    url: str = ""
+
+
+class Resource(Enum):
+    JOB = "jobs"
+    JOBRUN = "jobruns"
+    MODEL = "models"
+    MODEL_DEPLOYMENT = "modeldeployments"
+    MODEL_VERSION_SET = "model-version-sets"
+
+
+class DataScienceResource(Enum):
+    MODEL_DEPLOYMENT = "datasciencemodeldeployment"
+    MODEL = "datasciencemodel"
+
+
+class Tags(Enum):
+    TASK = "task"
+    LICENSE = "license"
+    ORGANIZATION = "organization"
+    AQUA_TAG = "OCI_AQUA"
+    AQUA_SERVICE_MODEL_TAG = "aqua_service_model"
+    AQUA_FINE_TUNED_MODEL_TAG = "aqua_fine_tuned_model"
+    AQUA_EVALUATION = "aqua_evaluation"
+    AQUA_FINE_TUNING = "aqua_finetuning"