oracle-ads 2.10.0__py3-none-any.whl → 2.11.0__py3-none-any.whl

ads/opctl/operator/lowcode/pii/model/guardrails.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*--
 
-# Copyright (c) 2023 Oracle and/or its affiliates.
+# Copyright (c) 2023, 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 import os
@@ -19,11 +19,11 @@ from ads.opctl.operator.lowcode.pii.model.report import (
 )
 from ads.opctl.operator.lowcode.pii.operator_config import PiiOperatorConfig
 from ads.opctl.operator.lowcode.pii.utils import (
-    _load_data,
-    _write_data,
     default_signer,
     get_output_name,
 )
+from ads.opctl.operator.lowcode.common.utils import load_data, write_data
+from ads.opctl.operator.lowcode.common.errors import InvalidParameterError
 
 
 class PIIGuardrail:
@@ -68,15 +68,20 @@ class PIIGuardrail:
         )
         self.datasets = None
 
-    def load_data(self, uri=None, storage_options=None):
+    def _load_data(self, uri=None, storage_options=None):
         """Loads input data."""
         input_data_uri = uri or self.spec.input_data.url
         logger.info(f"Loading input data from `{input_data_uri}` ...")
 
-        self.datasets = _load_data(
-            filename=input_data_uri,
-            storage_options=storage_options or self.storage_options,
-        )
+        try:
+            self.datasets = load_data(
+                data_spec=self.spec.input_data,
+                storage_options=storage_options or self.storage_options,
+            )
+        except InvalidParameterError as e:
+            e.args = e.args + ("Invalid Parameter: input_data",)
+            raise e
+
         return self
 
     def process(self, **kwargs):
@@ -92,12 +97,10 @@ class PIIGuardrail:
 
         if not data:
             try:
-                self.load_data()
+                self._load_data()
                 data = self.datasets
-            except Exception as e:
-                logger.warning(
-                    f"Failed to load data from `{self.spec.input_data.url}`."
-                )
+            except InvalidParameterError as e:
+                e.args = e.args + ("Invalid Parameter: input_data",)
                 raise e
 
         # process user data
@@ -110,10 +113,10 @@ class PIIGuardrail:
         # save output data
         if dst_uri:
             logger.info(f"Saving data into `{dst_uri}` ...")
-
-            _write_data(
+            write_data(
                 data=data.loc[:, data.columns != self.spec.target_column],
                 filename=dst_uri,
+                format=None,
                 storage_options=kwargs.pop("storage_options", None)
                 or self.storage_options,
             )

ads/opctl/operator/lowcode/pii/model/report.py

@@ -22,6 +22,11 @@ from ads.common.decorator.runtime_dependency import (
 )
 from ads.common.serializer import DataClassSerializable
 from ads.opctl import logger
+from ads.opctl.operator.lowcode.common.utils import (
+    disable_print,
+    enable_print,
+    human_time_friendly,
+)
 from ads.opctl.operator.lowcode.pii.constant import (
     DEFAULT_COLOR,
     DEFAULT_SHOW_ROWS,
@@ -31,12 +36,7 @@ from ads.opctl.operator.lowcode.pii.constant import (
     PII_REPORT_DESCRIPTION,
 )
 from ads.opctl.operator.lowcode.pii.operator_config import PiiOperatorConfig
-from ads.opctl.operator.lowcode.pii.utils import (
-    block_print,
-    compute_rate,
-    enable_print,
-    human_time_friendly,
-)
+from ads.opctl.operator.lowcode.pii.utils import compute_rate
 
 try:
     import datapane as dp
@@ -321,7 +321,7 @@ class PIIOperatorReport:
     def save_report(self, report_sections=None, report_uri=None, storage_options={}):
         with tempfile.TemporaryDirectory() as temp_dir:
             report_local_path = os.path.join(temp_dir, "___report.html")
-            block_print()
+            disable_print()
             dp.save_report(
                 report_sections or self.report_sections,
                 path=report_local_path,

ads/opctl/operator/lowcode/pii/operator_config.py

@@ -9,7 +9,7 @@ from dataclasses import dataclass, field
 from typing import Dict, List
 
 from ads.common.serializer import DataClassSerializable
-from ads.opctl.operator.common.operator_config import OperatorConfig
+from ads.opctl.operator.common.operator_config import OperatorConfig, InputData
 from ads.opctl.operator.common.utils import _load_yaml_from_uri
 from ads.opctl.operator.lowcode.pii.constant import (
     DEFAULT_SHOW_ROWS,
@@ -18,13 +18,6 @@ from ads.opctl.operator.lowcode.pii.constant import (
 )
 
 
-@dataclass(repr=True)
-class InputData(DataClassSerializable):
-    """Class representing operator specification input data details."""
-
-    url: str = None
-
-
 @dataclass(repr=True)
 class OutputDirectory(DataClassSerializable):
     """Class representing operator specification output directory details."""

ads/opctl/operator/lowcode/pii/utils.py

@@ -5,14 +5,6 @@
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 import os
-import sys
-
-import fsspec
-import pandas as pd
-
-from ads.common.object_storage_details import ObjectStorageDetails
-
-from .errors import PIIInputDataError
 
 
 def default_signer(**kwargs):
@@ -22,50 +14,6 @@ def default_signer(**kwargs):
     return default_signer(**kwargs)
 
 
-def _call_pandas_fsspec(pd_fn, filename, storage_options, **kwargs):
-    if fsspec.utils.get_protocol(filename) == "file":
-        return pd_fn(filename, **kwargs)
-
-    storage_options = storage_options or (
-        default_signer() if ObjectStorageDetails.is_oci_path(filename) else {}
-    )
-
-    return pd_fn(filename, storage_options=storage_options, **kwargs)
-
-
-def _load_data(filename, format=None, storage_options=None, columns=None, **kwargs):
-    if not format:
-        _, format = os.path.splitext(filename)
-        format = format[1:]
-    if format in ["json", "csv"]:
-        read_fn = getattr(pd, f"read_{format}")
-        data = _call_pandas_fsspec(read_fn, filename, storage_options=storage_options)
-    elif format in ["tsv"]:
-        data = _call_pandas_fsspec(
-            pd.read_csv, filename, storage_options=storage_options, sep="\t"
-        )
-    else:
-        raise PIIInputDataError(f"Unrecognized format: {format}")
-    if columns:
-        # keep only these columns, done after load because only CSV supports stream filtering
-        data = data[columns]
-    return data
-
-
-def _write_data(
-    data, filename, format=None, storage_options=None, index=False, **kwargs
-):
-    if not format:
-        _, format = os.path.splitext(filename)
-        format = format[1:]
-    if format in ["json", "csv"]:
-        write_fn = getattr(data, f"to_{format}")
-        return _call_pandas_fsspec(
-            write_fn, filename, index=index, storage_options=storage_options
-        )
-    raise PIIInputDataError(f"Unrecognized format: {format}")
-
-
 def get_output_name(given_name, target_name=None):
     """Add ``-out`` suffix to the src filename."""
     if not target_name:
@@ -93,33 +41,3 @@ def construct_filth_cls_name(name: str) -> str:
 ################
 def compute_rate(elapsed_time, num_unit):
     return elapsed_time / num_unit
-
-
-def human_time_friendly(seconds):
-    TIME_DURATION_UNITS = (
-        ("week", 60 * 60 * 24 * 7),
-        ("day", 60 * 60 * 24),
-        ("hour", 60 * 60),
-        ("min", 60),
-    )
-    if seconds == 0:
-        return "inf"
-    accumulator = []
-    for unit, div in TIME_DURATION_UNITS:
-        amount, seconds = divmod(float(seconds), div)
-        if amount > 0:
-            accumulator.append(
-                "{} {}{}".format(int(amount), unit, "" if amount == 1 else "s")
-            )
-    accumulator.append("{} secs".format(round(seconds, 2)))
-    return ", ".join(accumulator)
-
-
-# Disable
-def block_print():
-    sys.stdout = open(os.devnull, "w")
-
-
-# Restore
-def enable_print():
-    sys.stdout = sys.__stdout__

ads/opctl/operator/runtime/runtime.py

@@ -15,7 +15,8 @@ from cerberus import Validator
 from ads.common.extended_enum import ExtendedEnum
 from ads.common.serializer import DataClassSerializable
 from ads.opctl.operator.common.utils import _load_yaml_from_uri
-from ads.opctl.operator.common.errors import OperatorSchemaYamlError
+from ads.opctl.operator.common.errors import InvalidParameterError
+
 
 
 class OPERATOR_LOCAL_RUNTIME_TYPE(ExtendedEnum):
@@ -60,7 +61,7 @@ class Runtime(DataClassSerializable):
         result = validator.validate(obj_dict)
 
         if not result:
-            raise OperatorSchemaYamlError(json.dumps(validator.errors, indent=2))
+            raise InvalidParameterError(json.dumps(validator.errors, indent=2))
         return True
 
 

ads/telemetry/base.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (c) 2024 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import logging
+
+from ads import set_auth
+from ads.common import oci_client as oc
+from ads.common.auth import default_signer
+from ads.config import OCI_RESOURCE_PRINCIPAL_VERSION
+
+
+logger = logging.getLogger(__name__)
+class TelemetryBase:
+    """Base class for Telemetry Client."""
+
+    def __init__(self, bucket: str, namespace: str = None) -> None:
+        """Initializes the telemetry client.
+
+        Parameters
+        ----------
+        bucket : str
+            OCI object storage bucket name storing the telemetry objects.
+        namespace : str, optional
+            Namespace of the OCI object storage bucket, by default None.
+        """
+        if OCI_RESOURCE_PRINCIPAL_VERSION:
+            set_auth("resource_principal")
+        self._auth = default_signer()
+        self.os_client = oc.OCIClientFactory(**self._auth).object_storage
+        self.bucket = bucket
+        self._namespace = namespace
+        self._service_endpoint = None
+        logger.debug(f"Initialized Telemetry. Namespace: {self.namespace}, Bucket: {self.bucket}")
+
+
+    @property
+    def namespace(self) -> str:
+        """Gets the namespace of the object storage from the tenancy.
+
+        Returns
+        -------
+        str
+            The namespace of the tenancy.
+        """
+        if not self._namespace:
+            self._namespace = self.os_client.get_namespace().data
+        return self._namespace
+
+    @property
+    def service_endpoint(self):
+        """Gets the tenancy-specific endpoint.
+
+        Returns
+        -------
+        str
+            Tenancy-specific endpoint.
+        """
+        if not self._service_endpoint:
+            self._service_endpoint = self.os_client.base_client.endpoint
+        return self._service_endpoint

ads/telemetry/client.py

@@ -0,0 +1,105 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (c) 2024 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+
+import logging
+import threading
+import urllib.parse
+import requests
+from requests import Response
+from .base import TelemetryBase
+from ads.config import DEBUG_TELEMETRY
+
+
+logger = logging.getLogger(__name__)
+
+
+class TelemetryClient(TelemetryBase):
+    """Represents a telemetry python client providing functions to record an event.
+
+    Methods
+    -------
+    record_event(category: str = None, action: str = None, path: str = None, **kwargs) -> None
+        Send a head request to generate an event record.
+    record_event_async(category: str = None, action: str = None, path: str = None,  **kwargs)
+        Starts thread to send a head request to generate an event record.
+
+    Examples
+    --------
+    >>> import os
+    >>> import traceback
+    >>> from ads.telemetry.client import TelemetryClient
+    >>> AQUA_BUCKET = os.environ.get("AQUA_BUCKET", "service-managed-models")
+    >>> AQUA_BUCKET_NS = os.environ.get("AQUA_BUCKET_NS", "ociodscdev")
+    >>> telemetry = TelemetryClient(bucket=AQUA_BUCKET, namespace=AQUA_BUCKET_NS)
+    >>> telemetry.record_event_async(category="aqua/service/model", action="create") # records create action
+    >>> telemetry.record_event_async(category="aqua/service/model/create", action="shape", detail="VM.GPU.A10.1")
+    """
+
+    @staticmethod
+    def _encode_user_agent(**kwargs):
+        message = urllib.parse.urlencode(kwargs)
+        return message
+
+    def record_event(
+        self, category: str = None, action: str = None, detail: str = None, **kwargs
+    ) -> Response:
+        """Send a head request to generate an event record.
+
+        Parameters
+        ----------
+        category: (str)
+            Category of the event, which is also the path to the directory containing the object representing the event.
+        action: (str)
+            Filename of the object representing the event.
+        detail: (str)
+            Can be used to pass additional values, if required. When set, detail is converted to an action,
+            category and action are grouped together for telemetry parsing in the backend.
+        **kwargs:
+            Can be used to pass additional attributes like value that will be passed in the headers of the request.
+
+        Returns
+        -------
+        Response
+        """
+        try:
+            if not category or not action:
+                raise ValueError("Please specify the category and the action.")
+            if detail:
+                category, action = f"{category}/{action}", detail
+            endpoint = f"{self.service_endpoint}/n/{self.namespace}/b/{self.bucket}/o/telemetry/{category}/{action}"
+            headers = {"User-Agent": self._encode_user_agent(**kwargs)}
+            logger.debug(f"Sending telemetry to endpoint: {endpoint}")
+            signer = self._auth["signer"]
+            response = requests.head(endpoint, auth=signer, headers=headers)
+            logger.debug(f"Telemetry status code: {response.status_code}")
+            return response
+        except Exception as e:
+            if DEBUG_TELEMETRY:
+                logger.error(f"There is an error recording telemetry: {e}")
+
+    def record_event_async(
+        self, category: str = None, action: str = None, detail: str = None, **kwargs
+    ):
+        """Send a head request to generate an event record.
+
+        Parameters
+        ----------
+        category (str)
+            Category of the event, which is also the path to the directory containing the object representing the event.
+        action (str)
+            Filename of the object representing the event.
+
+        Returns
+        -------
+        Thread
+            A started thread to send a head request to generate an event record.
+        """
+        thread = threading.Thread(
+            target=self.record_event, args=(category, action, detail), kwargs=kwargs
+        )
+        thread.daemon = True
+        thread.start()
+        return thread

ads/telemetry/telemetry.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*--
 
-# Copyright (c) 2022, 2023 Oracle and/or its affiliates.
+# Copyright (c) 2022, 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 import os
@@ -102,7 +102,10 @@ def telemetry(
                 entry_point
             )
             try:
-                return func(*args, **{**kwargs, **{TELEMETRY_ARGUMENT_NAME: telemetry}})
+                # todo: inject telemetry arg and later update all functions that use the @telemetry
+                #   decorator to accept **kwargs. Comment the below line as some aqua apis don't support kwargs.
+                # return func(*args, **{**kwargs, **{TELEMETRY_ARGUMENT_NAME: telemetry}})
+                return func(*args, **kwargs)
             except:
                 raise
             finally:
@@ -178,7 +181,7 @@ class Telemetry:
         self: Telemetry
             An instance of the Telemetry.
         """
-        os.environ[self.environ_variable] = self._original_value
+        os.environ[self.environ_variable] = self._original_value or ""
         return self
 
     def clean(self) -> "Telemetry":

{oracle_ads-2.10.0.dist-info → oracle_ads-2.11.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: oracle_ads
-Version: 2.10.0
+Version: 2.11.0
 Summary: Oracle Accelerated Data Science SDK
 Keywords: Oracle Cloud Infrastructure,OCI,Machine Learning,ML,Artificial Intelligence,AI,Data Science,Cloud,Oracle
 Author: Oracle Data Science
@@ -24,13 +24,23 @@ Requires-Dist: matplotlib>=3.1.3
 Requires-Dist: numpy>=1.19.2
 Requires-Dist: oci>=2.113.0
 Requires-Dist: ocifs>=1.1.3
-Requires-Dist: pandas>1.2.1,<2.1
+Requires-Dist: pandas>1.2.1; python_version<'3.9'
+Requires-Dist: pandas>=2.2.0; python_version>='3.9'
 Requires-Dist: psutil>=5.7.2
 Requires-Dist: python_jsonschema_objects>=0.3.13
 Requires-Dist: requests
 Requires-Dist: scikit-learn>=1.0
 Requires-Dist: tabulate>=0.8.9
 Requires-Dist: tqdm>=4.59.0
+Requires-Dist: oracle_ads[opctl] ; extra == "anomaly"
+Requires-Dist: autots ; extra == "anomaly"
+Requires-Dist: oracle-automlx[forecasting]==23.4.1 ; extra == "anomaly"
+Requires-Dist: oracle-automlx[classic]==23.4.1 ; extra == "anomaly"
+Requires-Dist: oracledb ; extra == "anomaly"
+Requires-Dist: datapane ; extra == "anomaly"
+Requires-Dist: fire ; extra == "aqua"
+Requires-Dist: cachetools ; extra == "aqua"
+Requires-Dist: jupyter_server ; extra == "aqua"
 Requires-Dist: hdfs[kerberos] ; extra == "bds"
 Requires-Dist: ibis-framework[impala] ; extra == "bds"
 Requires-Dist: sqlalchemy ; extra == "bds"
@@ -58,19 +68,21 @@ Requires-Dist: holidays==0.21.13 ; extra == "forecast"
 Requires-Dist: neuralprophet ; extra == "forecast"
 Requires-Dist: numpy ; extra == "forecast"
 Requires-Dist: oci-cli ; extra == "forecast"
-Requires-Dist: optuna==2.9.0 ; extra == "forecast"
+Requires-Dist: optuna==3.1.0 ; extra == "forecast"
 Requires-Dist: oracle-ads ; extra == "forecast"
-Requires-Dist: oracle-automlx[forecasting]==23.2.3 ; extra == "forecast"
+Requires-Dist: oracle-automlx[forecasting]==23.4.1 ; extra == "forecast"
+Requires-Dist: oracle-automlx[classic]==23.4.1 ; extra == "forecast"
 Requires-Dist: pmdarima ; extra == "forecast"
 Requires-Dist: prophet ; extra == "forecast"
 Requires-Dist: shap ; extra == "forecast"
 Requires-Dist: sktime ; extra == "forecast"
 Requires-Dist: statsmodels ; extra == "forecast"
 Requires-Dist: plotly ; extra == "forecast"
+Requires-Dist: oracledb ; extra == "forecast"
 Requires-Dist: geopandas ; extra == "geo"
 Requires-Dist: oracle_ads[viz] ; extra == "geo"
 Requires-Dist: transformers ; extra == "huggingface"
-Requires-Dist: langchain>=0.0.295 ; extra == "llm"
+Requires-Dist: langchain>=0.1.10 ; extra == "llm"
 Requires-Dist: evaluate>=0.4.0 ; extra == "llm"
 Requires-Dist: ipython>=7.23.1, <8.0 ; extra == "notebook"
 Requires-Dist: ipywidgets~=7.6.3 ; extra == "notebook"
@@ -106,18 +118,42 @@ Requires-Dist: spacy==3.6.1 ; extra == "pii"
 Requires-Dist: pyspark>=3.0.0 ; extra == "spark"
 Requires-Dist: oracle_ads[viz] ; extra == "tensorflow"
 Requires-Dist: tensorflow ; extra == "tensorflow"
+Requires-Dist: dask==2023.5.0 ; extra == "testsuite" and ( python_version=='3.8')
+Requires-Dist: dask==2023.10.1 ; extra == "testsuite" and ( python_version>='3.9')
+Requires-Dist: arff ; extra == "testsuite"
+Requires-Dist: category_encoders==2.6.3 ; extra == "testsuite"
+Requires-Dist: cohere==4.53 ; extra == "testsuite"
+Requires-Dist: dask==2023.10.1 ; extra == "testsuite" and ( python_version>='3.9')
+Requires-Dist: dask==2023.5.0 ; extra == "testsuite" and ( python_version=='3.8')
+Requires-Dist: faiss-cpu ; extra == "testsuite"
+Requires-Dist: fastparquet==2024.2.0 ; extra == "testsuite"
+Requires-Dist: imbalanced-learn ; extra == "testsuite"
+Requires-Dist: lxml ; extra == "testsuite"
+Requires-Dist: mysql-connector-python ; extra == "testsuite"
+Requires-Dist: nltk ; extra == "testsuite"
+Requires-Dist: notebook==6.4.12 ; extra == "testsuite"
+Requires-Dist: opensearch-py ; extra == "testsuite"
+Requires-Dist: pdfplumber ; extra == "testsuite"
+Requires-Dist: py4j ; extra == "testsuite"
+Requires-Dist: pyarrow ; extra == "testsuite"
+Requires-Dist: statsmodels ; extra == "testsuite" and ( python_version=='3.8')
+Requires-Dist: statsmodels>=0.14.1 ; extra == "testsuite" and ( python_version>='3.9')
+Requires-Dist: tables ; extra == "testsuite"
+Requires-Dist: xlrd>=1.2.0 ; extra == "testsuite"
 Requires-Dist: spacy ; extra == "text"
 Requires-Dist: wordcloud>=1.8.1 ; extra == "text"
 Requires-Dist: oracle_ads[viz] ; extra == "torch"
 Requires-Dist: torch ; extra == "torch"
 Requires-Dist: torchvision ; extra == "torch"
-Requires-Dist: bokeh>=2.3.0, <=2.4.3 ; extra == "viz"
+Requires-Dist: bokeh>=3.0.0, <3.2.0 ; extra == "viz"
 Requires-Dist: folium>=0.12.1 ; extra == "viz"
 Requires-Dist: graphviz<0.17 ; extra == "viz"
 Requires-Dist: scipy>=1.5.4 ; extra == "viz"
 Requires-Dist: seaborn>=0.11.0 ; extra == "viz"
 Project-URL: Documentation, https://accelerated-data-science.readthedocs.io/en/latest/index.html
 Project-URL: Github, https://github.com/oracle/accelerated-data-science
+Provides-Extra: anomaly
+Provides-Extra: aqua
 Provides-Extra: bds
 Provides-Extra: boosted
 Provides-Extra: data
@@ -133,6 +169,7 @@ Provides-Extra: optuna
 Provides-Extra: pii
 Provides-Extra: spark
 Provides-Extra: tensorflow
+Provides-Extra: testsuite
 Provides-Extra: text
 Provides-Extra: torch
 Provides-Extra: viz
@@ -171,7 +208,7 @@ You have various options when installing ADS.
 To use the AI Forecast Operator, install the "forecast" dependencies using the following command:
 
 ```bash
-  python3 -m pip install 'oracle_ads[forecast]==2.9.0'
+  python3 -m pip install 'oracle_ads[forecast]>=2.9.0'
 ```
 
 ### Installing extras libraries