oracle-ads 2.13.8__py3-none-any.whl → 2.13.9rc0__py3-none-any.whl

ads/model/transformer/onnx_transformer.py

@@ -1,324 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*--
-
-# Copyright (c) 2022 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-import json
-import logging
-from typing import Dict, Union
-
-import numpy as np
-import pandas as pd
-
-pd.options.mode.chained_assignment = None
-
-
-# Note to developers: If you make any changes to this class, copy and paste those changes over to
-# templates/score_onnx.jinja2 and templates/score_onnx_new.jinja2. We do not yet have an automatic way of doing this.
-class ONNXTransformer(object):
-    """
-    This is a transformer to convert X [pandas.Dataframe, pd.Series] data into Onnx
-    readable dtypes and formats. It is Serializable, so it can be reloaded at another time.
-
-
-    Examples
-    --------
-    >>> from ads.model.transformer.onnx_transformer import ONNXTransformer
-    >>> onnx_data_transformer = ONNXTransformer()
-    >>> train_transformed = onnx_data_transformer.fit_transform(train.X, {"column_name1": "impute_value1", "column_name2": "impute_value2"}})
-    >>> test_transformed = onnx_data_transformer.transform(test.X)
-    """
-
-    def __init__(self):
-        self.impute_values = {}
-        self.dtypes = None
-        self._fitted = False
-
-    @staticmethod
-    def _handle_dtypes(X: Union[pd.DataFrame, pd.Series, np.ndarray, list]):
-        """Handles the dtypes for pandas dataframe and pandas Series.
-
-        Parameters
-        ----------
-        X : Union[pd.DataFrame, pd.Series, np.ndarray, list]
-            The Dataframe for the training data
-
-        Returns
-        -------
-        Union[pd.DataFrame, pd.Series, np.ndarray, list]
-            The transformed(numerical values are cast to float32) X data
-        """
-        # Data type cast could be expensive doing it in a for loop
-        # Especially with wide datasets
-        # So cast the numerical columns first, without loop
-        # Then impute missing values
-        if isinstance(X, pd.Series):
-            series_name = X.name if X.name else 0
-            _X = X.to_frame()
-            _X = ONNXTransformer._handle_dtypes_dataframe(_X)[series_name]
-        elif isinstance(X, pd.DataFrame):
-            _X = ONNXTransformer._handle_dtypes_dataframe(X)
-        elif isinstance(X, np.ndarray):
-            _X = ONNXTransformer._handle_dtypes_np_array(X)
-        else:
-            # if users convert pandas dataframe with mixed types to numpy array directly
-            # it will turn the whole numpy array into object even though some columns are
-            # numerical and some are not. In that case, we need to do extra work to identify
-            # which columns are really numerical which for now, we only convert to float32
-            # if numpy array is all numerical. else, nothing will be done.
-            _X = X
-        return _X
-
-    @staticmethod
-    def _handle_dtypes_dataframe(X: pd.DataFrame):
-        """handle the dtypes for pandas dataframe.
-
-        Parameters
-        ----------
-        X : pandas.DataFrame
-            The Dataframe for the training data
-
-        Returns
-        -------
-        pandas.DataFrame
-            The transformed X data
-        """
-        dict_astype = {}
-        for k, v in zip(X.columns, X.dtypes):
-            if "int" in str(v) or "float" in str(v) or "bool" in str(v):
-                dict_astype[k] = "float32"
-        _X = X.astype(dict_astype)
-        if len(dict_astype) > 0:
-            logging.warning("Numerical values in `X` are cast to float32.")
-        return _X
-
-    @staticmethod
-    def _handle_dtypes_np_array(X: np.ndarray):
-        """handle the dtypes for pandas dataframe.
-
-        Parameters
-        ----------
-        X : np.ndarray
-            The ndarray for the training data
-
-        Returns
-        -------
-        np.ndarray
-            The transformed X data
-        """
-        if "int" in str(X.dtype) or "float" in str(X.dtype) or "bool" in str(X.dtype):
-            _X = X.astype("float32")
-            logging.warning("Numerical values in `X` are cast to float32.")
-        else:
-            _X = X
-        return _X
-
-    def fit(
-        self,
-        X: Union[pd.DataFrame, pd.Series, np.ndarray, list],
-        impute_values: Dict = None,
-    ):
-        """
-        Fits the OnnxTransformer on the dataset
-        Parameters
-        ----------
-        X : Union[pandas.DataFrame, pandas.Series, np.ndarray, list]
-            The Dataframe for the training data
-
-        Returns
-        -------
-        Self: ads.Model
-            The fitted estimator
-        """
-        _X = ONNXTransformer._handle_dtypes(X)
-        if isinstance(_X, pd.DataFrame):
-            self.dtypes = _X.dtypes
-        elif isinstance(_X, np.ndarray):
-            self.dtypes = _X.dtype
-        self.impute_values = impute_values if impute_values else {}
-        self._fitted = True
-        return self
-
-    def transform(self, X: Union[pd.DataFrame, pd.Series, np.ndarray, list]):
-        """
-        Transforms the data for the OnnxTransformer.
-
-        Parameters
-        ----------
-        X: Union[pandas.DataFrame, pandas.Series, np.ndarray, list]
-            The Dataframe for the training data
-
-        Returns
-        -------
-        Union[pandas.DataFrame, pandas.Series, np.ndarray, list]
-            The transformed X data
-        """
-        assert self._fitted, "Call fit_transform first!"
-        if self.dtypes is not None and len(self.dtypes) > 0:
-            if isinstance(X, list):
-                _X = np.array(X).astype(self.dtypes).tolist()
-            else:
-                _X = X.astype(self.dtypes)
-        else:
-            _X = X
-        _X = ONNXTransformer._handle_missing_value(_X, impute_values=self.impute_values)
-        return _X
-
-    @staticmethod
-    def _handle_missing_value(
-        X: Union[pd.DataFrame, pd.Series, np.ndarray, list], impute_values: Dict
-    ):
-        """Impute missing values in X according to impute_values.
-
-        Parameters
-        ----------
-        X: Union[pandas.DataFrame, pandas.Series, np.ndarray, list]
-            The Dataframe for the training data
-
-        Raises
-        ------
-        Exception if X has only one dim, but imputed_values has multiple values.
-        NotImplemented if X has the data type that is not supported.
-
-        Returns
-        -------
-        Union[pandas.DataFrame, pd.Series, np.ndarray, list]
-            The transformed X data
-        """
-        if isinstance(X, np.ndarray):
-            X = ONNXTransformer._handle_missing_value_dataframe(
-                pd.DataFrame(X), impute_values=impute_values
-            ).values
-        elif isinstance(X, list):
-            X = ONNXTransformer._handle_missing_value_dataframe(
-                pd.DataFrame(X), impute_values=impute_values
-            ).values.tolist()
-        elif isinstance(X, pd.DataFrame):
-            X = ONNXTransformer._handle_missing_value_dataframe(
-                X, impute_values=impute_values
-            )
-        elif isinstance(X, pd.Series):
-            X = X.replace(r"^\s*$", np.NaN, regex=True)
-            if len(impute_values.keys()) == 1:
-                for key, val in impute_values.items():
-                    X = X.fillna(val)
-            else:
-                raise Exception(
-                    "Multiple imputed values are provided, but `X` has only one dim."
-                )
-        else:
-            raise NotImplemented(
-                f"{type(X)} is not supported. Convert `X` to pandas dataframe or numpy array."
-            )
-        return X
-
-    @staticmethod
-    def _handle_missing_value_dataframe(X: pd.DataFrame, impute_values: Dict):
-        for idx, val in impute_values.items():
-            if isinstance(idx, int):
-                X.iloc[:, idx] = (
-                    X.iloc[:, idx].replace(r"^\s*$", np.NaN, regex=True).fillna(val)
-                )
-            else:
-                X.loc[:, idx] = (
-                    X.loc[:, idx].replace(r"^\s*$", np.NaN, regex=True).fillna(val)
-                )
-        return X
-
-    def fit_transform(
-        self, X: Union[pd.DataFrame, pd.Series], impute_values: Dict = None
-    ):
-        """
-        Fits, then transforms the data
-        Parameters
-        ----------
-        X: Union[pandas.DataFrame, pandas.Series]
-            The Dataframe for the training data
-
-        Returns
-        -------
-        Union[pandas.DataFrame, pandas.Series]
-            The transformed X data
-        """
-        return self.fit(X, impute_values).transform(X)
-
-    def save(self, filename, **kwargs):
-        """
-        Saves the Onnx model to disk
-        Parameters
-        ----------
-        filename: Str
-            The filename location for where the model should be saved
-
-        Returns
-        -------
-        filename: Str
-            The filename where the model was saved
-        """
-        export_dict = {
-            "impute_values": {
-                "value": self.impute_values,
-                "dtype": str(type(self.impute_values)),
-            },
-            "dtypes": {}
-            if self.dtypes is None
-            else {
-                "value": {
-                    "index": list(self.dtypes.index),
-                    "values": [str(val) for val in self.dtypes.values],
-                }
-                if isinstance(self.dtypes, pd.Series)
-                else str(self.dtypes),
-                "dtype": str(type(self.dtypes)),
-            },
-            "_fitted": {"value": self._fitted, "dtype": str(type(self._fitted))},
-        }
-
-        with open(filename, "w") as f:
-            json.dump(export_dict, f, sort_keys=True, indent=4, separators=(",", ": "))
-        return filename
-
-    @staticmethod
-    def load(filename, **kwargs):
-        """
-        Loads the Onnx model to disk
-        Parameters
-        ----------
-        filename: Str
-            The filename location for where the model should be loaded
-
-        Returns
-        -------
-        onnx_transformer: ONNXTransformer
-            The loaded model
-        """
-        # Make sure you have  pandas, numpy, and sklearn imported
-        with open(filename, "r") as f:
-            export_dict = json.load(f)
-
-        onnx_transformer = ONNXTransformer()
-        for key in export_dict.keys():
-            if key not in ["impute_values", "dtypes"]:
-                try:
-                    setattr(onnx_transformer, key, export_dict[key]["value"])
-                except Exception as e:
-                    print(
-                        f"Warning: Failed to reload {key} from {filename} to OnnxTransformer."
-                    )
-                    raise e
-        if "value" in export_dict["dtypes"]:
-            if "index" in export_dict["dtypes"]["value"]:
-                onnx_transformer.dtypes = pd.Series(
-                    data=[
-                        np.dtype(val)
-                        for val in export_dict["dtypes"]["value"]["values"]
-                    ],
-                    index=export_dict["dtypes"]["value"]["index"],
-                )
-            else:
-                onnx_transformer.dtypes = export_dict["dtypes"]["value"]
-        else:
-            onnx_transformer.dtypes = {}
-        onnx_transformer.impute_values = export_dict["impute_values"]["value"]
-        return onnx_transformer

ads/mysqldb/__init__.py

@@ -1,5 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*--
-
-# Copyright (c) 2022 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/

ads/mysqldb/mysql_db.py

@@ -1,227 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*--
-
-# Copyright (c) 2022, 2023 Oracle and/or its affiliates
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-import ads
-import os
-import tempfile
-import zipfile
-from time import time
-from typing import Dict, Optional, List, Union, Iterator
-
-import pandas as pd
-import numpy as np
-
-import logging
-
-logger = logging.getLogger("ads.msql_connector")
-
-
-from mysql.connector import connection
-
-CURSOR_SIZE = 50000
-
-
-class MySQLRDBMSConnection(connection.MySQLConnection):
-    def __init__(
-        self,
-        *args,
-        **kwargs,
-    ):
-        if kwargs["user_name"]:
-            kwargs["user"] = kwargs.pop("user_name")
-        super().__init__(*args, **kwargs)
-
-    def insert(
-        self,
-        table_name: str,
-        df: pd.DataFrame,
-        if_exists: str,
-        batch_size=100000,
-        encoding="utf-8",
-    ):
-
-        if if_exists not in ["fail", "replace", "append"]:
-            raise ValueError(
-                f"Unknown option `if_exists`={if_exists}. Valid options are 'fail', 'replace', 'append'"
-            )
-        start_time = time()
-
-        df_orcl = df.copy()
-        # "object" type can be actual objects, or just plain strings, when inserting into the
-        # database we need to stringify these so they can be represented in a VARCHAR2 column
-
-        # object_columns = df_orcl.select_dtypes(include=["object"]).columns
-        # df_orcl = df_orcl.where(pd.notnull(df_orcl), None)
-        # df_orcl[object_columns] = df_orcl[object_columns].astype(str)
-
-        # prep column names for valid Oracle column names (alpha + # $ _)
-        df_orcl.columns = df_orcl.columns.str.replace(r"\W+", "_", regex=True)
-        table_exist = True
-        with self.cursor() as cursor:
-
-            if if_exists != "replace":
-                try:
-                    cursor.execute(f"SELECT 1 from {table_name} LIMIT 1")
-                    cursor.fetchall()
-                except Exception as e:
-                    table_exist = False
-                if if_exists == "fail" and table_exist:
-                    raise ValueError(
-                        f"Table {table_name} already exists. Set `if_exists`='replace' or 'append' to replace or append to the existing table"
-                    )
-            # Oracle doesn't have boolean so convert to 1/0
-            df_orcl = df_orcl.where(
-                df.applymap(type) != bool, df_orcl.replace({True: 1, False: 0})
-            )
-
-            type_mappings = {
-                "bool": "NUMBER(1)",
-                "int16": "INTEGER",
-                "int32": "INTEGER",
-                "int64": "INTEGER",
-                "float16": "FLOAT",
-                "float32": "FLOAT",
-                "float64": "FLOAT",
-                "datetime64": "DATETIME",
-            }
-            # add in any string types as Oracle's VARCHAR type setting length to accommodate longest
-            def get_max_str_len(df, column, encoding):
-                return df[column].dropna().str.encode(encoding).map(len).max()
-
-            longest_string_column = max(
-                (
-                    get_max_str_len(df_orcl, c, encoding)
-                    for c in df_orcl.select_dtypes(
-                        include=["object", "category"]
-                    ).columns
-                ),
-                default=0,
-            )
-
-            logger.debug(f"Max string column value: {longest_string_column}")
-
-            datatypes = {
-                c: f"VARCHAR({get_max_str_len(df_orcl, c, encoding)})"
-                for c in df_orcl.select_dtypes(include=["object", "category"]).columns
-            }
-
-            for df_type, orcl_type in type_mappings.items():
-
-                datatypes.update(
-                    {
-                        column: orcl_type
-                        for column in df_orcl.select_dtypes(include=df_type).columns
-                    }
-                )
-
-            if set(datatypes.keys()) != set(df_orcl.columns):
-                raise Exception(
-                    f"Unable to determine MySQL data type to use for column(s): {', '.join(set(df_orcl.columns)-set(datatypes.keys()))}"
-                )
-
-            # create table
-            if if_exists == "replace":
-                try:
-                    cursor.execute(f"drop table {table_name}")
-                except:
-                    logger.info(f"Table {table_name} does not exist")
-            if if_exists == "replace" or not table_exist:
-                sql = (
-                    f"create table {table_name} ("
-                    + ", ".join([f"{col} {datatypes[col]}" for col in df_orcl.columns])
-                    + ")"
-                )
-                logger.info(sql)
-                try:
-                    cursor.execute(sql)
-                except Exception as e:
-                    raise Exception(
-                        f"'{e if e.args else 'Unexpected error encountered'}' with query: '{sql}'"
-                    ) from e
-
-            # insert
-            bind_columns = ", ".join([f"{col}" for col in df_orcl.columns])
-            bind_variables = ",".join(["%s" for _ in df_orcl.columns])
-            sql = f"insert into {table_name}({bind_columns}) values({bind_variables})"
-
-            logger.info(sql)
-
-            # prevent buffer reallocation by locking in the longest string value
-            # cursor.setinputsizes(None, longest_string_column)
-
-            # replace NaN with None before turning into database records, important - don't
-            # do this earlier in the logic because it can change the pandas
-            # data types
-
-            record_data = list(
-                (
-                    [
-                        xr.to_pydatetime().strftime("%Y-%m-%d %H:%M:%S")
-                        if hasattr(xr, "to_pydatetime")
-                        else xr
-                        for xr in x
-                    ]
-                    for x in df_orcl.replace({np.nan: None}).itertuples(
-                        index=False, name=None
-                    )
-                )
-            )
-
-            def chunks(lst: List, batch_size: int):
-                """Yield successive batch_size chunks from lst."""
-                for i in range(0, len(lst), batch_size):
-                    yield lst[i : i + batch_size]
-
-            for batch in chunks(record_data, batch_size=batch_size):
-                cursor.executemany(sql, batch)
-
-            self.commit()
-
-            duration = time() - start_time
-            logger.info(
-                f"inserted {df_orcl.shape[0]} rows at {df_orcl.shape[0]/duration:.2f} rows/seconds"
-            )
-
-    def _fetch_by_batch(self, cursor, chunksize):
-        while True:
-            rows = cursor.fetchmany(chunksize)
-            if rows:
-                yield rows
-            else:
-                break
-
-    def query(
-        self, sql: str, bind_variables: Optional[Dict], chunksize=None
-    ) -> Union[pd.DataFrame, Iterator[pd.DataFrame]]:
-
-        start_time = time()
-
-        cursor = self.cursor(prepared=True)
-        cursor.arraysize = CURSOR_SIZE
-
-        cursor.execute(sql, bind_variables)
-        columns = [row[0] for row in cursor.description]
-
-        if chunksize:
-            logger.info(f"Chunksize is {chunksize}")
-            df = iter(
-                (
-                    pd.DataFrame(data=rows, columns=columns)
-                    for rows in self._fetch_by_batch(cursor, chunksize)
-                )
-            )
-
-        else:
-            df = pd.DataFrame(
-                cursor,
-                columns=columns,
-            )
-            duration = time() - start_time
-            logger.info(
-                f"fetched {df.shape[0]} rows at {df.shape[0]/duration:.2f} rows/seconds"
-            )
-
-        return df

ads/opctl/__init__.py

@@ -1,18 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8; -*-
-
-# Copyright (c) 2022, 2023 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-import logging
-import sys
-
-logger = logging.getLogger(__name__)
-handler = logging.StreamHandler(sys.stdout)
-# logger.addHandler(handler)
-
-logger.setLevel(logging.INFO)
-
-
-def set_log_level(level):
-    handler.setLevel(level)

ads/opctl/anomaly_detection.py

@@ -1,11 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright (c) 2025 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-from ads.opctl.operator.lowcode.anomaly.__main__ import operate
-from ads.opctl.operator.lowcode.anomaly.operator_config import AnomalyOperatorConfig
-
-if __name__ == "__main__":
-    config = AnomalyOperatorConfig()
-    operate(config)

ads/opctl/backend/__init__.py

@@ -1,5 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8; -*-
-
-# Copyright (c) 2022 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/